simplerubysteps 0.0.7 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41e3a9ed5b8d6c32357fa880c2826cdf18a16fb9195260c5d10c7063b7dcb474
-  data.tar.gz: '07180493d6f6456f41164cd74f1410fb8bcd6649a645d109011c85ec9133c463'
+  metadata.gz: 32057d9ea30569964077a231e6e5ed88aff396dabc03151c1ef77bbac237a7e2
+  data.tar.gz: 81d13b619c57a3b793ed7039a06e0ec2a730aa8844d70c8603f35277799090e0
 SHA512:
-  metadata.gz: 0620c89b84893c3e58078ab8dc66a211aaef5a52101a45e7392bde32bfdab3be3cc9d83bdee6e9adcfd1f3ecdc1e3420dd9068f58bea07550aad1993d0775a90
-  data.tar.gz: 50f9d29b51926176a2a3edc900694233f38d44511ef5115433cc756f88c74df5ffca90bc924406644580c43e1f9ca6e2b1e2f79187aa0b3c9eca003f14b2daae
+  metadata.gz: 68f92e9f3b4e7d0e5249e7df03f97a9e133ab51274117b2538b8672bf8e6221c6c8d2130679395ff0e1e57ee18f6e0bf4c321fe8ce034e0fe14c53ca37150e65
+  data.tar.gz: 1185695c5f0bc3c3d093cb7984c2947ec1ef2f4be737586f73ae058d2266ba90121d9d48b37608993877784eee729010867c15754309584beafe88c01d6b5836

data/README.md

@@ -1,55 +1,96 @@
 # Simplerubysteps
 
-Simplerubysteps simplifies the administration of AWS Step Functions with ruby (eventually - this is an early alpha version)
+Simplerubysteps makes it easy to manage AWS Step Functions with ruby.
 
-This is software in the experimental stage and should not really be used by anyone. Be warned.
+* Phase I (we are here): Experimenting and exploring the problem and solution domain. The aim is to explore the DSL capabilities and user experience of the automation tool. Things will work, but may change over time. The released gem versions of Simplerubysteps are early alpha versions and therefore should not be used by anyone in production.
+* Phase II: First release candidate (possibly rewritten from scratch)
+* Phase III: Maintain and evolve
 
-## Installation
+## Installation and Usage
 
-Prerequisites (for alpha version):
-* AWS CLI installed
+### Prerequisites
 
-## Usage
+* AWS CLI installed (mainly for debugging privileges)
+* Configured AWS CLI profile with sufficient permissions to create IAM roles and policies, create Lambda functions, create and run Step Functions state machines, run CloudWatch log queries, etc.
 
-### Install the gem and the simplerubysteps CLI
+### Install the gem and the srs CLI
 
 ```
 gem install simplerubysteps
 ```
 
-### Create AWS Step Functions State Machine with ruby DSL
+### Create an AWS Step Function State Machine with the simplerubysteps Ruby DSL
 
 ```
-cd samples/sample1
+mkdir -p samples/hello-world-2
+cd samples/hello-world-2
+
 vi workflow.rb
 ```
 
-### Create CloudFormation stack with Step Functions State Machine and supporting Lambda function
+#### Hello World State Machine (workflow.rb)
 
 ```
-export AWS_PROFILE=...
-cd samples/sample1
-simplerubysteps deploy
+require "simplerubysteps"
+include Simplerubysteps
+kind "EXPRESS"
+
+GERMAN_WORDS = ["Hallo"]
+
+def is_german?(word)
+  GERMAN_WORDS.include? word
+end
+
+task :start do
+  transition_to :german do |data|
+    is_german? data["hello"]
+  end
+
+  default_transition_to :english
+end
+
+task :german do
+  action do |data|
+    { hello_world: "#{data["hello"]} Welt!" }
+  end
+end
+
+task :english do
+  action do |data|
+    { hello_world: "#{data["hello"]} World!" }
+  end
+end
 ```
 
-### Trigger State Machine Execution and wait for completion
+### Deploy the Step Functions State Machine and the Lambda function that implements the Task Actions (with srs deploy)
+
+```
+export AWS_PROFILE=<AWS CLI profile name with sufficient privileges>
+cd samples/hello-world-2
 
+srs deploy
 ```
-export AWS_PROFILE=...          
-cd samples/sample1
 
-./start-directbranch.sh
+### Trigger State Machine executions (with srs start)
 
-./sample-task-worker.sh &
-./start-callbackbranch.sh
 ```
+export AWS_PROFILE=<AWS CLI profile name with sufficient privileges>
+cd samples/hello-world-2
 
-### Delete CloudFormation stack
+# Bellow: will print "Hello World!"
+echo '{"hello":"Hello"}'|srs start|jq -r ".output"|jq -r ".hello_world"
 
+# Bellow: will print "Hallo Welt!"
+echo '{"hello":"Hallo"}'|srs start|jq -r ".output"|jq -r ".hello_world"
 ```
-export AWS_PROFILE=...
-cd samples/sample1
-simplerubysteps destroy
+
+### Delete CloudFormation stack (with srs destroy)
+
+```
+export AWS_PROFILE=<AWS CLI profile name with sufficient privileges>
+cd samples/hello-world-2
+
+srs destroy
 ```
 
 ## Development
@@ -58,8 +99,10 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ### TODOs
 
-* Custom IAM policies per Lambda task
-* Workflow action unit test support
+* Custom IAM policies per Lambda task (e.g. to allow a task to send a message to an SQS queue)
+* Workflow task/action unit test support
+* Better error handling and reporting
+* Improved stack update strategy (e.g. renamed or added task scenario)
 * ...
 
 ## Contributing

data/exe/simplerubysteps

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-bin_dir = File.expand_path(File.dirname(__FILE__))
-include_path = File.join(bin_dir, "../lib/tool.rb")
+require "simplerubysteps"
+require "simplerubysteps/tool"
 
-require include_path
+Simplerubysteps::Tool.new.run

data/exe/srs

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "simplerubysteps"
+require "simplerubysteps/tool"
+
+Simplerubysteps::Tool.new.run

data/lib/function.rb

@@ -1,14 +1,17 @@
-require "json"
+require "simplerubysteps"
+
 require "./workflow.rb"
 
+include Simplerubysteps
+
 def handler(event:, context:)
   puts ENV.inspect if ENV["DEBUG"]
   puts event if ENV["DEBUG"]
   puts context.inspect if ENV["DEBUG"]
 
   if event["Token"]
-    $sm.states[event["Task"].to_sym].perform_action event["Input"], event["Token"]
+    $sm.states[ENV["task"].to_sym].perform_action event["Input"], event["Token"]
   else
-    $sm.states[event["Task"].to_sym].perform_action event["Input"]
+    $sm.states[ENV["task"].to_sym].perform_action event["Input"]
   end
 end

data/lib/{statemachine.yaml → simplerubysteps/cloudformation.rb}

@@ -1,66 +1,53 @@
+require "erb"
+
+module Simplerubysteps
+  CLOUDFORMATION_ERB_TEMPLATE = <<-YAML
 ---
 AWSTemplateFormatVersion: "2010-09-09"
-Parameters:
+
+<% if resources[:functions] %>
+Parameters:  
   LambdaS3:
     Description: LambdaS3.
-    Default: function.zip
-    Type: String  
+    Type: String
+<% end %>
+
+<% if resources[:state_machine] %>
   StepFunctionsS3:
     Description: StepFunctionsS3.
-    Default: my-state-machine-definition.json
-    Type: String    
-  DeployLambda:
-    Description: DeployLambda.
-    Default: "yes"
-    Type: String
-    AllowedValues:
-      - "yes"
-      - "no"
-    ConstraintDescription: must specify yes or no.
-  DeployStepfunctions:
-    Description: DeployStepfunctions.
-    Default: "yes"
     Type: String
-    AllowedValues:
-      - "yes"
-      - "no"
-    ConstraintDescription: must specify yes or no.    
   StateMachineType:
     Description: StateMachineType.
-    Default: STANDARD
-    Type: String    
-Conditions:
-  CreateLambda: !Equals 
-    - !Ref DeployLambda
-    - "yes"
-  CreateStepfunctions: !Equals 
-    - !Ref DeployStepfunctions
-    - "yes"    
+    Type: String
+<% end %>
+
 Resources:
-  DeployBucket: 
+  DeployBucket:
     Type: AWS::S3::Bucket
-  LambdaFunction:
-    Type: 'AWS::Lambda::Function'
-    Condition: CreateLambda
+
+<% if resources[:functions] %>
+<% resources[:functions].each_with_index do |resource, index| %>
+  LambdaFunction<%= index %>:
+    Type: "AWS::Lambda::Function"
     Properties:
       Code:
         S3Bucket: !Ref DeployBucket
         S3Key: !Ref LambdaS3
-      Handler: function.handler
-      Role: !GetAtt MyLambdaRole.Arn
+      Handler: function.handler      
+      Role: !GetAtt MyLambdaRole<%= index %>.Arn
       Runtime: ruby2.7
-  StepFunctionsStateMachine:
-    Type: 'AWS::StepFunctions::StateMachine'
-    Condition: CreateStepfunctions
+      Environment:
+        Variables:
+<% resource.each do |k, v| %>        
+          <%= k %>: <%= v %>
+<% end %>
+  LogGroup<%= index %>:
+    Type: AWS::Logs::LogGroup
     Properties:
-      DefinitionS3Location:
-        Bucket: !Ref DeployBucket
-        Key: !Ref StepFunctionsS3
-      RoleArn: !GetAtt StepFunctionsRole.Arn
-      StateMachineType: !Ref StateMachineType
-  MyLambdaRole:
+      LogGroupName: !Sub "/aws/lambda/${LambdaFunction<%= index %>}"
+      RetentionInDays: 7  
+  MyLambdaRole<%= index %>:
     Type: AWS::IAM::Role
-    Condition: CreateLambda
     Properties:
       AssumeRolePolicyDocument:
         Version: "2012-10-17"
@@ -81,9 +68,20 @@ Resources:
                   - logs:CreateLogStream
                   - logs:PutLogEvents
                 Resource: arn:aws:logs:*:*:*
+<% end %>
+<% end %>
+  
+<% if resources[:state_machine] %>
+  StepFunctionsStateMachine:
+    Type: "AWS::StepFunctions::StateMachine"
+    Properties:
+      DefinitionS3Location:
+        Bucket: !Ref DeployBucket
+        Key: !Ref StepFunctionsS3
+      RoleArn: !GetAtt StepFunctionsRole.Arn
+      StateMachineType: !Ref StateMachineType
   StepFunctionsRole:
     Type: AWS::IAM::Role
-    Condition: CreateStepfunctions
     Properties:
       AssumeRolePolicyDocument:
         Version: "2012-10-17"
@@ -106,22 +104,41 @@ Resources:
               - Effect: Allow
                 Action:
                   - lambda:InvokeFunction
-                Resource: !GetAtt LambdaFunction.Arn
+                Resource: 
+<% resources[:functions].each_with_index do |resource, index| %>                
+                  - !GetAtt LambdaFunction<%= index %>.Arn
+<% end %>                  
+<% end %>
+
 Outputs:
   DeployBucket:
     Value: !Ref DeployBucket
-  LambdaRoleARN:
-    Value: !GetAtt MyLambdaRole.Arn
-    Condition: CreateLambda
-  LambdaFunctionARN:
-    Value: !GetAtt LambdaFunction.Arn
-    Condition: CreateLambda
-  LambdaFunctionName:
-    Value: !Ref LambdaFunction
-    Condition: CreateLambda
-  StepFunctionsRoleARN:
-    Value: !GetAtt StepFunctionsRole.Arn
-    Condition: CreateStepfunctions
+
+<% if resources[:functions] %>    
+  LambdaCount:
+    Value: <%= resources[:functions].length %>
+<% resources[:functions].each_with_index do |resource, index| %>    
+  LambdaRoleARN<%= index %>:
+    Value: !GetAtt MyLambdaRole<%= index %>.Arn
+  LambdaFunctionARN<%= index %>:
+    Value: !GetAtt LambdaFunction<%= index %>.Arn
+  LambdaFunctionName<%= index %>:
+    Value: !Ref LambdaFunction<%= index %>
+<% end %>    
+<% end %>  
+
+<% if resources[:state_machine] %>
   StepFunctionsStateMachineARN:
     Value: !GetAtt StepFunctionsStateMachine.Arn
-    Condition: CreateStepfunctions
+  StateMachineType:
+    Value: !Ref StateMachineType
+  StepFunctionsRoleARN:
+    Value: !GetAtt StepFunctionsRole.Arn
+<% end %>    
+YAML
+
+  def self.cloudformation_yaml(resources)
+    template = ERB.new(CLOUDFORMATION_ERB_TEMPLATE)
+    template.result(binding)
+  end
+end

data/lib/simplerubysteps/dsl.rb

@@ -0,0 +1,86 @@
+require "json"
+
+module Simplerubysteps
+  $sm = StateMachine.new
+  $tasks = []
+
+  def kind(k)
+    $sm.kind = k
+  end
+
+  def task(name)
+    t = $sm.add Task.new(name)
+
+    $tasks.last.next = t if $tasks.last
+
+    $tasks.push t
+    yield if block_given?
+    $tasks.pop
+  end
+
+  def callback(name)
+    t = $sm.add Callback.new(name)
+
+    $tasks.last.next = t if $tasks.last
+
+    $tasks.push t
+    yield if block_given?
+    $tasks.pop
+  end
+
+  def action(&action_block)
+    $tasks.last.action &action_block
+  end
+
+  def transition(state)
+    $tasks.last.next = state
+  end
+
+  def transition_to(state, &condition_block)
+    choice = $tasks.last.implicit_choice
+
+    c = ChoiceItem.new({
+      :Variable => "$.#{choice.name}_#{state}",
+      :StringMatches => "yes",
+    })
+    c.next = state
+    c.implicit_condition_block = condition_block
+
+    choice.add c
+  end
+
+  def default_transition_to(state)
+    choice = $tasks.last.implicit_choice
+
+    choice.default = state
+  end
+
+  def choice(name)
+    t = $sm.add Choice.new(name)
+
+    $tasks.last.next = t if $tasks.last
+
+    $tasks.push t
+    yield if block_given?
+    $tasks.pop
+  end
+
+  def string_matches(var, match)
+    c = ChoiceItem.new({
+      :Variable => var,
+      :StringMatches => match,
+    })
+
+    $tasks.last.add c
+
+    $tasks.push c
+    yield if block_given?
+    $tasks.pop
+  end
+
+  def default
+    $tasks.push $tasks.last
+    yield if block_given?
+    $tasks.pop
+  end
+end

data/lib/simplerubysteps/model.rb

@@ -0,0 +1,204 @@
+module Simplerubysteps
+  $LAMBDA_FUNCTION_ARNS = ENV["LAMBDA_FUNCTION_ARNS"] ? ENV["LAMBDA_FUNCTION_ARNS"].split(",") : nil
+
+  def pop_function_arn
+    return "unknown" unless $LAMBDA_FUNCTION_ARNS
+    arn = $LAMBDA_FUNCTION_ARNS.first
+    $LAMBDA_FUNCTION_ARNS.delete arn
+    arn
+  end
+
+  def pop_function_name
+    return "unknown" unless pop_function_arn =~ /.+\:function\:(.+)/
+    $1
+  end
+
+  class StateMachine
+    attr_reader :states
+    attr_reader :start_at
+    attr_accessor :kind
+
+    def initialize()
+      @states = {}
+      @kind = "STANDARD"
+    end
+
+    def add(state)
+      @start_at = state.name unless @start_at
+
+      @states[state.name] = state
+      state.state_machine = self
+
+      state
+    end
+
+    def cloudformation_config
+      data = []
+      states.each do |name, state|
+        if state.is_a? Task or state.is_a? Callback
+          data.push({
+            task: name,
+          })
+        end
+      end
+      data
+    end
+
+    def render
+      {
+        :StartAt => @start_at,
+        :States => @states.map { |name, state| [name, state.render] }.to_h,
+      }
+    end
+  end
+
+  class State
+    attr_reader :name
+    attr_accessor :state_machine
+
+    def initialize(name)
+      @name = name
+      @dict = {}
+    end
+
+    def []=(key, value)
+      @dict[key] = value
+    end
+
+    def next=(state)
+      @dict[:Next] = (state.is_a? Symbol) ? state : state.name
+    end
+
+    def render
+      dict = @dict
+      dict[:End] = true unless dict[:Next]
+      dict
+    end
+  end
+
+  class Task < State
+    def initialize(name)
+      super
+      @dict[:Type] = "Task"
+      @dict[:Resource] = pop_function_arn
+      @dict[:Parameters] = {
+        :Task => name,
+        "Input.$" => "$",
+      }
+    end
+
+    def action(&action_block)
+      @action_block = action_block
+    end
+
+    def perform_action(input)
+      output = input # default: pass through
+
+      output = @action_block.call(input) if @action_block
+
+      if @implicit_choice
+        output = {} unless output
+        @implicit_choice.perform_action(output)
+      end
+
+      output
+    end
+
+    def implicit_choice
+      unless @implicit_choice
+        @implicit_choice = Choice.new("#{name}_choice")
+        $sm.add @implicit_choice
+        self.next = @implicit_choice
+      end
+      @implicit_choice
+    end
+  end
+
+  class Callback < State
+    def initialize(name)
+      super
+      @dict[:Type] = "Task"
+      @dict[:Resource] = "arn:aws:states:::lambda:invoke.waitForTaskToken"
+      @dict[:Parameters] = {
+        :FunctionName => pop_function_name,
+        :Payload => {
+          :Task => name,
+          "Input.$" => "$",
+          "Token.$" => "$$.Task.Token",
+        },
+      }
+    end
+
+    def action(&action_block)
+      @action_block = action_block
+    end
+
+    def perform_action(input, token)
+      @action_block.call(input, token) if @action_block
+    end
+  end
+
+  class ChoiceItem
+    attr_accessor :implicit_condition_block
+
+    def initialize(dict = {}, state = nil)
+      @dict = dict
+      self.next = state if state
+    end
+
+    def next=(state)
+      @dict[:Next] = (state.is_a? Symbol) ? state : state.name
+    end
+
+    def render
+      @dict
+    end
+
+    def perform_action(choice_name, output)
+      if @implicit_condition_block
+        output["#{choice_name}_#{@dict[:Next]}"] = @implicit_condition_block.call(output) ? "yes" : "no"
+      end
+    end
+  end
+
+  class Choice < State
+    attr_reader :choices
+
+    def initialize(name)
+      super
+      @choices = []
+      @dict[:Type] = "Choice"
+    end
+
+    def add(item)
+      @choices.push item
+    end
+
+    def add_string_matches(var, match, state)
+      add ChoiceItem.new({
+            :Variable => var,
+            :StringMatches => match,
+          }, state)
+    end
+
+    def default=(state)
+      @dict[:Default] = (state.is_a? Symbol) ? state : state.name
+    end
+
+    def next=(state)
+      self.default = state
+    end
+
+    def render
+      dict = @dict.clone
+      dict[:Choices] = @choices.map { |item| item.render }
+      dict
+    end
+
+    def perform_action(output)
+      @choices.each do |choice|
+        choice.perform_action name, output
+      end
+    end
+  end
+end