floe 0.11.3 → 0.13.0

data/lib/floe/workflow/intrinsic_function/transformer.rb

@@ -0,0 +1,266 @@
+# Disable rubocops against the `match` method, since this is a Parslet specific
+#   match method and not the typical `Object#match`.
+# rubocop:disable Performance/RegexpMatch, Performance/RedundantMatch
+
+require "parslet"
+
+module Floe
+  class Workflow
+    class IntrinsicFunction
+      class Transformer < Parslet::Transform
+        OptionalArg = Struct.new(:type)
+        VariadicArgs = Struct.new(:type)
+
+        class << self
+          def process_args(args, function, signature = nil)
+            args = resolve_args(args)
+            if signature
+              check_arity(args, function, signature)
+              check_types(args, function, signature)
+            end
+            args
+          end
+
+          private
+
+          def resolve_args(args)
+            if args.nil?
+              # 0 args
+              []
+            elsif args.kind_of?(Array)
+              # >1 arg
+              args.map { |a| a[:arg] }
+            else
+              # 1 arg
+              [args[:arg]]
+            end
+          end
+
+          def check_arity(args, function, signature)
+            if signature.any?(OptionalArg)
+              signature_required = signature.reject { |a| a.kind_of?(OptionalArg) }
+              signature_size = (signature_required.size..signature.size)
+
+              raise ArgumentError, "wrong number of arguments to #{function} (given #{args.size}, expected #{signature_size})" unless signature_size.include?(args.size)
+            elsif signature.any?(VariadicArgs)
+              signature_required = signature.reject { |a| a.kind_of?(VariadicArgs) }
+
+              raise ArgumentError, "wrong number of arguments to #{function} (given #{args.size}, expected at least #{signature_required.size})" unless args.size >= signature_required.size
+            else
+              raise ArgumentError, "wrong number of arguments to #{function} (given #{args.size}, expected #{signature.size})" unless signature.size == args.size
+            end
+          end
+
+          def check_types(args, function, signature)
+            # Adjust the signature for VariadicArgs to create a copy of the expected type for each given arg
+            if signature.last.kind_of?(VariadicArgs)
+              signature = signature[0..-2] + Array.new(args.size - signature.size + 1, signature.last.type)
+            end
+
+            args.zip(signature).each_with_index do |(arg, type), index|
+              type = type.type if type.kind_of?(OptionalArg)
+
+              if type.kind_of?(Array)
+                raise ArgumentError, "wrong type for argument #{index + 1} to #{function} (given #{arg.class}, expected one of #{type.join(", ")})" unless type.any? { |t| arg.kind_of?(t) }
+              else
+                raise ArgumentError, "wrong type for argument #{index + 1} to #{function} (given #{arg.class}, expected #{type})" unless arg.kind_of?(type)
+              end
+            end
+          end
+        end
+
+        rule(:null_literal  => simple(:v)) { nil }
+        rule(:true_literal  => simple(:v)) { true }
+        rule(:false_literal => simple(:v)) { false }
+
+        rule(:string   => simple(:v)) { v.to_s[1..-2] }
+        rule(:number   => simple(:v)) { v.match(/[eE.]/) ? Float(v) : Integer(v) }
+        rule(:jsonpath => simple(:v)) { Floe::Workflow::Path.value(v.to_s, context, input) }
+
+        STATES_FORMAT_PLACEHOLDER = /(?<!\\)\{\}/.freeze
+
+        rule(:states_format => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.Format", [String, VariadicArgs[[String, TrueClass, FalseClass, Integer, Float, NilClass]]])
+          str, *rest = *args
+
+          # TODO: Handle templates with escaped characters, including invalid templates
+          #   See https://states-language.net/#intrinsic-functions (number 6)
+
+          expected_args = str.scan(STATES_FORMAT_PLACEHOLDER).size
+          actual_args = rest.size
+          if expected_args != actual_args
+            raise ArgumentError, "number of arguments to States.Format do not match the occurrences of {} (given #{actual_args}, expected #{expected_args})"
+          end
+
+          rest.each do |arg|
+            str = str.sub(STATES_FORMAT_PLACEHOLDER, arg.nil? ? "null" : arg.to_s)
+          end
+
+          # TODO: Handle arguments that have escape characters within them but are interpolated
+          str.gsub!("\\'", "'")
+          str.gsub!("\\{", "{")
+          str.gsub!("\\}", "}")
+          str.gsub!("\\\\", "\\")
+
+          str
+        end
+
+        rule(:states_json_to_string => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.JsonToString", [Object])
+          json = args.first
+
+          JSON.generate(json)
+        end
+
+        rule(:states_string_to_json => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.StringToJson", [String])
+          str = args.first
+
+          JSON.parse(str)
+        rescue JSON::ParserError => e
+          raise ArgumentError, "invalid value for argument 1 to States.StringToJson (invalid json: #{e.message})"
+        end
+
+        rule(:states_array => {:args => subtree(:args)}) do
+          Transformer.process_args(args, "States.Array")
+        end
+
+        rule(:states_array_partition => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.ArrayPartition", [Array, Integer])
+          array, chunk = *args
+          raise ArgumentError, "invalid value for argument 2 to States.ArrayPartition (given #{chunk}, expected a positive Integer)" unless chunk.positive?
+
+          array.each_slice(chunk).to_a
+        end
+
+        rule(:states_array_contains => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.ArrayContains", [Array, Object])
+          array, target = *args
+
+          array.include?(target)
+        end
+
+        rule(:states_array_range => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.ArrayRange", [Integer, Integer, Integer])
+          range_begin, range_end, increment = *args
+          raise ArgumentError, "invalid value for argument 3 to States.ArrayRange (given #{increment}, expected a non-zero Integer)" if increment.zero?
+
+          (range_begin..range_end).step(increment).to_a
+        end
+
+        rule(:states_array_get_item => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.ArrayGetItem", [Array, Integer])
+          array, index = *args
+          raise ArgumentError, "invalid value for argument 2 to States.ArrayGetItem (given #{index}, expected 0 or a positive Integer)" unless index >= 0
+
+          array[index]
+        end
+
+        rule(:states_array_length => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.ArrayLength", [Array])
+          array = args.first
+
+          array.size
+        end
+
+        rule(:states_array_unique => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.ArrayUnique", [Array])
+          array = args.first
+
+          array.uniq
+        end
+
+        rule(:states_base64_encode => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.Base64Encode", [String])
+          str = args.first
+
+          require "base64"
+          Base64.strict_encode64(str).force_encoding("UTF-8")
+        end
+
+        rule(:states_base64_decode => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.Base64Decode", [String])
+          str = args.first
+
+          require "base64"
+          begin
+            Base64.strict_decode64(str)
+          rescue ArgumentError => err
+            raise ArgumentError, "invalid value for argument 1 to States.Base64Decode (#{err})"
+          end
+        end
+
+        rule(:states_hash => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.Hash", [Object, String])
+          data, algorithm = *args
+
+          if data.nil?
+            raise ArgumentError, "invalid value for argument 1 to States.Hash (given null, expected non-null)"
+          end
+
+          algorithms = %w[MD5 SHA-1 SHA-256 SHA-384 SHA-512]
+          unless algorithms.include?(algorithm)
+            raise ArgumentError, "invalid value for argument 2 to States.Hash (given #{algorithm.inspect}, expected one of #{algorithms.map(&:inspect).join(", ")})"
+          end
+
+          require "openssl"
+          algorithm = algorithm.sub("-", "")
+          data = JSON.generate(data) unless data.kind_of?(String)
+          OpenSSL::Digest.hexdigest(algorithm, data).force_encoding("UTF-8")
+        end
+
+        rule(:states_json_merge => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.JsonMerge", [Hash, Hash, [TrueClass, FalseClass]])
+          left, right, deep = *args
+
+          if deep
+            # NOTE: not implemented by AWS Step Functions and nuances not defined in docs
+            left.merge(right) { |_key, l, r| l.kind_of?(Hash) && r.kind_of?(Hash) ? l.merge(r) : r }
+          else
+            left.merge(right)
+          end
+        end
+
+        rule(:states_math_random => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.MathRandom", [Integer, Integer, OptionalArg[Integer]])
+          range_start, range_end, seed = *args
+          unless range_start < range_end
+            raise ArgumentError, "invalid values for arguments to States.MathRandom (start must be less than end)"
+          end
+
+          seed ||= Random.new_seed
+          Random.new(seed).rand(range_start..range_end)
+        end
+
+        rule(:states_math_add => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.MathAdd", [Integer, Integer])
+
+          args.sum
+        end
+
+        rule(:states_string_split => {:args => subtree(:args)}) do
+          args = Transformer.process_args(args(), "States.StringSplit", [String, String])
+          str, delimeter = *args
+
+          case delimeter.size
+          when 0
+            str.empty? ? [] : [str]
+          when 1
+            str.split(delimeter)
+          else
+            str.split(/[#{Regexp.escape(delimeter)}]+/)
+          end
+        end
+
+        rule(:states_uuid => {:args => subtree(:args)}) do
+          Transformer.process_args(args, "States.UUID", [])
+
+          require "securerandom"
+          SecureRandom.uuid
+        end
+      end
+    end
+  end
+end
+
+# rubocop:enable Performance/RegexpMatch, Performance/RedundantMatch

data/lib/floe/workflow/intrinsic_function.rb

@@ -0,0 +1,34 @@
+require "parslet"
+
+module Floe
+  class Workflow
+    class IntrinsicFunction
+      def self.value(payload, context = {}, input = {})
+        new(payload).value(context, input)
+      end
+
+      def self.intrinsic_function?(payload)
+        payload.start_with?("States.")
+      end
+
+      attr_reader :payload
+
+      def initialize(payload)
+        @payload = payload
+        @tree    = Parser.new.parse(payload)
+
+        Floe.logger.debug { "Parsed intrinsic function: #{payload.inspect} => #{tree.inspect}" }
+      rescue Parslet::ParseFailed => err
+        raise Floe::InvalidWorkflowError, err.message
+      end
+
+      def value(context = {}, input = {})
+        Transformer.new.apply(tree, :context => context, :input => input)
+      end
+
+      private
+
+      attr_reader :tree
+    end
+  end
+end

data/lib/floe/workflow/path.rb

@@ -4,6 +4,10 @@ module Floe
   class Workflow
     class Path
       class << self
+        def path?(payload)
+          payload.start_with?("$")
+        end
+
         def value(payload, context, input = {})
           new(payload).value(context, input)
         end
@@ -26,9 +30,22 @@ module Floe
             [input, payload]
           end
 
+        # If path is $ then just return the entire input
+        return obj if path == "$"
+
         results = JsonPath.on(obj, path)
+        case results.count
+        when 0
+          raise Floe::PathError, "Path [#{payload}] references an invalid value"
+        when 1
+          results.first
+        else
+          results
+        end
+      end
 
-        results.count < 2 ? results.first : results
+      def to_s
+        payload
       end
     end
   end

data/lib/floe/workflow/payload_template.rb

@@ -42,14 +42,17 @@ module Floe
       end
 
       def parse_payload_string(value)
-        value.start_with?("$") ? Path.new(value) : value
+        return Path.new(value)              if Path.path?(value)
+        return IntrinsicFunction.new(value) if IntrinsicFunction.intrinsic_function?(value)
+
+        value
       end
 
       def interpolate_value(value, context, inputs)
         case value
-        when Array then interpolate_value_array(value, context, inputs)
-        when Hash  then interpolate_value_hash(value, context, inputs)
-        when Path  then value.value(context, inputs)
+        when Array                   then interpolate_value_array(value, context, inputs)
+        when Hash                    then interpolate_value_hash(value, context, inputs)
+        when Path, IntrinsicFunction then value.value(context, inputs)
         else
           value
         end

data/lib/floe/workflow/retrier.rb

@@ -3,15 +3,21 @@
 module Floe
   class Workflow
     class Retrier
-      attr_reader :error_equals, :interval_seconds, :max_attempts, :backoff_rate
+      include ErrorMatcherMixin
+      include ValidationMixin
 
-      def initialize(payload)
-        @payload = payload
+      attr_reader :error_equals, :interval_seconds, :max_attempts, :backoff_rate, :name
+
+      def initialize(_workflow, name, payload)
+        @name             = name
+        @payload          = payload
 
         @error_equals     = payload["ErrorEquals"]
         @interval_seconds = payload["IntervalSeconds"] || 1.0
         @max_attempts     = payload["MaxAttempts"] || 3
         @backoff_rate     = payload["BackoffRate"] || 2.0
+
+        missing_field_error!("ErrorEquals") if !@error_equals.kind_of?(Array) || @error_equals.empty?
       end
 
       # @param [Integer] attempt 1 for the first attempt

data/lib/floe/workflow/state.rb

@@ -4,16 +4,18 @@ module Floe
   class Workflow
     class State
       include Logging
+      include ValidationMixin
 
       class << self
         def build!(workflow, name, payload)
           state_type = payload["Type"]
-          raise Floe::InvalidWorkflowError, "Missing \"Type\" field in state [#{name}]" if payload["Type"].nil?
+          missing_field_error!(name, "Type") if payload["Type"].nil?
+          invalid_field_error!(name[0..-2], "Name", name.last, "must be less than or equal to 80 characters") if name.last.length > 80
 
           begin
             klass = Floe::Workflow::States.const_get(state_type)
           rescue NameError
-            raise Floe::InvalidWorkflowError, "Invalid state type: [#{state_type}]"
+            invalid_field_error!(name, "Type", state_type, "is not valid")
           end
 
           klass.new(workflow, name, payload)
@@ -22,14 +24,11 @@ module Floe
 
       attr_reader :comment, :name, :type, :payload
 
-      def initialize(workflow, name, payload)
+      def initialize(_workflow, name, payload)
         @name     = name
         @payload  = payload
         @type     = payload["Type"]
         @comment  = payload["Comment"]
-
-        raise Floe::InvalidWorkflowError, "Missing \"Type\" field in state [#{name}]" if payload["Type"].nil?
-        raise Floe::InvalidWorkflowError, "State name [#{name[..79]}...] must be less than or equal to 80 characters" if name.length > 80
       end
 
       def wait(context, timeout: nil)
@@ -49,27 +48,45 @@ module Floe
         return Errno::EAGAIN unless ready?(context)
 
         finish(context)
+      rescue Floe::Error => e
+        mark_error(context, e)
       end
 
       def start(context)
+        mark_started(context)
+      end
+
+      def finish(context)
+        mark_finished(context)
+      end
+
+      def mark_started(context)
         context.state["EnteredTime"] = Time.now.utc.iso8601
 
-        logger.info("Running state: [#{long_name}] with input [#{context.input}]...")
+        logger.info("Running state: [#{long_name}] with input [#{context.json_input}]...")
       end
 
-      def finish(context)
-        finished_time     = Time.now.utc
-        entered_time      = Time.parse(context.state["EnteredTime"])
+      def mark_finished(context)
+        finished_time = Time.now.utc
+        entered_time  = Time.parse(context.state["EnteredTime"])
 
         context.state["FinishedTime"] ||= finished_time.iso8601
         context.state["Duration"]       = finished_time - entered_time
 
-        level = context.output&.[]("Error") ? :error : :info
-        logger.public_send(level, "Running state: [#{long_name}] with input [#{context.input}]...Complete #{context.next_state ? "- next state [#{context.next_state}]" : "workflow -"} output: [#{context.output}]")
+        level = context.failed? ? :error : :info
+        logger.public_send(level, "Running state: [#{long_name}] with input [#{context.json_input}]...Complete #{context.next_state ? "- next state [#{context.next_state}]" : "workflow -"} output: [#{context.json_output}]")
 
         0
       end
 
+      def mark_error(context, exception)
+        # InputPath or OutputPath were bad.
+        context.next_state = nil
+        context.output     = {"Error" => "States.Runtime", "Cause" => exception.message}
+        # Since finish threw an exception, super was never called. Calling that now.
+        mark_finished(context)
+      end
+
       def ready?(context)
         !context.state_started? || !running?(context)
       end
@@ -86,8 +103,12 @@ module Floe
         context.state["WaitUntil"] && Time.parse(context.state["WaitUntil"])
       end
 
+      def short_name
+        name.last
+      end
+
       def long_name
-        "#{@type}:#{name}"
+        "#{type}:#{short_name}"
       end
 
       private
@@ -95,7 +116,7 @@ module Floe
       def wait_until!(context, seconds: nil, time: nil)
         context.state["WaitUntil"] =
           if seconds
-            (Time.parse(context.state["EnteredTime"]) + seconds).iso8601
+            (Time.now + seconds).iso8601
           elsif time.kind_of?(String)
             time
           else

data/lib/floe/workflow/states/choice.rb

@@ -11,7 +11,7 @@ module Floe
 
           validate_state!(workflow)
 
-          @choices = payload["Choices"].map { |choice| ChoiceRule.build(choice) }
+          @choices = payload["Choices"].map.with_index { |choice, i| ChoiceRule.build(workflow, name + ["Choices", i.to_s], choice) }
           @default = payload["Default"]
 
           @input_path  = Path.new(payload.fetch("InputPath", "$"))
@@ -19,7 +19,8 @@ module Floe
         end
 
         def finish(context)
-          output     = output_path.value(context, context.input)
+          input      = input_path.value(context, context.input)
+          output     = output_path.value(context, input)
           next_state = choices.detect { |choice| choice.true?(context, output) }&.next || default
 
           context.next_state = next_state
@@ -43,12 +44,12 @@ module Floe
         end
 
         def validate_state_choices!
-          raise Floe::InvalidWorkflowError, "Choice state must have \"Choices\"" unless payload.key?("Choices")
-          raise Floe::InvalidWorkflowError, "\"Choices\" must be a non-empty array" unless payload["Choices"].kind_of?(Array) && !payload["Choices"].empty?
+          missing_field_error!("Choices") unless payload.key?("Choices")
+          invalid_field_error!("Choices", nil, "must be a non-empty array") unless payload["Choices"].kind_of?(Array) && !payload["Choices"].empty?
         end
 
         def validate_state_default!(workflow)
-          raise Floe::InvalidWorkflowError, "\"Default\" not in \"States\"" unless workflow.payload["States"].include?(payload["Default"])
+          invalid_field_error!("Default", payload["Default"], "is not found in \"States\"") if payload["Default"] && !workflow_state?(payload["Default"], workflow)
         end
       end
     end

data/lib/floe/workflow/states/fail.rb

@@ -4,7 +4,7 @@ module Floe
   class Workflow
     module States
       class Fail < Floe::Workflow::State
-        attr_reader :cause, :error
+        attr_reader :cause, :error, :cause_path, :error_path
 
         def initialize(workflow, name, payload)
           super

data/lib/floe/workflow/states/non_terminal_mixin.rb

@@ -12,8 +12,8 @@ module Floe
         end
 
         def validate_state_next!(workflow)
-          raise Floe::InvalidWorkflowError, "Missing \"Next\" field in state [#{name}]" if @next.nil? && !@end
-          raise Floe::InvalidWorkflowError, "\"Next\" [#{@next}] not in \"States\" for state [#{name}]" if @next && !workflow.payload["States"].key?(@next)
+          missing_field_error!("Next") if @next.nil? && !@end
+          invalid_field_error!("Next", @next, "is not found in \"States\"") if @next && !workflow_state?(@next, workflow)
         end
       end
     end

data/lib/floe/workflow/states/pass.rb

@@ -25,7 +25,8 @@ module Floe
         end
 
         def finish(context)
-          context.output = process_output(context, result)
+          input = result.nil? ? process_input(context) : result
+          context.output = process_output(context, input)
           super
         end
 

data/lib/floe/workflow/states/succeed.rb

@@ -6,9 +6,18 @@ module Floe
       class Succeed < Floe::Workflow::State
         attr_reader :input_path, :output_path
 
+        def initialize(workflow, name, payload)
+          super
+
+          @input_path  = Path.new(payload.fetch("InputPath", "$"))
+          @output_path = Path.new(payload.fetch("OutputPath", "$"))
+        end
+
         def finish(context)
+          input              = input_path.value(context, context.input)
+          context.output     = output_path.value(context, input)
           context.next_state = nil
-          context.output     = context.input
+
           super
         end
 

data/lib/floe/workflow/states/task.rb

@@ -18,10 +18,13 @@ module Floe
           @next              = payload["Next"]
           @end               = !!payload["End"]
           @resource          = payload["Resource"]
-          @runner            = Floe::Runner.for_resource(@resource)
+
+          missing_field_error!("Resource") unless @resource.kind_of?(String)
+          @runner = wrap_parser_error("Resource", @resource) { Floe::Runner.for_resource(@resource) }
+
           @timeout_seconds   = payload["TimeoutSeconds"]
-          @retry             = payload["Retry"].to_a.map { |retrier| Retrier.new(retrier) }
-          @catch             = payload["Catch"].to_a.map { |catcher| Catcher.new(catcher) }
+          @retry             = payload["Retry"].to_a.map.with_index { |retrier, i| Retrier.new(workflow, name + ["Retry", i.to_s], retrier) }
+          @catch             = payload["Catch"].to_a.map.with_index { |catcher, i| Catcher.new(workflow, name + ["Catch", i.to_s], catcher) }
           @input_path        = Path.new(payload.fetch("InputPath", "$"))
           @output_path       = Path.new(payload.fetch("OutputPath", "$"))
           @result_path       = ReferencePath.new(payload.fetch("ResultPath", "$"))
@@ -30,8 +33,6 @@ module Floe
           @credentials       = PayloadTemplate.new(payload["Credentials"])    if payload["Credentials"]
 
           validate_state!(workflow)
-        rescue ArgumentError => err
-          raise Floe::InvalidWorkflowError, err.message
         end
 
         def start(context)
@@ -82,11 +83,11 @@ module Floe
         end
 
         def find_retrier(error)
-          self.retry.detect { |r| (r.error_equals & [error, "States.ALL"]).any? }
+          self.retry.detect { |r| r.match_error?(error) }
         end
 
         def find_catcher(error)
-          self.catch.detect { |c| (c.error_equals & [error, "States.ALL"]).any? }
+          self.catch.detect { |c| c.match_error?(error) }
         end
 
         def retry_state!(context, error)
@@ -106,7 +107,7 @@ module Floe
           wait_until!(context, :seconds => retrier.sleep_duration(context["State"]["RetryCount"]))
           context.next_state = context.state_name
           context.output     = error
-          logger.info("Running state: [#{long_name}] with input [#{context.input}] got error[#{context.output}]...Retry - delay: #{wait_until(context)}")
+          logger.info("Running state: [#{long_name}] with input [#{context.json_input}] got error[#{context.json_output}]...Retry - delay: #{wait_until(context)}")
           true
         end
 
@@ -116,7 +117,7 @@ module Floe
 
           context.next_state = catcher.next
           context.output     = catcher.result_path.set(context.input, error)
-          logger.info("Running state: [#{long_name}] with input [#{context.input}]...CatchError - next state: [#{context.next_state}] output: [#{context.output}]")
+          logger.info("Running state: [#{long_name}] with input [#{context.json_input}]...CatchError - next state: [#{context.next_state}] output: [#{context.json_output}]")
 
           true
         end
@@ -126,7 +127,7 @@ module Floe
           # keeping in here for completeness
           context.next_state = nil
           context.output = error
-          logger.error("Running state: [#{long_name}] with input [#{context.input}]...Complete workflow - output: [#{context.output}]")
+          logger.error("Running state: [#{long_name}] with input [#{context.json_input}]...Complete workflow - output: [#{context.json_output}]")
         end
 
         def parse_error(output)