workato-connector-sdk 1.1.0 → 1.3.0

data/lib/workato/connector/sdk/connector.rb

@@ -1,6 +1,8 @@
 # typed: strict
 # frozen_string_literal: true
 
+using Workato::Extension::HashWithIndifferentAccess
+
 module Workato
   module Connector
     module Sdk
@@ -21,15 +23,10 @@ module Workato
 
         sig { params(definition: SorbetTypes::SourceHash, settings: SorbetTypes::SettingsHash).void }
         def initialize(definition, settings = {})
-          @source = T.let(definition.with_indifferent_access, HashWithIndifferentAccess)
-          @settings = T.let(settings.with_indifferent_access, HashWithIndifferentAccess)
-          @connection_source = T.let(@source[:connection] || {}.with_indifferent_access, HashWithIndifferentAccess)
-          @methods_source = T.let(@source[:methods] || {}.with_indifferent_access, HashWithIndifferentAccess)
-        end
-
-        sig { params(path: String, params: T::Hash[Symbol, T.untyped]).returns(T.untyped) }
-        def invoke(path, params = {})
-          InvokePath.new(path: path, connector: self, params: params).call
+          @source = T.let(HashWithIndifferentAccess.wrap(definition), HashWithIndifferentAccess)
+          @settings = T.let(HashWithIndifferentAccess.wrap(settings), HashWithIndifferentAccess)
+          @connection_source = T.let(HashWithIndifferentAccess.wrap(@source[:connection]), HashWithIndifferentAccess)
+          @methods_source = T.let(HashWithIndifferentAccess.wrap(@source[:methods]), HashWithIndifferentAccess)
         end
 
         sig { returns(T.nilable(String)) }
@@ -44,7 +41,8 @@ module Workato
             actions: source[:actions].presence || {},
             methods: methods_source,
             object_definitions: object_definitions,
-            connection: connection
+            connection: connection,
+            streams: streams
           )
         end
 
@@ -53,7 +51,8 @@ module Workato
           @methods = T.let(@methods, T.nilable(MethodsProxy))
           @methods ||= MethodsProxy.new(
             methods: methods_source,
-            connection: connection
+            connection: connection,
+            streams: streams
           )
         end
 
@@ -77,7 +76,8 @@ module Workato
             triggers: source[:triggers].presence || {},
             methods: methods_source,
             connection: connection,
-            object_definitions: object_definitions
+            object_definitions: object_definitions,
+            streams: streams
           )
         end
 
@@ -111,8 +111,34 @@ module Workato
           )
         end
 
+        sig { returns(Streams) }
+        def streams
+          @streams = T.let(@streams, T.nilable(Streams))
+          @streams ||= Streams.new(
+            streams: streams_sources,
+            methods: methods_source,
+            connection: connection
+          )
+        end
+
         private
 
+        sig { returns(HashWithIndifferentAccess) }
+        def streams_sources
+          @streams_sources = T.let(@streams_sources, T.nilable(HashWithIndifferentAccess))
+          return @streams_sources if @streams_sources
+
+          @streams_sources = HashWithIndifferentAccess.new
+          @streams_sources.merge!(source[:streams].presence || {})
+          (source[:actions] || {}).values.map do |action|
+            @streams_sources.merge!(action[:streams] || {})
+          end
+          (source[:trigger] || {}).values.map do |trigger|
+            @streams_sources.merge!(trigger[:streams] || {})
+          end
+          @streams_sources
+        end
+
         sig { returns(HashWithIndifferentAccess) }
         attr_reader :methods_source
 
@@ -131,18 +157,20 @@ module Workato
             actions: HashWithIndifferentAccess,
             object_definitions: ObjectDefinitions,
             methods: HashWithIndifferentAccess,
-            connection: Connection
+            connection: Connection,
+            streams: Streams
           ).void
         end
-        def initialize(actions:, object_definitions:, methods:, connection:)
+        def initialize(actions:, object_definitions:, methods:, connection:, streams:)
           @methods = methods
           @connection = connection
           @object_definitions = object_definitions
+          @streams = streams
           @actions = T.let({}, T::Hash[T.any(Symbol, String), Action])
           define_action_methods(actions)
         end
 
-        sig { params(action: T.any(Symbol, String)).returns(T.untyped) }
+        sig { params(action: T.any(Symbol, String)).returns(Action) }
         def [](action)
           public_send(action)
         end
@@ -155,6 +183,9 @@ module Workato
         sig { returns(Connection) }
         attr_reader :connection
 
+        sig { returns(Streams) }
+        attr_reader :streams
+
         sig { returns(ObjectDefinitions) }
         attr_reader :object_definitions
 
@@ -166,7 +197,8 @@ module Workato
                 action: definition,
                 object_definitions: object_definitions,
                 methods: methods,
-                connection: connection
+                connection: connection,
+                streams: streams
               )
               return @actions[action] if input_.nil?
 
@@ -182,12 +214,14 @@ module Workato
         sig do
           params(
             methods: HashWithIndifferentAccess,
-            connection: Connection
+            connection: Connection,
+            streams: Streams
           ).void
         end
-        def initialize(methods:, connection:)
+        def initialize(methods:, connection:, streams:)
           @methods = methods
           @connection = connection
+          @streams = streams
           @actions = T.let({}, T::Hash[T.any(Symbol, String), Action])
           define_action_methods
         end
@@ -200,6 +234,9 @@ module Workato
         sig { returns(Connection) }
         attr_reader :connection
 
+        sig { returns(Streams) }
+        attr_reader :streams
+
         sig { void }
         def define_action_methods
           methods.each do |method, _definition|
@@ -209,7 +246,8 @@ module Workato
                   execute: -> { T.unsafe(self).call(method, *args) }
                 },
                 methods: methods,
-                connection: connection
+                connection: connection,
+                streams: streams
               )
               T.must(@actions[method]).execute
             end
@@ -276,17 +314,24 @@ module Workato
             triggers: HashWithIndifferentAccess,
             object_definitions: ObjectDefinitions,
             methods: HashWithIndifferentAccess,
-            connection: Connection
+            connection: Connection,
+            streams: Streams
           ).void
         end
-        def initialize(triggers:, object_definitions:, methods:, connection:)
+        def initialize(triggers:, object_definitions:, methods:, connection:, streams:)
           @methods = methods
           @connection = connection
           @object_definitions = object_definitions
+          @streams = streams
           @triggers = T.let({}, T::Hash[T.any(Symbol, String), Trigger])
           define_trigger_methods(triggers)
         end
 
+        sig { params(trigger: T.any(Symbol, String)).returns(Trigger) }
+        def [](trigger)
+          public_send(trigger)
+        end
+
         private
 
         sig { returns(HashWithIndifferentAccess) }
@@ -295,6 +340,9 @@ module Workato
         sig { returns(Connection) }
         attr_reader :connection
 
+        sig { returns(Streams) }
+        attr_reader :streams
+
         sig { returns(ObjectDefinitions) }
         attr_reader :object_definitions
 
@@ -306,7 +354,8 @@ module Workato
                 trigger: definition,
                 object_definitions: object_definitions,
                 methods: methods,
-                connection: connection
+                connection: connection,
+                streams: streams
               )
 
               return @triggers[trigger] if input_.nil?
@@ -316,65 +365,6 @@ module Workato
           end
         end
       end
-
-      class InvokePath
-        extend T::Sig
-
-        sig do
-          params(
-            path: String,
-            connector: Connector,
-            params: T::Hash[Symbol, T.untyped]
-          ).void
-        end
-        def initialize(path:, connector:, params:)
-          @path = T.let(path, String)
-          @connector = T.let(connector, Connector)
-          @params = T.let(params, T::Hash[Symbol, T.untyped])
-        end
-
-        sig { returns(T.untyped) }
-        def call
-          invoke_path
-        end
-
-        private
-
-        sig { returns(String) }
-        attr_reader :path
-
-        sig { returns(Connector) }
-        attr_reader :connector
-
-        sig { returns(T::Hash[Symbol, T.untyped]) }
-        attr_reader :params
-
-        sig { returns(T.untyped) }
-        def invoke_path
-          methods = path.split('.')
-          method = methods.pop
-          raise ArgumentError, 'path is not found' unless method
-
-          object = methods.inject(connector) { |obj, m| obj.public_send(m) }
-          output = invoke_method(object, method)
-          if output.respond_to?(:invoke)
-            invoke_method(output, :invoke)
-          else
-            output
-          end
-        end
-
-        sig { params(object: T.untyped, method: T.any(Symbol, String)).returns(T.untyped) }
-        def invoke_method(object, method)
-          parameters = object.method(method).parameters.reject { |p| p[0] == :block }.map(&:second)
-          args = params.values_at(*parameters)
-          if parameters.last == :args
-            args = args.take(args.length - 1) + Array.wrap(args.last).flatten(1)
-          end
-          object.public_send(method, *args)
-        end
-      end
-      private_constant :InvokePath
     end
   end
 end

data/lib/workato/connector/sdk/core.rb

@@ -0,0 +1,62 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+require 'active_support'
+require 'active_support/core_ext/object/deep_dup'
+require 'active_support/core_ext/object/try'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/array/wrap'
+require 'active_support/core_ext/array/access'
+require 'active_support/core_ext/numeric/bytes'
+require 'active_support/core_ext/numeric/conversions'
+require 'active_support/core_ext/numeric/time'
+
+module Workato
+  module Connector
+    module Sdk
+      DEFAULT_MASTER_KEY_ENV = 'WORKATO_CONNECTOR_MASTER_KEY'
+      DEFAULT_MASTER_KEY_PATH = 'master.key'
+
+      DEFAULT_CONNECTOR_PATH = 'connector.rb'
+
+      DEFAULT_SETTINGS_PATH = 'settings.yaml'
+      DEFAULT_ENCRYPTED_SETTINGS_PATH = 'settings.yaml.enc'
+
+      DEFAULT_ACCOUNT_PROPERTIES_PATH = 'account_properties.yaml'
+      DEFAULT_ENCRYPTED_ACCOUNT_PROPERTIES_PATH = 'account_properties.yaml.enc'
+
+      DEFAULT_LOOKUP_TABLES_PATH = 'lookup_tables.yaml'
+
+      DEFAULT_TIME_ZONE = 'Pacific Time (US & Canada)'
+
+      DEFAULT_SCHEMAS_PATH = 'workato_schemas.json'
+
+      WORKATO_API_EMAIL_ENV = 'WORKATO_API_EMAIL'
+      WORKATO_API_TOKEN_ENV = 'WORKATO_API_TOKEN'
+
+      WORKATO_BASE_URL_ENV = 'WORKATO_BASE_URL'
+      DEFAULT_WORKATO_BASE_URL = 'https://app.workato.com'
+      WORKATO_BASE_URL = T.let(ENV.fetch(WORKATO_BASE_URL_ENV, DEFAULT_WORKATO_BASE_URL), String)
+    end
+  end
+end
+
+require_relative '../../extension/hash_with_indifferent_access'
+
+require_relative './errors'
+require_relative './account_properties'
+require_relative './operation'
+require_relative './connection'
+require_relative './stream'
+require_relative './streams'
+require_relative './action'
+require_relative './lookup_tables'
+require_relative './object_definitions'
+require_relative './request'
+require_relative './settings'
+require_relative './summarize'
+require_relative './trigger'
+require_relative './version'
+require_relative './workato_schemas'
+require_relative './connector'

data/lib/workato/connector/sdk/dsl/aws.rb

@@ -2,6 +2,9 @@
 # frozen_string_literal: true
 
 require 'aws-sigv4'
+require 'workato/utilities/xml'
+
+using Workato::Extension::HashWithIndifferentAccess
 
 module Workato
   module Connector
@@ -13,9 +16,9 @@ module Workato
           DUMMY_AWS_IAM_EXTERNAL_ID = 'dummy-aws-iam-external-id'
           DUMMY_AWS_WORKATO_ACCOUNT_ID = 'dummy-aws-workato-account-id'
 
-          AMAZON_ROLE_CLIENT_ID = ENV['AMAZON_ROLE_CLIENT_ID']
-          AMAZON_ROLE_CLIENT_KEY = ENV['AMAZON_ROLE_CLIENT_KEY']
-          AMAZON_ROLE_CLIENT_SECRET = ENV['AMAZON_ROLE_CLIENT_SECRET']
+          AMAZON_ROLE_CLIENT_ID = ENV.fetch('AMAZON_ROLE_CLIENT_ID', nil)
+          AMAZON_ROLE_CLIENT_KEY = ENV.fetch('AMAZON_ROLE_CLIENT_KEY', nil)
+          AMAZON_ROLE_CLIENT_SECRET = ENV.fetch('AMAZON_ROLE_CLIENT_SECRET', nil)
 
           WWW_FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded; charset=utf-8'
 
@@ -55,7 +58,7 @@ module Workato
                 method: method,
                 path: path,
                 params: params,
-                headers: (headers || {}).with_indifferent_access,
+                headers: HashWithIndifferentAccess.wrap(headers),
                 payload: payload
               )
 
@@ -134,7 +137,7 @@ module Workato
                 headers: headers,
                 method: :get
               )
-              response = Workato::Connector::Sdk::Xml.parse_xml_to_hash(response.body)
+              response = Workato::Utilities::Xml.parse_xml_to_hash(response.body)
 
               temp_credentials = response.dig('AssumeRoleResponse', 0, 'AssumeRoleResult', 0, 'Credentials', 0)
               {

data/lib/workato/connector/sdk/dsl/call.rb

@@ -6,7 +6,7 @@ module Workato
     module Sdk
       module Dsl
         module Call
-          def call(method, *args)
+          ruby2_keywords def call(method, *args)
             method_proc = @_methods[method]
 
             raise UndefinedMethodError, method unless method_proc

data/lib/workato/connector/sdk/dsl/csv_package.rb

@@ -0,0 +1,133 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'csv'
+
+module Workato
+  module Connector
+    module Sdk
+      CsvError = Class.new(Sdk::RuntimeError)
+
+      CsvFormatError = Class.new(CsvError)
+
+      class CsvFileTooBigError < CsvError
+        extend T::Sig
+
+        sig { returns(Integer) }
+        attr_reader :size
+
+        sig { returns(Integer) }
+        attr_reader :max
+
+        sig { params(size: Integer, max: Integer).void }
+        def initialize(size, max)
+          super("CSV file is too big. Max allowed: #{max.to_s(:human_size)}, got: #{size.to_s(:human_size)}")
+          @size = T.let(size, Integer)
+          @max = T.let(max, Integer)
+        end
+      end
+
+      class CsvFileTooManyLinesError < CsvError
+        extend T::Sig
+
+        sig { returns(Integer) }
+        attr_reader :max
+
+        sig { params(max: Integer).void }
+        def initialize(max)
+          super("CSV file has too many lines. Max allowed: #{max}")
+          @max = T.let(max, Integer)
+        end
+      end
+
+      module Dsl
+        class CsvPackage
+          extend T::Sig
+
+          MAX_FILE_SIZE_FOR_PARSE = T.let(30.megabytes, Integer)
+          private_constant :MAX_FILE_SIZE_FOR_PARSE
+
+          MAX_LINES_FOR_PARSE = 65_000
+          private_constant :MAX_LINES_FOR_PARSE
+
+          sig do
+            params(
+              str: String,
+              headers: T.any(T::Boolean, T::Array[String], String),
+              col_sep: T.nilable(String),
+              row_sep: T.nilable(String),
+              quote_char: T.nilable(String),
+              skip_blanks: T.nilable(T::Boolean),
+              skip_first_line: T::Boolean
+            ).returns(
+              T::Array[T::Hash[String, T.untyped]]
+            )
+          end
+          def parse(str, headers:, col_sep: nil, row_sep: nil, quote_char: nil, skip_blanks: nil,
+                    skip_first_line: false)
+            if str.bytesize > MAX_FILE_SIZE_FOR_PARSE
+              raise CsvFileTooBigError.new(str.bytesize, MAX_FILE_SIZE_FOR_PARSE)
+            end
+
+            index = 0
+            options = { col_sep: col_sep, row_sep: row_sep, quote_char: quote_char, headers: headers,
+                        skip_blanks: skip_blanks }.compact
+            Enumerator.new do |consumer|
+              CSV.parse(str, **options) do |row|
+                if index.zero? && skip_first_line
+                  index += 1
+                  next
+                end
+                if index == MAX_LINES_FOR_PARSE
+                  raise CsvFileTooManyLinesError, MAX_LINES_FOR_PARSE
+                end
+
+                index += 1
+                consumer.yield(T.cast(row, CSV::Row).to_hash)
+              end
+            end.to_a
+          rescue CSV::MalformedCSVError => e
+            raise CsvFormatError, e
+          rescue ArgumentError => e
+            raise Sdk::RuntimeError, e.message
+          end
+
+          sig do
+            params(
+              str: T.nilable(String),
+              headers: T.nilable(T::Array[String]),
+              col_sep: T.nilable(String),
+              row_sep: T.nilable(String),
+              quote_char: T.nilable(String),
+              force_quotes: T.nilable(T::Boolean),
+              blk: T.proc.params(csv: CSV).void
+            ).returns(
+              String
+            )
+          end
+          def generate(str = nil, headers: nil, col_sep: nil, row_sep: nil, quote_char: nil, force_quotes: nil, &blk)
+            options = { col_sep: col_sep, row_sep: row_sep, quote_char: quote_char, headers: headers,
+                        force_quotes: force_quotes }.compact
+            options[:write_headers] = options[:headers].present?
+
+            ::CSV.generate(str || String.new, **options, &blk)
+          rescue ArgumentError => e
+            raise Sdk::RuntimeError, e.message
+          end
+
+          private
+
+          T::Sig::WithoutRuntime.sig { params(symbol: T.any(String, Symbol), _args: T.untyped).void }
+          def method_missing(symbol, *_args)
+            raise UndefinedStdLibMethodError.new(symbol.to_s, 'workato.csv')
+          end
+
+          T::Sig::WithoutRuntime.sig { params(_args: T.untyped).returns(T::Boolean) }
+          def respond_to_missing?(*_args)
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/workato/connector/sdk/dsl/execution_context.rb

@@ -0,0 +1,45 @@
+# typed: true
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'active_support/core_ext/class/attribute'
+
+module Workato
+  module Connector
+    module Sdk
+      module Dsl
+        module ExecutionContext
+          extend T::Sig
+          extend T::Helpers
+          extend ActiveSupport::Concern
+
+          included do
+            T.bind(self, Class)
+
+            # encrypted safe recipe_id
+            class_attribute :recipe_id, instance_predicate: false, default: SecureRandom.hex(32)
+          end
+
+          sig { returns(T::Hash[Symbol, T.untyped]) }
+          def execution_context
+            @execution_context ||= {
+              recipe_id: recipe_id
+            }.compact
+          end
+
+          # mock unencrypted recipe_id for testing only
+          def recipe_id!
+            recipe_id.reverse
+          end
+
+          module ClassMethods
+            # mock unencrypted recipe_id for testing only
+            def recipe_id!
+              T.unsafe(self).recipe_id.reverse
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/workato/connector/sdk/dsl/http.rb

@@ -53,7 +53,7 @@ module Workato
               exception = nil
               begin
                 response = request.execute!
-              rescue RequestError, RuntimeError => e
+              rescue StandardError => e
                 exception = e.to_s
               end
               result[PARALLEL_SUCCESS_INDEX] &&= exception.nil?

data/lib/workato/connector/sdk/dsl/reinvoke_after.rb

@@ -0,0 +1,84 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Workato
+  module Connector
+    module Sdk
+      module Dsl
+        module ReinvokeAfter
+          extend T::Sig
+
+          sig do
+            params(
+              continue: T::Hash[T.untyped, T.untyped],
+              temp_output: T.nilable(T::Hash[T.untyped, T.untyped])
+            ).void
+          end
+          def checkpoint!(continue:, temp_output: nil)
+            # no-op
+          end
+
+          sig do
+            params(
+              seconds: T.any(Integer, Float),
+              continue: T::Hash[T.untyped, T.untyped],
+              temp_output: T.nilable(T::Hash[T.untyped, T.untyped])
+            ).void
+          end
+          def reinvoke_after(seconds:, continue:, temp_output: nil) # rubocop:disable Lint/UnusedMethodArgument
+            Kernel.throw REINVOKE_AFTER_SIGNAL, ReinvokeParams.new(
+              seconds: seconds,
+              continue: continue
+            )
+          end
+
+          private
+
+          MAX_REINVOKES = 5
+          private_constant :MAX_REINVOKES
+
+          REINVOKE_AFTER_SIGNAL = :reinvoke_after
+          private_constant :REINVOKE_AFTER_SIGNAL
+
+          class ReinvokeParams < T::Struct
+            prop :seconds, T.any(Float, Integer)
+            prop :continue, T::Hash[T.untyped, T.untyped]
+          end
+          private_constant :ReinvokeParams
+
+          sig { params(continue: T::Hash[T.any(Symbol, String), T.untyped], _blk: Proc).returns(T.untyped) }
+          def loop_reinvoke_after(continue, &_blk)
+            reinvokes_remaining = T.let(reinvoke_limit, Integer)
+
+            Kernel.loop do
+              reinvoke_after = Kernel.catch(REINVOKE_AFTER_SIGNAL) do
+                return yield(continue)
+              end
+
+              if reinvokes_remaining.zero?
+                Kernel.raise "Max number of reinvokes on SDK Gem reached. Current limit is #{reinvoke_limit}"
+              end
+
+              reinvokes_remaining -= 1
+
+              reinvoke_after = T.cast(reinvoke_after, ReinvokeParams)
+              reinvoke_sleep(reinvoke_after.seconds)
+              continue = reinvoke_after.continue
+            end
+          end
+
+          sig { params(seconds: T.any(Float, Integer)).void }
+          def reinvoke_sleep(seconds)
+            Kernel.sleep((ENV['WAIT_REINVOKE_AFTER'].presence || seconds).to_f)
+          end
+
+          sig { returns(Integer) }
+          def reinvoke_limit
+            @reinvoke_limit = T.let(@reinvoke_limit, T.nilable(Integer))
+            @reinvoke_limit ||= (ENV['MAX_REINVOKES'].presence || MAX_REINVOKES).to_i
+          end
+        end
+      end
+    end
+  end
+end