workato-connector-sdk 1.1.0 → 1.3.0

data/lib/workato/connector/sdk/stream.rb

@@ -0,0 +1,243 @@
+# typed: strict
+# frozen_string_literal: true
+
+using Workato::Extension::HashWithIndifferentAccess
+
+module Workato
+  module Connector
+    module Sdk
+      module SorbetTypes
+        StreamProc = T.type_alias do
+          T.proc.params(
+            input: SorbetTypes::StreamInputHash,
+            from: Integer,
+            to: Integer,
+            frame_size: Integer
+          ).returns(SorbetTypes::StreamOutput)
+        end
+
+        StreamInProc = T.type_alias do
+          T.proc.params(
+            data: T.untyped,
+            from: Integer,
+            to: T.nilable(Integer),
+            eof: T::Boolean,
+            next_from: T.nilable(Integer)
+          ).void
+        end
+
+        StreamInputHash = T.type_alias { T::Hash[T.any(Symbol, String), T.untyped] }
+
+        StreamOutput = T.type_alias { [T.untyped, T::Boolean] }
+      end
+
+      class Stream < Operation
+        extend T::Sig
+
+        DEFAULT_FRAME_SIZE = T.let(10.megabytes, Integer)
+
+        using BlockInvocationRefinements
+        include Dsl::ReinvokeAfter
+
+        sig do
+          params(
+            stream: SorbetTypes::StreamProc,
+            methods: SorbetTypes::SourceHash,
+            connection: Connection
+          ).void
+        end
+        def initialize(stream:, methods: {}, connection: Connection.new)
+          super(methods: methods, connection: connection)
+          @stream_proc = stream
+        end
+
+        sig do
+          params(
+            input: SorbetTypes::StreamInputHash,
+            from: Integer,
+            to: Integer,
+            frame_size: Integer
+          ).returns(SorbetTypes::StreamOutput)
+        end
+        def chunk(input = {}, from = 0, to = from + DEFAULT_FRAME_SIZE, frame_size = DEFAULT_FRAME_SIZE)
+          raise "'frame_size' must be a positive integer number" unless frame_size.positive?
+
+          stream_proc = @stream_proc
+          execute(nil, { input: input, from: from, to: to, size: frame_size }) do |_, input| # rubocop:disable Lint/ShadowingOuterLocalVariable
+            T.unsafe(self).instance_exec(input['input'], input['from'], input['to'], input['size'], &stream_proc)
+          end
+        end
+
+        sig { params(input: SorbetTypes::StreamInputHash, frame_size: Integer).returns(T.untyped) }
+        def invoke(input = {}, frame_size = DEFAULT_FRAME_SIZE)
+          proxy = Proxy.new(name: '', input: input, stream: self)
+          reader = Reader.new(stream: proxy, frame_size: frame_size)
+          data = T.let(nil, T.untyped)
+          reader.each_chunk do |chunk|
+            data = data.nil? ? chunk : data + chunk
+          end
+          data
+        end
+
+        class Reader
+          extend T::Sig
+
+          ProxyReadProc = T.type_alias do
+            T.proc.params(
+              data: T.untyped,
+              from: Integer,
+              eof: T::Boolean,
+              next_from: T.nilable(Integer)
+            ).void
+          end
+
+          sig do
+            params(
+              stream: T.any(Proxy, T::Hash[T.untyped, T.untyped], String),
+              from: T.nilable(Integer),
+              frame_size: T.nilable(Integer)
+            ).void
+          end
+          def initialize(stream:, from: nil, frame_size: nil)
+            @stream = T.let(
+              stream.is_a?(Hash) && stream[:__stream__] ? from_mock(stream) : stream,
+              T.any(Proxy, Mock, T::Hash[T.untyped, T.untyped], String)
+            )
+            @from = T.let(from || 0, Integer)
+            @frame_size = T.let(frame_size || DEFAULT_FRAME_SIZE, Integer)
+          end
+
+          sig { params(_blk: SorbetTypes::StreamInProc).void }
+          def each_chunk(&_blk)
+            case @stream
+            when Proxy, Mock
+              @stream.read(from: @from, frame_size: @frame_size) do |chunk, from, eof, next_from|
+                yield(chunk, from, calculate_byte_to(chunk, from), eof, next_from)
+              end
+            when Hash
+              chunk = @stream[:data][@from..]
+              yield(chunk, @from, calculate_byte_to(chunk, @from), @stream[:eof], nil)
+            else
+              chunk = @stream[@from..]
+              yield(@stream[@from..], @from, calculate_byte_to(chunk, @from), true, nil)
+            end
+          end
+
+          private
+
+          sig { params(chunk: T.untyped, from: Integer).returns(T.nilable(Integer)) }
+          def calculate_byte_to(chunk, from)
+            (chunk_size = chunk.try(:bytesize) || 0).zero? ? nil : from + chunk_size - 1
+          end
+
+          sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.any(Proxy, Mock)) }
+          def from_mock(hash)
+            case hash[:chunks]
+            when Proc
+              Proxy.new(
+                name: hash[:name],
+                input: HashWithIndifferentAccess.wrap(hash[:input] || {}),
+                stream: Stream.new(
+                  stream: hash[:chunks],
+                  connection: Connection.new(
+                    connection: hash[:connection] || {},
+                    settings: hash[:settings] || {}
+                  )
+                )
+              )
+            when Hash
+              Mock.new(chunks: hash[:chunks])
+            else
+              raise 'Mock streams with Proc or Hash. Read spec/examples/stream/connector_spec.rb for examples'
+            end
+          end
+        end
+
+        class Proxy
+          extend T::Sig
+
+          sig { params(name: String, input: SorbetTypes::StreamInputHash, stream: Stream).void }
+          def initialize(name:, input:, stream:)
+            @name = name
+            @input = input
+            @stream = stream
+          end
+
+          sig { params(_options: T.untyped).returns(T::Hash[String, T.untyped]) }
+          def as_json(_options = nil)
+            {
+              __stream__: true,
+              name: name,
+              input: input
+            }
+          end
+
+          sig { params(from: Integer, frame_size: Integer, _blk: Reader::ProxyReadProc).void }
+          def read(from:, frame_size:, &_blk)
+            next_from = from
+            loop do
+              res = read_chunk(next_from, frame_size)
+              yield(res.data, res.from, res.eof, res.next_from)
+              break if res.eof
+
+              next_from = T.must(res.next_from)
+            end
+          end
+
+          private
+
+          sig { returns(String) }
+          attr_reader :name
+
+          sig { returns(SorbetTypes::StreamInputHash) }
+          attr_reader :input
+
+          class Chunk < T::Struct
+            const :data, T.untyped
+            const :from, Integer
+            const :eof, T::Boolean
+            const :next_from, T.nilable(Integer)
+          end
+          private_constant :Chunk
+
+          sig { params(from: Integer, frame_size: Integer).returns(Chunk) }
+          def read_chunk(from, frame_size)
+            data, eof = @stream.chunk(input, from, from + frame_size - 1, frame_size)
+            next_from = from + (data&.length || 0)
+            next_from = nil if eof
+            Chunk.new(data: data, from: from, eof: eof, next_from: next_from)
+          end
+        end
+
+        class Mock
+          extend T::Sig
+
+          sig { params(chunks: T.untyped).void }
+          def initialize(chunks:)
+            @chunks = chunks
+          end
+
+          sig { params(from: Integer, frame_size: Integer, _blk: Reader::ProxyReadProc).void }
+          def read(from:, frame_size:, &_blk)
+            last_from = chunks.keys.last
+            chunks.each do |chunk_from, data|
+              next if chunk_from < from
+
+              eof = chunk_from == last_from
+              next_from = eof ? nil : chunk_from + frame_size
+
+              yield(data, chunk_from, eof, next_from)
+            end
+          end
+
+          private
+
+          sig { returns(T.untyped) }
+          attr_reader :chunks
+        end
+
+        private_constant :Mock
+      end
+    end
+  end
+end

data/lib/workato/connector/sdk/streams.rb

@@ -0,0 +1,71 @@
+# typed: strict
+# frozen_string_literal: true
+
+using Workato::Extension::HashWithIndifferentAccess
+
+module Workato
+  module Connector
+    module Sdk
+      class Streams
+        extend T::Sig
+
+        sig do
+          params(
+            streams: SorbetTypes::SourceHash,
+            methods: SorbetTypes::SourceHash,
+            connection: Connection
+          ).void
+        end
+        def initialize(streams: {}, methods: {}, connection: Connection.new)
+          @methods_source = methods
+          @connection = connection
+          @streams = T.let({}, T::Hash[T.any(Symbol, String), Stream])
+          @streams_source = streams
+          define_action_methods(streams)
+        end
+
+        sig { params(stream: T.any(String, Symbol)).returns(Stream) }
+        def [](stream)
+          @streams[stream] ||= Stream.new(
+            stream: @streams_source.fetch(stream),
+            methods: methods_source,
+            connection: connection
+          )
+        end
+
+        private
+
+        sig { params(streams_source: SorbetTypes::SourceHash).void }
+        def define_action_methods(streams_source)
+          streams_source.each do |stream, _stream_proc|
+            define_singleton_method(stream) do |input = {}, from = 0, to = nil, frame_size = Stream::DEFAULT_FRAME_SIZE|
+              to ||= from + frame_size
+              self[stream].chunk(input, from, to, frame_size)
+            end
+
+            define_singleton_method("#{stream}!") do |input = {}, frame_size = Stream::DEFAULT_FRAME_SIZE|
+              self[stream].invoke(input, frame_size)
+            end
+          end
+        end
+
+        sig { returns(SorbetTypes::SourceHash) }
+        attr_reader :methods_source
+
+        sig { returns(Connection) }
+        attr_reader :connection
+      end
+
+      class ProhibitedStreams < Streams
+        extend T::Sig
+
+        sig { void }
+        def initialize
+          @streams = Hash.new do
+            raise 'Streams are not available in this context. Access streams in actions or triggers'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/workato/connector/sdk/summarize.rb

@@ -34,8 +34,8 @@ module Workato
         attr_reader :paths
 
         def above_summarization_limit?(candidate)
-          candidate.is_a?(::Array) && candidate.length > ARRAY_SUMMARIZATION_LIMIT ||
-            candidate.is_a?(::String) && candidate.length > STRING_SUMMARIZATION_LIMIT
+          (candidate.is_a?(::Array) && candidate.length > ARRAY_SUMMARIZATION_LIMIT) ||
+            (candidate.is_a?(::String) && candidate.length > STRING_SUMMARIZATION_LIMIT)
         end
 
         def apply_summarization_limit(summarized, steps)

data/lib/workato/connector/sdk/trigger.rb

@@ -3,6 +3,8 @@
 
 require 'securerandom'
 
+using Workato::Extension::HashWithIndifferentAccess
+
 module Workato
   module Connector
     module Sdk
@@ -32,15 +34,18 @@ module Workato
             trigger: SorbetTypes::SourceHash,
             methods: SorbetTypes::SourceHash,
             connection: Connection,
-            object_definitions: T.nilable(ObjectDefinitions)
+            object_definitions: T.nilable(ObjectDefinitions),
+            streams: Streams
           ).void
         end
-        def initialize(trigger:, methods: {}, connection: Connection.new, object_definitions: nil)
+        def initialize(trigger:, methods: {}, connection: Connection.new, object_definitions: nil,
+                       streams: ProhibitedStreams.new)
           super(
             operation: trigger,
             connection: connection,
             methods: methods,
-            object_definitions: object_definitions
+            object_definitions: object_definitions,
+            streams: streams
           )
         end
 
@@ -66,7 +71,6 @@ module Workato
           ) do |connection, payload, eis, eos|
             instance_exec(connection, payload[:input], payload[:closure], eis, eos, &poll_proc)
           end
-          output.with_indifferent_access
           output[:events] = Array.wrap(output[:events])
                                  .reverse!
                                  .map! { |event| ::Hash.try_convert(event) || event }
@@ -117,7 +121,7 @@ module Workato
             headers: T::Hash[T.any(String, Symbol), T.untyped],
             params: T::Hash[T.any(String, Symbol), T.untyped],
             settings: T.nilable(SorbetTypes::SettingsHash),
-            webhook_subscribe_output: SorbetTypes::WebhookSubscribeOutputHash
+            webhook_subscribe_output: T.nilable(SorbetTypes::WebhookSubscribeOutputHash)
           ).returns(
             SorbetTypes::WebhookNotificationOutputHash
           )
@@ -133,16 +137,15 @@ module Workato
           webhook_subscribe_output = {}
         )
           connection.merge_settings!(settings) if settings
-          output = Dsl::WithDsl.execute(
-            connection,
-            input.with_indifferent_access,
+          output = global_dsl_context.execute(
+            HashWithIndifferentAccess.wrap(input),
             payload,
-            Array.wrap(extended_input_schema).map(&:with_indifferent_access),
-            Array.wrap(extended_output_schema).map(&:with_indifferent_access),
-            headers.with_indifferent_access,
-            params.with_indifferent_access,
+            Array.wrap(extended_input_schema).map { |i| HashWithIndifferentAccess.wrap(i) },
+            Array.wrap(extended_output_schema).map { |i| HashWithIndifferentAccess.wrap(i) },
+            HashWithIndifferentAccess.wrap(headers),
+            HashWithIndifferentAccess.wrap(params),
             connection.settings,
-            webhook_subscribe_output.with_indifferent_access,
+            HashWithIndifferentAccess.wrap(webhook_subscribe_output),
             &trigger[:webhook_notification]
           )
           if output.is_a?(::Array)
@@ -162,7 +165,7 @@ module Workato
             SorbetTypes::WebhookSubscribeOutputHash
           )
         end
-        def webhook_subscribe(webhook_url = '', settings = nil, input = {}, recipe_id = SecureRandom.uuid)
+        def webhook_subscribe(webhook_url = '', settings = nil, input = {}, recipe_id = recipe_id!)
           webhook_subscribe_proc = trigger[:webhook_subscribe]
           execute(settings, { input: input, webhook_url: webhook_url, recipe_id: recipe_id }) do |connection, payload|
             instance_exec(
@@ -189,7 +192,7 @@ module Workato
             payload: T::Hash[T.any(String, Symbol), T.untyped],
             headers: T::Hash[T.any(String, Symbol), T.untyped],
             params: T::Hash[T.any(String, Symbol), T.untyped],
-            webhook_subscribe_output: SorbetTypes::WebhookSubscribeOutputHash
+            webhook_subscribe_output: T.nilable(SorbetTypes::WebhookSubscribeOutputHash)
           ).returns(
             T.any(SorbetTypes::WebhookNotificationOutputHash, SorbetTypes::PollOutputHash)
           )
@@ -231,6 +234,11 @@ module Workato
         def webhook_notification?
           trigger[:webhook_notification].present?
         end
+
+        sig { returns(Dsl::WithDsl) }
+        def global_dsl_context
+          Dsl::WithDsl.new(connection, streams)
+        end
       end
     end
   end

data/lib/workato/connector/sdk/version.rb

@@ -4,7 +4,7 @@
 module Workato
   module Connector
     module Sdk
-      VERSION = '1.1.0'
+      VERSION = T.let(File.read(File.expand_path('../../../../VERSION', __dir__)).strip, String)
     end
   end
 end

data/lib/workato/connector/sdk.rb

@@ -2,48 +2,22 @@
 # frozen_string_literal: true
 
 require 'sorbet-runtime'
-::Method.prepend(T::CompatibilityPatches::MethodExtensions)
-
-module Workato
-  module Connector
-    module Sdk
-      DEFAULT_MASTER_KEY_ENV = 'WORKATO_CONNECTOR_MASTER_KEY'
-      DEFAULT_MASTER_KEY_PATH = 'master.key'
-
-      DEFAULT_CONNECTOR_PATH = 'connector.rb'
-
-      DEFAULT_SETTINGS_PATH = 'settings.yaml'
-      DEFAULT_ENCRYPTED_SETTINGS_PATH = 'settings.yaml.enc'
-
-      DEFAULT_ACCOUNT_PROPERTIES_PATH = 'account_properties.yaml'
-      DEFAULT_ENCRYPTED_ACCOUNT_PROPERTIES_PATH = 'account_properties.yaml.enc'
-
-      DEFAULT_LOOKUP_TABLES_PATH = 'lookup_tables.yaml'
-
-      DEFAULT_TIME_ZONE = 'Pacific Time (US & Canada)'
-
-      DEFAULT_SCHEMAS_PATH = 'workato_schemas.json'
-
-      WORKATO_API_EMAIL_ENV = 'WORKATO_API_EMAIL'
-      WORKATO_API_TOKEN_ENV = 'WORKATO_API_TOKEN'
-
-      WORKATO_BASE_URL_ENV = 'WORKATO_BASE_URL'
-      DEFAULT_WORKATO_BASE_URL = 'https://app.workato.com'
-      WORKATO_BASE_URL = T.let(ENV.fetch(WORKATO_BASE_URL_ENV, DEFAULT_WORKATO_BASE_URL), String)
-    end
-  end
-end
 
 # Global libs and monkey patches
 require 'active_support/all'
 require 'active_support/json'
+
+require_relative '../types/binary'
+
 require_relative '../extension/array'
 require_relative '../extension/case_sensitive_headers'
+require_relative '../extension/content_encoding_decoder'
 require_relative '../extension/currency'
 require_relative '../extension/date'
 require_relative '../extension/enumerable'
 require_relative '../extension/extra_chain_cert'
 require_relative '../extension/hash'
+require_relative '../extension/hash_with_indifferent_access'
 require_relative '../extension/integer'
 require_relative '../extension/nil_class'
 require_relative '../extension/object'
@@ -52,19 +26,19 @@ require_relative '../extension/string'
 require_relative '../extension/symbol'
 require_relative '../extension/time'
 
-require_relative './sdk/account_properties'
-require_relative './sdk/action'
-require_relative './sdk/connection'
-require_relative './sdk/connector'
-require_relative './sdk/dsl'
-require_relative './sdk/errors'
-require_relative './sdk/lookup_tables'
-require_relative './sdk/object_definitions'
-require_relative './sdk/operation'
-require_relative './sdk/request'
-require_relative './sdk/settings'
-require_relative './sdk/summarize'
-require_relative './sdk/trigger'
-require_relative './sdk/version'
-require_relative './sdk/workato_schemas'
-require_relative './sdk/xml'
+require_relative './sdk/core'
+
+begin
+  tz = ENV.fetch('TZ', nil)
+  if tz.present? && tz != 'UTC'
+    warn "WARNING: TZ environment variable is set to '#{tz}'. Set TZ=UTC for consistency with Workato platform'"
+  else
+    ENV['TZ'] = 'UTC'
+  end
+  Time.zone = Workato::Connector::Sdk::DEFAULT_TIME_ZONE
+rescue TZInfo::DataSourceNotFound
+  puts ''
+  puts "tzinfo-data is not present. Please install gem 'tzinfo-data' by 'gem install tzinfo-data'"
+  puts ''
+  exit!
+end

data/lib/workato/extension/array.rb

@@ -1,6 +1,8 @@
 # typed: false
 # frozen_string_literal: true
 
+require 'delegate'
+
 module Workato
   module Extension
     module Array

data/lib/workato/extension/case_sensitive_headers.rb

@@ -1,7 +1,6 @@
 # typed: false
 # frozen_string_literal: true
 
-require 'rest-client'
 require 'net/http'
 
 module Workato
@@ -22,31 +21,6 @@ module Workato
 
       ::Net::HTTPHeader.prepend Net::HTTPHeader
       ::Net::HTTPGenericRequest.prepend Net::HTTPHeader
-
-      module RestClient
-        module Request
-          attr_accessor :case_sensitive_headers
-
-          def processed_headers
-            return @processed_headers if case_sensitive_headers.blank?
-            return case_sensitive_headers if @processed_headers.blank?
-
-            @processed_headers.merge(case_sensitive_headers)
-          end
-
-          def execute(&block)
-            # With 2.0.0+, net/http accepts URI objects in requests and handles wrapping
-            # IPv6 addresses in [] for use in the Host request header.
-            net_http_request = net_http_request_class(method).new(uri, processed_headers)
-            net_http_request.case_sensitive_headers = case_sensitive_headers
-            transmit(uri, net_http_request, payload, &block)
-          ensure
-            payload&.close
-          end
-        end
-      end
-
-      ::RestClient::Request.prepend RestClient::Request
     end
   end
 end

data/lib/workato/extension/content_encoding_decoder.rb

@@ -0,0 +1,69 @@
+# typed: false
+# frozen_string_literal: true
+
+require 'rest-client'
+
+module Workato
+  module Extension
+    module ContentEncodingDecoder
+      module RestClient
+        module Response
+          def create(body, net_http_res, request, start_time)
+            body = decode_content_encoding(net_http_res, body)
+            super(body, net_http_res, request, start_time)
+          end
+
+          private
+
+          def decode_content_encoding(response, body)
+            content_encoding = response['content-encoding']
+
+            case content_encoding&.downcase
+            when 'deflate', 'gzip', 'x-gzip'
+              response.delete 'content-encoding'
+              return body if body.blank?
+
+              deflate_string(body).force_encoding(Encoding.default_external)
+            when 'none', 'identity'
+              response.delete 'content-encoding'
+              body
+            else
+              body
+            end
+          end
+
+          def deflate_string(body)
+            # Decodes all deflate, gzip or x-gzip
+            zstream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
+
+            zstream.inflate(body) + zstream.finish
+          rescue Zlib::DataError
+            # No luck with Zlib decompression. Let's try with raw deflate,
+            # like some broken web servers do. This part isn't compatible with Net::HTTP content-decoding
+            zstream.close
+
+            zstream = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+            zstream.inflate(body) + zstream.finish
+          ensure
+            zstream.close
+          end
+        end
+      end
+
+      ::RestClient::Response.singleton_class.prepend(RestClient::Response)
+
+      ::RestClient::Request.prepend(
+        Module.new do
+          def default_headers
+            # Should pass this header to be compatible with rest-client 2.0.2 version
+            # and rely on decode_content_encoding patch
+            # since net/http does not decompress response body if Content-Range is specified
+            # (see https://github.com/ruby/ruby/blob/27f6ad737b13062339df0a0c80449cf0dbc92ba5/lib/net/http/response.rb#L254)
+            # while the previous version of rest-client does.
+            super.tap { |headers| headers[:accept_encoding] = 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3' }
+          end
+        end
+      )
+    end
+  end
+end