multiwoven-integrations 0.1.28 → 0.1.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0972dc10bbb0d6d81f9903d0edb045f22f5d93bfa7d622f1c5758fd3b4fd2692'
-  data.tar.gz: 50ac2206ed0ba57413cc9ac334bb8d00acd8a263e7a3b78efe7a437983636331
+  metadata.gz: 3a8d265073e60d3503e2b8daa726d9e632d9562e128842f6c660d637216843b0
+  data.tar.gz: 7a81252a1beddb2b7138b8e13e187887fba27a14c35bef504627d915a136b85c
 SHA512:
-  metadata.gz: 8e9c3f011128710a4502dd51d5dad7934521fa80c923fd7c202721bcc63b206fa0e555b29f5ae5ad2dc4beb0619162b7f160e32a5e6ef78248b58ddb662af0ec
-  data.tar.gz: 4343882c80e4cfd50e008d6fbcd0636e0e92e87d2e08173f2dd73c3152448ac8c874f27e787f645be5c9e1ee46504c4a3c6b28ce36c63d660f02026471b2ba4b
+  metadata.gz: f369557a3a07bda33a0fd7683d02304484bf793010cdaf9c4da336ba6ffa75f5dd73fe558f5c2ff2de3b33d0ddd84ff81f14a47ec064cfddb184ff54d4fb62bd
+  data.tar.gz: 2aee5e9d654fcee1a782cd161d24162b03f1f24c1d436c3c9de72e43a23c57aa01b31816f3fe9b9ced43267534d32182017a30e0ca17fcc10d4f5a7714931c20

data/lib/multiwoven/integrations/core/destination_connector.rb

@@ -3,7 +3,9 @@
 module Multiwoven
   module Integrations::Core
     class DestinationConnector < BaseConnector
-      # records is transformed json payload send it to the destination
+      prepend RateLimiter
+
+      # Records are transformed json payload send it to the destination
       # SyncConfig is the Protocol::SyncConfig object
       def write(_sync_config, _records, _action = "insert")
         raise "Not implemented"

data/lib/multiwoven/integrations/core/rate_limiter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations::Core
+    module RateLimiter
+      def write(sync_config, records, action = "insert")
+        stream = sync_config.stream
+
+        @queue ||= Limiter::RateQueue.new(stream.request_rate_limit, interval: stream.rate_limit_unit_seconds) do
+          Integrations::Service.logger.info("Hit the limit for stream: #{stream.name}, waiting")
+        end
+
+        @queue.shift
+
+        super(sync_config, records, action)
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/core/utils.rb

@@ -72,6 +72,31 @@ module Multiwoven
       def success?(response)
         response && %w[200 201].include?(response.code.to_s)
       end
+
+      def build_catalog(catalog_json)
+        streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
+        Multiwoven::Integrations::Protocol::Catalog.new(
+          streams: streams,
+          request_rate_limit: catalog_json["request_rate_limit"] || 60,
+          request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
+          request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10
+        )
+      end
+
+      def build_stream(stream_json)
+        Multiwoven::Integrations::Protocol::Stream.new(
+          name: stream_json["name"],
+          url: stream_json["url"],
+          action: stream_json["action"],
+          request_method: stream_json["method"],
+          batch_support: stream_json["batch_support"] || false,
+          batch_size: stream_json["batch_size"] || 1,
+          json_schema: stream_json["json_schema"],
+          request_rate_limit: stream_json["request_rate_limit"].to_i,
+          request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
+          request_rate_concurrency: stream_json["request_rate_concurrency"].to_i
+        )
+      end
     end
   end
 end

data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb

@@ -25,23 +25,7 @@ module Multiwoven::Integrations::Destination
 
       def discover(_connection_config = nil)
         catalog_json = read_json(CATALOG_SPEC_PATH)
-
-        streams = catalog_json["streams"].map do |stream|
-          Multiwoven::Integrations::Protocol::Stream.new(
-            url: stream["url"],
-            name: stream["name"],
-            json_schema: stream["json_schema"],
-            request_method: stream["method"],
-            action: stream["action"],
-            batch_support: stream["batch_support"],
-            batch_size: stream["batch_size"]
-          )
-        end
-
-        catalog = Multiwoven::Integrations::Protocol::Catalog.new(
-          streams: streams
-        )
-
+        catalog = build_catalog(catalog_json)
         catalog.to_multiwoven_message
       rescue StandardError => e
         handle_exception(

data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json

@@ -1,4 +1,7 @@
 {
+  "request_rate_limit":  600,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
   "streams": [
     {
       "name": "audience",

data/lib/multiwoven/integrations/destination/klaviyo/client.rb

@@ -3,7 +3,7 @@
 module Multiwoven::Integrations::Destination
   module Klaviyo
     include Multiwoven::Integrations::Core
-    class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+    class Client < DestinationConnector
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
         api_key = connection_config[:private_api_key]
@@ -20,19 +20,7 @@ module Multiwoven::Integrations::Destination
       def discover(_connection_config = nil)
         catalog_json = read_json(CATALOG_SPEC_PATH)
 
-        streams = catalog_json["streams"].map do |stream|
-          Multiwoven::Integrations::Protocol::Stream.new(
-            name: stream["name"],
-            json_schema: stream["json_schema"],
-            url: stream["url"],
-            request_method: stream["method"],
-            action: stream["action"]
-          )
-        end
-
-        catalog = Multiwoven::Integrations::Protocol::Catalog.new(
-          streams: streams
-        )
+        catalog = build_catalog(catalog_json)
 
         catalog.to_multiwoven_message
       rescue StandardError => e

data/lib/multiwoven/integrations/destination/klaviyo/config/catalog.json

@@ -1,4 +1,7 @@
 {
+    "request_rate_limit":  600,
+    "request_rate_limit_unit": "minute",
+    "request_rate_concurrency": 10,
     "streams": [
       {
         "name": "profile",

data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb

@@ -21,7 +21,7 @@ module Multiwoven
           end
 
           def discover(_connection_config = nil)
-            catalog = build_catalog(load_catalog_streams)
+            catalog = build_catalog(load_catalog)
             catalog.to_multiwoven_message
           rescue StandardError => e
             handle_exception("SALESFORCE:CRM:DISCOVER:EXCEPTION", "error", e)
@@ -96,20 +96,8 @@ module Multiwoven
             ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
           end
 
-          def load_catalog_streams
-            catalog_json = read_json(CATALOG_SPEC_PATH)
-            catalog_json["streams"].map { |stream| build_stream(stream) }
-          end
-
-          def build_stream(stream)
-            Multiwoven::Integrations::Protocol::Stream.new(
-              name: stream["name"], json_schema: stream["json_schema"],
-              action: stream["action"]
-            )
-          end
-
-          def build_catalog(streams)
-            Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
           end
 
           def tracking_message(success, failure)

data/lib/multiwoven/integrations/destination/salesforce_crm/config/catalog.json

@@ -1,4 +1,7 @@
 {
+  "request_rate_limit":  100000,
+  "request_rate_limit_unit": "day",
+  "request_rate_concurrency": 10,
   "streams": [
     {
       "name": "Account",

data/lib/multiwoven/integrations/destination/slack/client.rb

@@ -19,7 +19,7 @@ module Multiwoven
           end
 
           def discover(_connection_config = nil)
-            catalog = build_catalog(load_catalog_streams)
+            catalog = build_catalog(load_catalog)
             catalog.to_multiwoven_message
           rescue StandardError => e
             handle_exception("SLACK:DISCOVER:EXCEPTION", "error", e)
@@ -97,20 +97,8 @@ module Multiwoven
             ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
           end
 
-          def load_catalog_streams
-            catalog_json = read_json(CATALOG_SPEC_PATH)
-            catalog_json["streams"].map { |stream| build_stream(stream) }
-          end
-
-          def build_stream(stream)
-            Multiwoven::Integrations::Protocol::Stream.new(
-              name: stream["name"], json_schema: stream["json_schema"],
-              action: stream["action"]
-            )
-          end
-
-          def build_catalog(streams)
-            Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
           end
 
           def tracking_message(success, failure)

data/lib/multiwoven/integrations/destination/slack/config/catalog.json

@@ -14,7 +14,9 @@
         "oneOf": [{ "required": ["text"] }]
       },
       "supported_sync_modes": ["full_refresh", "incremental"],
-      "source_defined_cursor": true
+      "request_rate_limit":  60,
+      "request_rate_limit_unit": "minute",
+      "request_rate_concurrency": 1
     }
   ]
 }

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -23,6 +23,7 @@ module Multiwoven
       "rate_limit", "connection_config"
     )
     LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
+    RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
 
     class ProtocolModel < Dry::Struct
       extend Multiwoven::Integrations::Core::Utils
@@ -108,21 +109,46 @@ module Multiwoven
       attribute? :action, StreamAction
       attribute :json_schema, Types::Hash
       attribute? :supported_sync_modes, Types::Array.of(SyncMode).optional
+
       # Applicable for database streams
       attribute? :source_defined_cursor, Types::Bool.optional
       attribute? :default_cursor_field, Types::Array.of(Types::String).optional
       attribute? :source_defined_primary_key, Types::Array.of(Types::Array.of(Types::String)).optional
+
       attribute? :namespace, Types::String.optional
       # Applicable for API streams
       attribute? :url, Types::String.optional
       attribute? :request_method, Types::String.optional
       attribute :batch_support, Types::Bool.default(false)
       attribute :batch_size, Types::Integer.default(1)
+
+      # Rate limits
+      attribute? :request_rate_limit, Types::Integer
+      attribute? :request_rate_limit_unit, RequestRateLimitingUnit
+      attribute? :request_rate_concurrency, Types::Integer
+
+      def rate_limit_unit_seconds
+        case request_rate_limit_unit
+        when "minute"
+          60 # Seconds in a minute
+        when "hour"
+          3600 # Seconds in an hour
+        when "day"
+          86_400 # Seconds in a day
+        else
+          1 # Default case, consider as seconds or handle as error
+        end
+      end
     end
 
     class Catalog < ProtocolModel
       attribute :streams, Types::Array.of(Stream)
 
+      # Rate limits
+      attribute? :request_rate_limit, Types::Integer.default(60)
+      attribute? :request_rate_limit_unit, RequestRateLimitingUnit
+      attribute? :request_rate_concurrency, Types::Integer.default(10)
+
       def to_multiwoven_message
         MultiwovenMessage.new(
           type: MultiwovenMessageType["catalog"],

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.28"
+    VERSION = "0.1.29"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/service.rb

@@ -3,11 +3,10 @@
 module Multiwoven
   module Integrations
     class Service
+      def initialize
+        yield(self.class.config) if block_given?
+      end
       class << self
-        def initialize
-          yield(config) if block_given?
-        end
-
         def connectors
           {
             source: build_connectors(

data/lib/multiwoven/integrations.rb

@@ -14,6 +14,7 @@ require "restforce"
 require "logger"
 require "slack-ruby-client"
 require "git"
+require "ruby-limiter"
 
 # Service
 require_relative "integrations/config"
@@ -23,6 +24,7 @@ require_relative "integrations/service"
 # Core
 require_relative "integrations/core/constants"
 require_relative "integrations/core/utils"
+require_relative "integrations/core/rate_limiter"
 require_relative "integrations/protocol/protocol"
 require_relative "integrations/core/base_connector"
 require_relative "integrations/core/source_connector"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.28
+  version: 0.1.29
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-02-19 00:00:00.000000000 Z
+date: 2024-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -150,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-limiter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-odbc
   requirement: !ruby/object:Gem::Requirement
@@ -299,6 +313,7 @@ files:
 - lib/multiwoven/integrations/core/constants.rb
 - lib/multiwoven/integrations/core/destination_connector.rb
 - lib/multiwoven/integrations/core/http_client.rb
+- lib/multiwoven/integrations/core/rate_limiter.rb
 - lib/multiwoven/integrations/core/source_connector.rb
 - lib/multiwoven/integrations/core/utils.rb
 - lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb