multiwoven-integrations 0.1.28 → 0.1.29

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '0972dc10bbb0d6d81f9903d0edb045f22f5d93bfa7d622f1c5758fd3b4fd2692'
4
- data.tar.gz: 50ac2206ed0ba57413cc9ac334bb8d00acd8a263e7a3b78efe7a437983636331
3
+ metadata.gz: 3a8d265073e60d3503e2b8daa726d9e632d9562e128842f6c660d637216843b0
4
+ data.tar.gz: 7a81252a1beddb2b7138b8e13e187887fba27a14c35bef504627d915a136b85c
5
5
  SHA512:
6
- metadata.gz: 8e9c3f011128710a4502dd51d5dad7934521fa80c923fd7c202721bcc63b206fa0e555b29f5ae5ad2dc4beb0619162b7f160e32a5e6ef78248b58ddb662af0ec
7
- data.tar.gz: 4343882c80e4cfd50e008d6fbcd0636e0e92e87d2e08173f2dd73c3152448ac8c874f27e787f645be5c9e1ee46504c4a3c6b28ce36c63d660f02026471b2ba4b
6
+ metadata.gz: f369557a3a07bda33a0fd7683d02304484bf793010cdaf9c4da336ba6ffa75f5dd73fe558f5c2ff2de3b33d0ddd84ff81f14a47ec064cfddb184ff54d4fb62bd
7
+ data.tar.gz: 2aee5e9d654fcee1a782cd161d24162b03f1f24c1d436c3c9de72e43a23c57aa01b31816f3fe9b9ced43267534d32182017a30e0ca17fcc10d4f5a7714931c20
@@ -3,7 +3,9 @@
3
3
  module Multiwoven
4
4
  module Integrations::Core
5
5
  class DestinationConnector < BaseConnector
6
- # records is transformed json payload send it to the destination
6
+ prepend RateLimiter
7
+
8
+ # Records are transformed json payload send it to the destination
7
9
  # SyncConfig is the Protocol::SyncConfig object
8
10
  def write(_sync_config, _records, _action = "insert")
9
11
  raise "Not implemented"
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven
4
+ module Integrations::Core
5
+ module RateLimiter
6
+ def write(sync_config, records, action = "insert")
7
+ stream = sync_config.stream
8
+
9
+ @queue ||= Limiter::RateQueue.new(stream.request_rate_limit, interval: stream.rate_limit_unit_seconds) do
10
+ Integrations::Service.logger.info("Hit the limit for stream: #{stream.name}, waiting")
11
+ end
12
+
13
+ @queue.shift
14
+
15
+ super(sync_config, records, action)
16
+ end
17
+ end
18
+ end
19
+ end
@@ -72,6 +72,31 @@ module Multiwoven
72
72
  def success?(response)
73
73
  response && %w[200 201].include?(response.code.to_s)
74
74
  end
75
+
76
+ def build_catalog(catalog_json)
77
+ streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
78
+ Multiwoven::Integrations::Protocol::Catalog.new(
79
+ streams: streams,
80
+ request_rate_limit: catalog_json["request_rate_limit"] || 60,
81
+ request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
82
+ request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10
83
+ )
84
+ end
85
+
86
+ def build_stream(stream_json)
87
+ Multiwoven::Integrations::Protocol::Stream.new(
88
+ name: stream_json["name"],
89
+ url: stream_json["url"],
90
+ action: stream_json["action"],
91
+ request_method: stream_json["method"],
92
+ batch_support: stream_json["batch_support"] || false,
93
+ batch_size: stream_json["batch_size"] || 1,
94
+ json_schema: stream_json["json_schema"],
95
+ request_rate_limit: stream_json["request_rate_limit"].to_i,
96
+ request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
97
+ request_rate_concurrency: stream_json["request_rate_concurrency"].to_i
98
+ )
99
+ end
75
100
  end
76
101
  end
77
102
  end
@@ -25,23 +25,7 @@ module Multiwoven::Integrations::Destination
25
25
 
26
26
  def discover(_connection_config = nil)
27
27
  catalog_json = read_json(CATALOG_SPEC_PATH)
28
-
29
- streams = catalog_json["streams"].map do |stream|
30
- Multiwoven::Integrations::Protocol::Stream.new(
31
- url: stream["url"],
32
- name: stream["name"],
33
- json_schema: stream["json_schema"],
34
- request_method: stream["method"],
35
- action: stream["action"],
36
- batch_support: stream["batch_support"],
37
- batch_size: stream["batch_size"]
38
- )
39
- end
40
-
41
- catalog = Multiwoven::Integrations::Protocol::Catalog.new(
42
- streams: streams
43
- )
44
-
28
+ catalog = build_catalog(catalog_json)
45
29
  catalog.to_multiwoven_message
46
30
  rescue StandardError => e
47
31
  handle_exception(
@@ -1,4 +1,7 @@
1
1
  {
2
+ "request_rate_limit": 600,
3
+ "request_rate_limit_unit": "minute",
4
+ "request_rate_concurrency": 10,
2
5
  "streams": [
3
6
  {
4
7
  "name": "audience",
@@ -3,7 +3,7 @@
3
3
  module Multiwoven::Integrations::Destination
4
4
  module Klaviyo
5
5
  include Multiwoven::Integrations::Core
6
- class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
6
+ class Client < DestinationConnector
7
7
  def check_connection(connection_config)
8
8
  connection_config = connection_config.with_indifferent_access
9
9
  api_key = connection_config[:private_api_key]
@@ -20,19 +20,7 @@ module Multiwoven::Integrations::Destination
20
20
  def discover(_connection_config = nil)
21
21
  catalog_json = read_json(CATALOG_SPEC_PATH)
22
22
 
23
- streams = catalog_json["streams"].map do |stream|
24
- Multiwoven::Integrations::Protocol::Stream.new(
25
- name: stream["name"],
26
- json_schema: stream["json_schema"],
27
- url: stream["url"],
28
- request_method: stream["method"],
29
- action: stream["action"]
30
- )
31
- end
32
-
33
- catalog = Multiwoven::Integrations::Protocol::Catalog.new(
34
- streams: streams
35
- )
23
+ catalog = build_catalog(catalog_json)
36
24
 
37
25
  catalog.to_multiwoven_message
38
26
  rescue StandardError => e
@@ -1,4 +1,7 @@
1
1
  {
2
+ "request_rate_limit": 600,
3
+ "request_rate_limit_unit": "minute",
4
+ "request_rate_concurrency": 10,
2
5
  "streams": [
3
6
  {
4
7
  "name": "profile",
@@ -21,7 +21,7 @@ module Multiwoven
21
21
  end
22
22
 
23
23
  def discover(_connection_config = nil)
24
- catalog = build_catalog(load_catalog_streams)
24
+ catalog = build_catalog(load_catalog)
25
25
  catalog.to_multiwoven_message
26
26
  rescue StandardError => e
27
27
  handle_exception("SALESFORCE:CRM:DISCOVER:EXCEPTION", "error", e)
@@ -96,20 +96,8 @@ module Multiwoven
96
96
  ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
97
97
  end
98
98
 
99
- def load_catalog_streams
100
- catalog_json = read_json(CATALOG_SPEC_PATH)
101
- catalog_json["streams"].map { |stream| build_stream(stream) }
102
- end
103
-
104
- def build_stream(stream)
105
- Multiwoven::Integrations::Protocol::Stream.new(
106
- name: stream["name"], json_schema: stream["json_schema"],
107
- action: stream["action"]
108
- )
109
- end
110
-
111
- def build_catalog(streams)
112
- Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
99
+ def load_catalog
100
+ read_json(CATALOG_SPEC_PATH)
113
101
  end
114
102
 
115
103
  def tracking_message(success, failure)
@@ -1,4 +1,7 @@
1
1
  {
2
+ "request_rate_limit": 100000,
3
+ "request_rate_limit_unit": "day",
4
+ "request_rate_concurrency": 10,
2
5
  "streams": [
3
6
  {
4
7
  "name": "Account",
@@ -19,7 +19,7 @@ module Multiwoven
19
19
  end
20
20
 
21
21
  def discover(_connection_config = nil)
22
- catalog = build_catalog(load_catalog_streams)
22
+ catalog = build_catalog(load_catalog)
23
23
  catalog.to_multiwoven_message
24
24
  rescue StandardError => e
25
25
  handle_exception("SLACK:DISCOVER:EXCEPTION", "error", e)
@@ -97,20 +97,8 @@ module Multiwoven
97
97
  ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
98
98
  end
99
99
 
100
- def load_catalog_streams
101
- catalog_json = read_json(CATALOG_SPEC_PATH)
102
- catalog_json["streams"].map { |stream| build_stream(stream) }
103
- end
104
-
105
- def build_stream(stream)
106
- Multiwoven::Integrations::Protocol::Stream.new(
107
- name: stream["name"], json_schema: stream["json_schema"],
108
- action: stream["action"]
109
- )
110
- end
111
-
112
- def build_catalog(streams)
113
- Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
100
+ def load_catalog
101
+ read_json(CATALOG_SPEC_PATH)
114
102
  end
115
103
 
116
104
  def tracking_message(success, failure)
@@ -14,7 +14,9 @@
14
14
  "oneOf": [{ "required": ["text"] }]
15
15
  },
16
16
  "supported_sync_modes": ["full_refresh", "incremental"],
17
- "source_defined_cursor": true
17
+ "request_rate_limit": 60,
18
+ "request_rate_limit_unit": "minute",
19
+ "request_rate_concurrency": 1
18
20
  }
19
21
  ]
20
22
  }
@@ -23,6 +23,7 @@ module Multiwoven
23
23
  "rate_limit", "connection_config"
24
24
  )
25
25
  LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
26
+ RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
26
27
 
27
28
  class ProtocolModel < Dry::Struct
28
29
  extend Multiwoven::Integrations::Core::Utils
@@ -108,21 +109,46 @@ module Multiwoven
108
109
  attribute? :action, StreamAction
109
110
  attribute :json_schema, Types::Hash
110
111
  attribute? :supported_sync_modes, Types::Array.of(SyncMode).optional
112
+
111
113
  # Applicable for database streams
112
114
  attribute? :source_defined_cursor, Types::Bool.optional
113
115
  attribute? :default_cursor_field, Types::Array.of(Types::String).optional
114
116
  attribute? :source_defined_primary_key, Types::Array.of(Types::Array.of(Types::String)).optional
117
+
115
118
  attribute? :namespace, Types::String.optional
116
119
  # Applicable for API streams
117
120
  attribute? :url, Types::String.optional
118
121
  attribute? :request_method, Types::String.optional
119
122
  attribute :batch_support, Types::Bool.default(false)
120
123
  attribute :batch_size, Types::Integer.default(1)
124
+
125
+ # Rate limits
126
+ attribute? :request_rate_limit, Types::Integer
127
+ attribute? :request_rate_limit_unit, RequestRateLimitingUnit
128
+ attribute? :request_rate_concurrency, Types::Integer
129
+
130
+ def rate_limit_unit_seconds
131
+ case request_rate_limit_unit
132
+ when "minute"
133
+ 60 # Seconds in a minute
134
+ when "hour"
135
+ 3600 # Seconds in an hour
136
+ when "day"
137
+ 86_400 # Seconds in a day
138
+ else
139
+ 1 # Default case, consider as seconds or handle as error
140
+ end
141
+ end
121
142
  end
122
143
 
123
144
  class Catalog < ProtocolModel
124
145
  attribute :streams, Types::Array.of(Stream)
125
146
 
147
+ # Rate limits
148
+ attribute? :request_rate_limit, Types::Integer.default(60)
149
+ attribute? :request_rate_limit_unit, RequestRateLimitingUnit
150
+ attribute? :request_rate_concurrency, Types::Integer.default(10)
151
+
126
152
  def to_multiwoven_message
127
153
  MultiwovenMessage.new(
128
154
  type: MultiwovenMessageType["catalog"],
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Multiwoven
4
4
  module Integrations
5
- VERSION = "0.1.28"
5
+ VERSION = "0.1.29"
6
6
 
7
7
  ENABLED_SOURCES = %w[
8
8
  Snowflake
@@ -3,11 +3,10 @@
3
3
  module Multiwoven
4
4
  module Integrations
5
5
  class Service
6
+ def initialize
7
+ yield(self.class.config) if block_given?
8
+ end
6
9
  class << self
7
- def initialize
8
- yield(config) if block_given?
9
- end
10
-
11
10
  def connectors
12
11
  {
13
12
  source: build_connectors(
@@ -14,6 +14,7 @@ require "restforce"
14
14
  require "logger"
15
15
  require "slack-ruby-client"
16
16
  require "git"
17
+ require "ruby-limiter"
17
18
 
18
19
  # Service
19
20
  require_relative "integrations/config"
@@ -23,6 +24,7 @@ require_relative "integrations/service"
23
24
  # Core
24
25
  require_relative "integrations/core/constants"
25
26
  require_relative "integrations/core/utils"
27
+ require_relative "integrations/core/rate_limiter"
26
28
  require_relative "integrations/protocol/protocol"
27
29
  require_relative "integrations/core/base_connector"
28
30
  require_relative "integrations/core/source_connector"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiwoven-integrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.28
4
+ version: 0.1.29
5
5
  platform: ruby
6
6
  authors:
7
7
  - Subin T P
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-02-19 00:00:00.000000000 Z
11
+ date: 2024-02-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -150,6 +150,20 @@ dependencies:
150
150
  - - ">="
151
151
  - !ruby/object:Gem::Version
152
152
  version: '0'
153
+ - !ruby/object:Gem::Dependency
154
+ name: ruby-limiter
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - ">="
158
+ - !ruby/object:Gem::Version
159
+ version: '0'
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - ">="
165
+ - !ruby/object:Gem::Version
166
+ version: '0'
153
167
  - !ruby/object:Gem::Dependency
154
168
  name: ruby-odbc
155
169
  requirement: !ruby/object:Gem::Requirement
@@ -299,6 +313,7 @@ files:
299
313
  - lib/multiwoven/integrations/core/constants.rb
300
314
  - lib/multiwoven/integrations/core/destination_connector.rb
301
315
  - lib/multiwoven/integrations/core/http_client.rb
316
+ - lib/multiwoven/integrations/core/rate_limiter.rb
302
317
  - lib/multiwoven/integrations/core/source_connector.rb
303
318
  - lib/multiwoven/integrations/core/utils.rb
304
319
  - lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb