multiwoven-integrations 0.1.27 → 0.1.29
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/base_connector.rb +2 -2
- data/lib/multiwoven/integrations/core/destination_connector.rb +3 -1
- data/lib/multiwoven/integrations/core/rate_limiter.rb +19 -0
- data/lib/multiwoven/integrations/core/utils.rb +25 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +1 -17
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json +3 -0
- data/lib/multiwoven/integrations/destination/klaviyo/client.rb +2 -14
- data/lib/multiwoven/integrations/destination/klaviyo/config/catalog.json +3 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb +3 -15
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/catalog.json +3 -0
- data/lib/multiwoven/integrations/destination/slack/client.rb +3 -15
- data/lib/multiwoven/integrations/destination/slack/config/catalog.json +3 -1
- data/lib/multiwoven/integrations/protocol/protocol.rb +26 -0
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations/service.rb +3 -4
- data/lib/multiwoven/integrations.rb +2 -0
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3a8d265073e60d3503e2b8daa726d9e632d9562e128842f6c660d637216843b0
|
4
|
+
data.tar.gz: 7a81252a1beddb2b7138b8e13e187887fba27a14c35bef504627d915a136b85c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f369557a3a07bda33a0fd7683d02304484bf793010cdaf9c4da336ba6ffa75f5dd73fe558f5c2ff2de3b33d0ddd84ff81f14a47ec064cfddb184ff54d4fb62bd
|
7
|
+
data.tar.gz: 2aee5e9d654fcee1a782cd161d24162b03f1f24c1d436c3c9de72e43a23c57aa01b31816f3fe9b9ced43267534d32182017a30e0ca17fcc10d4f5a7714931c20
|
@@ -27,10 +27,10 @@ module Multiwoven
|
|
27
27
|
def relative_path
|
28
28
|
path = Object.const_source_location(self.class.to_s)[0]
|
29
29
|
connector_folder = File.dirname(path)
|
30
|
-
marker = "multiwoven
|
30
|
+
marker = "/lib/multiwoven/integrations/"
|
31
31
|
parts = connector_folder.split(marker)
|
32
32
|
|
33
|
-
parts.last if parts.length > 1
|
33
|
+
marker + parts.last if parts.length > 1
|
34
34
|
end
|
35
35
|
|
36
36
|
# Connection config is a hash
|
@@ -3,7 +3,9 @@
|
|
3
3
|
module Multiwoven
|
4
4
|
module Integrations::Core
|
5
5
|
class DestinationConnector < BaseConnector
|
6
|
-
|
6
|
+
prepend RateLimiter
|
7
|
+
|
8
|
+
# Records are transformed json payload send it to the destination
|
7
9
|
# SyncConfig is the Protocol::SyncConfig object
|
8
10
|
def write(_sync_config, _records, _action = "insert")
|
9
11
|
raise "Not implemented"
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Multiwoven
|
4
|
+
module Integrations::Core
|
5
|
+
module RateLimiter
|
6
|
+
def write(sync_config, records, action = "insert")
|
7
|
+
stream = sync_config.stream
|
8
|
+
|
9
|
+
@queue ||= Limiter::RateQueue.new(stream.request_rate_limit, interval: stream.rate_limit_unit_seconds) do
|
10
|
+
Integrations::Service.logger.info("Hit the limit for stream: #{stream.name}, waiting")
|
11
|
+
end
|
12
|
+
|
13
|
+
@queue.shift
|
14
|
+
|
15
|
+
super(sync_config, records, action)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -72,6 +72,31 @@ module Multiwoven
|
|
72
72
|
def success?(response)
|
73
73
|
response && %w[200 201].include?(response.code.to_s)
|
74
74
|
end
|
75
|
+
|
76
|
+
def build_catalog(catalog_json)
|
77
|
+
streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
|
78
|
+
Multiwoven::Integrations::Protocol::Catalog.new(
|
79
|
+
streams: streams,
|
80
|
+
request_rate_limit: catalog_json["request_rate_limit"] || 60,
|
81
|
+
request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
|
82
|
+
request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10
|
83
|
+
)
|
84
|
+
end
|
85
|
+
|
86
|
+
def build_stream(stream_json)
|
87
|
+
Multiwoven::Integrations::Protocol::Stream.new(
|
88
|
+
name: stream_json["name"],
|
89
|
+
url: stream_json["url"],
|
90
|
+
action: stream_json["action"],
|
91
|
+
request_method: stream_json["method"],
|
92
|
+
batch_support: stream_json["batch_support"] || false,
|
93
|
+
batch_size: stream_json["batch_size"] || 1,
|
94
|
+
json_schema: stream_json["json_schema"],
|
95
|
+
request_rate_limit: stream_json["request_rate_limit"].to_i,
|
96
|
+
request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
|
97
|
+
request_rate_concurrency: stream_json["request_rate_concurrency"].to_i
|
98
|
+
)
|
99
|
+
end
|
75
100
|
end
|
76
101
|
end
|
77
102
|
end
|
@@ -25,23 +25,7 @@ module Multiwoven::Integrations::Destination
|
|
25
25
|
|
26
26
|
def discover(_connection_config = nil)
|
27
27
|
catalog_json = read_json(CATALOG_SPEC_PATH)
|
28
|
-
|
29
|
-
streams = catalog_json["streams"].map do |stream|
|
30
|
-
Multiwoven::Integrations::Protocol::Stream.new(
|
31
|
-
url: stream["url"],
|
32
|
-
name: stream["name"],
|
33
|
-
json_schema: stream["json_schema"],
|
34
|
-
request_method: stream["method"],
|
35
|
-
action: stream["action"],
|
36
|
-
batch_support: stream["batch_support"],
|
37
|
-
batch_size: stream["batch_size"]
|
38
|
-
)
|
39
|
-
end
|
40
|
-
|
41
|
-
catalog = Multiwoven::Integrations::Protocol::Catalog.new(
|
42
|
-
streams: streams
|
43
|
-
)
|
44
|
-
|
28
|
+
catalog = build_catalog(catalog_json)
|
45
29
|
catalog.to_multiwoven_message
|
46
30
|
rescue StandardError => e
|
47
31
|
handle_exception(
|
@@ -3,7 +3,7 @@
|
|
3
3
|
module Multiwoven::Integrations::Destination
|
4
4
|
module Klaviyo
|
5
5
|
include Multiwoven::Integrations::Core
|
6
|
-
class Client < DestinationConnector
|
6
|
+
class Client < DestinationConnector
|
7
7
|
def check_connection(connection_config)
|
8
8
|
connection_config = connection_config.with_indifferent_access
|
9
9
|
api_key = connection_config[:private_api_key]
|
@@ -20,19 +20,7 @@ module Multiwoven::Integrations::Destination
|
|
20
20
|
def discover(_connection_config = nil)
|
21
21
|
catalog_json = read_json(CATALOG_SPEC_PATH)
|
22
22
|
|
23
|
-
|
24
|
-
Multiwoven::Integrations::Protocol::Stream.new(
|
25
|
-
name: stream["name"],
|
26
|
-
json_schema: stream["json_schema"],
|
27
|
-
url: stream["url"],
|
28
|
-
request_method: stream["method"],
|
29
|
-
action: stream["action"]
|
30
|
-
)
|
31
|
-
end
|
32
|
-
|
33
|
-
catalog = Multiwoven::Integrations::Protocol::Catalog.new(
|
34
|
-
streams: streams
|
35
|
-
)
|
23
|
+
catalog = build_catalog(catalog_json)
|
36
24
|
|
37
25
|
catalog.to_multiwoven_message
|
38
26
|
rescue StandardError => e
|
@@ -21,7 +21,7 @@ module Multiwoven
|
|
21
21
|
end
|
22
22
|
|
23
23
|
def discover(_connection_config = nil)
|
24
|
-
catalog = build_catalog(
|
24
|
+
catalog = build_catalog(load_catalog)
|
25
25
|
catalog.to_multiwoven_message
|
26
26
|
rescue StandardError => e
|
27
27
|
handle_exception("SALESFORCE:CRM:DISCOVER:EXCEPTION", "error", e)
|
@@ -96,20 +96,8 @@ module Multiwoven
|
|
96
96
|
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
|
97
97
|
end
|
98
98
|
|
99
|
-
def
|
100
|
-
|
101
|
-
catalog_json["streams"].map { |stream| build_stream(stream) }
|
102
|
-
end
|
103
|
-
|
104
|
-
def build_stream(stream)
|
105
|
-
Multiwoven::Integrations::Protocol::Stream.new(
|
106
|
-
name: stream["name"], json_schema: stream["json_schema"],
|
107
|
-
action: stream["action"]
|
108
|
-
)
|
109
|
-
end
|
110
|
-
|
111
|
-
def build_catalog(streams)
|
112
|
-
Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
|
99
|
+
def load_catalog
|
100
|
+
read_json(CATALOG_SPEC_PATH)
|
113
101
|
end
|
114
102
|
|
115
103
|
def tracking_message(success, failure)
|
@@ -19,7 +19,7 @@ module Multiwoven
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def discover(_connection_config = nil)
|
22
|
-
catalog = build_catalog(
|
22
|
+
catalog = build_catalog(load_catalog)
|
23
23
|
catalog.to_multiwoven_message
|
24
24
|
rescue StandardError => e
|
25
25
|
handle_exception("SLACK:DISCOVER:EXCEPTION", "error", e)
|
@@ -97,20 +97,8 @@ module Multiwoven
|
|
97
97
|
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
|
98
98
|
end
|
99
99
|
|
100
|
-
def
|
101
|
-
|
102
|
-
catalog_json["streams"].map { |stream| build_stream(stream) }
|
103
|
-
end
|
104
|
-
|
105
|
-
def build_stream(stream)
|
106
|
-
Multiwoven::Integrations::Protocol::Stream.new(
|
107
|
-
name: stream["name"], json_schema: stream["json_schema"],
|
108
|
-
action: stream["action"]
|
109
|
-
)
|
110
|
-
end
|
111
|
-
|
112
|
-
def build_catalog(streams)
|
113
|
-
Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
|
100
|
+
def load_catalog
|
101
|
+
read_json(CATALOG_SPEC_PATH)
|
114
102
|
end
|
115
103
|
|
116
104
|
def tracking_message(success, failure)
|
@@ -23,6 +23,7 @@ module Multiwoven
|
|
23
23
|
"rate_limit", "connection_config"
|
24
24
|
)
|
25
25
|
LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
|
26
|
+
RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
|
26
27
|
|
27
28
|
class ProtocolModel < Dry::Struct
|
28
29
|
extend Multiwoven::Integrations::Core::Utils
|
@@ -108,21 +109,46 @@ module Multiwoven
|
|
108
109
|
attribute? :action, StreamAction
|
109
110
|
attribute :json_schema, Types::Hash
|
110
111
|
attribute? :supported_sync_modes, Types::Array.of(SyncMode).optional
|
112
|
+
|
111
113
|
# Applicable for database streams
|
112
114
|
attribute? :source_defined_cursor, Types::Bool.optional
|
113
115
|
attribute? :default_cursor_field, Types::Array.of(Types::String).optional
|
114
116
|
attribute? :source_defined_primary_key, Types::Array.of(Types::Array.of(Types::String)).optional
|
117
|
+
|
115
118
|
attribute? :namespace, Types::String.optional
|
116
119
|
# Applicable for API streams
|
117
120
|
attribute? :url, Types::String.optional
|
118
121
|
attribute? :request_method, Types::String.optional
|
119
122
|
attribute :batch_support, Types::Bool.default(false)
|
120
123
|
attribute :batch_size, Types::Integer.default(1)
|
124
|
+
|
125
|
+
# Rate limits
|
126
|
+
attribute? :request_rate_limit, Types::Integer
|
127
|
+
attribute? :request_rate_limit_unit, RequestRateLimitingUnit
|
128
|
+
attribute? :request_rate_concurrency, Types::Integer
|
129
|
+
|
130
|
+
def rate_limit_unit_seconds
|
131
|
+
case request_rate_limit_unit
|
132
|
+
when "minute"
|
133
|
+
60 # Seconds in a minute
|
134
|
+
when "hour"
|
135
|
+
3600 # Seconds in an hour
|
136
|
+
when "day"
|
137
|
+
86_400 # Seconds in a day
|
138
|
+
else
|
139
|
+
1 # Default case, consider as seconds or handle as error
|
140
|
+
end
|
141
|
+
end
|
121
142
|
end
|
122
143
|
|
123
144
|
class Catalog < ProtocolModel
|
124
145
|
attribute :streams, Types::Array.of(Stream)
|
125
146
|
|
147
|
+
# Rate limits
|
148
|
+
attribute? :request_rate_limit, Types::Integer.default(60)
|
149
|
+
attribute? :request_rate_limit_unit, RequestRateLimitingUnit
|
150
|
+
attribute? :request_rate_concurrency, Types::Integer.default(10)
|
151
|
+
|
126
152
|
def to_multiwoven_message
|
127
153
|
MultiwovenMessage.new(
|
128
154
|
type: MultiwovenMessageType["catalog"],
|
@@ -14,6 +14,7 @@ require "restforce"
|
|
14
14
|
require "logger"
|
15
15
|
require "slack-ruby-client"
|
16
16
|
require "git"
|
17
|
+
require "ruby-limiter"
|
17
18
|
|
18
19
|
# Service
|
19
20
|
require_relative "integrations/config"
|
@@ -23,6 +24,7 @@ require_relative "integrations/service"
|
|
23
24
|
# Core
|
24
25
|
require_relative "integrations/core/constants"
|
25
26
|
require_relative "integrations/core/utils"
|
27
|
+
require_relative "integrations/core/rate_limiter"
|
26
28
|
require_relative "integrations/protocol/protocol"
|
27
29
|
require_relative "integrations/core/base_connector"
|
28
30
|
require_relative "integrations/core/source_connector"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: multiwoven-integrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.29
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Subin T P
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-02-
|
11
|
+
date: 2024-02-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -150,6 +150,20 @@ dependencies:
|
|
150
150
|
- - ">="
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: ruby-limiter
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
153
167
|
- !ruby/object:Gem::Dependency
|
154
168
|
name: ruby-odbc
|
155
169
|
requirement: !ruby/object:Gem::Requirement
|
@@ -299,6 +313,7 @@ files:
|
|
299
313
|
- lib/multiwoven/integrations/core/constants.rb
|
300
314
|
- lib/multiwoven/integrations/core/destination_connector.rb
|
301
315
|
- lib/multiwoven/integrations/core/http_client.rb
|
316
|
+
- lib/multiwoven/integrations/core/rate_limiter.rb
|
302
317
|
- lib/multiwoven/integrations/core/source_connector.rb
|
303
318
|
- lib/multiwoven/integrations/core/utils.rb
|
304
319
|
- lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb
|