multiwoven-integrations 0.37.4 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/oauth_client_credentials.rb +107 -0
- data/lib/multiwoven/integrations/destination/http/client.rb +6 -3
- data/lib/multiwoven/integrations/destination/http/config/spec.json +49 -0
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations/source/http/client.rb +27 -10
- data/lib/multiwoven/integrations/source/http/config/spec.json +50 -1
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6517ac7cba3e8d4ef88e44ea54ba106168a1145ddd028fbb5d12ee15a6d24420
|
|
4
|
+
data.tar.gz: aeca6c1672dc44259ec5ef8b71e22e1c28c817d8e4ebf37ac5801fbfefc568ad
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b4c29de3371f2c52c9377da307c68cb18dbe21006602dcba6a39d172d452956276213f7efc56bf9990edf6cb6b5b81eaf1534a95ec8cb854f2cbdc9e8be6d77d
|
|
7
|
+
data.tar.gz: 3881622dda89bdcdee30bc421344201e8ea8d1c0bd4164911464beeabc5c522b03155edcc19bea8a5e8444f6ec145ef28f8dcd3c1b2d30a631a55925ac229c6b
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations::Core
|
|
5
|
+
# Shared OAuth2 client_credentials flow. Include in a connector client to:
|
|
6
|
+
# * inject `Authorization: Bearer <token>` when connection_config[:auth_type]
|
|
7
|
+
# is `oauth_client_credentials`
|
|
8
|
+
# * cache the token in the connector's `configuration` JSON and refresh it
|
|
9
|
+
# shortly before expiry
|
|
10
|
+
#
|
|
11
|
+
# The including class is expected to set `@connector_instance` (an object
|
|
12
|
+
# responding to `configuration` and `update!`) before making requests that
|
|
13
|
+
# should benefit from the cache. Without it, a fresh token is fetched every
|
|
14
|
+
# call — safe but wasteful.
|
|
15
|
+
module OauthClientCredentials
|
|
16
|
+
AUTH_TYPE_OAUTH_CLIENT_CREDENTIALS = "oauth_client_credentials"
|
|
17
|
+
# Refresh access tokens this many seconds before their advertised expiry,
|
|
18
|
+
# so a token that expires mid-request doesn't leave the connector.
|
|
19
|
+
TOKEN_EXPIRY_BUFFER_SECONDS = 300
|
|
20
|
+
|
|
21
|
+
def build_headers(connection_config)
|
|
22
|
+
headers = (connection_config[:headers] || {}).to_h.dup
|
|
23
|
+
return headers unless connection_config[:auth_type] == AUTH_TYPE_OAUTH_CLIENT_CREDENTIALS
|
|
24
|
+
|
|
25
|
+
headers["Authorization"] = "Bearer #{ensure_oauth_token(connection_config)}"
|
|
26
|
+
headers
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def ensure_oauth_token(connection_config)
|
|
30
|
+
cached = cached_oauth_token
|
|
31
|
+
return cached if cached
|
|
32
|
+
|
|
33
|
+
fetch_and_cache_oauth_token(connection_config)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def cached_oauth_token
|
|
39
|
+
config = connector_configuration
|
|
40
|
+
return nil unless config
|
|
41
|
+
|
|
42
|
+
token = config["oauth_access_token"]
|
|
43
|
+
expires_at = config["oauth_expires_at"]
|
|
44
|
+
return nil if token.nil? || token.to_s.empty? || expires_at.nil?
|
|
45
|
+
return nil if Time.parse(expires_at.to_s) <= Time.now + TOKEN_EXPIRY_BUFFER_SECONDS
|
|
46
|
+
|
|
47
|
+
token
|
|
48
|
+
rescue ArgumentError, TypeError
|
|
49
|
+
nil
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def fetch_and_cache_oauth_token(connection_config)
|
|
53
|
+
token_url = connection_config[:token_url]
|
|
54
|
+
client_id = connection_config[:client_id]
|
|
55
|
+
client_secret = connection_config[:client_secret]
|
|
56
|
+
raise ArgumentError, "OAuth token_url, client_id, and client_secret are required when auth_type is #{AUTH_TYPE_OAUTH_CLIENT_CREDENTIALS}" if token_url.to_s.empty? || client_id.to_s.empty? || client_secret.to_s.empty?
|
|
57
|
+
|
|
58
|
+
response = post_token_request(token_url, client_id, client_secret, connection_config[:scope])
|
|
59
|
+
raise "OAuth token request failed: #{response.code} #{response.body}" unless response.is_a?(Net::HTTPSuccess)
|
|
60
|
+
|
|
61
|
+
body = JSON.parse(response.body)
|
|
62
|
+
access_token = body["access_token"]
|
|
63
|
+
raise "OAuth token response missing 'access_token'" if access_token.to_s.empty?
|
|
64
|
+
|
|
65
|
+
expires_in = (body["expires_in"] || 3600).to_i
|
|
66
|
+
persist_oauth_token(access_token, Time.now + expires_in)
|
|
67
|
+
access_token
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def post_token_request(token_url, client_id, client_secret, scope)
|
|
71
|
+
uri = URI(token_url)
|
|
72
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
73
|
+
http.use_ssl = (uri.scheme == "https")
|
|
74
|
+
|
|
75
|
+
form = {
|
|
76
|
+
"grant_type" => "client_credentials",
|
|
77
|
+
"client_id" => client_id,
|
|
78
|
+
"client_secret" => client_secret
|
|
79
|
+
}
|
|
80
|
+
form["scope"] = scope unless scope.to_s.empty?
|
|
81
|
+
|
|
82
|
+
request = Net::HTTP::Post.new(uri)
|
|
83
|
+
request["Content-Type"] = "application/x-www-form-urlencoded"
|
|
84
|
+
request.body = URI.encode_www_form(form)
|
|
85
|
+
http.request(request)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def persist_oauth_token(access_token, expires_at)
|
|
89
|
+
return unless @connector_instance.respond_to?(:update!)
|
|
90
|
+
|
|
91
|
+
base = connector_configuration || {}
|
|
92
|
+
new_config = base.merge(
|
|
93
|
+
"oauth_access_token" => access_token,
|
|
94
|
+
"oauth_expires_at" => expires_at.iso8601
|
|
95
|
+
)
|
|
96
|
+
@connector_instance.update!(configuration: new_config)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def connector_configuration
|
|
100
|
+
return nil unless @connector_instance.respond_to?(:configuration)
|
|
101
|
+
|
|
102
|
+
cfg = @connector_instance.configuration
|
|
103
|
+
cfg.is_a?(Hash) ? cfg : nil
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -6,16 +6,18 @@ module Multiwoven
|
|
|
6
6
|
module Http
|
|
7
7
|
include Multiwoven::Integrations::Core
|
|
8
8
|
class Client < DestinationConnector
|
|
9
|
+
include Multiwoven::Integrations::Core::OauthClientCredentials
|
|
10
|
+
|
|
9
11
|
MAX_CHUNK_SIZE = 10
|
|
12
|
+
|
|
10
13
|
def check_connection(connection_config)
|
|
11
14
|
connection_config = connection_config.with_indifferent_access
|
|
12
15
|
destination_url = connection_config[:destination_url]
|
|
13
|
-
headers = connection_config[:headers]
|
|
14
16
|
request = Multiwoven::Integrations::Core::HttpClient.request(
|
|
15
17
|
destination_url,
|
|
16
18
|
HTTP_POST,
|
|
17
19
|
payload: {},
|
|
18
|
-
headers:
|
|
20
|
+
headers: build_headers(connection_config)
|
|
19
21
|
)
|
|
20
22
|
if success?(request)
|
|
21
23
|
success_status
|
|
@@ -43,8 +45,9 @@ module Multiwoven
|
|
|
43
45
|
|
|
44
46
|
def write(sync_config, records, _action = "create", _identifier_key = nil)
|
|
45
47
|
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
48
|
+
@connector_instance = sync_config&.destination&.connector_instance
|
|
46
49
|
url = connection_config[:destination_url]
|
|
47
|
-
headers = connection_config
|
|
50
|
+
headers = build_headers(connection_config)
|
|
48
51
|
log_message_array = []
|
|
49
52
|
write_success = 0
|
|
50
53
|
write_failure = 0
|
|
@@ -12,6 +12,14 @@
|
|
|
12
12
|
"title": "Destination url",
|
|
13
13
|
"order": 0
|
|
14
14
|
},
|
|
15
|
+
"auth_type": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"title": "Authentication Type",
|
|
18
|
+
"description": "Select 'oauth_client_credentials' to have Multiwoven fetch and refresh an OAuth2 access token automatically.",
|
|
19
|
+
"enum": ["none", "oauth_client_credentials"],
|
|
20
|
+
"default": "none",
|
|
21
|
+
"order": 2
|
|
22
|
+
},
|
|
15
23
|
"headers": {
|
|
16
24
|
"title": "Http Headers",
|
|
17
25
|
"order": 1,
|
|
@@ -19,6 +27,47 @@
|
|
|
19
27
|
"type": "string"
|
|
20
28
|
}
|
|
21
29
|
}
|
|
30
|
+
},
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"auth_type": {
|
|
33
|
+
"oneOf": [
|
|
34
|
+
{
|
|
35
|
+
"properties": {
|
|
36
|
+
"auth_type": { "enum": ["none"] }
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"properties": {
|
|
41
|
+
"auth_type": { "enum": ["oauth_client_credentials"] },
|
|
42
|
+
"token_url": {
|
|
43
|
+
"type": "string",
|
|
44
|
+
"title": "OAuth Token URL",
|
|
45
|
+
"description": "E.g. https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token",
|
|
46
|
+
"order": 3
|
|
47
|
+
},
|
|
48
|
+
"client_id": {
|
|
49
|
+
"type": "string",
|
|
50
|
+
"multiwoven_secret": true,
|
|
51
|
+
"title": "OAuth Client ID",
|
|
52
|
+
"order": 4
|
|
53
|
+
},
|
|
54
|
+
"client_secret": {
|
|
55
|
+
"type": "string",
|
|
56
|
+
"multiwoven_secret": true,
|
|
57
|
+
"title": "OAuth Client Secret",
|
|
58
|
+
"order": 5
|
|
59
|
+
},
|
|
60
|
+
"scope": {
|
|
61
|
+
"type": "string",
|
|
62
|
+
"title": "OAuth Scope",
|
|
63
|
+
"description": "Optional. E.g. https://graph.microsoft.com/.default",
|
|
64
|
+
"order": 6
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"required": ["token_url", "client_id", "client_secret"]
|
|
68
|
+
}
|
|
69
|
+
]
|
|
70
|
+
}
|
|
22
71
|
}
|
|
23
72
|
}
|
|
24
73
|
}
|
|
@@ -4,6 +4,8 @@ module Multiwoven::Integrations::Source
|
|
|
4
4
|
module Http
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
6
|
class Client < SourceConnector
|
|
7
|
+
include Multiwoven::Integrations::Core::OauthClientCredentials
|
|
8
|
+
|
|
7
9
|
def check_connection(connection_config)
|
|
8
10
|
connection_config = prepare_config(connection_config)
|
|
9
11
|
create_connection(connection_config)
|
|
@@ -17,7 +19,7 @@ module Multiwoven::Integrations::Source
|
|
|
17
19
|
url: @url,
|
|
18
20
|
http_method: connection_config[:http_method],
|
|
19
21
|
payload: connection_config[:request_format],
|
|
20
|
-
headers: connection_config
|
|
22
|
+
headers: build_headers(connection_config),
|
|
21
23
|
config: connection_config[:config],
|
|
22
24
|
params: connection_config[:params]
|
|
23
25
|
)
|
|
@@ -40,7 +42,7 @@ module Multiwoven::Integrations::Source
|
|
|
40
42
|
url: @url,
|
|
41
43
|
http_method: connection_config[:http_method],
|
|
42
44
|
payload: connection_config[:request_format],
|
|
43
|
-
headers: connection_config
|
|
45
|
+
headers: build_headers(connection_config),
|
|
44
46
|
config: connection_config[:config],
|
|
45
47
|
params: connection_config[:params]
|
|
46
48
|
)
|
|
@@ -55,8 +57,9 @@ module Multiwoven::Integrations::Source
|
|
|
55
57
|
def read(sync_config)
|
|
56
58
|
connection_config = sync_config.source.connection_specification
|
|
57
59
|
connection_config = connection_config.with_indifferent_access
|
|
60
|
+
@connector_instance = sync_config&.source&.connector_instance
|
|
58
61
|
connection_config = create_connection(connection_config)
|
|
59
|
-
if sync_config.increment_strategy_config
|
|
62
|
+
if sync_config.increment_strategy_config&.increment_strategy == "page"
|
|
60
63
|
@limit = sync_config.increment_strategy_config.limit
|
|
61
64
|
@offset = sync_config.increment_strategy_config.offset
|
|
62
65
|
else
|
|
@@ -82,6 +85,19 @@ module Multiwoven::Integrations::Source
|
|
|
82
85
|
end
|
|
83
86
|
end
|
|
84
87
|
|
|
88
|
+
def duplicate_request?(connection_config)
|
|
89
|
+
signature = [
|
|
90
|
+
@url,
|
|
91
|
+
connection_config[:http_method],
|
|
92
|
+
connection_config[:params],
|
|
93
|
+
connection_config[:request_format]
|
|
94
|
+
].to_json
|
|
95
|
+
return true if @last_request_signature == signature
|
|
96
|
+
|
|
97
|
+
@last_request_signature = signature
|
|
98
|
+
false
|
|
99
|
+
end
|
|
100
|
+
|
|
85
101
|
def create_connection(connection_config)
|
|
86
102
|
@url = "#{connection_config[:base_url]}#{connection_config[:path]}"
|
|
87
103
|
connection_config
|
|
@@ -113,11 +129,13 @@ module Multiwoven::Integrations::Source
|
|
|
113
129
|
def query(connection_config, query)
|
|
114
130
|
connection_config = prepare_config(connection_config)
|
|
115
131
|
build_paginated_request(connection_config, query)
|
|
132
|
+
return [] if duplicate_request?(connection_config)
|
|
133
|
+
|
|
116
134
|
response = send_request(
|
|
117
135
|
url: @url,
|
|
118
136
|
http_method: connection_config[:http_method],
|
|
119
137
|
payload: connection_config[:request_format],
|
|
120
|
-
headers: connection_config
|
|
138
|
+
headers: build_headers(connection_config),
|
|
121
139
|
config: connection_config[:config],
|
|
122
140
|
params: connection_config[:params] || {}
|
|
123
141
|
)
|
|
@@ -144,12 +162,11 @@ module Multiwoven::Integrations::Source
|
|
|
144
162
|
def parse_response(response_body, parse_response)
|
|
145
163
|
case parse_response
|
|
146
164
|
when Array
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
data = Hash[records[0].zip(row_values)]
|
|
165
|
+
headers = JsonPath.on(response_body, parse_response[0]).flatten(1)
|
|
166
|
+
values = JsonPath.on(response_body, parse_response[1])
|
|
167
|
+
rows = values.first.is_a?(Array) ? values : values.each_slice(headers.size).to_a
|
|
168
|
+
rows.map do |row_values|
|
|
169
|
+
data = Hash[headers.zip(row_values)]
|
|
153
170
|
RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
154
171
|
end
|
|
155
172
|
else
|
|
@@ -33,6 +33,14 @@
|
|
|
33
33
|
"order": 3,
|
|
34
34
|
"default": "/"
|
|
35
35
|
},
|
|
36
|
+
"auth_type": {
|
|
37
|
+
"type": "string",
|
|
38
|
+
"title": "Authentication Type",
|
|
39
|
+
"description": "Select 'oauth_client_credentials' to have Multiwoven fetch and refresh an OAuth2 access token automatically.",
|
|
40
|
+
"enum": ["none", "oauth_client_credentials"],
|
|
41
|
+
"default": "none",
|
|
42
|
+
"order": 14
|
|
43
|
+
},
|
|
36
44
|
"headers": {
|
|
37
45
|
"title": "HTTP Headers",
|
|
38
46
|
"description": "Custom headers to include in the HTTP request. Useful for authentication, content type specifications, and other request metadata.",
|
|
@@ -72,7 +80,7 @@
|
|
|
72
80
|
},
|
|
73
81
|
"parse_response": {
|
|
74
82
|
"title": "Parse Response",
|
|
75
|
-
"description": "
|
|
83
|
+
"description": "How to extract rows from the response. Three supported forms: (1) a JSONPath expression pointing to a single record whose keys become column names, e.g. `$.data[0]`; (2) a JSON array of two JSONPaths `[\"$.headers_path\", \"$.values_path\"]` where the first resolves to column names and the second to a flat stream of values sliced into rows; (3) the same two-path form where the values path resolves to a 2D array (array of row-arrays), useful for Microsoft Graph Excel range endpoints, e.g. `[\"$.values[0]\", \"$.values[1:]\"]`.",
|
|
76
84
|
"type": "string",
|
|
77
85
|
"x-response-format": true,
|
|
78
86
|
"order": 8,
|
|
@@ -112,6 +120,47 @@
|
|
|
112
120
|
"description": "The starting page number for pagination. This value is used to calculate the offset for the initial request.",
|
|
113
121
|
"order": 13
|
|
114
122
|
}
|
|
123
|
+
},
|
|
124
|
+
"dependencies": {
|
|
125
|
+
"auth_type": {
|
|
126
|
+
"oneOf": [
|
|
127
|
+
{
|
|
128
|
+
"properties": {
|
|
129
|
+
"auth_type": { "enum": ["none"] }
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
"properties": {
|
|
134
|
+
"auth_type": { "enum": ["oauth_client_credentials"] },
|
|
135
|
+
"token_url": {
|
|
136
|
+
"type": "string",
|
|
137
|
+
"title": "OAuth Token URL",
|
|
138
|
+
"description": "E.g. https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token",
|
|
139
|
+
"order": 15
|
|
140
|
+
},
|
|
141
|
+
"client_id": {
|
|
142
|
+
"type": "string",
|
|
143
|
+
"multiwoven_secret": true,
|
|
144
|
+
"title": "OAuth Client ID",
|
|
145
|
+
"order": 16
|
|
146
|
+
},
|
|
147
|
+
"client_secret": {
|
|
148
|
+
"type": "string",
|
|
149
|
+
"multiwoven_secret": true,
|
|
150
|
+
"title": "OAuth Client Secret",
|
|
151
|
+
"order": 17
|
|
152
|
+
},
|
|
153
|
+
"scope": {
|
|
154
|
+
"type": "string",
|
|
155
|
+
"title": "OAuth Scope",
|
|
156
|
+
"description": "Optional. E.g. https://graph.microsoft.com/.default",
|
|
157
|
+
"order": 18
|
|
158
|
+
}
|
|
159
|
+
},
|
|
160
|
+
"required": ["token_url", "client_id", "client_secret"]
|
|
161
|
+
}
|
|
162
|
+
]
|
|
163
|
+
}
|
|
115
164
|
}
|
|
116
165
|
}
|
|
117
166
|
}
|
|
@@ -64,6 +64,7 @@ require_relative "integrations/core/source_connector"
|
|
|
64
64
|
require_relative "integrations/core/destination_connector"
|
|
65
65
|
require_relative "integrations/core/http_helper"
|
|
66
66
|
require_relative "integrations/core/http_client"
|
|
67
|
+
require_relative "integrations/core/oauth_client_credentials"
|
|
67
68
|
require_relative "integrations/core/streaming_http_client"
|
|
68
69
|
require_relative "integrations/core/query_builder"
|
|
69
70
|
require_relative "integrations/core/unstructured_source_connector"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.38.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-07-
|
|
11
|
+
date: 2026-07-17 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -581,6 +581,7 @@ files:
|
|
|
581
581
|
- lib/multiwoven/integrations/core/fullrefresher.rb
|
|
582
582
|
- lib/multiwoven/integrations/core/http_client.rb
|
|
583
583
|
- lib/multiwoven/integrations/core/http_helper.rb
|
|
584
|
+
- lib/multiwoven/integrations/core/oauth_client_credentials.rb
|
|
584
585
|
- lib/multiwoven/integrations/core/query_builder.rb
|
|
585
586
|
- lib/multiwoven/integrations/core/rate_limiter.rb
|
|
586
587
|
- lib/multiwoven/integrations/core/source_connector.rb
|