multiwoven-integrations 0.37.5 → 0.38.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/oauth_client_credentials.rb +107 -0
- data/lib/multiwoven/integrations/destination/http/client.rb +6 -3
- data/lib/multiwoven/integrations/destination/http/config/spec.json +49 -0
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations/source/http/client.rb +6 -3
- data/lib/multiwoven/integrations/source/http/config/spec.json +49 -0
- data/lib/multiwoven/integrations/source/one_drive/client.rb +71 -18
- data/lib/multiwoven/integrations/source/one_drive/config/spec.json +6 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 828f206c7576ddd3e84774cc75175213d1bef9a78df725564c67ac37d5e474d8
|
|
4
|
+
data.tar.gz: 98b0930cef5d402c99ba5272a97cb918bb301573de8202cfaafe54d825f817e4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: afd08217ce63822cf47858221c70cda09b9755b17ce2e08e6fe7156b49b74a9c6c3525e52c2f07b6946ef27d7eccd0fe113846dd51bfa48467f3c8be1641c341
|
|
7
|
+
data.tar.gz: f0c1d17204b01e640d49ea0cb8747baead11ecbea7c4dd4e1c7a3aa9a938e8c60ca0cbb48044fa4a3b1aaed6cca51546928be77c206c4a263e965fca77cf74fa
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations::Core
|
|
5
|
+
# Shared OAuth2 client_credentials flow. Include in a connector client to:
|
|
6
|
+
# * inject `Authorization: Bearer <token>` when connection_config[:auth_type]
|
|
7
|
+
# is `oauth_client_credentials`
|
|
8
|
+
# * cache the token in the connector's `configuration` JSON and refresh it
|
|
9
|
+
# shortly before expiry
|
|
10
|
+
#
|
|
11
|
+
# The including class is expected to set `@connector_instance` (an object
|
|
12
|
+
# responding to `configuration` and `update!`) before making requests that
|
|
13
|
+
# should benefit from the cache. Without it, a fresh token is fetched every
|
|
14
|
+
# call — safe but wasteful.
|
|
15
|
+
module OauthClientCredentials
|
|
16
|
+
AUTH_TYPE_OAUTH_CLIENT_CREDENTIALS = "oauth_client_credentials"
|
|
17
|
+
# Refresh access tokens this many seconds before their advertised expiry,
|
|
18
|
+
# so a token that expires mid-request doesn't leave the connector.
|
|
19
|
+
TOKEN_EXPIRY_BUFFER_SECONDS = 300
|
|
20
|
+
|
|
21
|
+
def build_headers(connection_config)
|
|
22
|
+
headers = (connection_config[:headers] || {}).to_h.dup
|
|
23
|
+
return headers unless connection_config[:auth_type] == AUTH_TYPE_OAUTH_CLIENT_CREDENTIALS
|
|
24
|
+
|
|
25
|
+
headers["Authorization"] = "Bearer #{ensure_oauth_token(connection_config)}"
|
|
26
|
+
headers
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def ensure_oauth_token(connection_config)
|
|
30
|
+
cached = cached_oauth_token
|
|
31
|
+
return cached if cached
|
|
32
|
+
|
|
33
|
+
fetch_and_cache_oauth_token(connection_config)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def cached_oauth_token
|
|
39
|
+
config = connector_configuration
|
|
40
|
+
return nil unless config
|
|
41
|
+
|
|
42
|
+
token = config["oauth_access_token"]
|
|
43
|
+
expires_at = config["oauth_expires_at"]
|
|
44
|
+
return nil if token.nil? || token.to_s.empty? || expires_at.nil?
|
|
45
|
+
return nil if Time.parse(expires_at.to_s) <= Time.now + TOKEN_EXPIRY_BUFFER_SECONDS
|
|
46
|
+
|
|
47
|
+
token
|
|
48
|
+
rescue ArgumentError, TypeError
|
|
49
|
+
nil
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def fetch_and_cache_oauth_token(connection_config)
|
|
53
|
+
token_url = connection_config[:token_url]
|
|
54
|
+
client_id = connection_config[:client_id]
|
|
55
|
+
client_secret = connection_config[:client_secret]
|
|
56
|
+
raise ArgumentError, "OAuth token_url, client_id, and client_secret are required when auth_type is #{AUTH_TYPE_OAUTH_CLIENT_CREDENTIALS}" if token_url.to_s.empty? || client_id.to_s.empty? || client_secret.to_s.empty?
|
|
57
|
+
|
|
58
|
+
response = post_token_request(token_url, client_id, client_secret, connection_config[:scope])
|
|
59
|
+
raise "OAuth token request failed: #{response.code} #{response.body}" unless response.is_a?(Net::HTTPSuccess)
|
|
60
|
+
|
|
61
|
+
body = JSON.parse(response.body)
|
|
62
|
+
access_token = body["access_token"]
|
|
63
|
+
raise "OAuth token response missing 'access_token'" if access_token.to_s.empty?
|
|
64
|
+
|
|
65
|
+
expires_in = (body["expires_in"] || 3600).to_i
|
|
66
|
+
persist_oauth_token(access_token, Time.now + expires_in)
|
|
67
|
+
access_token
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def post_token_request(token_url, client_id, client_secret, scope)
|
|
71
|
+
uri = URI(token_url)
|
|
72
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
73
|
+
http.use_ssl = (uri.scheme == "https")
|
|
74
|
+
|
|
75
|
+
form = {
|
|
76
|
+
"grant_type" => "client_credentials",
|
|
77
|
+
"client_id" => client_id,
|
|
78
|
+
"client_secret" => client_secret
|
|
79
|
+
}
|
|
80
|
+
form["scope"] = scope unless scope.to_s.empty?
|
|
81
|
+
|
|
82
|
+
request = Net::HTTP::Post.new(uri)
|
|
83
|
+
request["Content-Type"] = "application/x-www-form-urlencoded"
|
|
84
|
+
request.body = URI.encode_www_form(form)
|
|
85
|
+
http.request(request)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def persist_oauth_token(access_token, expires_at)
|
|
89
|
+
return unless @connector_instance.respond_to?(:update!)
|
|
90
|
+
|
|
91
|
+
base = connector_configuration || {}
|
|
92
|
+
new_config = base.merge(
|
|
93
|
+
"oauth_access_token" => access_token,
|
|
94
|
+
"oauth_expires_at" => expires_at.iso8601
|
|
95
|
+
)
|
|
96
|
+
@connector_instance.update!(configuration: new_config)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def connector_configuration
|
|
100
|
+
return nil unless @connector_instance.respond_to?(:configuration)
|
|
101
|
+
|
|
102
|
+
cfg = @connector_instance.configuration
|
|
103
|
+
cfg.is_a?(Hash) ? cfg : nil
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -6,16 +6,18 @@ module Multiwoven
|
|
|
6
6
|
module Http
|
|
7
7
|
include Multiwoven::Integrations::Core
|
|
8
8
|
class Client < DestinationConnector
|
|
9
|
+
include Multiwoven::Integrations::Core::OauthClientCredentials
|
|
10
|
+
|
|
9
11
|
MAX_CHUNK_SIZE = 10
|
|
12
|
+
|
|
10
13
|
def check_connection(connection_config)
|
|
11
14
|
connection_config = connection_config.with_indifferent_access
|
|
12
15
|
destination_url = connection_config[:destination_url]
|
|
13
|
-
headers = connection_config[:headers]
|
|
14
16
|
request = Multiwoven::Integrations::Core::HttpClient.request(
|
|
15
17
|
destination_url,
|
|
16
18
|
HTTP_POST,
|
|
17
19
|
payload: {},
|
|
18
|
-
headers:
|
|
20
|
+
headers: build_headers(connection_config)
|
|
19
21
|
)
|
|
20
22
|
if success?(request)
|
|
21
23
|
success_status
|
|
@@ -43,8 +45,9 @@ module Multiwoven
|
|
|
43
45
|
|
|
44
46
|
def write(sync_config, records, _action = "create", _identifier_key = nil)
|
|
45
47
|
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
48
|
+
@connector_instance = sync_config&.destination&.connector_instance
|
|
46
49
|
url = connection_config[:destination_url]
|
|
47
|
-
headers = connection_config
|
|
50
|
+
headers = build_headers(connection_config)
|
|
48
51
|
log_message_array = []
|
|
49
52
|
write_success = 0
|
|
50
53
|
write_failure = 0
|
|
@@ -12,6 +12,14 @@
|
|
|
12
12
|
"title": "Destination url",
|
|
13
13
|
"order": 0
|
|
14
14
|
},
|
|
15
|
+
"auth_type": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"title": "Authentication Type",
|
|
18
|
+
"description": "Select 'oauth_client_credentials' to have Multiwoven fetch and refresh an OAuth2 access token automatically.",
|
|
19
|
+
"enum": ["none", "oauth_client_credentials"],
|
|
20
|
+
"default": "none",
|
|
21
|
+
"order": 2
|
|
22
|
+
},
|
|
15
23
|
"headers": {
|
|
16
24
|
"title": "Http Headers",
|
|
17
25
|
"order": 1,
|
|
@@ -19,6 +27,47 @@
|
|
|
19
27
|
"type": "string"
|
|
20
28
|
}
|
|
21
29
|
}
|
|
30
|
+
},
|
|
31
|
+
"dependencies": {
|
|
32
|
+
"auth_type": {
|
|
33
|
+
"oneOf": [
|
|
34
|
+
{
|
|
35
|
+
"properties": {
|
|
36
|
+
"auth_type": { "enum": ["none"] }
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
"properties": {
|
|
41
|
+
"auth_type": { "enum": ["oauth_client_credentials"] },
|
|
42
|
+
"token_url": {
|
|
43
|
+
"type": "string",
|
|
44
|
+
"title": "OAuth Token URL",
|
|
45
|
+
"description": "E.g. https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token",
|
|
46
|
+
"order": 3
|
|
47
|
+
},
|
|
48
|
+
"client_id": {
|
|
49
|
+
"type": "string",
|
|
50
|
+
"multiwoven_secret": true,
|
|
51
|
+
"title": "OAuth Client ID",
|
|
52
|
+
"order": 4
|
|
53
|
+
},
|
|
54
|
+
"client_secret": {
|
|
55
|
+
"type": "string",
|
|
56
|
+
"multiwoven_secret": true,
|
|
57
|
+
"title": "OAuth Client Secret",
|
|
58
|
+
"order": 5
|
|
59
|
+
},
|
|
60
|
+
"scope": {
|
|
61
|
+
"type": "string",
|
|
62
|
+
"title": "OAuth Scope",
|
|
63
|
+
"description": "Optional. E.g. https://graph.microsoft.com/.default",
|
|
64
|
+
"order": 6
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"required": ["token_url", "client_id", "client_secret"]
|
|
68
|
+
}
|
|
69
|
+
]
|
|
70
|
+
}
|
|
22
71
|
}
|
|
23
72
|
}
|
|
24
73
|
}
|
|
@@ -4,6 +4,8 @@ module Multiwoven::Integrations::Source
|
|
|
4
4
|
module Http
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
6
|
class Client < SourceConnector
|
|
7
|
+
include Multiwoven::Integrations::Core::OauthClientCredentials
|
|
8
|
+
|
|
7
9
|
def check_connection(connection_config)
|
|
8
10
|
connection_config = prepare_config(connection_config)
|
|
9
11
|
create_connection(connection_config)
|
|
@@ -17,7 +19,7 @@ module Multiwoven::Integrations::Source
|
|
|
17
19
|
url: @url,
|
|
18
20
|
http_method: connection_config[:http_method],
|
|
19
21
|
payload: connection_config[:request_format],
|
|
20
|
-
headers: connection_config
|
|
22
|
+
headers: build_headers(connection_config),
|
|
21
23
|
config: connection_config[:config],
|
|
22
24
|
params: connection_config[:params]
|
|
23
25
|
)
|
|
@@ -40,7 +42,7 @@ module Multiwoven::Integrations::Source
|
|
|
40
42
|
url: @url,
|
|
41
43
|
http_method: connection_config[:http_method],
|
|
42
44
|
payload: connection_config[:request_format],
|
|
43
|
-
headers: connection_config
|
|
45
|
+
headers: build_headers(connection_config),
|
|
44
46
|
config: connection_config[:config],
|
|
45
47
|
params: connection_config[:params]
|
|
46
48
|
)
|
|
@@ -55,6 +57,7 @@ module Multiwoven::Integrations::Source
|
|
|
55
57
|
def read(sync_config)
|
|
56
58
|
connection_config = sync_config.source.connection_specification
|
|
57
59
|
connection_config = connection_config.with_indifferent_access
|
|
60
|
+
@connector_instance = sync_config&.source&.connector_instance
|
|
58
61
|
connection_config = create_connection(connection_config)
|
|
59
62
|
if sync_config.increment_strategy_config&.increment_strategy == "page"
|
|
60
63
|
@limit = sync_config.increment_strategy_config.limit
|
|
@@ -132,7 +135,7 @@ module Multiwoven::Integrations::Source
|
|
|
132
135
|
url: @url,
|
|
133
136
|
http_method: connection_config[:http_method],
|
|
134
137
|
payload: connection_config[:request_format],
|
|
135
|
-
headers: connection_config
|
|
138
|
+
headers: build_headers(connection_config),
|
|
136
139
|
config: connection_config[:config],
|
|
137
140
|
params: connection_config[:params] || {}
|
|
138
141
|
)
|
|
@@ -33,6 +33,14 @@
|
|
|
33
33
|
"order": 3,
|
|
34
34
|
"default": "/"
|
|
35
35
|
},
|
|
36
|
+
"auth_type": {
|
|
37
|
+
"type": "string",
|
|
38
|
+
"title": "Authentication Type",
|
|
39
|
+
"description": "Select 'oauth_client_credentials' to have Multiwoven fetch and refresh an OAuth2 access token automatically.",
|
|
40
|
+
"enum": ["none", "oauth_client_credentials"],
|
|
41
|
+
"default": "none",
|
|
42
|
+
"order": 14
|
|
43
|
+
},
|
|
36
44
|
"headers": {
|
|
37
45
|
"title": "HTTP Headers",
|
|
38
46
|
"description": "Custom headers to include in the HTTP request. Useful for authentication, content type specifications, and other request metadata.",
|
|
@@ -112,6 +120,47 @@
|
|
|
112
120
|
"description": "The starting page number for pagination. This value is used to calculate the offset for the initial request.",
|
|
113
121
|
"order": 13
|
|
114
122
|
}
|
|
123
|
+
},
|
|
124
|
+
"dependencies": {
|
|
125
|
+
"auth_type": {
|
|
126
|
+
"oneOf": [
|
|
127
|
+
{
|
|
128
|
+
"properties": {
|
|
129
|
+
"auth_type": { "enum": ["none"] }
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
"properties": {
|
|
134
|
+
"auth_type": { "enum": ["oauth_client_credentials"] },
|
|
135
|
+
"token_url": {
|
|
136
|
+
"type": "string",
|
|
137
|
+
"title": "OAuth Token URL",
|
|
138
|
+
"description": "E.g. https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token",
|
|
139
|
+
"order": 15
|
|
140
|
+
},
|
|
141
|
+
"client_id": {
|
|
142
|
+
"type": "string",
|
|
143
|
+
"multiwoven_secret": true,
|
|
144
|
+
"title": "OAuth Client ID",
|
|
145
|
+
"order": 16
|
|
146
|
+
},
|
|
147
|
+
"client_secret": {
|
|
148
|
+
"type": "string",
|
|
149
|
+
"multiwoven_secret": true,
|
|
150
|
+
"title": "OAuth Client Secret",
|
|
151
|
+
"order": 17
|
|
152
|
+
},
|
|
153
|
+
"scope": {
|
|
154
|
+
"type": "string",
|
|
155
|
+
"title": "OAuth Scope",
|
|
156
|
+
"description": "Optional. E.g. https://graph.microsoft.com/.default",
|
|
157
|
+
"order": 18
|
|
158
|
+
}
|
|
159
|
+
},
|
|
160
|
+
"required": ["token_url", "client_id", "client_secret"]
|
|
161
|
+
}
|
|
162
|
+
]
|
|
163
|
+
}
|
|
115
164
|
}
|
|
116
165
|
}
|
|
117
166
|
}
|
|
@@ -82,6 +82,7 @@ module Multiwoven::Integrations::Source
|
|
|
82
82
|
@data_type = connection_config[:data_type]
|
|
83
83
|
@file_name = connection_config[:file_name]
|
|
84
84
|
@share_url = connection_config[:share_url]
|
|
85
|
+
@is_recursive = [true, "true"].include?(connection_config[:is_recursive])
|
|
85
86
|
stored_token = @connector_instance&.configuration&.dig("access_token")
|
|
86
87
|
@access_token = stored_token.presence || refresh_access_token
|
|
87
88
|
end
|
|
@@ -173,11 +174,12 @@ module Multiwoven::Integrations::Source
|
|
|
173
174
|
|
|
174
175
|
def list_files_in_folder(_connection_config)
|
|
175
176
|
files_in_folder.map do |file|
|
|
177
|
+
relative_path = relative_file_path(file)
|
|
176
178
|
RecordMessage.new(
|
|
177
179
|
data: {
|
|
178
180
|
element_id: file["id"],
|
|
179
181
|
file_name: file["name"],
|
|
180
|
-
file_path:
|
|
182
|
+
file_path: relative_path,
|
|
181
183
|
size: file["size"],
|
|
182
184
|
file_type: File.extname(file["name"]).sub(".", ""),
|
|
183
185
|
created_date: file["createdDateTime"],
|
|
@@ -190,10 +192,12 @@ module Multiwoven::Integrations::Source
|
|
|
190
192
|
end
|
|
191
193
|
|
|
192
194
|
def download_unstructured_file(_connection_config, file_path, sync_id)
|
|
193
|
-
|
|
194
|
-
file_item =
|
|
195
|
+
lookup_path = resolve_download_file_name(file_path)
|
|
196
|
+
file_item = find_file_item(lookup_path)
|
|
195
197
|
raise StandardError, "File not found." if file_item.nil?
|
|
196
198
|
|
|
199
|
+
file_name = file_item["name"]
|
|
200
|
+
relative_path = relative_file_path(file_item)
|
|
197
201
|
local_path = download_file_to_local(
|
|
198
202
|
file_name,
|
|
199
203
|
sync_id,
|
|
@@ -206,7 +210,7 @@ module Multiwoven::Integrations::Source
|
|
|
206
210
|
element_id: file_item["id"],
|
|
207
211
|
local_path: local_path,
|
|
208
212
|
file_name: file_name,
|
|
209
|
-
file_path:
|
|
213
|
+
file_path: relative_path,
|
|
210
214
|
size: file_item["size"],
|
|
211
215
|
file_type: File.extname(file_name).sub(".", ""),
|
|
212
216
|
created_date: file_item["createdDateTime"],
|
|
@@ -220,19 +224,35 @@ module Multiwoven::Integrations::Source
|
|
|
220
224
|
def files_in_folder
|
|
221
225
|
records = fetch_list_items
|
|
222
226
|
records["value"].select do |item|
|
|
223
|
-
item["folder"].blank? && matching_file_name?(item["name"])
|
|
227
|
+
item["folder"].blank? && matching_file_name?(item["name"], relative_file_path(item))
|
|
224
228
|
end
|
|
225
229
|
end
|
|
226
230
|
|
|
231
|
+
def find_file_item(lookup_path)
|
|
232
|
+
files = files_in_folder
|
|
233
|
+
exact_match = files.find do |item|
|
|
234
|
+
relative_path = relative_file_path(item)
|
|
235
|
+
relative_path == lookup_path
|
|
236
|
+
end
|
|
237
|
+
return exact_match if exact_match
|
|
238
|
+
return if lookup_path.include?("/")
|
|
239
|
+
|
|
240
|
+
files.find { |item| item["name"] == lookup_path }
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def relative_file_path(file)
|
|
244
|
+
file["relative_path"].presence || file["name"]
|
|
245
|
+
end
|
|
246
|
+
|
|
227
247
|
def resolve_download_file_name(file_path)
|
|
228
|
-
return
|
|
248
|
+
return file_path.to_s.strip unless file_path.to_s.start_with?("http")
|
|
229
249
|
|
|
230
250
|
@file_name.to_s.strip.presence || File.basename(file_path)
|
|
231
251
|
end
|
|
232
252
|
|
|
233
|
-
def matching_file_name?(name)
|
|
253
|
+
def matching_file_name?(name, relative_path = nil)
|
|
234
254
|
configured_name = @file_name.to_s.strip
|
|
235
|
-
configured_name.blank? || configured_name == name
|
|
255
|
+
configured_name.blank? || configured_name == name || configured_name == relative_path
|
|
236
256
|
end
|
|
237
257
|
|
|
238
258
|
def discover_stream_for_file(conn, file)
|
|
@@ -240,7 +260,7 @@ module Multiwoven::Integrations::Source
|
|
|
240
260
|
columns = build_discover_columns(describe_results)
|
|
241
261
|
|
|
242
262
|
Multiwoven::Integrations::Protocol::Stream.new(
|
|
243
|
-
name: stream_name_for(file
|
|
263
|
+
name: stream_name_for(file),
|
|
244
264
|
action: StreamAction["fetch"],
|
|
245
265
|
json_schema: convert_to_json_schema(columns)
|
|
246
266
|
)
|
|
@@ -289,9 +309,10 @@ module Multiwoven::Integrations::Source
|
|
|
289
309
|
def query(connection, query)
|
|
290
310
|
local_file = nil
|
|
291
311
|
file_name = extract_file_name_from_query(query)
|
|
312
|
+
local_basename = File.basename(file_name)
|
|
292
313
|
local_file = download_file_to_local(file_name, @sync_id)
|
|
293
314
|
|
|
294
|
-
file = read_local_file(connection,
|
|
315
|
+
file = read_local_file(connection, local_basename, local_file)
|
|
295
316
|
query = apply_local_file_to_query(query, file)
|
|
296
317
|
get_results(connection, query).map do |row|
|
|
297
318
|
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
@@ -393,7 +414,8 @@ module Multiwoven::Integrations::Source
|
|
|
393
414
|
end
|
|
394
415
|
|
|
395
416
|
def single_file_item_url(file_name)
|
|
396
|
-
|
|
417
|
+
# Preserve path separators for nested files; encode other unsafe chars.
|
|
418
|
+
encoded_file = file_name.to_s.split("/").map { |segment| URI::DEFAULT_PARSER.escape(segment) }.join("/")
|
|
397
419
|
|
|
398
420
|
if @share_url.present?
|
|
399
421
|
shared = shared_folder_reference
|
|
@@ -440,7 +462,35 @@ module Multiwoven::Integrations::Source
|
|
|
440
462
|
|
|
441
463
|
return { "value" => [fetch_shared_item_metadata] } if @share_url.present? && shared_folder_reference[:is_file]
|
|
442
464
|
|
|
443
|
-
|
|
465
|
+
collect_files_from_folder(list_items_url, recursive: @is_recursive)
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
# Lists files under the configured folder. When recursive is true, BFS over
|
|
469
|
+
# /children so nested folder files are included in syncs.
|
|
470
|
+
def collect_files_from_folder(root_url, recursive: false)
|
|
471
|
+
files = []
|
|
472
|
+
queue = [[root_url, ""]]
|
|
473
|
+
|
|
474
|
+
until queue.empty?
|
|
475
|
+
url, prefix = queue.shift
|
|
476
|
+
page = paginated_graph_collection(url)
|
|
477
|
+
|
|
478
|
+
page["value"].each do |item|
|
|
479
|
+
relative_path = prefix.empty? ? item["name"].to_s : "#{prefix}/#{item["name"]}"
|
|
480
|
+
|
|
481
|
+
if item["folder"].present?
|
|
482
|
+
next unless recursive
|
|
483
|
+
|
|
484
|
+
drive_id = item.dig("parentReference", "driveId") || @drive_id
|
|
485
|
+
queue << ["#{drive_item_url(drive_id, item["id"])}/children", relative_path]
|
|
486
|
+
else
|
|
487
|
+
item["relative_path"] = relative_path
|
|
488
|
+
files << item
|
|
489
|
+
end
|
|
490
|
+
end
|
|
491
|
+
end
|
|
492
|
+
|
|
493
|
+
{ "value" => files }
|
|
444
494
|
end
|
|
445
495
|
|
|
446
496
|
def fetch_shared_item_metadata
|
|
@@ -460,7 +510,9 @@ module Multiwoven::Integrations::Source
|
|
|
460
510
|
response = microsoft_graph_request(single_file_item_url(@file_name))
|
|
461
511
|
raise graph_api_error(response.body) unless success?(response)
|
|
462
512
|
|
|
463
|
-
JSON.parse(response.body)
|
|
513
|
+
item = JSON.parse(response.body)
|
|
514
|
+
item["relative_path"] = @file_name.to_s.strip
|
|
515
|
+
item
|
|
464
516
|
end
|
|
465
517
|
|
|
466
518
|
def paginated_graph_collection(url)
|
|
@@ -498,14 +550,15 @@ module Multiwoven::Integrations::Source
|
|
|
498
550
|
records["value"].select do |record|
|
|
499
551
|
record["folder"].blank? &&
|
|
500
552
|
SPREADSHEET_EXTENSIONS.include?(File.extname(record["name"].to_s).downcase) &&
|
|
501
|
-
matching_file_name?(record["name"])
|
|
553
|
+
matching_file_name?(record["name"], relative_file_path(record))
|
|
502
554
|
end
|
|
503
555
|
end
|
|
504
556
|
|
|
505
|
-
# Keep the
|
|
506
|
-
# `SELECT * FROM ${stream.name}`, and read_local_file keys off
|
|
507
|
-
|
|
508
|
-
|
|
557
|
+
# Keep the relative path (with extension) in the stream name — TableSelector
|
|
558
|
+
# generates `SELECT * FROM ${stream.name}`, and read_local_file keys off
|
|
559
|
+
# File.extname. Relative paths also disambiguate nested duplicates.
|
|
560
|
+
def stream_name_for(file)
|
|
561
|
+
relative_file_path(file)
|
|
509
562
|
end
|
|
510
563
|
|
|
511
564
|
def encode_sharing_url(url)
|
|
@@ -45,6 +45,12 @@
|
|
|
45
45
|
"title": "Share URL",
|
|
46
46
|
"description": "OneDrive or SharePoint sharing link for the folder to read from, skip if using User Name to access Root Folder."
|
|
47
47
|
},
|
|
48
|
+
"is_recursive": {
|
|
49
|
+
"type": "boolean",
|
|
50
|
+
"title": "Enable recursive",
|
|
51
|
+
"description": "Enables recursive folder traversal. When true, all files and subfolders are read. Default is false, reading only the specified folder.",
|
|
52
|
+
"default": false
|
|
53
|
+
},
|
|
48
54
|
"file_name": {
|
|
49
55
|
"type": "string",
|
|
50
56
|
"title": "File Name",
|
|
@@ -64,6 +64,7 @@ require_relative "integrations/core/source_connector"
|
|
|
64
64
|
require_relative "integrations/core/destination_connector"
|
|
65
65
|
require_relative "integrations/core/http_helper"
|
|
66
66
|
require_relative "integrations/core/http_client"
|
|
67
|
+
require_relative "integrations/core/oauth_client_credentials"
|
|
67
68
|
require_relative "integrations/core/streaming_http_client"
|
|
68
69
|
require_relative "integrations/core/query_builder"
|
|
69
70
|
require_relative "integrations/core/unstructured_source_connector"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.38.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-07-
|
|
11
|
+
date: 2026-07-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -581,6 +581,7 @@ files:
|
|
|
581
581
|
- lib/multiwoven/integrations/core/fullrefresher.rb
|
|
582
582
|
- lib/multiwoven/integrations/core/http_client.rb
|
|
583
583
|
- lib/multiwoven/integrations/core/http_helper.rb
|
|
584
|
+
- lib/multiwoven/integrations/core/oauth_client_credentials.rb
|
|
584
585
|
- lib/multiwoven/integrations/core/query_builder.rb
|
|
585
586
|
- lib/multiwoven/integrations/core/rate_limiter.rb
|
|
586
587
|
- lib/multiwoven/integrations/core/source_connector.rb
|