multiwoven-integrations 0.16.2 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/sftp/client.rb +133 -0
- data/lib/multiwoven/integrations/source/sftp/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/sftp/config/spec.json +59 -0
- data/lib/multiwoven/integrations/source/sftp/icon.svg +1 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1716352c3eca7da21c2291b6c0980c14c00e7acaaf2d1c806121d3b85c0628d8
|
|
4
|
+
data.tar.gz: a6096a5d2f71aa886d3e447d98b17c4e0d3104ec283ad1015770f9ae4aeeaed6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2b380bd351e7ab48a6675d05383ad57b15bf6d5f300d08064a36ffe32503280e97ae50f9a215229f431e5f84008117fe73924a4380c8bf3f242b7319ef1661e2
|
|
7
|
+
data.tar.gz: c5fb7462349698add62a5dfd25393b0c5655e0df671972afe56391734190b23d536a1e84f4e1dd03874aec5e0dc10533b4e887b585fbc6d43bcc7eba24be965d
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.17.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -21,6 +21,7 @@ module Multiwoven
|
|
|
21
21
|
VertexModel
|
|
22
22
|
HttpModel
|
|
23
23
|
OpenAI
|
|
24
|
+
Sftp
|
|
24
25
|
].freeze
|
|
25
26
|
|
|
26
27
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module Sftp
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
if @sftp.stat!(@remote_file_path)
|
|
11
|
+
success_status
|
|
12
|
+
else
|
|
13
|
+
failure_status(nil)
|
|
14
|
+
end
|
|
15
|
+
rescue StandardError => e
|
|
16
|
+
handle_exception(e, {
|
|
17
|
+
context: "SFTP:CHECK_CONNECTION:EXCEPTION",
|
|
18
|
+
type: "error"
|
|
19
|
+
})
|
|
20
|
+
failure_status(e)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def discover(connection_config)
|
|
24
|
+
connection_config = connection_config.with_indifferent_access
|
|
25
|
+
db = create_connection(connection_config)
|
|
26
|
+
@sftp.download!(@remote_file_path, @tempfile.path)
|
|
27
|
+
query = "SELECT * FROM read_csv_auto('#{@tempfile.path}')"
|
|
28
|
+
records = db.query(query).columns
|
|
29
|
+
catalog = Catalog.new(streams: create_streams(records.map(&:name)))
|
|
30
|
+
catalog.to_multiwoven_message
|
|
31
|
+
rescue StandardError => e
|
|
32
|
+
handle_exception(e, {
|
|
33
|
+
context: "SFTP:DISCOVER:EXCEPTION",
|
|
34
|
+
type: "error"
|
|
35
|
+
})
|
|
36
|
+
ensure
|
|
37
|
+
@tempfile&.close!
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def read(sync_config)
|
|
41
|
+
connection_config = sync_config.source.connection_specification
|
|
42
|
+
connection_config = connection_config.with_indifferent_access
|
|
43
|
+
conn = create_connection(connection_config)
|
|
44
|
+
query = sync_config.model.query
|
|
45
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
46
|
+
query(conn, query)
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
handle_exception(e, {
|
|
49
|
+
context: "SFTP:READ:EXCEPTION",
|
|
50
|
+
type: "error",
|
|
51
|
+
sync_id: sync_config.sync_id,
|
|
52
|
+
sync_run_id: sync_config.sync_run_id
|
|
53
|
+
})
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def create_connection(connection_config)
|
|
59
|
+
initialize_file_path(connection_config)
|
|
60
|
+
@sftp = with_sftp_client(connection_config)
|
|
61
|
+
conn = DuckDB::Database.open.connect
|
|
62
|
+
conn.execute(INSTALL_HTTPFS_QUERY)
|
|
63
|
+
conn
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def initialize_file_path(connection_config)
|
|
67
|
+
@remote_file_path = File.join(
|
|
68
|
+
connection_config[:file_path],
|
|
69
|
+
"#{connection_config[:file_name]}.#{connection_config[:format_type]}"
|
|
70
|
+
)
|
|
71
|
+
@tempfile = Tempfile.new(File.basename(@remote_file_path))
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def with_sftp_client(connection_config, &block)
|
|
75
|
+
Net::SFTP.start(
|
|
76
|
+
connection_config[:host],
|
|
77
|
+
connection_config[:username],
|
|
78
|
+
password: connection_config[:password],
|
|
79
|
+
port: connection_config.fetch(:port, 22), &block
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def get_results(conn, query)
|
|
84
|
+
results = conn.query(query)
|
|
85
|
+
hash_array_values(results)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def query(conn, query)
|
|
89
|
+
query_regex = /\ASELECT\s+(?<columns>[\w,\s]+)\s+FROM\s+\w+\s*(?:LIMIT\s+(?<limit>\d+))?\s*(?:OFFSET\s+(?<offset>\d+))?\z/i
|
|
90
|
+
match = query.match(query_regex)
|
|
91
|
+
columns = match[:columns] || "*"
|
|
92
|
+
offset = match[:offset].to_i || 0
|
|
93
|
+
limit = match[:limit]&.to_i || nil
|
|
94
|
+
@sftp.download!(@remote_file_path, @tempfile.path)
|
|
95
|
+
adjusted_query = "SELECT #{columns} FROM read_csv_auto('#{@tempfile.path}') OFFSET #{offset}"
|
|
96
|
+
adjusted_query += " LIMIT #{limit}" if limit
|
|
97
|
+
records = get_results(conn, adjusted_query)
|
|
98
|
+
records.map do |row|
|
|
99
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def hash_array_values(describe)
|
|
104
|
+
keys = describe.columns.map(&:name)
|
|
105
|
+
describe.map do |row|
|
|
106
|
+
Hash[keys.zip(row)]
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def create_streams(records)
|
|
111
|
+
group_by_table(records).map do |_, r|
|
|
112
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def group_by_table(records)
|
|
117
|
+
result = {}
|
|
118
|
+
records.each_with_index do |column, index|
|
|
119
|
+
table_name = @remote_file_path
|
|
120
|
+
column_data = {
|
|
121
|
+
column_name: column,
|
|
122
|
+
type: "string",
|
|
123
|
+
optional: true
|
|
124
|
+
}
|
|
125
|
+
result[index] ||= {}
|
|
126
|
+
result[index][:tablename] = table_name
|
|
127
|
+
result[index][:columns] = [column_data]
|
|
128
|
+
end
|
|
129
|
+
result
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Sftp",
|
|
4
|
+
"title": "SFTP",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "File Storage",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "source-sftp",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/sftp",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "SFTP",
|
|
7
|
+
"required": ["host", "username", "password", "file_path", "format_type" ],
|
|
8
|
+
"properties": {
|
|
9
|
+
"host": {
|
|
10
|
+
"title": "Host",
|
|
11
|
+
"description": "Hostname of the SFTP server.",
|
|
12
|
+
"type": "string",
|
|
13
|
+
"order": 0
|
|
14
|
+
},
|
|
15
|
+
"port": {
|
|
16
|
+
"title": "Port",
|
|
17
|
+
"description": "Port of the SFTP server.",
|
|
18
|
+
"type": "integer",
|
|
19
|
+
"minimum": 0,
|
|
20
|
+
"maximum": 65536,
|
|
21
|
+
"default": 22,
|
|
22
|
+
"order": 1
|
|
23
|
+
},
|
|
24
|
+
"username": {
|
|
25
|
+
"title": "User Name",
|
|
26
|
+
"description": "Username to use to access the SFTP server.",
|
|
27
|
+
"type": "string",
|
|
28
|
+
"order": 2
|
|
29
|
+
},
|
|
30
|
+
"password": {
|
|
31
|
+
"title": "Password",
|
|
32
|
+
"description": "Password associated with the username.",
|
|
33
|
+
"type": "string",
|
|
34
|
+
"multiwoven_secret": true,
|
|
35
|
+
"order": 3
|
|
36
|
+
},
|
|
37
|
+
"file_path": {
|
|
38
|
+
"title": "File path",
|
|
39
|
+
"type": "string",
|
|
40
|
+
"description": "Path to the directory where file is stored.",
|
|
41
|
+
"order": 4
|
|
42
|
+
},
|
|
43
|
+
"file_name": {
|
|
44
|
+
"title": "File Name",
|
|
45
|
+
"type": "string",
|
|
46
|
+
"description": "Name of the file to be written.",
|
|
47
|
+
"order": 5
|
|
48
|
+
},
|
|
49
|
+
"format_type": {
|
|
50
|
+
"title": "File Format Type",
|
|
51
|
+
"type": "string",
|
|
52
|
+
"description": "Format of the data output.",
|
|
53
|
+
"enum": ["csv"],
|
|
54
|
+
"default": "csv",
|
|
55
|
+
"order": 6
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="250" height="250" fill="none"><g clip-path="url(#a)"><path fill="#615EFF" d="M200.745 218.706a12.051 12.051 0 0 1-3.483 8.475 11.837 11.837 0 0 1-8.4 3.514H62.072c-3.15 0-6.173-1.261-8.403-3.507a12.053 12.053 0 0 1-3.49-8.472V31.485a12.08 12.08 0 0 1 .903-4.593 11.996 11.996 0 0 1 2.578-3.894 11.875 11.875 0 0 1 3.86-2.6 11.791 11.791 0 0 1 4.552-.912h81.371l57.302 58.96v140.26Z"/><path fill="#fff" d="M83.868 156.832c-.142-1.429-.75-2.539-1.822-3.33-1.074-.792-2.53-1.187-4.37-1.187-1.25 0-2.305.177-3.166.531-.861.343-1.522.821-1.981 1.435a3.457 3.457 0 0 0-.673 2.09c-.023.65.112 1.217.407 1.701.307.484.725.903 1.256 1.258.53.342 1.144.643 1.84.903a19.02 19.02 0 0 0 2.229.638l3.255.779c1.58.354 3.03.827 4.352 1.417 1.32.591 2.464 1.317 3.431 2.179a9.057 9.057 0 0 1 2.247 3.047c.542 1.169.82 2.509.831 4.021-.011 2.22-.578 4.145-1.698 5.775-1.109 1.617-2.712 2.875-4.811 3.773-2.088.885-4.606 1.328-7.554 1.328-2.925 0-5.472-.449-7.642-1.346-2.158-.898-3.845-2.226-5.06-3.986-1.202-1.771-1.833-3.962-1.892-6.572h7.412c.082 1.217.43 2.232 1.044 3.047.625.803 1.456 1.411 2.494 1.825 1.05.401 2.235.602 3.555.602 1.298 0 2.424-.189 3.38-.567.966-.378 1.715-.903 2.246-1.576.53-.673.796-1.447.796-2.321 0-.815-.242-1.5-.725-2.055-.472-.555-1.168-1.027-2.088-1.417-.908-.39-2.022-.744-3.343-1.063l-3.945-.992c-3.054-.744-5.466-1.907-7.235-3.489-1.77-1.583-2.648-3.714-2.636-6.395-.012-2.196.572-4.116 1.751-5.757 1.192-1.641 2.825-2.923 4.9-3.844 2.076-.921 4.435-1.382 7.076-1.382 2.69 0 5.036.461 7.04 1.382 2.018.921 3.586 2.203 4.706 3.844 1.12 1.641 1.699 3.543 1.734 5.704h-7.341Zm12.988 25.844v-36.278h23.988v6.324h-16.328v8.645h14.736v6.324h-14.736v14.985h-7.66Zm27.76-29.954v-6.324h29.754v6.324h-11.091v29.954h-7.571v-29.954h-11.092Zm34.654 29.954v-36.278h14.294c2.748 0 5.089.526 7.023 1.577 1.934 1.039 3.408 2.486 4.422 4.34 1.026 1.842 1.539 3.968 1.539 6.377s-.519 4.535-1.557 6.377c-1.037 1.842-2.541 3.277-4.51 4.304-1.958 1.028-4.329 1.541-7.112 1.541h-9.11v-6.146h7.872c1.474 0 2.689-.254 3.644-.762.967-.52 1.686-1.234 2.158-2.143.484-.921.725-1.978.725-3.171 0-1.205-.241-2.256-.725-3.153-.472-.91-1.191-1.612-2.158-2.108-.967-.508-2.194-.762-3.679-.762h-5.166v30.007h-7.66Z"/><path fill="#1A194D" d="m149.75 76.907 50.996 49.177V78.101h-45.173a11.33 11.33 0 0 1-5.643-1.433l-.18.24Z" opacity=".3"/><path fill="#C5C4FF" d="M200.746 78.35h-45.297a11.836 11.836 0 0 1-8.399-3.514 12.055 12.055 0 0 1-3.484-8.475v-47.2l57.18 59.189Z"/></g><defs><clipPath id="a"><path fill="#fff" d="M50 19h151v212H50z"/></clipPath></defs></svg>
|
|
@@ -74,6 +74,7 @@ require_relative "integrations/source/aws_sagemaker_model/client"
|
|
|
74
74
|
require_relative "integrations/source/google_vertex_model/client"
|
|
75
75
|
require_relative "integrations/source/http_model/client"
|
|
76
76
|
require_relative "integrations/source/open_ai/client"
|
|
77
|
+
require_relative "integrations/source/sftp/client"
|
|
77
78
|
|
|
78
79
|
# Destination
|
|
79
80
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.17.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-12-
|
|
11
|
+
date: 2024-12-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -758,6 +758,10 @@ files:
|
|
|
758
758
|
- lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json
|
|
759
759
|
- lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/icon.svg
|
|
760
760
|
- lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb
|
|
761
|
+
- lib/multiwoven/integrations/source/sftp/client.rb
|
|
762
|
+
- lib/multiwoven/integrations/source/sftp/config/meta.json
|
|
763
|
+
- lib/multiwoven/integrations/source/sftp/config/spec.json
|
|
764
|
+
- lib/multiwoven/integrations/source/sftp/icon.svg
|
|
761
765
|
- lib/multiwoven/integrations/source/snowflake/client.rb
|
|
762
766
|
- lib/multiwoven/integrations/source/snowflake/config/meta.json
|
|
763
767
|
- lib/multiwoven/integrations/source/snowflake/config/spec.json
|