multiwoven-integrations 0.1.41 → 0.1.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/utils.rb +2 -1
- data/lib/multiwoven/integrations/destination/sftp/client.rb +124 -0
- data/lib/multiwoven/integrations/destination/sftp/config/catalog.json +16 -0
- data/lib/multiwoven/integrations/destination/sftp/config/meta.json +16 -0
- data/lib/multiwoven/integrations/destination/sftp/config/spec.json +44 -0
- data/lib/multiwoven/integrations/destination/sftp/icon.svg +1 -0
- data/lib/multiwoven/integrations/protocol/protocol.rb +2 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations.rb +4 -0
- metadata +35 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: eee51b7ba03e877f2765da694f9ca7b4ede8e44e07ca74931d808f79fb828c06
|
|
4
|
+
data.tar.gz: 21a23a656e09175222172eea24e562c95ccc97e7bebe92443cc60678d68050bf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 388d743e298e3296fbd9bd12e5d29037d72bf4912f93ae8ae8b9d7f5285dbb2d39f39f1500f6ee3b6214e186564287eec8c3fe14d66f55cc53861585e3929e33
|
|
7
|
+
data.tar.gz: e527726e1e69c57c07d7d6ab28194f2a84dfbbffb941f61e2b5fca927cecf486edcb7fa2cecbbcc8ccc1713c396e1723161e5ea55f0cc8d310ce2d888f6de845
|
|
@@ -95,7 +95,8 @@ module Multiwoven
|
|
|
95
95
|
request_rate_limit: stream_json["request_rate_limit"].to_i,
|
|
96
96
|
request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
|
|
97
97
|
request_rate_concurrency: stream_json["request_rate_concurrency"].to_i,
|
|
98
|
-
supported_sync_modes: stream_json["supported_sync_modes"]
|
|
98
|
+
supported_sync_modes: stream_json["supported_sync_modes"],
|
|
99
|
+
schema_mode: stream_json["schema_mode"]
|
|
99
100
|
)
|
|
100
101
|
end
|
|
101
102
|
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Destination
|
|
4
|
+
module Sftp
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < DestinationConnector
|
|
7
|
+
prepend Multiwoven::Integrations::Core::Fullrefresher
|
|
8
|
+
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
9
|
+
|
|
10
|
+
def check_connection(connection_config)
|
|
11
|
+
connection_config = connection_config.with_indifferent_access
|
|
12
|
+
with_sftp_client(connection_config) do |sftp|
|
|
13
|
+
stream = SecureRandom.uuid
|
|
14
|
+
test_path = "/path/to/test/#{stream}"
|
|
15
|
+
test_file_operations(sftp, test_path)
|
|
16
|
+
return success_status
|
|
17
|
+
end
|
|
18
|
+
rescue StandardError => e
|
|
19
|
+
handle_exception("SFTP:CHECK_CONNECTION:EXCEPTION", "error", e)
|
|
20
|
+
failure_status(e)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def discover(_connection_config = nil)
|
|
24
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
25
|
+
|
|
26
|
+
catalog = build_catalog(catalog_json)
|
|
27
|
+
|
|
28
|
+
catalog.to_multiwoven_message
|
|
29
|
+
rescue StandardError => e
|
|
30
|
+
handle_exception(
|
|
31
|
+
"SFTP:DISCOVER:EXCEPTION",
|
|
32
|
+
"error",
|
|
33
|
+
e
|
|
34
|
+
)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def write(sync_config, records, _action = "insert")
|
|
38
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
39
|
+
file_path = generate_file_path(sync_config)
|
|
40
|
+
csv_content = generate_csv_content(records)
|
|
41
|
+
write_success = 0
|
|
42
|
+
write_failure = 0
|
|
43
|
+
# 10000 records in single
|
|
44
|
+
with_sftp_client(connection_config) do |sftp|
|
|
45
|
+
sftp.file.open(file_path, "w") { |file| file.puts(csv_content) }
|
|
46
|
+
write_success += records.size
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
|
|
49
|
+
write_failure += records.size
|
|
50
|
+
end
|
|
51
|
+
tracking_message(write_success, write_failure)
|
|
52
|
+
rescue StandardError => e
|
|
53
|
+
handle_exception(
|
|
54
|
+
"SFTP:WRITE:EXCEPTION",
|
|
55
|
+
"error",
|
|
56
|
+
e
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def clear_all_records(sync_config)
|
|
61
|
+
connection_specification = sync_config.destination.connection_specification.with_indifferent_access
|
|
62
|
+
with_sftp_client(connection_specification) do |sftp|
|
|
63
|
+
files = sftp.dir.glob(connection_specification[:destination_path], "*")
|
|
64
|
+
|
|
65
|
+
files.each do |file|
|
|
66
|
+
sftp.remove!(File.join(connection_specification[:destination_path], file.name))
|
|
67
|
+
end
|
|
68
|
+
if sftp.dir.entries(connection_specification[:destination_path]).size == 2
|
|
69
|
+
control_message("Successfully cleared data.", "succeeded")
|
|
70
|
+
else
|
|
71
|
+
control_message("Failed to clear data.", "failed")
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
rescue StandardError => e
|
|
75
|
+
control_message(e.message, "failed")
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
|
|
80
|
+
def generate_file_path(sync_config)
|
|
81
|
+
timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
|
|
82
|
+
file_name = "#{sync_config.stream.name}_#{timestamp}.csv"
|
|
83
|
+
File.join(sync_config.destination.connection_specification[:destination_path], file_name)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def generate_csv_content(records)
|
|
87
|
+
CSV.generate do |csv|
|
|
88
|
+
headers = records.first.keys
|
|
89
|
+
csv << headers
|
|
90
|
+
records.each { |record| csv << record.values_at(*headers) }
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def tracking_message(success, failure)
|
|
95
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
96
|
+
success: success, failed: failure
|
|
97
|
+
).to_multiwoven_message
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def with_sftp_client(connection_config, &block)
|
|
101
|
+
Net::SFTP.start(
|
|
102
|
+
connection_config[:host],
|
|
103
|
+
connection_config[:username],
|
|
104
|
+
password: connection_config[:password],
|
|
105
|
+
port: connection_config.fetch(:port, 22), &block
|
|
106
|
+
)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def test_file_operations(sftp, test_path)
|
|
110
|
+
sftp.file.open(test_path, "w") { |file| file.puts("connection_check") }
|
|
111
|
+
sftp.remove!(test_path)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def control_message(message, status)
|
|
115
|
+
ControlMessage.new(
|
|
116
|
+
type: "full_refresh",
|
|
117
|
+
emitted_at: Time.now.to_i,
|
|
118
|
+
status: ConnectionStatusType[status],
|
|
119
|
+
meta: { detail: message }
|
|
120
|
+
).to_multiwoven_message
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"request_rate_limit": 600,
|
|
3
|
+
"request_rate_limit_unit": "minute",
|
|
4
|
+
"request_rate_concurrency": 10,
|
|
5
|
+
"streams": [
|
|
6
|
+
{
|
|
7
|
+
"name": "sftp",
|
|
8
|
+
"batch_support": true,
|
|
9
|
+
"batch_size": 10000,
|
|
10
|
+
"action": "create",
|
|
11
|
+
"schema_mode": ["schemaless"],
|
|
12
|
+
"json_schema": {},
|
|
13
|
+
"supported_sync_modes": ["full_refresh","incremental"]
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Sftp",
|
|
4
|
+
"title": "SFTP",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"connectorSubtype": "file",
|
|
7
|
+
"category": "File Storage",
|
|
8
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
9
|
+
"github_issue_label": "destination-sftp",
|
|
10
|
+
"icon": "icon.svg",
|
|
11
|
+
"license": "MIT",
|
|
12
|
+
"release_stage": "alpha",
|
|
13
|
+
"support_level": "community",
|
|
14
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/klaviyo",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "SFTP",
|
|
7
|
+
"required": ["host", "username", "password", "destination_path"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"host": {
|
|
10
|
+
"title": "Host",
|
|
11
|
+
"description": "Hostname of the SFTP server.",
|
|
12
|
+
"type": "string",
|
|
13
|
+
"order": 0
|
|
14
|
+
},
|
|
15
|
+
"port": {
|
|
16
|
+
"title": "Port",
|
|
17
|
+
"description": "Port of the SFTP server.",
|
|
18
|
+
"type": "integer",
|
|
19
|
+
"minimum": 0,
|
|
20
|
+
"maximum": 65536,
|
|
21
|
+
"default": 22,
|
|
22
|
+
"order": 1
|
|
23
|
+
},
|
|
24
|
+
"username": {
|
|
25
|
+
"title": "User Name",
|
|
26
|
+
"description": "Username to use to access the SFTP server.",
|
|
27
|
+
"type": "string",
|
|
28
|
+
"order": 2
|
|
29
|
+
},
|
|
30
|
+
"password": {
|
|
31
|
+
"title": "Password",
|
|
32
|
+
"description": "Password associated with the username.",
|
|
33
|
+
"type": "string",
|
|
34
|
+
"order": 3
|
|
35
|
+
},
|
|
36
|
+
"destination_path": {
|
|
37
|
+
"title": "Destination path",
|
|
38
|
+
"type": "string",
|
|
39
|
+
"description": "Path to the directory where files will be written.",
|
|
40
|
+
"order": 4
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="250" height="250" fill="none"><g clip-path="url(#a)"><path fill="#615EFF" d="M200.745 218.706a12.051 12.051 0 0 1-3.483 8.475 11.837 11.837 0 0 1-8.4 3.514H62.072c-3.15 0-6.173-1.261-8.403-3.507a12.053 12.053 0 0 1-3.49-8.472V31.485a12.08 12.08 0 0 1 .903-4.593 11.996 11.996 0 0 1 2.578-3.894 11.875 11.875 0 0 1 3.86-2.6 11.791 11.791 0 0 1 4.552-.912h81.371l57.302 58.96v140.26Z"/><path fill="#fff" d="M83.868 156.832c-.142-1.429-.75-2.539-1.822-3.33-1.074-.792-2.53-1.187-4.37-1.187-1.25 0-2.305.177-3.166.531-.861.343-1.522.821-1.981 1.435a3.457 3.457 0 0 0-.673 2.09c-.023.65.112 1.217.407 1.701.307.484.725.903 1.256 1.258.53.342 1.144.643 1.84.903a19.02 19.02 0 0 0 2.229.638l3.255.779c1.58.354 3.03.827 4.352 1.417 1.32.591 2.464 1.317 3.431 2.179a9.057 9.057 0 0 1 2.247 3.047c.542 1.169.82 2.509.831 4.021-.011 2.22-.578 4.145-1.698 5.775-1.109 1.617-2.712 2.875-4.811 3.773-2.088.885-4.606 1.328-7.554 1.328-2.925 0-5.472-.449-7.642-1.346-2.158-.898-3.845-2.226-5.06-3.986-1.202-1.771-1.833-3.962-1.892-6.572h7.412c.082 1.217.43 2.232 1.044 3.047.625.803 1.456 1.411 2.494 1.825 1.05.401 2.235.602 3.555.602 1.298 0 2.424-.189 3.38-.567.966-.378 1.715-.903 2.246-1.576.53-.673.796-1.447.796-2.321 0-.815-.242-1.5-.725-2.055-.472-.555-1.168-1.027-2.088-1.417-.908-.39-2.022-.744-3.343-1.063l-3.945-.992c-3.054-.744-5.466-1.907-7.235-3.489-1.77-1.583-2.648-3.714-2.636-6.395-.012-2.196.572-4.116 1.751-5.757 1.192-1.641 2.825-2.923 4.9-3.844 2.076-.921 4.435-1.382 7.076-1.382 2.69 0 5.036.461 7.04 1.382 2.018.921 3.586 2.203 4.706 3.844 1.12 1.641 1.699 3.543 1.734 5.704h-7.341Zm12.988 25.844v-36.278h23.988v6.324h-16.328v8.645h14.736v6.324h-14.736v14.985h-7.66Zm27.76-29.954v-6.324h29.754v6.324h-11.091v29.954h-7.571v-29.954h-11.092Zm34.654 29.954v-36.278h14.294c2.748 0 5.089.526 7.023 1.577 1.934 1.039 3.408 2.486 4.422 4.34 1.026 1.842 1.539 3.968 1.539 6.377s-.519 4.535-1.557 6.377c-1.037 1.842-2.541 3.277-4.51 4.304-1.958 1.028-4.329 1.541-7.112 1.541h-9.11v-6.146h7.872c1.474 0 2.689-.254 3.644-.762.967-.52 1.686-1.234 2.158-2.143.484-.921.725-1.978.725-3.171 0-1.205-.241-2.256-.725-3.153-.472-.91-1.191-1.612-2.158-2.108-.967-.508-2.194-.762-3.679-.762h-5.166v30.007h-7.66Z"/><path fill="#1A194D" d="m149.75 76.907 50.996 49.177V78.101h-45.173a11.33 11.33 0 0 1-5.643-1.433l-.18.24Z" opacity=".3"/><path fill="#C5C4FF" d="M200.746 78.35h-45.297a11.836 11.836 0 0 1-8.399-3.514 12.055 12.055 0 0 1-3.484-8.475v-47.2l57.18 59.189Z"/></g><defs><clipPath id="a"><path fill="#fff" d="M50 19h151v212H50z"/></clipPath></defs></svg>
|
|
@@ -24,6 +24,7 @@ module Multiwoven
|
|
|
24
24
|
)
|
|
25
25
|
LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
|
|
26
26
|
RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
|
|
27
|
+
SchemaMode = Types::String.enum("schema", "schemaless")
|
|
27
28
|
|
|
28
29
|
class ProtocolModel < Dry::Struct
|
|
29
30
|
extend Multiwoven::Integrations::Core::Utils
|
|
@@ -124,6 +125,7 @@ module Multiwoven
|
|
|
124
125
|
attribute? :request_rate_limit, Types::Integer
|
|
125
126
|
attribute? :request_rate_limit_unit, RequestRateLimitingUnit
|
|
126
127
|
attribute? :request_rate_concurrency, Types::Integer
|
|
128
|
+
attribute? :schema_mode, Types::Array.of(SchemaMode).optional.default(["schema"])
|
|
127
129
|
|
|
128
130
|
def rate_limit_unit_seconds
|
|
129
131
|
case request_rate_limit_unit
|
|
@@ -19,6 +19,9 @@ require "hubspot-api-client"
|
|
|
19
19
|
require "google/apis/sheets_v4"
|
|
20
20
|
require "stringio"
|
|
21
21
|
require "stripe"
|
|
22
|
+
require "net/sftp"
|
|
23
|
+
require "csv"
|
|
24
|
+
require "securerandom"
|
|
22
25
|
|
|
23
26
|
# Service
|
|
24
27
|
require_relative "integrations/config"
|
|
@@ -54,6 +57,7 @@ require_relative "integrations/destination/google_sheets/client"
|
|
|
54
57
|
require_relative "integrations/destination/airtable/client"
|
|
55
58
|
require_relative "integrations/destination/stripe/client"
|
|
56
59
|
require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
|
|
60
|
+
require_relative "integrations/destination/sftp/client"
|
|
57
61
|
|
|
58
62
|
module Multiwoven
|
|
59
63
|
module Integrations
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.42
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-03-
|
|
11
|
+
date: 2024-03-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -38,6 +38,20 @@ dependencies:
|
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: csv
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
41
55
|
- !ruby/object:Gem::Dependency
|
|
42
56
|
name: dry-schema
|
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -136,6 +150,20 @@ dependencies:
|
|
|
136
150
|
- - ">="
|
|
137
151
|
- !ruby/object:Gem::Version
|
|
138
152
|
version: '0'
|
|
153
|
+
- !ruby/object:Gem::Dependency
|
|
154
|
+
name: net-sftp
|
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
|
156
|
+
requirements:
|
|
157
|
+
- - ">="
|
|
158
|
+
- !ruby/object:Gem::Version
|
|
159
|
+
version: '0'
|
|
160
|
+
type: :runtime
|
|
161
|
+
prerelease: false
|
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
163
|
+
requirements:
|
|
164
|
+
- - ">="
|
|
165
|
+
- !ruby/object:Gem::Version
|
|
166
|
+
version: '0'
|
|
139
167
|
- !ruby/object:Gem::Dependency
|
|
140
168
|
name: pg
|
|
141
169
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -395,6 +423,11 @@ files:
|
|
|
395
423
|
- lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json
|
|
396
424
|
- lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json
|
|
397
425
|
- lib/multiwoven/integrations/destination/salesforce_crm/icon.svg
|
|
426
|
+
- lib/multiwoven/integrations/destination/sftp/client.rb
|
|
427
|
+
- lib/multiwoven/integrations/destination/sftp/config/catalog.json
|
|
428
|
+
- lib/multiwoven/integrations/destination/sftp/config/meta.json
|
|
429
|
+
- lib/multiwoven/integrations/destination/sftp/config/spec.json
|
|
430
|
+
- lib/multiwoven/integrations/destination/sftp/icon.svg
|
|
398
431
|
- lib/multiwoven/integrations/destination/slack/client.rb
|
|
399
432
|
- lib/multiwoven/integrations/destination/slack/config/catalog.json
|
|
400
433
|
- lib/multiwoven/integrations/destination/slack/config/meta.json
|