multiwoven-integrations 0.1.41 → 0.1.42
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/utils.rb +2 -1
- data/lib/multiwoven/integrations/destination/sftp/client.rb +124 -0
- data/lib/multiwoven/integrations/destination/sftp/config/catalog.json +16 -0
- data/lib/multiwoven/integrations/destination/sftp/config/meta.json +16 -0
- data/lib/multiwoven/integrations/destination/sftp/config/spec.json +44 -0
- data/lib/multiwoven/integrations/destination/sftp/icon.svg +1 -0
- data/lib/multiwoven/integrations/protocol/protocol.rb +2 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations.rb +4 -0
- metadata +35 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: eee51b7ba03e877f2765da694f9ca7b4ede8e44e07ca74931d808f79fb828c06
|
|
4
|
+
data.tar.gz: 21a23a656e09175222172eea24e562c95ccc97e7bebe92443cc60678d68050bf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 388d743e298e3296fbd9bd12e5d29037d72bf4912f93ae8ae8b9d7f5285dbb2d39f39f1500f6ee3b6214e186564287eec8c3fe14d66f55cc53861585e3929e33
|
|
7
|
+
data.tar.gz: e527726e1e69c57c07d7d6ab28194f2a84dfbbffb941f61e2b5fca927cecf486edcb7fa2cecbbcc8ccc1713c396e1723161e5ea55f0cc8d310ce2d888f6de845
|
|
@@ -95,7 +95,8 @@ module Multiwoven
|
|
|
95
95
|
request_rate_limit: stream_json["request_rate_limit"].to_i,
|
|
96
96
|
request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
|
|
97
97
|
request_rate_concurrency: stream_json["request_rate_concurrency"].to_i,
|
|
98
|
-
supported_sync_modes: stream_json["supported_sync_modes"]
|
|
98
|
+
supported_sync_modes: stream_json["supported_sync_modes"],
|
|
99
|
+
schema_mode: stream_json["schema_mode"]
|
|
99
100
|
)
|
|
100
101
|
end
|
|
101
102
|
end
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Destination
|
|
4
|
+
module Sftp
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < DestinationConnector
|
|
7
|
+
prepend Multiwoven::Integrations::Core::Fullrefresher
|
|
8
|
+
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
9
|
+
|
|
10
|
+
def check_connection(connection_config)
|
|
11
|
+
connection_config = connection_config.with_indifferent_access
|
|
12
|
+
with_sftp_client(connection_config) do |sftp|
|
|
13
|
+
stream = SecureRandom.uuid
|
|
14
|
+
test_path = "/path/to/test/#{stream}"
|
|
15
|
+
test_file_operations(sftp, test_path)
|
|
16
|
+
return success_status
|
|
17
|
+
end
|
|
18
|
+
rescue StandardError => e
|
|
19
|
+
handle_exception("SFTP:CHECK_CONNECTION:EXCEPTION", "error", e)
|
|
20
|
+
failure_status(e)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def discover(_connection_config = nil)
|
|
24
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
25
|
+
|
|
26
|
+
catalog = build_catalog(catalog_json)
|
|
27
|
+
|
|
28
|
+
catalog.to_multiwoven_message
|
|
29
|
+
rescue StandardError => e
|
|
30
|
+
handle_exception(
|
|
31
|
+
"SFTP:DISCOVER:EXCEPTION",
|
|
32
|
+
"error",
|
|
33
|
+
e
|
|
34
|
+
)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def write(sync_config, records, _action = "insert")
|
|
38
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
39
|
+
file_path = generate_file_path(sync_config)
|
|
40
|
+
csv_content = generate_csv_content(records)
|
|
41
|
+
write_success = 0
|
|
42
|
+
write_failure = 0
|
|
43
|
+
# 10000 records in single
|
|
44
|
+
with_sftp_client(connection_config) do |sftp|
|
|
45
|
+
sftp.file.open(file_path, "w") { |file| file.puts(csv_content) }
|
|
46
|
+
write_success += records.size
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
|
|
49
|
+
write_failure += records.size
|
|
50
|
+
end
|
|
51
|
+
tracking_message(write_success, write_failure)
|
|
52
|
+
rescue StandardError => e
|
|
53
|
+
handle_exception(
|
|
54
|
+
"SFTP:WRITE:EXCEPTION",
|
|
55
|
+
"error",
|
|
56
|
+
e
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def clear_all_records(sync_config)
|
|
61
|
+
connection_specification = sync_config.destination.connection_specification.with_indifferent_access
|
|
62
|
+
with_sftp_client(connection_specification) do |sftp|
|
|
63
|
+
files = sftp.dir.glob(connection_specification[:destination_path], "*")
|
|
64
|
+
|
|
65
|
+
files.each do |file|
|
|
66
|
+
sftp.remove!(File.join(connection_specification[:destination_path], file.name))
|
|
67
|
+
end
|
|
68
|
+
if sftp.dir.entries(connection_specification[:destination_path]).size == 2
|
|
69
|
+
control_message("Successfully cleared data.", "succeeded")
|
|
70
|
+
else
|
|
71
|
+
control_message("Failed to clear data.", "failed")
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
rescue StandardError => e
|
|
75
|
+
control_message(e.message, "failed")
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
private
|
|
79
|
+
|
|
80
|
+
def generate_file_path(sync_config)
|
|
81
|
+
timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
|
|
82
|
+
file_name = "#{sync_config.stream.name}_#{timestamp}.csv"
|
|
83
|
+
File.join(sync_config.destination.connection_specification[:destination_path], file_name)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def generate_csv_content(records)
|
|
87
|
+
CSV.generate do |csv|
|
|
88
|
+
headers = records.first.keys
|
|
89
|
+
csv << headers
|
|
90
|
+
records.each { |record| csv << record.values_at(*headers) }
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def tracking_message(success, failure)
|
|
95
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
96
|
+
success: success, failed: failure
|
|
97
|
+
).to_multiwoven_message
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def with_sftp_client(connection_config, &block)
|
|
101
|
+
Net::SFTP.start(
|
|
102
|
+
connection_config[:host],
|
|
103
|
+
connection_config[:username],
|
|
104
|
+
password: connection_config[:password],
|
|
105
|
+
port: connection_config.fetch(:port, 22), &block
|
|
106
|
+
)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def test_file_operations(sftp, test_path)
|
|
110
|
+
sftp.file.open(test_path, "w") { |file| file.puts("connection_check") }
|
|
111
|
+
sftp.remove!(test_path)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def control_message(message, status)
|
|
115
|
+
ControlMessage.new(
|
|
116
|
+
type: "full_refresh",
|
|
117
|
+
emitted_at: Time.now.to_i,
|
|
118
|
+
status: ConnectionStatusType[status],
|
|
119
|
+
meta: { detail: message }
|
|
120
|
+
).to_multiwoven_message
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"request_rate_limit": 600,
|
|
3
|
+
"request_rate_limit_unit": "minute",
|
|
4
|
+
"request_rate_concurrency": 10,
|
|
5
|
+
"streams": [
|
|
6
|
+
{
|
|
7
|
+
"name": "sftp",
|
|
8
|
+
"batch_support": true,
|
|
9
|
+
"batch_size": 10000,
|
|
10
|
+
"action": "create",
|
|
11
|
+
"schema_mode": ["schemaless"],
|
|
12
|
+
"json_schema": {},
|
|
13
|
+
"supported_sync_modes": ["full_refresh","incremental"]
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Sftp",
|
|
4
|
+
"title": "SFTP",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"connectorSubtype": "file",
|
|
7
|
+
"category": "File Storage",
|
|
8
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
9
|
+
"github_issue_label": "destination-sftp",
|
|
10
|
+
"icon": "icon.svg",
|
|
11
|
+
"license": "MIT",
|
|
12
|
+
"release_stage": "alpha",
|
|
13
|
+
"support_level": "community",
|
|
14
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/klaviyo",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "SFTP",
|
|
7
|
+
"required": ["host", "username", "password", "destination_path"],
|
|
8
|
+
"properties": {
|
|
9
|
+
"host": {
|
|
10
|
+
"title": "Host",
|
|
11
|
+
"description": "Hostname of the SFTP server.",
|
|
12
|
+
"type": "string",
|
|
13
|
+
"order": 0
|
|
14
|
+
},
|
|
15
|
+
"port": {
|
|
16
|
+
"title": "Port",
|
|
17
|
+
"description": "Port of the SFTP server.",
|
|
18
|
+
"type": "integer",
|
|
19
|
+
"minimum": 0,
|
|
20
|
+
"maximum": 65536,
|
|
21
|
+
"default": 22,
|
|
22
|
+
"order": 1
|
|
23
|
+
},
|
|
24
|
+
"username": {
|
|
25
|
+
"title": "User Name",
|
|
26
|
+
"description": "Username to use to access the SFTP server.",
|
|
27
|
+
"type": "string",
|
|
28
|
+
"order": 2
|
|
29
|
+
},
|
|
30
|
+
"password": {
|
|
31
|
+
"title": "Password",
|
|
32
|
+
"description": "Password associated with the username.",
|
|
33
|
+
"type": "string",
|
|
34
|
+
"order": 3
|
|
35
|
+
},
|
|
36
|
+
"destination_path": {
|
|
37
|
+
"title": "Destination path",
|
|
38
|
+
"type": "string",
|
|
39
|
+
"description": "Path to the directory where files will be written.",
|
|
40
|
+
"order": 4
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="250" height="250" fill="none"><g clip-path="url(#a)"><path fill="#615EFF" d="M200.745 218.706a12.051 12.051 0 0 1-3.483 8.475 11.837 11.837 0 0 1-8.4 3.514H62.072c-3.15 0-6.173-1.261-8.403-3.507a12.053 12.053 0 0 1-3.49-8.472V31.485a12.08 12.08 0 0 1 .903-4.593 11.996 11.996 0 0 1 2.578-3.894 11.875 11.875 0 0 1 3.86-2.6 11.791 11.791 0 0 1 4.552-.912h81.371l57.302 58.96v140.26Z"/><path fill="#fff" d="M83.868 156.832c-.142-1.429-.75-2.539-1.822-3.33-1.074-.792-2.53-1.187-4.37-1.187-1.25 0-2.305.177-3.166.531-.861.343-1.522.821-1.981 1.435a3.457 3.457 0 0 0-.673 2.09c-.023.65.112 1.217.407 1.701.307.484.725.903 1.256 1.258.53.342 1.144.643 1.84.903a19.02 19.02 0 0 0 2.229.638l3.255.779c1.58.354 3.03.827 4.352 1.417 1.32.591 2.464 1.317 3.431 2.179a9.057 9.057 0 0 1 2.247 3.047c.542 1.169.82 2.509.831 4.021-.011 2.22-.578 4.145-1.698 5.775-1.109 1.617-2.712 2.875-4.811 3.773-2.088.885-4.606 1.328-7.554 1.328-2.925 0-5.472-.449-7.642-1.346-2.158-.898-3.845-2.226-5.06-3.986-1.202-1.771-1.833-3.962-1.892-6.572h7.412c.082 1.217.43 2.232 1.044 3.047.625.803 1.456 1.411 2.494 1.825 1.05.401 2.235.602 3.555.602 1.298 0 2.424-.189 3.38-.567.966-.378 1.715-.903 2.246-1.576.53-.673.796-1.447.796-2.321 0-.815-.242-1.5-.725-2.055-.472-.555-1.168-1.027-2.088-1.417-.908-.39-2.022-.744-3.343-1.063l-3.945-.992c-3.054-.744-5.466-1.907-7.235-3.489-1.77-1.583-2.648-3.714-2.636-6.395-.012-2.196.572-4.116 1.751-5.757 1.192-1.641 2.825-2.923 4.9-3.844 2.076-.921 4.435-1.382 7.076-1.382 2.69 0 5.036.461 7.04 1.382 2.018.921 3.586 2.203 4.706 3.844 1.12 1.641 1.699 3.543 1.734 5.704h-7.341Zm12.988 25.844v-36.278h23.988v6.324h-16.328v8.645h14.736v6.324h-14.736v14.985h-7.66Zm27.76-29.954v-6.324h29.754v6.324h-11.091v29.954h-7.571v-29.954h-11.092Zm34.654 29.954v-36.278h14.294c2.748 0 5.089.526 7.023 1.577 1.934 1.039 3.408 2.486 4.422 4.34 1.026 1.842 1.539 3.968 1.539 6.377s-.519 4.535-1.557 6.377c-1.037 1.842-2.541 3.277-4.51 4.304-1.958 1.028-4.329 1.541-7.112 1.541h-9.11v-6.146h7.872c1.474 0 2.689-.254 3.644-.762.967-.52 1.686-1.234 2.158-2.143.484-.921.725-1.978.725-3.171 0-1.205-.241-2.256-.725-3.153-.472-.91-1.191-1.612-2.158-2.108-.967-.508-2.194-.762-3.679-.762h-5.166v30.007h-7.66Z"/><path fill="#1A194D" d="m149.75 76.907 50.996 49.177V78.101h-45.173a11.33 11.33 0 0 1-5.643-1.433l-.18.24Z" opacity=".3"/><path fill="#C5C4FF" d="M200.746 78.35h-45.297a11.836 11.836 0 0 1-8.399-3.514 12.055 12.055 0 0 1-3.484-8.475v-47.2l57.18 59.189Z"/></g><defs><clipPath id="a"><path fill="#fff" d="M50 19h151v212H50z"/></clipPath></defs></svg>
|
|
@@ -24,6 +24,7 @@ module Multiwoven
|
|
|
24
24
|
)
|
|
25
25
|
LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
|
|
26
26
|
RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
|
|
27
|
+
SchemaMode = Types::String.enum("schema", "schemaless")
|
|
27
28
|
|
|
28
29
|
class ProtocolModel < Dry::Struct
|
|
29
30
|
extend Multiwoven::Integrations::Core::Utils
|
|
@@ -124,6 +125,7 @@ module Multiwoven
|
|
|
124
125
|
attribute? :request_rate_limit, Types::Integer
|
|
125
126
|
attribute? :request_rate_limit_unit, RequestRateLimitingUnit
|
|
126
127
|
attribute? :request_rate_concurrency, Types::Integer
|
|
128
|
+
attribute? :schema_mode, Types::Array.of(SchemaMode).optional.default(["schema"])
|
|
127
129
|
|
|
128
130
|
def rate_limit_unit_seconds
|
|
129
131
|
case request_rate_limit_unit
|
|
@@ -19,6 +19,9 @@ require "hubspot-api-client"
|
|
|
19
19
|
require "google/apis/sheets_v4"
|
|
20
20
|
require "stringio"
|
|
21
21
|
require "stripe"
|
|
22
|
+
require "net/sftp"
|
|
23
|
+
require "csv"
|
|
24
|
+
require "securerandom"
|
|
22
25
|
|
|
23
26
|
# Service
|
|
24
27
|
require_relative "integrations/config"
|
|
@@ -54,6 +57,7 @@ require_relative "integrations/destination/google_sheets/client"
|
|
|
54
57
|
require_relative "integrations/destination/airtable/client"
|
|
55
58
|
require_relative "integrations/destination/stripe/client"
|
|
56
59
|
require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
|
|
60
|
+
require_relative "integrations/destination/sftp/client"
|
|
57
61
|
|
|
58
62
|
module Multiwoven
|
|
59
63
|
module Integrations
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.42
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-03-
|
|
11
|
+
date: 2024-03-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -38,6 +38,20 @@ dependencies:
|
|
|
38
38
|
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: csv
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :runtime
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
41
55
|
- !ruby/object:Gem::Dependency
|
|
42
56
|
name: dry-schema
|
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -136,6 +150,20 @@ dependencies:
|
|
|
136
150
|
- - ">="
|
|
137
151
|
- !ruby/object:Gem::Version
|
|
138
152
|
version: '0'
|
|
153
|
+
- !ruby/object:Gem::Dependency
|
|
154
|
+
name: net-sftp
|
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
|
156
|
+
requirements:
|
|
157
|
+
- - ">="
|
|
158
|
+
- !ruby/object:Gem::Version
|
|
159
|
+
version: '0'
|
|
160
|
+
type: :runtime
|
|
161
|
+
prerelease: false
|
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
163
|
+
requirements:
|
|
164
|
+
- - ">="
|
|
165
|
+
- !ruby/object:Gem::Version
|
|
166
|
+
version: '0'
|
|
139
167
|
- !ruby/object:Gem::Dependency
|
|
140
168
|
name: pg
|
|
141
169
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -395,6 +423,11 @@ files:
|
|
|
395
423
|
- lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json
|
|
396
424
|
- lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json
|
|
397
425
|
- lib/multiwoven/integrations/destination/salesforce_crm/icon.svg
|
|
426
|
+
- lib/multiwoven/integrations/destination/sftp/client.rb
|
|
427
|
+
- lib/multiwoven/integrations/destination/sftp/config/catalog.json
|
|
428
|
+
- lib/multiwoven/integrations/destination/sftp/config/meta.json
|
|
429
|
+
- lib/multiwoven/integrations/destination/sftp/config/spec.json
|
|
430
|
+
- lib/multiwoven/integrations/destination/sftp/icon.svg
|
|
398
431
|
- lib/multiwoven/integrations/destination/slack/client.rb
|
|
399
432
|
- lib/multiwoven/integrations/destination/slack/config/catalog.json
|
|
400
433
|
- lib/multiwoven/integrations/destination/slack/config/meta.json
|