multiwoven-integrations 0.1.62 → 0.1.64
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/airtable/config/spec.json +1 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json +1 -0
- data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json +2 -1
- data/lib/multiwoven/integrations/destination/hubspot/config/spec.json +1 -0
- data/lib/multiwoven/integrations/destination/klaviyo/config/spec.json +2 -0
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/spec.json +3 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json +3 -0
- data/lib/multiwoven/integrations/destination/sftp/client.rb +48 -11
- data/lib/multiwoven/integrations/destination/sftp/config/spec.json +24 -1
- data/lib/multiwoven/integrations/destination/slack/config/spec.json +1 -0
- data/lib/multiwoven/integrations/destination/stripe/config/spec.json +1 -0
- data/lib/multiwoven/integrations/protocol/protocol.rb +2 -0
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations/source/bigquery/config/spec.json +2 -1
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json +3 -0
- data/lib/multiwoven/integrations.rb +1 -0
- data/multiwoven-integrations.gemspec +1 -0
- metadata +16 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a04192999a6c02ae4dd3cf752c22f602d9ed196e3a293f67c53727b89ec0ce3a
|
|
4
|
+
data.tar.gz: 955cab0486157d96c00ba540de776dd9d32687366ddd260bfa4e9a81c95e017d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 30a2e1bbd87278e54f40ce5c8d07630d4b4e4b3a1b7f077c5f1bfdf9e00d37dd402bbe3fbcfa477cf7fd331929a9f2e41d8506e7ac9388e3403ae73e2b474be3
|
|
7
|
+
data.tar.gz: 77840062ae81f7efa968919d7377a0e324c6b7720ffe3f237996a55c850e8f7dc68f2a241086780ea4d22d66a2b3c3a5d57e3045aa6c921a909d6e2a36584acc
|
|
@@ -9,11 +9,13 @@
|
|
|
9
9
|
"properties": {
|
|
10
10
|
"public_api_key": {
|
|
11
11
|
"type": "string",
|
|
12
|
+
"multiwoven_secret": true,
|
|
12
13
|
"title": "Public API Key",
|
|
13
14
|
"order": 0
|
|
14
15
|
},
|
|
15
16
|
"private_api_key": {
|
|
16
17
|
"type": "string",
|
|
18
|
+
"multiwoven_secret": true,
|
|
17
19
|
"title": "Private API Key",
|
|
18
20
|
"order": 1
|
|
19
21
|
}
|
data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/spec.json
CHANGED
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
},
|
|
22
22
|
"password": {
|
|
23
23
|
"type": "string",
|
|
24
|
+
"multiwoven_secret": true,
|
|
24
25
|
"title": "Password",
|
|
25
26
|
"order": 1
|
|
26
27
|
},
|
|
@@ -36,11 +37,13 @@
|
|
|
36
37
|
},
|
|
37
38
|
"client_secret": {
|
|
38
39
|
"type": "string",
|
|
40
|
+
"multiwoven_secret": true,
|
|
39
41
|
"title": "Client Secret",
|
|
40
42
|
"order": 4
|
|
41
43
|
},
|
|
42
44
|
"security_token": {
|
|
43
45
|
"type": "string",
|
|
46
|
+
"multiwoven_secret": true,
|
|
44
47
|
"title": "Security Token",
|
|
45
48
|
"order": 5
|
|
46
49
|
}
|
|
@@ -15,11 +15,13 @@
|
|
|
15
15
|
"properties": {
|
|
16
16
|
"access_token": {
|
|
17
17
|
"type": "string",
|
|
18
|
+
"multiwoven_secret": true,
|
|
18
19
|
"title": "Access Token",
|
|
19
20
|
"order": 0
|
|
20
21
|
},
|
|
21
22
|
"refresh_token": {
|
|
22
23
|
"type": "string",
|
|
24
|
+
"multiwoven_secret": true,
|
|
23
25
|
"title": "Refresh Token",
|
|
24
26
|
"order": 1
|
|
25
27
|
},
|
|
@@ -35,6 +37,7 @@
|
|
|
35
37
|
},
|
|
36
38
|
"client_secret": {
|
|
37
39
|
"type": "string",
|
|
40
|
+
"multiwoven_secret": true,
|
|
38
41
|
"title": "Client Secret",
|
|
39
42
|
"order": 4
|
|
40
43
|
}
|
|
@@ -39,27 +39,58 @@ module Multiwoven::Integrations::Destination
|
|
|
39
39
|
file_path = generate_file_path(sync_config)
|
|
40
40
|
local_file_name = generate_local_file_name(sync_config)
|
|
41
41
|
csv_content = generate_csv_content(records)
|
|
42
|
+
records_size = records.size
|
|
42
43
|
write_success = 0
|
|
43
|
-
write_failure = 0
|
|
44
44
|
|
|
45
|
+
case connection_config[:format][:compression_type]
|
|
46
|
+
when CompressionType.enum("zip")
|
|
47
|
+
write_success = write_compressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
|
|
48
|
+
when CompressionType.enum("un_compressed")
|
|
49
|
+
write_success = write_uncompressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
|
|
50
|
+
else
|
|
51
|
+
raise ArgumentError, "Unsupported compression type: #{connection_config[:format][:compression_type]}"
|
|
52
|
+
end
|
|
53
|
+
write_failure = records.size - write_success
|
|
54
|
+
tracking_message(write_success, write_failure)
|
|
55
|
+
rescue StandardError => e
|
|
56
|
+
handle_exception(
|
|
57
|
+
"SFTP:WRITE:EXCEPTION",
|
|
58
|
+
"error",
|
|
59
|
+
e
|
|
60
|
+
)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def write_compressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
|
|
64
|
+
write_success = 0
|
|
65
|
+
Tempfile.create([local_file_name, ".zip"]) do |tempfile|
|
|
66
|
+
Zip::File.open(tempfile.path, Zip::File::CREATE) do |zipfile|
|
|
67
|
+
zipfile.get_output_stream("#{local_file_name}.csv") { |f| f.write(csv_content) }
|
|
68
|
+
end
|
|
69
|
+
with_sftp_client(connection_config) do |sftp|
|
|
70
|
+
sftp.upload!(tempfile.path, file_path)
|
|
71
|
+
write_success = records_size
|
|
72
|
+
rescue StandardError => e
|
|
73
|
+
handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
|
|
74
|
+
write_success = 0
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
write_success
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def write_uncompressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
|
|
81
|
+
write_success = 0
|
|
45
82
|
Tempfile.create([local_file_name, ".csv"]) do |tempfile|
|
|
46
83
|
tempfile.write(csv_content)
|
|
47
84
|
tempfile.close
|
|
48
85
|
with_sftp_client(connection_config) do |sftp|
|
|
49
86
|
sftp.upload!(tempfile.path, file_path)
|
|
50
|
-
write_success
|
|
87
|
+
write_success = records_size
|
|
51
88
|
rescue StandardError => e
|
|
52
89
|
handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
|
|
53
|
-
|
|
90
|
+
write_success = 0
|
|
54
91
|
end
|
|
55
92
|
end
|
|
56
|
-
|
|
57
|
-
rescue StandardError => e
|
|
58
|
-
handle_exception(
|
|
59
|
-
"SFTP:WRITE:EXCEPTION",
|
|
60
|
-
"error",
|
|
61
|
-
e
|
|
62
|
-
)
|
|
93
|
+
write_success
|
|
63
94
|
end
|
|
64
95
|
|
|
65
96
|
def clear_all_records(sync_config)
|
|
@@ -82,7 +113,13 @@ module Multiwoven::Integrations::Destination
|
|
|
82
113
|
def generate_file_path(sync_config)
|
|
83
114
|
connection_specification = sync_config.destination.connection_specification.with_indifferent_access
|
|
84
115
|
timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
|
|
85
|
-
|
|
116
|
+
format = connection_specification[:format]
|
|
117
|
+
extension = if format[:compression_type] == "un_compressed"
|
|
118
|
+
format[:format_type]
|
|
119
|
+
else
|
|
120
|
+
format[:compression_type]
|
|
121
|
+
end
|
|
122
|
+
file_name = "#{connection_specification[:file_name]}_#{timestamp}.#{extension}"
|
|
86
123
|
File.join(connection_specification[:destination_path], file_name)
|
|
87
124
|
end
|
|
88
125
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
"connection_specification": {
|
|
5
5
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
6
|
"title": "SFTP",
|
|
7
|
-
"required": ["host", "username", "password", "destination_path"],
|
|
7
|
+
"required": ["host", "username", "password", "destination_path", "format" ],
|
|
8
8
|
"properties": {
|
|
9
9
|
"host": {
|
|
10
10
|
"title": "Host",
|
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
"title": "Password",
|
|
32
32
|
"description": "Password associated with the username.",
|
|
33
33
|
"type": "string",
|
|
34
|
+
"multiwoven_secret": true,
|
|
34
35
|
"order": 3
|
|
35
36
|
},
|
|
36
37
|
"destination_path": {
|
|
@@ -44,6 +45,28 @@
|
|
|
44
45
|
"type": "string",
|
|
45
46
|
"description": "Name of the file to be written.",
|
|
46
47
|
"order": 5
|
|
48
|
+
},
|
|
49
|
+
"format": {
|
|
50
|
+
"title": "Output Format",
|
|
51
|
+
"type": "object",
|
|
52
|
+
"description": "Format of the data output.",
|
|
53
|
+
"order": 6,
|
|
54
|
+
"required": ["format_type"],
|
|
55
|
+
"properties": {
|
|
56
|
+
"format_type": {
|
|
57
|
+
"title": "File Format Type",
|
|
58
|
+
"type": "string",
|
|
59
|
+
"enum": ["csv"],
|
|
60
|
+
"default": "csv"
|
|
61
|
+
},
|
|
62
|
+
"compression_type": {
|
|
63
|
+
"title": "Compression Type",
|
|
64
|
+
"description": "Whether the output files should be compressed.",
|
|
65
|
+
"type": "string",
|
|
66
|
+
"enum": ["un_compressed", "zip"],
|
|
67
|
+
"default": "un_compressed"
|
|
68
|
+
}
|
|
69
|
+
}
|
|
47
70
|
}
|
|
48
71
|
}
|
|
49
72
|
}
|
|
@@ -26,6 +26,8 @@ module Multiwoven
|
|
|
26
26
|
LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
|
|
27
27
|
RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
|
|
28
28
|
SchemaMode = Types::String.enum("schema", "schemaless")
|
|
29
|
+
FileFormatType = Types::String.enum("csv")
|
|
30
|
+
CompressionType = Types::String.enum("un_compressed", "zip")
|
|
29
31
|
|
|
30
32
|
class ProtocolModel < Dry::Struct
|
|
31
33
|
extend Multiwoven::Integrations::Core::Utils
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
},
|
|
23
23
|
"password": {
|
|
24
24
|
"type": "string",
|
|
25
|
+
"multiwoven_secret": true,
|
|
25
26
|
"title": "Password",
|
|
26
27
|
"order": 1
|
|
27
28
|
},
|
|
@@ -37,11 +38,13 @@
|
|
|
37
38
|
},
|
|
38
39
|
"client_secret": {
|
|
39
40
|
"type": "string",
|
|
41
|
+
"multiwoven_secret": true,
|
|
40
42
|
"title": "Client Secret",
|
|
41
43
|
"order": 4
|
|
42
44
|
},
|
|
43
45
|
"security_token": {
|
|
44
46
|
"type": "string",
|
|
47
|
+
"multiwoven_secret": true,
|
|
45
48
|
"title": "Security Token",
|
|
46
49
|
"order": 5
|
|
47
50
|
}
|
|
@@ -49,6 +49,7 @@ Gem::Specification.new do |spec|
|
|
|
49
49
|
spec.add_runtime_dependency "restforce"
|
|
50
50
|
spec.add_runtime_dependency "ruby-limiter"
|
|
51
51
|
spec.add_runtime_dependency "ruby-odbc"
|
|
52
|
+
spec.add_runtime_dependency "rubyzip"
|
|
52
53
|
spec.add_runtime_dependency "sequel"
|
|
53
54
|
spec.add_runtime_dependency "slack-ruby-client"
|
|
54
55
|
spec.add_runtime_dependency "stripe"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.64
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-05-03 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -234,6 +234,20 @@ dependencies:
|
|
|
234
234
|
- - ">="
|
|
235
235
|
- !ruby/object:Gem::Version
|
|
236
236
|
version: '0'
|
|
237
|
+
- !ruby/object:Gem::Dependency
|
|
238
|
+
name: rubyzip
|
|
239
|
+
requirement: !ruby/object:Gem::Requirement
|
|
240
|
+
requirements:
|
|
241
|
+
- - ">="
|
|
242
|
+
- !ruby/object:Gem::Version
|
|
243
|
+
version: '0'
|
|
244
|
+
type: :runtime
|
|
245
|
+
prerelease: false
|
|
246
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
247
|
+
requirements:
|
|
248
|
+
- - ">="
|
|
249
|
+
- !ruby/object:Gem::Version
|
|
250
|
+
version: '0'
|
|
237
251
|
- !ruby/object:Gem::Dependency
|
|
238
252
|
name: sequel
|
|
239
253
|
requirement: !ruby/object:Gem::Requirement
|