multiwoven-integrations 0.1.42 → 0.1.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/query_builder.rb +27 -0
- data/lib/multiwoven/integrations/core/utils.rb +3 -3
- data/lib/multiwoven/integrations/destination/postgresql/client.rb +123 -0
- data/lib/multiwoven/integrations/destination/postgresql/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/postgresql/config/spec.json +68 -0
- data/lib/multiwoven/integrations/destination/postgresql/icon.svg +20 -0
- data/lib/multiwoven/integrations/destination/sftp/client.rb +20 -9
- data/lib/multiwoven/integrations/destination/sftp/config/catalog.json +2 -2
- data/lib/multiwoven/integrations/protocol/protocol.rb +1 -1
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb +6 -1
- data/lib/multiwoven/integrations.rb +2 -0
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6afdc30d90ef04bf1c1d0f0ae9bc9c0c72005b9e805f0ee6dca950d6f50d375f
|
|
4
|
+
data.tar.gz: 435ab301cc93008ed6e5009c1675eceba8ffafbf6dbe09c6629196f7c3a3a705
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: daa60409b147ab79fb35350a8abcf8756d063e6615b37512256b779fdfd86e91492ddd8a6b70835e4c51d066cafce9694f7e918787a061f0afe8d72949027a45
|
|
7
|
+
data.tar.gz: a5f15cebfb4c8c85c1c9f8061d19a334769bf4cc4eaff430652b81f3be7d76359c2b0c82bd16a67eab178ab8999150bc5c469319e96a85c01c29f1b7f84ec041
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations::Core
|
|
5
|
+
class QueryBuilder
|
|
6
|
+
def self.perform(action, table, record, primary_key = nil)
|
|
7
|
+
case action.downcase
|
|
8
|
+
when "insert"
|
|
9
|
+
columns = record.keys.join(", ")
|
|
10
|
+
values = record.values.map { |value| "'#{value}'" }.join(", ")
|
|
11
|
+
# TODO: support bulk insert
|
|
12
|
+
"INSERT INTO #{table} (#{columns}) VALUES (#{values});"
|
|
13
|
+
when "update"
|
|
14
|
+
# Ensure primary key is a string and exists within record for the WHERE clause
|
|
15
|
+
return "Primary key '#{primary_key}' not found in record." if record[primary_key].nil?
|
|
16
|
+
|
|
17
|
+
primary_key_value = record.delete(primary_key) # Remove and return the primary key value
|
|
18
|
+
set_clause = record.map { |key, value| "#{key} = '#{value}'" }.join(", ")
|
|
19
|
+
where_clause = "#{primary_key} = '#{primary_key_value}'"
|
|
20
|
+
"UPDATE #{table} SET #{set_clause} WHERE #{where_clause};"
|
|
21
|
+
else
|
|
22
|
+
"Invalid action specified."
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -79,7 +79,8 @@ module Multiwoven
|
|
|
79
79
|
streams: streams,
|
|
80
80
|
request_rate_limit: catalog_json["request_rate_limit"] || 60,
|
|
81
81
|
request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
|
|
82
|
-
request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10
|
|
82
|
+
request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10,
|
|
83
|
+
schema_mode: catalog_json["schema_mode"] || ["schema"]
|
|
83
84
|
)
|
|
84
85
|
end
|
|
85
86
|
|
|
@@ -95,8 +96,7 @@ module Multiwoven
|
|
|
95
96
|
request_rate_limit: stream_json["request_rate_limit"].to_i,
|
|
96
97
|
request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
|
|
97
98
|
request_rate_concurrency: stream_json["request_rate_concurrency"].to_i,
|
|
98
|
-
supported_sync_modes: stream_json["supported_sync_modes"]
|
|
99
|
-
schema_mode: stream_json["schema_mode"]
|
|
99
|
+
supported_sync_modes: stream_json["supported_sync_modes"]
|
|
100
100
|
)
|
|
101
101
|
end
|
|
102
102
|
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pg"
|
|
4
|
+
|
|
5
|
+
module Multiwoven::Integrations::Destination
|
|
6
|
+
module Postgresql
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < DestinationConnector
|
|
9
|
+
def check_connection(connection_config)
|
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
|
11
|
+
create_connection(connection_config)
|
|
12
|
+
ConnectionStatus.new(
|
|
13
|
+
status: ConnectionStatusType["succeeded"]
|
|
14
|
+
).to_multiwoven_message
|
|
15
|
+
rescue PG::Error => e
|
|
16
|
+
ConnectionStatus.new(
|
|
17
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
18
|
+
).to_multiwoven_message
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def discover(connection_config)
|
|
22
|
+
connection_config = connection_config.with_indifferent_access
|
|
23
|
+
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
24
|
+
FROM information_schema.columns
|
|
25
|
+
WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
|
|
26
|
+
ORDER BY table_name, ordinal_position;"
|
|
27
|
+
|
|
28
|
+
db = create_connection(connection_config)
|
|
29
|
+
records = db.exec(query) do |result|
|
|
30
|
+
result.map do |row|
|
|
31
|
+
row
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
35
|
+
catalog.to_multiwoven_message
|
|
36
|
+
rescue StandardError => e
|
|
37
|
+
handle_exception(
|
|
38
|
+
"POSTGRESQL:DISCOVER:EXCEPTION",
|
|
39
|
+
"error",
|
|
40
|
+
e
|
|
41
|
+
)
|
|
42
|
+
ensure
|
|
43
|
+
db&.close
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def write(sync_config, records, action = "insert")
|
|
47
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
48
|
+
table_name = sync_config.stream.name
|
|
49
|
+
db = create_connection(connection_config)
|
|
50
|
+
|
|
51
|
+
write_success = 0
|
|
52
|
+
write_failure = 0
|
|
53
|
+
|
|
54
|
+
records.each do |record|
|
|
55
|
+
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record)
|
|
56
|
+
begin
|
|
57
|
+
db.exec(query)
|
|
58
|
+
write_success += 1
|
|
59
|
+
rescue StandardError => e
|
|
60
|
+
handle_exception("POSTGRESQL:RECORD:WRITE:EXCEPTION", "error", e)
|
|
61
|
+
write_failure += 1
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
tracking_message(write_success, write_failure)
|
|
65
|
+
rescue StandardError => e
|
|
66
|
+
handle_exception(
|
|
67
|
+
"POSTGRESQL:WRITE:EXCEPTION",
|
|
68
|
+
"error",
|
|
69
|
+
e
|
|
70
|
+
)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def query(connection, query)
|
|
76
|
+
connection.exec(query) do |result|
|
|
77
|
+
result.map do |row|
|
|
78
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def create_connection(connection_config)
|
|
84
|
+
raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
|
|
85
|
+
|
|
86
|
+
PG.connect(
|
|
87
|
+
host: connection_config[:host],
|
|
88
|
+
dbname: connection_config[:database],
|
|
89
|
+
user: connection_config[:credentials][:username],
|
|
90
|
+
password: connection_config[:credentials][:password],
|
|
91
|
+
port: connection_config[:port]
|
|
92
|
+
)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def create_streams(records)
|
|
96
|
+
group_by_table(records).map do |r|
|
|
97
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def group_by_table(records)
|
|
102
|
+
records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
|
|
103
|
+
{
|
|
104
|
+
tablename: table_name,
|
|
105
|
+
columns: columns.map do |column|
|
|
106
|
+
{
|
|
107
|
+
column_name: column["column_name"],
|
|
108
|
+
type: column["data_type"],
|
|
109
|
+
optional: column["is_nullable"] == "YES"
|
|
110
|
+
}
|
|
111
|
+
end
|
|
112
|
+
}
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def tracking_message(success, failure)
|
|
117
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
118
|
+
success: success, failed: failure
|
|
119
|
+
).to_multiwoven_message
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Postgresql",
|
|
4
|
+
"title": "PostgreSQL",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-postgresql",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/postgresql",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Postgresql",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["host", "port", "database", "schema"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"credentials": {
|
|
11
|
+
"title": "",
|
|
12
|
+
"type": "object",
|
|
13
|
+
"required": ["auth_type", "username", "password"],
|
|
14
|
+
"properties": {
|
|
15
|
+
"auth_type": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"default": "username/password",
|
|
18
|
+
"order": 0,
|
|
19
|
+
"readOnly": true
|
|
20
|
+
},
|
|
21
|
+
"username": {
|
|
22
|
+
"description": "Username refers to your individual PostgreSQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
|
|
23
|
+
"examples": ["POSTGRESQL_USER"],
|
|
24
|
+
"type": "string",
|
|
25
|
+
"title": "Username",
|
|
26
|
+
"order": 1
|
|
27
|
+
},
|
|
28
|
+
"password": {
|
|
29
|
+
"description": "This field requires the password associated with the user account specified in the preceding section.",
|
|
30
|
+
"type": "string",
|
|
31
|
+
"multiwoven_secret": true,
|
|
32
|
+
"title": "Password",
|
|
33
|
+
"order": 2
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"order": 0
|
|
37
|
+
},
|
|
38
|
+
"host": {
|
|
39
|
+
"description": "The hostname or IP address of your PostgreSQL server.",
|
|
40
|
+
"examples": ["127.0.0.1"],
|
|
41
|
+
"type": "string",
|
|
42
|
+
"title": "Host",
|
|
43
|
+
"order": 1
|
|
44
|
+
},
|
|
45
|
+
"port": {
|
|
46
|
+
"description": "The port number for your PostgreSQL server, which defaults to 5432, may vary based on your configuration. ",
|
|
47
|
+
"examples": ["5432"],
|
|
48
|
+
"type": "string",
|
|
49
|
+
"title": "Port",
|
|
50
|
+
"order": 2
|
|
51
|
+
},
|
|
52
|
+
"database": {
|
|
53
|
+
"description": "The specific PostgreSQL database to connect to.",
|
|
54
|
+
"examples": ["POSTGRESQL_DB"],
|
|
55
|
+
"type": "string",
|
|
56
|
+
"title": "Database",
|
|
57
|
+
"order": 3
|
|
58
|
+
},
|
|
59
|
+
"schema": {
|
|
60
|
+
"description": "The schema within the PostgreSQL database.",
|
|
61
|
+
"examples": ["POSTGRESQL_SCHEMA"],
|
|
62
|
+
"type": "string",
|
|
63
|
+
"title": "Schema",
|
|
64
|
+
"order": 4
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
|
3
|
+
<svg width="432.071pt" height="445.383pt" viewBox="0 0 432.071 445.383" xml:space="preserve" xmlns="http://www.w3.org/2000/svg">
|
|
4
|
+
<g id="orginal" style="fill-rule:nonzero;clip-rule:nonzero;stroke:#000000;stroke-miterlimit:4;">
|
|
5
|
+
</g>
|
|
6
|
+
<g id="Layer_x0020_3" style="fill-rule:nonzero;clip-rule:nonzero;fill:none;stroke:#FFFFFF;stroke-width:12.4651;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;">
|
|
7
|
+
<path style="fill:#000000;stroke:#000000;stroke-width:37.3953;stroke-linecap:butt;stroke-linejoin:miter;" d="M323.205,324.227c2.833-23.601,1.984-27.062,19.563-23.239l4.463,0.392c13.517,0.615,31.199-2.174,41.587-7c22.362-10.376,35.622-27.7,13.572-23.148c-50.297,10.376-53.755-6.655-53.755-6.655c53.111-78.803,75.313-178.836,56.149-203.322 C352.514-5.534,262.036,26.049,260.522,26.869l-0.482,0.089c-9.938-2.062-21.06-3.294-33.554-3.496c-22.761-0.374-40.032,5.967-53.133,15.904c0,0-161.408-66.498-153.899,83.628c1.597,31.936,45.777,241.655,98.47,178.31 c19.259-23.163,37.871-42.748,37.871-42.748c9.242,6.14,20.307,9.272,31.912,8.147l0.897-0.765c-0.281,2.876-0.157,5.689,0.359,9.019c-13.572,15.167-9.584,17.83-36.723,23.416c-27.457,5.659-11.326,15.734-0.797,18.367c12.768,3.193,42.305,7.716,62.268-20.224 l-0.795,3.188c5.325,4.26,4.965,30.619,5.72,49.452c0.756,18.834,2.017,36.409,5.856,46.771c3.839,10.36,8.369,37.05,44.036,29.406c29.809-6.388,52.6-15.582,54.677-101.107"/>
|
|
8
|
+
<path style="fill:#336791;stroke:none;" d="M402.395,271.23c-50.302,10.376-53.76-6.655-53.76-6.655c53.111-78.808,75.313-178.843,56.153-203.326c-52.27-66.785-142.752-35.2-144.262-34.38l-0.486,0.087c-9.938-2.063-21.06-3.292-33.56-3.496c-22.761-0.373-40.026,5.967-53.127,15.902 c0,0-161.411-66.495-153.904,83.63c1.597,31.938,45.776,241.657,98.471,178.312c19.26-23.163,37.869-42.748,37.869-42.748c9.243,6.14,20.308,9.272,31.908,8.147l0.901-0.765c-0.28,2.876-0.152,5.689,0.361,9.019c-13.575,15.167-9.586,17.83-36.723,23.416 c-27.459,5.659-11.328,15.734-0.796,18.367c12.768,3.193,42.307,7.716,62.266-20.224l-0.796,3.188c5.319,4.26,9.054,27.711,8.428,48.969c-0.626,21.259-1.044,35.854,3.147,47.254c4.191,11.4,8.368,37.05,44.042,29.406c29.809-6.388,45.256-22.942,47.405-50.555 c1.525-19.631,4.976-16.729,5.194-34.28l2.768-8.309c3.192-26.611,0.507-35.196,18.872-31.203l4.463,0.392c13.517,0.615,31.208-2.174,41.591-7c22.358-10.376,35.618-27.7,13.573-23.148z"/>
|
|
9
|
+
<path d="M215.866,286.484c-1.385,49.516,0.348,99.377,5.193,111.495c4.848,12.118,15.223,35.688,50.9,28.045c29.806-6.39,40.651-18.756,45.357-46.051c3.466-20.082,10.148-75.854,11.005-87.281"/>
|
|
10
|
+
<path d="M173.104,38.256c0,0-161.521-66.016-154.012,84.109c1.597,31.938,45.779,241.664,98.473,178.316c19.256-23.166,36.671-41.335,36.671-41.335"/>
|
|
11
|
+
<path d="M260.349,26.207c-5.591,1.753,89.848-34.889,144.087,34.417c19.159,24.484-3.043,124.519-56.153,203.329"/>
|
|
12
|
+
<path style="stroke-linejoin:bevel;" d="M348.282,263.953c0,0,3.461,17.036,53.764,6.653c22.04-4.552,8.776,12.774-13.577,23.155c-18.345,8.514-59.474,10.696-60.146-1.069c-1.729-30.355,21.647-21.133,19.96-28.739c-1.525-6.85-11.979-13.573-18.894-30.338 c-6.037-14.633-82.796-126.849,21.287-110.183c3.813-0.789-27.146-99.002-124.553-100.599c-97.385-1.597-94.19,119.762-94.19,119.762"/>
|
|
13
|
+
<path d="M188.604,274.334c-13.577,15.166-9.584,17.829-36.723,23.417c-27.459,5.66-11.326,15.733-0.797,18.365c12.768,3.195,42.307,7.718,62.266-20.229c6.078-8.509-0.036-22.086-8.385-25.547c-4.034-1.671-9.428-3.765-16.361,3.994z"/>
|
|
14
|
+
<path d="M187.715,274.069c-1.368-8.917,2.93-19.528,7.536-31.942c6.922-18.626,22.893-37.255,10.117-96.339c-9.523-44.029-73.396-9.163-73.436-3.193c-0.039,5.968,2.889,30.26-1.067,58.548c-5.162,36.913,23.488,68.132,56.479,64.938"/>
|
|
15
|
+
<path style="fill:#FFFFFF;stroke-width:4.155;stroke-linecap:butt;stroke-linejoin:miter;" d="M172.517,141.7c-0.288,2.039,3.733,7.48,8.976,8.207c5.234,0.73,9.714-3.522,9.998-5.559c0.284-2.039-3.732-4.285-8.977-5.015c-5.237-0.731-9.719,0.333-9.996,2.367z"/>
|
|
16
|
+
<path style="fill:#FFFFFF;stroke-width:2.0775;stroke-linecap:butt;stroke-linejoin:miter;" d="M331.941,137.543c0.284,2.039-3.732,7.48-8.976,8.207c-5.238,0.73-9.718-3.522-10.005-5.559c-0.277-2.039,3.74-4.285,8.979-5.015c5.239-0.73,9.718,0.333,10.002,2.368z"/>
|
|
17
|
+
<path d="M350.676,123.432c0.863,15.994-3.445,26.888-3.988,43.914c-0.804,24.748,11.799,53.074-7.191,81.435"/>
|
|
18
|
+
<path style="stroke-width:3;" d="M0,60.232"/>
|
|
19
|
+
</g>
|
|
20
|
+
</svg>
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Multiwoven::Integrations::Destination
|
|
4
4
|
module Sftp
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
|
-
class Client < DestinationConnector
|
|
6
|
+
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
|
7
7
|
prepend Multiwoven::Integrations::Core::Fullrefresher
|
|
8
8
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
9
9
|
|
|
@@ -37,16 +37,21 @@ module Multiwoven::Integrations::Destination
|
|
|
37
37
|
def write(sync_config, records, _action = "insert")
|
|
38
38
|
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
39
39
|
file_path = generate_file_path(sync_config)
|
|
40
|
+
local_file_name = generate_local_file_name(sync_config)
|
|
40
41
|
csv_content = generate_csv_content(records)
|
|
41
42
|
write_success = 0
|
|
42
43
|
write_failure = 0
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
44
|
+
|
|
45
|
+
Tempfile.create([local_file_name, ".csv"]) do |tempfile|
|
|
46
|
+
tempfile.write(csv_content)
|
|
47
|
+
tempfile.close
|
|
48
|
+
with_sftp_client(connection_config) do |sftp|
|
|
49
|
+
sftp.upload!(tempfile.path, file_path)
|
|
50
|
+
write_success += records.size
|
|
51
|
+
rescue StandardError => e
|
|
52
|
+
handle_exception("SFTP:RECORD:WRITE:EXCEPTION", "error", e)
|
|
53
|
+
write_failure += records.size
|
|
54
|
+
end
|
|
50
55
|
end
|
|
51
56
|
tracking_message(write_success, write_failure)
|
|
52
57
|
rescue StandardError => e
|
|
@@ -78,9 +83,15 @@ module Multiwoven::Integrations::Destination
|
|
|
78
83
|
private
|
|
79
84
|
|
|
80
85
|
def generate_file_path(sync_config)
|
|
86
|
+
connection_specification = sync_config.destination.connection_specification.with_indifferent_access
|
|
81
87
|
timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
|
|
82
88
|
file_name = "#{sync_config.stream.name}_#{timestamp}.csv"
|
|
83
|
-
File.join(
|
|
89
|
+
File.join(connection_specification[:destination_path], file_name)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def generate_local_file_name(sync_config)
|
|
93
|
+
timestamp = Time.now.strftime("%Y%m%d-%H%M%S")
|
|
94
|
+
"#{sync_config.stream.name}_#{timestamp}"
|
|
84
95
|
end
|
|
85
96
|
|
|
86
97
|
def generate_csv_content(records)
|
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
"request_rate_limit": 600,
|
|
3
3
|
"request_rate_limit_unit": "minute",
|
|
4
4
|
"request_rate_concurrency": 10,
|
|
5
|
+
"schema_mode": ["schemaless"],
|
|
5
6
|
"streams": [
|
|
6
7
|
{
|
|
7
8
|
"name": "sftp",
|
|
8
9
|
"batch_support": true,
|
|
9
|
-
"batch_size":
|
|
10
|
+
"batch_size": 100000,
|
|
10
11
|
"action": "create",
|
|
11
|
-
"schema_mode": ["schemaless"],
|
|
12
12
|
"json_schema": {},
|
|
13
13
|
"supported_sync_modes": ["full_refresh","incremental"]
|
|
14
14
|
}
|
|
@@ -125,7 +125,6 @@ module Multiwoven
|
|
|
125
125
|
attribute? :request_rate_limit, Types::Integer
|
|
126
126
|
attribute? :request_rate_limit_unit, RequestRateLimitingUnit
|
|
127
127
|
attribute? :request_rate_concurrency, Types::Integer
|
|
128
|
-
attribute? :schema_mode, Types::Array.of(SchemaMode).optional.default(["schema"])
|
|
129
128
|
|
|
130
129
|
def rate_limit_unit_seconds
|
|
131
130
|
case request_rate_limit_unit
|
|
@@ -148,6 +147,7 @@ module Multiwoven
|
|
|
148
147
|
attribute? :request_rate_limit, Types::Integer.default(60)
|
|
149
148
|
attribute? :request_rate_limit_unit, RequestRateLimitingUnit
|
|
150
149
|
attribute? :request_rate_concurrency, Types::Integer.default(10)
|
|
150
|
+
attribute? :schema_mode, Types::Array.of(SchemaMode).optional.default(["schema"])
|
|
151
151
|
|
|
152
152
|
def to_multiwoven_message
|
|
153
153
|
MultiwovenMessage.new(
|
|
@@ -39,8 +39,13 @@ module Multiwoven
|
|
|
39
39
|
def read(sync_config)
|
|
40
40
|
connection_config = sync_config.source.connection_specification.with_indifferent_access
|
|
41
41
|
initialize_client(connection_config)
|
|
42
|
+
return [] if sync_config.offset&.> 2000
|
|
43
|
+
|
|
44
|
+
# TODO: Salesforce imposes a limit on the use of OFFSET in SOQL queries, where you cannot skip(offset) more than 2000 records.
|
|
45
|
+
# This limitation can hinder the retrieval of large datasets in a single query.
|
|
46
|
+
# To overcome this, we need a cursor-based pagination strategy instead of relying on OFFSET.
|
|
47
|
+
# query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
42
48
|
query = sync_config.model.query
|
|
43
|
-
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
44
49
|
exclude_keys = ["attributes"]
|
|
45
50
|
queried_data = @client.query(query)
|
|
46
51
|
results = queried_data.map do |record|
|
|
@@ -38,6 +38,7 @@ require_relative "integrations/core/base_connector"
|
|
|
38
38
|
require_relative "integrations/core/source_connector"
|
|
39
39
|
require_relative "integrations/core/destination_connector"
|
|
40
40
|
require_relative "integrations/core/http_client"
|
|
41
|
+
require_relative "integrations/core/query_builder"
|
|
41
42
|
|
|
42
43
|
# Source
|
|
43
44
|
require_relative "integrations/source/snowflake/client"
|
|
@@ -58,6 +59,7 @@ require_relative "integrations/destination/airtable/client"
|
|
|
58
59
|
require_relative "integrations/destination/stripe/client"
|
|
59
60
|
require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
|
|
60
61
|
require_relative "integrations/destination/sftp/client"
|
|
62
|
+
require_relative "integrations/destination/postgresql/client"
|
|
61
63
|
|
|
62
64
|
module Multiwoven
|
|
63
65
|
module Integrations
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.44
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
@@ -384,6 +384,7 @@ files:
|
|
|
384
384
|
- lib/multiwoven/integrations/core/destination_connector.rb
|
|
385
385
|
- lib/multiwoven/integrations/core/fullrefresher.rb
|
|
386
386
|
- lib/multiwoven/integrations/core/http_client.rb
|
|
387
|
+
- lib/multiwoven/integrations/core/query_builder.rb
|
|
387
388
|
- lib/multiwoven/integrations/core/rate_limiter.rb
|
|
388
389
|
- lib/multiwoven/integrations/core/source_connector.rb
|
|
389
390
|
- lib/multiwoven/integrations/core/utils.rb
|
|
@@ -413,6 +414,10 @@ files:
|
|
|
413
414
|
- lib/multiwoven/integrations/destination/klaviyo/config/meta.json
|
|
414
415
|
- lib/multiwoven/integrations/destination/klaviyo/config/spec.json
|
|
415
416
|
- lib/multiwoven/integrations/destination/klaviyo/icon.svg
|
|
417
|
+
- lib/multiwoven/integrations/destination/postgresql/client.rb
|
|
418
|
+
- lib/multiwoven/integrations/destination/postgresql/config/meta.json
|
|
419
|
+
- lib/multiwoven/integrations/destination/postgresql/config/spec.json
|
|
420
|
+
- lib/multiwoven/integrations/destination/postgresql/icon.svg
|
|
416
421
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb
|
|
417
422
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json
|
|
418
423
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json
|