multiwoven-integrations 0.1.43 → 0.1.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/query_builder.rb +27 -0
- data/lib/multiwoven/integrations/destination/postgresql/client.rb +123 -0
- data/lib/multiwoven/integrations/destination/postgresql/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/postgresql/config/spec.json +68 -0
- data/lib/multiwoven/integrations/destination/postgresql/icon.svg +20 -0
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations.rb +2 -0
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6afdc30d90ef04bf1c1d0f0ae9bc9c0c72005b9e805f0ee6dca950d6f50d375f
|
|
4
|
+
data.tar.gz: 435ab301cc93008ed6e5009c1675eceba8ffafbf6dbe09c6629196f7c3a3a705
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: daa60409b147ab79fb35350a8abcf8756d063e6615b37512256b779fdfd86e91492ddd8a6b70835e4c51d066cafce9694f7e918787a061f0afe8d72949027a45
|
|
7
|
+
data.tar.gz: a5f15cebfb4c8c85c1c9f8061d19a334769bf4cc4eaff430652b81f3be7d76359c2b0c82bd16a67eab178ab8999150bc5c469319e96a85c01c29f1b7f84ec041
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations::Core
|
|
5
|
+
class QueryBuilder
|
|
6
|
+
def self.perform(action, table, record, primary_key = nil)
|
|
7
|
+
case action.downcase
|
|
8
|
+
when "insert"
|
|
9
|
+
columns = record.keys.join(", ")
|
|
10
|
+
values = record.values.map { |value| "'#{value}'" }.join(", ")
|
|
11
|
+
# TODO: support bulk insert
|
|
12
|
+
"INSERT INTO #{table} (#{columns}) VALUES (#{values});"
|
|
13
|
+
when "update"
|
|
14
|
+
# Ensure primary key is a string and exists within record for the WHERE clause
|
|
15
|
+
return "Primary key '#{primary_key}' not found in record." if record[primary_key].nil?
|
|
16
|
+
|
|
17
|
+
primary_key_value = record.delete(primary_key) # Remove and return the primary key value
|
|
18
|
+
set_clause = record.map { |key, value| "#{key} = '#{value}'" }.join(", ")
|
|
19
|
+
where_clause = "#{primary_key} = '#{primary_key_value}'"
|
|
20
|
+
"UPDATE #{table} SET #{set_clause} WHERE #{where_clause};"
|
|
21
|
+
else
|
|
22
|
+
"Invalid action specified."
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pg"
|
|
4
|
+
|
|
5
|
+
module Multiwoven::Integrations::Destination
|
|
6
|
+
module Postgresql
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < DestinationConnector
|
|
9
|
+
def check_connection(connection_config)
|
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
|
11
|
+
create_connection(connection_config)
|
|
12
|
+
ConnectionStatus.new(
|
|
13
|
+
status: ConnectionStatusType["succeeded"]
|
|
14
|
+
).to_multiwoven_message
|
|
15
|
+
rescue PG::Error => e
|
|
16
|
+
ConnectionStatus.new(
|
|
17
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
18
|
+
).to_multiwoven_message
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def discover(connection_config)
|
|
22
|
+
connection_config = connection_config.with_indifferent_access
|
|
23
|
+
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
24
|
+
FROM information_schema.columns
|
|
25
|
+
WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
|
|
26
|
+
ORDER BY table_name, ordinal_position;"
|
|
27
|
+
|
|
28
|
+
db = create_connection(connection_config)
|
|
29
|
+
records = db.exec(query) do |result|
|
|
30
|
+
result.map do |row|
|
|
31
|
+
row
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
35
|
+
catalog.to_multiwoven_message
|
|
36
|
+
rescue StandardError => e
|
|
37
|
+
handle_exception(
|
|
38
|
+
"POSTGRESQL:DISCOVER:EXCEPTION",
|
|
39
|
+
"error",
|
|
40
|
+
e
|
|
41
|
+
)
|
|
42
|
+
ensure
|
|
43
|
+
db&.close
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def write(sync_config, records, action = "insert")
|
|
47
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
48
|
+
table_name = sync_config.stream.name
|
|
49
|
+
db = create_connection(connection_config)
|
|
50
|
+
|
|
51
|
+
write_success = 0
|
|
52
|
+
write_failure = 0
|
|
53
|
+
|
|
54
|
+
records.each do |record|
|
|
55
|
+
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record)
|
|
56
|
+
begin
|
|
57
|
+
db.exec(query)
|
|
58
|
+
write_success += 1
|
|
59
|
+
rescue StandardError => e
|
|
60
|
+
handle_exception("POSTGRESQL:RECORD:WRITE:EXCEPTION", "error", e)
|
|
61
|
+
write_failure += 1
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
tracking_message(write_success, write_failure)
|
|
65
|
+
rescue StandardError => e
|
|
66
|
+
handle_exception(
|
|
67
|
+
"POSTGRESQL:WRITE:EXCEPTION",
|
|
68
|
+
"error",
|
|
69
|
+
e
|
|
70
|
+
)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def query(connection, query)
|
|
76
|
+
connection.exec(query) do |result|
|
|
77
|
+
result.map do |row|
|
|
78
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def create_connection(connection_config)
|
|
84
|
+
raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
|
|
85
|
+
|
|
86
|
+
PG.connect(
|
|
87
|
+
host: connection_config[:host],
|
|
88
|
+
dbname: connection_config[:database],
|
|
89
|
+
user: connection_config[:credentials][:username],
|
|
90
|
+
password: connection_config[:credentials][:password],
|
|
91
|
+
port: connection_config[:port]
|
|
92
|
+
)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def create_streams(records)
|
|
96
|
+
group_by_table(records).map do |r|
|
|
97
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def group_by_table(records)
|
|
102
|
+
records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
|
|
103
|
+
{
|
|
104
|
+
tablename: table_name,
|
|
105
|
+
columns: columns.map do |column|
|
|
106
|
+
{
|
|
107
|
+
column_name: column["column_name"],
|
|
108
|
+
type: column["data_type"],
|
|
109
|
+
optional: column["is_nullable"] == "YES"
|
|
110
|
+
}
|
|
111
|
+
end
|
|
112
|
+
}
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def tracking_message(success, failure)
|
|
117
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
118
|
+
success: success, failed: failure
|
|
119
|
+
).to_multiwoven_message
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Postgresql",
|
|
4
|
+
"title": "PostgreSQL",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-postgresql",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/postgresql",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Postgresql",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["host", "port", "database", "schema"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"credentials": {
|
|
11
|
+
"title": "",
|
|
12
|
+
"type": "object",
|
|
13
|
+
"required": ["auth_type", "username", "password"],
|
|
14
|
+
"properties": {
|
|
15
|
+
"auth_type": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"default": "username/password",
|
|
18
|
+
"order": 0,
|
|
19
|
+
"readOnly": true
|
|
20
|
+
},
|
|
21
|
+
"username": {
|
|
22
|
+
"description": "Username refers to your individual PostgreSQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
|
|
23
|
+
"examples": ["POSTGRESQL_USER"],
|
|
24
|
+
"type": "string",
|
|
25
|
+
"title": "Username",
|
|
26
|
+
"order": 1
|
|
27
|
+
},
|
|
28
|
+
"password": {
|
|
29
|
+
"description": "This field requires the password associated with the user account specified in the preceding section.",
|
|
30
|
+
"type": "string",
|
|
31
|
+
"multiwoven_secret": true,
|
|
32
|
+
"title": "Password",
|
|
33
|
+
"order": 2
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"order": 0
|
|
37
|
+
},
|
|
38
|
+
"host": {
|
|
39
|
+
"description": "The hostname or IP address of your PostgreSQL server.",
|
|
40
|
+
"examples": ["127.0.0.1"],
|
|
41
|
+
"type": "string",
|
|
42
|
+
"title": "Host",
|
|
43
|
+
"order": 1
|
|
44
|
+
},
|
|
45
|
+
"port": {
|
|
46
|
+
"description": "The port number for your PostgreSQL server, which defaults to 5432, may vary based on your configuration. ",
|
|
47
|
+
"examples": ["5432"],
|
|
48
|
+
"type": "string",
|
|
49
|
+
"title": "Port",
|
|
50
|
+
"order": 2
|
|
51
|
+
},
|
|
52
|
+
"database": {
|
|
53
|
+
"description": "The specific PostgreSQL database to connect to.",
|
|
54
|
+
"examples": ["POSTGRESQL_DB"],
|
|
55
|
+
"type": "string",
|
|
56
|
+
"title": "Database",
|
|
57
|
+
"order": 3
|
|
58
|
+
},
|
|
59
|
+
"schema": {
|
|
60
|
+
"description": "The schema within the PostgreSQL database.",
|
|
61
|
+
"examples": ["POSTGRESQL_SCHEMA"],
|
|
62
|
+
"type": "string",
|
|
63
|
+
"title": "Schema",
|
|
64
|
+
"order": 4
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
|
3
|
+
<svg width="432.071pt" height="445.383pt" viewBox="0 0 432.071 445.383" xml:space="preserve" xmlns="http://www.w3.org/2000/svg">
|
|
4
|
+
<g id="orginal" style="fill-rule:nonzero;clip-rule:nonzero;stroke:#000000;stroke-miterlimit:4;">
|
|
5
|
+
</g>
|
|
6
|
+
<g id="Layer_x0020_3" style="fill-rule:nonzero;clip-rule:nonzero;fill:none;stroke:#FFFFFF;stroke-width:12.4651;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;">
|
|
7
|
+
<path style="fill:#000000;stroke:#000000;stroke-width:37.3953;stroke-linecap:butt;stroke-linejoin:miter;" d="M323.205,324.227c2.833-23.601,1.984-27.062,19.563-23.239l4.463,0.392c13.517,0.615,31.199-2.174,41.587-7c22.362-10.376,35.622-27.7,13.572-23.148c-50.297,10.376-53.755-6.655-53.755-6.655c53.111-78.803,75.313-178.836,56.149-203.322 C352.514-5.534,262.036,26.049,260.522,26.869l-0.482,0.089c-9.938-2.062-21.06-3.294-33.554-3.496c-22.761-0.374-40.032,5.967-53.133,15.904c0,0-161.408-66.498-153.899,83.628c1.597,31.936,45.777,241.655,98.47,178.31 c19.259-23.163,37.871-42.748,37.871-42.748c9.242,6.14,20.307,9.272,31.912,8.147l0.897-0.765c-0.281,2.876-0.157,5.689,0.359,9.019c-13.572,15.167-9.584,17.83-36.723,23.416c-27.457,5.659-11.326,15.734-0.797,18.367c12.768,3.193,42.305,7.716,62.268-20.224 l-0.795,3.188c5.325,4.26,4.965,30.619,5.72,49.452c0.756,18.834,2.017,36.409,5.856,46.771c3.839,10.36,8.369,37.05,44.036,29.406c29.809-6.388,52.6-15.582,54.677-101.107"/>
|
|
8
|
+
<path style="fill:#336791;stroke:none;" d="M402.395,271.23c-50.302,10.376-53.76-6.655-53.76-6.655c53.111-78.808,75.313-178.843,56.153-203.326c-52.27-66.785-142.752-35.2-144.262-34.38l-0.486,0.087c-9.938-2.063-21.06-3.292-33.56-3.496c-22.761-0.373-40.026,5.967-53.127,15.902 c0,0-161.411-66.495-153.904,83.63c1.597,31.938,45.776,241.657,98.471,178.312c19.26-23.163,37.869-42.748,37.869-42.748c9.243,6.14,20.308,9.272,31.908,8.147l0.901-0.765c-0.28,2.876-0.152,5.689,0.361,9.019c-13.575,15.167-9.586,17.83-36.723,23.416 c-27.459,5.659-11.328,15.734-0.796,18.367c12.768,3.193,42.307,7.716,62.266-20.224l-0.796,3.188c5.319,4.26,9.054,27.711,8.428,48.969c-0.626,21.259-1.044,35.854,3.147,47.254c4.191,11.4,8.368,37.05,44.042,29.406c29.809-6.388,45.256-22.942,47.405-50.555 c1.525-19.631,4.976-16.729,5.194-34.28l2.768-8.309c3.192-26.611,0.507-35.196,18.872-31.203l4.463,0.392c13.517,0.615,31.208-2.174,41.591-7c22.358-10.376,35.618-27.7,13.573-23.148z"/>
|
|
9
|
+
<path d="M215.866,286.484c-1.385,49.516,0.348,99.377,5.193,111.495c4.848,12.118,15.223,35.688,50.9,28.045c29.806-6.39,40.651-18.756,45.357-46.051c3.466-20.082,10.148-75.854,11.005-87.281"/>
|
|
10
|
+
<path d="M173.104,38.256c0,0-161.521-66.016-154.012,84.109c1.597,31.938,45.779,241.664,98.473,178.316c19.256-23.166,36.671-41.335,36.671-41.335"/>
|
|
11
|
+
<path d="M260.349,26.207c-5.591,1.753,89.848-34.889,144.087,34.417c19.159,24.484-3.043,124.519-56.153,203.329"/>
|
|
12
|
+
<path style="stroke-linejoin:bevel;" d="M348.282,263.953c0,0,3.461,17.036,53.764,6.653c22.04-4.552,8.776,12.774-13.577,23.155c-18.345,8.514-59.474,10.696-60.146-1.069c-1.729-30.355,21.647-21.133,19.96-28.739c-1.525-6.85-11.979-13.573-18.894-30.338 c-6.037-14.633-82.796-126.849,21.287-110.183c3.813-0.789-27.146-99.002-124.553-100.599c-97.385-1.597-94.19,119.762-94.19,119.762"/>
|
|
13
|
+
<path d="M188.604,274.334c-13.577,15.166-9.584,17.829-36.723,23.417c-27.459,5.66-11.326,15.733-0.797,18.365c12.768,3.195,42.307,7.718,62.266-20.229c6.078-8.509-0.036-22.086-8.385-25.547c-4.034-1.671-9.428-3.765-16.361,3.994z"/>
|
|
14
|
+
<path d="M187.715,274.069c-1.368-8.917,2.93-19.528,7.536-31.942c6.922-18.626,22.893-37.255,10.117-96.339c-9.523-44.029-73.396-9.163-73.436-3.193c-0.039,5.968,2.889,30.26-1.067,58.548c-5.162,36.913,23.488,68.132,56.479,64.938"/>
|
|
15
|
+
<path style="fill:#FFFFFF;stroke-width:4.155;stroke-linecap:butt;stroke-linejoin:miter;" d="M172.517,141.7c-0.288,2.039,3.733,7.48,8.976,8.207c5.234,0.73,9.714-3.522,9.998-5.559c0.284-2.039-3.732-4.285-8.977-5.015c-5.237-0.731-9.719,0.333-9.996,2.367z"/>
|
|
16
|
+
<path style="fill:#FFFFFF;stroke-width:2.0775;stroke-linecap:butt;stroke-linejoin:miter;" d="M331.941,137.543c0.284,2.039-3.732,7.48-8.976,8.207c-5.238,0.73-9.718-3.522-10.005-5.559c-0.277-2.039,3.74-4.285,8.979-5.015c5.239-0.73,9.718,0.333,10.002,2.368z"/>
|
|
17
|
+
<path d="M350.676,123.432c0.863,15.994-3.445,26.888-3.988,43.914c-0.804,24.748,11.799,53.074-7.191,81.435"/>
|
|
18
|
+
<path style="stroke-width:3;" d="M0,60.232"/>
|
|
19
|
+
</g>
|
|
20
|
+
</svg>
|
|
@@ -38,6 +38,7 @@ require_relative "integrations/core/base_connector"
|
|
|
38
38
|
require_relative "integrations/core/source_connector"
|
|
39
39
|
require_relative "integrations/core/destination_connector"
|
|
40
40
|
require_relative "integrations/core/http_client"
|
|
41
|
+
require_relative "integrations/core/query_builder"
|
|
41
42
|
|
|
42
43
|
# Source
|
|
43
44
|
require_relative "integrations/source/snowflake/client"
|
|
@@ -58,6 +59,7 @@ require_relative "integrations/destination/airtable/client"
|
|
|
58
59
|
require_relative "integrations/destination/stripe/client"
|
|
59
60
|
require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
|
|
60
61
|
require_relative "integrations/destination/sftp/client"
|
|
62
|
+
require_relative "integrations/destination/postgresql/client"
|
|
61
63
|
|
|
62
64
|
module Multiwoven
|
|
63
65
|
module Integrations
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.44
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
@@ -384,6 +384,7 @@ files:
|
|
|
384
384
|
- lib/multiwoven/integrations/core/destination_connector.rb
|
|
385
385
|
- lib/multiwoven/integrations/core/fullrefresher.rb
|
|
386
386
|
- lib/multiwoven/integrations/core/http_client.rb
|
|
387
|
+
- lib/multiwoven/integrations/core/query_builder.rb
|
|
387
388
|
- lib/multiwoven/integrations/core/rate_limiter.rb
|
|
388
389
|
- lib/multiwoven/integrations/core/source_connector.rb
|
|
389
390
|
- lib/multiwoven/integrations/core/utils.rb
|
|
@@ -413,6 +414,10 @@ files:
|
|
|
413
414
|
- lib/multiwoven/integrations/destination/klaviyo/config/meta.json
|
|
414
415
|
- lib/multiwoven/integrations/destination/klaviyo/config/spec.json
|
|
415
416
|
- lib/multiwoven/integrations/destination/klaviyo/icon.svg
|
|
417
|
+
- lib/multiwoven/integrations/destination/postgresql/client.rb
|
|
418
|
+
- lib/multiwoven/integrations/destination/postgresql/config/meta.json
|
|
419
|
+
- lib/multiwoven/integrations/destination/postgresql/config/spec.json
|
|
420
|
+
- lib/multiwoven/integrations/destination/postgresql/icon.svg
|
|
416
421
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb
|
|
417
422
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json
|
|
418
423
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json
|