multiwoven-integrations 0.1.43 → 0.1.44
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/query_builder.rb +27 -0
- data/lib/multiwoven/integrations/destination/postgresql/client.rb +123 -0
- data/lib/multiwoven/integrations/destination/postgresql/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/postgresql/config/spec.json +68 -0
- data/lib/multiwoven/integrations/destination/postgresql/icon.svg +20 -0
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations.rb +2 -0
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6afdc30d90ef04bf1c1d0f0ae9bc9c0c72005b9e805f0ee6dca950d6f50d375f
|
|
4
|
+
data.tar.gz: 435ab301cc93008ed6e5009c1675eceba8ffafbf6dbe09c6629196f7c3a3a705
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: daa60409b147ab79fb35350a8abcf8756d063e6615b37512256b779fdfd86e91492ddd8a6b70835e4c51d066cafce9694f7e918787a061f0afe8d72949027a45
|
|
7
|
+
data.tar.gz: a5f15cebfb4c8c85c1c9f8061d19a334769bf4cc4eaff430652b81f3be7d76359c2b0c82bd16a67eab178ab8999150bc5c469319e96a85c01c29f1b7f84ec041
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations::Core
|
|
5
|
+
class QueryBuilder
|
|
6
|
+
def self.perform(action, table, record, primary_key = nil)
|
|
7
|
+
case action.downcase
|
|
8
|
+
when "insert"
|
|
9
|
+
columns = record.keys.join(", ")
|
|
10
|
+
values = record.values.map { |value| "'#{value}'" }.join(", ")
|
|
11
|
+
# TODO: support bulk insert
|
|
12
|
+
"INSERT INTO #{table} (#{columns}) VALUES (#{values});"
|
|
13
|
+
when "update"
|
|
14
|
+
# Ensure primary key is a string and exists within record for the WHERE clause
|
|
15
|
+
return "Primary key '#{primary_key}' not found in record." if record[primary_key].nil?
|
|
16
|
+
|
|
17
|
+
primary_key_value = record.delete(primary_key) # Remove and return the primary key value
|
|
18
|
+
set_clause = record.map { |key, value| "#{key} = '#{value}'" }.join(", ")
|
|
19
|
+
where_clause = "#{primary_key} = '#{primary_key_value}'"
|
|
20
|
+
"UPDATE #{table} SET #{set_clause} WHERE #{where_clause};"
|
|
21
|
+
else
|
|
22
|
+
"Invalid action specified."
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "pg"
|
|
4
|
+
|
|
5
|
+
module Multiwoven::Integrations::Destination
|
|
6
|
+
module Postgresql
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < DestinationConnector
|
|
9
|
+
def check_connection(connection_config)
|
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
|
11
|
+
create_connection(connection_config)
|
|
12
|
+
ConnectionStatus.new(
|
|
13
|
+
status: ConnectionStatusType["succeeded"]
|
|
14
|
+
).to_multiwoven_message
|
|
15
|
+
rescue PG::Error => e
|
|
16
|
+
ConnectionStatus.new(
|
|
17
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
18
|
+
).to_multiwoven_message
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def discover(connection_config)
|
|
22
|
+
connection_config = connection_config.with_indifferent_access
|
|
23
|
+
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
24
|
+
FROM information_schema.columns
|
|
25
|
+
WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
|
|
26
|
+
ORDER BY table_name, ordinal_position;"
|
|
27
|
+
|
|
28
|
+
db = create_connection(connection_config)
|
|
29
|
+
records = db.exec(query) do |result|
|
|
30
|
+
result.map do |row|
|
|
31
|
+
row
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
35
|
+
catalog.to_multiwoven_message
|
|
36
|
+
rescue StandardError => e
|
|
37
|
+
handle_exception(
|
|
38
|
+
"POSTGRESQL:DISCOVER:EXCEPTION",
|
|
39
|
+
"error",
|
|
40
|
+
e
|
|
41
|
+
)
|
|
42
|
+
ensure
|
|
43
|
+
db&.close
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def write(sync_config, records, action = "insert")
|
|
47
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
48
|
+
table_name = sync_config.stream.name
|
|
49
|
+
db = create_connection(connection_config)
|
|
50
|
+
|
|
51
|
+
write_success = 0
|
|
52
|
+
write_failure = 0
|
|
53
|
+
|
|
54
|
+
records.each do |record|
|
|
55
|
+
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record)
|
|
56
|
+
begin
|
|
57
|
+
db.exec(query)
|
|
58
|
+
write_success += 1
|
|
59
|
+
rescue StandardError => e
|
|
60
|
+
handle_exception("POSTGRESQL:RECORD:WRITE:EXCEPTION", "error", e)
|
|
61
|
+
write_failure += 1
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
tracking_message(write_success, write_failure)
|
|
65
|
+
rescue StandardError => e
|
|
66
|
+
handle_exception(
|
|
67
|
+
"POSTGRESQL:WRITE:EXCEPTION",
|
|
68
|
+
"error",
|
|
69
|
+
e
|
|
70
|
+
)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def query(connection, query)
|
|
76
|
+
connection.exec(query) do |result|
|
|
77
|
+
result.map do |row|
|
|
78
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def create_connection(connection_config)
|
|
84
|
+
raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
|
|
85
|
+
|
|
86
|
+
PG.connect(
|
|
87
|
+
host: connection_config[:host],
|
|
88
|
+
dbname: connection_config[:database],
|
|
89
|
+
user: connection_config[:credentials][:username],
|
|
90
|
+
password: connection_config[:credentials][:password],
|
|
91
|
+
port: connection_config[:port]
|
|
92
|
+
)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def create_streams(records)
|
|
96
|
+
group_by_table(records).map do |r|
|
|
97
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def group_by_table(records)
|
|
102
|
+
records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
|
|
103
|
+
{
|
|
104
|
+
tablename: table_name,
|
|
105
|
+
columns: columns.map do |column|
|
|
106
|
+
{
|
|
107
|
+
column_name: column["column_name"],
|
|
108
|
+
type: column["data_type"],
|
|
109
|
+
optional: column["is_nullable"] == "YES"
|
|
110
|
+
}
|
|
111
|
+
end
|
|
112
|
+
}
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def tracking_message(success, failure)
|
|
117
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
118
|
+
success: success, failed: failure
|
|
119
|
+
).to_multiwoven_message
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Postgresql",
|
|
4
|
+
"title": "PostgreSQL",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-postgresql",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/postgresql",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Postgresql",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["host", "port", "database", "schema"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"credentials": {
|
|
11
|
+
"title": "",
|
|
12
|
+
"type": "object",
|
|
13
|
+
"required": ["auth_type", "username", "password"],
|
|
14
|
+
"properties": {
|
|
15
|
+
"auth_type": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"default": "username/password",
|
|
18
|
+
"order": 0,
|
|
19
|
+
"readOnly": true
|
|
20
|
+
},
|
|
21
|
+
"username": {
|
|
22
|
+
"description": "Username refers to your individual PostgreSQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
|
|
23
|
+
"examples": ["POSTGRESQL_USER"],
|
|
24
|
+
"type": "string",
|
|
25
|
+
"title": "Username",
|
|
26
|
+
"order": 1
|
|
27
|
+
},
|
|
28
|
+
"password": {
|
|
29
|
+
"description": "This field requires the password associated with the user account specified in the preceding section.",
|
|
30
|
+
"type": "string",
|
|
31
|
+
"multiwoven_secret": true,
|
|
32
|
+
"title": "Password",
|
|
33
|
+
"order": 2
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"order": 0
|
|
37
|
+
},
|
|
38
|
+
"host": {
|
|
39
|
+
"description": "The hostname or IP address of your PostgreSQL server.",
|
|
40
|
+
"examples": ["127.0.0.1"],
|
|
41
|
+
"type": "string",
|
|
42
|
+
"title": "Host",
|
|
43
|
+
"order": 1
|
|
44
|
+
},
|
|
45
|
+
"port": {
|
|
46
|
+
"description": "The port number for your PostgreSQL server, which defaults to 5432, may vary based on your configuration. ",
|
|
47
|
+
"examples": ["5432"],
|
|
48
|
+
"type": "string",
|
|
49
|
+
"title": "Port",
|
|
50
|
+
"order": 2
|
|
51
|
+
},
|
|
52
|
+
"database": {
|
|
53
|
+
"description": "The specific PostgreSQL database to connect to.",
|
|
54
|
+
"examples": ["POSTGRESQL_DB"],
|
|
55
|
+
"type": "string",
|
|
56
|
+
"title": "Database",
|
|
57
|
+
"order": 3
|
|
58
|
+
},
|
|
59
|
+
"schema": {
|
|
60
|
+
"description": "The schema within the PostgreSQL database.",
|
|
61
|
+
"examples": ["POSTGRESQL_SCHEMA"],
|
|
62
|
+
"type": "string",
|
|
63
|
+
"title": "Schema",
|
|
64
|
+
"order": 4
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
|
3
|
+
<svg width="432.071pt" height="445.383pt" viewBox="0 0 432.071 445.383" xml:space="preserve" xmlns="http://www.w3.org/2000/svg">
|
|
4
|
+
<g id="orginal" style="fill-rule:nonzero;clip-rule:nonzero;stroke:#000000;stroke-miterlimit:4;">
|
|
5
|
+
</g>
|
|
6
|
+
<g id="Layer_x0020_3" style="fill-rule:nonzero;clip-rule:nonzero;fill:none;stroke:#FFFFFF;stroke-width:12.4651;stroke-linecap:round;stroke-linejoin:round;stroke-miterlimit:4;">
|
|
7
|
+
<path style="fill:#000000;stroke:#000000;stroke-width:37.3953;stroke-linecap:butt;stroke-linejoin:miter;" d="M323.205,324.227c2.833-23.601,1.984-27.062,19.563-23.239l4.463,0.392c13.517,0.615,31.199-2.174,41.587-7c22.362-10.376,35.622-27.7,13.572-23.148c-50.297,10.376-53.755-6.655-53.755-6.655c53.111-78.803,75.313-178.836,56.149-203.322 C352.514-5.534,262.036,26.049,260.522,26.869l-0.482,0.089c-9.938-2.062-21.06-3.294-33.554-3.496c-22.761-0.374-40.032,5.967-53.133,15.904c0,0-161.408-66.498-153.899,83.628c1.597,31.936,45.777,241.655,98.47,178.31 c19.259-23.163,37.871-42.748,37.871-42.748c9.242,6.14,20.307,9.272,31.912,8.147l0.897-0.765c-0.281,2.876-0.157,5.689,0.359,9.019c-13.572,15.167-9.584,17.83-36.723,23.416c-27.457,5.659-11.326,15.734-0.797,18.367c12.768,3.193,42.305,7.716,62.268-20.224 l-0.795,3.188c5.325,4.26,4.965,30.619,5.72,49.452c0.756,18.834,2.017,36.409,5.856,46.771c3.839,10.36,8.369,37.05,44.036,29.406c29.809-6.388,52.6-15.582,54.677-101.107"/>
|
|
8
|
+
<path style="fill:#336791;stroke:none;" d="M402.395,271.23c-50.302,10.376-53.76-6.655-53.76-6.655c53.111-78.808,75.313-178.843,56.153-203.326c-52.27-66.785-142.752-35.2-144.262-34.38l-0.486,0.087c-9.938-2.063-21.06-3.292-33.56-3.496c-22.761-0.373-40.026,5.967-53.127,15.902 c0,0-161.411-66.495-153.904,83.63c1.597,31.938,45.776,241.657,98.471,178.312c19.26-23.163,37.869-42.748,37.869-42.748c9.243,6.14,20.308,9.272,31.908,8.147l0.901-0.765c-0.28,2.876-0.152,5.689,0.361,9.019c-13.575,15.167-9.586,17.83-36.723,23.416 c-27.459,5.659-11.328,15.734-0.796,18.367c12.768,3.193,42.307,7.716,62.266-20.224l-0.796,3.188c5.319,4.26,9.054,27.711,8.428,48.969c-0.626,21.259-1.044,35.854,3.147,47.254c4.191,11.4,8.368,37.05,44.042,29.406c29.809-6.388,45.256-22.942,47.405-50.555 c1.525-19.631,4.976-16.729,5.194-34.28l2.768-8.309c3.192-26.611,0.507-35.196,18.872-31.203l4.463,0.392c13.517,0.615,31.208-2.174,41.591-7c22.358-10.376,35.618-27.7,13.573-23.148z"/>
|
|
9
|
+
<path d="M215.866,286.484c-1.385,49.516,0.348,99.377,5.193,111.495c4.848,12.118,15.223,35.688,50.9,28.045c29.806-6.39,40.651-18.756,45.357-46.051c3.466-20.082,10.148-75.854,11.005-87.281"/>
|
|
10
|
+
<path d="M173.104,38.256c0,0-161.521-66.016-154.012,84.109c1.597,31.938,45.779,241.664,98.473,178.316c19.256-23.166,36.671-41.335,36.671-41.335"/>
|
|
11
|
+
<path d="M260.349,26.207c-5.591,1.753,89.848-34.889,144.087,34.417c19.159,24.484-3.043,124.519-56.153,203.329"/>
|
|
12
|
+
<path style="stroke-linejoin:bevel;" d="M348.282,263.953c0,0,3.461,17.036,53.764,6.653c22.04-4.552,8.776,12.774-13.577,23.155c-18.345,8.514-59.474,10.696-60.146-1.069c-1.729-30.355,21.647-21.133,19.96-28.739c-1.525-6.85-11.979-13.573-18.894-30.338 c-6.037-14.633-82.796-126.849,21.287-110.183c3.813-0.789-27.146-99.002-124.553-100.599c-97.385-1.597-94.19,119.762-94.19,119.762"/>
|
|
13
|
+
<path d="M188.604,274.334c-13.577,15.166-9.584,17.829-36.723,23.417c-27.459,5.66-11.326,15.733-0.797,18.365c12.768,3.195,42.307,7.718,62.266-20.229c6.078-8.509-0.036-22.086-8.385-25.547c-4.034-1.671-9.428-3.765-16.361,3.994z"/>
|
|
14
|
+
<path d="M187.715,274.069c-1.368-8.917,2.93-19.528,7.536-31.942c6.922-18.626,22.893-37.255,10.117-96.339c-9.523-44.029-73.396-9.163-73.436-3.193c-0.039,5.968,2.889,30.26-1.067,58.548c-5.162,36.913,23.488,68.132,56.479,64.938"/>
|
|
15
|
+
<path style="fill:#FFFFFF;stroke-width:4.155;stroke-linecap:butt;stroke-linejoin:miter;" d="M172.517,141.7c-0.288,2.039,3.733,7.48,8.976,8.207c5.234,0.73,9.714-3.522,9.998-5.559c0.284-2.039-3.732-4.285-8.977-5.015c-5.237-0.731-9.719,0.333-9.996,2.367z"/>
|
|
16
|
+
<path style="fill:#FFFFFF;stroke-width:2.0775;stroke-linecap:butt;stroke-linejoin:miter;" d="M331.941,137.543c0.284,2.039-3.732,7.48-8.976,8.207c-5.238,0.73-9.718-3.522-10.005-5.559c-0.277-2.039,3.74-4.285,8.979-5.015c5.239-0.73,9.718,0.333,10.002,2.368z"/>
|
|
17
|
+
<path d="M350.676,123.432c0.863,15.994-3.445,26.888-3.988,43.914c-0.804,24.748,11.799,53.074-7.191,81.435"/>
|
|
18
|
+
<path style="stroke-width:3;" d="M0,60.232"/>
|
|
19
|
+
</g>
|
|
20
|
+
</svg>
|
|
@@ -38,6 +38,7 @@ require_relative "integrations/core/base_connector"
|
|
|
38
38
|
require_relative "integrations/core/source_connector"
|
|
39
39
|
require_relative "integrations/core/destination_connector"
|
|
40
40
|
require_relative "integrations/core/http_client"
|
|
41
|
+
require_relative "integrations/core/query_builder"
|
|
41
42
|
|
|
42
43
|
# Source
|
|
43
44
|
require_relative "integrations/source/snowflake/client"
|
|
@@ -58,6 +59,7 @@ require_relative "integrations/destination/airtable/client"
|
|
|
58
59
|
require_relative "integrations/destination/stripe/client"
|
|
59
60
|
require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
|
|
60
61
|
require_relative "integrations/destination/sftp/client"
|
|
62
|
+
require_relative "integrations/destination/postgresql/client"
|
|
61
63
|
|
|
62
64
|
module Multiwoven
|
|
63
65
|
module Integrations
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.44
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
@@ -384,6 +384,7 @@ files:
|
|
|
384
384
|
- lib/multiwoven/integrations/core/destination_connector.rb
|
|
385
385
|
- lib/multiwoven/integrations/core/fullrefresher.rb
|
|
386
386
|
- lib/multiwoven/integrations/core/http_client.rb
|
|
387
|
+
- lib/multiwoven/integrations/core/query_builder.rb
|
|
387
388
|
- lib/multiwoven/integrations/core/rate_limiter.rb
|
|
388
389
|
- lib/multiwoven/integrations/core/source_connector.rb
|
|
389
390
|
- lib/multiwoven/integrations/core/utils.rb
|
|
@@ -413,6 +414,10 @@ files:
|
|
|
413
414
|
- lib/multiwoven/integrations/destination/klaviyo/config/meta.json
|
|
414
415
|
- lib/multiwoven/integrations/destination/klaviyo/config/spec.json
|
|
415
416
|
- lib/multiwoven/integrations/destination/klaviyo/icon.svg
|
|
417
|
+
- lib/multiwoven/integrations/destination/postgresql/client.rb
|
|
418
|
+
- lib/multiwoven/integrations/destination/postgresql/config/meta.json
|
|
419
|
+
- lib/multiwoven/integrations/destination/postgresql/config/spec.json
|
|
420
|
+
- lib/multiwoven/integrations/destination/postgresql/icon.svg
|
|
416
421
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb
|
|
417
422
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json
|
|
418
423
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json
|