multiwoven-integrations 0.3.5 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/maria_db/client.rb +115 -0
- data/lib/multiwoven/integrations/destination/maria_db/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/maria_db/config/spec.json +48 -0
- data/lib/multiwoven/integrations/destination/maria_db/icon.svg +15 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/amazon_s3/client.rb +1 -2
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 35fa7c087990d28ecdd114eb72988b8270ac03eff77af82319a5a01b8b2410ee
|
|
4
|
+
data.tar.gz: a3e01fe37c6d3a01a996cc0b8a4976b9970fae04e8ff8299f1a82ca3907ce7b0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '09513942d7264e3bb71dc528daa4aa0a86efa9672178182a33c650a6bcd4138afc3ece53b439baf752aed3e89813a4f9873b0b62b38995fec96d18d5817553b0'
|
|
7
|
+
data.tar.gz: b3ff5e7eef73def73d1e749b6c6870aaf1128017d8b34d4f6a8e925b5b21b7031227fc48e82379bbb47ebc753b62f5c2e07754e90e67f16930352d369246cbd4
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Destination
|
|
4
|
+
module MariaDB
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < DestinationConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
11
|
+
rescue StandardError => e
|
|
12
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def discover(connection_config)
|
|
16
|
+
connection_config = connection_config.with_indifferent_access
|
|
17
|
+
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
18
|
+
FROM information_schema.columns
|
|
19
|
+
WHERE table_schema = '#{connection_config[:database]}'
|
|
20
|
+
ORDER BY table_name, ordinal_position;"
|
|
21
|
+
|
|
22
|
+
db = create_connection(connection_config)
|
|
23
|
+
records = db.fetch(query) do |result|
|
|
24
|
+
result.map do |row|
|
|
25
|
+
row
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
29
|
+
catalog.to_multiwoven_message
|
|
30
|
+
rescue StandardError => e
|
|
31
|
+
handle_exception(
|
|
32
|
+
"MARIA:DB:DISCOVER:EXCEPTION",
|
|
33
|
+
"error",
|
|
34
|
+
e
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def write(sync_config, records, action = "destination_insert")
|
|
39
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
40
|
+
table_name = sync_config.stream.name
|
|
41
|
+
primary_key = sync_config.model.primary_key
|
|
42
|
+
db = create_connection(connection_config)
|
|
43
|
+
|
|
44
|
+
write_success = 0
|
|
45
|
+
write_failure = 0
|
|
46
|
+
|
|
47
|
+
records.each do |record|
|
|
48
|
+
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
|
|
49
|
+
logger.debug("MARIA:DB:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
|
|
50
|
+
begin
|
|
51
|
+
db.run(query)
|
|
52
|
+
write_success += 1
|
|
53
|
+
rescue StandardError => e
|
|
54
|
+
handle_exception(e, {
|
|
55
|
+
context: "MARIA:DB:RECORD:WRITE:EXCEPTION",
|
|
56
|
+
type: "error",
|
|
57
|
+
sync_id: sync_config.sync_id,
|
|
58
|
+
sync_run_id: sync_config.sync_run_id
|
|
59
|
+
})
|
|
60
|
+
write_failure += 1
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
tracking_message(write_success, write_failure)
|
|
64
|
+
rescue StandardError => e
|
|
65
|
+
handle_exception(e, {
|
|
66
|
+
context: "MARIA:DB:RECORD:WRITE:EXCEPTION",
|
|
67
|
+
type: "error",
|
|
68
|
+
sync_id: sync_config.sync_id,
|
|
69
|
+
sync_run_id: sync_config.sync_run_id
|
|
70
|
+
})
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
private
|
|
74
|
+
|
|
75
|
+
def create_connection(connection_config)
|
|
76
|
+
Sequel.connect(
|
|
77
|
+
adapter: "mysql2",
|
|
78
|
+
host: connection_config[:host],
|
|
79
|
+
port: connection_config[:port],
|
|
80
|
+
user: connection_config[:username],
|
|
81
|
+
password: connection_config[:password],
|
|
82
|
+
database: connection_config[:database]
|
|
83
|
+
)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def create_streams(records)
|
|
87
|
+
group_by_table(records).map do |_, r|
|
|
88
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def group_by_table(records)
|
|
93
|
+
result = {}
|
|
94
|
+
records.each_with_index do |entry, index|
|
|
95
|
+
table_name = entry[:table_name]
|
|
96
|
+
column_data = {
|
|
97
|
+
column_name: entry[:column_name],
|
|
98
|
+
data_type: entry[:data_type],
|
|
99
|
+
is_nullable: entry[:is_nullable] == "YES"
|
|
100
|
+
}
|
|
101
|
+
result[index] ||= {}
|
|
102
|
+
result[index][:tablename] = table_name
|
|
103
|
+
result[index][:columns] = [column_data]
|
|
104
|
+
end
|
|
105
|
+
result
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def tracking_message(success, failure)
|
|
109
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
110
|
+
success: success, failed: failure
|
|
111
|
+
).to_multiwoven_message
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "MariaDB",
|
|
4
|
+
"title": "Maria DB",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/destination/mariadb",
|
|
8
|
+
"github_issue_label": "destination-maria-db",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/destination/mariadb",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Maria DB",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["host", "port", "username", "password", "database"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"host": {
|
|
12
|
+
"description": "The hostname or IP address of the server where the MariaDB database is hosted.",
|
|
13
|
+
"examples": ["localhost"],
|
|
14
|
+
"type": "string",
|
|
15
|
+
"title": "Host",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"port": {
|
|
19
|
+
"description": "The port number on which the MariaDB server is listening for connections.",
|
|
20
|
+
"examples": ["3306"],
|
|
21
|
+
"type": "string",
|
|
22
|
+
"title": "Port",
|
|
23
|
+
"order": 1
|
|
24
|
+
},
|
|
25
|
+
"username": {
|
|
26
|
+
"description": "The username used to authenticate and connect to the MariaDB database.",
|
|
27
|
+
"examples": ["root"],
|
|
28
|
+
"type": "string",
|
|
29
|
+
"title": "Username",
|
|
30
|
+
"order": 2
|
|
31
|
+
},
|
|
32
|
+
"password": {
|
|
33
|
+
"description": "The password corresponding to the username used for authentication.",
|
|
34
|
+
"type": "string",
|
|
35
|
+
"multiwoven_secret": true,
|
|
36
|
+
"title": "Password",
|
|
37
|
+
"order": 3
|
|
38
|
+
},
|
|
39
|
+
"database": {
|
|
40
|
+
"description": "The name of the specific database within the MariaDB server to connect to.",
|
|
41
|
+
"examples": ["test"],
|
|
42
|
+
"type": "string",
|
|
43
|
+
"title": "Database",
|
|
44
|
+
"order": 4
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewBox="0 0 309.88 252.72">
|
|
3
|
+
<defs>
|
|
4
|
+
<style>.cls-1{fill:#003545;}</style>
|
|
5
|
+
</defs>
|
|
6
|
+
<title>MDB-VLogo_RGB</title>
|
|
7
|
+
<path class="cls-1" d="M61.74,214.91,73,258.46h-8.5L57,227.09,42.86,258.46H35.68L21.62,227.27,13.9,258.46H5.58l11.16-43.55H24.1l15.2,33.72,15.14-33.72Z" transform="translate(-5.58 -6.4)"></path>
|
|
8
|
+
<path class="cls-1" d="M105.11,231.74v-5.07h8.21v31.79h-8.21v-5.12c-2.17,3.68-6.63,5.79-12.06,5.79-11.35,0-17.68-8-17.68-17.2,0-8.87,6-16,16.47-16C97.81,226,102.76,228,105.11,231.74Zm-21,10.62c0,5.85,3.68,10.86,10.8,10.86,6.87,0,10.61-4.89,10.61-10.68s-3.86-10.74-11-10.74C87.68,231.8,84.12,236.74,84.12,242.36Z" transform="translate(-5.58 -6.4)"></path>
|
|
9
|
+
<path class="cls-1" d="M131.83,258.46h-8.2V226.67h8.2v7.12A12.58,12.58,0,0,1,143.29,226a14.84,14.84,0,0,1,5.13.84l-1.75,6a18,18,0,0,0-4.34-.6c-5.92,0-10.5,4.46-10.5,11Z" transform="translate(-5.58 -6.4)"></path>
|
|
10
|
+
<path class="cls-1" d="M152.52,218.28a4.17,4.17,0,0,1,4.4-4.28,4.33,4.33,0,0,1,4.47,4.41,4.15,4.15,0,0,1-4.47,4.22A4.22,4.22,0,0,1,152.52,218.28Zm.36,8.39h8.21V249.9c0,1.44.3,2.47,1.5,2.47a8.85,8.85,0,0,0,1.57-.18l1.27,6a14.36,14.36,0,0,1-5.43,1c-3.44,0-7.12-1-7.12-8.81Z" transform="translate(-5.58 -6.4)"></path>
|
|
11
|
+
<path class="cls-1" d="M197.76,231.74v-5.07H206v31.79h-8.21v-5.12c-2.17,3.68-6.63,5.79-12.06,5.79-11.34,0-17.68-8-17.68-17.2,0-8.87,6-16,16.47-16C190.46,226,195.41,228,197.76,231.74Zm-21,10.62c0,5.85,3.68,10.86,10.8,10.86,6.88,0,10.62-4.89,10.62-10.68s-3.87-10.74-11-10.74C180.33,231.8,176.77,236.74,176.77,242.36Z" transform="translate(-5.58 -6.4)"></path>
|
|
12
|
+
<path class="cls-1" d="M218.57,214.91h19.67c16.53,0,24.79,9.11,24.67,21.77.12,13.16-9,21.78-23.22,21.78H218.57Zm5.43,3.86v35.89h15c13.15,0,18.16-8.87,18.16-18.1,0-10.43-6.27-17.79-18.16-17.79Z" transform="translate(-5.58 -6.4)"></path>
|
|
13
|
+
<path class="cls-1" d="M296.45,258.46h-25V214.91H294c8.62,0,16.83,1.62,16.71,11.28,0,6.81-4.23,8.69-8.69,9.41,6.33.54,10.14,4.58,10.14,11.1C312.2,256.47,303.63,258.46,296.45,258.46Zm-1.87-24.55c8.63,0,10.56-3.32,10.56-7.54,0-6.34-3.86-7.78-10.56-7.78H276.66v15.32Zm.24,3.68H276.66v17.19H295.6c5.31,0,10.92-1.75,10.92-8.44C306.52,238.62,300.07,237.59,294.82,237.59Z" transform="translate(-5.58 -6.4)"></path>
|
|
14
|
+
<path class="cls-1" d="M314.08,7.35a4.18,4.18,0,0,0-2.84-.95c-2.83,0-6.49,1.92-8.46,2.95l-.78.4a26.86,26.86,0,0,1-10.57,2.66c-3.76.12-7,.34-11.22.78-25,2.57-36.15,21.73-46.89,40.26C227.47,63.53,221.43,74,213.15,82a54.4,54.4,0,0,1-5.45,4.63C199.13,93,188.37,97.55,180,100.77c-8.06,3.08-16.86,5.85-25.36,8.53-7.79,2.45-15.14,4.77-21.9,7.28-3.05,1.13-5.64,2-7.93,2.76-6.16,2-10.6,3.53-17.09,8-2.53,1.73-5.07,3.6-6.79,5a71.62,71.62,0,0,0-13.55,14.27A84.25,84.25,0,0,1,76,160.27c-1.37,1.34-3.8,2-7.44,2-4.26,0-9.43-.88-14.9-1.81-5.64-1-11.47-1.95-16.47-1.95-4.06,0-7.17.66-9.49,2,0,0-3.91,2.28-5.56,5.23l1.62.73a33.21,33.21,0,0,1,6.92,5,34.72,34.72,0,0,0,7.2,5.12A6.18,6.18,0,0,1,40.1,178c-.68,1-1.68,2.29-2.73,3.67-5.77,7.55-9.14,12.32-7.21,14.92a6.07,6.07,0,0,0,3,.68c12.58,0,19.34-3.27,27.89-7.41,2.48-1.2,5-2.43,8-3.7,5-2.17,10.38-5.63,16.09-9.29C92.61,172,100.42,167,108,164.59a62.3,62.3,0,0,1,19.23-2.7c8,0,16.42,1.07,24.54,2.11,6.05.78,12.32,1.58,18.47,1.95,2.39.14,4.6.21,6.75.21a78.21,78.21,0,0,0,8.61-.45l.69-.24c4.31-2.65,6.33-8.34,8.29-13.84,1.26-3.54,2.32-6.72,4-8.74a2.55,2.55,0,0,1,.32-.27.4.4,0,0,1,.49.08s0,.05,0,.16c-1,21.51-9.66,35.17-18.42,47.31l-5.85,6.27s8.19,0,12.85-1.8c17-5.08,29.83-16.28,39.17-34.14a145.7,145.7,0,0,0,6.17-14.09c.16-.4,1.63-1.14,1.49.93-.05.61-.09,1.29-.14,2h0c0,.42,0,.85-.08,1.28-.24,3-.95,9.34-.95,9.34l5.25-2.81c12.66-8,22.42-24.14,29.82-49.25,3.08-10.46,5.34-20.85,7.33-30,2.38-11,4.43-20.43,6.78-24.09,3.69-5.74,9.32-9.62,14.77-13.39.74-.51,1.49-1,2.22-1.54,6.85-4.81,13.66-10.36,15.16-20.71l0-.23C316.05,10.22,315.13,8.25,314.08,7.35Z" transform="translate(-5.58 -6.4)"></path>
|
|
15
|
+
</svg>
|
|
@@ -55,8 +55,7 @@ module Multiwoven::Integrations::Source
|
|
|
55
55
|
if connection_config[:auth_type] == "user"
|
|
56
56
|
Aws::Credentials.new(connection_config[:access_id], connection_config[:secret_access])
|
|
57
57
|
elsif connection_config[:auth_type] == "role"
|
|
58
|
-
|
|
59
|
-
sts_client = Aws::STS::Client.new(region: connection_config[:region], credentials: credentials)
|
|
58
|
+
sts_client = Aws::STS::Client.new(region: connection_config[:region])
|
|
60
59
|
resp = sts_client.assume_role({
|
|
61
60
|
role_arn: connection_config[:arn],
|
|
62
61
|
role_session_name: session,
|
|
@@ -76,6 +76,7 @@ require_relative "integrations/destination/postgresql/client"
|
|
|
76
76
|
require_relative "integrations/destination/http/client"
|
|
77
77
|
require_relative "integrations/destination/zendesk/client"
|
|
78
78
|
require_relative "integrations/destination/iterable/client"
|
|
79
|
+
require_relative "integrations/destination/maria_db/client"
|
|
79
80
|
|
|
80
81
|
module Multiwoven
|
|
81
82
|
module Integrations
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-07-
|
|
11
|
+
date: 2024-07-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -522,6 +522,10 @@ files:
|
|
|
522
522
|
- lib/multiwoven/integrations/destination/klaviyo/config/meta.json
|
|
523
523
|
- lib/multiwoven/integrations/destination/klaviyo/config/spec.json
|
|
524
524
|
- lib/multiwoven/integrations/destination/klaviyo/icon.svg
|
|
525
|
+
- lib/multiwoven/integrations/destination/maria_db/client.rb
|
|
526
|
+
- lib/multiwoven/integrations/destination/maria_db/config/meta.json
|
|
527
|
+
- lib/multiwoven/integrations/destination/maria_db/config/spec.json
|
|
528
|
+
- lib/multiwoven/integrations/destination/maria_db/icon.svg
|
|
525
529
|
- lib/multiwoven/integrations/destination/postgresql/client.rb
|
|
526
530
|
- lib/multiwoven/integrations/destination/postgresql/config/meta.json
|
|
527
531
|
- lib/multiwoven/integrations/destination/postgresql/config/spec.json
|