multiwoven-integrations 0.2.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/protocol/protocol.rb +1 -1
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/maria_db/client.rb +92 -0
- data/lib/multiwoven/integrations/source/maria_db/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/maria_db/config/spec.json +48 -0
- data/lib/multiwoven/integrations/source/maria_db/icon.svg +15 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f9a7e28f8f3f1a559e5bb4feaec6a030201d447e1f1c7e90e8e8ced256274d5c
|
|
4
|
+
data.tar.gz: a16b584bbc112998e54b9a51785498411e98cb8156116f66e082afd55c1c4916
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f2e3b44f30db035ac8b87f1cc2ffbecc34aef53630ff7c5757193affaebc0749c827368c72cdb04444a00c0abc800c6c2b0cef97aa2484b532662292e8b0ccd0
|
|
7
|
+
data.tar.gz: 0f2650b28d03b4118dcb86d3533cadf6fec2bcb54c0b54f349e79aa7f276b50c9a574e6b3ef87c2b19dda810b18cefcf00dfb470706753e49f5e35c0a5d858a4
|
|
@@ -11,7 +11,7 @@ module Multiwoven
|
|
|
11
11
|
DestinationSyncMode = Types::String.enum("insert", "upsert")
|
|
12
12
|
ConnectorType = Types::String.enum("source", "destination")
|
|
13
13
|
ConnectorQueryType = Types::String.enum("raw_sql", "soql")
|
|
14
|
-
ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql")
|
|
14
|
+
ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector")
|
|
15
15
|
ConnectionStatusType = Types::String.enum("succeeded", "failed")
|
|
16
16
|
StreamType = Types::String.enum("static", "dynamic")
|
|
17
17
|
StreamAction = Types::String.enum("fetch", "create", "update", "delete")
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.3.1"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -14,6 +14,7 @@ module Multiwoven
|
|
|
14
14
|
AwsAthena
|
|
15
15
|
Clickhouse
|
|
16
16
|
AmazonS3
|
|
17
|
+
MariaDB
|
|
17
18
|
].freeze
|
|
18
19
|
|
|
19
20
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module MariaDB
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
11
|
+
rescue StandardError => e
|
|
12
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def discover(connection_config)
|
|
16
|
+
connection_config = connection_config.with_indifferent_access
|
|
17
|
+
query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:database]}' ORDER BY table_name, ordinal_position;"
|
|
18
|
+
db = create_connection(connection_config)
|
|
19
|
+
results = query_execution(db, query)
|
|
20
|
+
catalog = Catalog.new(streams: create_streams(results))
|
|
21
|
+
catalog.to_multiwoven_message
|
|
22
|
+
rescue StandardError => e
|
|
23
|
+
handle_exception(e, {
|
|
24
|
+
context: "MARIA:DB:DISCOVER:EXCEPTION",
|
|
25
|
+
type: "error"
|
|
26
|
+
})
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def read(sync_config)
|
|
30
|
+
connection_config = sync_config.source.connection_specification.with_indifferent_access
|
|
31
|
+
query = sync_config.model.query
|
|
32
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
33
|
+
db = create_connection(connection_config)
|
|
34
|
+
query(db, query)
|
|
35
|
+
rescue StandardError => e
|
|
36
|
+
handle_exception(e, {
|
|
37
|
+
context: "MARIA:DB:READ:EXCEPTION",
|
|
38
|
+
type: "error",
|
|
39
|
+
sync_id: sync_config.sync_id,
|
|
40
|
+
sync_run_id: sync_config.sync_run_id
|
|
41
|
+
})
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def create_connection(connection_config)
|
|
47
|
+
Sequel.connect(
|
|
48
|
+
adapter: "mysql2",
|
|
49
|
+
host: connection_config[:host],
|
|
50
|
+
port: connection_config[:port],
|
|
51
|
+
user: connection_config[:username],
|
|
52
|
+
password: connection_config[:password],
|
|
53
|
+
database: connection_config[:database]
|
|
54
|
+
)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def query_execution(db, query)
|
|
58
|
+
db.fetch(query).all
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def create_streams(records)
|
|
62
|
+
group_by_table(records).map do |_, r|
|
|
63
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def query(db, query)
|
|
68
|
+
records = []
|
|
69
|
+
query_execution(db, query).map do |row|
|
|
70
|
+
records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
71
|
+
end
|
|
72
|
+
records
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def group_by_table(records)
|
|
76
|
+
result = {}
|
|
77
|
+
records.each_with_index do |entry, index|
|
|
78
|
+
table_name = entry[:table_name]
|
|
79
|
+
column_data = {
|
|
80
|
+
column_name: entry[:column_name],
|
|
81
|
+
data_type: entry[:data_type],
|
|
82
|
+
is_nullable: entry[:is_nullable] == "YES"
|
|
83
|
+
}
|
|
84
|
+
result[index] ||= {}
|
|
85
|
+
result[index][:tablename] = table_name
|
|
86
|
+
result[index][:columns] = [column_data]
|
|
87
|
+
end
|
|
88
|
+
result
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "MariaDB",
|
|
4
|
+
"title": "Maria DB",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/sources/mariadb",
|
|
8
|
+
"github_issue_label": "source-maria-db",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/sources/mariadb",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Maria DB",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["host", "port", "username", "password", "database"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"host": {
|
|
12
|
+
"description": "The hostname or IP address of the server where the MariaDB database is hosted.",
|
|
13
|
+
"examples": ["localhost"],
|
|
14
|
+
"type": "string",
|
|
15
|
+
"title": "Host",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"port": {
|
|
19
|
+
"description": "The port number on which the MariaDB server is listening for connections.",
|
|
20
|
+
"examples": ["3306"],
|
|
21
|
+
"type": "string",
|
|
22
|
+
"title": "Port",
|
|
23
|
+
"order": 1
|
|
24
|
+
},
|
|
25
|
+
"username": {
|
|
26
|
+
"description": "The username used to authenticate and connect to the MariaDB database.",
|
|
27
|
+
"examples": ["root"],
|
|
28
|
+
"type": "string",
|
|
29
|
+
"title": "Username",
|
|
30
|
+
"order": 2
|
|
31
|
+
},
|
|
32
|
+
"password": {
|
|
33
|
+
"description": "The password corresponding to the username used for authentication.",
|
|
34
|
+
"type": "string",
|
|
35
|
+
"multiwoven_secret": true,
|
|
36
|
+
"title": "Password",
|
|
37
|
+
"order": 3
|
|
38
|
+
},
|
|
39
|
+
"database": {
|
|
40
|
+
"description": "The name of the specific database within the MariaDB server to connect to.",
|
|
41
|
+
"examples": ["mydatabase"],
|
|
42
|
+
"type": "string",
|
|
43
|
+
"title": "Database",
|
|
44
|
+
"order": 4
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewBox="0 0 309.88 252.72">
|
|
3
|
+
<defs>
|
|
4
|
+
<style>.cls-1{fill:#003545;}</style>
|
|
5
|
+
</defs>
|
|
6
|
+
<title>MDB-VLogo_RGB</title>
|
|
7
|
+
<path class="cls-1" d="M61.74,214.91,73,258.46h-8.5L57,227.09,42.86,258.46H35.68L21.62,227.27,13.9,258.46H5.58l11.16-43.55H24.1l15.2,33.72,15.14-33.72Z" transform="translate(-5.58 -6.4)"></path>
|
|
8
|
+
<path class="cls-1" d="M105.11,231.74v-5.07h8.21v31.79h-8.21v-5.12c-2.17,3.68-6.63,5.79-12.06,5.79-11.35,0-17.68-8-17.68-17.2,0-8.87,6-16,16.47-16C97.81,226,102.76,228,105.11,231.74Zm-21,10.62c0,5.85,3.68,10.86,10.8,10.86,6.87,0,10.61-4.89,10.61-10.68s-3.86-10.74-11-10.74C87.68,231.8,84.12,236.74,84.12,242.36Z" transform="translate(-5.58 -6.4)"></path>
|
|
9
|
+
<path class="cls-1" d="M131.83,258.46h-8.2V226.67h8.2v7.12A12.58,12.58,0,0,1,143.29,226a14.84,14.84,0,0,1,5.13.84l-1.75,6a18,18,0,0,0-4.34-.6c-5.92,0-10.5,4.46-10.5,11Z" transform="translate(-5.58 -6.4)"></path>
|
|
10
|
+
<path class="cls-1" d="M152.52,218.28a4.17,4.17,0,0,1,4.4-4.28,4.33,4.33,0,0,1,4.47,4.41,4.15,4.15,0,0,1-4.47,4.22A4.22,4.22,0,0,1,152.52,218.28Zm.36,8.39h8.21V249.9c0,1.44.3,2.47,1.5,2.47a8.85,8.85,0,0,0,1.57-.18l1.27,6a14.36,14.36,0,0,1-5.43,1c-3.44,0-7.12-1-7.12-8.81Z" transform="translate(-5.58 -6.4)"></path>
|
|
11
|
+
<path class="cls-1" d="M197.76,231.74v-5.07H206v31.79h-8.21v-5.12c-2.17,3.68-6.63,5.79-12.06,5.79-11.34,0-17.68-8-17.68-17.2,0-8.87,6-16,16.47-16C190.46,226,195.41,228,197.76,231.74Zm-21,10.62c0,5.85,3.68,10.86,10.8,10.86,6.88,0,10.62-4.89,10.62-10.68s-3.87-10.74-11-10.74C180.33,231.8,176.77,236.74,176.77,242.36Z" transform="translate(-5.58 -6.4)"></path>
|
|
12
|
+
<path class="cls-1" d="M218.57,214.91h19.67c16.53,0,24.79,9.11,24.67,21.77.12,13.16-9,21.78-23.22,21.78H218.57Zm5.43,3.86v35.89h15c13.15,0,18.16-8.87,18.16-18.1,0-10.43-6.27-17.79-18.16-17.79Z" transform="translate(-5.58 -6.4)"></path>
|
|
13
|
+
<path class="cls-1" d="M296.45,258.46h-25V214.91H294c8.62,0,16.83,1.62,16.71,11.28,0,6.81-4.23,8.69-8.69,9.41,6.33.54,10.14,4.58,10.14,11.1C312.2,256.47,303.63,258.46,296.45,258.46Zm-1.87-24.55c8.63,0,10.56-3.32,10.56-7.54,0-6.34-3.86-7.78-10.56-7.78H276.66v15.32Zm.24,3.68H276.66v17.19H295.6c5.31,0,10.92-1.75,10.92-8.44C306.52,238.62,300.07,237.59,294.82,237.59Z" transform="translate(-5.58 -6.4)"></path>
|
|
14
|
+
<path class="cls-1" d="M314.08,7.35a4.18,4.18,0,0,0-2.84-.95c-2.83,0-6.49,1.92-8.46,2.95l-.78.4a26.86,26.86,0,0,1-10.57,2.66c-3.76.12-7,.34-11.22.78-25,2.57-36.15,21.73-46.89,40.26C227.47,63.53,221.43,74,213.15,82a54.4,54.4,0,0,1-5.45,4.63C199.13,93,188.37,97.55,180,100.77c-8.06,3.08-16.86,5.85-25.36,8.53-7.79,2.45-15.14,4.77-21.9,7.28-3.05,1.13-5.64,2-7.93,2.76-6.16,2-10.6,3.53-17.09,8-2.53,1.73-5.07,3.6-6.79,5a71.62,71.62,0,0,0-13.55,14.27A84.25,84.25,0,0,1,76,160.27c-1.37,1.34-3.8,2-7.44,2-4.26,0-9.43-.88-14.9-1.81-5.64-1-11.47-1.95-16.47-1.95-4.06,0-7.17.66-9.49,2,0,0-3.91,2.28-5.56,5.23l1.62.73a33.21,33.21,0,0,1,6.92,5,34.72,34.72,0,0,0,7.2,5.12A6.18,6.18,0,0,1,40.1,178c-.68,1-1.68,2.29-2.73,3.67-5.77,7.55-9.14,12.32-7.21,14.92a6.07,6.07,0,0,0,3,.68c12.58,0,19.34-3.27,27.89-7.41,2.48-1.2,5-2.43,8-3.7,5-2.17,10.38-5.63,16.09-9.29C92.61,172,100.42,167,108,164.59a62.3,62.3,0,0,1,19.23-2.7c8,0,16.42,1.07,24.54,2.11,6.05.78,12.32,1.58,18.47,1.95,2.39.14,4.6.21,6.75.21a78.21,78.21,0,0,0,8.61-.45l.69-.24c4.31-2.65,6.33-8.34,8.29-13.84,1.26-3.54,2.32-6.72,4-8.74a2.55,2.55,0,0,1,.32-.27.4.4,0,0,1,.49.08s0,.05,0,.16c-1,21.51-9.66,35.17-18.42,47.31l-5.85,6.27s8.19,0,12.85-1.8c17-5.08,29.83-16.28,39.17-34.14a145.7,145.7,0,0,0,6.17-14.09c.16-.4,1.63-1.14,1.49.93-.05.61-.09,1.29-.14,2h0c0,.42,0,.85-.08,1.28-.24,3-.95,9.34-.95,9.34l5.25-2.81c12.66-8,22.42-24.14,29.82-49.25,3.08-10.46,5.34-20.85,7.33-30,2.38-11,4.43-20.43,6.78-24.09,3.69-5.74,9.32-9.62,14.77-13.39.74-.51,1.49-1,2.22-1.54,6.85-4.81,13.66-10.36,15.16-20.71l0-.23C316.05,10.22,315.13,8.25,314.08,7.35Z" transform="translate(-5.58 -6.4)"></path>
|
|
15
|
+
</svg>
|
|
@@ -58,6 +58,7 @@ require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
|
|
|
58
58
|
require_relative "integrations/source/aws_athena/client"
|
|
59
59
|
require_relative "integrations/source/clickhouse/client"
|
|
60
60
|
require_relative "integrations/source/amazon_s3/client"
|
|
61
|
+
require_relative "integrations/source/maria_db/client"
|
|
61
62
|
|
|
62
63
|
# Destination
|
|
63
64
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-06-
|
|
11
|
+
date: 2024-06-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -567,6 +567,10 @@ files:
|
|
|
567
567
|
- lib/multiwoven/integrations/source/databricks/config/meta.json
|
|
568
568
|
- lib/multiwoven/integrations/source/databricks/config/spec.json
|
|
569
569
|
- lib/multiwoven/integrations/source/databricks/icon.svg
|
|
570
|
+
- lib/multiwoven/integrations/source/maria_db/client.rb
|
|
571
|
+
- lib/multiwoven/integrations/source/maria_db/config/meta.json
|
|
572
|
+
- lib/multiwoven/integrations/source/maria_db/config/spec.json
|
|
573
|
+
- lib/multiwoven/integrations/source/maria_db/icon.svg
|
|
570
574
|
- lib/multiwoven/integrations/source/postgresql/client.rb
|
|
571
575
|
- lib/multiwoven/integrations/source/postgresql/config/meta.json
|
|
572
576
|
- lib/multiwoven/integrations/source/postgresql/config/spec.json
|