multiwoven-integrations 0.1.30 → 0.1.32
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/constants.rb +2 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/databricks/client.rb +98 -0
- data/lib/multiwoven/integrations/source/databricks/config/meta.json +16 -0
- data/lib/multiwoven/integrations/source/databricks/config/spec.json +55 -0
- data/lib/multiwoven/integrations/source/databricks/icon.svg +19 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4a489a81f1256c4b9b3330ff9a0bdf0e0de4ea6785adfb46ff5569598657c452
|
4
|
+
data.tar.gz: a87e81b569be6ff5f9a5828aaca24da6ca533f1b7f3e584c673186a4641b715a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3320dbac0e973597222396a356d32bcd40763b20a84cf022fff0b242efc170b7ccac7f8d977a7f37da66196e8def6a456a8e2a5bd4d116d3889bf4eca1b40e8e
|
7
|
+
data.tar.gz: 631c60c420a342bcfc5bf0f0e8f48905fd0de4466843d87a24769862b3d1ace7b9d52f419d3ffcdd8e06560eaa8a533408236967db105e5f708e2e30bebf53b3
|
@@ -12,9 +12,11 @@ module Multiwoven
|
|
12
12
|
CONNECTOR_SPEC_PATH = "config/spec.json"
|
13
13
|
CATALOG_SPEC_PATH = "config/catalog.json"
|
14
14
|
SNOWFLAKE_MAC_DRIVER_PATH = "/opt/snowflake/snowflakeodbc/lib/universal/libSnowflake.dylib"
|
15
|
+
DATABRICKS_MAC_DRIVER_PATH = "/Library/simba/spark/lib/libsparkodbc_sb64-universal.dylib"
|
15
16
|
MAIN_BRANCH_SHA = Git.ls_remote("https://github.com/Multiwoven/multiwoven-integrations")["head"][:sha]
|
16
17
|
|
17
18
|
SNOWFLAKE_DRIVER_PATH = ENV["SNOWFLAKE_DRIVER_PATH"] || SNOWFLAKE_MAC_DRIVER_PATH
|
19
|
+
DATABRICKS_DRIVER_PATH = ENV["DATABRICKS_DRIVER_PATH"] || DATABRICKS_MAC_DRIVER_PATH
|
18
20
|
|
19
21
|
# CONNECTORS
|
20
22
|
KLAVIYO_AUTH_ENDPOINT = "https://a.klaviyo.com/api/lists/"
|
@@ -0,0 +1,98 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Multiwoven::Integrations::Source
|
4
|
+
module Databricks
|
5
|
+
include Multiwoven::Integrations::Core
|
6
|
+
class Client < SourceConnector
|
7
|
+
def check_connection(connection_config)
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
9
|
+
create_connection(connection_config)
|
10
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
11
|
+
rescue Sequel::DatabaseConnectionError => e
|
12
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
13
|
+
end
|
14
|
+
|
15
|
+
def discover(connection_config)
|
16
|
+
connection_config = connection_config.with_indifferent_access
|
17
|
+
query = "SELECT table_name, column_name, data_type, is_nullable
|
18
|
+
FROM system.information_schema.columns
|
19
|
+
WHERE table_schema = \'#{connection_config[:schema]}\' AND table_catalog = \'#{connection_config[:catalog]}\'
|
20
|
+
ORDER BY table_name, ordinal_position;"
|
21
|
+
|
22
|
+
db = create_connection(connection_config)
|
23
|
+
|
24
|
+
records = []
|
25
|
+
db.fetch(query.gsub("\n", "")) do |row|
|
26
|
+
records << row
|
27
|
+
end
|
28
|
+
catalog = Catalog.new(streams: create_streams(records))
|
29
|
+
catalog.to_multiwoven_message
|
30
|
+
rescue StandardError => e
|
31
|
+
handle_exception(
|
32
|
+
"DATABRICKS:DISCOVER:EXCEPTION",
|
33
|
+
"error",
|
34
|
+
e
|
35
|
+
)
|
36
|
+
end
|
37
|
+
|
38
|
+
def read(sync_config)
|
39
|
+
connection_config = sync_config.source.connection_specification
|
40
|
+
connection_config = connection_config.with_indifferent_access
|
41
|
+
query = sync_config.model.query
|
42
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
43
|
+
|
44
|
+
db = create_connection(connection_config)
|
45
|
+
|
46
|
+
query(db, query)
|
47
|
+
rescue StandardError => e
|
48
|
+
handle_exception(
|
49
|
+
"DATABRICKS:READ:EXCEPTION",
|
50
|
+
"error",
|
51
|
+
e
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def query(connection, query)
|
58
|
+
records = []
|
59
|
+
connection.fetch(query) do |row|
|
60
|
+
records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
61
|
+
end
|
62
|
+
records
|
63
|
+
end
|
64
|
+
|
65
|
+
def create_connection(connection_config)
|
66
|
+
Sequel.odbc(drvconnect: generate_drvconnect(connection_config))
|
67
|
+
end
|
68
|
+
|
69
|
+
def generate_drvconnect(connection_config)
|
70
|
+
"Driver=#{DATABRICKS_DRIVER_PATH};
|
71
|
+
Host=#{connection_config[:host]};
|
72
|
+
PORT=#{connection_config[:port]};
|
73
|
+
SSL=1;
|
74
|
+
HTTPPath=#{connection_config[:http_path]};
|
75
|
+
PWD=#{connection_config[:access_token]};
|
76
|
+
UID=token;
|
77
|
+
ThriftTransport=2;AuthMech=3;
|
78
|
+
AllowSelfSignedServerCert=1;
|
79
|
+
CAIssuedCertNamesMismatch=1"
|
80
|
+
end
|
81
|
+
|
82
|
+
def create_streams(records)
|
83
|
+
group_by_table(records).map do |r|
|
84
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def group_by_table(records)
|
89
|
+
records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
|
90
|
+
{
|
91
|
+
tablename: table_name,
|
92
|
+
columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
|
93
|
+
}
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
{
|
2
|
+
"data": {
|
3
|
+
"name": "Databricks",
|
4
|
+
"title": "Databricks",
|
5
|
+
"connector_type": "source",
|
6
|
+
"category": "Data Warehouse",
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
8
|
+
"github_issue_label": "source-databricks",
|
9
|
+
"icon": "icon.svg",
|
10
|
+
"license": "MIT",
|
11
|
+
"release_stage": "alpha",
|
12
|
+
"support_level": "community",
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
14
|
+
}
|
15
|
+
}
|
16
|
+
|
@@ -0,0 +1,55 @@
|
|
1
|
+
{
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/databricks",
|
3
|
+
"stream_type": "dynamic",
|
4
|
+
"connection_specification": {
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
6
|
+
"title": "Databricks",
|
7
|
+
"type": "object",
|
8
|
+
"required": ["host", "port", "http_path", "catalog", "schema"],
|
9
|
+
"properties": {
|
10
|
+
"host": {
|
11
|
+
"title": "Server Hostname",
|
12
|
+
"description": "Server host name for the Databricks Cluster. It is different from the SQL Endpoint Cluster.",
|
13
|
+
"type": "string",
|
14
|
+
"examples": ["abc-12345678-wxyz.cloud.databricks.com"],
|
15
|
+
"order": 0
|
16
|
+
},
|
17
|
+
"port": {
|
18
|
+
"title": "Port",
|
19
|
+
"description": "",
|
20
|
+
"type": "string",
|
21
|
+
"default": "443",
|
22
|
+
"order": 1
|
23
|
+
},
|
24
|
+
"access_token": {
|
25
|
+
"title": "Personal Access Token",
|
26
|
+
"description": "",
|
27
|
+
"type": "string",
|
28
|
+
"multiwoven_secret": true,
|
29
|
+
"order": 2
|
30
|
+
},
|
31
|
+
"http_path": {
|
32
|
+
"title": "HTTP Path",
|
33
|
+
"description": "",
|
34
|
+
"examples": ["sql/protocolvx/o/1234567489/0000-1111111-abcd90"],
|
35
|
+
"type": "string",
|
36
|
+
"order": 3
|
37
|
+
},
|
38
|
+
"catalog": {
|
39
|
+
"description": "The name of the catalog",
|
40
|
+
"default": "hive_metastore",
|
41
|
+
"type": "string",
|
42
|
+
"title": "Databricks catalog",
|
43
|
+
"order": 4
|
44
|
+
},
|
45
|
+
"schema": {
|
46
|
+
"description": "The default schema tables are written.",
|
47
|
+
"default": "default",
|
48
|
+
"type": "string",
|
49
|
+
"title": "Database schema",
|
50
|
+
"order": 5
|
51
|
+
}
|
52
|
+
}
|
53
|
+
}
|
54
|
+
}
|
55
|
+
|
@@ -0,0 +1,19 @@
|
|
1
|
+
<svg version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 40.1 42" style="enable-background:new 0 0 40.1 42;" xml:space="preserve">
|
2
|
+
<style type="text/css">
|
3
|
+
.st0{fill:#FF3621;}
|
4
|
+
</style>
|
5
|
+
<metadata>
|
6
|
+
<sfw xmlns="ns_sfw;">
|
7
|
+
<slices>
|
8
|
+
</slices>
|
9
|
+
<sliceSourceBounds bottomLeftOrigin="true" height="42" width="40.1" x="-69.1" y="-10.5">
|
10
|
+
</sliceSourceBounds>
|
11
|
+
</sfw>
|
12
|
+
</metadata>
|
13
|
+
<g>
|
14
|
+
<path class="st0" d="M40.1,31.1v-7.4l-0.8-0.5L20.1,33.7l-18.2-10l0-4.3l18.2,9.9l20.1-10.9v-7.3l-0.8-0.5L20.1,21.2L2.6,11.6
|
15
|
+
L20.1,2l14.1,7.7l1.1-0.6V8.3L20.1,0L0,10.9V12L20.1,23l18.2-10v4.4l-18.2,10L0.8,16.8L0,17.3v7.4l20.1,10.9l18.2-9.9v4.3l-18.2,10
|
16
|
+
L0.8,29.5L0,30v1.1L20.1,42L40.1,31.1z">
|
17
|
+
</path>
|
18
|
+
</g>
|
19
|
+
</svg>
|
@@ -37,6 +37,7 @@ require_relative "integrations/source/snowflake/client"
|
|
37
37
|
require_relative "integrations/source/redshift/client"
|
38
38
|
require_relative "integrations/source/bigquery/client"
|
39
39
|
require_relative "integrations/source/postgresql/client"
|
40
|
+
require_relative "integrations/source/databricks/client"
|
40
41
|
|
41
42
|
# Destination
|
42
43
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: multiwoven-integrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.32
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Subin T P
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-03-
|
11
|
+
date: 2024-03-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -363,6 +363,10 @@ files:
|
|
363
363
|
- lib/multiwoven/integrations/source/bigquery/config/meta.json
|
364
364
|
- lib/multiwoven/integrations/source/bigquery/config/spec.json
|
365
365
|
- lib/multiwoven/integrations/source/bigquery/icon.svg
|
366
|
+
- lib/multiwoven/integrations/source/databricks/client.rb
|
367
|
+
- lib/multiwoven/integrations/source/databricks/config/meta.json
|
368
|
+
- lib/multiwoven/integrations/source/databricks/config/spec.json
|
369
|
+
- lib/multiwoven/integrations/source/databricks/icon.svg
|
366
370
|
- lib/multiwoven/integrations/source/postgresql/client.rb
|
367
371
|
- lib/multiwoven/integrations/source/postgresql/config/meta.json
|
368
372
|
- lib/multiwoven/integrations/source/postgresql/config/spec.json
|