multiwoven-integrations 0.1.30 → 0.1.31

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: de97ff2b2478ae4247a3027282f586bcc57c1d7db9de8aae698e8b83f0553f6a
4
- data.tar.gz: 861cdab018b60dfb1e8dd36b2f8a433f382eb358aa8a3645845185ac1f578019
3
+ metadata.gz: 8e3ffd041d8b25bfbae7c39e01491aad0d56d4a0e6b55707c57587b1a3b6993d
4
+ data.tar.gz: ed377a5d5c51060b7f288da572586194c1a846b5a10396c1a27f30cef960837d
5
5
  SHA512:
6
- metadata.gz: 53487427cd135f107cf333072798a53751e0d2fcc4bb92c5f42f8dafc88dfcd5c36f19c70f70daf4251f6b5c8770ef61992094258faeab0426eb73d22f8bc6ca
7
- data.tar.gz: 462bf61c0646edbd85ead0ecfc203cca0aa31a14a94bcdacb0a117b46b0c60af79d4ad5d5cc44be48cbe1697d01c23a934ce8ef241a78c6f69df870e4c28cdc7
6
+ metadata.gz: 4623a3a407aa222480eff99d1a862b0d59a1d1fe941255d0a31fd3539d5bda79faf87042eb281a9b0a15c254cc04ce0ed314d4f27bd8fa2798ce0e084afafde7
7
+ data.tar.gz: 8fc6cd88f2c293d2c84f0e923e25c9a57aea1fc21802e1d41dab5a3ee96e816deaefc635a4e06309b10128e3ce9ffc4b8e2d69274e95f1fd4e96f7315fe12bfb
@@ -12,9 +12,11 @@ module Multiwoven
12
12
  CONNECTOR_SPEC_PATH = "config/spec.json"
13
13
  CATALOG_SPEC_PATH = "config/catalog.json"
14
14
  SNOWFLAKE_MAC_DRIVER_PATH = "/opt/snowflake/snowflakeodbc/lib/universal/libSnowflake.dylib"
15
+ DATABRICKS_MAC_DRIVER_PATH = "/Library/simba/spark/lib/libsparkodbc_sb64-universal.dylib"
15
16
  MAIN_BRANCH_SHA = Git.ls_remote("https://github.com/Multiwoven/multiwoven-integrations")["head"][:sha]
16
17
 
17
18
  SNOWFLAKE_DRIVER_PATH = ENV["SNOWFLAKE_DRIVER_PATH"] || SNOWFLAKE_MAC_DRIVER_PATH
19
+ DATABRICKS_DRIVER_PATH = ENV["DATABRICKS_DRIVER_PATH"] || DATABRICKS_MAC_DRIVER_PATH
18
20
 
19
21
  # CONNECTORS
20
22
  KLAVIYO_AUTH_ENDPOINT = "https://a.klaviyo.com/api/lists/"
@@ -2,13 +2,14 @@
2
2
 
3
3
  module Multiwoven
4
4
  module Integrations
5
- VERSION = "0.1.30"
5
+ VERSION = "0.1.31"
6
6
 
7
7
  ENABLED_SOURCES = %w[
8
8
  Snowflake
9
9
  Redshift
10
10
  Bigquery
11
11
  Postgresql
12
+ Databricks
12
13
  ].freeze
13
14
 
14
15
  ENABLED_DESTINATIONS = %w[
@@ -0,0 +1,98 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven::Integrations::Source
4
+ module Databricks
5
+ include Multiwoven::Integrations::Core
6
+ class Client < SourceConnector
7
+ def check_connection(connection_config)
8
+ connection_config = connection_config.with_indifferent_access
9
+ create_connection(connection_config)
10
+ ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
11
+ rescue Sequel::DatabaseConnectionError => e
12
+ ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
13
+ end
14
+
15
+ def discover(connection_config)
16
+ connection_config = connection_config.with_indifferent_access
17
+ query = "SELECT table_name, column_name, data_type, is_nullable
18
+ FROM system.information_schema.columns
19
+ WHERE table_schema = \'#{connection_config[:schema]}\' AND table_catalog = \'#{connection_config[:catalog]}\'
20
+ ORDER BY table_name, ordinal_position;"
21
+
22
+ db = create_connection(connection_config)
23
+
24
+ records = []
25
+ db.fetch(query.gsub("\n", "")) do |row|
26
+ records << row
27
+ end
28
+ catalog = Catalog.new(streams: create_streams(records))
29
+ catalog.to_multiwoven_message
30
+ rescue StandardError => e
31
+ handle_exception(
32
+ "DATABRICKS:DISCOVER:EXCEPTION",
33
+ "error",
34
+ e
35
+ )
36
+ end
37
+
38
+ def read(sync_config)
39
+ connection_config = sync_config.source.connection_specification
40
+ connection_config = connection_config.with_indifferent_access
41
+ query = sync_config.model.query
42
+ query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
43
+
44
+ db = create_connection(connection_config)
45
+
46
+ query(db, query)
47
+ rescue StandardError => e
48
+ handle_exception(
49
+ "DATABRICKS:READ:EXCEPTION",
50
+ "error",
51
+ e
52
+ )
53
+ end
54
+
55
+ private
56
+
57
+ def query(connection, query)
58
+ records = []
59
+ connection.fetch(query) do |row|
60
+ records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
61
+ end
62
+ records
63
+ end
64
+
65
+ def create_connection(connection_config)
66
+ Sequel.odbc(drvconnect: generate_drvconnect(connection_config))
67
+ end
68
+
69
+ def generate_drvconnect(connection_config)
70
+ "Driver=#{DATABRICKS_DRIVER_PATH};
71
+ Host=#{connection_config[:host]};
72
+ PORT=#{connection_config[:port]};
73
+ SSL=1;
74
+ HTTPPath=#{connection_config[:http_path]};
75
+ PWD=#{connection_config[:access_token]};
76
+ UID=token;
77
+ ThriftTransport=2;AuthMech=3;
78
+ AllowSelfSignedServerCert=1;
79
+ CAIssuedCertNamesMismatch=1"
80
+ end
81
+
82
+ def create_streams(records)
83
+ group_by_table(records).map do |r|
84
+ Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
85
+ end
86
+ end
87
+
88
+ def group_by_table(records)
89
+ records.group_by { |entry| entry[:table_name] }.map do |table_name, columns|
90
+ {
91
+ tablename: table_name,
92
+ columns: columns.map { |column| { column_name: column[:column_name], type: column[:data_type], optional: column[:is_nullable] == "YES" } }
93
+ }
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
@@ -0,0 +1,16 @@
1
+ {
2
+ "data": {
3
+ "name": "Databricks",
4
+ "title": "Databricks",
5
+ "connector_type": "source",
6
+ "category": "Data Warehouse",
7
+ "documentation_url": "https://docs.mutliwoven.com",
8
+ "github_issue_label": "source-databricks",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
16
+
@@ -0,0 +1,60 @@
1
+ {
2
+ "documentation_url": "https://docs.multiwoven.com/integrations/sources/databricks",
3
+ "stream_type": "dynamic",
4
+ "connection_specification": {
5
+ "$schema": "http://json-schema.org/draft-07/schema#",
6
+ "title": "Databricks",
7
+ "type": "object",
8
+ "required": ["host", "port", "database", "schema"],
9
+ "properties": {
10
+ "title": "",
11
+ "type": "object",
12
+ "required": ["host", "port", "access_token", "http_path", "catalog", "schema"],
13
+ "properties": {
14
+ "host": {
15
+ "title": "Server Hostname",
16
+ "description": "Server host name for the Databricks Cluster. It is different from the SQL Endpoint Cluster.",
17
+ "type": "string",
18
+ "examples": "abc-12345678-wxyz.cloud.databricks.com",
19
+ "order": 0
20
+ },
21
+ "port": {
22
+ "title": "Port",
23
+ "description": "",
24
+ "type": "string",
25
+ "default": "443",
26
+ "order": 1
27
+ },
28
+ "access_token": {
29
+ "title": "Personal Access Token",
30
+ "description": "",
31
+ "type": "string",
32
+ "multiwoven_secret": true,
33
+ "order": 2
34
+ }
35
+ },
36
+ "http_path": {
37
+ "title": "HTTP Path",
38
+ "description": "",
39
+ "examples": "sql/protocolvx/o/1234567489/0000-1111111-abcd90",
40
+ "type": "string",
41
+ "order": 3
42
+ },
43
+ "catalog": {
44
+ "description": "The name of the catalog",
45
+ "default": "hive_metastore",
46
+ "type": "string",
47
+ "title": "Databricks catalog",
48
+ "order": 4
49
+ },
50
+ "schema": {
51
+ "description": " The default schema tables are written.",
52
+ "default": "default",
53
+ "type": "string",
54
+ "title": "Database schema",
55
+ "order": 5
56
+ }
57
+ }
58
+ }
59
+ }
60
+
@@ -0,0 +1,19 @@
1
+ <svg version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 40.1 42" style="enable-background:new 0 0 40.1 42;" xml:space="preserve">
2
+ <style type="text/css">
3
+ .st0{fill:#FF3621;}
4
+ </style>
5
+ <metadata>
6
+ <sfw xmlns="ns_sfw;">
7
+ <slices>
8
+ </slices>
9
+ <sliceSourceBounds bottomLeftOrigin="true" height="42" width="40.1" x="-69.1" y="-10.5">
10
+ </sliceSourceBounds>
11
+ </sfw>
12
+ </metadata>
13
+ <g>
14
+ <path class="st0" d="M40.1,31.1v-7.4l-0.8-0.5L20.1,33.7l-18.2-10l0-4.3l18.2,9.9l20.1-10.9v-7.3l-0.8-0.5L20.1,21.2L2.6,11.6
15
+ L20.1,2l14.1,7.7l1.1-0.6V8.3L20.1,0L0,10.9V12L20.1,23l18.2-10v4.4l-18.2,10L0.8,16.8L0,17.3v7.4l20.1,10.9l18.2-9.9v4.3l-18.2,10
16
+ L0.8,29.5L0,30v1.1L20.1,42L40.1,31.1z">
17
+ </path>
18
+ </g>
19
+ </svg>
@@ -37,6 +37,7 @@ require_relative "integrations/source/snowflake/client"
37
37
  require_relative "integrations/source/redshift/client"
38
38
  require_relative "integrations/source/bigquery/client"
39
39
  require_relative "integrations/source/postgresql/client"
40
+ require_relative "integrations/source/databricks/client"
40
41
 
41
42
  # Destination
42
43
  require_relative "integrations/destination/klaviyo/client"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiwoven-integrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.30
4
+ version: 0.1.31
5
5
  platform: ruby
6
6
  authors:
7
7
  - Subin T P
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-03-01 00:00:00.000000000 Z
11
+ date: 2024-03-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -363,6 +363,10 @@ files:
363
363
  - lib/multiwoven/integrations/source/bigquery/config/meta.json
364
364
  - lib/multiwoven/integrations/source/bigquery/config/spec.json
365
365
  - lib/multiwoven/integrations/source/bigquery/icon.svg
366
+ - lib/multiwoven/integrations/source/databricks/client.rb
367
+ - lib/multiwoven/integrations/source/databricks/config/meta.json
368
+ - lib/multiwoven/integrations/source/databricks/config/spec.json
369
+ - lib/multiwoven/integrations/source/databricks/icon.svg
366
370
  - lib/multiwoven/integrations/source/postgresql/client.rb
367
371
  - lib/multiwoven/integrations/source/postgresql/config/meta.json
368
372
  - lib/multiwoven/integrations/source/postgresql/config/spec.json