multiwoven-integrations 0.4.1 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/constants.rb +2 -0
- data/lib/multiwoven/integrations/destination/databricks_lakehouse/client.rb +147 -0
- data/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json +44 -0
- data/lib/multiwoven/integrations/destination/databricks_lakehouse/icon.svg +65 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/amazon_s3/client.rb +2 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8c247bae51bf614083121b6ed6f8b3ab674511bbc931bbc7c224ad857fd7443a
|
|
4
|
+
data.tar.gz: 3e8f42157f3c79293e9f896059f9b2d6b6c9ebc2f6a9bf55bbb1af0d2769417c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e44256ec88e53ed9942c91823772f9165e74e15f94c2013a850af18e165c646a94c41619eecbbc3c38ab4b3e0a5205bae44ded87b178bd27aaacc6a878edef7e
|
|
7
|
+
data.tar.gz: d06f6937d55c691ccbcb145bafc902b762a9e2782f7446fb3391a3c65d619e4a2c0be1439c1e39efc87f6bb0797115ae9077fc6f30d003d1d518e0ce2ed98536
|
|
@@ -16,6 +16,8 @@ module Multiwoven
|
|
|
16
16
|
JSON_SCHEMA_URL = "https://json-schema.org/draft-07/schema#"
|
|
17
17
|
|
|
18
18
|
# CONNECTORS
|
|
19
|
+
INSTALL_HTTPFS_QUERY = ENV["INSTALL_HTTPFS_QUERY"] || "INSTALL HTTPFS; LOAD HTTPFS;"
|
|
20
|
+
|
|
19
21
|
KLAVIYO_AUTH_ENDPOINT = "https://a.klaviyo.com/api/lists/"
|
|
20
22
|
KLAVIYO_AUTH_PAYLOAD = {
|
|
21
23
|
data: {
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations
|
|
5
|
+
module Destination
|
|
6
|
+
module DatabricksLakehouse
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < DestinationConnector
|
|
9
|
+
MAX_CHUNK_SIZE = 10
|
|
10
|
+
def check_connection(connection_config)
|
|
11
|
+
connection_config = connection_config.with_indifferent_access
|
|
12
|
+
db = create_connection(connection_config)
|
|
13
|
+
response = db.get("/api/2.0/clusters/list")
|
|
14
|
+
if response.status == 200
|
|
15
|
+
success_status
|
|
16
|
+
else
|
|
17
|
+
failure_status(nil)
|
|
18
|
+
end
|
|
19
|
+
rescue StandardError => e
|
|
20
|
+
handle_exception(e, {
|
|
21
|
+
context: "DATABRICKS:LAKEHOUSE:CHECK_CONNECTION:EXCEPTION",
|
|
22
|
+
type: "error"
|
|
23
|
+
})
|
|
24
|
+
failure_status(e)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def discover(connection_config)
|
|
28
|
+
connection_config = connection_config.with_indifferent_access
|
|
29
|
+
table_query = "SHOW TABLES IN #{connection_config[:catalog]}.#{connection_config[:schema]};"
|
|
30
|
+
db = create_connection(connection_config)
|
|
31
|
+
records = []
|
|
32
|
+
table_response = db.post("/api/2.0/sql/statements", generate_body(connection_config[:warehouse_id], table_query).to_json)
|
|
33
|
+
table_response_body = JSON.parse(table_response.body)
|
|
34
|
+
table_response_body["result"]["data_array"].each do |table|
|
|
35
|
+
table_name = table[1]
|
|
36
|
+
query = "DESCRIBE TABLE #{connection_config[:catalog]}.#{connection_config[:schema]}.#{table_name};"
|
|
37
|
+
column_response = db.post("/api/2.0/sql/statements", generate_body(connection_config[:warehouse_id], query).to_json)
|
|
38
|
+
column_response_body = JSON.parse(column_response.body)
|
|
39
|
+
records << [table_name, column_response_body["result"]["data_array"]]
|
|
40
|
+
end
|
|
41
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
42
|
+
catalog.to_multiwoven_message
|
|
43
|
+
rescue StandardError => e
|
|
44
|
+
handle_exception(
|
|
45
|
+
"DATABRICKS:LAKEHOUSE:DISCOVER:EXCEPTION",
|
|
46
|
+
"error",
|
|
47
|
+
e
|
|
48
|
+
)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def write(sync_config, records, action = "destination_insert")
|
|
52
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
53
|
+
table_name = "#{connection_config[:catalog]}.#{connection_config[:schema]}.#{sync_config.stream.name}"
|
|
54
|
+
primary_key = sync_config.model.primary_key
|
|
55
|
+
db = create_connection(connection_config)
|
|
56
|
+
write_success = 0
|
|
57
|
+
write_failure = 0
|
|
58
|
+
log_message_array = []
|
|
59
|
+
|
|
60
|
+
records.each do |record|
|
|
61
|
+
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
|
|
62
|
+
logger.debug("DATABRICKS:LAKEHOUSE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
|
|
63
|
+
begin
|
|
64
|
+
arg = ["/api/2.0/sql/statements", generate_body(connection_config[:warehouse_id], query)]
|
|
65
|
+
response = db.post("/api/2.0/sql/statements", generate_body(connection_config[:warehouse_id], query).to_json)
|
|
66
|
+
if response.status == 200
|
|
67
|
+
write_success += 1
|
|
68
|
+
else
|
|
69
|
+
write_failure += 1
|
|
70
|
+
end
|
|
71
|
+
log_message_array << log_request_response("info", arg, response)
|
|
72
|
+
rescue StandardError => e
|
|
73
|
+
handle_exception(e, {
|
|
74
|
+
context: "DATABRICKS:LAKEHOUSE:RECORD:WRITE:EXCEPTION",
|
|
75
|
+
type: "error",
|
|
76
|
+
sync_id: sync_config.sync_id,
|
|
77
|
+
sync_run_id: sync_config.sync_run_id
|
|
78
|
+
})
|
|
79
|
+
write_failure += 1
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
tracking_message(write_success, write_failure)
|
|
83
|
+
rescue StandardError => e
|
|
84
|
+
handle_exception(e, {
|
|
85
|
+
context: "DATABRICKS:LAKEHOUSE:RECORD:WRITE:EXCEPTION",
|
|
86
|
+
type: "error",
|
|
87
|
+
sync_id: sync_config.sync_id,
|
|
88
|
+
sync_run_id: sync_config.sync_run_id
|
|
89
|
+
})
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private
|
|
93
|
+
|
|
94
|
+
def create_connection(connection_config)
|
|
95
|
+
Faraday.new(url: connection_config[:host]) do |conn|
|
|
96
|
+
conn.headers["Authorization"] = "Bearer #{connection_config[:api_token]}"
|
|
97
|
+
conn.headers["Content-Type"] = "application/json"
|
|
98
|
+
conn.adapter Faraday.default_adapter
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def generate_body(warehouse_id, query)
|
|
103
|
+
{
|
|
104
|
+
warehouse_id: warehouse_id,
|
|
105
|
+
statement: query,
|
|
106
|
+
wait_timeout: "15s"
|
|
107
|
+
}
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def create_streams(records)
|
|
111
|
+
message = []
|
|
112
|
+
group_by_table(records).each_value do |r|
|
|
113
|
+
message << Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
114
|
+
end
|
|
115
|
+
message
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def group_by_table(records)
|
|
119
|
+
result = {}
|
|
120
|
+
records.each_with_index do |entries, index|
|
|
121
|
+
table_name = records[index][0]
|
|
122
|
+
column = []
|
|
123
|
+
entry_data = entries[1]
|
|
124
|
+
entry_data.each do |entry|
|
|
125
|
+
column << {
|
|
126
|
+
column_name: entry[0],
|
|
127
|
+
data_type: entry[1],
|
|
128
|
+
is_nullable: true
|
|
129
|
+
}
|
|
130
|
+
end
|
|
131
|
+
result[index] ||= {}
|
|
132
|
+
result[index][:tablename] = table_name
|
|
133
|
+
result[index][:columns] = column
|
|
134
|
+
end
|
|
135
|
+
result
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def tracking_message(success, failure)
|
|
139
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
140
|
+
success: success, failed: failure
|
|
141
|
+
).to_multiwoven_message
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "DatabricksLakehouse",
|
|
4
|
+
"title": "Databricks Lakehouse",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Marketing Automation",
|
|
7
|
+
"documentation_url": "https://docs.multiwoven.com/destinations/databricks_lakehouse",
|
|
8
|
+
"github_issue_label": "destination-databricks-lakehouse",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/databrick_lakehouse",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Databricks Lakehouse",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["host", "api_token", "warehouse_id", "catalog", "schema"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"host": {
|
|
11
|
+
"description": "The databrick lakehouse host domain.",
|
|
12
|
+
"type": "string",
|
|
13
|
+
"title": "Host",
|
|
14
|
+
"order": 0
|
|
15
|
+
},
|
|
16
|
+
"api_token": {
|
|
17
|
+
"description": "The databrick lakehouse api token.",
|
|
18
|
+
"type": "string",
|
|
19
|
+
"multiwoven_secret": true,
|
|
20
|
+
"title": "API Token",
|
|
21
|
+
"order": 1
|
|
22
|
+
},"warehouse_id": {
|
|
23
|
+
"description": "The databrick lakehouse warehouse ID.",
|
|
24
|
+
"type": "string",
|
|
25
|
+
"title": "Warehouse ID",
|
|
26
|
+
"order": 2
|
|
27
|
+
},
|
|
28
|
+
"catalog": {
|
|
29
|
+
"description": "The name of the catalog",
|
|
30
|
+
"default": "hive_metastore",
|
|
31
|
+
"type": "string",
|
|
32
|
+
"title": "Databricks catalog",
|
|
33
|
+
"order": 3
|
|
34
|
+
},
|
|
35
|
+
"schema": {
|
|
36
|
+
"description": "The default schema tables are written.",
|
|
37
|
+
"default": "default",
|
|
38
|
+
"type": "string",
|
|
39
|
+
"title": "Database schema",
|
|
40
|
+
"order": 4
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
|
2
|
+
<!-- Generator: Adobe Illustrator 18.0.0, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
|
3
|
+
|
|
4
|
+
<svg
|
|
5
|
+
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
|
6
|
+
xmlns:cc="http://creativecommons.org/ns#"
|
|
7
|
+
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
8
|
+
xmlns:svg="http://www.w3.org/2000/svg"
|
|
9
|
+
xmlns="http://www.w3.org/2000/svg"
|
|
10
|
+
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
|
11
|
+
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
|
12
|
+
version="1.1"
|
|
13
|
+
id="Layer_1"
|
|
14
|
+
x="0px"
|
|
15
|
+
y="0px"
|
|
16
|
+
viewBox="0 0 64 64"
|
|
17
|
+
enable-background="new 0 0 640 96"
|
|
18
|
+
xml:space="preserve"
|
|
19
|
+
sodipodi:docname="databricks-icon.svg"
|
|
20
|
+
width="64"
|
|
21
|
+
height="64"
|
|
22
|
+
inkscape:version="0.92.4 (5da689c313, 2019-01-14)"><metadata
|
|
23
|
+
id="metadata45"><rdf:RDF><cc:Work
|
|
24
|
+
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
|
|
25
|
+
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
|
|
26
|
+
id="defs43" /><sodipodi:namedview
|
|
27
|
+
pagecolor="#ffffff"
|
|
28
|
+
bordercolor="#666666"
|
|
29
|
+
borderopacity="1"
|
|
30
|
+
objecttolerance="10"
|
|
31
|
+
gridtolerance="10"
|
|
32
|
+
guidetolerance="10"
|
|
33
|
+
inkscape:pageopacity="0"
|
|
34
|
+
inkscape:pageshadow="2"
|
|
35
|
+
inkscape:window-width="1920"
|
|
36
|
+
inkscape:window-height="1001"
|
|
37
|
+
id="namedview41"
|
|
38
|
+
showgrid="false"
|
|
39
|
+
inkscape:zoom="2.996365"
|
|
40
|
+
inkscape:cx="53.009829"
|
|
41
|
+
inkscape:cy="48"
|
|
42
|
+
inkscape:window-x="-9"
|
|
43
|
+
inkscape:window-y="-9"
|
|
44
|
+
inkscape:window-maximized="1"
|
|
45
|
+
inkscape:current-layer="Layer_1" />
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
<g
|
|
50
|
+
id="g50"
|
|
51
|
+
transform="matrix(0.75294118,0,0,0.75294118,1.0541175,18.823529)"><path
|
|
52
|
+
style="fill:#db1905"
|
|
53
|
+
inkscape:connector-curvature="0"
|
|
54
|
+
id="path2"
|
|
55
|
+
d="M 0,24.8 V 38.5 L 41.1,60 82.2,38.5 v -13.7 0 L 67.7,17.4 82.2,9.8 v -13.6 0 0 -0.1 L 82.1,-3.8 41.1,-25 0.1,-3.9 H 0 V 9.8 L 14.5,17.4 0,24.8" /><polygon
|
|
56
|
+
transform="translate(0,-36)"
|
|
57
|
+
style="fill:#ff5224"
|
|
58
|
+
id="polygon4"
|
|
59
|
+
points="0,60.8 41.1,82.3 82.2,60.8 67.7,53.4 41.1,67.3 14.5,53.4 " /><polygon
|
|
60
|
+
transform="translate(0,-36)"
|
|
61
|
+
style="fill:#ff5224"
|
|
62
|
+
id="polygon6"
|
|
63
|
+
points="82.2,32.2 41.1,53.7 0,32.2 41.1,11 " /></g>
|
|
64
|
+
|
|
65
|
+
</svg>
|
|
@@ -74,6 +74,8 @@ module Multiwoven::Integrations::Source
|
|
|
74
74
|
@session_name = "preview-#{connection_config[:region]}-#{connection_config[:bucket]}" if @session_name.to_s.empty?
|
|
75
75
|
auth_data = get_auth_data(connection_config)
|
|
76
76
|
conn = DuckDB::Database.open.connect
|
|
77
|
+
# Install and/or Load the HTTPFS extension
|
|
78
|
+
conn.execute(INSTALL_HTTPFS_QUERY)
|
|
77
79
|
# Set up S3 configuration
|
|
78
80
|
secret_query = "
|
|
79
81
|
CREATE SECRET amazons3_source (
|
|
@@ -77,6 +77,7 @@ require_relative "integrations/destination/http/client"
|
|
|
77
77
|
require_relative "integrations/destination/zendesk/client"
|
|
78
78
|
require_relative "integrations/destination/iterable/client"
|
|
79
79
|
require_relative "integrations/destination/maria_db/client"
|
|
80
|
+
require_relative "integrations/destination/databricks_lakehouse/client"
|
|
80
81
|
|
|
81
82
|
module Multiwoven
|
|
82
83
|
module Integrations
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-07-
|
|
11
|
+
date: 2024-07-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -492,6 +492,10 @@ files:
|
|
|
492
492
|
- lib/multiwoven/integrations/destination/airtable/config/spec.json
|
|
493
493
|
- lib/multiwoven/integrations/destination/airtable/icon.svg
|
|
494
494
|
- lib/multiwoven/integrations/destination/airtable/schema_helper.rb
|
|
495
|
+
- lib/multiwoven/integrations/destination/databricks_lakehouse/client.rb
|
|
496
|
+
- lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json
|
|
497
|
+
- lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json
|
|
498
|
+
- lib/multiwoven/integrations/destination/databricks_lakehouse/icon.svg
|
|
495
499
|
- lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb
|
|
496
500
|
- lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json
|
|
497
501
|
- lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json
|