multiwoven-integrations 0.8.6 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/base_connector.rb +8 -0
- data/lib/multiwoven/integrations/core/constants.rb +3 -0
- data/lib/multiwoven/integrations/core/destination_connector.rb +0 -8
- data/lib/multiwoven/integrations/protocol/protocol.rb +3 -3
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/databrics_model/client.rb +87 -0
- data/lib/multiwoven/integrations/source/databrics_model/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/source/databrics_model/config/meta.json +16 -0
- data/lib/multiwoven/integrations/source/databrics_model/config/spec.json +47 -0
- data/lib/multiwoven/integrations/source/databrics_model/icon.svg +19 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: eb63f96519eb9556503c62850837b135658b10f62c3ec9c5dc8e9964dbc5b3fb
|
|
4
|
+
data.tar.gz: 59eba81bcc6f9b048971d8a96e4260e006d6ff1e00e72a6c4868ed4c17cebedf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2b5b50562d426959dd482948d8f4e499da80066ff7402af22a1cc9f7359d63236c6fb9d4c0a86b2ba669047ceeccedc20d1946d52d109243ef37de795d68d644
|
|
7
|
+
data.tar.gz: cbeacf1343d598cdc08ace21ccb52ddf1ad4862e468147a247b0d56fcebf6c651a53b13e3f11d08b0c280df3435b4fb9d9892aa88e060ab2b62fd0c75725cc98
|
|
@@ -66,6 +66,14 @@ module Multiwoven
|
|
|
66
66
|
message = error&.message || "failed"
|
|
67
67
|
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: message).to_multiwoven_message
|
|
68
68
|
end
|
|
69
|
+
|
|
70
|
+
def auth_headers(access_token)
|
|
71
|
+
{
|
|
72
|
+
"Accept" => "application/json",
|
|
73
|
+
"Authorization" => "Bearer #{access_token}",
|
|
74
|
+
"Content-Type" => "application/json"
|
|
75
|
+
}
|
|
76
|
+
end
|
|
69
77
|
end
|
|
70
78
|
end
|
|
71
79
|
end
|
|
@@ -47,6 +47,9 @@ module Multiwoven
|
|
|
47
47
|
MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
|
|
48
48
|
"workbook/worksheets/%<sheet_name>s/range(address='A1:Z1')/usedRange?$select=values"
|
|
49
49
|
|
|
50
|
+
DATABRICKS_HEALTH_URL = "https://%<databricks_host>s/api/2.0/serving-endpoints/%<endpoint_name>s"
|
|
51
|
+
DATABRICKS_SERVING_URL = "https://%<databricks_host>s/serving-endpoints/%<endpoint_name>s/invocations"
|
|
52
|
+
|
|
50
53
|
AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
|
|
51
54
|
AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
|
|
52
55
|
|
|
@@ -15,14 +15,6 @@ module Multiwoven
|
|
|
15
15
|
success: success, failed: failure, logs: log_message_array
|
|
16
16
|
).to_multiwoven_message
|
|
17
17
|
end
|
|
18
|
-
|
|
19
|
-
def auth_headers(access_token)
|
|
20
|
-
{
|
|
21
|
-
"Accept" => "application/json",
|
|
22
|
-
"Authorization" => "Bearer #{access_token}",
|
|
23
|
-
"Content-Type" => "application/json"
|
|
24
|
-
}
|
|
25
|
-
end
|
|
26
18
|
end
|
|
27
19
|
end
|
|
28
20
|
end
|
|
@@ -10,10 +10,10 @@ module Multiwoven
|
|
|
10
10
|
SyncStatus = Types::String.enum("started", "running", "complete", "incomplete")
|
|
11
11
|
DestinationSyncMode = Types::String.enum("insert", "upsert")
|
|
12
12
|
ConnectorType = Types::String.enum("source", "destination")
|
|
13
|
-
ConnectorQueryType = Types::String.enum("raw_sql", "soql")
|
|
14
|
-
ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector")
|
|
13
|
+
ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml")
|
|
14
|
+
ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml")
|
|
15
15
|
ConnectionStatusType = Types::String.enum("succeeded", "failed")
|
|
16
|
-
StreamType = Types::String.enum("static", "dynamic")
|
|
16
|
+
StreamType = Types::String.enum("static", "dynamic", "user_defined")
|
|
17
17
|
StreamAction = Types::String.enum("fetch", "create", "update", "delete")
|
|
18
18
|
MultiwovenMessageType = Types::String.enum(
|
|
19
19
|
"record", "log", "connector_spec",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.9.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -16,6 +16,7 @@ module Multiwoven
|
|
|
16
16
|
AmazonS3
|
|
17
17
|
MariaDB
|
|
18
18
|
Oracle
|
|
19
|
+
DatabricksModel
|
|
19
20
|
].freeze
|
|
20
21
|
|
|
21
22
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module DatabricksModel
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
url = build_url(DATABRICKS_HEALTH_URL, connection_config)
|
|
10
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
11
|
+
url,
|
|
12
|
+
HTTP_GET,
|
|
13
|
+
headers: auth_headers(connection_config[:token])
|
|
14
|
+
)
|
|
15
|
+
if success?(response)
|
|
16
|
+
success_status
|
|
17
|
+
else
|
|
18
|
+
failure_status(nil)
|
|
19
|
+
end
|
|
20
|
+
rescue StandardError => e
|
|
21
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def discover(_connection_config = nil)
|
|
25
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
26
|
+
catalog = build_catalog(catalog_json)
|
|
27
|
+
catalog.to_multiwoven_message
|
|
28
|
+
rescue StandardError => e
|
|
29
|
+
handle_exception(e, {
|
|
30
|
+
context: "DATABRICKS MODEL:DISCOVER:EXCEPTION",
|
|
31
|
+
type: "error"
|
|
32
|
+
})
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def read(sync_config)
|
|
36
|
+
connection_config = sync_config.source.connection_specification
|
|
37
|
+
connection_config = connection_config.with_indifferent_access
|
|
38
|
+
# The server checks the ConnectorQueryType.
|
|
39
|
+
# If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
|
|
40
|
+
# This query is then sent to the AI/ML model.
|
|
41
|
+
payload = JSON.parse(sync_config.model.query)
|
|
42
|
+
run_model(connection_config, payload)
|
|
43
|
+
rescue StandardError => e
|
|
44
|
+
handle_exception(e, {
|
|
45
|
+
context: "DATABRICKS MODEL:READ:EXCEPTION",
|
|
46
|
+
type: "error"
|
|
47
|
+
})
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def run_model(connection_config, payload)
|
|
53
|
+
connection_config = connection_config.with_indifferent_access
|
|
54
|
+
|
|
55
|
+
url = build_url(DATABRICKS_SERVING_URL, connection_config)
|
|
56
|
+
token = connection_config[:token]
|
|
57
|
+
response = send_request(url, token, payload)
|
|
58
|
+
process_response(response)
|
|
59
|
+
rescue StandardError => e
|
|
60
|
+
handle_exception(e, context: "DATABRICKS MODEL:RUN_MODEL:EXCEPTION", type: "error")
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def process_response(response)
|
|
64
|
+
if success?(response)
|
|
65
|
+
data = JSON.parse(response.body)
|
|
66
|
+
[RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
|
|
67
|
+
else
|
|
68
|
+
create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "request failed")
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def build_url(url, connection_config)
|
|
73
|
+
format(url, databricks_host: connection_config[:databricks_host],
|
|
74
|
+
endpoint_name: connection_config[:endpoint_name])
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def send_request(url, token, payload)
|
|
78
|
+
Multiwoven::Integrations::Core::HttpClient.request(
|
|
79
|
+
url,
|
|
80
|
+
HTTP_POST,
|
|
81
|
+
payload: payload,
|
|
82
|
+
headers: auth_headers(token)
|
|
83
|
+
)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "DatabricksModel",
|
|
4
|
+
"title": "Databricks Model",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "AI Model",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "source-databricks-foundation",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/databricks",
|
|
3
|
+
"stream_type": "user_defined",
|
|
4
|
+
"connector_query_type": "ai_ml",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Databricks Model",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["databricks_host", "token", "model_name"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"databricks_host": {
|
|
12
|
+
"title": "databricks_host",
|
|
13
|
+
"description": "databricks_host",
|
|
14
|
+
"type": "string",
|
|
15
|
+
"examples": ["app.databricks.com"],
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"token": {
|
|
19
|
+
"title": "Databricks Token",
|
|
20
|
+
"description": "personal access token",
|
|
21
|
+
"type": "string",
|
|
22
|
+
"multiwoven_secret": true,
|
|
23
|
+
"order": 1
|
|
24
|
+
},
|
|
25
|
+
"endpoint": {
|
|
26
|
+
"title": "Endpoint name",
|
|
27
|
+
"description": "Endpoint name",
|
|
28
|
+
"examples": ["databricks-dbrx-instruct"],
|
|
29
|
+
"type": "string",
|
|
30
|
+
"order": 2
|
|
31
|
+
},
|
|
32
|
+
"request_format": {
|
|
33
|
+
"title": "Request Format",
|
|
34
|
+
"description": "Sample Request Format",
|
|
35
|
+
"type": "string",
|
|
36
|
+
"order": 3
|
|
37
|
+
},
|
|
38
|
+
"response_format": {
|
|
39
|
+
"title": "Response Format",
|
|
40
|
+
"description": "Sample Response Format",
|
|
41
|
+
"type": "string",
|
|
42
|
+
"order": 4
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
<svg version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 40.1 42" style="enable-background:new 0 0 40.1 42;" xml:space="preserve">
|
|
2
|
+
<style type="text/css">
|
|
3
|
+
.st0{fill:#FF3621;}
|
|
4
|
+
</style>
|
|
5
|
+
<metadata>
|
|
6
|
+
<sfw xmlns="ns_sfw;">
|
|
7
|
+
<slices>
|
|
8
|
+
</slices>
|
|
9
|
+
<sliceSourceBounds bottomLeftOrigin="true" height="42" width="40.1" x="-69.1" y="-10.5">
|
|
10
|
+
</sliceSourceBounds>
|
|
11
|
+
</sfw>
|
|
12
|
+
</metadata>
|
|
13
|
+
<g>
|
|
14
|
+
<path class="st0" d="M40.1,31.1v-7.4l-0.8-0.5L20.1,33.7l-18.2-10l0-4.3l18.2,9.9l20.1-10.9v-7.3l-0.8-0.5L20.1,21.2L2.6,11.6
|
|
15
|
+
L20.1,2l14.1,7.7l1.1-0.6V8.3L20.1,0L0,10.9V12L20.1,23l18.2-10v4.4l-18.2,10L0.8,16.8L0,17.3v7.4l20.1,10.9l18.2-9.9v4.3l-18.2,10
|
|
16
|
+
L0.8,29.5L0,30v1.1L20.1,42L40.1,31.1z">
|
|
17
|
+
</path>
|
|
18
|
+
</g>
|
|
19
|
+
</svg>
|
|
@@ -62,6 +62,7 @@ require_relative "integrations/source/clickhouse/client"
|
|
|
62
62
|
require_relative "integrations/source/amazon_s3/client"
|
|
63
63
|
require_relative "integrations/source/maria_db/client"
|
|
64
64
|
require_relative "integrations/source/oracle_db/client"
|
|
65
|
+
require_relative "integrations/source/databrics_model/client"
|
|
65
66
|
|
|
66
67
|
# Destination
|
|
67
68
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.9.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
@@ -612,6 +612,11 @@ files:
|
|
|
612
612
|
- lib/multiwoven/integrations/source/databricks/config/meta.json
|
|
613
613
|
- lib/multiwoven/integrations/source/databricks/config/spec.json
|
|
614
614
|
- lib/multiwoven/integrations/source/databricks/icon.svg
|
|
615
|
+
- lib/multiwoven/integrations/source/databrics_model/client.rb
|
|
616
|
+
- lib/multiwoven/integrations/source/databrics_model/config/catalog.json
|
|
617
|
+
- lib/multiwoven/integrations/source/databrics_model/config/meta.json
|
|
618
|
+
- lib/multiwoven/integrations/source/databrics_model/config/spec.json
|
|
619
|
+
- lib/multiwoven/integrations/source/databrics_model/icon.svg
|
|
615
620
|
- lib/multiwoven/integrations/source/maria_db/client.rb
|
|
616
621
|
- lib/multiwoven/integrations/source/maria_db/config/meta.json
|
|
617
622
|
- lib/multiwoven/integrations/source/maria_db/config/spec.json
|