multiwoven-integrations 0.8.6 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/base_connector.rb +8 -0
- data/lib/multiwoven/integrations/core/constants.rb +3 -0
- data/lib/multiwoven/integrations/core/destination_connector.rb +0 -8
- data/lib/multiwoven/integrations/protocol/protocol.rb +3 -3
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/databrics_model/client.rb +87 -0
- data/lib/multiwoven/integrations/source/databrics_model/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/source/databrics_model/config/meta.json +16 -0
- data/lib/multiwoven/integrations/source/databrics_model/config/spec.json +47 -0
- data/lib/multiwoven/integrations/source/databrics_model/icon.svg +19 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: eb63f96519eb9556503c62850837b135658b10f62c3ec9c5dc8e9964dbc5b3fb
|
|
4
|
+
data.tar.gz: 59eba81bcc6f9b048971d8a96e4260e006d6ff1e00e72a6c4868ed4c17cebedf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2b5b50562d426959dd482948d8f4e499da80066ff7402af22a1cc9f7359d63236c6fb9d4c0a86b2ba669047ceeccedc20d1946d52d109243ef37de795d68d644
|
|
7
|
+
data.tar.gz: cbeacf1343d598cdc08ace21ccb52ddf1ad4862e468147a247b0d56fcebf6c651a53b13e3f11d08b0c280df3435b4fb9d9892aa88e060ab2b62fd0c75725cc98
|
|
@@ -66,6 +66,14 @@ module Multiwoven
|
|
|
66
66
|
message = error&.message || "failed"
|
|
67
67
|
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: message).to_multiwoven_message
|
|
68
68
|
end
|
|
69
|
+
|
|
70
|
+
def auth_headers(access_token)
|
|
71
|
+
{
|
|
72
|
+
"Accept" => "application/json",
|
|
73
|
+
"Authorization" => "Bearer #{access_token}",
|
|
74
|
+
"Content-Type" => "application/json"
|
|
75
|
+
}
|
|
76
|
+
end
|
|
69
77
|
end
|
|
70
78
|
end
|
|
71
79
|
end
|
|
@@ -47,6 +47,9 @@ module Multiwoven
|
|
|
47
47
|
MS_EXCEL_SHEET_RANGE_API = "https://graph.microsoft.com/v1.0/drives/%<drive_id>s/items/%<item_id>s/"\
|
|
48
48
|
"workbook/worksheets/%<sheet_name>s/range(address='A1:Z1')/usedRange?$select=values"
|
|
49
49
|
|
|
50
|
+
DATABRICKS_HEALTH_URL = "https://%<databricks_host>s/api/2.0/serving-endpoints/%<endpoint_name>s"
|
|
51
|
+
DATABRICKS_SERVING_URL = "https://%<databricks_host>s/serving-endpoints/%<endpoint_name>s/invocations"
|
|
52
|
+
|
|
50
53
|
AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
|
|
51
54
|
AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
|
|
52
55
|
|
|
@@ -15,14 +15,6 @@ module Multiwoven
|
|
|
15
15
|
success: success, failed: failure, logs: log_message_array
|
|
16
16
|
).to_multiwoven_message
|
|
17
17
|
end
|
|
18
|
-
|
|
19
|
-
def auth_headers(access_token)
|
|
20
|
-
{
|
|
21
|
-
"Accept" => "application/json",
|
|
22
|
-
"Authorization" => "Bearer #{access_token}",
|
|
23
|
-
"Content-Type" => "application/json"
|
|
24
|
-
}
|
|
25
|
-
end
|
|
26
18
|
end
|
|
27
19
|
end
|
|
28
20
|
end
|
|
@@ -10,10 +10,10 @@ module Multiwoven
|
|
|
10
10
|
SyncStatus = Types::String.enum("started", "running", "complete", "incomplete")
|
|
11
11
|
DestinationSyncMode = Types::String.enum("insert", "upsert")
|
|
12
12
|
ConnectorType = Types::String.enum("source", "destination")
|
|
13
|
-
ConnectorQueryType = Types::String.enum("raw_sql", "soql")
|
|
14
|
-
ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector")
|
|
13
|
+
ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml")
|
|
14
|
+
ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml")
|
|
15
15
|
ConnectionStatusType = Types::String.enum("succeeded", "failed")
|
|
16
|
-
StreamType = Types::String.enum("static", "dynamic")
|
|
16
|
+
StreamType = Types::String.enum("static", "dynamic", "user_defined")
|
|
17
17
|
StreamAction = Types::String.enum("fetch", "create", "update", "delete")
|
|
18
18
|
MultiwovenMessageType = Types::String.enum(
|
|
19
19
|
"record", "log", "connector_spec",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.9.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -16,6 +16,7 @@ module Multiwoven
|
|
|
16
16
|
AmazonS3
|
|
17
17
|
MariaDB
|
|
18
18
|
Oracle
|
|
19
|
+
DatabricksModel
|
|
19
20
|
].freeze
|
|
20
21
|
|
|
21
22
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module DatabricksModel
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
url = build_url(DATABRICKS_HEALTH_URL, connection_config)
|
|
10
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
11
|
+
url,
|
|
12
|
+
HTTP_GET,
|
|
13
|
+
headers: auth_headers(connection_config[:token])
|
|
14
|
+
)
|
|
15
|
+
if success?(response)
|
|
16
|
+
success_status
|
|
17
|
+
else
|
|
18
|
+
failure_status(nil)
|
|
19
|
+
end
|
|
20
|
+
rescue StandardError => e
|
|
21
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def discover(_connection_config = nil)
|
|
25
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
26
|
+
catalog = build_catalog(catalog_json)
|
|
27
|
+
catalog.to_multiwoven_message
|
|
28
|
+
rescue StandardError => e
|
|
29
|
+
handle_exception(e, {
|
|
30
|
+
context: "DATABRICKS MODEL:DISCOVER:EXCEPTION",
|
|
31
|
+
type: "error"
|
|
32
|
+
})
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def read(sync_config)
|
|
36
|
+
connection_config = sync_config.source.connection_specification
|
|
37
|
+
connection_config = connection_config.with_indifferent_access
|
|
38
|
+
# The server checks the ConnectorQueryType.
|
|
39
|
+
# If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
|
|
40
|
+
# This query is then sent to the AI/ML model.
|
|
41
|
+
payload = JSON.parse(sync_config.model.query)
|
|
42
|
+
run_model(connection_config, payload)
|
|
43
|
+
rescue StandardError => e
|
|
44
|
+
handle_exception(e, {
|
|
45
|
+
context: "DATABRICKS MODEL:READ:EXCEPTION",
|
|
46
|
+
type: "error"
|
|
47
|
+
})
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def run_model(connection_config, payload)
|
|
53
|
+
connection_config = connection_config.with_indifferent_access
|
|
54
|
+
|
|
55
|
+
url = build_url(DATABRICKS_SERVING_URL, connection_config)
|
|
56
|
+
token = connection_config[:token]
|
|
57
|
+
response = send_request(url, token, payload)
|
|
58
|
+
process_response(response)
|
|
59
|
+
rescue StandardError => e
|
|
60
|
+
handle_exception(e, context: "DATABRICKS MODEL:RUN_MODEL:EXCEPTION", type: "error")
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def process_response(response)
|
|
64
|
+
if success?(response)
|
|
65
|
+
data = JSON.parse(response.body)
|
|
66
|
+
[RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
|
|
67
|
+
else
|
|
68
|
+
create_log_message("DATABRICKS MODEL:RUN_MODEL", "error", "request failed")
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def build_url(url, connection_config)
|
|
73
|
+
format(url, databricks_host: connection_config[:databricks_host],
|
|
74
|
+
endpoint_name: connection_config[:endpoint_name])
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def send_request(url, token, payload)
|
|
78
|
+
Multiwoven::Integrations::Core::HttpClient.request(
|
|
79
|
+
url,
|
|
80
|
+
HTTP_POST,
|
|
81
|
+
payload: payload,
|
|
82
|
+
headers: auth_headers(token)
|
|
83
|
+
)
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "DatabricksModel",
|
|
4
|
+
"title": "Databricks Model",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "AI Model",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "source-databricks-foundation",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/databricks",
|
|
3
|
+
"stream_type": "user_defined",
|
|
4
|
+
"connector_query_type": "ai_ml",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Databricks Model",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["databricks_host", "token", "model_name"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"databricks_host": {
|
|
12
|
+
"title": "databricks_host",
|
|
13
|
+
"description": "databricks_host",
|
|
14
|
+
"type": "string",
|
|
15
|
+
"examples": ["app.databricks.com"],
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"token": {
|
|
19
|
+
"title": "Databricks Token",
|
|
20
|
+
"description": "personal access token",
|
|
21
|
+
"type": "string",
|
|
22
|
+
"multiwoven_secret": true,
|
|
23
|
+
"order": 1
|
|
24
|
+
},
|
|
25
|
+
"endpoint": {
|
|
26
|
+
"title": "Endpoint name",
|
|
27
|
+
"description": "Endpoint name",
|
|
28
|
+
"examples": ["databricks-dbrx-instruct"],
|
|
29
|
+
"type": "string",
|
|
30
|
+
"order": 2
|
|
31
|
+
},
|
|
32
|
+
"request_format": {
|
|
33
|
+
"title": "Request Format",
|
|
34
|
+
"description": "Sample Request Format",
|
|
35
|
+
"type": "string",
|
|
36
|
+
"order": 3
|
|
37
|
+
},
|
|
38
|
+
"response_format": {
|
|
39
|
+
"title": "Response Format",
|
|
40
|
+
"description": "Sample Response Format",
|
|
41
|
+
"type": "string",
|
|
42
|
+
"order": 4
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
<svg version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px" viewBox="0 0 40.1 42" style="enable-background:new 0 0 40.1 42;" xml:space="preserve">
|
|
2
|
+
<style type="text/css">
|
|
3
|
+
.st0{fill:#FF3621;}
|
|
4
|
+
</style>
|
|
5
|
+
<metadata>
|
|
6
|
+
<sfw xmlns="ns_sfw;">
|
|
7
|
+
<slices>
|
|
8
|
+
</slices>
|
|
9
|
+
<sliceSourceBounds bottomLeftOrigin="true" height="42" width="40.1" x="-69.1" y="-10.5">
|
|
10
|
+
</sliceSourceBounds>
|
|
11
|
+
</sfw>
|
|
12
|
+
</metadata>
|
|
13
|
+
<g>
|
|
14
|
+
<path class="st0" d="M40.1,31.1v-7.4l-0.8-0.5L20.1,33.7l-18.2-10l0-4.3l18.2,9.9l20.1-10.9v-7.3l-0.8-0.5L20.1,21.2L2.6,11.6
|
|
15
|
+
L20.1,2l14.1,7.7l1.1-0.6V8.3L20.1,0L0,10.9V12L20.1,23l18.2-10v4.4l-18.2,10L0.8,16.8L0,17.3v7.4l20.1,10.9l18.2-9.9v4.3l-18.2,10
|
|
16
|
+
L0.8,29.5L0,30v1.1L20.1,42L40.1,31.1z">
|
|
17
|
+
</path>
|
|
18
|
+
</g>
|
|
19
|
+
</svg>
|
|
@@ -62,6 +62,7 @@ require_relative "integrations/source/clickhouse/client"
|
|
|
62
62
|
require_relative "integrations/source/amazon_s3/client"
|
|
63
63
|
require_relative "integrations/source/maria_db/client"
|
|
64
64
|
require_relative "integrations/source/oracle_db/client"
|
|
65
|
+
require_relative "integrations/source/databrics_model/client"
|
|
65
66
|
|
|
66
67
|
# Destination
|
|
67
68
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.9.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
@@ -612,6 +612,11 @@ files:
|
|
|
612
612
|
- lib/multiwoven/integrations/source/databricks/config/meta.json
|
|
613
613
|
- lib/multiwoven/integrations/source/databricks/config/spec.json
|
|
614
614
|
- lib/multiwoven/integrations/source/databricks/icon.svg
|
|
615
|
+
- lib/multiwoven/integrations/source/databrics_model/client.rb
|
|
616
|
+
- lib/multiwoven/integrations/source/databrics_model/config/catalog.json
|
|
617
|
+
- lib/multiwoven/integrations/source/databrics_model/config/meta.json
|
|
618
|
+
- lib/multiwoven/integrations/source/databrics_model/config/spec.json
|
|
619
|
+
- lib/multiwoven/integrations/source/databrics_model/icon.svg
|
|
615
620
|
- lib/multiwoven/integrations/source/maria_db/client.rb
|
|
616
621
|
- lib/multiwoven/integrations/source/maria_db/config/meta.json
|
|
617
622
|
- lib/multiwoven/integrations/source/maria_db/config/spec.json
|