multiwoven-integrations 0.10.1 → 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/constants.rb +3 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/google_vertex_model/client.rb +83 -0
- data/lib/multiwoven/integrations/source/google_vertex_model/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/source/google_vertex_model/config/meta.json +16 -0
- data/lib/multiwoven/integrations/source/google_vertex_model/config/spec.json +103 -0
- data/lib/multiwoven/integrations/source/google_vertex_model/icon.svg +2 -0
- data/lib/multiwoven/integrations.rb +3 -0
- data/multiwoven-integrations.gemspec +2 -0
- metadata +35 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8a76de4bbbe54dda8a7a063b0e2ad56aa79cb68625661cb7396b1a23db6e8ac6
|
|
4
|
+
data.tar.gz: 486ad225768b3fe8732f80deeb35d25e3f2e9e73ffa652d6ea592e51288dbdc6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '03249a470b67cac1c549d7c7682e80f0f6e1b6c62b41c547687b1b03cbf11cc8a691b5cdfc1fa05259df58d0ed192d020f466d8136b1ce6c582d9bbc0187f245'
|
|
7
|
+
data.tar.gz: b30b3fe2853752f86478aaf80b300c6f9bb038bd72a2b6a2f5507819ef15d8686a2182eb29ee99136191d31be9efcf58eede56fa9ebc12edfe6d97d255b6f091
|
|
@@ -50,6 +50,9 @@ module Multiwoven
|
|
|
50
50
|
DATABRICKS_HEALTH_URL = "https://%<databricks_host>s/api/2.0/serving-endpoints/%<endpoint_name>s"
|
|
51
51
|
DATABRICKS_SERVING_URL = "https://%<databricks_host>s/serving-endpoints/%<endpoint_name>s/invocations"
|
|
52
52
|
|
|
53
|
+
GOOGLE_VERTEX_ENDPOINT_SERVICE_URL = "%<region>s-aiplatform.googleapis.com"
|
|
54
|
+
GOOGLE_VERTEX_MODEL_NAME = "projects/%<project_id>s/locations/%<region>s/endpoints/%<endpoint_id>s"
|
|
55
|
+
|
|
53
56
|
# HTTP
|
|
54
57
|
HTTP_GET = "GET"
|
|
55
58
|
HTTP_POST = "POST"
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.11.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -18,6 +18,7 @@ module Multiwoven
|
|
|
18
18
|
Oracle
|
|
19
19
|
DatabricksModel
|
|
20
20
|
AwsSagemakerModel
|
|
21
|
+
VertexModel
|
|
21
22
|
].freeze
|
|
22
23
|
|
|
23
24
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module VertexModel
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
@client.get_endpoint(name: build_url(GOOGLE_VERTEX_MODEL_NAME, connection_config))
|
|
11
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
12
|
+
rescue StandardError => e
|
|
13
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def discover(_connection_config = nil)
|
|
17
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
18
|
+
catalog = build_catalog(catalog_json)
|
|
19
|
+
catalog.to_multiwoven_message
|
|
20
|
+
rescue StandardError => e
|
|
21
|
+
handle_exception(e, {
|
|
22
|
+
context: "GOOGLE:VERTEX MODEL:DISCOVER:EXCEPTION",
|
|
23
|
+
type: "error"
|
|
24
|
+
})
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def read(sync_config)
|
|
28
|
+
connection_config = sync_config.source.connection_specification
|
|
29
|
+
connection_config = connection_config.with_indifferent_access
|
|
30
|
+
# The server checks the ConnectorQueryType.
|
|
31
|
+
# If it's "ai_ml," the server calculates the payload and passes it as a query in the sync config model protocol.
|
|
32
|
+
# This query is then sent to the AI/ML model.
|
|
33
|
+
payload = JSON.parse(sync_config.model.query)
|
|
34
|
+
run_model(connection_config, payload)
|
|
35
|
+
rescue StandardError => e
|
|
36
|
+
handle_exception(e, {
|
|
37
|
+
context: "GOOGLE:VERTEX MODEL:READ:EXCEPTION",
|
|
38
|
+
type: "error"
|
|
39
|
+
})
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def create_connection(connection_config)
|
|
45
|
+
Google::Cloud::AIPlatform::V1::EndpointService::Client.configure do |config|
|
|
46
|
+
config.endpoint = build_url(GOOGLE_VERTEX_ENDPOINT_SERVICE_URL, connection_config)
|
|
47
|
+
config.credentials = connection_config["credentials_json"]
|
|
48
|
+
end
|
|
49
|
+
Google::Cloud::AIPlatform::V1::PredictionService::Client.configure do |config|
|
|
50
|
+
config.endpoint = build_url(GOOGLE_VERTEX_ENDPOINT_SERVICE_URL, connection_config)
|
|
51
|
+
config.credentials = connection_config["credentials_json"]
|
|
52
|
+
end
|
|
53
|
+
@client = Google::Cloud::AIPlatform::V1::EndpointService::Client.new
|
|
54
|
+
@endpoint = Google::Cloud::AIPlatform::V1::PredictionService::Client.new
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def run_model(connection_config, payload)
|
|
58
|
+
create_connection(connection_config)
|
|
59
|
+
http_body = Google::Api::HttpBody.new(data: JSON.generate(payload))
|
|
60
|
+
response = @endpoint.raw_predict(endpoint: build_url(GOOGLE_VERTEX_MODEL_NAME, connection_config), http_body: http_body)
|
|
61
|
+
process_response(response)
|
|
62
|
+
rescue StandardError => e
|
|
63
|
+
handle_exception(e, context: "GOOGLE:VERTEX MODEL:RUN_MODEL:EXCEPTION", type: "error")
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def process_response(response)
|
|
67
|
+
data = JSON.parse(response.data)
|
|
68
|
+
[RecordMessage.new(data: data, emitted_at: Time.now.to_i).to_multiwoven_message]
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def build_url(url, connection_config)
|
|
72
|
+
case url
|
|
73
|
+
when GOOGLE_VERTEX_MODEL_NAME
|
|
74
|
+
format(url, project_id: connection_config[:project_id],
|
|
75
|
+
region: connection_config[:region],
|
|
76
|
+
endpoint_id: connection_config[:endpoint_id])
|
|
77
|
+
when GOOGLE_VERTEX_ENDPOINT_SERVICE_URL
|
|
78
|
+
format(url, region: connection_config[:region])
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "VertexModel",
|
|
4
|
+
"title": "Google Vertex",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "AI Model",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "source-vertex",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/google_vertex-model",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Google Vertex Model",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["project_id", "endpoint_id", "location", "credentials_json", "request_format", "response_format"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"project_id": {
|
|
12
|
+
"type": "string",
|
|
13
|
+
"description": "The project ID of the endpoint",
|
|
14
|
+
"title": "Project ID",
|
|
15
|
+
"order": 0
|
|
16
|
+
},
|
|
17
|
+
"endpoint_id": {
|
|
18
|
+
"type": "string",
|
|
19
|
+
"description": "The endpoint ID of the trained model",
|
|
20
|
+
"title": "Endpoint ID",
|
|
21
|
+
"order": 1
|
|
22
|
+
},
|
|
23
|
+
"region":{
|
|
24
|
+
"type": "string",
|
|
25
|
+
"description": "The region where the trained model is stored",
|
|
26
|
+
"title": "Region",
|
|
27
|
+
"order": 2
|
|
28
|
+
},
|
|
29
|
+
"request_format":{
|
|
30
|
+
"title": "Request Format",
|
|
31
|
+
"description": "Sample Request Format",
|
|
32
|
+
"type": "string",
|
|
33
|
+
"order": 3
|
|
34
|
+
},
|
|
35
|
+
"response_format": {
|
|
36
|
+
"title": "Response Format",
|
|
37
|
+
"description": "Sample Response Format",
|
|
38
|
+
"type": "string",
|
|
39
|
+
"order": 4
|
|
40
|
+
},
|
|
41
|
+
"credentials_json": {
|
|
42
|
+
"type": "object",
|
|
43
|
+
"description": "You can get the keys from the Google Cloud web console. First, go to the IAM page and select Service Accounts from the left menu. Next, locate your service account in the list, click on its Keys tab, and then click Add Key. Lastly, click Create new key and select JSON.",
|
|
44
|
+
"title": "",
|
|
45
|
+
"properties": {
|
|
46
|
+
"type": {
|
|
47
|
+
"type": "string",
|
|
48
|
+
"enum": ["service_account"]
|
|
49
|
+
},
|
|
50
|
+
"project_id": {
|
|
51
|
+
"type": "string"
|
|
52
|
+
},
|
|
53
|
+
"private_key_id": {
|
|
54
|
+
"type": "string"
|
|
55
|
+
},
|
|
56
|
+
"private_key": {
|
|
57
|
+
"type": "string",
|
|
58
|
+
"multiwoven_secret": true
|
|
59
|
+
},
|
|
60
|
+
"client_email": {
|
|
61
|
+
"type": "string",
|
|
62
|
+
"format": "email"
|
|
63
|
+
},
|
|
64
|
+
"client_id": {
|
|
65
|
+
"type": "string"
|
|
66
|
+
},
|
|
67
|
+
"auth_uri": {
|
|
68
|
+
"type": "string",
|
|
69
|
+
"format": "uri"
|
|
70
|
+
},
|
|
71
|
+
"token_uri": {
|
|
72
|
+
"type": "string",
|
|
73
|
+
"format": "uri"
|
|
74
|
+
},
|
|
75
|
+
"auth_provider_x509_cert_url": {
|
|
76
|
+
"type": "string",
|
|
77
|
+
"format": "uri"
|
|
78
|
+
},
|
|
79
|
+
"client_x509_cert_url": {
|
|
80
|
+
"type": "string",
|
|
81
|
+
"format": "uri"
|
|
82
|
+
},
|
|
83
|
+
"universe_domain": {
|
|
84
|
+
"type": "string"
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
"required": [
|
|
88
|
+
"type",
|
|
89
|
+
"project_id",
|
|
90
|
+
"private_key_id",
|
|
91
|
+
"private_key",
|
|
92
|
+
"client_email",
|
|
93
|
+
"client_id",
|
|
94
|
+
"auth_uri",
|
|
95
|
+
"token_uri",
|
|
96
|
+
"auth_provider_x509_cert_url",
|
|
97
|
+
"client_x509_cert_url",
|
|
98
|
+
"universe_domain"
|
|
99
|
+
]
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
|
|
2
|
+
<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path d="M20,13.89A.77.77,0,0,0,19,13.73l-7,5.14v.22a.72.72,0,1,1,0,1.43v0a.74.74,0,0,0,.45-.15l7.41-5.47A.76.76,0,0,0,20,13.89Z" style="fill:#669df6"/><path d="M12,20.52a.72.72,0,0,1,0-1.43h0v-.22L5,13.73a.76.76,0,0,0-1,.16.74.74,0,0,0,.16,1l7.41,5.47a.73.73,0,0,0,.44.15v0Z" style="fill:#aecbfa"/><path d="M12,18.34a1.47,1.47,0,1,0,1.47,1.47A1.47,1.47,0,0,0,12,18.34Zm0,2.18a.72.72,0,1,1,.72-.71A.71.71,0,0,1,12,20.52Z" style="fill:#4285f4"/><path d="M6,6.11a.76.76,0,0,1-.75-.75V3.48a.76.76,0,1,1,1.51,0V5.36A.76.76,0,0,1,6,6.11Z" style="fill:#aecbfa"/><circle cx="5.98" cy="12" r="0.76" style="fill:#aecbfa"/><circle cx="5.98" cy="9.79" r="0.76" style="fill:#aecbfa"/><circle cx="5.98" cy="7.57" r="0.76" style="fill:#aecbfa"/><path d="M18,8.31a.76.76,0,0,1-.75-.76V5.67a.75.75,0,1,1,1.5,0V7.55A.75.75,0,0,1,18,8.31Z" style="fill:#4285f4"/><circle cx="18.02" cy="12.01" r="0.76" style="fill:#4285f4"/><circle cx="18.02" cy="9.76" r="0.76" style="fill:#4285f4"/><circle cx="18.02" cy="3.48" r="0.76" style="fill:#4285f4"/><path d="M12,15a.76.76,0,0,1-.75-.75V12.34a.76.76,0,0,1,1.51,0v1.89A.76.76,0,0,1,12,15Z" style="fill:#669df6"/><circle cx="12" cy="16.45" r="0.76" style="fill:#669df6"/><circle cx="12" cy="10.14" r="0.76" style="fill:#669df6"/><circle cx="12" cy="7.92" r="0.76" style="fill:#669df6"/><path d="M15,10.54a.76.76,0,0,1-.75-.75V7.91a.76.76,0,1,1,1.51,0V9.79A.76.76,0,0,1,15,10.54Z" style="fill:#4285f4"/><circle cx="15.01" cy="5.69" r="0.76" style="fill:#4285f4"/><circle cx="15.01" cy="14.19" r="0.76" style="fill:#4285f4"/><circle cx="15.01" cy="11.97" r="0.76" style="fill:#4285f4"/><circle cx="8.99" cy="14.19" r="0.76" style="fill:#aecbfa"/><circle cx="8.99" cy="7.92" r="0.76" style="fill:#aecbfa"/><circle cx="8.99" cy="5.69" r="0.76" style="fill:#aecbfa"/><path d="M9,12.73A.76.76,0,0,1,8.24,12V10.1a.75.75,0,1,1,1.5,0V12A.75.75,0,0,1,9,12.73Z" style="fill:#aecbfa"/></svg>
|
|
@@ -34,6 +34,8 @@ require "aws-sdk-sts"
|
|
|
34
34
|
require "ruby-oci8"
|
|
35
35
|
require "aws-sdk-sagemaker"
|
|
36
36
|
require "aws-sdk-sagemakerruntime"
|
|
37
|
+
require "google/cloud/ai_platform/v1"
|
|
38
|
+
require "grpc"
|
|
37
39
|
|
|
38
40
|
# Service
|
|
39
41
|
require_relative "integrations/config"
|
|
@@ -66,6 +68,7 @@ require_relative "integrations/source/maria_db/client"
|
|
|
66
68
|
require_relative "integrations/source/oracle_db/client"
|
|
67
69
|
require_relative "integrations/source/databrics_model/client"
|
|
68
70
|
require_relative "integrations/source/aws_sagemaker_model/client"
|
|
71
|
+
require_relative "integrations/source/google_vertex_model/client"
|
|
69
72
|
|
|
70
73
|
# Destination
|
|
71
74
|
require_relative "integrations/destination/klaviyo/client"
|
|
@@ -47,7 +47,9 @@ Gem::Specification.new do |spec|
|
|
|
47
47
|
spec.add_runtime_dependency "duckdb"
|
|
48
48
|
spec.add_runtime_dependency "git"
|
|
49
49
|
spec.add_runtime_dependency "google-apis-sheets_v4"
|
|
50
|
+
spec.add_runtime_dependency "google-cloud-ai_platform-v1"
|
|
50
51
|
spec.add_runtime_dependency "google-cloud-bigquery"
|
|
52
|
+
spec.add_runtime_dependency "grpc"
|
|
51
53
|
spec.add_runtime_dependency "hubspot-api-client"
|
|
52
54
|
spec.add_runtime_dependency "iterable-api-client"
|
|
53
55
|
spec.add_runtime_dependency "net-sftp"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.11.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-09-
|
|
11
|
+
date: 2024-09-09 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -206,6 +206,20 @@ dependencies:
|
|
|
206
206
|
- - ">="
|
|
207
207
|
- !ruby/object:Gem::Version
|
|
208
208
|
version: '0'
|
|
209
|
+
- !ruby/object:Gem::Dependency
|
|
210
|
+
name: google-cloud-ai_platform-v1
|
|
211
|
+
requirement: !ruby/object:Gem::Requirement
|
|
212
|
+
requirements:
|
|
213
|
+
- - ">="
|
|
214
|
+
- !ruby/object:Gem::Version
|
|
215
|
+
version: '0'
|
|
216
|
+
type: :runtime
|
|
217
|
+
prerelease: false
|
|
218
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
219
|
+
requirements:
|
|
220
|
+
- - ">="
|
|
221
|
+
- !ruby/object:Gem::Version
|
|
222
|
+
version: '0'
|
|
209
223
|
- !ruby/object:Gem::Dependency
|
|
210
224
|
name: google-cloud-bigquery
|
|
211
225
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -220,6 +234,20 @@ dependencies:
|
|
|
220
234
|
- - ">="
|
|
221
235
|
- !ruby/object:Gem::Version
|
|
222
236
|
version: '0'
|
|
237
|
+
- !ruby/object:Gem::Dependency
|
|
238
|
+
name: grpc
|
|
239
|
+
requirement: !ruby/object:Gem::Requirement
|
|
240
|
+
requirements:
|
|
241
|
+
- - ">="
|
|
242
|
+
- !ruby/object:Gem::Version
|
|
243
|
+
version: '0'
|
|
244
|
+
type: :runtime
|
|
245
|
+
prerelease: false
|
|
246
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
247
|
+
requirements:
|
|
248
|
+
- - ">="
|
|
249
|
+
- !ruby/object:Gem::Version
|
|
250
|
+
version: '0'
|
|
223
251
|
- !ruby/object:Gem::Dependency
|
|
224
252
|
name: hubspot-api-client
|
|
225
253
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -650,6 +678,11 @@ files:
|
|
|
650
678
|
- lib/multiwoven/integrations/source/databrics_model/config/meta.json
|
|
651
679
|
- lib/multiwoven/integrations/source/databrics_model/config/spec.json
|
|
652
680
|
- lib/multiwoven/integrations/source/databrics_model/icon.svg
|
|
681
|
+
- lib/multiwoven/integrations/source/google_vertex_model/client.rb
|
|
682
|
+
- lib/multiwoven/integrations/source/google_vertex_model/config/catalog.json
|
|
683
|
+
- lib/multiwoven/integrations/source/google_vertex_model/config/meta.json
|
|
684
|
+
- lib/multiwoven/integrations/source/google_vertex_model/config/spec.json
|
|
685
|
+
- lib/multiwoven/integrations/source/google_vertex_model/icon.svg
|
|
653
686
|
- lib/multiwoven/integrations/source/maria_db/client.rb
|
|
654
687
|
- lib/multiwoven/integrations/source/maria_db/config/meta.json
|
|
655
688
|
- lib/multiwoven/integrations/source/maria_db/config/spec.json
|