multiwoven-integrations 0.21.2 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/constants.rb +2 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/watsonx_data/client.rb +146 -0
- data/lib/multiwoven/integrations/source/watsonx_data/config/meta.json +16 -0
- data/lib/multiwoven/integrations/source/watsonx_data/config/spec.json +72 -0
- data/lib/multiwoven/integrations/source/watsonx_data/icon.svg +1 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fc3297301450525e523cde59f46f1371bd67fa92a31a0a3135ae9818563a9b31
|
|
4
|
+
data.tar.gz: 3168abc5dcd112f1cfd86b4daeb19d7501aba221261e5db75d1b10976bd95e6b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e141fddcc210447d6dcf88cae35e051050965d0d3548e7a30f8a4b164e9a8df5fdad9e07d3b44d0d5f5edc714f6e3f52d457b3959a74c8363e88414802198d59
|
|
7
|
+
data.tar.gz: e32081015f5afeeeb76c393d8770b6e87901b4146681c2f9419ce75e2a8674c7d306566da0e4ca9d9cb43294019aa986540b094010dfb4a7dcefec4c5cc78558
|
|
@@ -60,6 +60,8 @@ module Multiwoven
|
|
|
60
60
|
WATSONX_GENERATION_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v1/deployments/%<deployment_id>s/text/generation?version=%<version>s"
|
|
61
61
|
WATSONX_STREAM_DEPLOYMENT_URL = "https://%<region>s.ml.cloud.ibm.com/ml/v1/deployments/%<deployment_id>s/text/generation_stream?version=%<version>s"
|
|
62
62
|
|
|
63
|
+
WATSONX_DATA_QUERIES_URL = "https://%<region>s.lakehouse.cloud.ibm.com/lakehouse/api/v2/queries/execute/%<engine_id>s"
|
|
64
|
+
|
|
63
65
|
# HTTP
|
|
64
66
|
HTTP_GET = "GET"
|
|
65
67
|
HTTP_POST = "POST"
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module WatsonxData
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
API_VERSION = "2021-05-01"
|
|
7
|
+
class Client < SourceConnector
|
|
8
|
+
def check_connection(connection_config)
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
response = execute_query(connection_config, "show catalogs")
|
|
11
|
+
success?(response) ? success_status : failure_status(nil)
|
|
12
|
+
rescue StandardError => e
|
|
13
|
+
handle_exception(e, { context: "WATSONX DATA:CHECK_CONNECTION:EXCEPTION", type: "error" })
|
|
14
|
+
failure_status(e)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def discover(connection_config)
|
|
18
|
+
connection_config = connection_config.with_indifferent_access
|
|
19
|
+
query = "SELECT table_name, column_name,
|
|
20
|
+
data_type,
|
|
21
|
+
is_nullable
|
|
22
|
+
FROM information_schema.columns
|
|
23
|
+
WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
|
|
24
|
+
ORDER BY table_name, ordinal_position"
|
|
25
|
+
response = execute_query(connection_config, query)
|
|
26
|
+
records = JSON.parse(response.body)["response"]["result"]
|
|
27
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
28
|
+
catalog.to_multiwoven_message
|
|
29
|
+
rescue StandardError => e
|
|
30
|
+
handle_exception(e, { context: "WATSONX DATA:DISCOVER:EXCEPTION", type: "error" })
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def read(sync_config)
|
|
34
|
+
connection_config = sync_config.source.connection_specification
|
|
35
|
+
connection_config = connection_config.with_indifferent_access
|
|
36
|
+
query = sync_config.model.query
|
|
37
|
+
if connection_config[:engine] == "presto"
|
|
38
|
+
query = batched_query_for_presto(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
39
|
+
else
|
|
40
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
41
|
+
end
|
|
42
|
+
query(connection_config, query)
|
|
43
|
+
rescue StandardError => e
|
|
44
|
+
handle_exception(e, { context: "WATSONX DATA:READ:EXCEPTION", type: "error" })
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def batched_query_for_presto(query, limit, offset)
|
|
50
|
+
<<~SQL
|
|
51
|
+
SELECT * FROM (
|
|
52
|
+
SELECT *, ROW_NUMBER() OVER () as rownum FROM ( #{query} ) subquery
|
|
53
|
+
) t
|
|
54
|
+
WHERE rownum > #{offset}
|
|
55
|
+
LIMIT #{limit}
|
|
56
|
+
SQL
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def execute_query(connection_config, query)
|
|
60
|
+
connection_config.with_indifferent_access
|
|
61
|
+
get_access_token(connection_config[:api_key])
|
|
62
|
+
url = format(
|
|
63
|
+
WATSONX_DATA_QUERIES_URL,
|
|
64
|
+
region: connection_config[:region],
|
|
65
|
+
engine_id: connection_config[:engine_id]
|
|
66
|
+
)
|
|
67
|
+
headers = auth_headers(@access_token)
|
|
68
|
+
headers["AuthInstanceId"] = connection_config[:auth_instance_id]
|
|
69
|
+
send_request(
|
|
70
|
+
url: url,
|
|
71
|
+
http_method: HTTP_POST,
|
|
72
|
+
payload: {
|
|
73
|
+
sql_string: query,
|
|
74
|
+
catalog_name: connection_config[:database],
|
|
75
|
+
schema_name: connection_config[:schema]
|
|
76
|
+
},
|
|
77
|
+
headers: headers,
|
|
78
|
+
config: connection_config[:config]
|
|
79
|
+
)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def query(connection, query)
|
|
83
|
+
response = execute_query(connection, query)
|
|
84
|
+
response = JSON.parse(response.body).with_indifferent_access
|
|
85
|
+
records = response[:response][:result]
|
|
86
|
+
records.map do |row|
|
|
87
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def create_connection(connection_config)
|
|
92
|
+
connection_config
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def create_streams(records)
|
|
96
|
+
group_by_table(records).map do |r|
|
|
97
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def group_by_table(records)
|
|
102
|
+
records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
|
|
103
|
+
{
|
|
104
|
+
tablename: table_name,
|
|
105
|
+
columns: columns.map do |column|
|
|
106
|
+
{
|
|
107
|
+
column_name: column["column_name"],
|
|
108
|
+
type: column["data_type"],
|
|
109
|
+
optional: column["is_nullable"] == "YES"
|
|
110
|
+
}
|
|
111
|
+
end
|
|
112
|
+
}
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def get_access_token(api_key)
|
|
117
|
+
cache = defined?(Rails) && Rails.respond_to?(:cache) ? Rails.cache : ActiveSupport::Cache::MemoryStore.new
|
|
118
|
+
cache_key = "watsonx_data_#{api_key}"
|
|
119
|
+
cached_token = cache.read(cache_key)
|
|
120
|
+
if cached_token
|
|
121
|
+
@access_token = cached_token
|
|
122
|
+
else
|
|
123
|
+
new_token = get_iam_token(api_key)
|
|
124
|
+
# puts new_token
|
|
125
|
+
# max expiration is 3 minutes. No way to make it higher
|
|
126
|
+
cache.write(cache_key, new_token, expires_in: 180)
|
|
127
|
+
@access_token = new_token
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def get_iam_token(api_key)
|
|
132
|
+
uri = URI("https://iam.cloud.ibm.com/identity/token")
|
|
133
|
+
request = Net::HTTP::Post.new(uri)
|
|
134
|
+
request["Content-Type"] = "application/x-www-form-urlencoded"
|
|
135
|
+
request.body = "grant_type=urn:ibm:params:oauth:grant-type:apikey&apikey=#{api_key}"
|
|
136
|
+
response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
|
|
137
|
+
http.request(request)
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
raise "Failed to get IAM token: #{response.body}" unless response.is_a?(Net::HTTPSuccess)
|
|
141
|
+
|
|
142
|
+
JSON.parse(response.body)["access_token"]
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "WatsonxData",
|
|
4
|
+
"title": "WatsonX Data Endpoint",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"documentation_url": "https://docs.mutltiwoven.com",
|
|
8
|
+
"github_issue_label": "source-watsonx-data-endpoint",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/source/watsonx-data-endpoint",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "WatsonX Data Endpoint",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["api_key","region","engine","engine_id","auth_instance_id","database","schema"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"api_key": {
|
|
12
|
+
"type": "string",
|
|
13
|
+
"multiwoven_secret": true,
|
|
14
|
+
"title": "API Key",
|
|
15
|
+
"order": 0
|
|
16
|
+
},
|
|
17
|
+
"region": {
|
|
18
|
+
"description": "WatsonX Data region",
|
|
19
|
+
"type": "string",
|
|
20
|
+
"title": "Region",
|
|
21
|
+
"order": 1
|
|
22
|
+
},
|
|
23
|
+
"engine": {
|
|
24
|
+
"description": "Which engine is being used? (Presto/Spark)",
|
|
25
|
+
"type": "string",
|
|
26
|
+
"title": "Engine",
|
|
27
|
+
"enum": ["presto"],
|
|
28
|
+
"default": "presto",
|
|
29
|
+
"order": 2
|
|
30
|
+
},
|
|
31
|
+
"engine_id": {
|
|
32
|
+
"description": "Engine id",
|
|
33
|
+
"type": "string",
|
|
34
|
+
"title": "Engine Id",
|
|
35
|
+
"order": 3
|
|
36
|
+
},
|
|
37
|
+
"auth_instance_id": {
|
|
38
|
+
"description": "WatsonX Data Instance CRN",
|
|
39
|
+
"type": "string",
|
|
40
|
+
"title": "Instance CRN",
|
|
41
|
+
"order": 4
|
|
42
|
+
},
|
|
43
|
+
"database": {
|
|
44
|
+
"description": "The specific database to connect to.",
|
|
45
|
+
"type": "string",
|
|
46
|
+
"title": "Database",
|
|
47
|
+
"order": 5
|
|
48
|
+
},
|
|
49
|
+
"schema": {
|
|
50
|
+
"description": "The schema within the database.",
|
|
51
|
+
"type": "string",
|
|
52
|
+
"title": "Schema",
|
|
53
|
+
"order": 6
|
|
54
|
+
},
|
|
55
|
+
"config": {
|
|
56
|
+
"title": "",
|
|
57
|
+
"type": "object",
|
|
58
|
+
"properties": {
|
|
59
|
+
"timeout": {
|
|
60
|
+
"type": "string",
|
|
61
|
+
"default": "30",
|
|
62
|
+
"title": "HTTP Timeout",
|
|
63
|
+
"description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
|
|
64
|
+
"order": 0
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"order": 7
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg id="Watsonx-Data--Streamline-Carbon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" height="16" width="16"><desc>Watsonx Data Streamline Icon: https://streamlinehq.com</desc><defs></defs><path d="M26 24c-1.1046 0-2 .8954-2 2 0 .0764.0142.1488.0225.2229C21.7417 28.0192 18.9433 29 16 29c-2.7746 0-5.3432-.881-7.4566-2.3676.2576.0261.517.0444.7798.0444C13.5561 26.6768 17 23.233 17 19h-2c0 3.1304-2.5464 5.6768-5.6768 5.6768-2.2111 0-4.1977-1.2816-5.1318-3.2725-.1365-.2972-.2595-.6007-.3738-.9094C4.4778 20.8169 5.2174 21 6 21c2.7568 0 5-2.2432 5-5v-2H9v2c0 1.6543-1.3457 3-3 3s-3-1.3457-3-3c0-2.1152.4917-4.1328 1.4619-5.9956l-1.7744-.9238C1.5835 11.2017 1 13.5943 1 16c0 8.271 6.729 15 15 15 3.3744 0 6.5818-1.1193 9.2048-3.1662.244.106.5123.1662.7952.1662 1.1046 0 2-.8954 2-2s-.8954-2-2-2Z" fill="#000000"></path><path transform="rotate(90 22 22)" d="M21 21h2v2h-2Z" fill="#000000"></path><path transform="rotate(90 16 16)" d="M15 15h2v2h-2Z" fill="#000000"></path><path transform="rotate(-90 10 10)" d="M9 9h2v2H9Z" fill="#000000"></path><path d="M16 1c-3.3744 0-6.5818 1.1193-9.2048 3.1662C6.5512 4.0602 6.2829 4 6 4c-1.1046 0-2 .8954-2 2s.8954 2 2 2 2-.8954 2-2c0-.0764-.0142-.1488-.0225-.2229C10.2583 3.9808 13.0567 3 16 3c2.7708 0 5.3363.8784 7.4481 2.3613-.249-.0242-.5005-.038-.7547-.038-4.2329 0-7.6768 3.4438-7.6768 7.6768h2c0-3.1304 2.5464-5.6768 5.6768-5.6768 2.0554 0 3.9068 1.0953 4.9186 2.8651.2153.4283.4053.8701.5729 1.3237C27.5234 11.1887 26.7844 11 26 11c-2.7568 0-5 2.2432-5 5v2h2v-2c0-1.6543 1.3457-3 3-3s3 1.3457 3 3c0 2.1152-.4917 4.1328-1.4619 5.9956l1.7744.9238C30.4165 20.7983 31 18.4057 31 16c0-8.271-6.729-15-15-15Z" fill="#000000"></path><path id="_Transparent_Rectangle_" d="M0 0h32v32H0Z" fill="none"></path></svg>
|
|
@@ -76,6 +76,7 @@ require_relative "integrations/source/http_model/client"
|
|
|
76
76
|
require_relative "integrations/source/open_ai/client"
|
|
77
77
|
require_relative "integrations/source/sftp/client"
|
|
78
78
|
require_relative "integrations/source/watsonx_ai/client"
|
|
79
|
+
require_relative "integrations/source/watsonx_data/client"
|
|
79
80
|
require_relative "integrations/source/anthropic/client"
|
|
80
81
|
|
|
81
82
|
# Destination
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.22.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-
|
|
11
|
+
date: 2025-04-08 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -786,6 +786,10 @@ files:
|
|
|
786
786
|
- lib/multiwoven/integrations/source/watsonx_ai/config/meta.json
|
|
787
787
|
- lib/multiwoven/integrations/source/watsonx_ai/config/spec.json
|
|
788
788
|
- lib/multiwoven/integrations/source/watsonx_ai/icon.svg
|
|
789
|
+
- lib/multiwoven/integrations/source/watsonx_data/client.rb
|
|
790
|
+
- lib/multiwoven/integrations/source/watsonx_data/config/meta.json
|
|
791
|
+
- lib/multiwoven/integrations/source/watsonx_data/config/spec.json
|
|
792
|
+
- lib/multiwoven/integrations/source/watsonx_data/icon.svg
|
|
789
793
|
- multiwoven-integrations.gemspec
|
|
790
794
|
- sig/multiwoven/integrations.rbs
|
|
791
795
|
homepage: https://www.multiwoven.com/
|