multiwoven-integrations 0.6.0 → 0.7.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json +2 -2
- data/lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json +1 -1
- data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json +7 -8
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/databricks/config/spec.json +1 -1
- data/lib/multiwoven/integrations/source/oracle_db/client.rb +127 -0
- data/lib/multiwoven/integrations/source/oracle_db/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/oracle_db/config/spec.json +47 -0
- data/lib/multiwoven/integrations/source/oracle_db/icon.svg +4 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e8eae28bf7587692f1d8e9fd6dcb4bdc3ced95ae23fe3edc8e88e13851fa4cb4
|
|
4
|
+
data.tar.gz: 4e0ba7c79a2f18ebf64b6821908086f275d55701d63d103d595efd7ea9ba2b0b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 730507c7e931280de1628ade5269fd87ca7c5cc60a7c49127fcbb1438048cc4f6f620f3601af293439a68063596134909bb1fb4a34e37171c2888c7d5af39f2d
|
|
7
|
+
data.tar.gz: 4aa67fd119b1dbd8d8bc491ac9afd97ed826b74eefa4600e7bae550ddae4caedf7b3e194a1a101b31117fc8534dc7ae7ec800a6212160653fd1010c855e42769
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"data": {
|
|
3
3
|
"name": "DatabricksLakehouse",
|
|
4
|
-
"title": "Databricks
|
|
4
|
+
"title": "Databricks Datawarehouse",
|
|
5
5
|
"connector_type": "destination",
|
|
6
|
-
"category": "
|
|
6
|
+
"category": "Database",
|
|
7
7
|
"documentation_url": "https://docs.multiwoven.com/destinations/databricks_lakehouse",
|
|
8
8
|
"github_issue_label": "destination-databricks-lakehouse",
|
|
9
9
|
"icon": "icon.svg",
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"stream_type": "static",
|
|
4
4
|
"connection_specification": {
|
|
5
5
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
-
"title": "Databricks
|
|
6
|
+
"title": "Databricks Datawarehouse",
|
|
7
7
|
"type": "object",
|
|
8
8
|
"required": ["host", "api_token", "warehouse_id", "catalog", "schema"],
|
|
9
9
|
"properties": {
|
|
@@ -23,7 +23,8 @@
|
|
|
23
23
|
"type": "string"
|
|
24
24
|
},
|
|
25
25
|
"private_key_id": {
|
|
26
|
-
"type": "string"
|
|
26
|
+
"type": "string",
|
|
27
|
+
"multiwoven_secret": true
|
|
27
28
|
},
|
|
28
29
|
"private_key": {
|
|
29
30
|
"type": "string",
|
|
@@ -34,19 +35,17 @@
|
|
|
34
35
|
"format": "email"
|
|
35
36
|
},
|
|
36
37
|
"client_id": {
|
|
37
|
-
"type": "string"
|
|
38
|
+
"type": "string",
|
|
39
|
+
"multiwoven_secret": true
|
|
38
40
|
},
|
|
39
41
|
"auth_uri": {
|
|
40
|
-
"type": "string"
|
|
41
|
-
"format": "uri"
|
|
42
|
+
"type": "string"
|
|
42
43
|
},
|
|
43
44
|
"token_uri": {
|
|
44
|
-
"type": "string"
|
|
45
|
-
"format": "uri"
|
|
45
|
+
"type": "string"
|
|
46
46
|
},
|
|
47
47
|
"auth_provider_x509_cert_url": {
|
|
48
|
-
"type": "string"
|
|
49
|
-
"format": "uri"
|
|
48
|
+
"type": "string"
|
|
50
49
|
},
|
|
51
50
|
"client_x509_cert_url": {
|
|
52
51
|
"type": "string",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.7.1"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -15,6 +15,7 @@ module Multiwoven
|
|
|
15
15
|
Clickhouse
|
|
16
16
|
AmazonS3
|
|
17
17
|
MariaDB
|
|
18
|
+
Oracle
|
|
18
19
|
].freeze
|
|
19
20
|
|
|
20
21
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
"connector_query_type": "raw_sql",
|
|
5
5
|
"connection_specification": {
|
|
6
6
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
-
"title": "Databricks",
|
|
7
|
+
"title": "Databricks Datawarehouse",
|
|
8
8
|
"type": "object",
|
|
9
9
|
"required": ["host", "port", "http_path", "catalog", "schema"],
|
|
10
10
|
"properties": {
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module Oracle
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
ConnectionStatus.new(
|
|
11
|
+
status: ConnectionStatusType["succeeded"]
|
|
12
|
+
).to_multiwoven_message
|
|
13
|
+
rescue StandardError => e
|
|
14
|
+
ConnectionStatus.new(
|
|
15
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
16
|
+
).to_multiwoven_message
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def discover(connection_config)
|
|
20
|
+
records = []
|
|
21
|
+
connection_config = connection_config.with_indifferent_access
|
|
22
|
+
query = "SELECT table_name, column_name, data_type, nullable
|
|
23
|
+
FROM all_tab_columns
|
|
24
|
+
WHERE owner = '#{connection_config[:username].upcase}'
|
|
25
|
+
ORDER BY table_name, column_id"
|
|
26
|
+
conn = create_connection(connection_config)
|
|
27
|
+
cursor = conn.exec(query)
|
|
28
|
+
while (row = cursor.fetch)
|
|
29
|
+
records << row
|
|
30
|
+
end
|
|
31
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
32
|
+
catalog.to_multiwoven_message
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
handle_exception(
|
|
35
|
+
"ORACLE:DISCOVER:EXCEPTION",
|
|
36
|
+
"error",
|
|
37
|
+
e
|
|
38
|
+
)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def read(sync_config)
|
|
42
|
+
connection_config = sync_config.source.connection_specification.with_indifferent_access
|
|
43
|
+
query = sync_config.model.query
|
|
44
|
+
db = create_connection(connection_config)
|
|
45
|
+
query(db, query)
|
|
46
|
+
rescue StandardError => e
|
|
47
|
+
handle_exception(e, {
|
|
48
|
+
context: "ORACLE:READ:EXCEPTION",
|
|
49
|
+
type: "error",
|
|
50
|
+
sync_id: sync_config.sync_id,
|
|
51
|
+
sync_run_id: sync_config.sync_run_id
|
|
52
|
+
})
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def create_connection(connection_config)
|
|
58
|
+
OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def create_streams(records)
|
|
62
|
+
group_by_table(records).map do |_, r|
|
|
63
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def query(connection, query)
|
|
68
|
+
records = []
|
|
69
|
+
query = reformat_query(query)
|
|
70
|
+
cursor = connection.exec(query)
|
|
71
|
+
columns = cursor.get_col_names
|
|
72
|
+
while (row = cursor.fetch)
|
|
73
|
+
data_hash = columns.zip(row).to_h
|
|
74
|
+
records << RecordMessage.new(data: data_hash, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
75
|
+
end
|
|
76
|
+
records
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def group_by_table(records)
|
|
80
|
+
result = {}
|
|
81
|
+
records.each_with_index do |entry, index|
|
|
82
|
+
table_name = entry[0]
|
|
83
|
+
column_data = {
|
|
84
|
+
column_name: entry[1],
|
|
85
|
+
data_type: entry[2],
|
|
86
|
+
is_nullable: entry[3] == "Y"
|
|
87
|
+
}
|
|
88
|
+
result[index] ||= {}
|
|
89
|
+
result[index][:tablename] = table_name
|
|
90
|
+
result[index][:columns] = [column_data]
|
|
91
|
+
end
|
|
92
|
+
result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
|
|
93
|
+
{ tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def reformat_query(sql_query)
|
|
98
|
+
offset = nil
|
|
99
|
+
limit = nil
|
|
100
|
+
|
|
101
|
+
sql_query = sql_query.gsub(";", "")
|
|
102
|
+
|
|
103
|
+
if sql_query.match?(/LIMIT (\d+)/i)
|
|
104
|
+
limit = sql_query.match(/LIMIT (\d+)/i)[1].to_i
|
|
105
|
+
sql_query.sub!(/LIMIT \d+/i, "")
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
if sql_query.match?(/OFFSET (\d+)/i)
|
|
109
|
+
offset = sql_query.match(/OFFSET (\d+)/i)[1].to_i
|
|
110
|
+
sql_query.sub!(/OFFSET \d+/i, "")
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
sql_query.strip!
|
|
114
|
+
|
|
115
|
+
if offset && limit
|
|
116
|
+
"#{sql_query} OFFSET #{offset} ROWS FETCH NEXT #{limit} ROWS ONLY"
|
|
117
|
+
elsif offset
|
|
118
|
+
"#{sql_query} OFFSET #{offset} ROWS"
|
|
119
|
+
elsif limit
|
|
120
|
+
"#{sql_query} FETCH NEXT #{limit} ROWS ONLY"
|
|
121
|
+
else
|
|
122
|
+
sql_query
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Oracle",
|
|
4
|
+
"title": "Oracle",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
|
|
8
|
+
"github_issue_label": "source-oracle",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Oracle",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["host", "port", "sid", "username", "password"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"host": {
|
|
12
|
+
"description": "The Oracle host.",
|
|
13
|
+
"examples": ["localhost"],
|
|
14
|
+
"type": "string",
|
|
15
|
+
"title": "Host",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"port": {
|
|
19
|
+
"description": "The Oracle port number.",
|
|
20
|
+
"examples": ["1521"],
|
|
21
|
+
"type": "string",
|
|
22
|
+
"title": "Port",
|
|
23
|
+
"order": 1
|
|
24
|
+
},
|
|
25
|
+
"sid": {
|
|
26
|
+
"description": "The name of your service in Oracle.",
|
|
27
|
+
"examples": ["ORCLPDB1"],
|
|
28
|
+
"type": "string",
|
|
29
|
+
"title": "SID",
|
|
30
|
+
"order": 2
|
|
31
|
+
},
|
|
32
|
+
"username": {
|
|
33
|
+
"description": "The username used to authenticate and connect.",
|
|
34
|
+
"type": "string",
|
|
35
|
+
"title": "Username",
|
|
36
|
+
"order": 3
|
|
37
|
+
},
|
|
38
|
+
"password": {
|
|
39
|
+
"description": "The password corresponding to the username used for authentication.",
|
|
40
|
+
"type": "string",
|
|
41
|
+
"multiwoven_secret": true,
|
|
42
|
+
"title": "Password",
|
|
43
|
+
"order": 4
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
|
|
2
|
+
<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
|
|
3
|
+
<path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
|
|
4
|
+
</svg>
|
|
@@ -61,6 +61,7 @@ require_relative "integrations/source/aws_athena/client"
|
|
|
61
61
|
require_relative "integrations/source/clickhouse/client"
|
|
62
62
|
require_relative "integrations/source/amazon_s3/client"
|
|
63
63
|
require_relative "integrations/source/maria_db/client"
|
|
64
|
+
require_relative "integrations/source/oracle_db/client"
|
|
64
65
|
|
|
65
66
|
# Destination
|
|
66
67
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.7.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-08-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -611,6 +611,10 @@ files:
|
|
|
611
611
|
- lib/multiwoven/integrations/source/maria_db/config/meta.json
|
|
612
612
|
- lib/multiwoven/integrations/source/maria_db/config/spec.json
|
|
613
613
|
- lib/multiwoven/integrations/source/maria_db/icon.svg
|
|
614
|
+
- lib/multiwoven/integrations/source/oracle_db/client.rb
|
|
615
|
+
- lib/multiwoven/integrations/source/oracle_db/config/meta.json
|
|
616
|
+
- lib/multiwoven/integrations/source/oracle_db/config/spec.json
|
|
617
|
+
- lib/multiwoven/integrations/source/oracle_db/icon.svg
|
|
614
618
|
- lib/multiwoven/integrations/source/postgresql/client.rb
|
|
615
619
|
- lib/multiwoven/integrations/source/postgresql/config/meta.json
|
|
616
620
|
- lib/multiwoven/integrations/source/postgresql/config/spec.json
|