multiwoven-integrations 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/databricks_lakehouse/config/meta.json +2 -2
- data/lib/multiwoven/integrations/destination/databricks_lakehouse/config/spec.json +1 -1
- data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json +7 -8
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/databricks/config/spec.json +1 -1
- data/lib/multiwoven/integrations/source/oracle_db/client.rb +127 -0
- data/lib/multiwoven/integrations/source/oracle_db/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/oracle_db/config/spec.json +47 -0
- data/lib/multiwoven/integrations/source/oracle_db/icon.svg +4 -0
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e8eae28bf7587692f1d8e9fd6dcb4bdc3ced95ae23fe3edc8e88e13851fa4cb4
|
|
4
|
+
data.tar.gz: 4e0ba7c79a2f18ebf64b6821908086f275d55701d63d103d595efd7ea9ba2b0b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 730507c7e931280de1628ade5269fd87ca7c5cc60a7c49127fcbb1438048cc4f6f620f3601af293439a68063596134909bb1fb4a34e37171c2888c7d5af39f2d
|
|
7
|
+
data.tar.gz: 4aa67fd119b1dbd8d8bc491ac9afd97ed826b74eefa4600e7bae550ddae4caedf7b3e194a1a101b31117fc8534dc7ae7ec800a6212160653fd1010c855e42769
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"data": {
|
|
3
3
|
"name": "DatabricksLakehouse",
|
|
4
|
-
"title": "Databricks
|
|
4
|
+
"title": "Databricks Datawarehouse",
|
|
5
5
|
"connector_type": "destination",
|
|
6
|
-
"category": "
|
|
6
|
+
"category": "Database",
|
|
7
7
|
"documentation_url": "https://docs.multiwoven.com/destinations/databricks_lakehouse",
|
|
8
8
|
"github_issue_label": "destination-databricks-lakehouse",
|
|
9
9
|
"icon": "icon.svg",
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"stream_type": "static",
|
|
4
4
|
"connection_specification": {
|
|
5
5
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
-
"title": "Databricks
|
|
6
|
+
"title": "Databricks Datawarehouse",
|
|
7
7
|
"type": "object",
|
|
8
8
|
"required": ["host", "api_token", "warehouse_id", "catalog", "schema"],
|
|
9
9
|
"properties": {
|
|
@@ -23,7 +23,8 @@
|
|
|
23
23
|
"type": "string"
|
|
24
24
|
},
|
|
25
25
|
"private_key_id": {
|
|
26
|
-
"type": "string"
|
|
26
|
+
"type": "string",
|
|
27
|
+
"multiwoven_secret": true
|
|
27
28
|
},
|
|
28
29
|
"private_key": {
|
|
29
30
|
"type": "string",
|
|
@@ -34,19 +35,17 @@
|
|
|
34
35
|
"format": "email"
|
|
35
36
|
},
|
|
36
37
|
"client_id": {
|
|
37
|
-
"type": "string"
|
|
38
|
+
"type": "string",
|
|
39
|
+
"multiwoven_secret": true
|
|
38
40
|
},
|
|
39
41
|
"auth_uri": {
|
|
40
|
-
"type": "string"
|
|
41
|
-
"format": "uri"
|
|
42
|
+
"type": "string"
|
|
42
43
|
},
|
|
43
44
|
"token_uri": {
|
|
44
|
-
"type": "string"
|
|
45
|
-
"format": "uri"
|
|
45
|
+
"type": "string"
|
|
46
46
|
},
|
|
47
47
|
"auth_provider_x509_cert_url": {
|
|
48
|
-
"type": "string"
|
|
49
|
-
"format": "uri"
|
|
48
|
+
"type": "string"
|
|
50
49
|
},
|
|
51
50
|
"client_x509_cert_url": {
|
|
52
51
|
"type": "string",
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.7.1"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -15,6 +15,7 @@ module Multiwoven
|
|
|
15
15
|
Clickhouse
|
|
16
16
|
AmazonS3
|
|
17
17
|
MariaDB
|
|
18
|
+
Oracle
|
|
18
19
|
].freeze
|
|
19
20
|
|
|
20
21
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
"connector_query_type": "raw_sql",
|
|
5
5
|
"connection_specification": {
|
|
6
6
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
-
"title": "Databricks",
|
|
7
|
+
"title": "Databricks Datawarehouse",
|
|
8
8
|
"type": "object",
|
|
9
9
|
"required": ["host", "port", "http_path", "catalog", "schema"],
|
|
10
10
|
"properties": {
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module Oracle
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
ConnectionStatus.new(
|
|
11
|
+
status: ConnectionStatusType["succeeded"]
|
|
12
|
+
).to_multiwoven_message
|
|
13
|
+
rescue StandardError => e
|
|
14
|
+
ConnectionStatus.new(
|
|
15
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
16
|
+
).to_multiwoven_message
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def discover(connection_config)
|
|
20
|
+
records = []
|
|
21
|
+
connection_config = connection_config.with_indifferent_access
|
|
22
|
+
query = "SELECT table_name, column_name, data_type, nullable
|
|
23
|
+
FROM all_tab_columns
|
|
24
|
+
WHERE owner = '#{connection_config[:username].upcase}'
|
|
25
|
+
ORDER BY table_name, column_id"
|
|
26
|
+
conn = create_connection(connection_config)
|
|
27
|
+
cursor = conn.exec(query)
|
|
28
|
+
while (row = cursor.fetch)
|
|
29
|
+
records << row
|
|
30
|
+
end
|
|
31
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
32
|
+
catalog.to_multiwoven_message
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
handle_exception(
|
|
35
|
+
"ORACLE:DISCOVER:EXCEPTION",
|
|
36
|
+
"error",
|
|
37
|
+
e
|
|
38
|
+
)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def read(sync_config)
|
|
42
|
+
connection_config = sync_config.source.connection_specification.with_indifferent_access
|
|
43
|
+
query = sync_config.model.query
|
|
44
|
+
db = create_connection(connection_config)
|
|
45
|
+
query(db, query)
|
|
46
|
+
rescue StandardError => e
|
|
47
|
+
handle_exception(e, {
|
|
48
|
+
context: "ORACLE:READ:EXCEPTION",
|
|
49
|
+
type: "error",
|
|
50
|
+
sync_id: sync_config.sync_id,
|
|
51
|
+
sync_run_id: sync_config.sync_run_id
|
|
52
|
+
})
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def create_connection(connection_config)
|
|
58
|
+
OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def create_streams(records)
|
|
62
|
+
group_by_table(records).map do |_, r|
|
|
63
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def query(connection, query)
|
|
68
|
+
records = []
|
|
69
|
+
query = reformat_query(query)
|
|
70
|
+
cursor = connection.exec(query)
|
|
71
|
+
columns = cursor.get_col_names
|
|
72
|
+
while (row = cursor.fetch)
|
|
73
|
+
data_hash = columns.zip(row).to_h
|
|
74
|
+
records << RecordMessage.new(data: data_hash, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
75
|
+
end
|
|
76
|
+
records
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def group_by_table(records)
|
|
80
|
+
result = {}
|
|
81
|
+
records.each_with_index do |entry, index|
|
|
82
|
+
table_name = entry[0]
|
|
83
|
+
column_data = {
|
|
84
|
+
column_name: entry[1],
|
|
85
|
+
data_type: entry[2],
|
|
86
|
+
is_nullable: entry[3] == "Y"
|
|
87
|
+
}
|
|
88
|
+
result[index] ||= {}
|
|
89
|
+
result[index][:tablename] = table_name
|
|
90
|
+
result[index][:columns] = [column_data]
|
|
91
|
+
end
|
|
92
|
+
result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
|
|
93
|
+
{ tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def reformat_query(sql_query)
|
|
98
|
+
offset = nil
|
|
99
|
+
limit = nil
|
|
100
|
+
|
|
101
|
+
sql_query = sql_query.gsub(";", "")
|
|
102
|
+
|
|
103
|
+
if sql_query.match?(/LIMIT (\d+)/i)
|
|
104
|
+
limit = sql_query.match(/LIMIT (\d+)/i)[1].to_i
|
|
105
|
+
sql_query.sub!(/LIMIT \d+/i, "")
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
if sql_query.match?(/OFFSET (\d+)/i)
|
|
109
|
+
offset = sql_query.match(/OFFSET (\d+)/i)[1].to_i
|
|
110
|
+
sql_query.sub!(/OFFSET \d+/i, "")
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
sql_query.strip!
|
|
114
|
+
|
|
115
|
+
if offset && limit
|
|
116
|
+
"#{sql_query} OFFSET #{offset} ROWS FETCH NEXT #{limit} ROWS ONLY"
|
|
117
|
+
elsif offset
|
|
118
|
+
"#{sql_query} OFFSET #{offset} ROWS"
|
|
119
|
+
elsif limit
|
|
120
|
+
"#{sql_query} FETCH NEXT #{limit} ROWS ONLY"
|
|
121
|
+
else
|
|
122
|
+
sql_query
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Oracle",
|
|
4
|
+
"title": "Oracle",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
|
|
8
|
+
"github_issue_label": "source-oracle",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Oracle",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["host", "port", "sid", "username", "password"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"host": {
|
|
12
|
+
"description": "The Oracle host.",
|
|
13
|
+
"examples": ["localhost"],
|
|
14
|
+
"type": "string",
|
|
15
|
+
"title": "Host",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"port": {
|
|
19
|
+
"description": "The Oracle port number.",
|
|
20
|
+
"examples": ["1521"],
|
|
21
|
+
"type": "string",
|
|
22
|
+
"title": "Port",
|
|
23
|
+
"order": 1
|
|
24
|
+
},
|
|
25
|
+
"sid": {
|
|
26
|
+
"description": "The name of your service in Oracle.",
|
|
27
|
+
"examples": ["ORCLPDB1"],
|
|
28
|
+
"type": "string",
|
|
29
|
+
"title": "SID",
|
|
30
|
+
"order": 2
|
|
31
|
+
},
|
|
32
|
+
"username": {
|
|
33
|
+
"description": "The username used to authenticate and connect.",
|
|
34
|
+
"type": "string",
|
|
35
|
+
"title": "Username",
|
|
36
|
+
"order": 3
|
|
37
|
+
},
|
|
38
|
+
"password": {
|
|
39
|
+
"description": "The password corresponding to the username used for authentication.",
|
|
40
|
+
"type": "string",
|
|
41
|
+
"multiwoven_secret": true,
|
|
42
|
+
"title": "Password",
|
|
43
|
+
"order": 4
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
|
|
2
|
+
<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
|
|
3
|
+
<path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
|
|
4
|
+
</svg>
|
|
@@ -61,6 +61,7 @@ require_relative "integrations/source/aws_athena/client"
|
|
|
61
61
|
require_relative "integrations/source/clickhouse/client"
|
|
62
62
|
require_relative "integrations/source/amazon_s3/client"
|
|
63
63
|
require_relative "integrations/source/maria_db/client"
|
|
64
|
+
require_relative "integrations/source/oracle_db/client"
|
|
64
65
|
|
|
65
66
|
# Destination
|
|
66
67
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.7.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-08-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -611,6 +611,10 @@ files:
|
|
|
611
611
|
- lib/multiwoven/integrations/source/maria_db/config/meta.json
|
|
612
612
|
- lib/multiwoven/integrations/source/maria_db/config/spec.json
|
|
613
613
|
- lib/multiwoven/integrations/source/maria_db/icon.svg
|
|
614
|
+
- lib/multiwoven/integrations/source/oracle_db/client.rb
|
|
615
|
+
- lib/multiwoven/integrations/source/oracle_db/config/meta.json
|
|
616
|
+
- lib/multiwoven/integrations/source/oracle_db/config/spec.json
|
|
617
|
+
- lib/multiwoven/integrations/source/oracle_db/icon.svg
|
|
614
618
|
- lib/multiwoven/integrations/source/postgresql/client.rb
|
|
615
619
|
- lib/multiwoven/integrations/source/postgresql/config/meta.json
|
|
616
620
|
- lib/multiwoven/integrations/source/postgresql/config/spec.json
|