multiwoven-integrations 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/maria_db/client.rb +3 -1
- data/lib/multiwoven/integrations/destination/oracle_db/client.rb +112 -0
- data/lib/multiwoven/integrations/destination/oracle_db/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/oracle_db/config/spec.json +47 -0
- data/lib/multiwoven/integrations/destination/oracle_db/icon.svg +4 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations.rb +2 -0
- data/multiwoven-integrations.gemspec +1 -0
- metadata +20 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a92fb5b20e3b6fae0fea1e7cbffe45c849db3da972627e1756ce44089bcb064c
|
|
4
|
+
data.tar.gz: 1caef02be1a6d299ad7f3676cd9cce821ddb04af53dc7d1e804520d241612ccb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 00c63f04d92804636a152d245435d9d7bb06744947c8d8e2c84dac1b4ed84cd80b546e73ad6d4f5d92c7b8e20b76e078afebc8864d8d678c6e434cbf81aba456
|
|
7
|
+
data.tar.gz: 6b1c65e3d1463089bfc0237a7443d2504147ab3d1cde5bcbf126b3ea0a1c7d2937920ebd79a54fabb7978118191bf6df1146d0db955aa75ca5aef27a28ceaeb9
|
|
@@ -102,7 +102,9 @@ module Multiwoven::Integrations::Destination
|
|
|
102
102
|
result[index][:tablename] = table_name
|
|
103
103
|
result[index][:columns] = [column_data]
|
|
104
104
|
end
|
|
105
|
-
result
|
|
105
|
+
result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
|
|
106
|
+
{ tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
|
|
107
|
+
end
|
|
106
108
|
end
|
|
107
109
|
|
|
108
110
|
def tracking_message(success, failure)
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Destination
|
|
4
|
+
module Oracle
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < DestinationConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
ConnectionStatus.new(
|
|
11
|
+
status: ConnectionStatusType["succeeded"]
|
|
12
|
+
).to_multiwoven_message
|
|
13
|
+
rescue StandardError => e
|
|
14
|
+
ConnectionStatus.new(
|
|
15
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
16
|
+
).to_multiwoven_message
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def discover(connection_config)
|
|
20
|
+
records = []
|
|
21
|
+
connection_config = connection_config.with_indifferent_access
|
|
22
|
+
query = "SELECT table_name, column_name, data_type, nullable
|
|
23
|
+
FROM all_tab_columns
|
|
24
|
+
WHERE owner = '#{connection_config[:username].upcase}'
|
|
25
|
+
ORDER BY table_name, column_id"
|
|
26
|
+
conn = create_connection(connection_config)
|
|
27
|
+
cursor = conn.exec(query)
|
|
28
|
+
while (row = cursor.fetch)
|
|
29
|
+
records << row
|
|
30
|
+
end
|
|
31
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
32
|
+
catalog.to_multiwoven_message
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
handle_exception(
|
|
35
|
+
"ORACLE:DISCOVER:EXCEPTION",
|
|
36
|
+
"error",
|
|
37
|
+
e
|
|
38
|
+
)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def write(sync_config, records, action = "destination_insert")
|
|
42
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
43
|
+
table_name = sync_config.stream.name
|
|
44
|
+
primary_key = sync_config.model.primary_key
|
|
45
|
+
conn = create_connection(connection_config)
|
|
46
|
+
|
|
47
|
+
write_success = 0
|
|
48
|
+
write_failure = 0
|
|
49
|
+
log_message_array = []
|
|
50
|
+
|
|
51
|
+
records.each do |record|
|
|
52
|
+
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
|
|
53
|
+
query = query.gsub(";", "")
|
|
54
|
+
logger.debug("ORACLE:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
|
|
55
|
+
begin
|
|
56
|
+
response = conn.exec(query)
|
|
57
|
+
conn.exec("COMMIT")
|
|
58
|
+
write_success += 1
|
|
59
|
+
log_message_array << log_request_response("info", query, response)
|
|
60
|
+
rescue StandardError => e
|
|
61
|
+
handle_exception(e, {
|
|
62
|
+
context: "ORACLE:RECORD:WRITE:EXCEPTION",
|
|
63
|
+
type: "error",
|
|
64
|
+
sync_id: sync_config.sync_id,
|
|
65
|
+
sync_run_id: sync_config.sync_run_id
|
|
66
|
+
})
|
|
67
|
+
write_failure += 1
|
|
68
|
+
log_message_array << log_request_response("error", query, e.message)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
tracking_message(write_success, write_failure, log_message_array)
|
|
72
|
+
rescue StandardError => e
|
|
73
|
+
handle_exception(e, {
|
|
74
|
+
context: "ORACLE:RECORD:WRITE:EXCEPTION",
|
|
75
|
+
type: "error",
|
|
76
|
+
sync_id: sync_config.sync_id,
|
|
77
|
+
sync_run_id: sync_config.sync_run_id
|
|
78
|
+
})
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def create_connection(connection_config)
|
|
84
|
+
OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def create_streams(records)
|
|
88
|
+
group_by_table(records).map do |_, r|
|
|
89
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def group_by_table(records)
|
|
94
|
+
result = {}
|
|
95
|
+
records.each_with_index do |entry, index|
|
|
96
|
+
table_name = entry[0]
|
|
97
|
+
column_data = {
|
|
98
|
+
column_name: entry[1],
|
|
99
|
+
data_type: entry[2],
|
|
100
|
+
is_nullable: entry[3] == "Y"
|
|
101
|
+
}
|
|
102
|
+
result[index] ||= {}
|
|
103
|
+
result[index][:tablename] = table_name
|
|
104
|
+
result[index][:columns] = [column_data]
|
|
105
|
+
end
|
|
106
|
+
result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
|
|
107
|
+
{ tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Oracle",
|
|
4
|
+
"title": "Oracle",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle",
|
|
8
|
+
"github_issue_label": "destination-oracle",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/destination/oracle",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Oracle",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["host", "port", "sid", "username", "password"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"host": {
|
|
12
|
+
"description": "The Oracle host.",
|
|
13
|
+
"examples": ["localhost"],
|
|
14
|
+
"type": "string",
|
|
15
|
+
"title": "Host",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"port": {
|
|
19
|
+
"description": "The Oracle port number.",
|
|
20
|
+
"examples": ["1521"],
|
|
21
|
+
"type": "string",
|
|
22
|
+
"title": "Port",
|
|
23
|
+
"order": 1
|
|
24
|
+
},
|
|
25
|
+
"sid": {
|
|
26
|
+
"description": "The name of your service in Oracle.",
|
|
27
|
+
"examples": ["ORCLPDB1"],
|
|
28
|
+
"type": "string",
|
|
29
|
+
"title": "SID",
|
|
30
|
+
"order": 2
|
|
31
|
+
},
|
|
32
|
+
"username": {
|
|
33
|
+
"description": "The username used to authenticate and connect.",
|
|
34
|
+
"type": "string",
|
|
35
|
+
"title": "Username",
|
|
36
|
+
"order": 3
|
|
37
|
+
},
|
|
38
|
+
"password": {
|
|
39
|
+
"description": "The password corresponding to the username used for authentication.",
|
|
40
|
+
"type": "string",
|
|
41
|
+
"multiwoven_secret": true,
|
|
42
|
+
"title": "Password",
|
|
43
|
+
"order": 4
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
|
|
2
|
+
<svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
|
|
3
|
+
<path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
|
|
4
|
+
</svg>
|
|
@@ -31,6 +31,7 @@ require "aws-sdk-s3"
|
|
|
31
31
|
require "duckdb"
|
|
32
32
|
require "iterable-api-client"
|
|
33
33
|
require "aws-sdk-sts"
|
|
34
|
+
require "ruby-oci8"
|
|
34
35
|
|
|
35
36
|
# Service
|
|
36
37
|
require_relative "integrations/config"
|
|
@@ -78,6 +79,7 @@ require_relative "integrations/destination/zendesk/client"
|
|
|
78
79
|
require_relative "integrations/destination/iterable/client"
|
|
79
80
|
require_relative "integrations/destination/maria_db/client"
|
|
80
81
|
require_relative "integrations/destination/databricks_lakehouse/client"
|
|
82
|
+
require_relative "integrations/destination/oracle_db/client"
|
|
81
83
|
|
|
82
84
|
module Multiwoven
|
|
83
85
|
module Integrations
|
|
@@ -53,6 +53,7 @@ Gem::Specification.new do |spec|
|
|
|
53
53
|
spec.add_runtime_dependency "rake"
|
|
54
54
|
spec.add_runtime_dependency "restforce"
|
|
55
55
|
spec.add_runtime_dependency "ruby-limiter"
|
|
56
|
+
spec.add_runtime_dependency "ruby-oci8"
|
|
56
57
|
spec.add_runtime_dependency "ruby-odbc"
|
|
57
58
|
spec.add_runtime_dependency "rubyzip"
|
|
58
59
|
spec.add_runtime_dependency "sequel"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-07-
|
|
11
|
+
date: 2024-07-26 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -290,6 +290,20 @@ dependencies:
|
|
|
290
290
|
- - ">="
|
|
291
291
|
- !ruby/object:Gem::Version
|
|
292
292
|
version: '0'
|
|
293
|
+
- !ruby/object:Gem::Dependency
|
|
294
|
+
name: ruby-oci8
|
|
295
|
+
requirement: !ruby/object:Gem::Requirement
|
|
296
|
+
requirements:
|
|
297
|
+
- - ">="
|
|
298
|
+
- !ruby/object:Gem::Version
|
|
299
|
+
version: '0'
|
|
300
|
+
type: :runtime
|
|
301
|
+
prerelease: false
|
|
302
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
303
|
+
requirements:
|
|
304
|
+
- - ">="
|
|
305
|
+
- !ruby/object:Gem::Version
|
|
306
|
+
version: '0'
|
|
293
307
|
- !ruby/object:Gem::Dependency
|
|
294
308
|
name: ruby-odbc
|
|
295
309
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -530,6 +544,10 @@ files:
|
|
|
530
544
|
- lib/multiwoven/integrations/destination/maria_db/config/meta.json
|
|
531
545
|
- lib/multiwoven/integrations/destination/maria_db/config/spec.json
|
|
532
546
|
- lib/multiwoven/integrations/destination/maria_db/icon.svg
|
|
547
|
+
- lib/multiwoven/integrations/destination/oracle_db/client.rb
|
|
548
|
+
- lib/multiwoven/integrations/destination/oracle_db/config/meta.json
|
|
549
|
+
- lib/multiwoven/integrations/destination/oracle_db/config/spec.json
|
|
550
|
+
- lib/multiwoven/integrations/destination/oracle_db/icon.svg
|
|
533
551
|
- lib/multiwoven/integrations/destination/postgresql/client.rb
|
|
534
552
|
- lib/multiwoven/integrations/destination/postgresql/config/meta.json
|
|
535
553
|
- lib/multiwoven/integrations/destination/postgresql/config/spec.json
|