multiwoven-integrations 0.6.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a92fb5b20e3b6fae0fea1e7cbffe45c849db3da972627e1756ce44089bcb064c
4
- data.tar.gz: 1caef02be1a6d299ad7f3676cd9cce821ddb04af53dc7d1e804520d241612ccb
3
+ metadata.gz: e8eae28bf7587692f1d8e9fd6dcb4bdc3ced95ae23fe3edc8e88e13851fa4cb4
4
+ data.tar.gz: 4e0ba7c79a2f18ebf64b6821908086f275d55701d63d103d595efd7ea9ba2b0b
5
5
  SHA512:
6
- metadata.gz: 00c63f04d92804636a152d245435d9d7bb06744947c8d8e2c84dac1b4ed84cd80b546e73ad6d4f5d92c7b8e20b76e078afebc8864d8d678c6e434cbf81aba456
7
- data.tar.gz: 6b1c65e3d1463089bfc0237a7443d2504147ab3d1cde5bcbf126b3ea0a1c7d2937920ebd79a54fabb7978118191bf6df1146d0db955aa75ca5aef27a28ceaeb9
6
+ metadata.gz: 730507c7e931280de1628ade5269fd87ca7c5cc60a7c49127fcbb1438048cc4f6f620f3601af293439a68063596134909bb1fb4a34e37171c2888c7d5af39f2d
7
+ data.tar.gz: 4aa67fd119b1dbd8d8bc491ac9afd97ed826b74eefa4600e7bae550ddae4caedf7b3e194a1a101b31117fc8534dc7ae7ec800a6212160653fd1010c855e42769
@@ -1,9 +1,9 @@
1
1
  {
2
2
  "data": {
3
3
  "name": "DatabricksLakehouse",
4
- "title": "Databricks Lakehouse",
4
+ "title": "Databricks Datawarehouse",
5
5
  "connector_type": "destination",
6
- "category": "Marketing Automation",
6
+ "category": "Database",
7
7
  "documentation_url": "https://docs.multiwoven.com/destinations/databricks_lakehouse",
8
8
  "github_issue_label": "destination-databricks-lakehouse",
9
9
  "icon": "icon.svg",
@@ -3,7 +3,7 @@
3
3
  "stream_type": "static",
4
4
  "connection_specification": {
5
5
  "$schema": "http://json-schema.org/draft-07/schema#",
6
- "title": "Databricks Lakehouse",
6
+ "title": "Databricks Datawarehouse",
7
7
  "type": "object",
8
8
  "required": ["host", "api_token", "warehouse_id", "catalog", "schema"],
9
9
  "properties": {
@@ -23,7 +23,8 @@
23
23
  "type": "string"
24
24
  },
25
25
  "private_key_id": {
26
- "type": "string"
26
+ "type": "string",
27
+ "multiwoven_secret": true
27
28
  },
28
29
  "private_key": {
29
30
  "type": "string",
@@ -34,19 +35,17 @@
34
35
  "format": "email"
35
36
  },
36
37
  "client_id": {
37
- "type": "string"
38
+ "type": "string",
39
+ "multiwoven_secret": true
38
40
  },
39
41
  "auth_uri": {
40
- "type": "string",
41
- "format": "uri"
42
+ "type": "string"
42
43
  },
43
44
  "token_uri": {
44
- "type": "string",
45
- "format": "uri"
45
+ "type": "string"
46
46
  },
47
47
  "auth_provider_x509_cert_url": {
48
- "type": "string",
49
- "format": "uri"
48
+ "type": "string"
50
49
  },
51
50
  "client_x509_cert_url": {
52
51
  "type": "string",
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Multiwoven
4
4
  module Integrations
5
- VERSION = "0.6.0"
5
+ VERSION = "0.7.1"
6
6
 
7
7
  ENABLED_SOURCES = %w[
8
8
  Snowflake
@@ -15,6 +15,7 @@ module Multiwoven
15
15
  Clickhouse
16
16
  AmazonS3
17
17
  MariaDB
18
+ Oracle
18
19
  ].freeze
19
20
 
20
21
  ENABLED_DESTINATIONS = %w[
@@ -4,7 +4,7 @@
4
4
  "connector_query_type": "raw_sql",
5
5
  "connection_specification": {
6
6
  "$schema": "http://json-schema.org/draft-07/schema#",
7
- "title": "Databricks",
7
+ "title": "Databricks Datawarehouse",
8
8
  "type": "object",
9
9
  "required": ["host", "port", "http_path", "catalog", "schema"],
10
10
  "properties": {
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven::Integrations::Source
4
+ module Oracle
5
+ include Multiwoven::Integrations::Core
6
+ class Client < SourceConnector
7
+ def check_connection(connection_config)
8
+ connection_config = connection_config.with_indifferent_access
9
+ create_connection(connection_config)
10
+ ConnectionStatus.new(
11
+ status: ConnectionStatusType["succeeded"]
12
+ ).to_multiwoven_message
13
+ rescue StandardError => e
14
+ ConnectionStatus.new(
15
+ status: ConnectionStatusType["failed"], message: e.message
16
+ ).to_multiwoven_message
17
+ end
18
+
19
+ def discover(connection_config)
20
+ records = []
21
+ connection_config = connection_config.with_indifferent_access
22
+ query = "SELECT table_name, column_name, data_type, nullable
23
+ FROM all_tab_columns
24
+ WHERE owner = '#{connection_config[:username].upcase}'
25
+ ORDER BY table_name, column_id"
26
+ conn = create_connection(connection_config)
27
+ cursor = conn.exec(query)
28
+ while (row = cursor.fetch)
29
+ records << row
30
+ end
31
+ catalog = Catalog.new(streams: create_streams(records))
32
+ catalog.to_multiwoven_message
33
+ rescue StandardError => e
34
+ handle_exception(
35
+ "ORACLE:DISCOVER:EXCEPTION",
36
+ "error",
37
+ e
38
+ )
39
+ end
40
+
41
+ def read(sync_config)
42
+ connection_config = sync_config.source.connection_specification.with_indifferent_access
43
+ query = sync_config.model.query
44
+ db = create_connection(connection_config)
45
+ query(db, query)
46
+ rescue StandardError => e
47
+ handle_exception(e, {
48
+ context: "ORACLE:READ:EXCEPTION",
49
+ type: "error",
50
+ sync_id: sync_config.sync_id,
51
+ sync_run_id: sync_config.sync_run_id
52
+ })
53
+ end
54
+
55
+ private
56
+
57
+ def create_connection(connection_config)
58
+ OCI8.new(connection_config[:username], connection_config[:password], "#{connection_config[:host]}:#{connection_config[:port]}/#{connection_config[:sid]}")
59
+ end
60
+
61
+ def create_streams(records)
62
+ group_by_table(records).map do |_, r|
63
+ Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
64
+ end
65
+ end
66
+
67
+ def query(connection, query)
68
+ records = []
69
+ query = reformat_query(query)
70
+ cursor = connection.exec(query)
71
+ columns = cursor.get_col_names
72
+ while (row = cursor.fetch)
73
+ data_hash = columns.zip(row).to_h
74
+ records << RecordMessage.new(data: data_hash, emitted_at: Time.now.to_i).to_multiwoven_message
75
+ end
76
+ records
77
+ end
78
+
79
+ def group_by_table(records)
80
+ result = {}
81
+ records.each_with_index do |entry, index|
82
+ table_name = entry[0]
83
+ column_data = {
84
+ column_name: entry[1],
85
+ data_type: entry[2],
86
+ is_nullable: entry[3] == "Y"
87
+ }
88
+ result[index] ||= {}
89
+ result[index][:tablename] = table_name
90
+ result[index][:columns] = [column_data]
91
+ end
92
+ result.values.group_by { |entry| entry[:tablename] }.transform_values do |entries|
93
+ { tablename: entries.first[:tablename], columns: entries.flat_map { |entry| entry[:columns] } }
94
+ end
95
+ end
96
+
97
+ def reformat_query(sql_query)
98
+ offset = nil
99
+ limit = nil
100
+
101
+ sql_query = sql_query.gsub(";", "")
102
+
103
+ if sql_query.match?(/LIMIT (\d+)/i)
104
+ limit = sql_query.match(/LIMIT (\d+)/i)[1].to_i
105
+ sql_query.sub!(/LIMIT \d+/i, "")
106
+ end
107
+
108
+ if sql_query.match?(/OFFSET (\d+)/i)
109
+ offset = sql_query.match(/OFFSET (\d+)/i)[1].to_i
110
+ sql_query.sub!(/OFFSET \d+/i, "")
111
+ end
112
+
113
+ sql_query.strip!
114
+
115
+ if offset && limit
116
+ "#{sql_query} OFFSET #{offset} ROWS FETCH NEXT #{limit} ROWS ONLY"
117
+ elsif offset
118
+ "#{sql_query} OFFSET #{offset} ROWS"
119
+ elsif limit
120
+ "#{sql_query} FETCH NEXT #{limit} ROWS ONLY"
121
+ else
122
+ sql_query
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "Oracle",
4
+ "title": "Oracle",
5
+ "connector_type": "source",
6
+ "category": "Database",
7
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
8
+ "github_issue_label": "source-oracle",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,47 @@
1
+ {
2
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/source/oracle",
3
+ "stream_type": "dynamic",
4
+ "connector_query_type": "raw_sql",
5
+ "connection_specification": {
6
+ "$schema": "http://json-schema.org/draft-07/schema#",
7
+ "title": "Oracle",
8
+ "type": "object",
9
+ "required": ["host", "port", "sid", "username", "password"],
10
+ "properties": {
11
+ "host": {
12
+ "description": "The Oracle host.",
13
+ "examples": ["localhost"],
14
+ "type": "string",
15
+ "title": "Host",
16
+ "order": 0
17
+ },
18
+ "port": {
19
+ "description": "The Oracle port number.",
20
+ "examples": ["1521"],
21
+ "type": "string",
22
+ "title": "Port",
23
+ "order": 1
24
+ },
25
+ "sid": {
26
+ "description": "The name of your service in Oracle.",
27
+ "examples": ["ORCLPDB1"],
28
+ "type": "string",
29
+ "title": "SID",
30
+ "order": 2
31
+ },
32
+ "username": {
33
+ "description": "The username used to authenticate and connect.",
34
+ "type": "string",
35
+ "title": "Username",
36
+ "order": 3
37
+ },
38
+ "password": {
39
+ "description": "The password corresponding to the username used for authentication.",
40
+ "type": "string",
41
+ "multiwoven_secret": true,
42
+ "title": "Password",
43
+ "order": 4
44
+ }
45
+ }
46
+ }
47
+ }
@@ -0,0 +1,4 @@
1
+ <?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
2
+ <svg width="800px" height="800px" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg">
3
+ <path fill="#F00" fill-rule="evenodd" d="M7.957359,18.9123664 C4.11670252,18.9123664 1,15.803458 1,11.9617373 C1,8.12000773 4.11670252,5 7.957359,5 L16.0437948,5 C19.8855156,5 23,8.12000773 23,11.9617373 C23,15.803458 19.8855156,18.9123664 16.0437948,18.9123664 L7.957359,18.9123664 L7.957359,18.9123664 Z M15.8639176,16.4585488 C18.352201,16.4585488 20.3674397,14.448858 20.3674397,11.9617373 C20.3674397,9.47460595 18.352201,7.45381934 15.8639176,7.45381934 L8.1360824,7.45381934 C5.64895285,7.45381934 3.63255855,9.47460595 3.63255855,11.9617373 C3.63255855,14.448858 5.64895285,16.4585488 8.1360824,16.4585488 L15.8639176,16.4585488 L15.8639176,16.4585488 Z"/>
4
+ </svg>
@@ -61,6 +61,7 @@ require_relative "integrations/source/aws_athena/client"
61
61
  require_relative "integrations/source/clickhouse/client"
62
62
  require_relative "integrations/source/amazon_s3/client"
63
63
  require_relative "integrations/source/maria_db/client"
64
+ require_relative "integrations/source/oracle_db/client"
64
65
 
65
66
  # Destination
66
67
  require_relative "integrations/destination/klaviyo/client"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiwoven-integrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.7.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Subin T P
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-07-26 00:00:00.000000000 Z
11
+ date: 2024-08-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -611,6 +611,10 @@ files:
611
611
  - lib/multiwoven/integrations/source/maria_db/config/meta.json
612
612
  - lib/multiwoven/integrations/source/maria_db/config/spec.json
613
613
  - lib/multiwoven/integrations/source/maria_db/icon.svg
614
+ - lib/multiwoven/integrations/source/oracle_db/client.rb
615
+ - lib/multiwoven/integrations/source/oracle_db/config/meta.json
616
+ - lib/multiwoven/integrations/source/oracle_db/config/spec.json
617
+ - lib/multiwoven/integrations/source/oracle_db/icon.svg
614
618
  - lib/multiwoven/integrations/source/postgresql/client.rb
615
619
  - lib/multiwoven/integrations/source/postgresql/config/meta.json
616
620
  - lib/multiwoven/integrations/source/postgresql/config/spec.json