multiwoven-integrations 0.1.68 → 0.1.69
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/clickhouse/client.rb +102 -0
- data/lib/multiwoven/integrations/source/clickhouse/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/clickhouse/config/spec.json +42 -0
- data/lib/multiwoven/integrations/source/clickhouse/icon.svg +4 -0
- data/lib/multiwoven/integrations.rb +3 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 504054c37b59618cb9743ef91bb97069a89346c37ca31f70aca89947542f8261
|
|
4
|
+
data.tar.gz: 978b116bd88a9a313840f94ea6010bc0e07dcfab31d3f9f7ed6ab3c751a3f159
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1add5f808f5ba6774635129869a4c2470f2f7c0f1ec71bf43e09187ed482423d32b41d31f436159240f0ee751b785ac2a9504bbaf83c616eaf0146b4a0974094
|
|
7
|
+
data.tar.gz: e97ea09b8de2d4170102549a965f5606a3563b8b621b4be8114acdfa5aca05c6899a0d79925faf3578f952057febc16764534a48cf08cd59d81d6f561ec6602c
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.1.
|
|
5
|
+
VERSION = "0.1.69"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -12,6 +12,7 @@ module Multiwoven
|
|
|
12
12
|
Databricks
|
|
13
13
|
SalesforceConsumerGoodsCloud
|
|
14
14
|
AwsAthena
|
|
15
|
+
Clickhouse
|
|
15
16
|
].freeze
|
|
16
17
|
|
|
17
18
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module Clickhouse
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < SourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
create_connection(connection_config)
|
|
10
|
+
ConnectionStatus.new(
|
|
11
|
+
status: ConnectionStatusType["succeeded"]
|
|
12
|
+
).to_multiwoven_message
|
|
13
|
+
rescue StandardError => e
|
|
14
|
+
ConnectionStatus.new(
|
|
15
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
16
|
+
).to_multiwoven_message
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def discover(connection_config)
|
|
20
|
+
connection_config = connection_config.with_indifferent_access
|
|
21
|
+
query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:database]}' ORDER BY table_name, ordinal_position;"
|
|
22
|
+
db = create_connection(connection_config)
|
|
23
|
+
records = query_execution(db, query)
|
|
24
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
25
|
+
catalog.to_multiwoven_message
|
|
26
|
+
rescue StandardError => e
|
|
27
|
+
handle_exception(
|
|
28
|
+
"CLICKHOUSE:DISCOVER:EXCEPTION",
|
|
29
|
+
"error",
|
|
30
|
+
e
|
|
31
|
+
)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def read(sync_config)
|
|
35
|
+
connection_config = sync_config.source.connection_specification
|
|
36
|
+
connection_config = connection_config.with_indifferent_access
|
|
37
|
+
query = sync_config.model.query
|
|
38
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
39
|
+
db = create_connection(connection_config)
|
|
40
|
+
query(db, query)
|
|
41
|
+
rescue StandardError => e
|
|
42
|
+
handle_exception(
|
|
43
|
+
"CLICKHOUSE:READ:EXCEPTION",
|
|
44
|
+
"error",
|
|
45
|
+
e
|
|
46
|
+
)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def query(connection, query)
|
|
52
|
+
query_execution(connection, query).map do |row|
|
|
53
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def create_connection(connection_config)
|
|
58
|
+
@auth_token = Base64.strict_encode64("#{connection_config[:username]}:#{connection_config[:password]}")
|
|
59
|
+
Faraday.new(connection_config[:host]) do |faraday|
|
|
60
|
+
faraday.request :url_encoded
|
|
61
|
+
faraday.adapter Faraday.default_adapter
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def query_execution(connection, query)
|
|
66
|
+
response = connection.post do |req|
|
|
67
|
+
req.url "/"
|
|
68
|
+
req.headers["Authorization"] = "Basic #{@auth_token}"
|
|
69
|
+
req.headers["Content-Type"] = "text/plain"
|
|
70
|
+
req.body = query
|
|
71
|
+
end
|
|
72
|
+
column_names = query[/SELECT (.*?) FROM/i, 1].split(",").map(&:strip)
|
|
73
|
+
response.body.strip.split("\n").map do |row|
|
|
74
|
+
columns = row.split("\t")
|
|
75
|
+
column_names.zip(columns).to_h
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def create_streams(records)
|
|
80
|
+
group_by_table(records).map do |_, r|
|
|
81
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def group_by_table(records)
|
|
86
|
+
result = {}
|
|
87
|
+
records.each_with_index do |entry, index|
|
|
88
|
+
table_name = entry["table_name"]
|
|
89
|
+
column_data = {
|
|
90
|
+
column_name: entry["column_name"],
|
|
91
|
+
data_type: entry["data_type"].gsub(/Nullable\((\w+)\)/, '\1').downcase.gsub!(/\d+/, ""),
|
|
92
|
+
is_nullable: entry["is_nullable"] == "1"
|
|
93
|
+
}
|
|
94
|
+
result[index] ||= {}
|
|
95
|
+
result[index][:tablename] = table_name
|
|
96
|
+
result[index][:columns] = [column_data]
|
|
97
|
+
end
|
|
98
|
+
result
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Clickhouse",
|
|
4
|
+
"title": "ClickHouse",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/clickhouse",
|
|
8
|
+
"github_issue_label": "source-clickhouse",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/clickhouse",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "ClickHouse",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["url", "username", "password", "database"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"url": {
|
|
12
|
+
"description": "The ClickHouse host url to connect.",
|
|
13
|
+
"examples": ["tu61szglca.us-west-2.aws.clickhouse.cloud"],
|
|
14
|
+
"type": "string",
|
|
15
|
+
"title": "Personal URL",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"username": {
|
|
19
|
+
"description": "The username for ClickHouse.",
|
|
20
|
+
"examples": ["Default"],
|
|
21
|
+
"type": "string",
|
|
22
|
+
"title": "Username",
|
|
23
|
+
"order": 1
|
|
24
|
+
},
|
|
25
|
+
"password": {
|
|
26
|
+
"description": "The password for ClickHouse.",
|
|
27
|
+
"examples": ["Default"],
|
|
28
|
+
"type": "string",
|
|
29
|
+
"multiwoven_secret": true,
|
|
30
|
+
"title": "Password",
|
|
31
|
+
"order": 2
|
|
32
|
+
},
|
|
33
|
+
"database": {
|
|
34
|
+
"description": "The ClickHouse database.",
|
|
35
|
+
"examples": ["default"],
|
|
36
|
+
"type": "string",
|
|
37
|
+
"title": "Database",
|
|
38
|
+
"order": 3
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
@@ -25,6 +25,8 @@ require "securerandom"
|
|
|
25
25
|
require "aws-sdk-athena"
|
|
26
26
|
require "zip"
|
|
27
27
|
require "zendesk_api"
|
|
28
|
+
require "faraday"
|
|
29
|
+
require "base64"
|
|
28
30
|
|
|
29
31
|
# Service
|
|
30
32
|
require_relative "integrations/config"
|
|
@@ -51,6 +53,7 @@ require_relative "integrations/source/postgresql/client"
|
|
|
51
53
|
require_relative "integrations/source/databricks/client"
|
|
52
54
|
require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
|
|
53
55
|
require_relative "integrations/source/aws_athena/client"
|
|
56
|
+
require_relative "integrations/source/clickhouse/client"
|
|
54
57
|
|
|
55
58
|
# Destination
|
|
56
59
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.69
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-05-
|
|
11
|
+
date: 2024-05-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -508,6 +508,10 @@ files:
|
|
|
508
508
|
- lib/multiwoven/integrations/source/bigquery/config/meta.json
|
|
509
509
|
- lib/multiwoven/integrations/source/bigquery/config/spec.json
|
|
510
510
|
- lib/multiwoven/integrations/source/bigquery/icon.svg
|
|
511
|
+
- lib/multiwoven/integrations/source/clickhouse/client.rb
|
|
512
|
+
- lib/multiwoven/integrations/source/clickhouse/config/meta.json
|
|
513
|
+
- lib/multiwoven/integrations/source/clickhouse/config/spec.json
|
|
514
|
+
- lib/multiwoven/integrations/source/clickhouse/icon.svg
|
|
511
515
|
- lib/multiwoven/integrations/source/databricks/client.rb
|
|
512
516
|
- lib/multiwoven/integrations/source/databricks/config/meta.json
|
|
513
517
|
- lib/multiwoven/integrations/source/databricks/config/spec.json
|