multiwoven-integrations 0.1.67 → 0.1.69
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/config.rb +2 -1
- data/lib/multiwoven/integrations/core/utils.rb +6 -1
- data/lib/multiwoven/integrations/destination/zendesk/config/catalog.json +107 -107
- data/lib/multiwoven/integrations/rollout.rb +3 -2
- data/lib/multiwoven/integrations/service.rb +4 -0
- data/lib/multiwoven/integrations/source/clickhouse/client.rb +102 -0
- data/lib/multiwoven/integrations/source/clickhouse/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/clickhouse/config/spec.json +42 -0
- data/lib/multiwoven/integrations/source/clickhouse/icon.svg +4 -0
- data/lib/multiwoven/integrations.rb +4 -1
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 504054c37b59618cb9743ef91bb97069a89346c37ca31f70aca89947542f8261
|
4
|
+
data.tar.gz: 978b116bd88a9a313840f94ea6010bc0e07dcfab31d3f9f7ed6ab3c751a3f159
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1add5f808f5ba6774635129869a4c2470f2f7c0f1ec71bf43e09187ed482423d32b41d31f436159240f0ee751b785ac2a9504bbaf83c616eaf0146b4a0974094
|
7
|
+
data.tar.gz: e97ea09b8de2d4170102549a965f5606a3563b8b621b4be8114acdfa5aca05c6899a0d79925faf3578f952057febc16764534a48cf08cd59d81d6f561ec6602c
|
@@ -3,10 +3,11 @@
|
|
3
3
|
module Multiwoven
|
4
4
|
module Integrations
|
5
5
|
class Config
|
6
|
-
attr_accessor :logger
|
6
|
+
attr_accessor :logger, :exception_reporter
|
7
7
|
|
8
8
|
def initialize(params = {})
|
9
9
|
@logger = params[:logger]
|
10
|
+
@exception_reporter = params[:exception_reporter]
|
10
11
|
end
|
11
12
|
end
|
12
13
|
end
|
@@ -48,6 +48,11 @@ module Multiwoven
|
|
48
48
|
Integrations::Service.logger
|
49
49
|
end
|
50
50
|
|
51
|
+
def report_exception(exception)
|
52
|
+
reporter = Integrations::Service.exception_reporter
|
53
|
+
reporter&.report(exception)
|
54
|
+
end
|
55
|
+
|
51
56
|
def create_log_message(context, type, exception)
|
52
57
|
Integrations::Protocol::LogMessage.new(
|
53
58
|
name: context,
|
@@ -60,7 +65,7 @@ module Multiwoven
|
|
60
65
|
logger.error(
|
61
66
|
"#{context}: #{exception.message}"
|
62
67
|
)
|
63
|
-
|
68
|
+
report_exception(exception)
|
64
69
|
create_log_message(context, type, exception)
|
65
70
|
end
|
66
71
|
|
@@ -1,110 +1,110 @@
|
|
1
1
|
{
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
}
|
49
|
-
},
|
50
|
-
"supported_sync_modes": [
|
51
|
-
"incremental"
|
52
|
-
],
|
53
|
-
"source_defined_cursor": true,
|
54
|
-
"default_cursor_field": [
|
55
|
-
"updated_at"
|
56
|
-
],
|
57
|
-
"source_defined_primary_key": [
|
58
|
-
[
|
59
|
-
"id"
|
60
|
-
]
|
61
|
-
]
|
62
|
-
},
|
63
|
-
{
|
64
|
-
"name": "users",
|
65
|
-
"action": "create",
|
66
|
-
"json_schema": {
|
67
|
-
"type": "object",
|
68
|
-
"additionalProperties": true,
|
69
|
-
"properties": {
|
70
|
-
"id": {
|
71
|
-
"type": "integer"
|
72
|
-
},
|
73
|
-
"name": {
|
74
|
-
"type": "string"
|
75
|
-
},
|
76
|
-
"email": {
|
77
|
-
"type": "string"
|
78
|
-
},
|
79
|
-
"role": {
|
80
|
-
"type": "string"
|
81
|
-
},
|
82
|
-
"last_login_at": {
|
83
|
-
"type": "string",
|
84
|
-
"format": "date-time"
|
85
|
-
},
|
86
|
-
"created_at": {
|
87
|
-
"type": "string",
|
88
|
-
"format": "date-time"
|
89
|
-
},
|
90
|
-
"updated_at": {
|
91
|
-
"type": "string",
|
92
|
-
"format": "date-time"
|
93
|
-
}
|
94
|
-
}
|
95
|
-
},
|
96
|
-
"supported_sync_modes": [
|
97
|
-
"incremental"
|
98
|
-
],
|
99
|
-
"source_defined_cursor": true,
|
100
|
-
"default_cursor_field": [
|
101
|
-
"updated_at"
|
102
|
-
],
|
103
|
-
"source_defined_primary_key": [
|
104
|
-
[
|
105
|
-
"id"
|
106
|
-
]
|
107
|
-
]
|
2
|
+
"request_rate_limit": 700,
|
3
|
+
"request_rate_limit_unit": "minute",
|
4
|
+
"request_rate_concurrency": 10,
|
5
|
+
"streams": [
|
6
|
+
{
|
7
|
+
"name": "tickets",
|
8
|
+
"action": "create",
|
9
|
+
"json_schema": {
|
10
|
+
"type": "object",
|
11
|
+
"additionalProperties": true,
|
12
|
+
"properties": {
|
13
|
+
"id": {
|
14
|
+
"type": "integer"
|
15
|
+
},
|
16
|
+
"subject": {
|
17
|
+
"type": "string"
|
18
|
+
},
|
19
|
+
"description": {
|
20
|
+
"type": "string"
|
21
|
+
},
|
22
|
+
"status": {
|
23
|
+
"type": "string"
|
24
|
+
},
|
25
|
+
"priority": {
|
26
|
+
"type": "string"
|
27
|
+
},
|
28
|
+
"requester_id": {
|
29
|
+
"type": "integer"
|
30
|
+
},
|
31
|
+
"assignee_id": {
|
32
|
+
"type": "integer"
|
33
|
+
},
|
34
|
+
"tags": {
|
35
|
+
"type": "array",
|
36
|
+
"items": {
|
37
|
+
"type": "string"
|
38
|
+
}
|
39
|
+
},
|
40
|
+
"created_at": {
|
41
|
+
"type": "string",
|
42
|
+
"format": "date-time"
|
43
|
+
},
|
44
|
+
"updated_at": {
|
45
|
+
"type": "string",
|
46
|
+
"format": "date-time"
|
47
|
+
}
|
108
48
|
}
|
109
|
-
|
49
|
+
},
|
50
|
+
"supported_sync_modes": [
|
51
|
+
"incremental"
|
52
|
+
],
|
53
|
+
"source_defined_cursor": true,
|
54
|
+
"default_cursor_field": [
|
55
|
+
"updated_at"
|
56
|
+
],
|
57
|
+
"source_defined_primary_key": [
|
58
|
+
[
|
59
|
+
"id"
|
60
|
+
]
|
61
|
+
]
|
62
|
+
},
|
63
|
+
{
|
64
|
+
"name": "users",
|
65
|
+
"action": "create",
|
66
|
+
"json_schema": {
|
67
|
+
"type": "object",
|
68
|
+
"additionalProperties": true,
|
69
|
+
"properties": {
|
70
|
+
"id": {
|
71
|
+
"type": "integer"
|
72
|
+
},
|
73
|
+
"name": {
|
74
|
+
"type": "string"
|
75
|
+
},
|
76
|
+
"email": {
|
77
|
+
"type": "string"
|
78
|
+
},
|
79
|
+
"role": {
|
80
|
+
"type": "string"
|
81
|
+
},
|
82
|
+
"last_login_at": {
|
83
|
+
"type": "string",
|
84
|
+
"format": "date-time"
|
85
|
+
},
|
86
|
+
"created_at": {
|
87
|
+
"type": "string",
|
88
|
+
"format": "date-time"
|
89
|
+
},
|
90
|
+
"updated_at": {
|
91
|
+
"type": "string",
|
92
|
+
"format": "date-time"
|
93
|
+
}
|
94
|
+
}
|
95
|
+
},
|
96
|
+
"supported_sync_modes": [
|
97
|
+
"incremental"
|
98
|
+
],
|
99
|
+
"source_defined_cursor": true,
|
100
|
+
"default_cursor_field": [
|
101
|
+
"updated_at"
|
102
|
+
],
|
103
|
+
"source_defined_primary_key": [
|
104
|
+
[
|
105
|
+
"id"
|
106
|
+
]
|
107
|
+
]
|
108
|
+
}
|
109
|
+
]
|
110
110
|
}
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module Multiwoven
|
4
4
|
module Integrations
|
5
|
-
VERSION = "0.1.
|
5
|
+
VERSION = "0.1.69"
|
6
6
|
|
7
7
|
ENABLED_SOURCES = %w[
|
8
8
|
Snowflake
|
@@ -12,6 +12,7 @@ module Multiwoven
|
|
12
12
|
Databricks
|
13
13
|
SalesforceConsumerGoodsCloud
|
14
14
|
AwsAthena
|
15
|
+
Clickhouse
|
15
16
|
].freeze
|
16
17
|
|
17
18
|
ENABLED_DESTINATIONS = %w[
|
@@ -26,8 +27,8 @@ module Multiwoven
|
|
26
27
|
SalesforceConsumerGoodsCloud
|
27
28
|
Sftp
|
28
29
|
Postgresql
|
29
|
-
Zendesk
|
30
30
|
Http
|
31
|
+
Zendesk
|
31
32
|
].freeze
|
32
33
|
end
|
33
34
|
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Multiwoven::Integrations::Source
|
4
|
+
module Clickhouse
|
5
|
+
include Multiwoven::Integrations::Core
|
6
|
+
class Client < SourceConnector
|
7
|
+
def check_connection(connection_config)
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
9
|
+
create_connection(connection_config)
|
10
|
+
ConnectionStatus.new(
|
11
|
+
status: ConnectionStatusType["succeeded"]
|
12
|
+
).to_multiwoven_message
|
13
|
+
rescue StandardError => e
|
14
|
+
ConnectionStatus.new(
|
15
|
+
status: ConnectionStatusType["failed"], message: e.message
|
16
|
+
).to_multiwoven_message
|
17
|
+
end
|
18
|
+
|
19
|
+
def discover(connection_config)
|
20
|
+
connection_config = connection_config.with_indifferent_access
|
21
|
+
query = "SELECT table_name, column_name, data_type, is_nullable FROM information_schema.columns WHERE table_schema = '#{connection_config[:database]}' ORDER BY table_name, ordinal_position;"
|
22
|
+
db = create_connection(connection_config)
|
23
|
+
records = query_execution(db, query)
|
24
|
+
catalog = Catalog.new(streams: create_streams(records))
|
25
|
+
catalog.to_multiwoven_message
|
26
|
+
rescue StandardError => e
|
27
|
+
handle_exception(
|
28
|
+
"CLICKHOUSE:DISCOVER:EXCEPTION",
|
29
|
+
"error",
|
30
|
+
e
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
def read(sync_config)
|
35
|
+
connection_config = sync_config.source.connection_specification
|
36
|
+
connection_config = connection_config.with_indifferent_access
|
37
|
+
query = sync_config.model.query
|
38
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
39
|
+
db = create_connection(connection_config)
|
40
|
+
query(db, query)
|
41
|
+
rescue StandardError => e
|
42
|
+
handle_exception(
|
43
|
+
"CLICKHOUSE:READ:EXCEPTION",
|
44
|
+
"error",
|
45
|
+
e
|
46
|
+
)
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def query(connection, query)
|
52
|
+
query_execution(connection, query).map do |row|
|
53
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def create_connection(connection_config)
|
58
|
+
@auth_token = Base64.strict_encode64("#{connection_config[:username]}:#{connection_config[:password]}")
|
59
|
+
Faraday.new(connection_config[:host]) do |faraday|
|
60
|
+
faraday.request :url_encoded
|
61
|
+
faraday.adapter Faraday.default_adapter
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def query_execution(connection, query)
|
66
|
+
response = connection.post do |req|
|
67
|
+
req.url "/"
|
68
|
+
req.headers["Authorization"] = "Basic #{@auth_token}"
|
69
|
+
req.headers["Content-Type"] = "text/plain"
|
70
|
+
req.body = query
|
71
|
+
end
|
72
|
+
column_names = query[/SELECT (.*?) FROM/i, 1].split(",").map(&:strip)
|
73
|
+
response.body.strip.split("\n").map do |row|
|
74
|
+
columns = row.split("\t")
|
75
|
+
column_names.zip(columns).to_h
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def create_streams(records)
|
80
|
+
group_by_table(records).map do |_, r|
|
81
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
def group_by_table(records)
|
86
|
+
result = {}
|
87
|
+
records.each_with_index do |entry, index|
|
88
|
+
table_name = entry["table_name"]
|
89
|
+
column_data = {
|
90
|
+
column_name: entry["column_name"],
|
91
|
+
data_type: entry["data_type"].gsub(/Nullable\((\w+)\)/, '\1').downcase.gsub!(/\d+/, ""),
|
92
|
+
is_nullable: entry["is_nullable"] == "1"
|
93
|
+
}
|
94
|
+
result[index] ||= {}
|
95
|
+
result[index][:tablename] = table_name
|
96
|
+
result[index][:columns] = [column_data]
|
97
|
+
end
|
98
|
+
result
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
{
|
2
|
+
"data": {
|
3
|
+
"name": "Clickhouse",
|
4
|
+
"title": "ClickHouse",
|
5
|
+
"connector_type": "source",
|
6
|
+
"category": "Data Warehouse",
|
7
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/clickhouse",
|
8
|
+
"github_issue_label": "source-clickhouse",
|
9
|
+
"icon": "icon.svg",
|
10
|
+
"license": "MIT",
|
11
|
+
"release_stage": "alpha",
|
12
|
+
"support_level": "community",
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
14
|
+
}
|
15
|
+
}
|
@@ -0,0 +1,42 @@
|
|
1
|
+
{
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/clickhouse",
|
3
|
+
"stream_type": "dynamic",
|
4
|
+
"connector_query_type": "raw_sql",
|
5
|
+
"connection_specification": {
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
7
|
+
"title": "ClickHouse",
|
8
|
+
"type": "object",
|
9
|
+
"required": ["url", "username", "password", "database"],
|
10
|
+
"properties": {
|
11
|
+
"url": {
|
12
|
+
"description": "The ClickHouse host url to connect.",
|
13
|
+
"examples": ["tu61szglca.us-west-2.aws.clickhouse.cloud"],
|
14
|
+
"type": "string",
|
15
|
+
"title": "Personal URL",
|
16
|
+
"order": 0
|
17
|
+
},
|
18
|
+
"username": {
|
19
|
+
"description": "The username for ClickHouse.",
|
20
|
+
"examples": ["Default"],
|
21
|
+
"type": "string",
|
22
|
+
"title": "Username",
|
23
|
+
"order": 1
|
24
|
+
},
|
25
|
+
"password": {
|
26
|
+
"description": "The password for ClickHouse.",
|
27
|
+
"examples": ["Default"],
|
28
|
+
"type": "string",
|
29
|
+
"multiwoven_secret": true,
|
30
|
+
"title": "Password",
|
31
|
+
"order": 2
|
32
|
+
},
|
33
|
+
"database": {
|
34
|
+
"description": "The ClickHouse database.",
|
35
|
+
"examples": ["default"],
|
36
|
+
"type": "string",
|
37
|
+
"title": "Database",
|
38
|
+
"order": 3
|
39
|
+
}
|
40
|
+
}
|
41
|
+
}
|
42
|
+
}
|
@@ -25,6 +25,8 @@ require "securerandom"
|
|
25
25
|
require "aws-sdk-athena"
|
26
26
|
require "zip"
|
27
27
|
require "zendesk_api"
|
28
|
+
require "faraday"
|
29
|
+
require "base64"
|
28
30
|
|
29
31
|
# Service
|
30
32
|
require_relative "integrations/config"
|
@@ -51,6 +53,7 @@ require_relative "integrations/source/postgresql/client"
|
|
51
53
|
require_relative "integrations/source/databricks/client"
|
52
54
|
require_relative "integrations/source/salesforce_consumer_goods_cloud/client"
|
53
55
|
require_relative "integrations/source/aws_athena/client"
|
56
|
+
require_relative "integrations/source/clickhouse/client"
|
54
57
|
|
55
58
|
# Destination
|
56
59
|
require_relative "integrations/destination/klaviyo/client"
|
@@ -64,8 +67,8 @@ require_relative "integrations/destination/stripe/client"
|
|
64
67
|
require_relative "integrations/destination/salesforce_consumer_goods_cloud/client"
|
65
68
|
require_relative "integrations/destination/sftp/client"
|
66
69
|
require_relative "integrations/destination/postgresql/client"
|
67
|
-
require_relative "integrations/destination/zendesk/client"
|
68
70
|
require_relative "integrations/destination/http/client"
|
71
|
+
require_relative "integrations/destination/zendesk/client"
|
69
72
|
|
70
73
|
module Multiwoven
|
71
74
|
module Integrations
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: multiwoven-integrations
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.69
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Subin T P
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-05-
|
11
|
+
date: 2024-05-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -508,6 +508,10 @@ files:
|
|
508
508
|
- lib/multiwoven/integrations/source/bigquery/config/meta.json
|
509
509
|
- lib/multiwoven/integrations/source/bigquery/config/spec.json
|
510
510
|
- lib/multiwoven/integrations/source/bigquery/icon.svg
|
511
|
+
- lib/multiwoven/integrations/source/clickhouse/client.rb
|
512
|
+
- lib/multiwoven/integrations/source/clickhouse/config/meta.json
|
513
|
+
- lib/multiwoven/integrations/source/clickhouse/config/spec.json
|
514
|
+
- lib/multiwoven/integrations/source/clickhouse/icon.svg
|
511
515
|
- lib/multiwoven/integrations/source/databricks/client.rb
|
512
516
|
- lib/multiwoven/integrations/source/databricks/config/meta.json
|
513
517
|
- lib/multiwoven/integrations/source/databricks/config/spec.json
|