multiwoven-integrations 0.1.11 → 0.1.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/base_connector.rb +4 -1
- data/lib/multiwoven/integrations/core/constants.rb +1 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json +19 -19
- data/lib/multiwoven/integrations/destination/klaviyo/config/spec.json +18 -18
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json +40 -34
- data/lib/multiwoven/integrations/destination/slack/config/spec.json +1 -1
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations/service.rb +0 -4
- data/lib/multiwoven/integrations/source/bigquery/client.rb +2 -2
- data/lib/multiwoven/integrations/source/bigquery/config/spec.json +60 -7
- data/lib/multiwoven/integrations/source/redshift/config/meta.json +1 -1
- data/lib/multiwoven/integrations/source/redshift/config/spec.json +6 -6
- data/lib/multiwoven/integrations/source/snowflake/config/spec.json +5 -5
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d065a58bb18a6388dabdd7b99b14114899d7a3eeb5e43ca5ae09acd34e49a0a4
|
|
4
|
+
data.tar.gz: 973cdedae00a16ab5bace810325440faa0b58551226d494298aab201c5c3ecd4
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c67ab22eb39ba5588161da5566e1b60cb09a951836f02b0da0d0b5f797921ecdf10fb578f8bff3ef49d3cd2a81ac3f1cfbe3541ef84c4c61b4dc2b651de4ec47
|
|
7
|
+
data.tar.gz: f5f27ea2767a96ee4081a7b7c7a7504f3ff30843ba941d0a0f85e2728d46e4d8523427308c6c639e54a8d1121e603ea7567bc94fde714017787aa44d62ee8785
|
|
@@ -15,7 +15,10 @@ module Multiwoven
|
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
def meta_data
|
|
18
|
-
|
|
18
|
+
client_meta_data = read_json(META_DATA_PATH).to_h
|
|
19
|
+
icon_name = client_meta_data.with_indifferent_access["data"]["icon"]
|
|
20
|
+
icon_url = "https://raw.githubusercontent.com/Multiwoven/multiwoven-integrations/#{MAIN_BRANCH_SHA}/assets/images/connectors/#{icon_name}"
|
|
21
|
+
@meta_data ||= client_meta_data.merge!({ "icon": icon_url })
|
|
19
22
|
end
|
|
20
23
|
|
|
21
24
|
def check_connection(_connection_config)
|
|
@@ -12,6 +12,7 @@ module Multiwoven
|
|
|
12
12
|
CONNECTOR_SPEC_PATH = "config/spec.json"
|
|
13
13
|
CATALOG_SPEC_PATH = "config/catalog.json"
|
|
14
14
|
SNOWFLAKE_MAC_DRIVER_PATH = "/opt/snowflake/snowflakeodbc/lib/universal/libSnowflake.dylib"
|
|
15
|
+
MAIN_BRANCH_SHA = Git.ls_remote("https://github.com/Multiwoven/multiwoven-integrations")["head"][:sha]
|
|
15
16
|
|
|
16
17
|
SNOWFLAKE_DRIVER_PATH = ENV["SNOWFLAKE_DRIVER_PATH"] || SNOWFLAKE_MAC_DRIVER_PATH
|
|
17
18
|
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
}
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/facebook",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Facebook Custom Audiences",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["access_token", "ad_account_id"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"access_token": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"title": "Access Token",
|
|
13
|
+
"order": 0
|
|
14
|
+
},
|
|
15
|
+
"ad_account_id": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"title": "Ad Account Id",
|
|
18
|
+
"order": 1
|
|
20
19
|
}
|
|
21
20
|
}
|
|
22
|
-
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -1,22 +1,22 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
}
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/klaviyo",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Klaviyo",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["public_api_key", "private_api_key"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"public_api_key": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"title": "Public API Key",
|
|
13
|
+
"order": 0
|
|
14
|
+
},
|
|
15
|
+
"private_api_key": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"title": "Private API Key",
|
|
18
|
+
"order": 1
|
|
20
19
|
}
|
|
21
20
|
}
|
|
22
21
|
}
|
|
22
|
+
}
|
|
@@ -1,37 +1,43 @@
|
|
|
1
1
|
{
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
"
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/salesforce_crm",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Salesforce CRM",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": [
|
|
9
|
+
"access_token",
|
|
10
|
+
"refresh_token",
|
|
11
|
+
"instance_url",
|
|
12
|
+
"client_id",
|
|
13
|
+
"client_secret"
|
|
14
|
+
],
|
|
15
|
+
"properties": {
|
|
16
|
+
"access_token": {
|
|
17
|
+
"type": "string",
|
|
18
|
+
"title": "Access Token",
|
|
19
|
+
"order": 0
|
|
20
|
+
},
|
|
21
|
+
"refresh_token": {
|
|
22
|
+
"type": "string",
|
|
23
|
+
"title": "Refresh Token",
|
|
24
|
+
"order": 1
|
|
25
|
+
},
|
|
26
|
+
"instance_url": {
|
|
27
|
+
"type": "string",
|
|
28
|
+
"title": "Instance URL",
|
|
29
|
+
"order": 2
|
|
30
|
+
},
|
|
31
|
+
"client_id": {
|
|
32
|
+
"type": "string",
|
|
33
|
+
"title": "Client ID",
|
|
34
|
+
"order": 3
|
|
35
|
+
},
|
|
36
|
+
"client_secret": {
|
|
37
|
+
"type": "string",
|
|
38
|
+
"title": "Client Secret",
|
|
39
|
+
"order": 4
|
|
35
40
|
}
|
|
36
41
|
}
|
|
37
|
-
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
5
|
class Service
|
|
6
|
-
MAIN_BRANCH_SHA = Git.ls_remote("https://github.com/Multiwoven/multiwoven-integrations")["head"][:sha]
|
|
7
6
|
class << self
|
|
8
7
|
def initialize
|
|
9
8
|
yield(config) if block_given?
|
|
@@ -40,10 +39,7 @@ module Multiwoven
|
|
|
40
39
|
enabled_connectors.map do |connector|
|
|
41
40
|
client = connector_class(type, connector).new
|
|
42
41
|
connector_spec = { connector_spec: client.connector_spec.to_h }
|
|
43
|
-
icon_name = client.meta_data.with_indifferent_access["data"]["icon"]
|
|
44
|
-
icon_url = "https://raw.githubusercontent.com/Multiwoven/multiwoven-integrations/#{MAIN_BRANCH_SHA}/assets/images/connectors/#{icon_name}"
|
|
45
42
|
client.meta_data["data"].to_h.merge!(connector_spec)
|
|
46
|
-
client.meta_data["data"].to_h.merge!({ "icon": icon_url })
|
|
47
43
|
end
|
|
48
44
|
end
|
|
49
45
|
|
|
@@ -52,9 +52,9 @@ module Multiwoven::Integrations::Source
|
|
|
52
52
|
|
|
53
53
|
bigquery = create_connection(connection_config)
|
|
54
54
|
records = []
|
|
55
|
-
results = bigquery.query
|
|
55
|
+
results = bigquery.query(query) || []
|
|
56
56
|
results.each do |row|
|
|
57
|
-
records << RecordMessage.new(data: row, emitted_at: Time.now.to_i)
|
|
57
|
+
records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
records
|
|
@@ -3,25 +3,78 @@
|
|
|
3
3
|
"stream_type": "dynamic",
|
|
4
4
|
"connection_specification": {
|
|
5
5
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
-
"title": "BigQuery
|
|
6
|
+
"title": "Google BigQuery",
|
|
7
7
|
"type": "object",
|
|
8
8
|
"required": ["project_id", "credentials_json"],
|
|
9
9
|
"properties": {
|
|
10
10
|
"project_id": {
|
|
11
11
|
"type": "string",
|
|
12
|
-
"description": "The
|
|
12
|
+
"description": "The project ID, serving as the unique identifier for the Google Cloud project associated with your BigQuery data warehouse, can be located within the Google Cloud web console. This is achieved by either selecting your project name from the top navigation bar or by navigating to the project settings page.",
|
|
13
13
|
"title": "Project ID"
|
|
14
14
|
},
|
|
15
15
|
"dataset_id": {
|
|
16
16
|
"type": "string",
|
|
17
|
-
"description": "The dataset ID to search for tables and views. If you are only loading data from one dataset, setting this option could result in much faster schema discovery.",
|
|
17
|
+
"description": "The dataset ID to search for tables and views. If you are only loading data from one dataset, setting this option could result in much faster schema discovery. You can get ",
|
|
18
18
|
"title": "Default Dataset ID"
|
|
19
19
|
},
|
|
20
20
|
"credentials_json": {
|
|
21
|
-
"type": "
|
|
22
|
-
"description": "
|
|
23
|
-
"title": "
|
|
24
|
-
"
|
|
21
|
+
"type": "object",
|
|
22
|
+
"description": "You can get the keys from the Google Cloud web console. First, go to the IAM page and select Service Accounts from the left menu. Next, locate your service account in the list, click on its Keys tab, and then click Add Key. Lastly, click Create new key and select JSON.",
|
|
23
|
+
"title": "",
|
|
24
|
+
"properties": {
|
|
25
|
+
"type": {
|
|
26
|
+
"type": "string",
|
|
27
|
+
"enum": ["service_account"]
|
|
28
|
+
},
|
|
29
|
+
"project_id": {
|
|
30
|
+
"type": "string"
|
|
31
|
+
},
|
|
32
|
+
"private_key_id": {
|
|
33
|
+
"type": "string"
|
|
34
|
+
},
|
|
35
|
+
"private_key": {
|
|
36
|
+
"type": "string"
|
|
37
|
+
},
|
|
38
|
+
"client_email": {
|
|
39
|
+
"type": "string",
|
|
40
|
+
"format": "email"
|
|
41
|
+
},
|
|
42
|
+
"client_id": {
|
|
43
|
+
"type": "string"
|
|
44
|
+
},
|
|
45
|
+
"auth_uri": {
|
|
46
|
+
"type": "string",
|
|
47
|
+
"format": "uri"
|
|
48
|
+
},
|
|
49
|
+
"token_uri": {
|
|
50
|
+
"type": "string",
|
|
51
|
+
"format": "uri"
|
|
52
|
+
},
|
|
53
|
+
"auth_provider_x509_cert_url": {
|
|
54
|
+
"type": "string",
|
|
55
|
+
"format": "uri"
|
|
56
|
+
},
|
|
57
|
+
"client_x509_cert_url": {
|
|
58
|
+
"type": "string",
|
|
59
|
+
"format": "uri"
|
|
60
|
+
},
|
|
61
|
+
"universe_domain": {
|
|
62
|
+
"type": "string"
|
|
63
|
+
}
|
|
64
|
+
},
|
|
65
|
+
"required": [
|
|
66
|
+
"type",
|
|
67
|
+
"project_id",
|
|
68
|
+
"private_key_id",
|
|
69
|
+
"private_key",
|
|
70
|
+
"client_email",
|
|
71
|
+
"client_id",
|
|
72
|
+
"auth_uri",
|
|
73
|
+
"token_uri",
|
|
74
|
+
"auth_provider_x509_cert_url",
|
|
75
|
+
"client_x509_cert_url",
|
|
76
|
+
"universe_domain"
|
|
77
|
+
]
|
|
25
78
|
}
|
|
26
79
|
}
|
|
27
80
|
}
|
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
"stream_type": "dynamic",
|
|
4
4
|
"connection_specification": {
|
|
5
5
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
-
"title": "Redshift
|
|
6
|
+
"title": "Amazon Redshift",
|
|
7
7
|
"type": "object",
|
|
8
8
|
"required": ["host", "port", "database", "schema"],
|
|
9
9
|
"properties": {
|
|
10
10
|
"credentials": {
|
|
11
|
-
"title": "
|
|
11
|
+
"title": "",
|
|
12
12
|
"type": "object",
|
|
13
13
|
"required": ["auth_type", "username", "password"],
|
|
14
14
|
"properties": {
|
|
@@ -19,14 +19,14 @@
|
|
|
19
19
|
"readOnly": true
|
|
20
20
|
},
|
|
21
21
|
"username": {
|
|
22
|
-
"description": "
|
|
22
|
+
"description": "Username refers to your individual Redshift login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
|
|
23
23
|
"examples": ["REDSHIFT_USER"],
|
|
24
24
|
"type": "string",
|
|
25
25
|
"title": "Username",
|
|
26
26
|
"order": 1
|
|
27
27
|
},
|
|
28
28
|
"password": {
|
|
29
|
-
"description": "
|
|
29
|
+
"description": "This field requires the password associated with the user account specified in the preceding section.",
|
|
30
30
|
"type": "string",
|
|
31
31
|
"multiwoven_secret": true,
|
|
32
32
|
"title": "Password",
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
"order": 0
|
|
37
37
|
},
|
|
38
38
|
"host": {
|
|
39
|
-
"description": "The
|
|
39
|
+
"description": "The hostname or IP address of your Redshift cluster represents a critical connectivity parameter. To retrieve this information, access the Redshift web console, proceed to the Clusters panel, and select your specific cluster. Within the cluster's details, locate and copy the Endpoint string, ensuring to omit the port number and database name appended at the conclusion.",
|
|
40
40
|
"examples": [
|
|
41
41
|
"example-redshift-cluster.abcdefg.us-west-2.redshift.amazonaws.com"
|
|
42
42
|
],
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
"order": 1
|
|
46
46
|
},
|
|
47
47
|
"port": {
|
|
48
|
-
"description": "The port
|
|
48
|
+
"description": "The port number for your Redshift cluster, which defaults to 5439, may vary based on your configuration. To verify the specific port number assigned to your cluster, access the Redshift web console, proceed to the Clusters panel, and select your cluster. You can find the port number displayed within the Properties tab.",
|
|
49
49
|
"examples": ["5439"],
|
|
50
50
|
"type": "string",
|
|
51
51
|
"title": "Port",
|
|
@@ -3,12 +3,12 @@
|
|
|
3
3
|
"stream_type": "dynamic",
|
|
4
4
|
"connection_specification": {
|
|
5
5
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
-
"title": "Snowflake
|
|
6
|
+
"title": "Snowflake",
|
|
7
7
|
"type": "object",
|
|
8
8
|
"required": ["host", "role", "warehouse", "database"],
|
|
9
9
|
"properties": {
|
|
10
10
|
"credentials": {
|
|
11
|
-
"title": "
|
|
11
|
+
"title": "",
|
|
12
12
|
"type": "object",
|
|
13
13
|
"required": ["auth_type", "username", "password"],
|
|
14
14
|
"properties": {
|
|
@@ -19,14 +19,14 @@
|
|
|
19
19
|
"readOnly": true
|
|
20
20
|
},
|
|
21
21
|
"username": {
|
|
22
|
-
"description": "The username
|
|
23
|
-
"examples": ["
|
|
22
|
+
"description": "The username you created to allow multiwoven to access the database.",
|
|
23
|
+
"examples": ["MULTIWOVEN_USER"],
|
|
24
24
|
"type": "string",
|
|
25
25
|
"title": "Username",
|
|
26
26
|
"order": 1
|
|
27
27
|
},
|
|
28
28
|
"password": {
|
|
29
|
-
"description": "The password
|
|
29
|
+
"description": "The password associated with the username.",
|
|
30
30
|
"type": "string",
|
|
31
31
|
"multiwoven_secret": true,
|
|
32
32
|
"title": "Password",
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.12
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-02-
|
|
11
|
+
date: 2024-02-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -362,7 +362,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
362
362
|
- !ruby/object:Gem::Version
|
|
363
363
|
version: '0'
|
|
364
364
|
requirements: []
|
|
365
|
-
rubygems_version: 3.
|
|
365
|
+
rubygems_version: 3.5.3
|
|
366
366
|
signing_key:
|
|
367
367
|
specification_version: 4
|
|
368
368
|
summary: Integration suite for open source reverse ETL platform
|