multiwoven-integrations 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.vscode/settings.json +5 -0
- data/lib/multiwoven/integrations/core/base_connector.rb +3 -3
- data/lib/multiwoven/integrations/core/constants.rb +2 -0
- data/lib/multiwoven/integrations/core/source_connector.rb +17 -0
- data/lib/multiwoven/integrations/core/utils.rb +8 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +123 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json +38 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json +22 -0
- data/lib/multiwoven/integrations/destination/klaviyo/client.rb +11 -3
- data/lib/multiwoven/integrations/destination/klaviyo/config/catalog.json +86 -88
- data/lib/multiwoven/integrations/destination/klaviyo/config/spec.json +2 -2
- data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb +127 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/catalog.json +317 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json +37 -0
- data/lib/multiwoven/integrations/protocol/protocol.rb +2 -0
- data/lib/multiwoven/integrations/rollout.rb +3 -1
- data/lib/multiwoven/integrations/source/bigquery/client.rb +6 -0
- data/lib/multiwoven/integrations/source/redshift/client.rb +5 -0
- data/lib/multiwoven/integrations/source/snowflake/client.rb +5 -0
- data/lib/multiwoven/integrations.rb +4 -0
- data/multiwoven-integrations.gemspec +8 -6
- metadata +32 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e21d943bffd7e4ad698ecfc778df04a421e79c9020808e61ef30ac90c622a8ac
|
|
4
|
+
data.tar.gz: 7838b443c4aa37a523369d0b6f1bf919b0b7e110cb16fb3a5fa7f72e274776ff
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 83dc8ffa3e34b4ff1d73a0f0b09304ff57765af634b1356f85d8f374e9aaa7acb81accecbfbdbe74ea0e3c96b32cf91ae28adb8fb7885b2a0008b3f59432cd34
|
|
7
|
+
data.tar.gz: 2cbe6766d9729ed2c3505403b0c197c8ae8dbb3d31a107fffcc3bb474032921295724b2446be80df9798d6776d901006a524929b4ae56593c92ae72816879cec
|
|
@@ -30,10 +30,10 @@ module Multiwoven
|
|
|
30
30
|
private
|
|
31
31
|
|
|
32
32
|
def read_json(file_path)
|
|
33
|
-
|
|
34
|
-
connector_folder =
|
|
33
|
+
path = Object.const_source_location(self.class.to_s)[0]
|
|
34
|
+
connector_folder = File.dirname(path)
|
|
35
35
|
file_path = File.join(
|
|
36
|
-
|
|
36
|
+
"#{connector_folder}/",
|
|
37
37
|
file_path
|
|
38
38
|
)
|
|
39
39
|
file_contents = File.read(file_path)
|
|
@@ -7,6 +7,23 @@ module Multiwoven
|
|
|
7
7
|
raise "Not implemented"
|
|
8
8
|
# return list of RecordMessage
|
|
9
9
|
end
|
|
10
|
+
|
|
11
|
+
private
|
|
12
|
+
|
|
13
|
+
def batched_query(sql_query, limit, offset)
|
|
14
|
+
offset = offset.to_i
|
|
15
|
+
limit = limit.to_i
|
|
16
|
+
raise ArgumentError, "Offset and limit must be non-negative" if offset.negative? || limit.negative?
|
|
17
|
+
|
|
18
|
+
# Removing any trailing semicolons
|
|
19
|
+
sql_query.chomp!(";")
|
|
20
|
+
|
|
21
|
+
# Checking if the query already has a LIMIT clause
|
|
22
|
+
raise ArgumentError, "Query already contains a LIMIT clause" if sql_query.match?(/LIMIT \d+/i)
|
|
23
|
+
|
|
24
|
+
# Appending the LIMIT and OFFSET clauses to the SQL query
|
|
25
|
+
"#{sql_query} LIMIT #{limit} OFFSET #{offset}"
|
|
26
|
+
end
|
|
10
27
|
end
|
|
11
28
|
end
|
|
12
29
|
end
|
|
@@ -63,6 +63,14 @@ module Multiwoven
|
|
|
63
63
|
|
|
64
64
|
create_log_message(context, type, exception)
|
|
65
65
|
end
|
|
66
|
+
|
|
67
|
+
def extract_data(record_object, properties)
|
|
68
|
+
record_object.data.select { |key, _| properties.key?(key.to_s) }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def success?(response)
|
|
72
|
+
response && %w[200 201].include?(response.code.to_s)
|
|
73
|
+
end
|
|
66
74
|
end
|
|
67
75
|
end
|
|
68
76
|
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Destination
|
|
4
|
+
module FacebookCustomAudience
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
access_token = connection_config[:access_token]
|
|
10
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
11
|
+
FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS,
|
|
12
|
+
HTTP_GET,
|
|
13
|
+
headers: auth_headers(access_token)
|
|
14
|
+
)
|
|
15
|
+
if success?(response)
|
|
16
|
+
ad_account_exists?(response, connection_config[:ad_account_id])
|
|
17
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
18
|
+
else
|
|
19
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"]).to_multiwoven_message
|
|
20
|
+
end
|
|
21
|
+
rescue StandardError => e
|
|
22
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def discover(_connection_config = nil)
|
|
26
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
27
|
+
|
|
28
|
+
streams = catalog_json["streams"].map do |stream|
|
|
29
|
+
Multiwoven::Integrations::Protocol::Stream.new(
|
|
30
|
+
audience_id: stream["audience_id"],
|
|
31
|
+
url: stream["url"],
|
|
32
|
+
name: stream["name"],
|
|
33
|
+
json_schema: stream["json_schema"],
|
|
34
|
+
request_method: stream["method"],
|
|
35
|
+
action: stream["action"]
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
catalog = Multiwoven::Integrations::Protocol::Catalog.new(
|
|
40
|
+
streams: streams
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
catalog.to_multiwoven_message
|
|
44
|
+
rescue StandardError => e
|
|
45
|
+
handle_exception(
|
|
46
|
+
"FACEBOOK AUDIENCE:DISCOVER:EXCEPTION",
|
|
47
|
+
"error",
|
|
48
|
+
e
|
|
49
|
+
)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def write(sync_config, records, _action = "insert")
|
|
53
|
+
url = sync_config.stream.url
|
|
54
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
55
|
+
connection_config = connection_config.with_indifferent_access
|
|
56
|
+
access_token = connection_config[:access_token]
|
|
57
|
+
|
|
58
|
+
write_success = 0
|
|
59
|
+
write_failure = 0
|
|
60
|
+
records.each do |record|
|
|
61
|
+
schema, data = extract_schema_and_data(record.with_indifferent_access[:data])
|
|
62
|
+
payload = {
|
|
63
|
+
"payload" => {
|
|
64
|
+
"schema" => schema,
|
|
65
|
+
"data" => [data]
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
69
|
+
url,
|
|
70
|
+
sync_config.stream.request_method,
|
|
71
|
+
payload: payload,
|
|
72
|
+
headers: auth_headers(access_token)
|
|
73
|
+
)
|
|
74
|
+
if success?(response)
|
|
75
|
+
write_success += 1
|
|
76
|
+
else
|
|
77
|
+
write_failure += 1
|
|
78
|
+
end
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
logger.error(
|
|
81
|
+
"FACEBOOK:RECORD:WRITE:FAILURE: #{e.message}"
|
|
82
|
+
)
|
|
83
|
+
write_failure += 1
|
|
84
|
+
end
|
|
85
|
+
tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
86
|
+
success: write_success,
|
|
87
|
+
failed: write_failure
|
|
88
|
+
)
|
|
89
|
+
tracker.to_multiwoven_message
|
|
90
|
+
rescue StandardError => e
|
|
91
|
+
# TODO: Handle rate limiting seperately
|
|
92
|
+
handle_exception(
|
|
93
|
+
"FACEBOOK:WRITE:EXCEPTION",
|
|
94
|
+
"error",
|
|
95
|
+
e
|
|
96
|
+
)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def extract_schema_and_data(data)
|
|
100
|
+
[data.keys, data.values]
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def auth_headers(access_token)
|
|
104
|
+
{
|
|
105
|
+
"Accept" => "application/json",
|
|
106
|
+
"Authorization" => "Bearer #{access_token}",
|
|
107
|
+
"Content-Type" => "application/json"
|
|
108
|
+
}
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def ad_account_exists?(response, ad_account_id)
|
|
112
|
+
return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" }
|
|
113
|
+
|
|
114
|
+
raise ArgumentError, "Ad account not found in business account"
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def extract_data(response)
|
|
118
|
+
response_body = response.body
|
|
119
|
+
JSON.parse(response_body)["data"] if response_body
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"streams": [
|
|
3
|
+
{
|
|
4
|
+
"name": "audience",
|
|
5
|
+
"action": "create",
|
|
6
|
+
"method": "POST",
|
|
7
|
+
"audience_id": "audience_id",
|
|
8
|
+
"url": "https://graph.facebook.com/v18.0/{audience_id}/users",
|
|
9
|
+
"json_schema": {
|
|
10
|
+
"type": "object",
|
|
11
|
+
"additionalProperties": true,
|
|
12
|
+
"properties": {
|
|
13
|
+
"CT": { "type": ["string", "null"], "default": null },
|
|
14
|
+
"COUNTRY": { "type": ["string", "null"], "default": null },
|
|
15
|
+
"DOBD": { "type": ["string", "null"], "default": null },
|
|
16
|
+
"DOBM": { "type": ["string", "null"], "default": null },
|
|
17
|
+
"DOBY": { "type": ["string", "null"], "default": null },
|
|
18
|
+
"EMAIL": { "type": ["string", "null"], "default": null },
|
|
19
|
+
"EXTERN_ID": { "type": ["string", "null"], "default": null },
|
|
20
|
+
"FI": { "type": ["string", "null"], "default": null },
|
|
21
|
+
"FN": { "type": ["string", "null"], "default": null },
|
|
22
|
+
"GEN": { "type": ["string", "null"], "default": null },
|
|
23
|
+
"LN": { "type": ["string", "null"], "default": null },
|
|
24
|
+
"LOOKALIKE_VALUE": { "type": ["number", "null"], "default": null },
|
|
25
|
+
"MADID": { "type": ["string", "null"], "default": null },
|
|
26
|
+
"PAGEUID": { "type": ["string", "null"], "default": null },
|
|
27
|
+
"PHONE": { "type": ["string", "null"], "default": null },
|
|
28
|
+
"ST": { "type": ["string", "null"], "default": null },
|
|
29
|
+
"ZIP": { "type": ["string", "null"], "default": null }
|
|
30
|
+
}
|
|
31
|
+
},
|
|
32
|
+
"supported_sync_modes": ["full_refresh", "incremental"],
|
|
33
|
+
"source_defined_cursor": true,
|
|
34
|
+
"default_cursor_field": ["updated"],
|
|
35
|
+
"source_defined_primary_key": [["email"]]
|
|
36
|
+
}
|
|
37
|
+
]
|
|
38
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data":
|
|
3
|
+
{
|
|
4
|
+
"name": "Facebook",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"connector_subtype": "API",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-facebook",
|
|
9
|
+
"icon": "facebook.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/facebook",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Facebook Destination Spec",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["access_token","ad_account_id"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"access_token": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"title": "Access Token",
|
|
13
|
+
"order": 0
|
|
14
|
+
},
|
|
15
|
+
"ad_account_id": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"title": "Ad Account Id",
|
|
18
|
+
"order": 1
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
module Multiwoven::Integrations::Destination
|
|
4
4
|
module Klaviyo
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
|
-
class Client < DestinationConnector
|
|
6
|
+
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
|
7
7
|
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
8
9
|
api_key = connection_config[:private_api_key]
|
|
9
10
|
|
|
10
11
|
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
@@ -16,7 +17,7 @@ module Multiwoven::Integrations::Destination
|
|
|
16
17
|
parse_response(response)
|
|
17
18
|
end
|
|
18
19
|
|
|
19
|
-
def discover
|
|
20
|
+
def discover(_connection_config = nil)
|
|
20
21
|
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
21
22
|
|
|
22
23
|
streams = catalog_json["streams"].map do |stream|
|
|
@@ -24,7 +25,7 @@ module Multiwoven::Integrations::Destination
|
|
|
24
25
|
name: stream["name"],
|
|
25
26
|
json_schema: stream["json_schema"],
|
|
26
27
|
url: stream["url"],
|
|
27
|
-
|
|
28
|
+
request_method: stream["method"],
|
|
28
29
|
action: stream["action"]
|
|
29
30
|
)
|
|
30
31
|
end
|
|
@@ -44,12 +45,19 @@ module Multiwoven::Integrations::Destination
|
|
|
44
45
|
|
|
45
46
|
def write(sync_config, records, _action = "insert")
|
|
46
47
|
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
48
|
+
connection_config = connection_config.with_indifferent_access
|
|
47
49
|
url = sync_config.stream.url
|
|
50
|
+
|
|
48
51
|
request_method = sync_config.stream.request_method
|
|
49
52
|
|
|
50
53
|
write_success = 0
|
|
51
54
|
write_failure = 0
|
|
52
55
|
records.each do |record|
|
|
56
|
+
# pre process payload
|
|
57
|
+
# Add hardcode values into payload
|
|
58
|
+
record["data"] ||= {}
|
|
59
|
+
record["data"]["type"] = sync_config.stream.name
|
|
60
|
+
|
|
53
61
|
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
54
62
|
url,
|
|
55
63
|
request_method,
|
|
@@ -6,103 +6,101 @@
|
|
|
6
6
|
"url": "https://a.klaviyo.com/api/profiles",
|
|
7
7
|
"method": "POST",
|
|
8
8
|
"json_schema": {
|
|
9
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
9
10
|
"type": "object",
|
|
10
|
-
"additionalProperties": true,
|
|
11
11
|
"properties": {
|
|
12
|
-
"
|
|
13
|
-
"type":
|
|
14
|
-
"null",
|
|
15
|
-
"string"
|
|
16
|
-
]
|
|
17
|
-
},
|
|
18
|
-
"id": {
|
|
19
|
-
"type": "string"
|
|
20
|
-
},
|
|
21
|
-
"updated": {
|
|
22
|
-
"type": [
|
|
23
|
-
"null",
|
|
24
|
-
"string"
|
|
25
|
-
],
|
|
26
|
-
"format": "date-time"
|
|
27
|
-
},
|
|
28
|
-
"attributes": {
|
|
29
|
-
"type": [
|
|
30
|
-
"null",
|
|
31
|
-
"object"
|
|
32
|
-
],
|
|
33
|
-
"additionalProperties": true,
|
|
12
|
+
"data": {
|
|
13
|
+
"type": "object",
|
|
34
14
|
"properties": {
|
|
35
|
-
"
|
|
36
|
-
"type":
|
|
37
|
-
|
|
38
|
-
"
|
|
39
|
-
]
|
|
40
|
-
},
|
|
41
|
-
"phone_number": {
|
|
42
|
-
"type": [
|
|
43
|
-
"null",
|
|
44
|
-
"string"
|
|
45
|
-
]
|
|
46
|
-
},
|
|
47
|
-
"first_name": {
|
|
48
|
-
"type": [
|
|
49
|
-
"null",
|
|
50
|
-
"string"
|
|
51
|
-
]
|
|
52
|
-
},
|
|
53
|
-
"last_name": {
|
|
54
|
-
"type": [
|
|
55
|
-
"null",
|
|
56
|
-
"string"
|
|
15
|
+
"type": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"enum": [
|
|
18
|
+
"profile"
|
|
57
19
|
]
|
|
58
20
|
},
|
|
59
|
-
"
|
|
60
|
-
"type":
|
|
61
|
-
|
|
62
|
-
"
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
"
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
21
|
+
"attributes": {
|
|
22
|
+
"type": "object",
|
|
23
|
+
"properties": {
|
|
24
|
+
"email": {
|
|
25
|
+
"type": "string",
|
|
26
|
+
"format": "email"
|
|
27
|
+
},
|
|
28
|
+
"phone_number": {
|
|
29
|
+
"type": "string"
|
|
30
|
+
},
|
|
31
|
+
"external_id": {
|
|
32
|
+
"type": "string",
|
|
33
|
+
"format": "uuid"
|
|
34
|
+
},
|
|
35
|
+
"first_name": {
|
|
36
|
+
"type": "string"
|
|
37
|
+
},
|
|
38
|
+
"last_name": {
|
|
39
|
+
"type": "string"
|
|
40
|
+
},
|
|
41
|
+
"organization": {
|
|
42
|
+
"type": "string"
|
|
43
|
+
},
|
|
44
|
+
"title": {
|
|
45
|
+
"type": "string"
|
|
46
|
+
},
|
|
47
|
+
"image": {
|
|
48
|
+
"type": "string",
|
|
49
|
+
"format": "uri"
|
|
50
|
+
},
|
|
51
|
+
"location": {
|
|
52
|
+
"type": "object",
|
|
53
|
+
"properties": {
|
|
54
|
+
"address1": {
|
|
55
|
+
"type": "string"
|
|
56
|
+
},
|
|
57
|
+
"address2": {
|
|
58
|
+
"type": "string"
|
|
59
|
+
},
|
|
60
|
+
"city": {
|
|
61
|
+
"type": "string"
|
|
62
|
+
},
|
|
63
|
+
"country": {
|
|
64
|
+
"type": "string"
|
|
65
|
+
},
|
|
66
|
+
"region": {
|
|
67
|
+
"type": "string"
|
|
68
|
+
},
|
|
69
|
+
"zip": {
|
|
70
|
+
"type": "string"
|
|
71
|
+
},
|
|
72
|
+
"timezone": {
|
|
73
|
+
"type": "string"
|
|
74
|
+
},
|
|
75
|
+
"ip": {
|
|
76
|
+
"type": "string",
|
|
77
|
+
"format": "ipv4"
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
},
|
|
82
|
+
"properties": {
|
|
83
|
+
"type": "object",
|
|
84
|
+
"additionalProperties": {
|
|
85
|
+
"type": "string"
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
},
|
|
89
|
+
"required": [
|
|
90
|
+
"email",
|
|
91
|
+
"phone_number"
|
|
76
92
|
]
|
|
77
|
-
},
|
|
78
|
-
"last_event_date": {
|
|
79
|
-
"type": [
|
|
80
|
-
"null",
|
|
81
|
-
"string"
|
|
82
|
-
],
|
|
83
|
-
"format": "date-time"
|
|
84
93
|
}
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
"null",
|
|
90
|
-
"object"
|
|
91
|
-
]
|
|
92
|
-
},
|
|
93
|
-
"relationships": {
|
|
94
|
-
"type": [
|
|
95
|
-
"null",
|
|
96
|
-
"object"
|
|
97
|
-
]
|
|
98
|
-
},
|
|
99
|
-
"segments": {
|
|
100
|
-
"type": [
|
|
101
|
-
"null",
|
|
102
|
-
"object"
|
|
94
|
+
},
|
|
95
|
+
"required": [
|
|
96
|
+
"type",
|
|
97
|
+
"attributes"
|
|
103
98
|
]
|
|
104
99
|
}
|
|
105
|
-
}
|
|
100
|
+
},
|
|
101
|
+
"required": [
|
|
102
|
+
"data"
|
|
103
|
+
]
|
|
106
104
|
},
|
|
107
105
|
"supported_sync_modes": [
|
|
108
106
|
"full_refresh",
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "stringio"
|
|
4
|
+
|
|
5
|
+
module Multiwoven
|
|
6
|
+
module Integrations
|
|
7
|
+
module Destination
|
|
8
|
+
module SalesforceCrm
|
|
9
|
+
include Multiwoven::Integrations::Core
|
|
10
|
+
|
|
11
|
+
API_VERSION = "59.0"
|
|
12
|
+
|
|
13
|
+
class Client < DestinationConnector
|
|
14
|
+
def check_connection(connection_config)
|
|
15
|
+
connection_config = connection_config.with_indifferent_access
|
|
16
|
+
initialize_client(connection_config)
|
|
17
|
+
authenticate_client
|
|
18
|
+
success_status
|
|
19
|
+
rescue StandardError => e
|
|
20
|
+
failure_status(e)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def discover(_connection_config = nil)
|
|
24
|
+
catalog = build_catalog(load_catalog_streams)
|
|
25
|
+
catalog.to_multiwoven_message
|
|
26
|
+
rescue StandardError => e
|
|
27
|
+
handle_exception("SALESFORCE:CRM:DISCOVER:EXCEPTION", "error", e)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def write(sync_config, records, _action = "create")
|
|
31
|
+
initialize_client(sync_config[:destination][:connection_specification])
|
|
32
|
+
process_records(records, sync_config[:stream])
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
handle_exception("SALESFORCE:CRM:WRITE:EXCEPTION", "error", e)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def initialize_client(config)
|
|
40
|
+
config = config.with_indifferent_access
|
|
41
|
+
@client = Restforce.new(oauth_token: config[:access_token],
|
|
42
|
+
refresh_token: config[:refresh_token],
|
|
43
|
+
instance_url: config[:instance_url],
|
|
44
|
+
client_id: config[:client_id],
|
|
45
|
+
client_secret: config[:client_secret],
|
|
46
|
+
authentication_callback: proc { |x| log_debug(x.to_s) },
|
|
47
|
+
api_version: API_VERSION)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def process_records(records, stream)
|
|
51
|
+
write_success = 0
|
|
52
|
+
write_failure = 0
|
|
53
|
+
properties = stream[:json_schema][:properties]
|
|
54
|
+
records.each do |record_object|
|
|
55
|
+
record = extract_data(record_object, properties)
|
|
56
|
+
process_record(stream, record)
|
|
57
|
+
write_success += 1
|
|
58
|
+
rescue StandardError => e
|
|
59
|
+
handle_exception("SALESFORCE:CRM:WRITE:EXCEPTION", "error", e)
|
|
60
|
+
write_failure += 1
|
|
61
|
+
end
|
|
62
|
+
tracking_message(write_success, write_failure)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def process_record(stream, record)
|
|
66
|
+
send_data_to_salesforce(stream[:action], stream[:name], record)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def send_data_to_salesforce(action, stream_name, record = {})
|
|
70
|
+
method_name = "#{action}!"
|
|
71
|
+
args = build_args(action, stream_name, record)
|
|
72
|
+
@client.send(method_name, *args)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def build_args(action, stream_name, record)
|
|
76
|
+
case action
|
|
77
|
+
when :upsert
|
|
78
|
+
[stream_name, record[:external_key], record]
|
|
79
|
+
when :destroy
|
|
80
|
+
[stream_name, record[:id]]
|
|
81
|
+
else
|
|
82
|
+
[stream_name, record]
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def authenticate_client
|
|
87
|
+
@client.authenticate!
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def success_status
|
|
91
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def failure_status(error)
|
|
95
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def load_catalog_streams
|
|
99
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
100
|
+
catalog_json["streams"].map { |stream| build_stream(stream) }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def build_stream(stream)
|
|
104
|
+
Multiwoven::Integrations::Protocol::Stream.new(
|
|
105
|
+
name: stream["name"], json_schema: stream["json_schema"],
|
|
106
|
+
action: stream["action"]
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def build_catalog(streams)
|
|
111
|
+
Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def tracking_message(success, failure)
|
|
115
|
+
Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
116
|
+
success: success, failed: failure
|
|
117
|
+
).to_multiwoven_message
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def log_debug(message)
|
|
121
|
+
Multiwoven::Integrations::Service.logger.debug(message)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
{
|
|
2
|
+
"streams": [
|
|
3
|
+
{
|
|
4
|
+
"name": "Account",
|
|
5
|
+
"action": "create",
|
|
6
|
+
"json_schema": {
|
|
7
|
+
"type": "object",
|
|
8
|
+
"additionalProperties": true,
|
|
9
|
+
"properties": {
|
|
10
|
+
"Id": {
|
|
11
|
+
"type": "string"
|
|
12
|
+
},
|
|
13
|
+
"IsDeleted": {
|
|
14
|
+
"type": "boolean"
|
|
15
|
+
},
|
|
16
|
+
"MasterRecordId": {
|
|
17
|
+
"type": ["string", "null"]
|
|
18
|
+
},
|
|
19
|
+
"Name": {
|
|
20
|
+
"type": "string"
|
|
21
|
+
},
|
|
22
|
+
"Type": {
|
|
23
|
+
"type": ["string", "null"],
|
|
24
|
+
"enum": [
|
|
25
|
+
"Prospect",
|
|
26
|
+
"Customer - Direct",
|
|
27
|
+
"Customer - Channel",
|
|
28
|
+
"Channel Partner / Reseller",
|
|
29
|
+
"Installation Partner",
|
|
30
|
+
"Technology Partner",
|
|
31
|
+
"Other"
|
|
32
|
+
]
|
|
33
|
+
},
|
|
34
|
+
"ParentId": {
|
|
35
|
+
"type": ["string", "null"]
|
|
36
|
+
},
|
|
37
|
+
"BillingStreet": {
|
|
38
|
+
"type": ["string", "null"]
|
|
39
|
+
},
|
|
40
|
+
"BillingCity": {
|
|
41
|
+
"type": ["string", "null"]
|
|
42
|
+
},
|
|
43
|
+
"BillingState": {
|
|
44
|
+
"type": ["string", "null"]
|
|
45
|
+
},
|
|
46
|
+
"BillingPostalCode": {
|
|
47
|
+
"type": ["string", "null"]
|
|
48
|
+
},
|
|
49
|
+
"BillingCountry": {
|
|
50
|
+
"type": ["string", "null"]
|
|
51
|
+
},
|
|
52
|
+
"BillingLatitude": {
|
|
53
|
+
"type": ["number", "null"]
|
|
54
|
+
},
|
|
55
|
+
"BillingLongitude": {
|
|
56
|
+
"type": ["number", "null"]
|
|
57
|
+
},
|
|
58
|
+
"BillingGeocodeAccuracy": {
|
|
59
|
+
"type": ["string", "null"],
|
|
60
|
+
"enum": [
|
|
61
|
+
"Address",
|
|
62
|
+
"NearAddress",
|
|
63
|
+
"Block",
|
|
64
|
+
"Street",
|
|
65
|
+
"ExtendedZip",
|
|
66
|
+
"Zip",
|
|
67
|
+
"Neighborhood",
|
|
68
|
+
"City",
|
|
69
|
+
"County",
|
|
70
|
+
"State",
|
|
71
|
+
"Unknown"
|
|
72
|
+
]
|
|
73
|
+
},
|
|
74
|
+
"BillingAddress": {
|
|
75
|
+
"type": ["string", "null"],
|
|
76
|
+
"enum": [
|
|
77
|
+
"Address",
|
|
78
|
+
"NearAddress",
|
|
79
|
+
"Block",
|
|
80
|
+
"Street",
|
|
81
|
+
"ExtendedZip",
|
|
82
|
+
"Zip",
|
|
83
|
+
"Neighborhood",
|
|
84
|
+
"City",
|
|
85
|
+
"County",
|
|
86
|
+
"State",
|
|
87
|
+
"Unknown"
|
|
88
|
+
]
|
|
89
|
+
},
|
|
90
|
+
"ShippingStreet": {
|
|
91
|
+
"type": ["string", "null"]
|
|
92
|
+
},
|
|
93
|
+
"ShippingCity": {
|
|
94
|
+
"type": ["string", "null"]
|
|
95
|
+
},
|
|
96
|
+
"ShippingState": {
|
|
97
|
+
"type": ["string", "null"]
|
|
98
|
+
},
|
|
99
|
+
"ShippingPostalCode": {
|
|
100
|
+
"type": ["string", "null"]
|
|
101
|
+
},
|
|
102
|
+
"ShippingCountry": {
|
|
103
|
+
"type": ["string", "null"]
|
|
104
|
+
},
|
|
105
|
+
"ShippingLatitude": {
|
|
106
|
+
"type": ["number", "null"]
|
|
107
|
+
},
|
|
108
|
+
"ShippingLongitude": {
|
|
109
|
+
"type": ["number", "null"]
|
|
110
|
+
},
|
|
111
|
+
"ShippingGeocodeAccuracy": {
|
|
112
|
+
"type": ["string", "null"]
|
|
113
|
+
},
|
|
114
|
+
"ShippingAddress": {
|
|
115
|
+
"type": ["string", "null"]
|
|
116
|
+
},
|
|
117
|
+
"Phone": {
|
|
118
|
+
"type": ["string", "null"]
|
|
119
|
+
},
|
|
120
|
+
"Fax": {
|
|
121
|
+
"type": ["string", "null"]
|
|
122
|
+
},
|
|
123
|
+
"AccountNumber": {
|
|
124
|
+
"type": ["string", "null"]
|
|
125
|
+
},
|
|
126
|
+
"Website": {
|
|
127
|
+
"type": ["string", "null"],
|
|
128
|
+
"format": "uri"
|
|
129
|
+
},
|
|
130
|
+
"PhotoUrl": {
|
|
131
|
+
"type": ["string", "null"],
|
|
132
|
+
"format": "uri"
|
|
133
|
+
},
|
|
134
|
+
"Sic": {
|
|
135
|
+
"type": ["string", "null"]
|
|
136
|
+
},
|
|
137
|
+
"Industry": {
|
|
138
|
+
"type": ["string", "null"],
|
|
139
|
+
"enum": [
|
|
140
|
+
"Agriculture",
|
|
141
|
+
"Apparel",
|
|
142
|
+
"Banking",
|
|
143
|
+
"Biotechnology",
|
|
144
|
+
"Chemicals",
|
|
145
|
+
"Communications",
|
|
146
|
+
"Construction",
|
|
147
|
+
"Consulting",
|
|
148
|
+
"Education",
|
|
149
|
+
"Electronics",
|
|
150
|
+
"Energy",
|
|
151
|
+
"Engineering",
|
|
152
|
+
"Entertainment",
|
|
153
|
+
"Environmental",
|
|
154
|
+
"Finance",
|
|
155
|
+
"Food & Beverage",
|
|
156
|
+
"Government",
|
|
157
|
+
"Healthcare",
|
|
158
|
+
"Hospitality",
|
|
159
|
+
"Insurance",
|
|
160
|
+
"Machinery",
|
|
161
|
+
"Manufacturing",
|
|
162
|
+
"Media",
|
|
163
|
+
"Not For Profit",
|
|
164
|
+
"Recreation",
|
|
165
|
+
"Retail",
|
|
166
|
+
"Shipping",
|
|
167
|
+
"Technology",
|
|
168
|
+
"Telecommunications",
|
|
169
|
+
"Transportation",
|
|
170
|
+
"Utilities",
|
|
171
|
+
"Other"
|
|
172
|
+
]
|
|
173
|
+
},
|
|
174
|
+
"AnnualRevenue": {
|
|
175
|
+
"type": ["number", "null"]
|
|
176
|
+
},
|
|
177
|
+
"NumberOfEmployees": {
|
|
178
|
+
"type": ["integer", "null"]
|
|
179
|
+
},
|
|
180
|
+
"Ownership": {
|
|
181
|
+
"type": ["string", "null"],
|
|
182
|
+
"enum": ["Public", "Private", "Subsidiary", "Other"]
|
|
183
|
+
},
|
|
184
|
+
"TickerSymbol": {
|
|
185
|
+
"type": ["string", "null"]
|
|
186
|
+
},
|
|
187
|
+
"Description": {
|
|
188
|
+
"type": ["string", "null"]
|
|
189
|
+
},
|
|
190
|
+
"Rating": {
|
|
191
|
+
"type": ["string", "null"],
|
|
192
|
+
"enum": ["Hot", "Warm", "Cold"]
|
|
193
|
+
},
|
|
194
|
+
"Site": {
|
|
195
|
+
"type": ["string", "null"]
|
|
196
|
+
},
|
|
197
|
+
"OwnerId": {
|
|
198
|
+
"type": "string"
|
|
199
|
+
},
|
|
200
|
+
"CreatedDate": {
|
|
201
|
+
"type": "string",
|
|
202
|
+
"format": "date-time"
|
|
203
|
+
},
|
|
204
|
+
"CreatedById": {
|
|
205
|
+
"type": "string"
|
|
206
|
+
},
|
|
207
|
+
"LastModifiedDate": {
|
|
208
|
+
"type": "string",
|
|
209
|
+
"format": "date-time"
|
|
210
|
+
},
|
|
211
|
+
"LastModifiedById": {
|
|
212
|
+
"type": "string"
|
|
213
|
+
},
|
|
214
|
+
"SystemModstamp": {
|
|
215
|
+
"type": "string",
|
|
216
|
+
"format": "date-time"
|
|
217
|
+
},
|
|
218
|
+
"LastActivityDate": {
|
|
219
|
+
"type": ["string", "null"],
|
|
220
|
+
"format": "date"
|
|
221
|
+
},
|
|
222
|
+
"LastViewedDate": {
|
|
223
|
+
"type": ["string", "null"],
|
|
224
|
+
"format": "date-time"
|
|
225
|
+
},
|
|
226
|
+
"LastReferencedDate": {
|
|
227
|
+
"type": ["string", "null"],
|
|
228
|
+
"format": "date-time"
|
|
229
|
+
},
|
|
230
|
+
"Jigsaw": {
|
|
231
|
+
"type": ["string", "null"]
|
|
232
|
+
},
|
|
233
|
+
"JigsawCompanyId": {
|
|
234
|
+
"type": ["string", "null"]
|
|
235
|
+
},
|
|
236
|
+
"CleanStatus": {
|
|
237
|
+
"type": ["string", "null"],
|
|
238
|
+
"enum": [
|
|
239
|
+
"In Sync",
|
|
240
|
+
"Different",
|
|
241
|
+
"Reviewed",
|
|
242
|
+
"Not Found",
|
|
243
|
+
"Inactive",
|
|
244
|
+
"Not Compared",
|
|
245
|
+
"Select Match",
|
|
246
|
+
"Skipped"
|
|
247
|
+
]
|
|
248
|
+
},
|
|
249
|
+
"AccountSource": {
|
|
250
|
+
"type": ["string", "null"],
|
|
251
|
+
"enum": [
|
|
252
|
+
"Web",
|
|
253
|
+
"Phone Inquiry",
|
|
254
|
+
"Partner Referral",
|
|
255
|
+
"Purchased List",
|
|
256
|
+
"Other"
|
|
257
|
+
]
|
|
258
|
+
},
|
|
259
|
+
"DunsNumber": {
|
|
260
|
+
"type": ["string", "null"]
|
|
261
|
+
},
|
|
262
|
+
"Tradestyle": {
|
|
263
|
+
"type": ["string", "null"]
|
|
264
|
+
},
|
|
265
|
+
"NaicsCode": {
|
|
266
|
+
"type": ["string", "null"]
|
|
267
|
+
},
|
|
268
|
+
"NaicsDesc": {
|
|
269
|
+
"type": ["string", "null"]
|
|
270
|
+
},
|
|
271
|
+
"YearStarted": {
|
|
272
|
+
"type": ["string", "null"]
|
|
273
|
+
},
|
|
274
|
+
"SicDesc": {
|
|
275
|
+
"type": ["string", "null"]
|
|
276
|
+
},
|
|
277
|
+
"DandbCompanyId": {
|
|
278
|
+
"type": ["string", "null"]
|
|
279
|
+
},
|
|
280
|
+
"OperatingHoursId": {
|
|
281
|
+
"type": ["string", "null"]
|
|
282
|
+
},
|
|
283
|
+
"CustomerPriority__c": {
|
|
284
|
+
"type": ["string", "null"],
|
|
285
|
+
"enum": ["High", "Low", "Medium"]
|
|
286
|
+
},
|
|
287
|
+
"SLA__c": {
|
|
288
|
+
"type": ["string", "null"],
|
|
289
|
+
"enum": ["Gold", "Silver", "Platinum", "Bronze"]
|
|
290
|
+
},
|
|
291
|
+
"Active__c": {
|
|
292
|
+
"type": ["string", "null"],
|
|
293
|
+
"enum": ["Yes", "No"]
|
|
294
|
+
},
|
|
295
|
+
"NumberofLocations__c": {
|
|
296
|
+
"type": ["number", "null"]
|
|
297
|
+
},
|
|
298
|
+
"UpsellOpportunity__c": {
|
|
299
|
+
"type": ["string", "null"],
|
|
300
|
+
"enum": ["Yes", "No", "Maybe"]
|
|
301
|
+
},
|
|
302
|
+
"SLASerialNumber__c": {
|
|
303
|
+
"type": ["string", "null"]
|
|
304
|
+
},
|
|
305
|
+
"SLAExpirationDate__c": {
|
|
306
|
+
"type": ["string", "null"],
|
|
307
|
+
"format": "date"
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
},
|
|
311
|
+
"supported_sync_modes": ["full_refresh", "incremental"],
|
|
312
|
+
"source_defined_cursor": true,
|
|
313
|
+
"default_cursor_field": ["updated"],
|
|
314
|
+
"source_defined_primary_key": [["Id"]]
|
|
315
|
+
}
|
|
316
|
+
]
|
|
317
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data":
|
|
3
|
+
{
|
|
4
|
+
"name": "Salesforce CRM",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"connector_subtype": "API",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-salesforce-crm",
|
|
9
|
+
"icon": "salesforce.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/salesforce_crm",
|
|
3
|
+
"stream_type": "static",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Salesforce Destination Spec",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["access_token", "refresh_token", "instance_url", "client_id", "client_secret"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"access_token": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"title": "Access Token",
|
|
13
|
+
"order": 0
|
|
14
|
+
},
|
|
15
|
+
"refresh_token": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"title": "Refresh Token",
|
|
18
|
+
"order": 1
|
|
19
|
+
},
|
|
20
|
+
"instance_url": {
|
|
21
|
+
"type": "string",
|
|
22
|
+
"title": "Instance URL",
|
|
23
|
+
"order": 2
|
|
24
|
+
},
|
|
25
|
+
"client_id": {
|
|
26
|
+
"type": "string",
|
|
27
|
+
"title": "Client ID",
|
|
28
|
+
"order": 3
|
|
29
|
+
},
|
|
30
|
+
"client_secret": {
|
|
31
|
+
"type": "string",
|
|
32
|
+
"title": "Client Secret",
|
|
33
|
+
"order": 4
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.1.
|
|
5
|
+
VERSION = "0.1.2"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -12,6 +12,8 @@ module Multiwoven
|
|
|
12
12
|
|
|
13
13
|
ENABLED_DESTINATIONS = %w[
|
|
14
14
|
Klaviyo
|
|
15
|
+
SalesforceCrm
|
|
16
|
+
FacebookCustomAudience
|
|
15
17
|
].freeze
|
|
16
18
|
end
|
|
17
19
|
end
|
|
@@ -7,6 +7,7 @@ module Multiwoven::Integrations::Source
|
|
|
7
7
|
include Multiwoven::Integrations::Core
|
|
8
8
|
class Client < SourceConnector
|
|
9
9
|
def check_connection(connection_config)
|
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
|
10
11
|
bigquery = create_connection(connection_config)
|
|
11
12
|
bigquery.datasets
|
|
12
13
|
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
@@ -15,6 +16,7 @@ module Multiwoven::Integrations::Source
|
|
|
15
16
|
end
|
|
16
17
|
|
|
17
18
|
def discover(connection_config)
|
|
19
|
+
connection_config = connection_config.with_indifferent_access
|
|
18
20
|
bigquery = create_connection(connection_config)
|
|
19
21
|
target_dataset_id = connection_config["dataset_id"]
|
|
20
22
|
records = bigquery.datasets.flat_map do |dataset|
|
|
@@ -43,7 +45,11 @@ module Multiwoven::Integrations::Source
|
|
|
43
45
|
|
|
44
46
|
def read(sync_config)
|
|
45
47
|
connection_config = sync_config.source.connection_specification
|
|
48
|
+
connection_config = connection_config.with_indifferent_access
|
|
46
49
|
query = sync_config.model.query
|
|
50
|
+
|
|
51
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
52
|
+
|
|
47
53
|
bigquery = create_connection(connection_config)
|
|
48
54
|
records = []
|
|
49
55
|
results = bigquery.query query
|
|
@@ -7,6 +7,7 @@ module Multiwoven::Integrations::Source
|
|
|
7
7
|
include Multiwoven::Integrations::Core
|
|
8
8
|
class Client < SourceConnector
|
|
9
9
|
def check_connection(connection_config)
|
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
|
10
11
|
create_connection(connection_config)
|
|
11
12
|
ConnectionStatus.new(
|
|
12
13
|
status: ConnectionStatusType["succeeded"]
|
|
@@ -18,6 +19,7 @@ module Multiwoven::Integrations::Source
|
|
|
18
19
|
end
|
|
19
20
|
|
|
20
21
|
def discover(connection_config)
|
|
22
|
+
connection_config = connection_config.with_indifferent_access
|
|
21
23
|
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
22
24
|
FROM information_schema.columns
|
|
23
25
|
WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
|
|
@@ -43,7 +45,10 @@ module Multiwoven::Integrations::Source
|
|
|
43
45
|
|
|
44
46
|
def read(sync_config)
|
|
45
47
|
connection_config = sync_config.source.connection_specification
|
|
48
|
+
connection_config = connection_config.with_indifferent_access
|
|
46
49
|
query = sync_config.model.query
|
|
50
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
51
|
+
|
|
47
52
|
db = create_connection(connection_config)
|
|
48
53
|
records = db.exec(query) do |result|
|
|
49
54
|
result.map do |row|
|
|
@@ -5,6 +5,7 @@ module Multiwoven::Integrations::Source
|
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
6
|
class Client < SourceConnector
|
|
7
7
|
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
8
9
|
create_connection(connection_config)
|
|
9
10
|
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
|
10
11
|
rescue Sequel::DatabaseConnectionError => e
|
|
@@ -12,6 +13,7 @@ module Multiwoven::Integrations::Source
|
|
|
12
13
|
end
|
|
13
14
|
|
|
14
15
|
def discover(connection_config)
|
|
16
|
+
connection_config = connection_config.with_indifferent_access
|
|
15
17
|
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
16
18
|
FROM information_schema.columns
|
|
17
19
|
WHERE table_schema = \'#{connection_config[:schema]}\' AND table_catalog = \'#{connection_config[:database]}\'
|
|
@@ -35,7 +37,10 @@ module Multiwoven::Integrations::Source
|
|
|
35
37
|
|
|
36
38
|
def read(sync_config)
|
|
37
39
|
connection_config = sync_config.source.connection_specification
|
|
40
|
+
connection_config = connection_config.with_indifferent_access
|
|
38
41
|
query = sync_config.model.query
|
|
42
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
43
|
+
|
|
39
44
|
db = create_connection(connection_config)
|
|
40
45
|
|
|
41
46
|
records = []
|
|
@@ -10,6 +10,8 @@ require "byebug"
|
|
|
10
10
|
require "net/http"
|
|
11
11
|
require "uri"
|
|
12
12
|
require "active_support/core_ext/hash/indifferent_access"
|
|
13
|
+
require "restforce"
|
|
14
|
+
require "logger"
|
|
13
15
|
|
|
14
16
|
# Service
|
|
15
17
|
require_relative "integrations/config"
|
|
@@ -32,6 +34,8 @@ require_relative "integrations/source/bigquery/client"
|
|
|
32
34
|
|
|
33
35
|
# Destination
|
|
34
36
|
require_relative "integrations/destination/klaviyo/client"
|
|
37
|
+
require_relative "integrations/destination/salesforce_crm/client"
|
|
38
|
+
require_relative "integrations/destination/facebook_custom_audience/client"
|
|
35
39
|
|
|
36
40
|
module Multiwoven
|
|
37
41
|
module Integrations
|
|
@@ -8,17 +8,18 @@ Gem::Specification.new do |spec|
|
|
|
8
8
|
spec.authors = ["Subin T P"]
|
|
9
9
|
spec.email = ["subin@multiwoven.com"]
|
|
10
10
|
|
|
11
|
-
spec.summary = "
|
|
12
|
-
spec.description = "
|
|
11
|
+
spec.summary = "Integration suite for open source reverse ETL platform"
|
|
12
|
+
spec.description = "Multiwoven Integrations is a comprehensive Ruby gem designed to facilitate seamless connectivity between various data sources and SaaS platforms."
|
|
13
|
+
|
|
13
14
|
spec.homepage = "https://www.multiwoven.com/"
|
|
14
15
|
spec.license = "MIT"
|
|
15
16
|
spec.required_ruby_version = ">= 2.6.0"
|
|
16
17
|
|
|
17
18
|
# spec.metadata["allowed_push_host"] = nil
|
|
18
|
-
|
|
19
|
+
spec.metadata["github_repo"] = "https://github.com/Multiwoven/multiwoven-integrations"
|
|
19
20
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
20
|
-
spec.metadata["source_code_uri"] = "https://
|
|
21
|
-
spec.metadata["changelog_uri"] = "https://
|
|
21
|
+
spec.metadata["source_code_uri"] = "https://github.com/Multiwoven/multiwoven-integrations"
|
|
22
|
+
spec.metadata["changelog_uri"] = "https://github.com/Multiwoven/multiwoven-integrations/blob/master/CHANGELOG.md"
|
|
22
23
|
|
|
23
24
|
# Specify which files should be added to the gem when it is released.
|
|
24
25
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
@@ -39,7 +40,8 @@ Gem::Specification.new do |spec|
|
|
|
39
40
|
spec.add_runtime_dependency "google-cloud-bigquery"
|
|
40
41
|
spec.add_runtime_dependency "pg"
|
|
41
42
|
spec.add_runtime_dependency "rake"
|
|
42
|
-
spec.add_runtime_dependency "
|
|
43
|
+
spec.add_runtime_dependency "restforce"
|
|
44
|
+
spec.add_runtime_dependency "ruby-odbc"
|
|
43
45
|
spec.add_runtime_dependency "sequel"
|
|
44
46
|
|
|
45
47
|
spec.add_development_dependency "byebug"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-01-
|
|
11
|
+
date: 2024-01-23 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -109,7 +109,21 @@ dependencies:
|
|
|
109
109
|
- !ruby/object:Gem::Version
|
|
110
110
|
version: '0'
|
|
111
111
|
- !ruby/object:Gem::Dependency
|
|
112
|
-
name:
|
|
112
|
+
name: restforce
|
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
|
114
|
+
requirements:
|
|
115
|
+
- - ">="
|
|
116
|
+
- !ruby/object:Gem::Version
|
|
117
|
+
version: '0'
|
|
118
|
+
type: :runtime
|
|
119
|
+
prerelease: false
|
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - ">="
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '0'
|
|
125
|
+
- !ruby/object:Gem::Dependency
|
|
126
|
+
name: ruby-odbc
|
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
|
114
128
|
requirements:
|
|
115
129
|
- - ">="
|
|
@@ -220,7 +234,8 @@ dependencies:
|
|
|
220
234
|
- - ">="
|
|
221
235
|
- !ruby/object:Gem::Version
|
|
222
236
|
version: '0'
|
|
223
|
-
description:
|
|
237
|
+
description: Multiwoven Integrations is a comprehensive Ruby gem designed to facilitate
|
|
238
|
+
seamless connectivity between various data sources and SaaS platforms.
|
|
224
239
|
email:
|
|
225
240
|
- subin@multiwoven.com
|
|
226
241
|
executables: []
|
|
@@ -230,6 +245,7 @@ files:
|
|
|
230
245
|
- ".rspec"
|
|
231
246
|
- ".rubocop.yml"
|
|
232
247
|
- ".ruby-version"
|
|
248
|
+
- ".vscode/settings.json"
|
|
233
249
|
- CHANGELOG.md
|
|
234
250
|
- CODE_OF_CONDUCT.md
|
|
235
251
|
- LICENSE.txt
|
|
@@ -243,10 +259,18 @@ files:
|
|
|
243
259
|
- lib/multiwoven/integrations/core/http_client.rb
|
|
244
260
|
- lib/multiwoven/integrations/core/source_connector.rb
|
|
245
261
|
- lib/multiwoven/integrations/core/utils.rb
|
|
262
|
+
- lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb
|
|
263
|
+
- lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json
|
|
264
|
+
- lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json
|
|
265
|
+
- lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json
|
|
246
266
|
- lib/multiwoven/integrations/destination/klaviyo/client.rb
|
|
247
267
|
- lib/multiwoven/integrations/destination/klaviyo/config/catalog.json
|
|
248
268
|
- lib/multiwoven/integrations/destination/klaviyo/config/meta.json
|
|
249
269
|
- lib/multiwoven/integrations/destination/klaviyo/config/spec.json
|
|
270
|
+
- lib/multiwoven/integrations/destination/salesforce_crm/client.rb
|
|
271
|
+
- lib/multiwoven/integrations/destination/salesforce_crm/config/catalog.json
|
|
272
|
+
- lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json
|
|
273
|
+
- lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json
|
|
250
274
|
- lib/multiwoven/integrations/protocol/protocol.json
|
|
251
275
|
- lib/multiwoven/integrations/protocol/protocol.rb
|
|
252
276
|
- lib/multiwoven/integrations/rollout.rb
|
|
@@ -266,9 +290,10 @@ homepage: https://www.multiwoven.com/
|
|
|
266
290
|
licenses:
|
|
267
291
|
- MIT
|
|
268
292
|
metadata:
|
|
293
|
+
github_repo: https://github.com/Multiwoven/multiwoven-integrations
|
|
269
294
|
homepage_uri: https://www.multiwoven.com/
|
|
270
|
-
source_code_uri: https://
|
|
271
|
-
changelog_uri: https://
|
|
295
|
+
source_code_uri: https://github.com/Multiwoven/multiwoven-integrations
|
|
296
|
+
changelog_uri: https://github.com/Multiwoven/multiwoven-integrations/blob/master/CHANGELOG.md
|
|
272
297
|
post_install_message:
|
|
273
298
|
rdoc_options: []
|
|
274
299
|
require_paths:
|
|
@@ -287,5 +312,5 @@ requirements: []
|
|
|
287
312
|
rubygems_version: 3.4.22
|
|
288
313
|
signing_key:
|
|
289
314
|
specification_version: 4
|
|
290
|
-
summary:
|
|
315
|
+
summary: Integration suite for open source reverse ETL platform
|
|
291
316
|
test_files: []
|