etl-integrations 0.1.81
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +34 -0
- data/.ruby-version +1 -0
- data/.vscode/settings.json +5 -0
- data/CHANGELOG.md +38 -0
- data/CODE_OF_CONDUCT.md +84 -0
- data/LICENSE.txt +21 -0
- data/README.md +105 -0
- data/Rakefile +12 -0
- data/lib/multiwoven/integrations/config.rb +13 -0
- data/lib/multiwoven/integrations/core/base_connector.rb +70 -0
- data/lib/multiwoven/integrations/core/constants.rb +46 -0
- data/lib/multiwoven/integrations/core/destination_connector.rb +14 -0
- data/lib/multiwoven/integrations/core/fullrefresher.rb +19 -0
- data/lib/multiwoven/integrations/core/http_client.rb +34 -0
- data/lib/multiwoven/integrations/core/query_builder.rb +27 -0
- data/lib/multiwoven/integrations/core/rate_limiter.rb +19 -0
- data/lib/multiwoven/integrations/core/source_connector.rb +38 -0
- data/lib/multiwoven/integrations/core/utils.rb +104 -0
- data/lib/multiwoven/integrations/destination/airtable/client.rb +153 -0
- data/lib/multiwoven/integrations/destination/airtable/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/destination/airtable/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/airtable/config/spec.json +22 -0
- data/lib/multiwoven/integrations/destination/airtable/icon.svg +6 -0
- data/lib/multiwoven/integrations/destination/airtable/schema_helper.rb +141 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +124 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json +42 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json +27 -0
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/icon.svg +23 -0
- data/lib/multiwoven/integrations/destination/google_sheets/client.rb +231 -0
- data/lib/multiwoven/integrations/destination/google_sheets/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/destination/google_sheets/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json +74 -0
- data/lib/multiwoven/integrations/destination/google_sheets/icon.svg +1 -0
- data/lib/multiwoven/integrations/destination/hubspot/client.rb +107 -0
- data/lib/multiwoven/integrations/destination/hubspot/config/catalog.json +351 -0
- data/lib/multiwoven/integrations/destination/hubspot/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/hubspot/config/spec.json +17 -0
- data/lib/multiwoven/integrations/destination/hubspot/icon.svg +5 -0
- data/lib/multiwoven/integrations/destination/klaviyo/client.rb +116 -0
- data/lib/multiwoven/integrations/destination/klaviyo/config/catalog.json +103 -0
- data/lib/multiwoven/integrations/destination/klaviyo/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/klaviyo/config/spec.json +22 -0
- data/lib/multiwoven/integrations/destination/klaviyo/icon.svg +6 -0
- data/lib/multiwoven/integrations/destination/postgresql/client.rb +123 -0
- data/lib/multiwoven/integrations/destination/postgresql/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/postgresql/config/spec.json +68 -0
- data/lib/multiwoven/integrations/destination/postgresql/icon.svg +20 -0
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb +114 -0
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json +16 -0
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/spec.json +49 -0
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/icon.svg +16 -0
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb +132 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb +117 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/catalog.json +320 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json +43 -0
- data/lib/multiwoven/integrations/destination/salesforce_crm/icon.svg +16 -0
- data/lib/multiwoven/integrations/destination/sftp/client.rb +133 -0
- data/lib/multiwoven/integrations/destination/sftp/config/catalog.json +16 -0
- data/lib/multiwoven/integrations/destination/sftp/config/meta.json +16 -0
- data/lib/multiwoven/integrations/destination/sftp/config/spec.json +50 -0
- data/lib/multiwoven/integrations/destination/sftp/icon.svg +1 -0
- data/lib/multiwoven/integrations/destination/slack/client.rb +114 -0
- data/lib/multiwoven/integrations/destination/slack/config/catalog.json +22 -0
- data/lib/multiwoven/integrations/destination/slack/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/slack/config/spec.json +22 -0
- data/lib/multiwoven/integrations/destination/slack/icon.svg +26 -0
- data/lib/multiwoven/integrations/destination/stripe/client.rb +82 -0
- data/lib/multiwoven/integrations/destination/stripe/config/catalog.json +128 -0
- data/lib/multiwoven/integrations/destination/stripe/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/stripe/config/spec.json +17 -0
- data/lib/multiwoven/integrations/destination/stripe/icon.svg +10 -0
- data/lib/multiwoven/integrations/destination/tally/client.rb +151 -0
- data/lib/multiwoven/integrations/destination/tally/config/catalog.json +62 -0
- data/lib/multiwoven/integrations/destination/tally/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/tally/config/spec.json +45 -0
- data/lib/multiwoven/integrations/destination/tally/icon.svg +7 -0
- data/lib/multiwoven/integrations/protocol/protocol.json +189 -0
- data/lib/multiwoven/integrations/protocol/protocol.rb +216 -0
- data/lib/multiwoven/integrations/rollout.rb +32 -0
- data/lib/multiwoven/integrations/service.rb +79 -0
- data/lib/multiwoven/integrations/source/bigquery/client.rb +98 -0
- data/lib/multiwoven/integrations/source/bigquery/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/bigquery/config/spec.json +82 -0
- data/lib/multiwoven/integrations/source/bigquery/icon.svg +1 -0
- data/lib/multiwoven/integrations/source/databricks/client.rb +98 -0
- data/lib/multiwoven/integrations/source/databricks/config/meta.json +16 -0
- data/lib/multiwoven/integrations/source/databricks/config/spec.json +56 -0
- data/lib/multiwoven/integrations/source/databricks/icon.svg +19 -0
- data/lib/multiwoven/integrations/source/postgresql/client.rb +109 -0
- data/lib/multiwoven/integrations/source/postgresql/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/postgresql/config/spec.json +69 -0
- data/lib/multiwoven/integrations/source/postgresql/icon.svg +20 -0
- data/lib/multiwoven/integrations/source/redshift/client.rb +109 -0
- data/lib/multiwoven/integrations/source/redshift/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/redshift/config/spec.json +71 -0
- data/lib/multiwoven/integrations/source/redshift/icon.svg +15 -0
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb +123 -0
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/meta.json +17 -0
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json +50 -0
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/icon.svg +16 -0
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb +130 -0
- data/lib/multiwoven/integrations/source/snowflake/client.rb +92 -0
- data/lib/multiwoven/integrations/source/snowflake/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/snowflake/config/spec.json +82 -0
- data/lib/multiwoven/integrations/source/snowflake/icon.svg +10 -0
- data/lib/multiwoven/integrations/source/zoho_books/client.rb +155 -0
- data/lib/multiwoven/integrations/source/zoho_books/config/meta.json +16 -0
- data/lib/multiwoven/integrations/source/zoho_books/config/spec.json +43 -0
- data/lib/multiwoven/integrations/source/zoho_books/icon.svg +16 -0
- data/lib/multiwoven/integrations.rb +71 -0
- data/multiwoven-integrations-0.1.68.gem +0 -0
- data/sig/multiwoven/integrations.rbs +6 -0
- metadata +515 -0
@@ -0,0 +1,104 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Multiwoven
|
4
|
+
module Integrations::Core
|
5
|
+
module Utils
|
6
|
+
def keys_to_symbols(hash)
|
7
|
+
if hash.is_a?(Hash)
|
8
|
+
hash.each_with_object({}) do |(key, value), result|
|
9
|
+
result[key.to_sym] = keys_to_symbols(value)
|
10
|
+
end
|
11
|
+
elsif hash.is_a?(Array)
|
12
|
+
hash.map { |item| keys_to_symbols(item) }
|
13
|
+
else
|
14
|
+
hash
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def convert_to_json_schema(column_definitions)
|
19
|
+
json_schema = {
|
20
|
+
"type" => "object",
|
21
|
+
"properties" => {}
|
22
|
+
}
|
23
|
+
|
24
|
+
column_definitions.each do |column|
|
25
|
+
column_name = column[:column_name]
|
26
|
+
type = column[:type]
|
27
|
+
optional = column[:optional]
|
28
|
+
json_type = map_type_to_json_schema(type)
|
29
|
+
json_schema["properties"][column_name] = {
|
30
|
+
"type" => json_type
|
31
|
+
}
|
32
|
+
json_schema["properties"][column_name]["type"] = [json_type, "null"] if optional
|
33
|
+
end
|
34
|
+
|
35
|
+
json_schema
|
36
|
+
end
|
37
|
+
|
38
|
+
def map_type_to_json_schema(type)
|
39
|
+
case type
|
40
|
+
when "NUMBER"
|
41
|
+
"integer"
|
42
|
+
else
|
43
|
+
"string" # Default type
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def logger
|
48
|
+
Integrations::Service.logger
|
49
|
+
end
|
50
|
+
|
51
|
+
def create_log_message(context, type, exception)
|
52
|
+
Integrations::Protocol::LogMessage.new(
|
53
|
+
name: context,
|
54
|
+
level: type,
|
55
|
+
message: exception.message
|
56
|
+
).to_multiwoven_message
|
57
|
+
end
|
58
|
+
|
59
|
+
def handle_exception(context, type, exception)
|
60
|
+
logger.error(
|
61
|
+
"#{context}: #{exception.message}"
|
62
|
+
)
|
63
|
+
|
64
|
+
create_log_message(context, type, exception)
|
65
|
+
end
|
66
|
+
|
67
|
+
def extract_data(record_object, properties)
|
68
|
+
data_attributes = record_object.with_indifferent_access
|
69
|
+
data_attributes.select { |key, _| properties.key?(key.to_sym) }
|
70
|
+
end
|
71
|
+
|
72
|
+
def success?(response)
|
73
|
+
response && %w[200 201].include?(response.code.to_s)
|
74
|
+
end
|
75
|
+
|
76
|
+
def build_catalog(catalog_json)
|
77
|
+
streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
|
78
|
+
Multiwoven::Integrations::Protocol::Catalog.new(
|
79
|
+
streams: streams,
|
80
|
+
request_rate_limit: catalog_json["request_rate_limit"] || 60,
|
81
|
+
request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
|
82
|
+
request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10,
|
83
|
+
schema_mode: catalog_json["schema_mode"] || "schema"
|
84
|
+
)
|
85
|
+
end
|
86
|
+
|
87
|
+
def build_stream(stream_json)
|
88
|
+
Multiwoven::Integrations::Protocol::Stream.new(
|
89
|
+
name: stream_json["name"],
|
90
|
+
url: stream_json["url"],
|
91
|
+
action: stream_json["action"],
|
92
|
+
request_method: stream_json["method"],
|
93
|
+
batch_support: stream_json["batch_support"] || false,
|
94
|
+
batch_size: stream_json["batch_size"] || 1,
|
95
|
+
json_schema: stream_json["json_schema"],
|
96
|
+
request_rate_limit: stream_json["request_rate_limit"].to_i,
|
97
|
+
request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
|
98
|
+
request_rate_concurrency: stream_json["request_rate_concurrency"].to_i,
|
99
|
+
supported_sync_modes: stream_json["supported_sync_modes"]
|
100
|
+
)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,153 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "schema_helper"
|
4
|
+
module Multiwoven
|
5
|
+
module Integrations
|
6
|
+
module Destination
|
7
|
+
module Airtable
|
8
|
+
include Multiwoven::Integrations::Core
|
9
|
+
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
10
|
+
prepend Multiwoven::Integrations::Core::RateLimiter
|
11
|
+
MAX_CHUNK_SIZE = 10
|
12
|
+
def check_connection(connection_config)
|
13
|
+
connection_config = connection_config.with_indifferent_access
|
14
|
+
bases = Multiwoven::Integrations::Core::HttpClient.request(
|
15
|
+
AIRTABLE_BASES_ENDPOINT,
|
16
|
+
HTTP_GET,
|
17
|
+
headers: auth_headers(connection_config[:api_key])
|
18
|
+
)
|
19
|
+
if success?(bases)
|
20
|
+
base_id_exists?(bases, connection_config[:base_id])
|
21
|
+
success_status
|
22
|
+
else
|
23
|
+
failure_status(nil)
|
24
|
+
end
|
25
|
+
rescue StandardError => e
|
26
|
+
failure_status(e)
|
27
|
+
end
|
28
|
+
|
29
|
+
def discover(connection_config)
|
30
|
+
connection_config = connection_config.with_indifferent_access
|
31
|
+
base_id = connection_config[:base_id]
|
32
|
+
api_key = connection_config[:api_key]
|
33
|
+
|
34
|
+
bases = Multiwoven::Integrations::Core::HttpClient.request(
|
35
|
+
AIRTABLE_BASES_ENDPOINT,
|
36
|
+
HTTP_GET,
|
37
|
+
headers: auth_headers(api_key)
|
38
|
+
)
|
39
|
+
|
40
|
+
base = extract_bases(bases).find { |b| b["id"] == base_id }
|
41
|
+
base_name = base["name"]
|
42
|
+
|
43
|
+
schema = Multiwoven::Integrations::Core::HttpClient.request(
|
44
|
+
AIRTABLE_GET_BASE_SCHEMA_ENDPOINT.gsub("{baseId}", base_id),
|
45
|
+
HTTP_GET,
|
46
|
+
headers: auth_headers(api_key)
|
47
|
+
)
|
48
|
+
|
49
|
+
catalog = build_catalog_from_schema(extract_body(schema), base_id, base_name)
|
50
|
+
catalog.to_multiwoven_message
|
51
|
+
rescue StandardError => e
|
52
|
+
handle_exception("AIRTABLE:DISCOVER:EXCEPTION", "error", e)
|
53
|
+
end
|
54
|
+
|
55
|
+
def write(sync_config, records, _action = "create")
|
56
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
57
|
+
api_key = connection_config[:api_key]
|
58
|
+
url = sync_config.stream.url
|
59
|
+
write_success = 0
|
60
|
+
write_failure = 0
|
61
|
+
records.each_slice(MAX_CHUNK_SIZE) do |chunk|
|
62
|
+
payload = create_payload(chunk)
|
63
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
64
|
+
url,
|
65
|
+
sync_config.stream.request_method,
|
66
|
+
payload: payload,
|
67
|
+
headers: auth_headers(api_key)
|
68
|
+
)
|
69
|
+
if success?(response)
|
70
|
+
write_success += chunk.size
|
71
|
+
else
|
72
|
+
write_failure += chunk.size
|
73
|
+
end
|
74
|
+
rescue StandardError => e
|
75
|
+
handle_exception("AIRTABLE:RECORD:WRITE:EXCEPTION", "error", e)
|
76
|
+
write_failure += chunk.size
|
77
|
+
end
|
78
|
+
|
79
|
+
tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
80
|
+
success: write_success,
|
81
|
+
failed: write_failure
|
82
|
+
)
|
83
|
+
tracker.to_multiwoven_message
|
84
|
+
rescue StandardError => e
|
85
|
+
handle_exception("AIRTABLE:WRITE:EXCEPTION", "error", e)
|
86
|
+
end
|
87
|
+
|
88
|
+
private
|
89
|
+
|
90
|
+
def create_payload(records)
|
91
|
+
{
|
92
|
+
"records" => records.map do |record|
|
93
|
+
{
|
94
|
+
"fields" => record
|
95
|
+
}
|
96
|
+
end
|
97
|
+
}
|
98
|
+
end
|
99
|
+
|
100
|
+
def auth_headers(access_token)
|
101
|
+
{
|
102
|
+
"Accept" => "application/json",
|
103
|
+
"Authorization" => "Bearer #{access_token}",
|
104
|
+
"Content-Type" => "application/json"
|
105
|
+
}
|
106
|
+
end
|
107
|
+
|
108
|
+
def base_id_exists?(bases, base_id)
|
109
|
+
return if extract_data(bases).any? { |base| base["id"] == base_id }
|
110
|
+
|
111
|
+
raise ArgumentError, "base_id not found"
|
112
|
+
end
|
113
|
+
|
114
|
+
def extract_bases(response)
|
115
|
+
response_body = extract_body(response)
|
116
|
+
response_body["bases"] if response_body
|
117
|
+
end
|
118
|
+
|
119
|
+
def extract_body(response)
|
120
|
+
response_body = response.body
|
121
|
+
JSON.parse(response_body) if response_body
|
122
|
+
end
|
123
|
+
|
124
|
+
def load_catalog
|
125
|
+
read_json(CATALOG_SPEC_PATH)
|
126
|
+
end
|
127
|
+
|
128
|
+
def create_stream(table, base_id, base_name)
|
129
|
+
{
|
130
|
+
name: "#{base_name}/#{SchemaHelper.clean_name(table["name"])}",
|
131
|
+
action: "create",
|
132
|
+
method: HTTP_POST,
|
133
|
+
url: "#{AIRTABLE_URL_BASE}#{base_id}/#{table["id"]}",
|
134
|
+
json_schema: SchemaHelper.get_json_schema(table),
|
135
|
+
supported_sync_modes: %w[incremental],
|
136
|
+
batch_support: true,
|
137
|
+
batch_size: 10
|
138
|
+
|
139
|
+
}.with_indifferent_access
|
140
|
+
end
|
141
|
+
|
142
|
+
def build_catalog_from_schema(schema, base_id, base_name)
|
143
|
+
catalog = build_catalog(load_catalog)
|
144
|
+
schema["tables"].each do |table|
|
145
|
+
catalog.streams << build_stream(create_stream(table, base_id, base_name))
|
146
|
+
end
|
147
|
+
catalog
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
{
|
2
|
+
"data": {
|
3
|
+
"name": "Airtable",
|
4
|
+
"title": "airtable",
|
5
|
+
"connector_type": "destination",
|
6
|
+
"category": "Productivity Tools",
|
7
|
+
"documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/airtable",
|
8
|
+
"github_issue_label": "destination-airtable",
|
9
|
+
"icon": "icon.svg",
|
10
|
+
"license": "MIT",
|
11
|
+
"release_stage": "alpha",
|
12
|
+
"support_level": "community",
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
14
|
+
}
|
15
|
+
}
|
@@ -0,0 +1,22 @@
|
|
1
|
+
{
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/airtable",
|
3
|
+
"stream_type": "dynamic",
|
4
|
+
"connection_specification": {
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
6
|
+
"title": "Airtable",
|
7
|
+
"type": "object",
|
8
|
+
"required": ["api_key", "base_id"],
|
9
|
+
"properties": {
|
10
|
+
"api_key": {
|
11
|
+
"type": "string",
|
12
|
+
"title": "Personal access token",
|
13
|
+
"order": 0
|
14
|
+
},
|
15
|
+
"base_id": {
|
16
|
+
"type": "string",
|
17
|
+
"title": "Airtable base id",
|
18
|
+
"order": 1
|
19
|
+
}
|
20
|
+
}
|
21
|
+
}
|
22
|
+
}
|
@@ -0,0 +1,6 @@
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="27" shape-rendering="geometricPrecision" viewBox="0 0 200 170">
|
2
|
+
<path fill="#FCB400" d="M90.039 12.367L24.079 39.66c-3.667 1.519-3.63 6.729.062 8.192l66.235 26.266a24.575 24.575 0 0018.12 0l66.236-26.266c3.69-1.463 3.729-6.673.06-8.191l-65.958-27.294a24.578 24.578 0 00-18.795 0"></path>
|
3
|
+
<path fill="#18BFFF" d="M105.312 88.46v65.617c0 3.12 3.147 5.258 6.048 4.108l73.806-28.648a4.418 4.418 0 002.79-4.108V59.813c0-3.121-3.147-5.258-6.048-4.108l-73.806 28.648a4.42 4.42 0 00-2.79 4.108"></path>
|
4
|
+
<path fill="#F82B60" d="M88.078 91.846l-21.904 10.576-2.224 1.075-46.238 22.155c-2.93 1.414-6.672-.722-6.672-3.978V60.088c0-1.178.604-2.195 1.414-2.96a5.024 5.024 0 011.12-.84c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
|
5
|
+
<path fill="rgba(0, 0, 0, 0.25)" d="M88.078 91.846l-21.904 10.576-53.72-45.295a5.024 5.024 0 011.12-.839c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
|
6
|
+
</svg>
|
@@ -0,0 +1,141 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Multiwoven
|
4
|
+
module Integrations
|
5
|
+
module Destination
|
6
|
+
module Airtable
|
7
|
+
module SchemaHelper
|
8
|
+
include Core::Constants
|
9
|
+
|
10
|
+
module_function
|
11
|
+
|
12
|
+
def clean_name(name_str)
|
13
|
+
name_str.strip.gsub(" ", "_")
|
14
|
+
end
|
15
|
+
|
16
|
+
def get_json_schema(table)
|
17
|
+
fields = table["fields"] || {}
|
18
|
+
properties = fields.each_with_object({}) do |field, props|
|
19
|
+
name, schema = process_field(field)
|
20
|
+
props[name] = schema
|
21
|
+
end
|
22
|
+
|
23
|
+
build_schema(properties)
|
24
|
+
end
|
25
|
+
|
26
|
+
def process_field(field)
|
27
|
+
name = clean_name(field.fetch("name", ""))
|
28
|
+
original_type = field.fetch("type", "")
|
29
|
+
options = field.fetch("options", {})
|
30
|
+
|
31
|
+
schema = determine_schema(original_type, options)
|
32
|
+
[name, schema]
|
33
|
+
end
|
34
|
+
|
35
|
+
def determine_schema(original_type, options)
|
36
|
+
if COMPLEX_AIRTABLE_TYPES.keys.include?(original_type)
|
37
|
+
complex_type = deep_copy(COMPLEX_AIRTABLE_TYPES[original_type])
|
38
|
+
adjust_complex_type(original_type, complex_type, options)
|
39
|
+
elsif SIMPLE_AIRTABLE_TYPES.keys.include?(original_type)
|
40
|
+
simple_type_schema(original_type, options)
|
41
|
+
else
|
42
|
+
SCHEMA_TYPES[:STRING]
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def adjust_complex_type(original_type, complex_type, options)
|
47
|
+
exec_type = options.dig("result", "type") || "simpleText"
|
48
|
+
if complex_type == SCHEMA_TYPES[:ARRAY_WITH_ANY]
|
49
|
+
adjust_array_with_any(original_type, complex_type, exec_type, options)
|
50
|
+
else
|
51
|
+
complex_type
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def adjust_array_with_any(original_type, complex_type, exec_type, options)
|
56
|
+
if original_type == "formula" && %w[number currency percent duration].include?(exec_type)
|
57
|
+
complex_type = SCHEMA_TYPES[:NUMBER]
|
58
|
+
elsif original_type == "formula" && ARRAY_FORMULAS.none? { |x| options.fetch("formula", "").start_with?(x) }
|
59
|
+
complex_type = SCHEMA_TYPES[:STRING]
|
60
|
+
elsif SIMPLE_AIRTABLE_TYPES.keys.include?(exec_type)
|
61
|
+
complex_type["items"] = deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
|
62
|
+
else
|
63
|
+
complex_type["items"] = SCHEMA_TYPES[:STRING]
|
64
|
+
end
|
65
|
+
complex_type
|
66
|
+
end
|
67
|
+
|
68
|
+
def simple_type_schema(original_type, options)
|
69
|
+
exec_type = options.dig("result", "type") || original_type
|
70
|
+
deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
|
71
|
+
end
|
72
|
+
|
73
|
+
def build_schema(properties)
|
74
|
+
{
|
75
|
+
"$schema" => JSON_SCHEMA_URL,
|
76
|
+
"type" => "object",
|
77
|
+
"additionalProperties" => true,
|
78
|
+
"properties" => properties
|
79
|
+
}
|
80
|
+
end
|
81
|
+
|
82
|
+
def deep_copy(object)
|
83
|
+
Marshal.load(Marshal.dump(object))
|
84
|
+
end
|
85
|
+
|
86
|
+
SCHEMA_TYPES = {
|
87
|
+
STRING: { "type": %w[null string] },
|
88
|
+
NUMBER: { "type": %w[null number] },
|
89
|
+
BOOLEAN: { "type": %w[null boolean] },
|
90
|
+
DATE: { "type": %w[null string], "format": "date" },
|
91
|
+
DATETIME: { "type": %w[null string], "format": "date-time" },
|
92
|
+
ARRAY_WITH_STRINGS: { "type": %w[null array], "items": { "type": %w[null string] } },
|
93
|
+
ARRAY_WITH_ANY: { "type": %w[null array], "items": {} }
|
94
|
+
}.freeze.with_indifferent_access
|
95
|
+
|
96
|
+
SIMPLE_AIRTABLE_TYPES = {
|
97
|
+
"multipleAttachments" => SCHEMA_TYPES[:STRING],
|
98
|
+
"autoNumber" => SCHEMA_TYPES[:NUMBER],
|
99
|
+
"barcode" => SCHEMA_TYPES[:STRING],
|
100
|
+
"button" => SCHEMA_TYPES[:STRING],
|
101
|
+
"checkbox" => :BOOLEAN,
|
102
|
+
"singleCollaborator" => SCHEMA_TYPES[:STRING],
|
103
|
+
"count" => SCHEMA_TYPES[:NUMBER],
|
104
|
+
"createdBy" => SCHEMA_TYPES[:STRING],
|
105
|
+
"createdTime" => SCHEMA_TYPES[:DATETIME],
|
106
|
+
"currency" => SCHEMA_TYPES[:NUMBER],
|
107
|
+
"email" => SCHEMA_TYPES[:STRING],
|
108
|
+
"date" => SCHEMA_TYPES[:DATE],
|
109
|
+
"dateTime" => SCHEMA_TYPES[:DATETIME],
|
110
|
+
"duration" => SCHEMA_TYPES[:NUMBER],
|
111
|
+
"lastModifiedBy" => SCHEMA_TYPES[:STRING],
|
112
|
+
"lastModifiedTime" => SCHEMA_TYPES[:DATETIME],
|
113
|
+
"multipleRecordLinks" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
|
114
|
+
"multilineText" => SCHEMA_TYPES[:STRING],
|
115
|
+
"multipleCollaborators" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
|
116
|
+
"multipleSelects" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
|
117
|
+
"number" => SCHEMA_TYPES[:NUMBER],
|
118
|
+
"percent" => SCHEMA_TYPES[:NUMBER],
|
119
|
+
"phoneNumber" => SCHEMA_TYPES[:STRING],
|
120
|
+
"rating" => SCHEMA_TYPES[:NUMBER],
|
121
|
+
"richText" => SCHEMA_TYPES[:STRING],
|
122
|
+
"singleLineText" => SCHEMA_TYPES[:STRING],
|
123
|
+
"singleSelect" => SCHEMA_TYPES[:STRING],
|
124
|
+
"externalSyncSource" => SCHEMA_TYPES[:STRING],
|
125
|
+
"url" => SCHEMA_TYPES[:STRING],
|
126
|
+
"simpleText" => SCHEMA_TYPES[:STRING]
|
127
|
+
}.freeze
|
128
|
+
|
129
|
+
COMPLEX_AIRTABLE_TYPES = {
|
130
|
+
"formula" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
|
131
|
+
"lookup" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
|
132
|
+
"multipleLookupValues" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
|
133
|
+
"rollup" => SCHEMA_TYPES[:ARRAY_WITH_ANY]
|
134
|
+
}.freeze.with_indifferent_access
|
135
|
+
|
136
|
+
ARRAY_FORMULAS = %w[ARRAYCOMPACT ARRAYFLATTEN ARRAYUNIQUE ARRAYSLICE].freeze
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Multiwoven::Integrations::Destination
|
4
|
+
module FacebookCustomAudience
|
5
|
+
include Multiwoven::Integrations::Core
|
6
|
+
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
7
|
+
prepend Multiwoven::Integrations::Core::RateLimiter
|
8
|
+
MAX_CHUNK_SIZE = 10_000
|
9
|
+
def check_connection(connection_config)
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
11
|
+
access_token = connection_config[:access_token]
|
12
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
13
|
+
FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS,
|
14
|
+
HTTP_GET,
|
15
|
+
headers: auth_headers(access_token)
|
16
|
+
)
|
17
|
+
if success?(response)
|
18
|
+
ad_account_exists?(response, connection_config[:ad_account_id])
|
19
|
+
ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
|
20
|
+
else
|
21
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"]).to_multiwoven_message
|
22
|
+
end
|
23
|
+
rescue StandardError => e
|
24
|
+
ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
|
25
|
+
end
|
26
|
+
|
27
|
+
def discover(_connection_config = nil)
|
28
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
29
|
+
catalog = build_catalog(catalog_json)
|
30
|
+
catalog.to_multiwoven_message
|
31
|
+
rescue StandardError => e
|
32
|
+
handle_exception(
|
33
|
+
"FACEBOOK AUDIENCE:DISCOVER:EXCEPTION",
|
34
|
+
"error",
|
35
|
+
e
|
36
|
+
)
|
37
|
+
end
|
38
|
+
|
39
|
+
def write(sync_config, records, _action = "insert")
|
40
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
41
|
+
access_token = connection_config[:access_token]
|
42
|
+
url = generate_url(sync_config, connection_config)
|
43
|
+
write_success = 0
|
44
|
+
write_failure = 0
|
45
|
+
records.each_slice(MAX_CHUNK_SIZE) do |chunk|
|
46
|
+
payload = create_payload(chunk, sync_config.stream.json_schema.with_indifferent_access)
|
47
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
48
|
+
url,
|
49
|
+
sync_config.stream.request_method,
|
50
|
+
payload: payload,
|
51
|
+
headers: auth_headers(access_token)
|
52
|
+
)
|
53
|
+
if success?(response)
|
54
|
+
write_success += chunk.size
|
55
|
+
else
|
56
|
+
write_failure += chunk.size
|
57
|
+
end
|
58
|
+
rescue StandardError => e
|
59
|
+
handle_exception("FACEBOOK:RECORD:WRITE:EXCEPTION", "error", e)
|
60
|
+
write_failure += chunk.size
|
61
|
+
end
|
62
|
+
|
63
|
+
tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
64
|
+
success: write_success,
|
65
|
+
failed: write_failure
|
66
|
+
)
|
67
|
+
tracker.to_multiwoven_message
|
68
|
+
rescue StandardError => e
|
69
|
+
handle_exception("FACEBOOK:WRITE:EXCEPTION", "error", e)
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
def generate_url(sync_config, connection_config)
|
75
|
+
sync_config.stream.url.gsub("{audience_id}", connection_config[:audience_id])
|
76
|
+
end
|
77
|
+
|
78
|
+
def create_payload(records, json_schema)
|
79
|
+
schema, data = extract_schema_and_data(records, json_schema)
|
80
|
+
{
|
81
|
+
"payload" => {
|
82
|
+
"schema" => schema,
|
83
|
+
"data" => data
|
84
|
+
}
|
85
|
+
}
|
86
|
+
end
|
87
|
+
|
88
|
+
def extract_schema_and_data(records, json_schema)
|
89
|
+
schema_properties = json_schema[:properties]
|
90
|
+
schema = records.first.keys.map(&:to_s).map(&:upcase)
|
91
|
+
data = []
|
92
|
+
records.each do |record|
|
93
|
+
encrypted_data_array = []
|
94
|
+
record.with_indifferent_access.each do |key, value|
|
95
|
+
schema_key = key.upcase
|
96
|
+
encrypted_value = schema_properties[schema_key] && schema_properties[schema_key]["x-hashRequired"] ? Digest::SHA256.hexdigest(value.to_s) : value
|
97
|
+
encrypted_data_array << encrypted_value
|
98
|
+
end
|
99
|
+
data << encrypted_data_array
|
100
|
+
end
|
101
|
+
[schema, data]
|
102
|
+
end
|
103
|
+
|
104
|
+
def auth_headers(access_token)
|
105
|
+
{
|
106
|
+
"Accept" => "application/json",
|
107
|
+
"Authorization" => "Bearer #{access_token}",
|
108
|
+
"Content-Type" => "application/json"
|
109
|
+
}
|
110
|
+
end
|
111
|
+
|
112
|
+
def ad_account_exists?(response, ad_account_id)
|
113
|
+
return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" }
|
114
|
+
|
115
|
+
raise ArgumentError, "Ad account not found in business account"
|
116
|
+
end
|
117
|
+
|
118
|
+
def extract_data(response)
|
119
|
+
response_body = response.body
|
120
|
+
JSON.parse(response_body)["data"] if response_body
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
{
|
2
|
+
"request_rate_limit": 600,
|
3
|
+
"request_rate_limit_unit": "minute",
|
4
|
+
"request_rate_concurrency": 10,
|
5
|
+
"streams": [
|
6
|
+
{
|
7
|
+
"name": "audience",
|
8
|
+
"action": "create",
|
9
|
+
"method": "POST",
|
10
|
+
"batch_support": true,
|
11
|
+
"batch_size": 10000,
|
12
|
+
"url": "https://graph.facebook.com/v18.0/{audience_id}/users",
|
13
|
+
"json_schema": {
|
14
|
+
"type": "object",
|
15
|
+
"additionalProperties": true,
|
16
|
+
"properties": {
|
17
|
+
"EMAIL": { "type": ["string", "null"], "default": null, "title": "Email", "x-hashRequired": true },
|
18
|
+
"PHONE": { "type": ["string", "null"], "default": null, "title": "Phone", "x-hashRequired": true },
|
19
|
+
"GEN": { "type": ["string", "null"], "default": null, "title": "Gender", "x-hashRequired": true },
|
20
|
+
"FI": { "type": ["string", "null"], "default": null, "title": "First Initial", "x-hashRequired": true },
|
21
|
+
"FN": { "type": ["string", "null"], "default": null, "title": "First Name", "x-hashRequired": true },
|
22
|
+
"DOBD": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Day)", "x-hashRequired": true },
|
23
|
+
"DOBM": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Month)", "x-hashRequired": true },
|
24
|
+
"DOBY": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Year)", "x-hashRequired": true },
|
25
|
+
"LN": { "type": ["string", "null"], "default": null, "title": "Last Name", "x-hashRequired": true },
|
26
|
+
"CT": { "type": ["string", "null"], "default": null, "title": "City", "x-hashRequired": true },
|
27
|
+
"ST": { "type": ["string", "null"], "default": null, "title": "State", "x-hashRequired": true },
|
28
|
+
"ZIP": { "type": ["string", "null"], "default": null, "title": "Zip Code", "x-hashRequired": true },
|
29
|
+
"COUNTRY": { "type": ["string", "null"], "default": null, "title": "Country", "x-hashRequired": true },
|
30
|
+
"EXTERN_ID": { "type": ["string", "null"], "default": null, "title": "External ID", "x-hashRequired": false},
|
31
|
+
"LOOKALIKE_VALUE": { "type": ["number", "null"], "default": null, "title": "Lookalike Value", "x-hashRequired": false },
|
32
|
+
"MADID": { "type": ["string", "null"], "default": null, "title": "Mobile ID", "x-hashRequired": false },
|
33
|
+
"PAGEUID": { "type": ["string", "null"], "default": null, "title": "Page-Scoped ID", "x-hashRequired": false }
|
34
|
+
}
|
35
|
+
},
|
36
|
+
"supported_sync_modes": ["incremental"],
|
37
|
+
"source_defined_cursor": true,
|
38
|
+
"default_cursor_field": ["updated"],
|
39
|
+
"source_defined_primary_key": [["email"]]
|
40
|
+
}
|
41
|
+
]
|
42
|
+
}
|
@@ -0,0 +1,15 @@
|
|
1
|
+
{
|
2
|
+
"data": {
|
3
|
+
"name": "FacebookCustomAudience",
|
4
|
+
"title": "Facebook Custom Audiences",
|
5
|
+
"connector_type": "destination",
|
6
|
+
"category": "Adtech",
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
8
|
+
"github_issue_label": "destination-facebook",
|
9
|
+
"icon": "icon.svg",
|
10
|
+
"license": "MIT",
|
11
|
+
"release_stage": "alpha",
|
12
|
+
"support_level": "community",
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
14
|
+
}
|
15
|
+
}
|
@@ -0,0 +1,27 @@
|
|
1
|
+
{
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/facebook",
|
3
|
+
"stream_type": "static",
|
4
|
+
"connection_specification": {
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
6
|
+
"title": "Facebook Custom Audiences",
|
7
|
+
"type": "object",
|
8
|
+
"required": ["access_token", "ad_account_id", "audience_id"],
|
9
|
+
"properties": {
|
10
|
+
"access_token": {
|
11
|
+
"type": "string",
|
12
|
+
"title": "Access Token",
|
13
|
+
"order": 0
|
14
|
+
},
|
15
|
+
"ad_account_id": {
|
16
|
+
"type": "string",
|
17
|
+
"title": "Ad Account Id",
|
18
|
+
"order": 1
|
19
|
+
},
|
20
|
+
"audience_id": {
|
21
|
+
"type": "string",
|
22
|
+
"title": "Audience Id",
|
23
|
+
"order": 2
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
27
|
+
}
|