etl-integrations 0.1.81

Sign up to get free protection for your applications and to get access to all the features.
Files changed (119) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/.rubocop.yml +34 -0
  4. data/.ruby-version +1 -0
  5. data/.vscode/settings.json +5 -0
  6. data/CHANGELOG.md +38 -0
  7. data/CODE_OF_CONDUCT.md +84 -0
  8. data/LICENSE.txt +21 -0
  9. data/README.md +105 -0
  10. data/Rakefile +12 -0
  11. data/lib/multiwoven/integrations/config.rb +13 -0
  12. data/lib/multiwoven/integrations/core/base_connector.rb +70 -0
  13. data/lib/multiwoven/integrations/core/constants.rb +46 -0
  14. data/lib/multiwoven/integrations/core/destination_connector.rb +14 -0
  15. data/lib/multiwoven/integrations/core/fullrefresher.rb +19 -0
  16. data/lib/multiwoven/integrations/core/http_client.rb +34 -0
  17. data/lib/multiwoven/integrations/core/query_builder.rb +27 -0
  18. data/lib/multiwoven/integrations/core/rate_limiter.rb +19 -0
  19. data/lib/multiwoven/integrations/core/source_connector.rb +38 -0
  20. data/lib/multiwoven/integrations/core/utils.rb +104 -0
  21. data/lib/multiwoven/integrations/destination/airtable/client.rb +153 -0
  22. data/lib/multiwoven/integrations/destination/airtable/config/catalog.json +6 -0
  23. data/lib/multiwoven/integrations/destination/airtable/config/meta.json +15 -0
  24. data/lib/multiwoven/integrations/destination/airtable/config/spec.json +22 -0
  25. data/lib/multiwoven/integrations/destination/airtable/icon.svg +6 -0
  26. data/lib/multiwoven/integrations/destination/airtable/schema_helper.rb +141 -0
  27. data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +124 -0
  28. data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json +42 -0
  29. data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json +15 -0
  30. data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json +27 -0
  31. data/lib/multiwoven/integrations/destination/facebook_custom_audience/icon.svg +23 -0
  32. data/lib/multiwoven/integrations/destination/google_sheets/client.rb +231 -0
  33. data/lib/multiwoven/integrations/destination/google_sheets/config/catalog.json +6 -0
  34. data/lib/multiwoven/integrations/destination/google_sheets/config/meta.json +15 -0
  35. data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json +74 -0
  36. data/lib/multiwoven/integrations/destination/google_sheets/icon.svg +1 -0
  37. data/lib/multiwoven/integrations/destination/hubspot/client.rb +107 -0
  38. data/lib/multiwoven/integrations/destination/hubspot/config/catalog.json +351 -0
  39. data/lib/multiwoven/integrations/destination/hubspot/config/meta.json +15 -0
  40. data/lib/multiwoven/integrations/destination/hubspot/config/spec.json +17 -0
  41. data/lib/multiwoven/integrations/destination/hubspot/icon.svg +5 -0
  42. data/lib/multiwoven/integrations/destination/klaviyo/client.rb +116 -0
  43. data/lib/multiwoven/integrations/destination/klaviyo/config/catalog.json +103 -0
  44. data/lib/multiwoven/integrations/destination/klaviyo/config/meta.json +15 -0
  45. data/lib/multiwoven/integrations/destination/klaviyo/config/spec.json +22 -0
  46. data/lib/multiwoven/integrations/destination/klaviyo/icon.svg +6 -0
  47. data/lib/multiwoven/integrations/destination/postgresql/client.rb +123 -0
  48. data/lib/multiwoven/integrations/destination/postgresql/config/meta.json +15 -0
  49. data/lib/multiwoven/integrations/destination/postgresql/config/spec.json +68 -0
  50. data/lib/multiwoven/integrations/destination/postgresql/icon.svg +20 -0
  51. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb +114 -0
  52. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json +6 -0
  53. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json +16 -0
  54. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/spec.json +49 -0
  55. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/icon.svg +16 -0
  56. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb +132 -0
  57. data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb +117 -0
  58. data/lib/multiwoven/integrations/destination/salesforce_crm/config/catalog.json +320 -0
  59. data/lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json +15 -0
  60. data/lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json +43 -0
  61. data/lib/multiwoven/integrations/destination/salesforce_crm/icon.svg +16 -0
  62. data/lib/multiwoven/integrations/destination/sftp/client.rb +133 -0
  63. data/lib/multiwoven/integrations/destination/sftp/config/catalog.json +16 -0
  64. data/lib/multiwoven/integrations/destination/sftp/config/meta.json +16 -0
  65. data/lib/multiwoven/integrations/destination/sftp/config/spec.json +50 -0
  66. data/lib/multiwoven/integrations/destination/sftp/icon.svg +1 -0
  67. data/lib/multiwoven/integrations/destination/slack/client.rb +114 -0
  68. data/lib/multiwoven/integrations/destination/slack/config/catalog.json +22 -0
  69. data/lib/multiwoven/integrations/destination/slack/config/meta.json +15 -0
  70. data/lib/multiwoven/integrations/destination/slack/config/spec.json +22 -0
  71. data/lib/multiwoven/integrations/destination/slack/icon.svg +26 -0
  72. data/lib/multiwoven/integrations/destination/stripe/client.rb +82 -0
  73. data/lib/multiwoven/integrations/destination/stripe/config/catalog.json +128 -0
  74. data/lib/multiwoven/integrations/destination/stripe/config/meta.json +15 -0
  75. data/lib/multiwoven/integrations/destination/stripe/config/spec.json +17 -0
  76. data/lib/multiwoven/integrations/destination/stripe/icon.svg +10 -0
  77. data/lib/multiwoven/integrations/destination/tally/client.rb +151 -0
  78. data/lib/multiwoven/integrations/destination/tally/config/catalog.json +62 -0
  79. data/lib/multiwoven/integrations/destination/tally/config/meta.json +15 -0
  80. data/lib/multiwoven/integrations/destination/tally/config/spec.json +45 -0
  81. data/lib/multiwoven/integrations/destination/tally/icon.svg +7 -0
  82. data/lib/multiwoven/integrations/protocol/protocol.json +189 -0
  83. data/lib/multiwoven/integrations/protocol/protocol.rb +216 -0
  84. data/lib/multiwoven/integrations/rollout.rb +32 -0
  85. data/lib/multiwoven/integrations/service.rb +79 -0
  86. data/lib/multiwoven/integrations/source/bigquery/client.rb +98 -0
  87. data/lib/multiwoven/integrations/source/bigquery/config/meta.json +15 -0
  88. data/lib/multiwoven/integrations/source/bigquery/config/spec.json +82 -0
  89. data/lib/multiwoven/integrations/source/bigquery/icon.svg +1 -0
  90. data/lib/multiwoven/integrations/source/databricks/client.rb +98 -0
  91. data/lib/multiwoven/integrations/source/databricks/config/meta.json +16 -0
  92. data/lib/multiwoven/integrations/source/databricks/config/spec.json +56 -0
  93. data/lib/multiwoven/integrations/source/databricks/icon.svg +19 -0
  94. data/lib/multiwoven/integrations/source/postgresql/client.rb +109 -0
  95. data/lib/multiwoven/integrations/source/postgresql/config/meta.json +15 -0
  96. data/lib/multiwoven/integrations/source/postgresql/config/spec.json +69 -0
  97. data/lib/multiwoven/integrations/source/postgresql/icon.svg +20 -0
  98. data/lib/multiwoven/integrations/source/redshift/client.rb +109 -0
  99. data/lib/multiwoven/integrations/source/redshift/config/meta.json +15 -0
  100. data/lib/multiwoven/integrations/source/redshift/config/spec.json +71 -0
  101. data/lib/multiwoven/integrations/source/redshift/icon.svg +15 -0
  102. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb +123 -0
  103. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/catalog.json +6 -0
  104. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/meta.json +17 -0
  105. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json +50 -0
  106. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/icon.svg +16 -0
  107. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb +130 -0
  108. data/lib/multiwoven/integrations/source/snowflake/client.rb +92 -0
  109. data/lib/multiwoven/integrations/source/snowflake/config/meta.json +15 -0
  110. data/lib/multiwoven/integrations/source/snowflake/config/spec.json +82 -0
  111. data/lib/multiwoven/integrations/source/snowflake/icon.svg +10 -0
  112. data/lib/multiwoven/integrations/source/zoho_books/client.rb +155 -0
  113. data/lib/multiwoven/integrations/source/zoho_books/config/meta.json +16 -0
  114. data/lib/multiwoven/integrations/source/zoho_books/config/spec.json +43 -0
  115. data/lib/multiwoven/integrations/source/zoho_books/icon.svg +16 -0
  116. data/lib/multiwoven/integrations.rb +71 -0
  117. data/multiwoven-integrations-0.1.68.gem +0 -0
  118. data/sig/multiwoven/integrations.rbs +6 -0
  119. metadata +515 -0
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven
4
+ module Integrations::Core
5
+ module Utils
6
+ def keys_to_symbols(hash)
7
+ if hash.is_a?(Hash)
8
+ hash.each_with_object({}) do |(key, value), result|
9
+ result[key.to_sym] = keys_to_symbols(value)
10
+ end
11
+ elsif hash.is_a?(Array)
12
+ hash.map { |item| keys_to_symbols(item) }
13
+ else
14
+ hash
15
+ end
16
+ end
17
+
18
+ def convert_to_json_schema(column_definitions)
19
+ json_schema = {
20
+ "type" => "object",
21
+ "properties" => {}
22
+ }
23
+
24
+ column_definitions.each do |column|
25
+ column_name = column[:column_name]
26
+ type = column[:type]
27
+ optional = column[:optional]
28
+ json_type = map_type_to_json_schema(type)
29
+ json_schema["properties"][column_name] = {
30
+ "type" => json_type
31
+ }
32
+ json_schema["properties"][column_name]["type"] = [json_type, "null"] if optional
33
+ end
34
+
35
+ json_schema
36
+ end
37
+
38
+ def map_type_to_json_schema(type)
39
+ case type
40
+ when "NUMBER"
41
+ "integer"
42
+ else
43
+ "string" # Default type
44
+ end
45
+ end
46
+
47
+ def logger
48
+ Integrations::Service.logger
49
+ end
50
+
51
+ def create_log_message(context, type, exception)
52
+ Integrations::Protocol::LogMessage.new(
53
+ name: context,
54
+ level: type,
55
+ message: exception.message
56
+ ).to_multiwoven_message
57
+ end
58
+
59
+ def handle_exception(context, type, exception)
60
+ logger.error(
61
+ "#{context}: #{exception.message}"
62
+ )
63
+
64
+ create_log_message(context, type, exception)
65
+ end
66
+
67
+ def extract_data(record_object, properties)
68
+ data_attributes = record_object.with_indifferent_access
69
+ data_attributes.select { |key, _| properties.key?(key.to_sym) }
70
+ end
71
+
72
+ def success?(response)
73
+ response && %w[200 201].include?(response.code.to_s)
74
+ end
75
+
76
+ def build_catalog(catalog_json)
77
+ streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
78
+ Multiwoven::Integrations::Protocol::Catalog.new(
79
+ streams: streams,
80
+ request_rate_limit: catalog_json["request_rate_limit"] || 60,
81
+ request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
82
+ request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10,
83
+ schema_mode: catalog_json["schema_mode"] || "schema"
84
+ )
85
+ end
86
+
87
+ def build_stream(stream_json)
88
+ Multiwoven::Integrations::Protocol::Stream.new(
89
+ name: stream_json["name"],
90
+ url: stream_json["url"],
91
+ action: stream_json["action"],
92
+ request_method: stream_json["method"],
93
+ batch_support: stream_json["batch_support"] || false,
94
+ batch_size: stream_json["batch_size"] || 1,
95
+ json_schema: stream_json["json_schema"],
96
+ request_rate_limit: stream_json["request_rate_limit"].to_i,
97
+ request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
98
+ request_rate_concurrency: stream_json["request_rate_concurrency"].to_i,
99
+ supported_sync_modes: stream_json["supported_sync_modes"]
100
+ )
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "schema_helper"
4
+ module Multiwoven
5
+ module Integrations
6
+ module Destination
7
+ module Airtable
8
+ include Multiwoven::Integrations::Core
9
+ class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
10
+ prepend Multiwoven::Integrations::Core::RateLimiter
11
+ MAX_CHUNK_SIZE = 10
12
+ def check_connection(connection_config)
13
+ connection_config = connection_config.with_indifferent_access
14
+ bases = Multiwoven::Integrations::Core::HttpClient.request(
15
+ AIRTABLE_BASES_ENDPOINT,
16
+ HTTP_GET,
17
+ headers: auth_headers(connection_config[:api_key])
18
+ )
19
+ if success?(bases)
20
+ base_id_exists?(bases, connection_config[:base_id])
21
+ success_status
22
+ else
23
+ failure_status(nil)
24
+ end
25
+ rescue StandardError => e
26
+ failure_status(e)
27
+ end
28
+
29
+ def discover(connection_config)
30
+ connection_config = connection_config.with_indifferent_access
31
+ base_id = connection_config[:base_id]
32
+ api_key = connection_config[:api_key]
33
+
34
+ bases = Multiwoven::Integrations::Core::HttpClient.request(
35
+ AIRTABLE_BASES_ENDPOINT,
36
+ HTTP_GET,
37
+ headers: auth_headers(api_key)
38
+ )
39
+
40
+ base = extract_bases(bases).find { |b| b["id"] == base_id }
41
+ base_name = base["name"]
42
+
43
+ schema = Multiwoven::Integrations::Core::HttpClient.request(
44
+ AIRTABLE_GET_BASE_SCHEMA_ENDPOINT.gsub("{baseId}", base_id),
45
+ HTTP_GET,
46
+ headers: auth_headers(api_key)
47
+ )
48
+
49
+ catalog = build_catalog_from_schema(extract_body(schema), base_id, base_name)
50
+ catalog.to_multiwoven_message
51
+ rescue StandardError => e
52
+ handle_exception("AIRTABLE:DISCOVER:EXCEPTION", "error", e)
53
+ end
54
+
55
+ def write(sync_config, records, _action = "create")
56
+ connection_config = sync_config.destination.connection_specification.with_indifferent_access
57
+ api_key = connection_config[:api_key]
58
+ url = sync_config.stream.url
59
+ write_success = 0
60
+ write_failure = 0
61
+ records.each_slice(MAX_CHUNK_SIZE) do |chunk|
62
+ payload = create_payload(chunk)
63
+ response = Multiwoven::Integrations::Core::HttpClient.request(
64
+ url,
65
+ sync_config.stream.request_method,
66
+ payload: payload,
67
+ headers: auth_headers(api_key)
68
+ )
69
+ if success?(response)
70
+ write_success += chunk.size
71
+ else
72
+ write_failure += chunk.size
73
+ end
74
+ rescue StandardError => e
75
+ handle_exception("AIRTABLE:RECORD:WRITE:EXCEPTION", "error", e)
76
+ write_failure += chunk.size
77
+ end
78
+
79
+ tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
80
+ success: write_success,
81
+ failed: write_failure
82
+ )
83
+ tracker.to_multiwoven_message
84
+ rescue StandardError => e
85
+ handle_exception("AIRTABLE:WRITE:EXCEPTION", "error", e)
86
+ end
87
+
88
+ private
89
+
90
+ def create_payload(records)
91
+ {
92
+ "records" => records.map do |record|
93
+ {
94
+ "fields" => record
95
+ }
96
+ end
97
+ }
98
+ end
99
+
100
+ def auth_headers(access_token)
101
+ {
102
+ "Accept" => "application/json",
103
+ "Authorization" => "Bearer #{access_token}",
104
+ "Content-Type" => "application/json"
105
+ }
106
+ end
107
+
108
+ def base_id_exists?(bases, base_id)
109
+ return if extract_data(bases).any? { |base| base["id"] == base_id }
110
+
111
+ raise ArgumentError, "base_id not found"
112
+ end
113
+
114
+ def extract_bases(response)
115
+ response_body = extract_body(response)
116
+ response_body["bases"] if response_body
117
+ end
118
+
119
+ def extract_body(response)
120
+ response_body = response.body
121
+ JSON.parse(response_body) if response_body
122
+ end
123
+
124
+ def load_catalog
125
+ read_json(CATALOG_SPEC_PATH)
126
+ end
127
+
128
+ def create_stream(table, base_id, base_name)
129
+ {
130
+ name: "#{base_name}/#{SchemaHelper.clean_name(table["name"])}",
131
+ action: "create",
132
+ method: HTTP_POST,
133
+ url: "#{AIRTABLE_URL_BASE}#{base_id}/#{table["id"]}",
134
+ json_schema: SchemaHelper.get_json_schema(table),
135
+ supported_sync_modes: %w[incremental],
136
+ batch_support: true,
137
+ batch_size: 10
138
+
139
+ }.with_indifferent_access
140
+ end
141
+
142
+ def build_catalog_from_schema(schema, base_id, base_name)
143
+ catalog = build_catalog(load_catalog)
144
+ schema["tables"].each do |table|
145
+ catalog.streams << build_stream(create_stream(table, base_id, base_name))
146
+ end
147
+ catalog
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,6 @@
1
+ {
2
+ "request_rate_limit": 300,
3
+ "request_rate_limit_unit": "minute",
4
+ "request_rate_concurrency": 10,
5
+ "streams": []
6
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "Airtable",
4
+ "title": "airtable",
5
+ "connector_type": "destination",
6
+ "category": "Productivity Tools",
7
+ "documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/airtable",
8
+ "github_issue_label": "destination-airtable",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "documentation_url": "https://docs.multiwoven.com/integrations/destination/airtable",
3
+ "stream_type": "dynamic",
4
+ "connection_specification": {
5
+ "$schema": "http://json-schema.org/draft-07/schema#",
6
+ "title": "Airtable",
7
+ "type": "object",
8
+ "required": ["api_key", "base_id"],
9
+ "properties": {
10
+ "api_key": {
11
+ "type": "string",
12
+ "title": "Personal access token",
13
+ "order": 0
14
+ },
15
+ "base_id": {
16
+ "type": "string",
17
+ "title": "Airtable base id",
18
+ "order": 1
19
+ }
20
+ }
21
+ }
22
+ }
@@ -0,0 +1,6 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" width="32" height="27" shape-rendering="geometricPrecision" viewBox="0 0 200 170">
2
+ <path fill="#FCB400" d="M90.039 12.367L24.079 39.66c-3.667 1.519-3.63 6.729.062 8.192l66.235 26.266a24.575 24.575 0 0018.12 0l66.236-26.266c3.69-1.463 3.729-6.673.06-8.191l-65.958-27.294a24.578 24.578 0 00-18.795 0"></path>
3
+ <path fill="#18BFFF" d="M105.312 88.46v65.617c0 3.12 3.147 5.258 6.048 4.108l73.806-28.648a4.418 4.418 0 002.79-4.108V59.813c0-3.121-3.147-5.258-6.048-4.108l-73.806 28.648a4.42 4.42 0 00-2.79 4.108"></path>
4
+ <path fill="#F82B60" d="M88.078 91.846l-21.904 10.576-2.224 1.075-46.238 22.155c-2.93 1.414-6.672-.722-6.672-3.978V60.088c0-1.178.604-2.195 1.414-2.96a5.024 5.024 0 011.12-.84c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
5
+ <path fill="rgba(0, 0, 0, 0.25)" d="M88.078 91.846l-21.904 10.576-53.72-45.295a5.024 5.024 0 011.12-.839c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
6
+ </svg>
@@ -0,0 +1,141 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven
4
+ module Integrations
5
+ module Destination
6
+ module Airtable
7
+ module SchemaHelper
8
+ include Core::Constants
9
+
10
+ module_function
11
+
12
+ def clean_name(name_str)
13
+ name_str.strip.gsub(" ", "_")
14
+ end
15
+
16
+ def get_json_schema(table)
17
+ fields = table["fields"] || {}
18
+ properties = fields.each_with_object({}) do |field, props|
19
+ name, schema = process_field(field)
20
+ props[name] = schema
21
+ end
22
+
23
+ build_schema(properties)
24
+ end
25
+
26
+ def process_field(field)
27
+ name = clean_name(field.fetch("name", ""))
28
+ original_type = field.fetch("type", "")
29
+ options = field.fetch("options", {})
30
+
31
+ schema = determine_schema(original_type, options)
32
+ [name, schema]
33
+ end
34
+
35
+ def determine_schema(original_type, options)
36
+ if COMPLEX_AIRTABLE_TYPES.keys.include?(original_type)
37
+ complex_type = deep_copy(COMPLEX_AIRTABLE_TYPES[original_type])
38
+ adjust_complex_type(original_type, complex_type, options)
39
+ elsif SIMPLE_AIRTABLE_TYPES.keys.include?(original_type)
40
+ simple_type_schema(original_type, options)
41
+ else
42
+ SCHEMA_TYPES[:STRING]
43
+ end
44
+ end
45
+
46
+ def adjust_complex_type(original_type, complex_type, options)
47
+ exec_type = options.dig("result", "type") || "simpleText"
48
+ if complex_type == SCHEMA_TYPES[:ARRAY_WITH_ANY]
49
+ adjust_array_with_any(original_type, complex_type, exec_type, options)
50
+ else
51
+ complex_type
52
+ end
53
+ end
54
+
55
+ def adjust_array_with_any(original_type, complex_type, exec_type, options)
56
+ if original_type == "formula" && %w[number currency percent duration].include?(exec_type)
57
+ complex_type = SCHEMA_TYPES[:NUMBER]
58
+ elsif original_type == "formula" && ARRAY_FORMULAS.none? { |x| options.fetch("formula", "").start_with?(x) }
59
+ complex_type = SCHEMA_TYPES[:STRING]
60
+ elsif SIMPLE_AIRTABLE_TYPES.keys.include?(exec_type)
61
+ complex_type["items"] = deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
62
+ else
63
+ complex_type["items"] = SCHEMA_TYPES[:STRING]
64
+ end
65
+ complex_type
66
+ end
67
+
68
+ def simple_type_schema(original_type, options)
69
+ exec_type = options.dig("result", "type") || original_type
70
+ deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
71
+ end
72
+
73
+ def build_schema(properties)
74
+ {
75
+ "$schema" => JSON_SCHEMA_URL,
76
+ "type" => "object",
77
+ "additionalProperties" => true,
78
+ "properties" => properties
79
+ }
80
+ end
81
+
82
+ def deep_copy(object)
83
+ Marshal.load(Marshal.dump(object))
84
+ end
85
+
86
+ SCHEMA_TYPES = {
87
+ STRING: { "type": %w[null string] },
88
+ NUMBER: { "type": %w[null number] },
89
+ BOOLEAN: { "type": %w[null boolean] },
90
+ DATE: { "type": %w[null string], "format": "date" },
91
+ DATETIME: { "type": %w[null string], "format": "date-time" },
92
+ ARRAY_WITH_STRINGS: { "type": %w[null array], "items": { "type": %w[null string] } },
93
+ ARRAY_WITH_ANY: { "type": %w[null array], "items": {} }
94
+ }.freeze.with_indifferent_access
95
+
96
+ SIMPLE_AIRTABLE_TYPES = {
97
+ "multipleAttachments" => SCHEMA_TYPES[:STRING],
98
+ "autoNumber" => SCHEMA_TYPES[:NUMBER],
99
+ "barcode" => SCHEMA_TYPES[:STRING],
100
+ "button" => SCHEMA_TYPES[:STRING],
101
+ "checkbox" => :BOOLEAN,
102
+ "singleCollaborator" => SCHEMA_TYPES[:STRING],
103
+ "count" => SCHEMA_TYPES[:NUMBER],
104
+ "createdBy" => SCHEMA_TYPES[:STRING],
105
+ "createdTime" => SCHEMA_TYPES[:DATETIME],
106
+ "currency" => SCHEMA_TYPES[:NUMBER],
107
+ "email" => SCHEMA_TYPES[:STRING],
108
+ "date" => SCHEMA_TYPES[:DATE],
109
+ "dateTime" => SCHEMA_TYPES[:DATETIME],
110
+ "duration" => SCHEMA_TYPES[:NUMBER],
111
+ "lastModifiedBy" => SCHEMA_TYPES[:STRING],
112
+ "lastModifiedTime" => SCHEMA_TYPES[:DATETIME],
113
+ "multipleRecordLinks" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
114
+ "multilineText" => SCHEMA_TYPES[:STRING],
115
+ "multipleCollaborators" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
116
+ "multipleSelects" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
117
+ "number" => SCHEMA_TYPES[:NUMBER],
118
+ "percent" => SCHEMA_TYPES[:NUMBER],
119
+ "phoneNumber" => SCHEMA_TYPES[:STRING],
120
+ "rating" => SCHEMA_TYPES[:NUMBER],
121
+ "richText" => SCHEMA_TYPES[:STRING],
122
+ "singleLineText" => SCHEMA_TYPES[:STRING],
123
+ "singleSelect" => SCHEMA_TYPES[:STRING],
124
+ "externalSyncSource" => SCHEMA_TYPES[:STRING],
125
+ "url" => SCHEMA_TYPES[:STRING],
126
+ "simpleText" => SCHEMA_TYPES[:STRING]
127
+ }.freeze
128
+
129
+ COMPLEX_AIRTABLE_TYPES = {
130
+ "formula" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
131
+ "lookup" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
132
+ "multipleLookupValues" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
133
+ "rollup" => SCHEMA_TYPES[:ARRAY_WITH_ANY]
134
+ }.freeze.with_indifferent_access
135
+
136
+ ARRAY_FORMULAS = %w[ARRAYCOMPACT ARRAYFLATTEN ARRAYUNIQUE ARRAYSLICE].freeze
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end
@@ -0,0 +1,124 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven::Integrations::Destination
4
+ module FacebookCustomAudience
5
+ include Multiwoven::Integrations::Core
6
+ class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
7
+ prepend Multiwoven::Integrations::Core::RateLimiter
8
+ MAX_CHUNK_SIZE = 10_000
9
+ def check_connection(connection_config)
10
+ connection_config = connection_config.with_indifferent_access
11
+ access_token = connection_config[:access_token]
12
+ response = Multiwoven::Integrations::Core::HttpClient.request(
13
+ FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS,
14
+ HTTP_GET,
15
+ headers: auth_headers(access_token)
16
+ )
17
+ if success?(response)
18
+ ad_account_exists?(response, connection_config[:ad_account_id])
19
+ ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
20
+ else
21
+ ConnectionStatus.new(status: ConnectionStatusType["failed"]).to_multiwoven_message
22
+ end
23
+ rescue StandardError => e
24
+ ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
25
+ end
26
+
27
+ def discover(_connection_config = nil)
28
+ catalog_json = read_json(CATALOG_SPEC_PATH)
29
+ catalog = build_catalog(catalog_json)
30
+ catalog.to_multiwoven_message
31
+ rescue StandardError => e
32
+ handle_exception(
33
+ "FACEBOOK AUDIENCE:DISCOVER:EXCEPTION",
34
+ "error",
35
+ e
36
+ )
37
+ end
38
+
39
+ def write(sync_config, records, _action = "insert")
40
+ connection_config = sync_config.destination.connection_specification.with_indifferent_access
41
+ access_token = connection_config[:access_token]
42
+ url = generate_url(sync_config, connection_config)
43
+ write_success = 0
44
+ write_failure = 0
45
+ records.each_slice(MAX_CHUNK_SIZE) do |chunk|
46
+ payload = create_payload(chunk, sync_config.stream.json_schema.with_indifferent_access)
47
+ response = Multiwoven::Integrations::Core::HttpClient.request(
48
+ url,
49
+ sync_config.stream.request_method,
50
+ payload: payload,
51
+ headers: auth_headers(access_token)
52
+ )
53
+ if success?(response)
54
+ write_success += chunk.size
55
+ else
56
+ write_failure += chunk.size
57
+ end
58
+ rescue StandardError => e
59
+ handle_exception("FACEBOOK:RECORD:WRITE:EXCEPTION", "error", e)
60
+ write_failure += chunk.size
61
+ end
62
+
63
+ tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
64
+ success: write_success,
65
+ failed: write_failure
66
+ )
67
+ tracker.to_multiwoven_message
68
+ rescue StandardError => e
69
+ handle_exception("FACEBOOK:WRITE:EXCEPTION", "error", e)
70
+ end
71
+
72
+ private
73
+
74
+ def generate_url(sync_config, connection_config)
75
+ sync_config.stream.url.gsub("{audience_id}", connection_config[:audience_id])
76
+ end
77
+
78
+ def create_payload(records, json_schema)
79
+ schema, data = extract_schema_and_data(records, json_schema)
80
+ {
81
+ "payload" => {
82
+ "schema" => schema,
83
+ "data" => data
84
+ }
85
+ }
86
+ end
87
+
88
+ def extract_schema_and_data(records, json_schema)
89
+ schema_properties = json_schema[:properties]
90
+ schema = records.first.keys.map(&:to_s).map(&:upcase)
91
+ data = []
92
+ records.each do |record|
93
+ encrypted_data_array = []
94
+ record.with_indifferent_access.each do |key, value|
95
+ schema_key = key.upcase
96
+ encrypted_value = schema_properties[schema_key] && schema_properties[schema_key]["x-hashRequired"] ? Digest::SHA256.hexdigest(value.to_s) : value
97
+ encrypted_data_array << encrypted_value
98
+ end
99
+ data << encrypted_data_array
100
+ end
101
+ [schema, data]
102
+ end
103
+
104
+ def auth_headers(access_token)
105
+ {
106
+ "Accept" => "application/json",
107
+ "Authorization" => "Bearer #{access_token}",
108
+ "Content-Type" => "application/json"
109
+ }
110
+ end
111
+
112
+ def ad_account_exists?(response, ad_account_id)
113
+ return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" }
114
+
115
+ raise ArgumentError, "Ad account not found in business account"
116
+ end
117
+
118
+ def extract_data(response)
119
+ response_body = response.body
120
+ JSON.parse(response_body)["data"] if response_body
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,42 @@
1
+ {
2
+ "request_rate_limit": 600,
3
+ "request_rate_limit_unit": "minute",
4
+ "request_rate_concurrency": 10,
5
+ "streams": [
6
+ {
7
+ "name": "audience",
8
+ "action": "create",
9
+ "method": "POST",
10
+ "batch_support": true,
11
+ "batch_size": 10000,
12
+ "url": "https://graph.facebook.com/v18.0/{audience_id}/users",
13
+ "json_schema": {
14
+ "type": "object",
15
+ "additionalProperties": true,
16
+ "properties": {
17
+ "EMAIL": { "type": ["string", "null"], "default": null, "title": "Email", "x-hashRequired": true },
18
+ "PHONE": { "type": ["string", "null"], "default": null, "title": "Phone", "x-hashRequired": true },
19
+ "GEN": { "type": ["string", "null"], "default": null, "title": "Gender", "x-hashRequired": true },
20
+ "FI": { "type": ["string", "null"], "default": null, "title": "First Initial", "x-hashRequired": true },
21
+ "FN": { "type": ["string", "null"], "default": null, "title": "First Name", "x-hashRequired": true },
22
+ "DOBD": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Day)", "x-hashRequired": true },
23
+ "DOBM": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Month)", "x-hashRequired": true },
24
+ "DOBY": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Year)", "x-hashRequired": true },
25
+ "LN": { "type": ["string", "null"], "default": null, "title": "Last Name", "x-hashRequired": true },
26
+ "CT": { "type": ["string", "null"], "default": null, "title": "City", "x-hashRequired": true },
27
+ "ST": { "type": ["string", "null"], "default": null, "title": "State", "x-hashRequired": true },
28
+ "ZIP": { "type": ["string", "null"], "default": null, "title": "Zip Code", "x-hashRequired": true },
29
+ "COUNTRY": { "type": ["string", "null"], "default": null, "title": "Country", "x-hashRequired": true },
30
+ "EXTERN_ID": { "type": ["string", "null"], "default": null, "title": "External ID", "x-hashRequired": false},
31
+ "LOOKALIKE_VALUE": { "type": ["number", "null"], "default": null, "title": "Lookalike Value", "x-hashRequired": false },
32
+ "MADID": { "type": ["string", "null"], "default": null, "title": "Mobile ID", "x-hashRequired": false },
33
+ "PAGEUID": { "type": ["string", "null"], "default": null, "title": "Page-Scoped ID", "x-hashRequired": false }
34
+ }
35
+ },
36
+ "supported_sync_modes": ["incremental"],
37
+ "source_defined_cursor": true,
38
+ "default_cursor_field": ["updated"],
39
+ "source_defined_primary_key": [["email"]]
40
+ }
41
+ ]
42
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "FacebookCustomAudience",
4
+ "title": "Facebook Custom Audiences",
5
+ "connector_type": "destination",
6
+ "category": "Adtech",
7
+ "documentation_url": "https://docs.mutliwoven.com",
8
+ "github_issue_label": "destination-facebook",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,27 @@
1
+ {
2
+ "documentation_url": "https://docs.multiwoven.com/integrations/destination/facebook",
3
+ "stream_type": "static",
4
+ "connection_specification": {
5
+ "$schema": "http://json-schema.org/draft-07/schema#",
6
+ "title": "Facebook Custom Audiences",
7
+ "type": "object",
8
+ "required": ["access_token", "ad_account_id", "audience_id"],
9
+ "properties": {
10
+ "access_token": {
11
+ "type": "string",
12
+ "title": "Access Token",
13
+ "order": 0
14
+ },
15
+ "ad_account_id": {
16
+ "type": "string",
17
+ "title": "Ad Account Id",
18
+ "order": 1
19
+ },
20
+ "audience_id": {
21
+ "type": "string",
22
+ "title": "Audience Id",
23
+ "order": 2
24
+ }
25
+ }
26
+ }
27
+ }