etl-integrations 0.1.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. checksums.yaml +7 -0
  2. data/.rspec +3 -0
  3. data/.rubocop.yml +34 -0
  4. data/.ruby-version +1 -0
  5. data/.vscode/settings.json +5 -0
  6. data/CHANGELOG.md +38 -0
  7. data/CODE_OF_CONDUCT.md +84 -0
  8. data/LICENSE.txt +21 -0
  9. data/README.md +105 -0
  10. data/Rakefile +12 -0
  11. data/lib/multiwoven/integrations/config.rb +13 -0
  12. data/lib/multiwoven/integrations/core/base_connector.rb +70 -0
  13. data/lib/multiwoven/integrations/core/constants.rb +46 -0
  14. data/lib/multiwoven/integrations/core/destination_connector.rb +14 -0
  15. data/lib/multiwoven/integrations/core/fullrefresher.rb +19 -0
  16. data/lib/multiwoven/integrations/core/http_client.rb +34 -0
  17. data/lib/multiwoven/integrations/core/query_builder.rb +27 -0
  18. data/lib/multiwoven/integrations/core/rate_limiter.rb +19 -0
  19. data/lib/multiwoven/integrations/core/source_connector.rb +38 -0
  20. data/lib/multiwoven/integrations/core/utils.rb +104 -0
  21. data/lib/multiwoven/integrations/destination/airtable/client.rb +153 -0
  22. data/lib/multiwoven/integrations/destination/airtable/config/catalog.json +6 -0
  23. data/lib/multiwoven/integrations/destination/airtable/config/meta.json +15 -0
  24. data/lib/multiwoven/integrations/destination/airtable/config/spec.json +22 -0
  25. data/lib/multiwoven/integrations/destination/airtable/icon.svg +6 -0
  26. data/lib/multiwoven/integrations/destination/airtable/schema_helper.rb +141 -0
  27. data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +124 -0
  28. data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json +42 -0
  29. data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json +15 -0
  30. data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json +27 -0
  31. data/lib/multiwoven/integrations/destination/facebook_custom_audience/icon.svg +23 -0
  32. data/lib/multiwoven/integrations/destination/google_sheets/client.rb +231 -0
  33. data/lib/multiwoven/integrations/destination/google_sheets/config/catalog.json +6 -0
  34. data/lib/multiwoven/integrations/destination/google_sheets/config/meta.json +15 -0
  35. data/lib/multiwoven/integrations/destination/google_sheets/config/spec.json +74 -0
  36. data/lib/multiwoven/integrations/destination/google_sheets/icon.svg +1 -0
  37. data/lib/multiwoven/integrations/destination/hubspot/client.rb +107 -0
  38. data/lib/multiwoven/integrations/destination/hubspot/config/catalog.json +351 -0
  39. data/lib/multiwoven/integrations/destination/hubspot/config/meta.json +15 -0
  40. data/lib/multiwoven/integrations/destination/hubspot/config/spec.json +17 -0
  41. data/lib/multiwoven/integrations/destination/hubspot/icon.svg +5 -0
  42. data/lib/multiwoven/integrations/destination/klaviyo/client.rb +116 -0
  43. data/lib/multiwoven/integrations/destination/klaviyo/config/catalog.json +103 -0
  44. data/lib/multiwoven/integrations/destination/klaviyo/config/meta.json +15 -0
  45. data/lib/multiwoven/integrations/destination/klaviyo/config/spec.json +22 -0
  46. data/lib/multiwoven/integrations/destination/klaviyo/icon.svg +6 -0
  47. data/lib/multiwoven/integrations/destination/postgresql/client.rb +123 -0
  48. data/lib/multiwoven/integrations/destination/postgresql/config/meta.json +15 -0
  49. data/lib/multiwoven/integrations/destination/postgresql/config/spec.json +68 -0
  50. data/lib/multiwoven/integrations/destination/postgresql/icon.svg +20 -0
  51. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb +114 -0
  52. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json +6 -0
  53. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json +16 -0
  54. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/spec.json +49 -0
  55. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/icon.svg +16 -0
  56. data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb +132 -0
  57. data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb +117 -0
  58. data/lib/multiwoven/integrations/destination/salesforce_crm/config/catalog.json +320 -0
  59. data/lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json +15 -0
  60. data/lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json +43 -0
  61. data/lib/multiwoven/integrations/destination/salesforce_crm/icon.svg +16 -0
  62. data/lib/multiwoven/integrations/destination/sftp/client.rb +133 -0
  63. data/lib/multiwoven/integrations/destination/sftp/config/catalog.json +16 -0
  64. data/lib/multiwoven/integrations/destination/sftp/config/meta.json +16 -0
  65. data/lib/multiwoven/integrations/destination/sftp/config/spec.json +50 -0
  66. data/lib/multiwoven/integrations/destination/sftp/icon.svg +1 -0
  67. data/lib/multiwoven/integrations/destination/slack/client.rb +114 -0
  68. data/lib/multiwoven/integrations/destination/slack/config/catalog.json +22 -0
  69. data/lib/multiwoven/integrations/destination/slack/config/meta.json +15 -0
  70. data/lib/multiwoven/integrations/destination/slack/config/spec.json +22 -0
  71. data/lib/multiwoven/integrations/destination/slack/icon.svg +26 -0
  72. data/lib/multiwoven/integrations/destination/stripe/client.rb +82 -0
  73. data/lib/multiwoven/integrations/destination/stripe/config/catalog.json +128 -0
  74. data/lib/multiwoven/integrations/destination/stripe/config/meta.json +15 -0
  75. data/lib/multiwoven/integrations/destination/stripe/config/spec.json +17 -0
  76. data/lib/multiwoven/integrations/destination/stripe/icon.svg +10 -0
  77. data/lib/multiwoven/integrations/destination/tally/client.rb +151 -0
  78. data/lib/multiwoven/integrations/destination/tally/config/catalog.json +62 -0
  79. data/lib/multiwoven/integrations/destination/tally/config/meta.json +15 -0
  80. data/lib/multiwoven/integrations/destination/tally/config/spec.json +45 -0
  81. data/lib/multiwoven/integrations/destination/tally/icon.svg +7 -0
  82. data/lib/multiwoven/integrations/protocol/protocol.json +189 -0
  83. data/lib/multiwoven/integrations/protocol/protocol.rb +216 -0
  84. data/lib/multiwoven/integrations/rollout.rb +32 -0
  85. data/lib/multiwoven/integrations/service.rb +79 -0
  86. data/lib/multiwoven/integrations/source/bigquery/client.rb +98 -0
  87. data/lib/multiwoven/integrations/source/bigquery/config/meta.json +15 -0
  88. data/lib/multiwoven/integrations/source/bigquery/config/spec.json +82 -0
  89. data/lib/multiwoven/integrations/source/bigquery/icon.svg +1 -0
  90. data/lib/multiwoven/integrations/source/databricks/client.rb +98 -0
  91. data/lib/multiwoven/integrations/source/databricks/config/meta.json +16 -0
  92. data/lib/multiwoven/integrations/source/databricks/config/spec.json +56 -0
  93. data/lib/multiwoven/integrations/source/databricks/icon.svg +19 -0
  94. data/lib/multiwoven/integrations/source/postgresql/client.rb +109 -0
  95. data/lib/multiwoven/integrations/source/postgresql/config/meta.json +15 -0
  96. data/lib/multiwoven/integrations/source/postgresql/config/spec.json +69 -0
  97. data/lib/multiwoven/integrations/source/postgresql/icon.svg +20 -0
  98. data/lib/multiwoven/integrations/source/redshift/client.rb +109 -0
  99. data/lib/multiwoven/integrations/source/redshift/config/meta.json +15 -0
  100. data/lib/multiwoven/integrations/source/redshift/config/spec.json +71 -0
  101. data/lib/multiwoven/integrations/source/redshift/icon.svg +15 -0
  102. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb +123 -0
  103. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/catalog.json +6 -0
  104. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/meta.json +17 -0
  105. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/config/spec.json +50 -0
  106. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/icon.svg +16 -0
  107. data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb +130 -0
  108. data/lib/multiwoven/integrations/source/snowflake/client.rb +92 -0
  109. data/lib/multiwoven/integrations/source/snowflake/config/meta.json +15 -0
  110. data/lib/multiwoven/integrations/source/snowflake/config/spec.json +82 -0
  111. data/lib/multiwoven/integrations/source/snowflake/icon.svg +10 -0
  112. data/lib/multiwoven/integrations/source/zoho_books/client.rb +155 -0
  113. data/lib/multiwoven/integrations/source/zoho_books/config/meta.json +16 -0
  114. data/lib/multiwoven/integrations/source/zoho_books/config/spec.json +43 -0
  115. data/lib/multiwoven/integrations/source/zoho_books/icon.svg +16 -0
  116. data/lib/multiwoven/integrations.rb +71 -0
  117. data/multiwoven-integrations-0.1.68.gem +0 -0
  118. data/sig/multiwoven/integrations.rbs +6 -0
  119. metadata +515 -0
@@ -0,0 +1,104 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven
4
+ module Integrations::Core
5
+ module Utils
6
+ def keys_to_symbols(hash)
7
+ if hash.is_a?(Hash)
8
+ hash.each_with_object({}) do |(key, value), result|
9
+ result[key.to_sym] = keys_to_symbols(value)
10
+ end
11
+ elsif hash.is_a?(Array)
12
+ hash.map { |item| keys_to_symbols(item) }
13
+ else
14
+ hash
15
+ end
16
+ end
17
+
18
+ def convert_to_json_schema(column_definitions)
19
+ json_schema = {
20
+ "type" => "object",
21
+ "properties" => {}
22
+ }
23
+
24
+ column_definitions.each do |column|
25
+ column_name = column[:column_name]
26
+ type = column[:type]
27
+ optional = column[:optional]
28
+ json_type = map_type_to_json_schema(type)
29
+ json_schema["properties"][column_name] = {
30
+ "type" => json_type
31
+ }
32
+ json_schema["properties"][column_name]["type"] = [json_type, "null"] if optional
33
+ end
34
+
35
+ json_schema
36
+ end
37
+
38
+ def map_type_to_json_schema(type)
39
+ case type
40
+ when "NUMBER"
41
+ "integer"
42
+ else
43
+ "string" # Default type
44
+ end
45
+ end
46
+
47
+ def logger
48
+ Integrations::Service.logger
49
+ end
50
+
51
+ def create_log_message(context, type, exception)
52
+ Integrations::Protocol::LogMessage.new(
53
+ name: context,
54
+ level: type,
55
+ message: exception.message
56
+ ).to_multiwoven_message
57
+ end
58
+
59
+ def handle_exception(context, type, exception)
60
+ logger.error(
61
+ "#{context}: #{exception.message}"
62
+ )
63
+
64
+ create_log_message(context, type, exception)
65
+ end
66
+
67
+ def extract_data(record_object, properties)
68
+ data_attributes = record_object.with_indifferent_access
69
+ data_attributes.select { |key, _| properties.key?(key.to_sym) }
70
+ end
71
+
72
+ def success?(response)
73
+ response && %w[200 201].include?(response.code.to_s)
74
+ end
75
+
76
+ def build_catalog(catalog_json)
77
+ streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
78
+ Multiwoven::Integrations::Protocol::Catalog.new(
79
+ streams: streams,
80
+ request_rate_limit: catalog_json["request_rate_limit"] || 60,
81
+ request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
82
+ request_rate_concurrency: catalog_json["request_rate_concurrency"] || 10,
83
+ schema_mode: catalog_json["schema_mode"] || "schema"
84
+ )
85
+ end
86
+
87
+ def build_stream(stream_json)
88
+ Multiwoven::Integrations::Protocol::Stream.new(
89
+ name: stream_json["name"],
90
+ url: stream_json["url"],
91
+ action: stream_json["action"],
92
+ request_method: stream_json["method"],
93
+ batch_support: stream_json["batch_support"] || false,
94
+ batch_size: stream_json["batch_size"] || 1,
95
+ json_schema: stream_json["json_schema"],
96
+ request_rate_limit: stream_json["request_rate_limit"].to_i,
97
+ request_rate_limit_unit: stream_json["request_rate_limit_unit"] || "minute",
98
+ request_rate_concurrency: stream_json["request_rate_concurrency"].to_i,
99
+ supported_sync_modes: stream_json["supported_sync_modes"]
100
+ )
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,153 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "schema_helper"
4
+ module Multiwoven
5
+ module Integrations
6
+ module Destination
7
+ module Airtable
8
+ include Multiwoven::Integrations::Core
9
+ class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
10
+ prepend Multiwoven::Integrations::Core::RateLimiter
11
+ MAX_CHUNK_SIZE = 10
12
+ def check_connection(connection_config)
13
+ connection_config = connection_config.with_indifferent_access
14
+ bases = Multiwoven::Integrations::Core::HttpClient.request(
15
+ AIRTABLE_BASES_ENDPOINT,
16
+ HTTP_GET,
17
+ headers: auth_headers(connection_config[:api_key])
18
+ )
19
+ if success?(bases)
20
+ base_id_exists?(bases, connection_config[:base_id])
21
+ success_status
22
+ else
23
+ failure_status(nil)
24
+ end
25
+ rescue StandardError => e
26
+ failure_status(e)
27
+ end
28
+
29
+ def discover(connection_config)
30
+ connection_config = connection_config.with_indifferent_access
31
+ base_id = connection_config[:base_id]
32
+ api_key = connection_config[:api_key]
33
+
34
+ bases = Multiwoven::Integrations::Core::HttpClient.request(
35
+ AIRTABLE_BASES_ENDPOINT,
36
+ HTTP_GET,
37
+ headers: auth_headers(api_key)
38
+ )
39
+
40
+ base = extract_bases(bases).find { |b| b["id"] == base_id }
41
+ base_name = base["name"]
42
+
43
+ schema = Multiwoven::Integrations::Core::HttpClient.request(
44
+ AIRTABLE_GET_BASE_SCHEMA_ENDPOINT.gsub("{baseId}", base_id),
45
+ HTTP_GET,
46
+ headers: auth_headers(api_key)
47
+ )
48
+
49
+ catalog = build_catalog_from_schema(extract_body(schema), base_id, base_name)
50
+ catalog.to_multiwoven_message
51
+ rescue StandardError => e
52
+ handle_exception("AIRTABLE:DISCOVER:EXCEPTION", "error", e)
53
+ end
54
+
55
+ def write(sync_config, records, _action = "create")
56
+ connection_config = sync_config.destination.connection_specification.with_indifferent_access
57
+ api_key = connection_config[:api_key]
58
+ url = sync_config.stream.url
59
+ write_success = 0
60
+ write_failure = 0
61
+ records.each_slice(MAX_CHUNK_SIZE) do |chunk|
62
+ payload = create_payload(chunk)
63
+ response = Multiwoven::Integrations::Core::HttpClient.request(
64
+ url,
65
+ sync_config.stream.request_method,
66
+ payload: payload,
67
+ headers: auth_headers(api_key)
68
+ )
69
+ if success?(response)
70
+ write_success += chunk.size
71
+ else
72
+ write_failure += chunk.size
73
+ end
74
+ rescue StandardError => e
75
+ handle_exception("AIRTABLE:RECORD:WRITE:EXCEPTION", "error", e)
76
+ write_failure += chunk.size
77
+ end
78
+
79
+ tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
80
+ success: write_success,
81
+ failed: write_failure
82
+ )
83
+ tracker.to_multiwoven_message
84
+ rescue StandardError => e
85
+ handle_exception("AIRTABLE:WRITE:EXCEPTION", "error", e)
86
+ end
87
+
88
+ private
89
+
90
+ def create_payload(records)
91
+ {
92
+ "records" => records.map do |record|
93
+ {
94
+ "fields" => record
95
+ }
96
+ end
97
+ }
98
+ end
99
+
100
+ def auth_headers(access_token)
101
+ {
102
+ "Accept" => "application/json",
103
+ "Authorization" => "Bearer #{access_token}",
104
+ "Content-Type" => "application/json"
105
+ }
106
+ end
107
+
108
+ def base_id_exists?(bases, base_id)
109
+ return if extract_data(bases).any? { |base| base["id"] == base_id }
110
+
111
+ raise ArgumentError, "base_id not found"
112
+ end
113
+
114
+ def extract_bases(response)
115
+ response_body = extract_body(response)
116
+ response_body["bases"] if response_body
117
+ end
118
+
119
+ def extract_body(response)
120
+ response_body = response.body
121
+ JSON.parse(response_body) if response_body
122
+ end
123
+
124
+ def load_catalog
125
+ read_json(CATALOG_SPEC_PATH)
126
+ end
127
+
128
+ def create_stream(table, base_id, base_name)
129
+ {
130
+ name: "#{base_name}/#{SchemaHelper.clean_name(table["name"])}",
131
+ action: "create",
132
+ method: HTTP_POST,
133
+ url: "#{AIRTABLE_URL_BASE}#{base_id}/#{table["id"]}",
134
+ json_schema: SchemaHelper.get_json_schema(table),
135
+ supported_sync_modes: %w[incremental],
136
+ batch_support: true,
137
+ batch_size: 10
138
+
139
+ }.with_indifferent_access
140
+ end
141
+
142
+ def build_catalog_from_schema(schema, base_id, base_name)
143
+ catalog = build_catalog(load_catalog)
144
+ schema["tables"].each do |table|
145
+ catalog.streams << build_stream(create_stream(table, base_id, base_name))
146
+ end
147
+ catalog
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+ end
@@ -0,0 +1,6 @@
1
+ {
2
+ "request_rate_limit": 300,
3
+ "request_rate_limit_unit": "minute",
4
+ "request_rate_concurrency": 10,
5
+ "streams": []
6
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "Airtable",
4
+ "title": "airtable",
5
+ "connector_type": "destination",
6
+ "category": "Productivity Tools",
7
+ "documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/airtable",
8
+ "github_issue_label": "destination-airtable",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,22 @@
1
+ {
2
+ "documentation_url": "https://docs.multiwoven.com/integrations/destination/airtable",
3
+ "stream_type": "dynamic",
4
+ "connection_specification": {
5
+ "$schema": "http://json-schema.org/draft-07/schema#",
6
+ "title": "Airtable",
7
+ "type": "object",
8
+ "required": ["api_key", "base_id"],
9
+ "properties": {
10
+ "api_key": {
11
+ "type": "string",
12
+ "title": "Personal access token",
13
+ "order": 0
14
+ },
15
+ "base_id": {
16
+ "type": "string",
17
+ "title": "Airtable base id",
18
+ "order": 1
19
+ }
20
+ }
21
+ }
22
+ }
@@ -0,0 +1,6 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" width="32" height="27" shape-rendering="geometricPrecision" viewBox="0 0 200 170">
2
+ <path fill="#FCB400" d="M90.039 12.367L24.079 39.66c-3.667 1.519-3.63 6.729.062 8.192l66.235 26.266a24.575 24.575 0 0018.12 0l66.236-26.266c3.69-1.463 3.729-6.673.06-8.191l-65.958-27.294a24.578 24.578 0 00-18.795 0"></path>
3
+ <path fill="#18BFFF" d="M105.312 88.46v65.617c0 3.12 3.147 5.258 6.048 4.108l73.806-28.648a4.418 4.418 0 002.79-4.108V59.813c0-3.121-3.147-5.258-6.048-4.108l-73.806 28.648a4.42 4.42 0 00-2.79 4.108"></path>
4
+ <path fill="#F82B60" d="M88.078 91.846l-21.904 10.576-2.224 1.075-46.238 22.155c-2.93 1.414-6.672-.722-6.672-3.978V60.088c0-1.178.604-2.195 1.414-2.96a5.024 5.024 0 011.12-.84c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
5
+ <path fill="rgba(0, 0, 0, 0.25)" d="M88.078 91.846l-21.904 10.576-53.72-45.295a5.024 5.024 0 011.12-.839c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
6
+ </svg>
@@ -0,0 +1,141 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven
4
+ module Integrations
5
+ module Destination
6
+ module Airtable
7
+ module SchemaHelper
8
+ include Core::Constants
9
+
10
+ module_function
11
+
12
+ def clean_name(name_str)
13
+ name_str.strip.gsub(" ", "_")
14
+ end
15
+
16
+ def get_json_schema(table)
17
+ fields = table["fields"] || {}
18
+ properties = fields.each_with_object({}) do |field, props|
19
+ name, schema = process_field(field)
20
+ props[name] = schema
21
+ end
22
+
23
+ build_schema(properties)
24
+ end
25
+
26
+ def process_field(field)
27
+ name = clean_name(field.fetch("name", ""))
28
+ original_type = field.fetch("type", "")
29
+ options = field.fetch("options", {})
30
+
31
+ schema = determine_schema(original_type, options)
32
+ [name, schema]
33
+ end
34
+
35
+ def determine_schema(original_type, options)
36
+ if COMPLEX_AIRTABLE_TYPES.keys.include?(original_type)
37
+ complex_type = deep_copy(COMPLEX_AIRTABLE_TYPES[original_type])
38
+ adjust_complex_type(original_type, complex_type, options)
39
+ elsif SIMPLE_AIRTABLE_TYPES.keys.include?(original_type)
40
+ simple_type_schema(original_type, options)
41
+ else
42
+ SCHEMA_TYPES[:STRING]
43
+ end
44
+ end
45
+
46
+ def adjust_complex_type(original_type, complex_type, options)
47
+ exec_type = options.dig("result", "type") || "simpleText"
48
+ if complex_type == SCHEMA_TYPES[:ARRAY_WITH_ANY]
49
+ adjust_array_with_any(original_type, complex_type, exec_type, options)
50
+ else
51
+ complex_type
52
+ end
53
+ end
54
+
55
+ def adjust_array_with_any(original_type, complex_type, exec_type, options)
56
+ if original_type == "formula" && %w[number currency percent duration].include?(exec_type)
57
+ complex_type = SCHEMA_TYPES[:NUMBER]
58
+ elsif original_type == "formula" && ARRAY_FORMULAS.none? { |x| options.fetch("formula", "").start_with?(x) }
59
+ complex_type = SCHEMA_TYPES[:STRING]
60
+ elsif SIMPLE_AIRTABLE_TYPES.keys.include?(exec_type)
61
+ complex_type["items"] = deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
62
+ else
63
+ complex_type["items"] = SCHEMA_TYPES[:STRING]
64
+ end
65
+ complex_type
66
+ end
67
+
68
+ def simple_type_schema(original_type, options)
69
+ exec_type = options.dig("result", "type") || original_type
70
+ deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
71
+ end
72
+
73
+ def build_schema(properties)
74
+ {
75
+ "$schema" => JSON_SCHEMA_URL,
76
+ "type" => "object",
77
+ "additionalProperties" => true,
78
+ "properties" => properties
79
+ }
80
+ end
81
+
82
+ def deep_copy(object)
83
+ Marshal.load(Marshal.dump(object))
84
+ end
85
+
86
+ SCHEMA_TYPES = {
87
+ STRING: { "type": %w[null string] },
88
+ NUMBER: { "type": %w[null number] },
89
+ BOOLEAN: { "type": %w[null boolean] },
90
+ DATE: { "type": %w[null string], "format": "date" },
91
+ DATETIME: { "type": %w[null string], "format": "date-time" },
92
+ ARRAY_WITH_STRINGS: { "type": %w[null array], "items": { "type": %w[null string] } },
93
+ ARRAY_WITH_ANY: { "type": %w[null array], "items": {} }
94
+ }.freeze.with_indifferent_access
95
+
96
+ SIMPLE_AIRTABLE_TYPES = {
97
+ "multipleAttachments" => SCHEMA_TYPES[:STRING],
98
+ "autoNumber" => SCHEMA_TYPES[:NUMBER],
99
+ "barcode" => SCHEMA_TYPES[:STRING],
100
+ "button" => SCHEMA_TYPES[:STRING],
101
+ "checkbox" => :BOOLEAN,
102
+ "singleCollaborator" => SCHEMA_TYPES[:STRING],
103
+ "count" => SCHEMA_TYPES[:NUMBER],
104
+ "createdBy" => SCHEMA_TYPES[:STRING],
105
+ "createdTime" => SCHEMA_TYPES[:DATETIME],
106
+ "currency" => SCHEMA_TYPES[:NUMBER],
107
+ "email" => SCHEMA_TYPES[:STRING],
108
+ "date" => SCHEMA_TYPES[:DATE],
109
+ "dateTime" => SCHEMA_TYPES[:DATETIME],
110
+ "duration" => SCHEMA_TYPES[:NUMBER],
111
+ "lastModifiedBy" => SCHEMA_TYPES[:STRING],
112
+ "lastModifiedTime" => SCHEMA_TYPES[:DATETIME],
113
+ "multipleRecordLinks" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
114
+ "multilineText" => SCHEMA_TYPES[:STRING],
115
+ "multipleCollaborators" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
116
+ "multipleSelects" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
117
+ "number" => SCHEMA_TYPES[:NUMBER],
118
+ "percent" => SCHEMA_TYPES[:NUMBER],
119
+ "phoneNumber" => SCHEMA_TYPES[:STRING],
120
+ "rating" => SCHEMA_TYPES[:NUMBER],
121
+ "richText" => SCHEMA_TYPES[:STRING],
122
+ "singleLineText" => SCHEMA_TYPES[:STRING],
123
+ "singleSelect" => SCHEMA_TYPES[:STRING],
124
+ "externalSyncSource" => SCHEMA_TYPES[:STRING],
125
+ "url" => SCHEMA_TYPES[:STRING],
126
+ "simpleText" => SCHEMA_TYPES[:STRING]
127
+ }.freeze
128
+
129
+ COMPLEX_AIRTABLE_TYPES = {
130
+ "formula" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
131
+ "lookup" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
132
+ "multipleLookupValues" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
133
+ "rollup" => SCHEMA_TYPES[:ARRAY_WITH_ANY]
134
+ }.freeze.with_indifferent_access
135
+
136
+ ARRAY_FORMULAS = %w[ARRAYCOMPACT ARRAYFLATTEN ARRAYUNIQUE ARRAYSLICE].freeze
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end
@@ -0,0 +1,124 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven::Integrations::Destination
4
+ module FacebookCustomAudience
5
+ include Multiwoven::Integrations::Core
6
+ class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
7
+ prepend Multiwoven::Integrations::Core::RateLimiter
8
+ MAX_CHUNK_SIZE = 10_000
9
+ def check_connection(connection_config)
10
+ connection_config = connection_config.with_indifferent_access
11
+ access_token = connection_config[:access_token]
12
+ response = Multiwoven::Integrations::Core::HttpClient.request(
13
+ FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS,
14
+ HTTP_GET,
15
+ headers: auth_headers(access_token)
16
+ )
17
+ if success?(response)
18
+ ad_account_exists?(response, connection_config[:ad_account_id])
19
+ ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
20
+ else
21
+ ConnectionStatus.new(status: ConnectionStatusType["failed"]).to_multiwoven_message
22
+ end
23
+ rescue StandardError => e
24
+ ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
25
+ end
26
+
27
+ def discover(_connection_config = nil)
28
+ catalog_json = read_json(CATALOG_SPEC_PATH)
29
+ catalog = build_catalog(catalog_json)
30
+ catalog.to_multiwoven_message
31
+ rescue StandardError => e
32
+ handle_exception(
33
+ "FACEBOOK AUDIENCE:DISCOVER:EXCEPTION",
34
+ "error",
35
+ e
36
+ )
37
+ end
38
+
39
+ def write(sync_config, records, _action = "insert")
40
+ connection_config = sync_config.destination.connection_specification.with_indifferent_access
41
+ access_token = connection_config[:access_token]
42
+ url = generate_url(sync_config, connection_config)
43
+ write_success = 0
44
+ write_failure = 0
45
+ records.each_slice(MAX_CHUNK_SIZE) do |chunk|
46
+ payload = create_payload(chunk, sync_config.stream.json_schema.with_indifferent_access)
47
+ response = Multiwoven::Integrations::Core::HttpClient.request(
48
+ url,
49
+ sync_config.stream.request_method,
50
+ payload: payload,
51
+ headers: auth_headers(access_token)
52
+ )
53
+ if success?(response)
54
+ write_success += chunk.size
55
+ else
56
+ write_failure += chunk.size
57
+ end
58
+ rescue StandardError => e
59
+ handle_exception("FACEBOOK:RECORD:WRITE:EXCEPTION", "error", e)
60
+ write_failure += chunk.size
61
+ end
62
+
63
+ tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
64
+ success: write_success,
65
+ failed: write_failure
66
+ )
67
+ tracker.to_multiwoven_message
68
+ rescue StandardError => e
69
+ handle_exception("FACEBOOK:WRITE:EXCEPTION", "error", e)
70
+ end
71
+
72
+ private
73
+
74
+ def generate_url(sync_config, connection_config)
75
+ sync_config.stream.url.gsub("{audience_id}", connection_config[:audience_id])
76
+ end
77
+
78
+ def create_payload(records, json_schema)
79
+ schema, data = extract_schema_and_data(records, json_schema)
80
+ {
81
+ "payload" => {
82
+ "schema" => schema,
83
+ "data" => data
84
+ }
85
+ }
86
+ end
87
+
88
+ def extract_schema_and_data(records, json_schema)
89
+ schema_properties = json_schema[:properties]
90
+ schema = records.first.keys.map(&:to_s).map(&:upcase)
91
+ data = []
92
+ records.each do |record|
93
+ encrypted_data_array = []
94
+ record.with_indifferent_access.each do |key, value|
95
+ schema_key = key.upcase
96
+ encrypted_value = schema_properties[schema_key] && schema_properties[schema_key]["x-hashRequired"] ? Digest::SHA256.hexdigest(value.to_s) : value
97
+ encrypted_data_array << encrypted_value
98
+ end
99
+ data << encrypted_data_array
100
+ end
101
+ [schema, data]
102
+ end
103
+
104
+ def auth_headers(access_token)
105
+ {
106
+ "Accept" => "application/json",
107
+ "Authorization" => "Bearer #{access_token}",
108
+ "Content-Type" => "application/json"
109
+ }
110
+ end
111
+
112
+ def ad_account_exists?(response, ad_account_id)
113
+ return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" }
114
+
115
+ raise ArgumentError, "Ad account not found in business account"
116
+ end
117
+
118
+ def extract_data(response)
119
+ response_body = response.body
120
+ JSON.parse(response_body)["data"] if response_body
121
+ end
122
+ end
123
+ end
124
+ end
@@ -0,0 +1,42 @@
1
+ {
2
+ "request_rate_limit": 600,
3
+ "request_rate_limit_unit": "minute",
4
+ "request_rate_concurrency": 10,
5
+ "streams": [
6
+ {
7
+ "name": "audience",
8
+ "action": "create",
9
+ "method": "POST",
10
+ "batch_support": true,
11
+ "batch_size": 10000,
12
+ "url": "https://graph.facebook.com/v18.0/{audience_id}/users",
13
+ "json_schema": {
14
+ "type": "object",
15
+ "additionalProperties": true,
16
+ "properties": {
17
+ "EMAIL": { "type": ["string", "null"], "default": null, "title": "Email", "x-hashRequired": true },
18
+ "PHONE": { "type": ["string", "null"], "default": null, "title": "Phone", "x-hashRequired": true },
19
+ "GEN": { "type": ["string", "null"], "default": null, "title": "Gender", "x-hashRequired": true },
20
+ "FI": { "type": ["string", "null"], "default": null, "title": "First Initial", "x-hashRequired": true },
21
+ "FN": { "type": ["string", "null"], "default": null, "title": "First Name", "x-hashRequired": true },
22
+ "DOBD": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Day)", "x-hashRequired": true },
23
+ "DOBM": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Month)", "x-hashRequired": true },
24
+ "DOBY": { "type": ["string", "null"], "default": null, "title": "Date of Birth (Year)", "x-hashRequired": true },
25
+ "LN": { "type": ["string", "null"], "default": null, "title": "Last Name", "x-hashRequired": true },
26
+ "CT": { "type": ["string", "null"], "default": null, "title": "City", "x-hashRequired": true },
27
+ "ST": { "type": ["string", "null"], "default": null, "title": "State", "x-hashRequired": true },
28
+ "ZIP": { "type": ["string", "null"], "default": null, "title": "Zip Code", "x-hashRequired": true },
29
+ "COUNTRY": { "type": ["string", "null"], "default": null, "title": "Country", "x-hashRequired": true },
30
+ "EXTERN_ID": { "type": ["string", "null"], "default": null, "title": "External ID", "x-hashRequired": false},
31
+ "LOOKALIKE_VALUE": { "type": ["number", "null"], "default": null, "title": "Lookalike Value", "x-hashRequired": false },
32
+ "MADID": { "type": ["string", "null"], "default": null, "title": "Mobile ID", "x-hashRequired": false },
33
+ "PAGEUID": { "type": ["string", "null"], "default": null, "title": "Page-Scoped ID", "x-hashRequired": false }
34
+ }
35
+ },
36
+ "supported_sync_modes": ["incremental"],
37
+ "source_defined_cursor": true,
38
+ "default_cursor_field": ["updated"],
39
+ "source_defined_primary_key": [["email"]]
40
+ }
41
+ ]
42
+ }
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "FacebookCustomAudience",
4
+ "title": "Facebook Custom Audiences",
5
+ "connector_type": "destination",
6
+ "category": "Adtech",
7
+ "documentation_url": "https://docs.mutliwoven.com",
8
+ "github_issue_label": "destination-facebook",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,27 @@
1
+ {
2
+ "documentation_url": "https://docs.multiwoven.com/integrations/destination/facebook",
3
+ "stream_type": "static",
4
+ "connection_specification": {
5
+ "$schema": "http://json-schema.org/draft-07/schema#",
6
+ "title": "Facebook Custom Audiences",
7
+ "type": "object",
8
+ "required": ["access_token", "ad_account_id", "audience_id"],
9
+ "properties": {
10
+ "access_token": {
11
+ "type": "string",
12
+ "title": "Access Token",
13
+ "order": 0
14
+ },
15
+ "ad_account_id": {
16
+ "type": "string",
17
+ "title": "Ad Account Id",
18
+ "order": 1
19
+ },
20
+ "audience_id": {
21
+ "type": "string",
22
+ "title": "Audience Id",
23
+ "order": 2
24
+ }
25
+ }
26
+ }
27
+ }