multiwoven-integrations 0.1.34 → 0.1.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32abb3bef8987a8a76828570507e207089efb5c7c59dd21093774beeef21a265
-  data.tar.gz: 9920eb296b386ef3add818a1e7b59ef26e7e8c32aee8a21e8295b82deebc4d44
+  metadata.gz: 154bbe9f4e36d95f4cd7fdb8094077e2ff724a3eef95ec76264950747d619fe2
+  data.tar.gz: b3da4210a997a0143b637a32d4d4b0dc6ba741742e5a1f73a3a465c8187479de
 SHA512:
-  metadata.gz: 33846b75d3a91aedf93fa367d0cd2385f7dd06066015a782cd149dd1cedefd055f29c9c007062988b13a18b9eb4efac1afd250a4b47531c1fcfd5d0b0f5619c9
-  data.tar.gz: 6faa53d0929c0d5ae12da7467e56573e86e1ff3b1898f96762150440a19fff351796198e2fe03f72a12a7008762391ac4714a3f5d5a9703602e6acaf328d27ba
+  metadata.gz: '052295ce22be1a08561f2d907bb1974c465302f2ec12cd6ae50afa976354bc13eb9ac0d178cd1b9eca86945cb18c4cb316ef05a44e6deac14ccc49f03fb57029'
+  data.tar.gz: 35e541a5b3ea70f9761ff0c54b527a6334fcc91d5321018d4ce926163f8611eb4a4db714a2b36ad01c8e869854544ec63142bef2e2e99bd7cc0d3d2e022ebe2d

data/lib/multiwoven/integrations/core/constants.rb

@@ -18,6 +18,8 @@ module Multiwoven
       SNOWFLAKE_DRIVER_PATH = ENV["SNOWFLAKE_DRIVER_PATH"] || SNOWFLAKE_MAC_DRIVER_PATH
       DATABRICKS_DRIVER_PATH = ENV["DATABRICKS_DRIVER_PATH"] || DATABRICKS_MAC_DRIVER_PATH
 
+      JSON_SCHEMA_URL = "https://json-schema.org/draft-07/schema#"
+
       # CONNECTORS
       KLAVIYO_AUTH_ENDPOINT = "https://a.klaviyo.com/api/lists/"
       KLAVIYO_AUTH_PAYLOAD = {
@@ -31,14 +33,16 @@ module Multiwoven
 
       FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS = "https://graph.facebook.com/v18.0/me/adaccounts?fields=id,name"
 
+      AIRTABLE_URL_BASE = "https://api.airtable.com/v0/"
+      AIRTABLE_BASES_ENDPOINT = "https://api.airtable.com/v0/meta/bases"
+      AIRTABLE_GET_BASE_SCHEMA_ENDPOINT = "https://api.airtable.com/v0/meta/bases/{baseId}/tables"
+
       # HTTP
       HTTP_GET = "GET"
       HTTP_POST = "POST"
       HTTP_PUT = "PUT"
       HTTP_DELETE = "DELETE"
 
-      JSON_SCHEMA_URL = "http://json-schema.org/draft-07/schema#"
-
       # google sheets
       GOOGLE_SHEETS_SCOPE = "https://www.googleapis.com/auth/drive"
       GOOGLE_SPREADSHEET_ID_REGEX = %r{/d/([-\w]{20,})/}.freeze

data/lib/multiwoven/integrations/destination/airtable/client.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require_relative "schema_helper"
+module Multiwoven
+  module Integrations
+    module Destination
+      module Airtable
+        include Multiwoven::Integrations::Core
+        class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+          MAX_CHUNK_SIZE = 10
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            bases = Multiwoven::Integrations::Core::HttpClient.request(
+              AIRTABLE_BASES_ENDPOINT,
+              HTTP_GET,
+              headers: auth_headers(connection_config[:api_key])
+            )
+            if success?(bases)
+              base_id_exists?(bases, connection_config[:base_id])
+              success_status
+            else
+              failure_status(nil)
+            end
+          rescue StandardError => e
+            failure_status(e)
+          end
+
+          def discover(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            base_id = connection_config[:base_id]
+            api_key = connection_config[:api_key]
+
+            bases = Multiwoven::Integrations::Core::HttpClient.request(
+              AIRTABLE_BASES_ENDPOINT,
+              HTTP_GET,
+              headers: auth_headers(api_key)
+            )
+
+            base = extract_bases(bases).find { |b| b["id"] == base_id }
+            base_name = base["name"]
+
+            schema = Multiwoven::Integrations::Core::HttpClient.request(
+              AIRTABLE_GET_BASE_SCHEMA_ENDPOINT.gsub("{baseId}", base_id),
+              HTTP_GET,
+              headers: auth_headers(api_key)
+            )
+
+            catalog = build_catalog_from_schema(extract_body(schema), base_id, base_name)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception("AIRTABLE:DISCOVER:EXCEPTION", "error", e)
+          end
+
+          def write(sync_config, records, _action = "create")
+            connection_config = sync_config.destination.connection_specification.with_indifferent_access
+            api_key = connection_config[:api_key]
+            url = sync_config.stream.url
+            write_success = 0
+            write_failure = 0
+            records.each_slice(MAX_CHUNK_SIZE) do |chunk|
+              payload = create_payload(chunk)
+              response = Multiwoven::Integrations::Core::HttpClient.request(
+                url,
+                sync_config.stream.request_method,
+                payload: payload,
+                headers: auth_headers(api_key)
+              )
+              if success?(response)
+                write_success += chunk.size
+              else
+                write_failure += chunk.size
+              end
+            rescue StandardError => e
+              handle_exception("AIRTABLE:RECORD:WRITE:EXCEPTION", "error", e)
+              write_failure += chunk.size
+            end
+
+            tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
+              success: write_success,
+              failed: write_failure
+            )
+            tracker.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception("AIRTABLE:WRITE:EXCEPTION", "error", e)
+          end
+
+          private
+
+          def create_payload(records)
+            {
+              "records" => records.map do |record|
+                {
+                  "fields" => record
+                }
+              end
+            }
+          end
+
+          def auth_headers(access_token)
+            {
+              "Accept" => "application/json",
+              "Authorization" => "Bearer #{access_token}",
+              "Content-Type" => "application/json"
+            }
+          end
+
+          def base_id_exists?(bases, base_id)
+            return if extract_data(bases).any? { |base| base["id"] == base_id }
+
+            raise ArgumentError, "base_id not found"
+          end
+
+          def extract_bases(response)
+            response_body = extract_body(response)
+            response_body["bases"] if response_body
+          end
+
+          def extract_body(response)
+            response_body = response.body
+            JSON.parse(response_body) if response_body
+          end
+
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
+          end
+
+          def create_stream(table, base_id, base_name)
+            {
+              name: "#{base_name}/#{SchemaHelper.clean_name(table["name"])}",
+              action: "create",
+              method: HTTP_POST,
+              url: "#{AIRTABLE_URL_BASE}#{base_id}/#{table["id"]}",
+              json_schema: SchemaHelper.get_json_schema(table),
+              supported_sync_modes: %w[incremental],
+              batch_support: true,
+              batch_size: 10
+
+            }.with_indifferent_access
+          end
+
+          def build_catalog_from_schema(schema, base_id, base_name)
+            catalog = build_catalog(load_catalog)
+            schema["tables"].each do |table|
+              catalog.streams << build_stream(create_stream(table, base_id, base_name))
+            end
+            catalog
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/airtable/config/catalog.json

@@ -0,0 +1,6 @@
+{
+  "request_rate_limit": 300,
+  "request_rate_limit_unit": "minute",
+  "request_rate_concurrency": 10,
+  "streams": []
+}

data/lib/multiwoven/integrations/destination/airtable/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data": {
+    "name": "Airtable",
+    "title": "airtable",
+    "connector_type": "destination",
+    "category": "Productivity Tools",
+    "documentation_url": "https://docs.multiwoven.com/destinations/productivity-tools/airtable",
+    "github_issue_label": "destination-airtable",
+    "icon": "icon.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/destination/airtable/config/spec.json

@@ -0,0 +1,22 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/destination/airtable",
+  "stream_type": "dynamic",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Airtable",
+    "type": "object",
+    "required": ["api_key", "base_id"],
+    "properties": {
+      "api_key": {
+        "type": "string",
+        "title": "Personal access token",
+        "order": 0
+      },
+      "base_id": {
+        "type": "string",
+        "title": "Airtable base id",
+        "order": 1
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/destination/airtable/icon.svg

@@ -0,0 +1,6 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="32" height="27" shape-rendering="geometricPrecision" viewBox="0 0 200 170">
+  <path fill="#FCB400" d="M90.039 12.367L24.079 39.66c-3.667 1.519-3.63 6.729.062 8.192l66.235 26.266a24.575 24.575 0 0018.12 0l66.236-26.266c3.69-1.463 3.729-6.673.06-8.191l-65.958-27.294a24.578 24.578 0 00-18.795 0"></path>
+  <path fill="#18BFFF" d="M105.312 88.46v65.617c0 3.12 3.147 5.258 6.048 4.108l73.806-28.648a4.418 4.418 0 002.79-4.108V59.813c0-3.121-3.147-5.258-6.048-4.108l-73.806 28.648a4.42 4.42 0 00-2.79 4.108"></path>
+  <path fill="#F82B60" d="M88.078 91.846l-21.904 10.576-2.224 1.075-46.238 22.155c-2.93 1.414-6.672-.722-6.672-3.978V60.088c0-1.178.604-2.195 1.414-2.96a5.024 5.024 0 011.12-.84c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
+  <path fill="rgba(0, 0, 0, 0.25)" d="M88.078 91.846l-21.904 10.576-53.72-45.295a5.024 5.024 0 011.12-.839c1.104-.663 2.68-.84 4.02-.31L87.71 83.76c3.564 1.414 3.844 6.408.368 8.087"></path>
+</svg>

data/lib/multiwoven/integrations/destination/airtable/schema_helper.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module Airtable
+        module SchemaHelper
+          include Core::Constants
+
+          module_function
+
+          def clean_name(name_str)
+            name_str.strip.gsub(" ", "_")
+          end
+
+          def get_json_schema(table)
+            fields = table["fields"] || {}
+            properties = fields.each_with_object({}) do |field, props|
+              name, schema = process_field(field)
+              props[name] = schema
+            end
+
+            build_schema(properties)
+          end
+
+          def process_field(field)
+            name = clean_name(field.fetch("name", ""))
+            original_type = field.fetch("type", "")
+            options = field.fetch("options", {})
+
+            schema = determine_schema(original_type, options)
+            [name, schema]
+          end
+
+          def determine_schema(original_type, options)
+            if COMPLEX_AIRTABLE_TYPES.keys.include?(original_type)
+              complex_type = deep_copy(COMPLEX_AIRTABLE_TYPES[original_type])
+              adjust_complex_type(original_type, complex_type, options)
+            elsif SIMPLE_AIRTABLE_TYPES.keys.include?(original_type)
+              simple_type_schema(original_type, options)
+            else
+              SCHEMA_TYPES[:STRING]
+            end
+          end
+
+          def adjust_complex_type(original_type, complex_type, options)
+            exec_type = options.dig("result", "type") || "simpleText"
+            if complex_type == SCHEMA_TYPES[:ARRAY_WITH_ANY]
+              adjust_array_with_any(original_type, complex_type, exec_type, options)
+            else
+              complex_type
+            end
+          end
+
+          def adjust_array_with_any(original_type, complex_type, exec_type, options)
+            if original_type == "formula" && %w[number currency percent duration].include?(exec_type)
+              complex_type = SCHEMA_TYPES[:NUMBER]
+            elsif original_type == "formula" && ARRAY_FORMULAS.none? { |x| options.fetch("formula", "").start_with?(x) }
+              complex_type = SCHEMA_TYPES[:STRING]
+            elsif SIMPLE_AIRTABLE_TYPES.keys.include?(exec_type)
+              complex_type["items"] = deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
+            else
+              complex_type["items"] = SCHEMA_TYPES[:STRING]
+            end
+            complex_type
+          end
+
+          def simple_type_schema(original_type, options)
+            exec_type = options.dig("result", "type") || original_type
+            deep_copy(SIMPLE_AIRTABLE_TYPES[exec_type])
+          end
+
+          def build_schema(properties)
+            {
+              "$schema" => JSON_SCHEMA_URL,
+              "type" => "object",
+              "additionalProperties" => true,
+              "properties" => properties
+            }
+          end
+
+          def deep_copy(object)
+            Marshal.load(Marshal.dump(object))
+          end
+
+          SCHEMA_TYPES = {
+            STRING: { "type": %w[null string] },
+            NUMBER: { "type": %w[null number] },
+            BOOLEAN: { "type": %w[null boolean] },
+            DATE: { "type": %w[null string], "format": "date" },
+            DATETIME: { "type": %w[null string], "format": "date-time" },
+            ARRAY_WITH_STRINGS: { "type": %w[null array], "items": { "type": %w[null string] } },
+            ARRAY_WITH_ANY: { "type": %w[null array], "items": {} }
+          }.freeze.with_indifferent_access
+
+          SIMPLE_AIRTABLE_TYPES = {
+            "multipleAttachments" => SCHEMA_TYPES[:STRING],
+            "autoNumber" => SCHEMA_TYPES[:NUMBER],
+            "barcode" => SCHEMA_TYPES[:STRING],
+            "button" => SCHEMA_TYPES[:STRING],
+            "checkbox" => :BOOLEAN,
+            "singleCollaborator" => SCHEMA_TYPES[:STRING],
+            "count" => SCHEMA_TYPES[:NUMBER],
+            "createdBy" => SCHEMA_TYPES[:STRING],
+            "createdTime" => SCHEMA_TYPES[:DATETIME],
+            "currency" => SCHEMA_TYPES[:NUMBER],
+            "email" => SCHEMA_TYPES[:STRING],
+            "date" => SCHEMA_TYPES[:DATE],
+            "dateTime" => SCHEMA_TYPES[:DATETIME],
+            "duration" => SCHEMA_TYPES[:NUMBER],
+            "lastModifiedBy" => SCHEMA_TYPES[:STRING],
+            "lastModifiedTime" => SCHEMA_TYPES[:DATETIME],
+            "multipleRecordLinks" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
+            "multilineText" => SCHEMA_TYPES[:STRING],
+            "multipleCollaborators" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
+            "multipleSelects" => SCHEMA_TYPES[:ARRAY_WITH_STRINGS],
+            "number" => SCHEMA_TYPES[:NUMBER],
+            "percent" => SCHEMA_TYPES[:NUMBER],
+            "phoneNumber" => SCHEMA_TYPES[:STRING],
+            "rating" => SCHEMA_TYPES[:NUMBER],
+            "richText" => SCHEMA_TYPES[:STRING],
+            "singleLineText" => SCHEMA_TYPES[:STRING],
+            "singleSelect" => SCHEMA_TYPES[:STRING],
+            "externalSyncSource" => SCHEMA_TYPES[:STRING],
+            "url" => SCHEMA_TYPES[:STRING],
+            "simpleText" => SCHEMA_TYPES[:STRING]
+          }.freeze
+
+          COMPLEX_AIRTABLE_TYPES = {
+            "formula" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
+            "lookup" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
+            "multipleLookupValues" => SCHEMA_TYPES[:ARRAY_WITH_ANY],
+            "rollup" => SCHEMA_TYPES[:ARRAY_WITH_ANY]
+          }.freeze.with_indifferent_access
+
+          ARRAY_FORMULAS = %w[ARRAYCOMPACT ARRAYFLATTEN ARRAYUNIQUE ARRAYSLICE].freeze
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.34"
+    VERSION = "0.1.35"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -19,6 +19,7 @@ module Multiwoven
       Slack
       Hubspot
       GoogleSheets
+      Airtable
     ].freeze
   end
 end

data/lib/multiwoven/integrations.rb

@@ -48,6 +48,7 @@ require_relative "integrations/destination/facebook_custom_audience/client"
 require_relative "integrations/destination/slack/client"
 require_relative "integrations/destination/hubspot/client"
 require_relative "integrations/destination/google_sheets/client"
+require_relative "integrations/destination/airtable/client"
 
 module Multiwoven
   module Integrations

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.34
+  version: 0.1.35
 platform: ruby
 authors:
 - Subin T P
@@ -330,6 +330,12 @@ files:
 - lib/multiwoven/integrations/core/rate_limiter.rb
 - lib/multiwoven/integrations/core/source_connector.rb
 - lib/multiwoven/integrations/core/utils.rb
+- lib/multiwoven/integrations/destination/airtable/client.rb
+- lib/multiwoven/integrations/destination/airtable/config/catalog.json
+- lib/multiwoven/integrations/destination/airtable/config/meta.json
+- lib/multiwoven/integrations/destination/airtable/config/spec.json
+- lib/multiwoven/integrations/destination/airtable/icon.svg
+- lib/multiwoven/integrations/destination/airtable/schema_helper.rb
 - lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb
 - lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json
 - lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json