multiwoven-integrations 0.1.1 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e33f303a54e624644de0dbaa80056632e61c7834f0b192ba2e041e6c3d131e6
-  data.tar.gz: dd3e29285e988665a08b6e71b1986810af068916bd2ebdd4f74a3bacab6b1385
+  metadata.gz: 2ca3845d4b648bffe8c785fdd1cc0c206099deb7d7414700d528a25ec2fb72f9
+  data.tar.gz: 319139931a5c9a723a379c3aa116d3d6a825235fc6b80359abf6c19104fcc815
 SHA512:
-  metadata.gz: f538a941eae0f2d00d754d494d70c319f4bf44444ff4463e36010dbd53e25633b114f7399edd5a1199720eb736d20f64c8e956df4b349776e173ed1f6bc537c1
-  data.tar.gz: 56fe3d7902f215f67403d21bcb857f8d18d87f6be0082dd875489e4d60d74a753178a2436eacbc4405f7c3baa8b1fbf267bfe3ab09cc9a292137c413e5432bd3
+  metadata.gz: f5a96c27ef09fb593a0f000bbefa835fb79ec2c7876a7d84635013f757d8a31a52f7f5f32b791a9539dc6494af0092faf28ca8c13cccc1b4465e94eeba49490a
+  data.tar.gz: a9ddefdc6259d2d160ea118e1571ed1facc8cb39d533f490692e085a12d160b2d8ec8138ab1801353c0e31eb172111ffa51d1c49af9b1b54c9a1645b7a9315f1

data/.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+  "cSpell.words": [
+    "multiwoven"
+  ]
+}

data/lib/multiwoven/integrations/core/base_connector.rb

@@ -30,10 +30,10 @@ module Multiwoven
       private
 
       def read_json(file_path)
-        # TODO: Rethink. Hack to find connector folder
-        connector_folder = self.class.to_s.split("::")[2..3].map(&:downcase).join("/")
+        path = Object.const_source_location(self.class.to_s)[0]
+        connector_folder = File.dirname(path)
         file_path = File.join(
-          INTEGRATIONS_PATH, "#{connector_folder}/",
+          "#{connector_folder}/",
           file_path
         )
         file_contents = File.read(file_path)

data/lib/multiwoven/integrations/core/constants.rb

@@ -24,6 +24,8 @@ module Multiwoven
         }
       }.freeze
 
+      FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS = "https://graph.facebook.com/v18.0/me/adaccounts?fields=id,name"
+
       # HTTP
       HTTP_GET = "GET"
       HTTP_POST = "POST"

data/lib/multiwoven/integrations/core/source_connector.rb

@@ -7,6 +7,23 @@ module Multiwoven
         raise "Not implemented"
         # return list of RecordMessage
       end
+
+      private
+
+      def batched_query(sql_query, limit, offset)
+        offset = offset.to_i
+        limit = limit.to_i
+        raise ArgumentError, "Offset and limit must be non-negative" if offset.negative? || limit.negative?
+
+        # Removing any trailing semicolons
+        sql_query.chomp!(";")
+
+        # Checking if the query already has a LIMIT clause
+        raise ArgumentError, "Query already contains a LIMIT clause" if sql_query.match?(/LIMIT \d+/i)
+
+        # Appending the LIMIT and OFFSET clauses to the SQL query
+        "#{sql_query} LIMIT #{limit} OFFSET #{offset}"
+      end
     end
   end
 end

data/lib/multiwoven/integrations/core/utils.rb

@@ -63,6 +63,14 @@ module Multiwoven
 
         create_log_message(context, type, exception)
       end
+
+      def extract_data(record_object, properties)
+        record_object.data.select { |key, _| properties.key?(key.to_s) }
+      end
+
+      def success?(response)
+        response && %w[200 201].include?(response.code.to_s)
+      end
     end
   end
 end

data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Multiwoven::Integrations::Destination
+  module FacebookCustomAudience
+    include Multiwoven::Integrations::Core
+    class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+      def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
+        access_token = connection_config[:access_token]
+        response = Multiwoven::Integrations::Core::HttpClient.request(
+          FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS,
+          HTTP_GET,
+          headers: auth_headers(access_token)
+        )
+        if success?(response)
+          ad_account_exists?(response, connection_config[:ad_account_id])
+          ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+        else
+          ConnectionStatus.new(status: ConnectionStatusType["failed"]).to_multiwoven_message
+        end
+      rescue StandardError => e
+        ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
+      end
+
+      def discover(_connection_config = nil)
+        catalog_json = read_json(CATALOG_SPEC_PATH)
+
+        streams = catalog_json["streams"].map do |stream|
+          Multiwoven::Integrations::Protocol::Stream.new(
+            audience_id: stream["audience_id"],
+            url: stream["url"],
+            name: stream["name"],
+            json_schema: stream["json_schema"],
+            request_method: stream["method"],
+            action: stream["action"]
+          )
+        end
+
+        catalog = Multiwoven::Integrations::Protocol::Catalog.new(
+          streams: streams
+        )
+
+        catalog.to_multiwoven_message
+      rescue StandardError => e
+        handle_exception(
+          "FACEBOOK AUDIENCE:DISCOVER:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def write(sync_config, records, _action = "insert")
+        url = sync_config.stream.url
+        connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        connection_config = connection_config.with_indifferent_access
+        access_token = connection_config[:access_token]
+
+        write_success = 0
+        write_failure = 0
+        records.each do |record|
+          schema, data = extract_schema_and_data(record.with_indifferent_access[:data])
+          payload = {
+            "payload" => {
+              "schema" => schema,
+              "data" => [data]
+            }
+          }
+          response = Multiwoven::Integrations::Core::HttpClient.request(
+            url,
+            sync_config.stream.request_method,
+            payload: payload,
+            headers: auth_headers(access_token)
+          )
+          if success?(response)
+            write_success += 1
+          else
+            write_failure += 1
+          end
+        rescue StandardError => e
+          logger.error(
+            "FACEBOOK:RECORD:WRITE:FAILURE: #{e.message}"
+          )
+          write_failure += 1
+        end
+        tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
+          success: write_success,
+          failed: write_failure
+        )
+        tracker.to_multiwoven_message
+      rescue StandardError => e
+        # TODO: Handle rate limiting seperately
+        handle_exception(
+          "FACEBOOK:WRITE:EXCEPTION",
+          "error",
+          e
+        )
+      end
+
+      def extract_schema_and_data(data)
+        [data.keys, data.values]
+      end
+
+      def auth_headers(access_token)
+        {
+          "Accept" => "application/json",
+          "Authorization" => "Bearer #{access_token}",
+          "Content-Type" => "application/json"
+        }
+      end
+
+      def ad_account_exists?(response, ad_account_id)
+        return if extract_data(response).any? { |ad_account| ad_account["id"] == "act_#{ad_account_id}" }
+
+        raise ArgumentError, "Ad account not found in business account"
+      end
+
+      def extract_data(response)
+        response_body = response.body
+        JSON.parse(response_body)["data"] if response_body
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json

@@ -0,0 +1,38 @@
+{
+  "streams": [
+    {
+      "name": "audience",
+      "action": "create",
+      "method": "POST",
+      "audience_id": "audience_id",
+      "url": "https://graph.facebook.com/v18.0/{audience_id}/users",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "properties": {
+          "CT": { "type": ["string", "null"], "default": null },
+          "COUNTRY": { "type": ["string", "null"], "default": null },
+          "DOBD": { "type": ["string", "null"], "default": null },
+          "DOBM": { "type": ["string", "null"], "default": null },
+          "DOBY": { "type": ["string", "null"], "default": null },
+          "EMAIL": { "type": ["string", "null"], "default": null },
+          "EXTERN_ID": { "type": ["string", "null"], "default": null },
+          "FI": { "type": ["string", "null"], "default": null },
+          "FN": { "type": ["string", "null"], "default": null },
+          "GEN": { "type": ["string", "null"], "default": null },
+          "LN": { "type": ["string", "null"], "default": null },
+          "LOOKALIKE_VALUE": { "type": ["number", "null"], "default": null },
+          "MADID": { "type": ["string", "null"], "default": null },
+          "PAGEUID": { "type": ["string", "null"], "default": null },
+          "PHONE": { "type": ["string", "null"], "default": null },
+          "ST": { "type": ["string", "null"], "default": null },
+          "ZIP": { "type": ["string", "null"], "default": null }
+        }
+      },
+      "supported_sync_modes": ["full_refresh", "incremental"],
+      "source_defined_cursor": true,
+      "default_cursor_field": ["updated"],
+      "source_defined_primary_key": [["email"]]
+    }
+  ]
+}

data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data":
+    {
+      "name": "Facebook",
+      "connector_type": "destination",
+      "category": "Adtech",
+      "documentation_url": "https://docs.mutliwoven.com",
+      "github_issue_label": "destination-facebook",
+      "icon": "facebook.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "multiwoven"]
+    }
+}

data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/spec.json

@@ -0,0 +1,22 @@
+{
+    "documentation_url": "https://docs.multiwoven.com/integrations/destination/facebook",
+    "stream_type": "static",
+    "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "Facebook Destination Spec",
+      "type": "object",
+      "required": ["access_token","ad_account_id"],
+      "properties": {
+        "access_token": {
+          "type": "string",
+          "title": "Access Token",
+          "order": 0
+        },
+        "ad_account_id": {
+          "type": "string",
+          "title": "Ad Account Id",
+          "order": 1
+        }
+      }
+    }
+  }

data/lib/multiwoven/integrations/destination/klaviyo/client.rb

@@ -3,8 +3,9 @@
 module Multiwoven::Integrations::Destination
   module Klaviyo
     include Multiwoven::Integrations::Core
-    class Client < DestinationConnector
+    class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
       def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
         api_key = connection_config[:private_api_key]
 
         response = Multiwoven::Integrations::Core::HttpClient.request(
@@ -16,7 +17,7 @@ module Multiwoven::Integrations::Destination
         parse_response(response)
       end
 
-      def discover
+      def discover(_connection_config = nil)
         catalog_json = read_json(CATALOG_SPEC_PATH)
 
         streams = catalog_json["streams"].map do |stream|
@@ -24,7 +25,7 @@ module Multiwoven::Integrations::Destination
             name: stream["name"],
             json_schema: stream["json_schema"],
             url: stream["url"],
-            method: stream["method"],
+            request_method: stream["method"],
             action: stream["action"]
           )
         end
@@ -44,12 +45,19 @@ module Multiwoven::Integrations::Destination
 
       def write(sync_config, records, _action = "insert")
         connection_config = sync_config.destination.connection_specification.with_indifferent_access
+        connection_config = connection_config.with_indifferent_access
         url = sync_config.stream.url
+
         request_method = sync_config.stream.request_method
 
         write_success = 0
         write_failure = 0
         records.each do |record|
+          # pre process payload
+          # Add hardcode values into payload
+          record["data"] ||= {}
+          record["data"]["type"] = sync_config.stream.name
+
           response = Multiwoven::Integrations::Core::HttpClient.request(
             url,
             request_method,

data/lib/multiwoven/integrations/destination/klaviyo/config/catalog.json

@@ -6,103 +6,101 @@
         "url": "https://a.klaviyo.com/api/profiles",
         "method": "POST",
         "json_schema": {
+          "$schema": "http://json-schema.org/draft-07/schema#",
           "type": "object",
-          "additionalProperties": true,
           "properties": {
-            "type": {
-              "type": [
-                "null",
-                "string"
-              ]
-            },
-            "id": {
-              "type": "string"
-            },
-            "updated": {
-              "type": [
-                "null",
-                "string"
-              ],
-              "format": "date-time"
-            },
-            "attributes": {
-              "type": [
-                "null",
-                "object"
-              ],
-              "additionalProperties": true,
+            "data": {
+              "type": "object",
               "properties": {
-                "email": {
-                  "type": [
-                    "null",
-                    "string"
-                  ]
-                },
-                "phone_number": {
-                  "type": [
-                    "null",
-                    "string"
-                  ]
-                },
-                "first_name": {
-                  "type": [
-                    "null",
-                    "string"
-                  ]
-                },
-                "last_name": {
-                  "type": [
-                    "null",
-                    "string"
+                "type": {
+                  "type": "string",
+                  "enum": [
+                    "profile"
                   ]
                 },
-                "properties": {
-                  "type": [
-                    "null",
-                    "object"
-                  ],
-                  "additionalProperties": true
-                },
-                "organization": {
-                  "type": [
-                    "null",
-                    "string"
-                  ]
-                },
-                "title": {
-                  "type": [
-                    "null",
-                    "string"
+                "attributes": {
+                  "type": "object",
+                  "properties": {
+                    "email": {
+                      "type": "string",
+                      "format": "email"
+                    },
+                    "phone_number": {
+                      "type": "string"
+                    },
+                    "external_id": {
+                      "type": "string",
+                      "format": "uuid"
+                    },
+                    "first_name": {
+                      "type": "string"
+                    },
+                    "last_name": {
+                      "type": "string"
+                    },
+                    "organization": {
+                      "type": "string"
+                    },
+                    "title": {
+                      "type": "string"
+                    },
+                    "image": {
+                      "type": "string",
+                      "format": "uri"
+                    },
+                    "location": {
+                      "type": "object",
+                      "properties": {
+                        "address1": {
+                          "type": "string"
+                        },
+                        "address2": {
+                          "type": "string"
+                        },
+                        "city": {
+                          "type": "string"
+                        },
+                        "country": {
+                          "type": "string"
+                        },
+                        "region": {
+                          "type": "string"
+                        },
+                        "zip": {
+                          "type": "string"
+                        },
+                        "timezone": {
+                          "type": "string"
+                        },
+                        "ip": {
+                          "type": "string",
+                          "format": "ipv4"
+                        }
+                      }
+          
+                    },
+                    "properties": {
+                      "type": "object",
+                      "additionalProperties": {
+                        "type": "string"
+                      }
+                    }
+                  },
+                  "required": [
+                    "email",
+                    "phone_number"
                   ]
-                },
-                "last_event_date": {
-                  "type": [
-                    "null",
-                    "string"
-                  ],
-                  "format": "date-time"
                 }
-              }
-            },
-            "links": {
-              "type": [
-                "null",
-                "object"
-              ]
-            },
-            "relationships": {
-              "type": [
-                "null",
-                "object"
-              ]
-            },
-            "segments": {
-              "type": [
-                "null",
-                "object"
+              },
+              "required": [
+                "type",
+                "attributes"
               ]
             }
-          }
+          },
+          "required": [
+            "data"
+          ]
         },
         "supported_sync_modes": [
           "full_refresh",

data/lib/multiwoven/integrations/destination/klaviyo/config/meta.json

@@ -3,7 +3,7 @@
     {
       "name": "Klaviyo",
       "connector_type": "destination",
-      "connector_subtype": "API",
+      "category": "Marketing Automation",
       "documentation_url": "https://docs.mutliwoven.com",
       "github_issue_label": "destination-klaviyo",
       "icon": "klaviyo.svg",

data/lib/multiwoven/integrations/destination/klaviyo/config/spec.json

@@ -7,7 +7,7 @@
       "type": "object",
       "required": ["public_api_key", "private_api_key"],
       "properties": {
-        "public_api_keyhost": {
+        "public_api_key": {
           "type": "string",
           "title": "Public API Key",
           "order": 0
@@ -19,4 +19,4 @@
         }
       }
     }
-  }
+  }

data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module SalesforceCrm
+        include Multiwoven::Integrations::Core
+
+        API_VERSION = "59.0"
+
+        class Client < DestinationConnector
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            initialize_client(connection_config)
+            authenticate_client
+            success_status
+          rescue StandardError => e
+            failure_status(e)
+          end
+
+          def discover(_connection_config = nil)
+            catalog = build_catalog(load_catalog_streams)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception("SALESFORCE:CRM:DISCOVER:EXCEPTION", "error", e)
+          end
+
+          def write(sync_config, records, _action = "create")
+            initialize_client(sync_config[:destination][:connection_specification])
+            process_records(records, sync_config[:stream])
+          rescue StandardError => e
+            handle_exception("SALESFORCE:CRM:WRITE:EXCEPTION", "error", e)
+          end
+
+          private
+
+          def initialize_client(config)
+            config = config.with_indifferent_access
+            @client = Restforce.new(oauth_token: config[:access_token],
+                                    refresh_token: config[:refresh_token],
+                                    instance_url: config[:instance_url],
+                                    client_id: config[:client_id],
+                                    client_secret: config[:client_secret],
+                                    authentication_callback: proc { |x| log_debug(x.to_s) },
+                                    api_version: API_VERSION)
+          end
+
+          def process_records(records, stream)
+            write_success = 0
+            write_failure = 0
+            properties = stream[:json_schema][:properties]
+            records.each do |record_object|
+              record = extract_data(record_object, properties)
+              process_record(stream, record)
+              write_success += 1
+            rescue StandardError => e
+              handle_exception("SALESFORCE:CRM:WRITE:EXCEPTION", "error", e)
+              write_failure += 1
+            end
+            tracking_message(write_success, write_failure)
+          end
+
+          def process_record(stream, record)
+            send_data_to_salesforce(stream[:action], stream[:name], record)
+          end
+
+          def send_data_to_salesforce(action, stream_name, record = {})
+            method_name = "#{action}!"
+            args = build_args(action, stream_name, record)
+            @client.send(method_name, *args)
+          end
+
+          def build_args(action, stream_name, record)
+            case action
+            when :upsert
+              [stream_name, record[:external_key], record]
+            when :destroy
+              [stream_name, record[:id]]
+            else
+              [stream_name, record]
+            end
+          end
+
+          def authenticate_client
+            @client.authenticate!
+          end
+
+          def success_status
+            ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+          end
+
+          def failure_status(error)
+            ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
+          end
+
+          def load_catalog_streams
+            catalog_json = read_json(CATALOG_SPEC_PATH)
+            catalog_json["streams"].map { |stream| build_stream(stream) }
+          end
+
+          def build_stream(stream)
+            Multiwoven::Integrations::Protocol::Stream.new(
+              name: stream["name"], json_schema: stream["json_schema"],
+              action: stream["action"]
+            )
+          end
+
+          def build_catalog(streams)
+            Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
+          end
+
+          def tracking_message(success, failure)
+            Multiwoven::Integrations::Protocol::TrackingMessage.new(
+              success: success, failed: failure
+            ).to_multiwoven_message
+          end
+
+          def log_debug(message)
+            Multiwoven::Integrations::Service.logger.debug(message)
+          end
+        end
+      end
+    end
+  end
+end