multiwoven-integrations 0.1.1 → 0.1.6

data/lib/multiwoven/integrations/destination/salesforce_crm/config/catalog.json

@@ -0,0 +1,317 @@
+{
+  "streams": [
+    {
+      "name": "Account",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": true,
+        "properties": {
+          "Id": {
+            "type": "string"
+          },
+          "IsDeleted": {
+            "type": "boolean"
+          },
+          "MasterRecordId": {
+            "type": ["string", "null"]
+          },
+          "Name": {
+            "type": "string"
+          },
+          "Type": {
+            "type": ["string", "null"],
+            "enum": [
+              "Prospect",
+              "Customer - Direct",
+              "Customer - Channel",
+              "Channel Partner / Reseller",
+              "Installation Partner",
+              "Technology Partner",
+              "Other"
+            ]
+          },
+          "ParentId": {
+            "type": ["string", "null"]
+          },
+          "BillingStreet": {
+            "type": ["string", "null"]
+          },
+          "BillingCity": {
+            "type": ["string", "null"]
+          },
+          "BillingState": {
+            "type": ["string", "null"]
+          },
+          "BillingPostalCode": {
+            "type": ["string", "null"]
+          },
+          "BillingCountry": {
+            "type": ["string", "null"]
+          },
+          "BillingLatitude": {
+            "type": ["number", "null"]
+          },
+          "BillingLongitude": {
+            "type": ["number", "null"]
+          },
+          "BillingGeocodeAccuracy": {
+            "type": ["string", "null"],
+            "enum": [
+              "Address",
+              "NearAddress",
+              "Block",
+              "Street",
+              "ExtendedZip",
+              "Zip",
+              "Neighborhood",
+              "City",
+              "County",
+              "State",
+              "Unknown"
+            ]
+          },
+          "BillingAddress": {
+            "type": ["string", "null"],
+            "enum": [
+              "Address",
+              "NearAddress",
+              "Block",
+              "Street",
+              "ExtendedZip",
+              "Zip",
+              "Neighborhood",
+              "City",
+              "County",
+              "State",
+              "Unknown"
+            ]
+          },
+          "ShippingStreet": {
+            "type": ["string", "null"]
+          },
+          "ShippingCity": {
+            "type": ["string", "null"]
+          },
+          "ShippingState": {
+            "type": ["string", "null"]
+          },
+          "ShippingPostalCode": {
+            "type": ["string", "null"]
+          },
+          "ShippingCountry": {
+            "type": ["string", "null"]
+          },
+          "ShippingLatitude": {
+            "type": ["number", "null"]
+          },
+          "ShippingLongitude": {
+            "type": ["number", "null"]
+          },
+          "ShippingGeocodeAccuracy": {
+            "type": ["string", "null"]
+          },
+          "ShippingAddress": {
+            "type": ["string", "null"]
+          },
+          "Phone": {
+            "type": ["string", "null"]
+          },
+          "Fax": {
+            "type": ["string", "null"]
+          },
+          "AccountNumber": {
+            "type": ["string", "null"]
+          },
+          "Website": {
+            "type": ["string", "null"],
+            "format": "uri"
+          },
+          "PhotoUrl": {
+            "type": ["string", "null"],
+            "format": "uri"
+          },
+          "Sic": {
+            "type": ["string", "null"]
+          },
+          "Industry": {
+            "type": ["string", "null"],
+            "enum": [
+              "Agriculture",
+              "Apparel",
+              "Banking",
+              "Biotechnology",
+              "Chemicals",
+              "Communications",
+              "Construction",
+              "Consulting",
+              "Education",
+              "Electronics",
+              "Energy",
+              "Engineering",
+              "Entertainment",
+              "Environmental",
+              "Finance",
+              "Food & Beverage",
+              "Government",
+              "Healthcare",
+              "Hospitality",
+              "Insurance",
+              "Machinery",
+              "Manufacturing",
+              "Media",
+              "Not For Profit",
+              "Recreation",
+              "Retail",
+              "Shipping",
+              "Technology",
+              "Telecommunications",
+              "Transportation",
+              "Utilities",
+              "Other"
+            ]
+          },
+          "AnnualRevenue": {
+            "type": ["number", "null"]
+          },
+          "NumberOfEmployees": {
+            "type": ["integer", "null"]
+          },
+          "Ownership": {
+            "type": ["string", "null"],
+            "enum": ["Public", "Private", "Subsidiary", "Other"]
+          },
+          "TickerSymbol": {
+            "type": ["string", "null"]
+          },
+          "Description": {
+            "type": ["string", "null"]
+          },
+          "Rating": {
+            "type": ["string", "null"],
+            "enum": ["Hot", "Warm", "Cold"]
+          },
+          "Site": {
+            "type": ["string", "null"]
+          },
+          "OwnerId": {
+            "type": "string"
+          },
+          "CreatedDate": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "CreatedById": {
+            "type": "string"
+          },
+          "LastModifiedDate": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "LastModifiedById": {
+            "type": "string"
+          },
+          "SystemModstamp": {
+            "type": "string",
+            "format": "date-time"
+          },
+          "LastActivityDate": {
+            "type": ["string", "null"],
+            "format": "date"
+          },
+          "LastViewedDate": {
+            "type": ["string", "null"],
+            "format": "date-time"
+          },
+          "LastReferencedDate": {
+            "type": ["string", "null"],
+            "format": "date-time"
+          },
+          "Jigsaw": {
+            "type": ["string", "null"]
+          },
+          "JigsawCompanyId": {
+            "type": ["string", "null"]
+          },
+          "CleanStatus": {
+            "type": ["string", "null"],
+            "enum": [
+              "In Sync",
+              "Different",
+              "Reviewed",
+              "Not Found",
+              "Inactive",
+              "Not Compared",
+              "Select Match",
+              "Skipped"
+            ]
+          },
+          "AccountSource": {
+            "type": ["string", "null"],
+            "enum": [
+              "Web",
+              "Phone Inquiry",
+              "Partner Referral",
+              "Purchased List",
+              "Other"
+            ]
+          },
+          "DunsNumber": {
+            "type": ["string", "null"]
+          },
+          "Tradestyle": {
+            "type": ["string", "null"]
+          },
+          "NaicsCode": {
+            "type": ["string", "null"]
+          },
+          "NaicsDesc": {
+            "type": ["string", "null"]
+          },
+          "YearStarted": {
+            "type": ["string", "null"]
+          },
+          "SicDesc": {
+            "type": ["string", "null"]
+          },
+          "DandbCompanyId": {
+            "type": ["string", "null"]
+          },
+          "OperatingHoursId": {
+            "type": ["string", "null"]
+          },
+          "CustomerPriority__c": {
+            "type": ["string", "null"],
+            "enum": ["High", "Low", "Medium"]
+          },
+          "SLA__c": {
+            "type": ["string", "null"],
+            "enum": ["Gold", "Silver", "Platinum", "Bronze"]
+          },
+          "Active__c": {
+            "type": ["string", "null"],
+            "enum": ["Yes", "No"]
+          },
+          "NumberofLocations__c": {
+            "type": ["number", "null"]
+          },
+          "UpsellOpportunity__c": {
+            "type": ["string", "null"],
+            "enum": ["Yes", "No", "Maybe"]
+          },
+          "SLASerialNumber__c": {
+            "type": ["string", "null"]
+          },
+          "SLAExpirationDate__c": {
+            "type": ["string", "null"],
+            "format": "date"
+          }
+        }
+      },
+      "supported_sync_modes": ["full_refresh", "incremental"],
+      "source_defined_cursor": true,
+      "default_cursor_field": ["updated"],
+      "source_defined_primary_key": [["Id"]]
+    }
+  ]
+}

data/lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json

@@ -0,0 +1,15 @@
+{
+  "data":
+    {
+      "name": "Salesforce CRM",
+      "connector_type": "destination",
+      "category": "CRM",
+      "documentation_url": "https://docs.mutliwoven.com",
+      "github_issue_label": "destination-salesforce-crm",
+      "icon": "salesforce.svg",
+      "license": "MIT",
+      "release_stage": "alpha",
+      "support_level": "community",
+      "tags": ["language:ruby", "multiwoven"]
+    }
+}

data/lib/multiwoven/integrations/destination/salesforce_crm/config/spec.json

@@ -0,0 +1,37 @@
+{
+    "documentation_url": "https://docs.multiwoven.com/integrations/destination/salesforce_crm",
+    "stream_type": "static",
+    "connection_specification": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "title": "Salesforce Destination Spec",
+      "type": "object",
+      "required": ["access_token", "refresh_token", "instance_url", "client_id", "client_secret"],
+      "properties": {
+        "access_token": {
+          "type": "string",
+          "title": "Access Token",
+          "order": 0
+        },
+        "refresh_token": {
+          "type": "string",
+          "title": "Refresh Token",
+          "order": 1
+        },
+        "instance_url": {
+          "type": "string",
+          "title": "Instance URL",
+          "order": 2
+        },
+        "client_id": {
+          "type": "string",
+          "title": "Client ID",
+          "order": 3
+        },
+        "client_secret": {
+          "type": "string",
+          "title": "Client Secret",
+          "order": 4
+        }
+      }
+    }
+  }

data/lib/multiwoven/integrations/destination/slack/client.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module Slack
+        include Multiwoven::Integrations::Core
+
+        class Client < DestinationConnector
+          attr_accessor :channel_id
+
+          def check_connection(connection_config)
+            configure_slack(connection_config[:api_token])
+            client = ::Slack::Web::Client.new
+            client.auth_test
+            success_status
+          rescue StandardError => e
+            failure_status(e)
+          end
+
+          def discover(_connection_config = nil)
+            catalog = build_catalog(load_catalog_streams)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception("SLACK:DISCOVER:EXCEPTION", "error", e)
+          end
+
+          def write(sync_config, records, action = "create")
+            # Currently as we only create a message for each record in slack, we are not using actions.
+            # This will be changed in future.
+
+            @action = sync_config.stream.action || action
+            connection_config = sync_config.destination.connection_specification.with_indifferent_access
+            configure_slack(connection_config[:api_token])
+            @client = ::Slack::Web::Client.new
+            @channel_id = connection_config[:channel_id]
+            process_records(records, sync_config.stream)
+          rescue StandardError => e
+            handle_exception("SLACK:WRITE:EXCEPTION", "error", e)
+          end
+
+          private
+
+          def configure_slack(api_token)
+            ::Slack.configure do |config|
+              config.token = api_token
+            end
+          end
+
+          def process_records(records, stream)
+            write_success = 0
+            write_failure = 0
+            records.each do |record_object|
+              process_record(stream, record_object.with_indifferent_access)
+              write_success += 1
+            rescue StandardError => e
+              write_failure += 1
+              handle_exception("SLACK:CRM:WRITE:EXCEPTION", "error", e)
+            end
+            tracking_message(write_success, write_failure)
+          end
+
+          def process_record(stream, record)
+            send_data_to_slack(stream[:name], record)
+          end
+
+          def send_data_to_slack(stream_name, record = {})
+            args = build_args(stream_name, record)
+            @client.send(stream_name, **args)
+          end
+
+          def build_args(stream_name, record)
+            case stream_name
+            when "chat_postMessage"
+              { channel: channel_id, text: slack_code_block(record[:data]) }
+            else
+              raise "Stream name not found: #{stream_name}"
+            end
+          end
+
+          def slack_code_block(data)
+            longest_key = data.keys.map(&:to_s).max_by(&:length).length
+            table_str = "```\n"
+            data.each do |key, value|
+              table_str += "#{key.to_s.ljust(longest_key)} : #{value}\n"
+            end
+            table_str += "```"
+
+            table_str
+          end
+
+          def success_status
+            ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+          end
+
+          def failure_status(error)
+            ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
+          end
+
+          def load_catalog_streams
+            catalog_json = read_json(CATALOG_SPEC_PATH)
+            catalog_json["streams"].map { |stream| build_stream(stream) }
+          end
+
+          def build_stream(stream)
+            Multiwoven::Integrations::Protocol::Stream.new(
+              name: stream["name"], json_schema: stream["json_schema"],
+              action: stream["action"]
+            )
+          end
+
+          def build_catalog(streams)
+            Multiwoven::Integrations::Protocol::Catalog.new(streams: streams)
+          end
+
+          def tracking_message(success, failure)
+            Multiwoven::Integrations::Protocol::TrackingMessage.new(
+              success: success, failed: failure
+            ).to_multiwoven_message
+          end
+        end
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/slack/config/catalog.json

@@ -0,0 +1,71 @@
+{
+  "streams": [
+    {
+      "name": "chat_postMessage",
+      "action": "create",
+      "json_schema": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "text": {
+            "type": ["string", "null"]
+          },
+          "attachments": {
+            "type": ["string", "null"]
+          },
+          "blocks": {
+            "type": ["array", "null"],
+            "items": {
+              "type": "string"
+            }
+          },
+          "as_user": {
+            "type": ["boolean", "null"]
+          },
+          "icon_emoji": {
+            "type": ["string", "null"]
+          },
+          "icon_url": {
+            "type": ["string", "null"]
+          },
+          "link_names": {
+            "type": ["boolean", "null"]
+          },
+          "metadata": {
+            "type": ["string", "null"]
+          },
+          "mrkdwn": {
+            "type": ["boolean", "null"]
+          },
+          "parse": {
+            "type": ["string", "null"]
+          },
+          "reply_broadcast": {
+            "type": ["boolean", "null"]
+          },
+          "thread_ts": {
+            "type": ["string", "null"]
+          },
+          "unfurl_links": {
+            "type": ["boolean", "null"]
+          },
+          "unfurl_media": {
+            "type": ["boolean", "null"]
+          },
+          "username": {
+            "type": ["string", "null"]
+          }
+        },
+        "oneOf": [
+          { "required": ["text"] },
+          { "required": ["attachments"] },
+          { "required": ["blocks"] }
+        ]
+      },
+      "supported_sync_modes": ["full_refresh", "incremental"],
+      "source_defined_cursor": true,
+      "default_cursor_field": ["updated"],
+      "source_defined_primary_key": [["Id"]]
+    }
+  ]
+}

data/lib/multiwoven/integrations/destination/slack/config/meta.json

@@ -0,0 +1,14 @@
+{
+  "data": {
+    "name": "Slack",
+    "connector_type": "destination",
+    "category": "Team Collaboration",
+    "documentation_url": "https://docs.mutliwoven.com",
+    "github_issue_label": "destination-slack",
+    "icon": "slack.svg",
+    "license": "MIT",
+    "release_stage": "alpha",
+    "support_level": "community",
+    "tags": ["language:ruby", "multiwoven"]
+  }
+}

data/lib/multiwoven/integrations/destination/slack/config/spec.json

@@ -0,0 +1,22 @@
+{
+  "documentation_url": "https://docs.multiwoven.com/integrations/destination/slack",
+  "stream_type": "static",
+  "connection_specification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Slack Destination Spec",
+    "type": "object",
+    "required": ["api_token"],
+    "properties": {
+      "api_token": {
+        "type": "string",
+        "title": "API Token",
+        "order": 0
+      },
+      "channel_id": {
+        "type": "string",
+        "title": "Channel ID",
+        "order": 1
+      }
+    }
+  }
+}

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -130,6 +130,8 @@ module Multiwoven
     end
 
     class SyncConfig < ProtocolModel
+      attr_accessor :offset, :limit
+
       attribute :source, Connector
       attribute :destination, Connector
       attribute :model, Model

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.1"
+    VERSION = "0.1.6"
 
     ENABLED_SOURCES = %w[
       Snowflake
@@ -12,6 +12,9 @@ module Multiwoven
 
     ENABLED_DESTINATIONS = %w[
       Klaviyo
+      SalesforceCrm
+      FacebookCustomAudience
+      Slack
     ].freeze
   end
 end

data/lib/multiwoven/integrations/source/bigquery/client.rb

@@ -7,6 +7,7 @@ module Multiwoven::Integrations::Source
     include Multiwoven::Integrations::Core
     class Client < SourceConnector
       def check_connection(connection_config)
+        connection_config = connection_config.with_indifferent_access
         bigquery = create_connection(connection_config)
         bigquery.datasets
         ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
@@ -15,6 +16,7 @@ module Multiwoven::Integrations::Source
       end
 
       def discover(connection_config)
+        connection_config = connection_config.with_indifferent_access
         bigquery = create_connection(connection_config)
         target_dataset_id = connection_config["dataset_id"]
         records = bigquery.datasets.flat_map do |dataset|
@@ -43,7 +45,11 @@ module Multiwoven::Integrations::Source
 
       def read(sync_config)
         connection_config = sync_config.source.connection_specification
+        connection_config = connection_config.with_indifferent_access
         query = sync_config.model.query
+
+        query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
+
         bigquery = create_connection(connection_config)
         records = []
         results = bigquery.query query

data/lib/multiwoven/integrations/source/bigquery/config/meta.json

@@ -3,7 +3,7 @@
     {
       "name": "BigQuery",
       "connector_type": "source",
-      "connector_subtype": "database",
+      "category": "Data Warehouse",
       "documentation_url": "https://docs.mutliwoven.com",
       "github_issue_label": "source-bigquery",
       "icon": "bigquery.svg",