multiwoven-integrations 0.1.36 → 0.1.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 818c8606d1ef1b50bd14765b3ed85f687ef5e16cdd975324bfbf28ed2b371070
-  data.tar.gz: 0bd41ef2fe575ff51d5ea5878522509de3aa6ae0b3592074c15ce3de5be97c51
+  metadata.gz: be7368d653a73843a7d204b4dd7793c2ce7d5ba7f7e1e3b49818e15aff85367a
+  data.tar.gz: 21c792560111def1edad270dfcba87627818fbf4763c159053abf0114438e035
 SHA512:
-  metadata.gz: dfaa4c53e72fbf3fc64d2d746bf4fd632c19be834af900a93eb88060fc1a84156b1849b645768ca41ebd3037f205a7b4f79430738eddb7f607b526fb9424c08b
-  data.tar.gz: dd7dc877f6ad86082975cb9dbdc714bc5e9dc623196178e6c9c11b601deab967fa107d1d218d5dc03e43b6b6c580403d45bc470fd364a109a8a1cb6e8a793520
+  metadata.gz: b294140a41825ef76941958c46ec86a976e9073813b8a641663d249d518dd77d208d1c229bc20b7e76336196aa8abef7504ee92ac8a4100681b76fef05a2251e
+  data.tar.gz: f72981b9d03124e508d4cc1087ef13b876ee731d037bfaf4c4142bbc13c9a3227f248b0bdf07579f95591549c0800ce6439fdcd94a36510537589186512f9018

data/lib/multiwoven/integrations/core/destination_connector.rb

@@ -3,8 +3,6 @@
 module Multiwoven
   module Integrations::Core
     class DestinationConnector < BaseConnector
-      prepend RateLimiter
-
       # Records are transformed json payload send it to the destination
       # SyncConfig is the Protocol::SyncConfig object
       def write(_sync_config, _records, _action = "insert")

data/lib/multiwoven/integrations/core/fullrefresher.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations::Core
+    module Fullrefresher
+      def write(sync_config, records, action = "insert")
+        if sync_config && sync_config.sync_mode == "full_refresh" && !@full_refreshed
+          response = clear_all_records(sync_config)
+          return response unless response &&
+                                 response.control.status == Multiwoven::Integrations::Protocol::ConnectionStatusType["succeeded"]
+
+          @full_refreshed = true
+        end
+
+        super(sync_config, records, action)
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/airtable/client.rb

@@ -7,6 +7,7 @@ module Multiwoven
       module Airtable
         include Multiwoven::Integrations::Core
         class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+          prepend Multiwoven::Integrations::Core::RateLimiter
           MAX_CHUNK_SIZE = 10
           def check_connection(connection_config)
             connection_config = connection_config.with_indifferent_access

data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb

@@ -4,6 +4,7 @@ module Multiwoven::Integrations::Destination
   module FacebookCustomAudience
     include Multiwoven::Integrations::Core
     class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+      prepend Multiwoven::Integrations::Core::RateLimiter
       MAX_CHUNK_SIZE = 10_000
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access

data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json

@@ -33,7 +33,7 @@
           "PAGEUID": { "type": ["string", "null"], "default": null, "title": "Page-Scoped ID", "x-hashRequired": false }
         }
       },
-      "supported_sync_modes": ["full_refresh", "incremental"],
+      "supported_sync_modes": ["incremental"],
       "source_defined_cursor": true,
       "default_cursor_field": ["updated"],
       "source_defined_primary_key": [["email"]]

data/lib/multiwoven/integrations/destination/google_sheets/client.rb

@@ -7,9 +7,12 @@ module Multiwoven
         include Multiwoven::Integrations::Core
 
         class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+          prepend Multiwoven::Integrations::Core::Fullrefresher
+          prepend Multiwoven::Integrations::Core::RateLimiter
           MAX_CHUNK_SIZE = 10_000
 
           def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
             authorize_client(connection_config)
             fetch_google_spread_sheets(connection_config)
             success_status
@@ -19,6 +22,7 @@ module Multiwoven
           end
 
           def discover(connection_config)
+            connection_config = connection_config.with_indifferent_access
             authorize_client(connection_config)
             spreadsheets = fetch_google_spread_sheets(connection_config)
             catalog = build_catalog_from_spreadsheets(spreadsheets, connection_config)
@@ -34,6 +38,24 @@ module Multiwoven
             handle_exception("GOOGLE_SHEETS:CRM:WRITE:EXCEPTION", "error", e)
           end
 
+          def clear_all_records(sync_config)
+            setup_write_environment(sync_config, "clear")
+            connection_specification = sync_config.destination.connection_specification.with_indifferent_access
+            spreadsheet = fetch_google_spread_sheets(connection_specification)
+            sheet_ids = spreadsheet.sheets.map(&:properties).map(&:sheet_id)
+
+            delete_extra_sheets(sheet_ids)
+
+            unless sheet_ids.empty?
+              clear_response = clear_sheet_data(spreadsheet.sheets.first.properties.title)
+              return control_message("Successfully cleared data.", "succeeded") if clear_response&.cleared_range
+            end
+
+            control_message("Failed to clear data.", "failed")
+          rescue StandardError => e
+            control_message(e.message, "failed")
+          end
+
           private
 
           # To define the level of access granted to your app, you need to identify and declare authorization scopes which is provided by google scopse https://developers.google.com/sheets/api/scopes
@@ -99,7 +121,7 @@ module Multiwoven
               batch_support: true,
               batch_size: 10_000,
               json_schema: generate_properties_schema(column_names),
-              supported_sync_modes: %w[incremental]
+              supported_sync_modes: %w[incremental full_refresh]
             }.with_indifferent_access
           end
 
@@ -113,8 +135,9 @@ module Multiwoven
 
           def setup_write_environment(sync_config, action)
             @action = sync_config.stream.action || action
-            @spreadsheet_id = extract_spreadsheet_id(sync_config.destination.connection_specification[:spreadsheet_link])
-            authorize_client(sync_config.destination.connection_specification)
+            connection_specification = sync_config.destination.connection_specification.with_indifferent_access
+            @spreadsheet_id = extract_spreadsheet_id(connection_specification[:spreadsheet_link])
+            authorize_client(connection_specification)
           end
 
           def extract_spreadsheet_id(link)
@@ -176,6 +199,31 @@ module Multiwoven
               success: success, failed: failure
             ).to_multiwoven_message
           end
+
+          def delete_extra_sheets(sheet_ids)
+            # Leave one sheet intact as a spreadsheet must have at least one sheet.
+            # Delete all other sheets.
+            (sheet_ids.length - 1).times do |i|
+              request = Google::Apis::SheetsV4::BatchUpdateSpreadsheetRequest.new(
+                requests: [{ delete_sheet: { sheet_id: sheet_ids[i + 1] } }]
+              )
+              @client.batch_update_spreadsheet(@spreadsheet_id, request)
+            end
+          end
+
+          def clear_sheet_data(sheet_title)
+            clear_request = Google::Apis::SheetsV4::ClearValuesRequest.new
+            @client&.clear_values(@spreadsheet_id, "#{sheet_title}!A2:Z", clear_request)
+          end
+
+          def control_message(message, status)
+            ControlMessage.new(
+              type: "full_refresh",
+              emitted_at: Time.now.to_i,
+              status: ConnectionStatusType[status],
+              meta: { detail: message }
+            ).to_multiwoven_message
+          end
         end
       end
     end

data/lib/multiwoven/integrations/destination/hubspot/client.rb

@@ -9,6 +9,7 @@ module Multiwoven
         include Multiwoven::Integrations::Core
 
         class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
           def check_connection(connection_config)
             connection_config = connection_config.with_indifferent_access
             initialize_client(connection_config)

data/lib/multiwoven/integrations/destination/klaviyo/client.rb

@@ -4,6 +4,7 @@ module Multiwoven::Integrations::Destination
   module Klaviyo
     include Multiwoven::Integrations::Core
     class Client < DestinationConnector
+      prepend Multiwoven::Integrations::Core::RateLimiter
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
         api_key = connection_config[:private_api_key]

data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module Multiwoven
+  module Integrations
+    module Destination
+      module SalesforceConsumerGoodsCloud
+        include Multiwoven::Integrations::Core
+
+        API_VERSION = "59.0"
+
+        class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
+          def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
+            initialize_client(connection_config)
+            authenticate_client
+            success_status
+          rescue StandardError => e
+            failure_status(e)
+          end
+
+          def discover(_connection_config = nil)
+            catalog = build_catalog(load_catalog)
+            catalog.to_multiwoven_message
+          rescue StandardError => e
+            handle_exception("SALESFORCE:CONSUMER:GOODS:ClOUD:DISCOVER:EXCEPTION", "error", e)
+          end
+
+          def write(sync_config, records, action = "create")
+            @action = sync_config.stream.action || action
+            initialize_client(sync_config.destination.connection_specification)
+            process_records(records, sync_config.stream)
+          rescue StandardError => e
+            handle_exception("SALESFORCE:CONSUMER:GOODS:ClOUD:WRITE:EXCEPTION", "error", e)
+          end
+
+          private
+
+          def initialize_client(config)
+            config = config.with_indifferent_access
+            @client = Restforce.new(oauth_token: config[:access_token],
+                                    refresh_token: config[:refresh_token],
+                                    instance_url: config[:instance_url],
+                                    client_id: config[:client_id],
+                                    client_secret: config[:client_secret],
+                                    authentication_callback: proc { |x| log_debug(x.to_s) },
+                                    api_version: API_VERSION)
+          end
+
+          def process_records(records, stream)
+            write_success = 0
+            write_failure = 0
+            properties = stream.json_schema[:properties]
+            records.each do |record_object|
+              record = extract_data(record_object, properties)
+              process_record(stream, record)
+              write_success += 1
+            rescue StandardError => e
+              handle_exception("SALESFORCE:CONSUMER:GOODS:ClOUD:WRITE:EXCEPTION", "error", e)
+              write_failure += 1
+            end
+            tracking_message(write_success, write_failure)
+          end
+
+          def process_record(stream, record)
+            send_data_to_salesforce(stream.name, record)
+          end
+
+          def send_data_to_salesforce(stream_name, record = {})
+            method_name = "#{@action}!"
+            args = build_args(@action, stream_name, record)
+            @client.send(method_name, *args)
+          end
+
+          def build_args(action, stream_name, record)
+            case action
+            when :upsert
+              [stream_name, record[:external_key], record]
+            when :destroy
+              [stream_name, record[:id]]
+            else
+              [stream_name, record]
+            end
+          end
+
+          def authenticate_client
+            @client.authenticate!
+          end
+
+          def success_status
+            ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
+          end
+
+          def failure_status(error)
+            ConnectionStatus.new(status: ConnectionStatusType["failed"], message: error.message).to_multiwoven_message
+          end
+
+          def load_catalog
+            read_json(CATALOG_SPEC_PATH)
+          end
+
+          def tracking_message(success, failure)
+            Multiwoven::Integrations::Protocol::TrackingMessage.new(
+              success: success, failed: failure
+            ).to_multiwoven_message
+          end
+
+          def log_debug(message)
+            Multiwoven::Integrations::Service.logger.debug(message)
+          end
+        end
+      end
+    end
+  end
+end