multiwoven-integrations 0.1.36 → 0.1.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 818c8606d1ef1b50bd14765b3ed85f687ef5e16cdd975324bfbf28ed2b371070
-  data.tar.gz: 0bd41ef2fe575ff51d5ea5878522509de3aa6ae0b3592074c15ce3de5be97c51
+  metadata.gz: 13fe187a421657cfb45280d87ae9fdd88dfb57b684290aba50869fc73bbd19c2
+  data.tar.gz: 8d08feb79f2798e6955c482af4ec2ecaccc35ff723d60191c6efcfc85f173972
 SHA512:
-  metadata.gz: dfaa4c53e72fbf3fc64d2d746bf4fd632c19be834af900a93eb88060fc1a84156b1849b645768ca41ebd3037f205a7b4f79430738eddb7f607b526fb9424c08b
-  data.tar.gz: dd7dc877f6ad86082975cb9dbdc714bc5e9dc623196178e6c9c11b601deab967fa107d1d218d5dc03e43b6b6c580403d45bc470fd364a109a8a1cb6e8a793520
+  metadata.gz: b828ceef6251ca0859da2c5fe74e763c2a3f366cd461839456b74d4791eec94e482d7cfa1673ff76511f3e48dc8e7d09a0b216ff2f8a47585ff69cb628745a96
+  data.tar.gz: 639488c9e265492f3e98b69cfd5010998c569a99c70bb11f2cbfbed92499f7b4c2eacf4f2febe0698cafebc6d039f3e18834eb6e8d010ff137f9603ac060ce6e

data/lib/multiwoven/integrations/core/destination_connector.rb

@@ -3,8 +3,6 @@
 module Multiwoven
   module Integrations::Core
     class DestinationConnector < BaseConnector
-      prepend RateLimiter
-
       # Records are transformed json payload send it to the destination
       # SyncConfig is the Protocol::SyncConfig object
       def write(_sync_config, _records, _action = "insert")

data/lib/multiwoven/integrations/core/fullrefresher.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Multiwoven
+  module Integrations::Core
+    module Fullrefresher
+      def write(sync_config, records, action = "insert")
+        if sync_config && sync_config.sync_mode == "full_refresh" && !@full_refreshed
+          response = clear_all_records(sync_config)
+          return response unless response &&
+                                 response.control.status == Multiwoven::Integrations::Protocol::ConnectionStatusType["succeeded"]
+
+          @full_refreshed = true
+        end
+
+        super(sync_config, records, action)
+      end
+    end
+  end
+end

data/lib/multiwoven/integrations/destination/airtable/client.rb

@@ -7,6 +7,7 @@ module Multiwoven
       module Airtable
         include Multiwoven::Integrations::Core
         class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+          prepend Multiwoven::Integrations::Core::RateLimiter
           MAX_CHUNK_SIZE = 10
           def check_connection(connection_config)
             connection_config = connection_config.with_indifferent_access

data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb

@@ -4,6 +4,7 @@ module Multiwoven::Integrations::Destination
   module FacebookCustomAudience
     include Multiwoven::Integrations::Core
     class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+      prepend Multiwoven::Integrations::Core::RateLimiter
       MAX_CHUNK_SIZE = 10_000
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access

data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/catalog.json

@@ -33,7 +33,7 @@
           "PAGEUID": { "type": ["string", "null"], "default": null, "title": "Page-Scoped ID", "x-hashRequired": false }
         }
       },
-      "supported_sync_modes": ["full_refresh", "incremental"],
+      "supported_sync_modes": ["incremental"],
       "source_defined_cursor": true,
       "default_cursor_field": ["updated"],
       "source_defined_primary_key": [["email"]]

data/lib/multiwoven/integrations/destination/google_sheets/client.rb

@@ -7,9 +7,12 @@ module Multiwoven
         include Multiwoven::Integrations::Core
 
         class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+          prepend Multiwoven::Integrations::Core::Fullrefresher
+          prepend Multiwoven::Integrations::Core::RateLimiter
           MAX_CHUNK_SIZE = 10_000
 
           def check_connection(connection_config)
+            connection_config = connection_config.with_indifferent_access
             authorize_client(connection_config)
             fetch_google_spread_sheets(connection_config)
             success_status
@@ -19,6 +22,7 @@ module Multiwoven
           end
 
           def discover(connection_config)
+            connection_config = connection_config.with_indifferent_access
             authorize_client(connection_config)
             spreadsheets = fetch_google_spread_sheets(connection_config)
             catalog = build_catalog_from_spreadsheets(spreadsheets, connection_config)
@@ -34,6 +38,24 @@ module Multiwoven
             handle_exception("GOOGLE_SHEETS:CRM:WRITE:EXCEPTION", "error", e)
           end
 
+          def clear_all_records(sync_config)
+            setup_write_environment(sync_config, "clear")
+            connection_specification = sync_config.destination.connection_specification.with_indifferent_access
+            spreadsheet = fetch_google_spread_sheets(connection_specification)
+            sheet_ids = spreadsheet.sheets.map(&:properties).map(&:sheet_id)
+
+            delete_extra_sheets(sheet_ids)
+
+            unless sheet_ids.empty?
+              clear_response = clear_sheet_data(spreadsheet.sheets.first.properties.title)
+              return control_message("Successfully cleared data.", "succeeded") if clear_response&.cleared_range
+            end
+
+            control_message("Failed to clear data.", "failed")
+          rescue StandardError => e
+            control_message(e.message, "failed")
+          end
+
           private
 
           # To define the level of access granted to your app, you need to identify and declare authorization scopes which is provided by google scopse https://developers.google.com/sheets/api/scopes
@@ -99,7 +121,7 @@ module Multiwoven
               batch_support: true,
               batch_size: 10_000,
               json_schema: generate_properties_schema(column_names),
-              supported_sync_modes: %w[incremental]
+              supported_sync_modes: %w[incremental full_refresh]
             }.with_indifferent_access
           end
 
@@ -113,8 +135,9 @@ module Multiwoven
 
           def setup_write_environment(sync_config, action)
             @action = sync_config.stream.action || action
-            @spreadsheet_id = extract_spreadsheet_id(sync_config.destination.connection_specification[:spreadsheet_link])
-            authorize_client(sync_config.destination.connection_specification)
+            connection_specification = sync_config.destination.connection_specification.with_indifferent_access
+            @spreadsheet_id = extract_spreadsheet_id(connection_specification[:spreadsheet_link])
+            authorize_client(connection_specification)
           end
 
           def extract_spreadsheet_id(link)
@@ -176,6 +199,31 @@ module Multiwoven
               success: success, failed: failure
             ).to_multiwoven_message
           end
+
+          def delete_extra_sheets(sheet_ids)
+            # Leave one sheet intact as a spreadsheet must have at least one sheet.
+            # Delete all other sheets.
+            (sheet_ids.length - 1).times do |i|
+              request = Google::Apis::SheetsV4::BatchUpdateSpreadsheetRequest.new(
+                requests: [{ delete_sheet: { sheet_id: sheet_ids[i + 1] } }]
+              )
+              @client.batch_update_spreadsheet(@spreadsheet_id, request)
+            end
+          end
+
+          def clear_sheet_data(sheet_title)
+            clear_request = Google::Apis::SheetsV4::ClearValuesRequest.new
+            @client&.clear_values(@spreadsheet_id, "#{sheet_title}!A2:Z", clear_request)
+          end
+
+          def control_message(message, status)
+            ControlMessage.new(
+              type: "full_refresh",
+              emitted_at: Time.now.to_i,
+              status: ConnectionStatusType[status],
+              meta: { detail: message }
+            ).to_multiwoven_message
+          end
         end
       end
     end

data/lib/multiwoven/integrations/destination/hubspot/client.rb

@@ -9,6 +9,7 @@ module Multiwoven
         include Multiwoven::Integrations::Core
 
         class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
           def check_connection(connection_config)
             connection_config = connection_config.with_indifferent_access
             initialize_client(connection_config)

data/lib/multiwoven/integrations/destination/klaviyo/client.rb

@@ -4,6 +4,7 @@ module Multiwoven::Integrations::Destination
   module Klaviyo
     include Multiwoven::Integrations::Core
     class Client < DestinationConnector
+      prepend Multiwoven::Integrations::Core::RateLimiter
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
         api_key = connection_config[:private_api_key]

data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb

@@ -11,6 +11,7 @@ module Multiwoven
         API_VERSION = "59.0"
 
         class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
           def check_connection(connection_config)
             connection_config = connection_config.with_indifferent_access
             initialize_client(connection_config)

data/lib/multiwoven/integrations/destination/slack/client.rb

@@ -7,6 +7,7 @@ module Multiwoven
         include Multiwoven::Integrations::Core
 
         class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
           attr_accessor :channel_id
 
           def check_connection(connection_config)

data/lib/multiwoven/integrations/destination/stripe/client.rb

@@ -9,6 +9,7 @@ module Multiwoven
         API_VERSION = "59.0"
 
         class Client < DestinationConnector
+          prepend Multiwoven::Integrations::Core::RateLimiter
           def check_connection(connection_config)
             connection_config = connection_config.with_indifferent_access
             initialize_client(connection_config)

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -20,7 +20,7 @@ module Multiwoven
       "tracking"
     )
     ControlMessageType = Types::String.enum(
-      "rate_limit", "connection_config"
+      "rate_limit", "connection_config", "full_refresh"
     )
     LogLevel = Types::String.enum("fatal", "error", "warn", "info", "debug", "trace")
     RequestRateLimitingUnit = Types::String.default("minute").enum("minute", "hour", "day")
@@ -170,6 +170,7 @@ module Multiwoven
     class ControlMessage < ProtocolModel
       attribute :type, ControlMessageType
       attribute :emitted_at, Types::Integer
+      attribute? :status, ConnectionStatusType.optional
       attribute? :meta, Types::Hash
 
       def to_multiwoven_message

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.36"
+    VERSION = "0.1.37"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations.rb

@@ -29,6 +29,7 @@ require_relative "integrations/service"
 require_relative "integrations/core/constants"
 require_relative "integrations/core/utils"
 require_relative "integrations/core/rate_limiter"
+require_relative "integrations/core/fullrefresher"
 require_relative "integrations/protocol/protocol"
 require_relative "integrations/core/base_connector"
 require_relative "integrations/core/source_connector"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.36
+  version: 0.1.37
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-03-13 00:00:00.000000000 Z
+date: 2024-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: google-apis-sheets_v4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: google-cloud-bigquery
   requirement: !ruby/object:Gem::Requirement
@@ -220,6 +234,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: stripe
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -326,6 +354,7 @@ files:
 - lib/multiwoven/integrations/core/base_connector.rb
 - lib/multiwoven/integrations/core/constants.rb
 - lib/multiwoven/integrations/core/destination_connector.rb
+- lib/multiwoven/integrations/core/fullrefresher.rb
 - lib/multiwoven/integrations/core/http_client.rb
 - lib/multiwoven/integrations/core/rate_limiter.rb
 - lib/multiwoven/integrations/core/source_connector.rb