multiwoven-integrations 0.1.17 → 0.1.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b5d95619c96cf27279deaccb0a69eb119fb5da9acfcc2fde885e8fac38cee42
-  data.tar.gz: 6091f680309d4300be69e94496bf3400274b7934f2c32d639cce7a49f487bf86
+  metadata.gz: 429328f071eab809197f11563a5c87a91ca1d83db1f89f67edc001b2d8aa0d08
+  data.tar.gz: fdbe36a9021b0eae79cff29ec7002859f03993e354d61a45e01024b293ea5835
 SHA512:
-  metadata.gz: '095149b7a10dcd6f7ab18c201ecfb38515e73f863c675a3f85f7f73878aecfadab80cde348676e3ebeb03096ebf26faa57618f903a992eaa07825afd11a24119'
-  data.tar.gz: 5ffb66ff3a80658096f672693b9d927fa1a49cfcae85bdce9a5fc1524be48fb45e3f646e2c478a69e3447ba56017656c628976ad01dad486beab8a8043f0ee76
+  metadata.gz: 3b17d88517b0b196bef171d72abb99f8ae9faa8e1b8e62c4e9296e86d821025544c742e35a526b5cd1a35a31e8f1959183648a24dcb8ac0269ec34dca5de87e0
+  data.tar.gz: 96d942afc5d2a4015534b57d72b2f8f35fdb91bec1004e5d180ecf451d46a5cc980fdfca406335de9d7ef522e041c49dcba459d56f297ead36223e701f389ac3

data/lib/multiwoven/integrations/core/base_connector.rb

@@ -10,6 +10,7 @@ module Multiwoven
       def connector_spec
         @connector_spec ||= begin
           spec_json = keys_to_symbols(read_json(CONNECTOR_SPEC_PATH)).to_json
+          # returns Protocol::ConnectorSpecification
           ConnectorSpecification.from_json(spec_json)
         end
       end
@@ -19,17 +20,20 @@ module Multiwoven
         icon_name = client_meta_data[:data][:icon]
         icon_url = "https://raw.githubusercontent.com/Multiwoven/multiwoven-integrations/#{MAIN_BRANCH_SHA}/assets/images/connectors/#{icon_name}"
         client_meta_data[:data][:icon] = icon_url
-
+        # returns hash
         @meta_data ||= client_meta_data
       end
 
+      # Connection config is a hash
       def check_connection(_connection_config)
         raise "Not implemented"
         # returns Protocol.ConnectionStatus
       end
 
+      # Connection config is a hash
       def discover(_connection_config)
         raise "Not implemented"
+        # returns Protocol::Catalog
       end
 
       private

data/lib/multiwoven/integrations/core/destination_connector.rb

@@ -3,9 +3,11 @@
 module Multiwoven
   module Integrations::Core
     class DestinationConnector < BaseConnector
+      # records is transformed json payload send it to the destination
+      # SyncConfig is the Protocol::SyncConfig object
       def write(_sync_config, _records, _action = "insert")
         raise "Not implemented"
-        # return list of record message
+        # return Protocol::TrackingMessage
       end
     end
   end

data/lib/multiwoven/integrations/core/source_connector.rb

@@ -3,10 +3,12 @@
 module Multiwoven
   module Integrations::Core
     class SourceConnector < BaseConnector
+      # accepts Protocol::SyncConfig
       def read(_sync_config)
         raise "Not implemented"
         # setup sync configs
         # call query(connection, query)
+        # Returns list of RecordMessage
       end
 
       private

data/lib/multiwoven/integrations/core/utils.rb

@@ -65,7 +65,7 @@ module Multiwoven
       end
 
       def extract_data(record_object, properties)
-        data_attributes = record_object.with_indifferent_access[:data][:attributes]
+        data_attributes = record_object.with_indifferent_access
         data_attributes.select { |key, _| properties.key?(key.to_sym) }
       end
 

data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb

@@ -3,7 +3,8 @@
 module Multiwoven::Integrations::Destination
   module FacebookCustomAudience
     include Multiwoven::Integrations::Core
-    class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
+    class Client < DestinationConnector  # rubocop:disable Metrics/ClassLength
+      MAX_CHUNK_SIZE = 10_000
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
         access_token = connection_config[:access_token]
@@ -54,8 +55,8 @@ module Multiwoven::Integrations::Destination
         url = generate_url(sync_config, connection_config)
         write_success = 0
         write_failure = 0
-        records.each do |record|
-          payload = create_payload(record.with_indifferent_access[:data][:attributes], sync_config.stream.json_schema.with_indifferent_access)
+        records.each_slice(MAX_CHUNK_SIZE) do |chunk|
+          payload = create_payload(chunk, sync_config.stream.json_schema.with_indifferent_access)
           response = Multiwoven::Integrations::Core::HttpClient.request(
             url,
             sync_config.stream.request_method,
@@ -63,28 +64,22 @@ module Multiwoven::Integrations::Destination
             headers: auth_headers(access_token)
           )
           if success?(response)
-            write_success += 1
+            write_success += chunk.size
           else
-            write_failure += 1
+            write_failure += chunk.size
           end
         rescue StandardError => e
-          logger.error(
-            "FACEBOOK:RECORD:WRITE:FAILURE: #{e.message}"
-          )
-          write_failure += 1
+          handle_exception("FACEBOOK:RECORD:WRITE:EXCEPTION", "error", e)
+          write_failure += chunk.size
         end
+
         tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
           success: write_success,
           failed: write_failure
         )
         tracker.to_multiwoven_message
       rescue StandardError => e
-        # TODO: Handle rate limiting seperately
-        handle_exception(
-          "FACEBOOK:WRITE:EXCEPTION",
-          "error",
-          e
-        )
+        handle_exception("FACEBOOK:WRITE:EXCEPTION", "error", e)
       end
 
       private
@@ -93,28 +88,30 @@ module Multiwoven::Integrations::Destination
         sync_config.stream.url.gsub("{audience_id}", connection_config[:audience_id])
       end
 
-      def create_payload(record_data, json_schema)
-        schema, data = extract_schema_and_data(record_data, json_schema)
+      def create_payload(records, json_schema)
+        schema, data = extract_schema_and_data(records, json_schema)
         {
           "payload" => {
             "schema" => schema,
-            "data" => [data]
+            "data" => data
           }
         }
       end
 
-      def extract_schema_and_data(data, json_schema)
+      def extract_schema_and_data(records, json_schema)
         schema_properties = json_schema[:properties]
-        schema = data.keys.map(&:upcase)
-        encrypted_data_array = []
-
-        data.each do |key, value|
-          schema_key = key.upcase
-          encrypted_value = schema_properties[schema_key] && schema_properties[schema_key]["x-hashRequired"] ? Digest::SHA256.hexdigest(value.to_s) : value
-          encrypted_data_array << encrypted_value
+        schema = records.first.keys.map(&:to_s).map(&:upcase)
+        data = []
+        records.each do |record|
+          encrypted_data_array = []
+          record.with_indifferent_access.each do |key, value|
+            schema_key = key.upcase
+            encrypted_value = schema_properties[schema_key] && schema_properties[schema_key]["x-hashRequired"] ? Digest::SHA256.hexdigest(value.to_s) : value
+            encrypted_data_array << encrypted_value
+          end
+          data << encrypted_data_array
         end
-
-        [schema, encrypted_data_array]
+        [schema, data]
       end
 
       def auth_headers(access_token)

data/lib/multiwoven/integrations/destination/slack/client.rb

@@ -72,7 +72,7 @@ module Multiwoven
           def build_args(stream_name, record)
             case stream_name
             when "chat_postMessage"
-              { channel: channel_id, text: slack_code_block(record[:data][:attributes]) }
+              { channel: channel_id, text: slack_code_block(record) }
             else
               raise "Stream name not found: #{stream_name}"
             end

data/lib/multiwoven/integrations/destination/slack/config/catalog.json

@@ -9,63 +9,12 @@
         "properties": {
           "text": {
             "type": ["string", "null"]
-          },
-          "attachments": {
-            "type": ["string", "null"]
-          },
-          "blocks": {
-            "type": ["array", "null"],
-            "items": {
-              "type": "string"
-            }
-          },
-          "as_user": {
-            "type": ["boolean", "null"]
-          },
-          "icon_emoji": {
-            "type": ["string", "null"]
-          },
-          "icon_url": {
-            "type": ["string", "null"]
-          },
-          "link_names": {
-            "type": ["boolean", "null"]
-          },
-          "metadata": {
-            "type": ["string", "null"]
-          },
-          "mrkdwn": {
-            "type": ["boolean", "null"]
-          },
-          "parse": {
-            "type": ["string", "null"]
-          },
-          "reply_broadcast": {
-            "type": ["boolean", "null"]
-          },
-          "thread_ts": {
-            "type": ["string", "null"]
-          },
-          "unfurl_links": {
-            "type": ["boolean", "null"]
-          },
-          "unfurl_media": {
-            "type": ["boolean", "null"]
-          },
-          "username": {
-            "type": ["string", "null"]
           }
         },
-        "oneOf": [
-          { "required": ["text"] },
-          { "required": ["attachments"] },
-          { "required": ["blocks"] }
-        ]
+        "oneOf": [{ "required": ["text"] }]
       },
       "supported_sync_modes": ["full_refresh", "incremental"],
-      "source_defined_cursor": true,
-      "default_cursor_field": ["updated"],
-      "source_defined_primary_key": [["Id"]]
+      "source_defined_cursor": true
     }
   ]
 }

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.17"
+    VERSION = "0.1.19"
 
     ENABLED_SOURCES = %w[
       Snowflake

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.17
+  version: 0.1.19
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-02-06 00:00:00.000000000 Z
+date: 2024-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -362,7 +362,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.4.1
 signing_key:
 specification_version: 4
 summary: Integration suite for open source reverse ETL platform