multiwoven-integrations 0.3.3 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e8263bcb86ca81c8262642cd939c8079e671e7cb03a29a905c24f564888038c5
-  data.tar.gz: cf5ba4ee0ad1f25967592e72369b3e0452d09715a036b70685d48ee4f29568de
+  metadata.gz: 50ba719dc1cdacb377fe3caeadc0948944da43f2fce377d7db93f9ca3d5f1bea
+  data.tar.gz: 0f04916affd15488b2e5f7fe67b3031747d193b851dfa31e5085065bc8e6048e
 SHA512:
-  metadata.gz: a0c976f1fd891d8474dbc113bce838c0a566e0127e49829c61c37042b479b3303185193beb66a204a752a87f8a7ec82b512d51fba5437e5239e1fc8a889ddcb9
-  data.tar.gz: 0ecd0c38c46db77c5e6c9ea6d8ef94cfd3325f7cb9e5f1a2d86ae6d22c307305595cac210d5d6710843806d8c8891c1b04bc7cf663efcb7e5b42cd3bbe45e39f
+  metadata.gz: e1a4bb4360df030a2a8124579a1d95b84ee9ae3b2f64eb59a9e4502d481d1f5a8470216981c0303868cdca22dff9c8d8a14ca7bf01547277c4bf4bb6d72da78c
+  data.tar.gz: 6251c28f1bbbd38beeed2e1008715a146ad0d2fd7826c60c7508dc177e43bdf4f3bf7a792b7976c031fbfad9a0d77946ab301949cbaf9eb95acc5024596af3d8

data/lib/multiwoven/integrations/core/constants.rb

@@ -34,6 +34,9 @@ module Multiwoven
       AIRTABLE_BASES_ENDPOINT = "https://api.airtable.com/v0/meta/bases"
       AIRTABLE_GET_BASE_SCHEMA_ENDPOINT = "https://api.airtable.com/v0/meta/bases/{baseId}/tables"
 
+      AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
+      AWS_SECRET_ACCESS_KEY = ENV["AWS_SECRET_ACCESS_KEY"]
+
       # HTTP
       HTTP_GET = "GET"
       HTTP_POST = "POST"

data/lib/multiwoven/integrations/core/destination_connector.rb

@@ -9,6 +9,12 @@ module Multiwoven
         raise "Not implemented"
         # return Protocol::TrackingMessage
       end
+
+      def tracking_message(success, failure, log_message_array)
+        Multiwoven::Integrations::Protocol::TrackingMessage.new(
+          success: success, failed: failure, logs: log_message_array
+        ).to_multiwoven_message
+      end
     end
   end
 end

data/lib/multiwoven/integrations/core/utils.rb

@@ -53,6 +53,14 @@ module Multiwoven
         reporter&.report(exception, meta)
       end
 
+      def log_request_response(level, request, response)
+        Integrations::Protocol::LogMessage.new(
+          name: self.class.name,
+          level: level,
+          message: { request: request.to_s, response: response.to_s, level: level }.to_json
+        )
+      end
+
       def create_log_message(context, type, exception)
         Integrations::Protocol::LogMessage.new(
           name: context,

data/lib/multiwoven/integrations/destination/postgresql/client.rb

@@ -46,6 +46,7 @@ module Multiwoven::Integrations::Destination
         connection_config = sync_config.destination.connection_specification.with_indifferent_access
         table_name = sync_config.stream.name
         primary_key = sync_config.model.primary_key
+        log_message_array = []
         db = create_connection(connection_config)
 
         write_success = 0
@@ -55,8 +56,9 @@ module Multiwoven::Integrations::Destination
           query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
           logger.debug("POSTGRESQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
           begin
-            db.exec(query)
+            response = db.exec(query)
             write_success += 1
+            log_message_array << log_request_response("info", query, response)
           rescue StandardError => e
             handle_exception(e, {
                                context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
@@ -65,9 +67,10 @@ module Multiwoven::Integrations::Destination
                                sync_run_id: sync_config.sync_run_id
                              })
             write_failure += 1
+            log_message_array << log_request_response("error", query, e.message)
           end
         end
-        tracking_message(write_success, write_failure)
+        tracking_message(write_success, write_failure, log_message_array)
       rescue StandardError => e
         handle_exception(e, {
                            context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
@@ -119,12 +122,6 @@ module Multiwoven::Integrations::Destination
           }
         end
       end
-
-      def tracking_message(success, failure)
-        Multiwoven::Integrations::Protocol::TrackingMessage.new(
-          success: success, failed: failure
-        ).to_multiwoven_message
-      end
     end
   end
 end

data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb

@@ -80,10 +80,14 @@ module Multiwoven
             write_success = 0
             write_failure = 0
             properties = stream.json_schema[:properties]
+            log_message_array = []
+
             records.each do |record_object|
               record = extract_data(record_object, properties)
-              process_record(stream, record)
+              args = [stream.name, "Id", record]
+              response = send_data_to_salesforce(args)
               write_success += 1
+              log_message_array << log_request_response("info", args, response)
             rescue StandardError => e
               # TODO: add sync_id and sync run id to the logs
               handle_exception(e, {
@@ -93,18 +97,14 @@ module Multiwoven
                                  sync_run_id: @sync_config.sync_run_id
                                })
               write_failure += 1
+              log_message_array << log_request_response("error", args, e.message)
             end
-            tracking_message(write_success, write_failure)
-          end
-
-          def process_record(stream, record)
-            send_data_to_salesforce(stream.name, record)
+            tracking_message(write_success, write_failure, log_message_array)
           end
 
-          def send_data_to_salesforce(stream_name, record = {})
+          def send_data_to_salesforce(args)
             method_name = "upsert!"
-            args = [stream_name, "Id", record]
-            @logger.debug("sync_id: #{@sync_config.sync_id}, sync_run_id: #{@sync_config.sync_run_id}, record: #{record}")
+            @logger.debug("sync_id: #{@sync_config.sync_id}, sync_run_id: #{@sync_config.sync_run_id}, args: #{args}")
             @client.send(method_name, *args)
           end
 
@@ -124,12 +124,6 @@ module Multiwoven
             read_json(CATALOG_SPEC_PATH)
           end
 
-          def tracking_message(success, failure)
-            Multiwoven::Integrations::Protocol::TrackingMessage.new(
-              success: success, failed: failure
-            ).to_multiwoven_message
-          end
-
           def log_debug(message)
             Multiwoven::Integrations::Service.logger.debug(message)
           end

data/lib/multiwoven/integrations/protocol/protocol.rb

@@ -197,6 +197,7 @@ module Multiwoven
       attribute :success, Types::Integer.default(0)
       attribute :failed, Types::Integer.default(0)
       attribute? :meta, Types::Hash
+      attribute? :logs, Types::Array.of(LogMessage)
 
       def to_multiwoven_message
         MultiwovenMessage.new(

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.3.3"
+    VERSION = "0.3.5"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/amazon_s3/client.rb

@@ -50,15 +50,17 @@ module Multiwoven::Integrations::Source
       private
 
       def get_auth_data(connection_config)
-        session = @session_name
+        session = @session_name.gsub(/\s+/, "-")
         @session_name = ""
         if connection_config[:auth_type] == "user"
           Aws::Credentials.new(connection_config[:access_id], connection_config[:secret_access])
         elsif connection_config[:auth_type] == "role"
-          sts_client = Aws::STS::Client.new(region: connection_config[:region])
+          credentials = Aws::Credentials.new(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+          sts_client = Aws::STS::Client.new(region: connection_config[:region], credentials: credentials)
           resp = sts_client.assume_role({
                                           role_arn: connection_config[:arn],
-                                          role_session_name: session
+                                          role_session_name: session,
+                                          external_id: connection_config[:external_id]
                                         })
           Aws::Credentials.new(
             resp.credentials.access_key_id,

data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json

@@ -29,6 +29,7 @@
         "region",
         "bucket",
         "arn",
+        "external_id",
         "file_type"
       ]
     },
@@ -53,16 +54,22 @@
         "title": "IAM Role ARN",
         "order": 1
       },
+      "external_id": {
+        "type": "string",
+        "title": "External Id",
+        "description": "Unique ID that allows handshake between AWS accounts.",
+        "order": 2
+      },
       "access_id": {
         "type": "string",
         "title": "Access Id",
-        "order": 2
+        "order": 3
       },
       "secret_access": {
         "type": "string",
         "title": "Secret Access",
         "multiwoven_secret": true,
-        "order": 3
+        "order": 4
       },
       "region": {
         "description": "AWS region",
@@ -71,13 +78,13 @@
         ],
         "type": "string",
         "title": "Region",
-        "order": 4
+        "order": 5
       },
       "bucket": {
         "description": "Bucket Name",
         "type": "string",
         "title": "Bucket",
-        "order": 5
+        "order": 6
       },
       "path": {
         "description": "Path to csv or parquet files",
@@ -86,7 +93,7 @@
         ],
         "type": "string",
         "title": "Path",
-        "order": 6
+        "order": 7
       },
       "file_type": {
         "description": "The type of file to read",
@@ -96,7 +103,7 @@
           "csv",
           "parquet"
         ],
-        "order": 7
+        "order": 8
       }
     }
   }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.5
 platform: ruby
 authors:
 - Subin T P
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-06-28 00:00:00.000000000 Z
+date: 2024-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport