multiwoven-integrations 0.34.19 → 0.34.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5cd1d108a22615bf693a33e70ab969742427dfe1f6af6f8d6d35e9a6fda301d5
-  data.tar.gz: 226cf6853b8f6c211937336ffb1551a02436dfaf7142a3e217044799b5108797
+  metadata.gz: fc1a7c208f3183e532a2eb7d7e71db0a19fe1fc8c559d27036f9d6bf147eaf10
+  data.tar.gz: e5ebf0e7895beaf61fe4cfaa4d41b414e9315c0c26725c3cb4a91cb4c1cad3eb
 SHA512:
-  metadata.gz: 9111231a712e0be82ed18bb6b283bcbed0a676ea5dafe2179f69cd80d70a68b799c873792d6d6c1b7b5e7b892411a4f7f5c8664075afc695557ef9245001f8e0
-  data.tar.gz: '084dadfad1d504fe10a896e6cf20d4ebbac8b8a492675dda566b79d641d03b307c06dbae903d757b25a2596652727693efb52cd99c274e60cc676d590d5e3dd9'
+  metadata.gz: 3abb6be05b20a2d7bbdbcff69a59fb29e62129286f8473552080fbcff421b12f720ce089831b4a8c218c4003b788de49982207d2dc86d7c2ee8b7cdc8764588e
+  data.tar.gz: 7c3d59f96b267950075467c07228e2ed157eabc2a5365e584ad5a7fbd2a6057a58be2f82a64ac6d760f2735a21b0f59bdba1489106ab86fad3c9977f560ff29b

data/lib/multiwoven/integrations/destination/postgresql/client.rb

@@ -6,6 +6,8 @@ module Multiwoven::Integrations::Destination
   module Postgresql
     include Multiwoven::Integrations::Core
     class Client < DestinationConnector
+      MAX_CHUNK_SIZE = 10_000
+
       def check_connection(connection_config)
         connection_config = connection_config.with_indifferent_access
         create_connection(connection_config)
@@ -50,30 +52,34 @@ module Multiwoven::Integrations::Destination
         raw_table = sync_config.stream.name
         table_name = qualify_table(connection_config[:schema], raw_table)
         primary_key = sync_config.model.primary_key
-        log_message_array = []
         db = create_connection(connection_config)
 
         write_success = 0
         write_failure = 0
+        log_message_array = []
 
-        records.each do |record|
-          query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
-          logger.debug("POSTGRESQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
-          begin
-            response = db.exec(query)
+        records.each_slice(MAX_CHUNK_SIZE) do |chunk|
+          bulk_write(db, table_name, chunk, primary_key, action)
+          write_success += chunk.size
+          log_message_array << log_request_response("info", "bulk_#{action}", "#{chunk.size} rows")
+        rescue StandardError => e
+          logger.warn("POSTGRESQL:BULK_WRITE:FALLBACK chunk_size=#{chunk.size} error=#{e.message}")
+          chunk.each do |record|
+            response = bulk_write(db, table_name, [record], primary_key, action)
             write_success += 1
-            log_message_array << log_request_response("info", query, response)
-          rescue StandardError => e
-            handle_exception(e, {
+            log_message_array << log_request_response("info", "fallback_#{action}", response)
+          rescue StandardError => individual_error
+            handle_exception(individual_error, {
                                context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
                                type: "error",
                                sync_id: sync_config.sync_id,
                                sync_run_id: sync_config.sync_run_id
                              })
             write_failure += 1
-            log_message_array << log_request_response("error", query, e.message)
+            log_message_array << log_request_response("error", "fallback_#{action}", individual_error.message)
           end
         end
+
         tracking_message(write_success, write_failure, log_message_array)
       rescue StandardError => e
         handle_exception(e, {
@@ -82,10 +88,48 @@ module Multiwoven::Integrations::Destination
                            sync_id: sync_config.sync_id,
                            sync_run_id: sync_config.sync_run_id
                          })
+      ensure
+        db&.close
       end
 
       private
 
+      def bulk_write(db, table_name, records, primary_key, action)
+        return if records.empty?
+
+        columns = records.flat_map(&:keys).uniq
+        col_list = columns.map { |c| quote_ident(c) }.join(", ")
+
+        values_clauses = records.map do |record|
+          vals = columns.map { |col| escape_value(db, record[col]) }
+          "(#{vals.join(", ")})"
+        end
+
+        sql = "INSERT INTO #{table_name} (#{col_list}) VALUES #{values_clauses.join(", ")}"
+        sql += build_upsert_clause(columns, primary_key) if action.to_s == "destination_update"
+        db.exec(sql)
+      end
+
+      def build_upsert_clause(columns, primary_key)
+        return "" unless primary_key.present?
+
+        update_cols = columns.reject { |c| c.to_s == primary_key.to_s }
+        return " ON CONFLICT (#{quote_ident(primary_key)}) DO NOTHING" if update_cols.empty?
+
+        set_clause = update_cols.map { |c| "#{quote_ident(c)} = EXCLUDED.#{quote_ident(c)}" }.join(", ")
+        " ON CONFLICT (#{quote_ident(primary_key)}) DO UPDATE SET #{set_clause}"
+      end
+
+      def escape_value(db, value)
+        return "NULL" if value.nil?
+
+        "'#{db.escape_string(value.to_s)}'"
+      end
+
+      def quote_ident(name)
+        PG::Connection.quote_ident(name.to_s)
+      end
+
       def query(connection, query)
         connection.exec(query) do |result|
           result.map do |row|
@@ -108,7 +152,13 @@ module Multiwoven::Integrations::Destination
 
       def create_streams(records)
         group_by_table(records).map do |r|
-          Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
+          Multiwoven::Integrations::Protocol::Stream.new(
+            name: r[:tablename],
+            action: StreamAction["fetch"],
+            json_schema: convert_to_json_schema(r[:columns]),
+            batch_support: true,
+            batch_size: MAX_CHUNK_SIZE
+          )
         end
       end
 

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.34.19"
+    VERSION = "0.34.20"
 
     ENABLED_SOURCES = %w[
       Snowflake

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.34.19
+  version: 0.34.20
 platform: ruby
 authors:
 - Subin T P
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-02-24 00:00:00.000000000 Z
+date: 2026-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport