data_drain 0.1.6 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb6e8dad588e5512397bb9ed8f99b50bb32dcacf6eb493fbef2dfcf0cb4f8289
-  data.tar.gz: 241eb6c7e340b25d95757f0e663489eba65cb1b9293426c81148c45c78ea534f
+  metadata.gz: 7b52a43c3e135f0ef36762bd0453e5ef39b4ba64872415cfff86a3407edadd1f
+  data.tar.gz: f2c77c7173d8a554f3c5aad5fdd4116ce52f91e5c286277be19fab9dfc178153
 SHA512:
-  metadata.gz: 04ca70eca218c7a9aadd292c4b8def9b4895c2b223a2d46f4297fdd4d078b1d8fd05153ce7410fda35f6783242dddfc1a25993c3b2b3db1aad81572da119cdf2
-  data.tar.gz: dd168e87460ac02365bbc7f1d9f80050c6a103de1f8f0820545933458cce7aa736c6edb4f9c5e8292f436253082ab43ba874182b14709aeb71430cece9a6f56b
+  metadata.gz: 3ae238656bb38fdc270654ee7939bb8713b13c97237921302e1568f201c51aaab3cc5489d13e85b3a24f3642083759713e6b581336b3d848c9c4fff7ef84bf1f
+  data.tar.gz: a91df374b1ad62de9c2aa4d091da45c852cfecab7a3784ed2e1946fe8d82fdcdbde4800825d569774727a5e50cd6acc4c527f1302ee6ddb1572b68780b553115

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## [Unreleased]
 
+## [0.1.8] - 2026-03-16
+
+- Fix: Se cambió la cadena de conexión de DuckDB a formato URI para propagar el timeout de sesión en el ATTACH.
+
+## [0.1.7] - 2026-03-16
+
+- Se agrego soporte para idle_in_transaction_session_timeout.
+
 ## [0.1.6] - 2026-03-16
 
 - Se agrego el tem_directory para duckdb.

data/README.md

@@ -47,12 +47,26 @@ DataDrain.configure do |config|
   config.db_pass = ENV.fetch('DB_PASS', '')
   config.db_name = ENV.fetch('DB_NAME', 'core_production')
 
-  # Rendimiento y Tuning
+  # Rendimiento y Tuning de Postgres
   config.batch_size     = 5000 # Registros a borrar por transacción
   config.throttle_delay = 0.5  # Segundos de pausa entre borrados
+  
+  # Timeout de inactividad de transacciones en PostgreSQL (en milisegundos).
+  # Útil establecerlo en 0 para evitar que la conexión se cierre prematuramente 
+  # durante el borrado de grandes volúmenes de datos.
+  config.idle_in_transaction_session_timeout = 0
+  
   config.logger         = Rails.logger
 
+  # Tuning de DuckDB
+  # Límite máximo de RAM para las consultas en memoria de DuckDB (ej. '2GB', '512MB').
+  # Evita que el proceso OOM (Out Of Memory) si el contenedor o servidor tiene memoria limitada.
   config.limit_ram      = '2GB'
+  
+  # Directorio temporal de DuckDB para desbordar memoria (spill to disk) durante
+  # transformaciones pesadas o creación de archivos Parquet masivos.
+  # Es muy recomendable que este directorio resida en un disco SSD/NVMe rápido.
+  config.tmp_directory  = '/tmp/duckdb_work'
 end
 ```
 

data/lib/data_drain/configuration.rb

@@ -8,7 +8,8 @@ module DataDrain
     attr_accessor :storage_mode, :aws_region,
       :aws_access_key_id, :aws_secret_access_key,
       :db_host, :db_port, :db_user, :db_pass, :db_name,
-      :batch_size, :throttle_delay, :logger, :limit_ram, :tmp_directory
+      :batch_size, :throttle_delay, :logger, :limit_ram, :tmp_directory,
+      :idle_in_transaction_session_timeout
 
     def initialize
       @storage_mode   = :local
@@ -18,12 +19,13 @@ module DataDrain
       @throttle_delay = 0.5
       @limit_ram      = nil # eg 2GB
       @tmp_directory  = nil # eg /tmp/duckdb_work
+      @idle_in_transaction_session_timeout = 0
       @logger         = Logger.new($stdout)
     end
 
     # @return [String] Cadena de conexión optimizada para DuckDB.
     def duckdb_connection_string
-      "host=#{@db_host} port=#{@db_port} dbname=#{@db_name} user=#{@db_user} password=#{@db_pass}"
+      "postgresql://#{@db_user}:#{@db_pass}@#{@db_host}:#{@db_port}/#{@db_name}?options=-c%20idle_in_transaction_session_timeout%3D#{@idle_in_transaction_session_timeout}"
     end
   end
 end

data/lib/data_drain/engine.rb

@@ -157,6 +157,10 @@ module DataDrain
         dbname:   @config.db_name
       )
 
+      if @config.idle_in_transaction_session_timeout.present?
+        conn.exec("SET idle_in_transaction_session_timeout = #{@config.idle_in_transaction_session_timeout};")
+      end
+
       loop do
         sql = <<~SQL
           DELETE FROM #{@table_name}

data/lib/data_drain/file_ingestor.rb

@@ -37,6 +37,9 @@ module DataDrain
         return false
       end
 
+      @duckdb.query("SET max_memory='#{@config.limit_ram}';") if @config.limit_ram.present?
+      @duckdb.query("SET temp_directory='#{@config.tmp_directory}'") if @config.tmp_directory.present?
+
       @adapter.setup_duckdb(@duckdb)
 
       # Determinamos la función lectora de DuckDB según la extensión del archivo

data/lib/data_drain/record.rb

@@ -30,6 +30,11 @@ module DataDrain
       Thread.current[:data_drain_duckdb_conn] ||= begin
         db = DuckDB::Database.open(":memory:")
         conn = db.connect
+
+        config = DataDrain.configuration
+        conn.query("SET max_memory='#{config.limit_ram}';") if config.limit_ram.present?
+        conn.query("SET temp_directory='#{config.tmp_directory}'") if config.tmp_directory.present?
+
         DataDrain::Storage.adapter.setup_duckdb(conn)
         conn
       end

data/lib/data_drain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataDrain
-  VERSION = "0.1.6"
+  VERSION = "0.1.8"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_drain
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.8
 platform: ruby
 authors:
 - Gabriel