data_drain 0.1.5 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05fca5dc293bebcb71247b8659d52933d8a5baca6b73e38ba9d9b21236c02643
-  data.tar.gz: c7d89c3fb45dc4deb3b871bd403a2e1b39172977893882df2658deb8eafcb22f
+  metadata.gz: a5a4767539f7a1e03be113cbdb218eba708228e2d4255734bd6150a913e414a0
+  data.tar.gz: d7a5d26b5ce4cb545a5e2132e09d72b65569aad6c2716b681f9c07dcfea80747
 SHA512:
-  metadata.gz: fce9cc7ecc7008136c456ed3442bdd4b6c8171e0b28c37a1ba6bc897866b3e4469dfd7af8285a9b1022c70f00cf8e14a0e16a8fd280038f2a0b604418471be4b
-  data.tar.gz: 766e0ea20985a90e39fd84727318b6b1a753bad695af44f78bbf94da088f3aef06c1b428811af27fdb46466c9c694a70974ef113f9ee9bdb90b3a616fe353f9c
+  metadata.gz: 5b2c335c98d509d951d6fdea54ad3068c2fd539a88033ef0d4be8e6c12127d019b24086b02eb60c31e63d65ba99bd51daa2d10e595b46c21a01ebdc1516082dc
+  data.tar.gz: 95a6fe6c0b97862c624668a8123383f0206744bfecf09f04e57bc17cc95eba27ea86f0f5d083dd8e948f9490eb640189f8bfcc7b9a39a2c924a419cbe54edda4

data/CHANGELOG.md

@@ -1,8 +1,16 @@
 ## [Unreleased]
 
+## [0.1.7] - 2026-03-16
+
+- Se agrego soporte para idle_in_transaction_session_timeout.
+
+## [0.1.6] - 2026-03-16
+
+- Se agrego el tem_directory para duckdb.
+
 ## [0.1.5] - 2026-03-16
 
-- Se agrego el attacha para duckdb.
+- Se agrego el attach para duckdb.
 
 ## [0.1.4] - 2026-03-16
 

data/README.md

@@ -47,12 +47,26 @@ DataDrain.configure do |config|
   config.db_pass = ENV.fetch('DB_PASS', '')
   config.db_name = ENV.fetch('DB_NAME', 'core_production')
 
-  # Rendimiento y Tuning
+  # Rendimiento y Tuning de Postgres
   config.batch_size     = 5000 # Registros a borrar por transacción
   config.throttle_delay = 0.5  # Segundos de pausa entre borrados
+  
+  # Timeout de inactividad de transacciones en PostgreSQL (en milisegundos).
+  # Útil establecerlo en 0 para evitar que la conexión se cierre prematuramente 
+  # durante el borrado de grandes volúmenes de datos.
+  config.idle_in_transaction_session_timeout = 0
+  
   config.logger         = Rails.logger
 
+  # Tuning de DuckDB
+  # Límite máximo de RAM para las consultas en memoria de DuckDB (ej. '2GB', '512MB').
+  # Evita que el proceso OOM (Out Of Memory) si el contenedor o servidor tiene memoria limitada.
   config.limit_ram      = '2GB'
+  
+  # Directorio temporal de DuckDB para desbordar memoria (spill to disk) durante
+  # transformaciones pesadas o creación de archivos Parquet masivos.
+  # Es muy recomendable que este directorio resida en un disco SSD/NVMe rápido.
+  config.tmp_directory  = '/tmp/duckdb_work'
 end
 ```
 

data/lib/data_drain/configuration.rb

@@ -8,7 +8,8 @@ module DataDrain
     attr_accessor :storage_mode, :aws_region,
       :aws_access_key_id, :aws_secret_access_key,
       :db_host, :db_port, :db_user, :db_pass, :db_name,
-      :batch_size, :throttle_delay, :logger, :limit_ram
+      :batch_size, :throttle_delay, :logger, :limit_ram, :tmp_directory,
+      :idle_in_transaction_session_timeout
 
     def initialize
       @storage_mode   = :local
@@ -17,6 +18,8 @@ module DataDrain
       @batch_size     = 5000
       @throttle_delay = 0.5
       @limit_ram      = nil # eg 2GB
+      @tmp_directory  = nil # eg /tmp/duckdb_work
+      @idle_in_transaction_session_timeout = 0
       @logger         = Logger.new($stdout)
     end
 

data/lib/data_drain/engine.rb

@@ -81,6 +81,7 @@ module DataDrain
     def setup_duckdb
       @duckdb.query("INSTALL postgres; LOAD postgres;")
       @duckdb.query("SET max_memory='#{@config.limit_ram}';") if @config.limit_ram.present?
+      @duckdb.query("SET temp_directory='#{@config.tmp_directory}'") if @config.tmp_directory.present?
       @duckdb.query("ATTACH '#{@config.duckdb_connection_string}' AS pg_source (TYPE POSTGRES, READ_ONLY)")
 
       # 💡 Magia del Adapter: Él sabe si cargar httpfs y setear credenciales o no hacer nada
@@ -156,6 +157,10 @@ module DataDrain
         dbname:   @config.db_name
       )
 
+      if @config.idle_in_transaction_session_timeout.present?
+        conn.exec("SET idle_in_transaction_session_timeout = #{@config.idle_in_transaction_session_timeout};")
+      end
+
       loop do
         sql = <<~SQL
           DELETE FROM #{@table_name}

data/lib/data_drain/file_ingestor.rb

@@ -37,6 +37,9 @@ module DataDrain
         return false
       end
 
+      @duckdb.query("SET max_memory='#{@config.limit_ram}';") if @config.limit_ram.present?
+      @duckdb.query("SET temp_directory='#{@config.tmp_directory}'") if @config.tmp_directory.present?
+
       @adapter.setup_duckdb(@duckdb)
 
       # Determinamos la función lectora de DuckDB según la extensión del archivo

data/lib/data_drain/record.rb

@@ -30,6 +30,11 @@ module DataDrain
       Thread.current[:data_drain_duckdb_conn] ||= begin
         db = DuckDB::Database.open(":memory:")
         conn = db.connect
+
+        config = DataDrain.configuration
+        conn.query("SET max_memory='#{config.limit_ram}';") if config.limit_ram.present?
+        conn.query("SET temp_directory='#{config.tmp_directory}'") if config.tmp_directory.present?
+
         DataDrain::Storage.adapter.setup_duckdb(conn)
         conn
       end

data/lib/data_drain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataDrain
-  VERSION = "0.1.5"
+  VERSION = "0.1.7"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_drain
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.7
 platform: ruby
 authors:
 - Gabriel