data_drain 0.3.0 → 0.3.1

data/lib/data_drain/engine.rb

@@ -50,6 +50,7 @@ module DataDrain
       @duckdb = database.connect
     end
 
+    # @return [Boolean] true si el flujo completó exitosamente, false si falló
     def call
       @durations = {}
       start_time = monotonic
@@ -158,7 +159,7 @@ module DataDrain
     # @api private
     # @return [Integer]
     def get_postgres_count
-      pg_sql = "SELECT COUNT(*) AS row_count FROM public.#{@table_name} WHERE #{base_where_sql}"
+      pg_sql = "SELECT count() AS row_count FROM public.#{@table_name} WHERE #{base_where_sql}"
       pg_sql = pg_sql.gsub("'", "''")
       query = "SELECT row_count FROM postgres_query('pg_source', '#{pg_sql}')"
       @duckdb.query(query).first.first
@@ -203,7 +204,7 @@ module DataDrain
 
       begin
         query = <<~SQL
-          SELECT COUNT(*)
+          SELECT count()
           FROM read_parquet('#{archive_path}')
           WHERE #{base_where_sql}
         SQL

data/lib/data_drain/file_ingestor.rb

@@ -82,7 +82,7 @@ module DataDrain
 
     # @api private
     def step_count_source
-      source_count = timed(:source_query) { @duckdb.query("SELECT COUNT(*) FROM #{@reader_function}").first.first }
+      source_count = timed(:source_query) { @duckdb.query("SELECT count() FROM #{@reader_function}").first.first }
       safe_log(:info, "file_ingestor.count", {
                  source_path: @source_path,
                  count: source_count,

data/lib/data_drain/observability.rb

@@ -7,6 +7,8 @@ module DataDrain
   # Este módulo es genérico y puede ser utilizado en otras gemas.
   # @api private
   module Observability
+    # Regex para detectar claves sensibles en logs y enmascararlas preventivamente.
+    # @!visibility private
     SENSITIVE_KEY_PATTERN = /password|passwd|pass|secret|token|api_key|apikey|auth|credential|private_key/i
 
     private

data/lib/data_drain/storage/base.rb

@@ -54,6 +54,18 @@ module DataDrain
       def destroy_partitions(bucket, folder_name, partition_keys, partitions)
         raise NotImplementedError, "#{self.class} debe implementar #destroy_partitions"
       end
+
+      protected
+
+      # @param bucket [String]
+      # @param folder_name [String]
+      # @param partition_path [String, nil]
+      # @return [String] path sin prefix de protocolo ni sufijo glob
+      def build_path_base(bucket, folder_name, partition_path)
+        base = File.join(bucket, folder_name)
+        base = File.join(base, partition_path) if partition_path && !partition_path.empty?
+        base
+      end
     end
   end
 end

data/lib/data_drain/storage/local.rb

@@ -24,9 +24,7 @@ module DataDrain
       # @param partition_path [String, nil]
       # @return [String]
       def build_path(bucket, folder_name, partition_path)
-        base = File.join(bucket, folder_name)
-        base = File.join(base, partition_path) if partition_path && !partition_path.empty?
-        "#{base}/**/*.parquet"
+        "#{build_path_base(bucket, folder_name, partition_path)}/**/*.parquet"
       end
 
       # @param bucket [String]

data/lib/data_drain/storage/s3.rb

@@ -1,7 +1,11 @@
 # frozen_string_literal: true
 
+require "aws-sdk-s3"
+
 module DataDrain
   module Storage
+    # Adaptador de almacenamiento para Amazon S3.
+    # Configura credenciales en DuckDB y provee destrucción de particiones vía AWS SDK.
     class S3 < Base
       # Carga la extensión httpfs en DuckDB e inyecta las credenciales de AWS.
       # Si aws_access_key_id y aws_secret_access_key están seteados, usa
@@ -19,9 +23,7 @@ module DataDrain
       # @param partition_path [String, nil]
       # @return [String]
       def build_path(bucket, folder_name, partition_path)
-        base = File.join(bucket, folder_name)
-        base = File.join(base, partition_path) if partition_path && !partition_path.empty?
-        "s3://#{base}/**/*.parquet"
+        "s3://#{build_path_base(bucket, folder_name, partition_path)}/**/*.parquet"
       end
 
       # @param bucket [String]

data/lib/data_drain/types/json_type.rb

@@ -3,6 +3,7 @@
 require "json"
 
 module DataDrain
+  # Tipos personalizados para ActiveModel registrados por DataDrain.
   module Types
     # Tipo personalizado para ActiveModel que maneja la conversión de
     # cadenas JSON de DuckDB hacia Hashes de Ruby.

data/lib/data_drain/validations.rb

@@ -3,6 +3,8 @@
 module DataDrain
   # Módulo de validación de configuración para prevenir errores de uso.
   module Validations
+    # Regex que valida identificadores SQL (tablas, columnas, etc.).
+    # Permite letras, guiones bajos y números (no al inicio).
     IDENTIFIER_REGEX = /\A[a-zA-Z_][a-zA-Z0-9_]*\z/
 
     module_function

data/lib/data_drain/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module DataDrain
-  VERSION = "0.3.0"
+  # @return [String] versión semver de la gema
+  VERSION = "0.3.1"
 end

data/lib/data_drain.rb

@@ -31,6 +31,7 @@ module DataDrain
     end
 
     # @api private
+    # @return [void]
     def reset_configuration!
       @configuration = Configuration.new
       DataDrain::Storage.reset_adapter!

data/skill/references/antipatrones.md

@@ -195,6 +195,16 @@ logger.debug("query=#{expensive_serialize(obj)}")  # Siempre evalúa, incluso si
 logger.debug { "query=#{expensive_serialize(obj)}" }
 ```
 
+**Ejemplo real en DataDrain:**
+
+```ruby
+# Incorrecto — el query puede tener MB de datos en partition_by, se serializa siempre:
+logger.debug("export_query=#{query}")
+
+# Correcto — solo se serializa si DEBUG está activo:
+logger.debug { "export_query=#{query}" }
+```
+
 ---
 
 ## 12. Asumir que `Record.connection` se puede cerrar manualmente

data/skill/references/postgres-tuning.md

@@ -121,6 +121,20 @@ Migrar a tabla particionada cambia DataDrain de "DELETE masivo throttled" a
 DataDrain no detecta particiones automáticamente (futuro item). Hoy el
 operador decide.
 
+## Tuning de parámetros DataDrain por tamaño
+
+| Filas tabla | `batch_size` | `throttle_delay` | `vacuum_after_purge` | `slow_batch_threshold_s` |
+|------------|-------------|-----------------|---------------------|-------------------------|
+| <1M | 5000 | 0.1 | false | 30 |
+| 1M-100M | 5000 | 0.5 | true | 30 |
+| 100M-1B | 10000 | 1.0 | true | 60 |
+| >1B | migrar a particionamiento (ver arriba) | | | |
+
+Contexto operacional:
+- **OLTP concurrente**: `throttle_delay` alto (≥0.5s) para no saturar la DB.
+- **Tablas frías** (sin queries de usuarios): `throttle_delay` 0 OK.
+- **`slow_batch_threshold_s`** alto en tablas grandes porque cada batch tarda más legítimamente.
+
 ## Referencias
 
 - Skill: `.agents/skills/postgresql-optimization/SKILL.md`

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_drain
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Gabriel
@@ -100,7 +100,11 @@ files:
 - docs/IMPROVEMENT_PLAN.md
 - docs/execution/archive/v0.2.0.agente-review.md
 - docs/execution/archive/v0.2.0.md
+- docs/execution/archive/v0.3.0-OBSERVACIONES.md
+- docs/execution/archive/v0.3.0.md
 - docs/execution/v0.2.2.md
+- docs/execution/v0.3.1-OBSERVACIONES.md
+- docs/execution/v0.3.1.md
 - docs/glue_pyspark_example.py
 - lib/data_drain.rb
 - lib/data_drain/configuration.rb
@@ -135,7 +139,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="