data_drain 0.2.1 → 0.3.0

data/lib/data_drain/record.rb

@@ -38,14 +38,14 @@ module DataDrain
 
       entry[:conn]&.close
       entry[:db]&.close
-    rescue StandardError # rubocop:disable Lint/SuppressedException
+    rescue StandardError
+      nil
     end
 
     # Retorna la conexión persistente a DuckDB en memoria para el hilo (Thread) actual.
     # Esto previene tener que recargar extensiones (como httpfs) en cada consulta.
     #
     # @return [DuckDB::Connection] Conexión activa a DuckDB.
-    # rubocop:disable Metrics/AbcSize
     def self.connection
       Thread.current[:data_drain_duckdb] ||= begin
         db = DuckDB::Database.open(":memory:")
@@ -56,11 +56,13 @@ module DataDrain
         conn.query("SET temp_directory='#{config.tmp_directory}'") if config.tmp_directory.present?
 
         DataDrain::Storage.adapter.setup_duckdb(conn)
+
+        conn.query("SET lock_configuration=true;")
+
         { db: db, conn: conn }
       end
       Thread.current[:data_drain_duckdb][:conn]
     end
-    # rubocop:enable Metrics/AbcSize
 
     # Consulta registros en el Data Lake filtrando por claves de partición.
     #
@@ -137,22 +139,14 @@ module DataDrain
       # @param sql [String]
       # @param columns [Array<String>]
       # @return [Array<DataDrain::Record>]
-      # rubocop:disable Metrics/MethodLength
       def execute_and_instantiate(sql, columns)
         @logger = DataDrain.configuration.logger
-        begin
-          result = connection.query(sql)
-        rescue DuckDB::Error => e
-          safe_log(:warn, "record.parquet_not_found", exception_metadata(e))
-          return []
-        end
-
-        result.map do |row|
-          attributes_hash = columns.zip(row).to_h
-          new(attributes_hash)
-        end
+        result = connection.query(sql)
+        result.map { |row| new(columns.zip(row).to_h) }
+      rescue DuckDB::Error => e
+        safe_log(:warn, "record.parquet_not_found", exception_metadata(e))
+        []
       end
     end
-    # rubocop:enable Metrics/MethodLength
   end
 end

data/lib/data_drain/storage/s3.rb

@@ -2,9 +2,7 @@
 
 module DataDrain
   module Storage
-    # Implementación del adaptador de almacenamiento para Amazon S3.
     class S3 < Base
-      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
       # Carga la extensión httpfs en DuckDB e inyecta las credenciales de AWS.
       # Si aws_access_key_id y aws_secret_access_key están seteados, usa
       # credenciales explícitas. Si no, usa credential_chain (IAM role, env vars,
@@ -16,42 +14,6 @@ module DataDrain
         create_s3_secret(connection)
       end
 
-      private
-
-      # @param connection [DuckDB::Connection]
-      # @raise [DataDrain::ConfigurationError]
-      def create_s3_secret(connection)
-        region = @config.aws_region
-        raise DataDrain::ConfigurationError, "aws_region es obligatorio para storage_mode=:s3" if region.nil?
-
-        if @config.aws_access_key_id && @config.aws_secret_access_key
-          connection.query(<<~SQL)
-            CREATE OR REPLACE SECRET data_drain_s3 (
-              TYPE S3,
-              KEY_ID '#{escape_sql(@config.aws_access_key_id)}',
-              SECRET '#{escape_sql(@config.aws_secret_access_key)}',
-              REGION '#{escape_sql(region)}'
-            );
-          SQL
-        else
-          connection.query(<<~SQL)
-            CREATE OR REPLACE SECRET data_drain_s3 (
-              TYPE S3,
-              PROVIDER credential_chain,
-              REGION '#{escape_sql(region)}'
-            );
-          SQL
-        end
-      end
-
-      # @param value [String]
-      # @return [String]
-      def escape_sql(value)
-        value.to_s.gsub("'", "''")
-      end
-
-      public
-
       # @param bucket [String]
       # @param folder_name [String]
       # @param partition_path [String, nil]
@@ -68,33 +30,87 @@ module DataDrain
       # @param partitions [Hash]
       # @return [Integer]
       def destroy_partitions(bucket, folder_name, partition_keys, partitions)
-        client = Aws::S3::Client.new(
+        client = s3_client
+        prefix, pattern_regex = build_destroy_pattern(folder_name, partition_keys, partitions)
+        objects = collect_matching_objects(client, bucket, prefix, pattern_regex)
+        delete_in_batches(client, bucket, objects)
+      end
+
+      private
+
+      # @return [Aws::S3::Client]
+      def s3_client
+        Aws::S3::Client.new(
           region: @config.aws_region,
           access_key_id: @config.aws_access_key_id,
           secret_access_key: @config.aws_secret_access_key
         )
+      end
 
+      # @param folder_name [String]
+      # @param partition_keys [Array<Symbol>]
+      # @param partitions [Hash]
+      # @return [Array(String, Regexp)] prefix y pattern_regex
+      def build_destroy_pattern(folder_name, partition_keys, partitions)
         regex_parts = partition_keys.map do |key|
           val = partitions[key]
           val.nil? || val.to_s.empty? ? "#{key}=[^/]+" : "#{key}=#{val}"
         end
-        pattern_regex = Regexp.new("^#{folder_name}/#{regex_parts.join("/")}")
+        pattern = Regexp.new("^#{folder_name}/#{regex_parts.join("/")}")
 
-        objects_to_delete = []
         prefix = "#{folder_name}/"
         first_key = partition_keys.first
         prefix += "#{first_key}=#{partitions[first_key]}/" if partitions[first_key]
 
+        [prefix, pattern]
+      end
+
+      # @param client [Aws::S3::Client]
+      # @param bucket [String]
+      # @param prefix [String]
+      # @param pattern_regex [Regexp]
+      # @return [Array<Hash>]
+      def collect_matching_objects(client, bucket, prefix, pattern_regex)
+        objects = []
         client.list_objects_v2(bucket: bucket, prefix: prefix).each do |response|
           response.contents.each do |obj|
-            objects_to_delete << { key: obj.key } if obj.key.match?(pattern_regex)
+            objects << { key: obj.key } if obj.key.match?(pattern_regex)
           end
         end
+        objects
+      end
+
+      # @param connection [DuckDB::Connection]
+      # @raise [DataDrain::ConfigurationError]
+      def create_s3_secret(connection)
+        region = @config.aws_region
+        raise DataDrain::ConfigurationError, "aws_region es obligatorio para storage_mode=:s3" if region.nil?
 
-        delete_in_batches(client, bucket, objects_to_delete)
+        if @config.aws_access_key_id && @config.aws_secret_access_key
+          connection.query(<<~SQL)
+            CREATE OR REPLACE SECRET data_drain_s3 (
+              TYPE S3,
+              KEY_ID '#{escape_sql(@config.aws_access_key_id)}',
+              SECRET '#{escape_sql(@config.aws_secret_access_key)}',
+              REGION '#{escape_sql(region)}'
+            );
+          SQL
+        else
+          connection.query(<<~SQL)
+            CREATE OR REPLACE SECRET data_drain_s3 (
+              TYPE S3,
+              PROVIDER credential_chain,
+              REGION '#{escape_sql(region)}'
+            );
+          SQL
+        end
       end
 
-      private
+      # @param value [String]
+      # @return [String]
+      def escape_sql(value)
+        value.to_s.gsub("'", "''")
+      end
 
       # @param client [Aws::S3::Client]
       # @param bucket [String]
@@ -111,6 +127,5 @@ module DataDrain
         deleted_count
       end
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
   end
 end

data/lib/data_drain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataDrain
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/lib/data_drain.rb

@@ -7,6 +7,7 @@ require_relative "data_drain/configuration"
 require_relative "data_drain/validations"
 require_relative "data_drain/storage"
 require_relative "data_drain/observability"
+require_relative "data_drain/observability/timing"
 require_relative "data_drain/engine"
 require_relative "data_drain/record"
 require_relative "data_drain/file_ingestor"

data/skill/SKILL.md

@@ -213,3 +213,4 @@ Catálogo completo en [Antipatrones](references/antipatrones.md). Resumen de los
 - [API Detallada](references/api-detallada.md) — Firmas completas, parámetros, retornos y comportamientos de cada clase pública.
 - [Eventos y Telemetría](references/eventos-telemetria.md) — Catálogo completo de eventos KV emitidos por la gema.
 - [Antipatrones](references/antipatrones.md) — Qué NO hacer y alternativas correctas.
+- [Postgres Tuning](references/postgres-tuning.md) — Índices, VACUUM, particionamiento y diagnóstico por tamaño de tabla.

data/skill/references/antipatrones.md

@@ -234,9 +234,26 @@ DataDrain::Engine.new(
 
 **Incorrecto:**
 ```ruby
-DataDrain::GlueRunner.run_and_wait("job", args)  # Asumir que retorna en X minutos
+DataDrain::GlueRunner.run_and_wait("job", args)  # Sin timeout, puede bloquearse
 ```
 
-**Razón:** El loop de polling no tiene timeout máximo. Si Glue queda colgado en `RUNNING` indefinidamente, `run_and_wait` bloquea para siempre.
+**Razón:** Si Glue queda colgado en `RUNNING`, bloquea indefinidamente.
 
-**Alternativa:** Envolver en `Timeout.timeout(N)` en el caller, o monitorear el job desde fuera (CloudWatch alarm). Mejor aún: futura mejora de la gema agregar `max_wait_seconds`.
+**Alternativa:** Usar `max_wait_seconds:` (desde v0.2.2):
+```ruby
+GlueRunner.run_and_wait("job", args, max_wait_seconds: 3600)  # 1h max
+```
+
+---
+
+## 15. Llamar `Engine.new` con configuración incompleta
+
+**Incorrecto:**
+```ruby
+DataDrain::Engine.new(table_name: "versions", start_date: ..., end_date: ...)
+# donde DataDrain.configuration no tiene db_name seteado
+```
+
+**Razón:** Desde v0.2.2, `Engine#initialize` llama `config.validate_for_engine!` que verifica `db_host`, `db_user`, `db_name`. Si alguno falta, levanta `DataDrain::ConfigurationError`.
+
+**Alternativa:** Asegurar que `db_name`, `db_user` y `db_host` estén seteados en `DataDrain.configure` antes de llamar `Engine.new`. Si se usa auth peer/trust, `db_pass` puede ser `nil`.

data/skill/references/api-detallada.md

@@ -40,7 +40,20 @@ Atributos (`attr_accessor`):
 ### `#duckdb_connection_string`
 Retorna URI: `postgresql://user:pass@host:port/db?options=-c%20idle_in_transaction_session_timeout%3D<val>`
 
-**No hay validaciones automáticas.** Caller debe garantizar consistencia (ej. credenciales AWS si `storage_mode = :s3`).
+### `#validate!`
+Valida invariantes generales. Llamada automáticamente por `FileIngestor#initialize` y `GlueRunner.run_and_wait`.
+
+Raises `DataDrain::ConfigurationError` si:
+- `storage_mode` no es `:local` ni `:s3`
+- `storage_mode == :s3` y `aws_region` es nil o vacío
+
+### `#validate_for_engine!`
+Valida invariantes de Engine. Además de `#validate!`, verifica `db_host`, `db_user`, `db_name` no nil ni vacíos.
+
+Llamada automáticamente por `Engine#initialize`.
+
+**No valida `db_pass`** — puede ser nil con auth peer/trust (sockets locales) o IAM (RDS).
+**No valida `db_port`** — tiene default `5432`, nunca nil tras `Configuration#initialize`.
 
 ---
 
@@ -164,24 +177,24 @@ Formato: `#<Class attr1: val1, attr2: val2, ...>`.
 
 ## `DataDrain::GlueRunner`
 
-### `.run_and_wait(job_name, arguments = {}, polling_interval: 30) → true`
+### `.run_and_wait(job_name, arguments = {}, polling_interval: 30, max_wait_seconds: nil) → true`
 
 | Parámetro | Tipo | Descripción |
 |-----------|------|-------------|
 | `job_name` | String | Nombre del Job en consola AWS |
 | `arguments` | Hash | Args con prefijo `--` (ej. `"--start_date" => "..."`) |
 | `polling_interval` | Integer | Segundos entre chequeos. Default `30` |
+| `max_wait_seconds` | Integer, nil | Timeout máximo. nil = sin límite. Default `nil` |
 
 Flujo:
 1. `Aws::Glue::Client.new(region: config.aws_region)`
 2. `start_job_run` → captura `run_id`
 3. Loop: `get_job_run`, evalúa `job_run_state`:
+   - Si `max_wait_seconds` excede → log `glue_runner.timeout`, `raise DataDrain::Error`
    - `SUCCEEDED` → log `glue_runner.complete`, retorna `true`
    - `FAILED|STOPPED|TIMEOUT` → log `glue_runner.failed` (incluye `error_message` truncado a 200 chars), `raise RuntimeError`
    - Otro → log `glue_runner.polling`, `sleep polling_interval`
 
-No tiene timeout máximo. Si Glue queda colgado en `RUNNING`, esto bloquea indefinidamente.
-
 ---
 
 ## `DataDrain::Storage`
@@ -220,7 +233,7 @@ Diseñado para `include` (instance methods, requiere `@logger`) o `extend` (clas
 ### `#safe_log(level, event, metadata = {})` (privado)
 - Si `@logger` es nil, no-op.
 - Construye `fields = { component: observability_name, event: event }.merge(metadata)`.
-- Filtra valores cuyas keys sean `:password|:token|:secret|:api_key|:auth` → `[FILTERED]`.
+- Filtra valores cuyas keys matcheen `SENSITIVE_KEY_PATTERN = /password|passwd|pass|secret|token|api_key|apikey|auth|credential|private_key/i` → `[FILTERED]`. Aplica a claves exactas (`password`) y variantes (`db_password`, `aws_secret_access_key`, `bearer_token`, etc.).
 - Emite `@logger.send(level) { "k1=v1 k2=v2 ..." }`.
 - `rescue StandardError` silencioso (resilience).
 

data/skill/references/eventos-telemetria.md

@@ -128,6 +128,11 @@ Catálogo completo de eventos KV emitidos por DataDrain. Formato Wispro-Observab
 **Campos:** `job`, `run_id`, `status`, `duration_s`, `error_message` (si Glue lo provee, truncado a 200 chars).
 **Consecuencia:** `raise RuntimeError`.
 
+### `glue_runner.timeout`
+**Nivel:** ERROR. Emite cuando `max_wait_seconds` excede antes de `SUCCEEDED`.
+**Campos:** `job`, `run_id`, `max_wait_seconds`.
+**Consecuencia:** `raise DataDrain::Error`.
+
 ---
 
 ## Ejemplos reales

data/skill/references/postgres-tuning.md

@@ -0,0 +1,129 @@
+# Postgres Tuning para DataDrain
+
+Guía operacional para tablas que DataDrain archiva y purga. Cubre índices,
+VACUUM, particionamiento y diagnóstico.
+
+## Tabla de decisión por tamaño
+
+| Tamaño | Estrategia |
+|--------|-----------|
+| <10GB | Índice composite `(created_at, pk)` con `CREATE INDEX CONCURRENTLY` |
+| 10-100GB | Mismo + `SET maintenance_work_mem='4GB'` + checklist |
+| 100GB-1TB | Particionamiento declarativo por mes |
+| >1TB | Particionamiento obligatorio + `DROP PARTITION` reemplaza DELETE |
+
+## Índice recomendado
+
+Para tablas <100GB, DataDrain se beneficia de un índice composite:
+
+    CREATE INDEX CONCURRENTLY idx_versions_created_at_id
+    ON versions (created_at, id);
+
+El DELETE en batches usa `WHERE created_at >= X AND created_at < Y` + `IN (SELECT id LIMIT N)`.
+El índice composite lo convierte en index scan por rango + acceso directo al id.
+
+### Checklist pre-`CREATE INDEX CONCURRENTLY`
+
+- [ ] Tamaño actual: `SELECT pg_size_pretty(pg_total_relation_size('versions'));`
+- [ ] Espacio libre disco (>2x tabla)
+- [ ] `SET maintenance_work_mem = '4GB';` (sesión)
+- [ ] `SET statement_timeout = 0;`
+- [ ] Ventana de baja carga
+- [ ] Plan rollback: `DROP INDEX CONCURRENTLY` si satura I/O
+
+### Riesgos de `CONCURRENTLY`
+
+1. **Dos pasadas** (puede tardar horas en 500GB)
+2. **I/O sostenido** (satura IOPS en EBS gp3 sin provisioned)
+3. **Puede fallar y dejar índice INVALID** → recuperar con `DROP INDEX CONCURRENTLY idx; CREATE INDEX CONCURRENTLY idx ...`
+4. **Espacio en disco alto** durante build (sort externo si `maintenance_work_mem` bajo)
+
+## VACUUM ANALYZE post-purga
+
+En tablas no particionadas, purgar millones de rows deja dead tuples.
+Sin VACUUM, el espacio no se libera y los seq scan recorren páginas vacías.
+
+    VACUUM ANALYZE versions;
+
+Item 5 del roadmap agrega `config.vacuum_after_purge` para automatizar esto.
+Hasta v0.3.0, correr manualmente después de cada `Engine#call` en tablas
+grandes no particionadas.
+
+**NO usar `VACUUM FULL`** — bloquea la tabla entera (ACCESS EXCLUSIVE lock).
+
+## Diagnóstico de purga lenta
+
+    -- Plan del DELETE en lotes
+    EXPLAIN (ANALYZE, BUFFERS)
+    DELETE FROM versions
+    WHERE id IN (
+      SELECT id FROM versions
+      WHERE created_at >= '2026-01-01' AND created_at < '2026-02-01'
+      LIMIT 5000
+    );
+
+    -- Sesiones activas sobre la tabla
+    SELECT pid, state, wait_event, query_start, query
+    FROM pg_stat_activity
+    WHERE query LIKE '%versions%'
+      AND state != 'idle';
+
+    -- Estadísticas de la tabla
+    SELECT relname, n_live_tup, n_dead_tup, last_vacuum, last_autovacuum
+    FROM pg_stat_user_tables
+    WHERE relname = 'versions';
+
+    -- Top queries lentas (requiere pg_stat_statements)
+    SELECT substring(query, 1, 100) AS query, calls, mean_exec_time, rows
+    FROM pg_stat_statements
+    WHERE query LIKE '%versions%'
+    ORDER BY mean_exec_time DESC
+    LIMIT 10;
+
+## Particionamiento declarativo (tablas > 100GB)
+
+Migrar a tabla particionada cambia DataDrain de "DELETE masivo throttled" a
+"DROP PARTITION instantáneo".
+
+### Setup
+
+    -- 1. Crear tabla particionada (vacía, misma estructura que versions)
+    CREATE TABLE versions_new (
+      id UUID PRIMARY KEY,
+      created_at TIMESTAMP NOT NULL,
+      ... -- resto de columnas
+    ) PARTITION BY RANGE (created_at);
+
+    -- 2. Crear partición por mes
+    CREATE TABLE versions_2026_03 PARTITION OF versions_new
+      FOR VALUES FROM ('2026-03-01') TO ('2026-04-01');
+
+    -- 3. Migrar datos (lotes, una partición por vez)
+    INSERT INTO versions_2026_03
+    SELECT * FROM versions
+    WHERE created_at >= '2026-03-01' AND created_at < '2026-04-01';
+
+    -- 4. Swap nombres (downtime mínimo)
+    BEGIN;
+      ALTER TABLE versions RENAME TO versions_old;
+      ALTER TABLE versions_new RENAME TO versions;
+    COMMIT;
+
+### Beneficio para DataDrain
+
+    -- v0.2.x: DELETE en lotes, VACUUM después, horas en TB
+    DataDrain::Engine.new(...).call
+
+    -- Con particiones: DataDrain sigue funcionando pero si el rango
+    -- coincide con una partición, el operador puede hacer:
+    DROP TABLE versions_2026_03;  -- instantáneo, sin bloat
+
+DataDrain no detecta particiones automáticamente (futuro item). Hoy el
+operador decide.
+
+## Referencias
+
+- Skill: `.agents/skills/postgresql-optimization/SKILL.md`
+- PG docs: https://www.postgresql.org/docs/current/ddl-partitioning.html
+- Item 5 roadmap (VACUUM automático): ../IMPROVEMENT_PLAN.md#item-5
+- Item 11b roadmap (warning runtime): ../IMPROVEMENT_PLAN.md#item-11b

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_drain
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Gabriel
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-14 00:00:00.000000000 Z
+date: 2026-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -100,6 +100,7 @@ files:
 - docs/IMPROVEMENT_PLAN.md
 - docs/execution/archive/v0.2.0.agente-review.md
 - docs/execution/archive/v0.2.0.md
+- docs/execution/v0.2.2.md
 - docs/glue_pyspark_example.py
 - lib/data_drain.rb
 - lib/data_drain/configuration.rb
@@ -108,6 +109,7 @@ files:
 - lib/data_drain/file_ingestor.rb
 - lib/data_drain/glue_runner.rb
 - lib/data_drain/observability.rb
+- lib/data_drain/observability/timing.rb
 - lib/data_drain/record.rb
 - lib/data_drain/storage.rb
 - lib/data_drain/storage/base.rb
@@ -121,6 +123,7 @@ files:
 - skill/references/antipatrones.md
 - skill/references/api-detallada.md
 - skill/references/eventos-telemetria.md
+- skill/references/postgres-tuning.md
 homepage: https://github.com/gedera/data_drain
 licenses: []
 metadata: {}