data_drain 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ce021179463aeff31239f3bba35fc8b663633bfd648a2584a723b64b5d444387
-  data.tar.gz: b41a91f9a7792f952402c7ac2c9dc981e8372178df200c66e9d7fd8c63753c9c
+  metadata.gz: 8fc6bdf2a5a21825f9bc07185c0fc259056646c6084de2a42fd2d4e3198af4f2
+  data.tar.gz: aa7623b2ec910da4491c38904aa4623fd48a466ccc97a38124f29aafd9677655
 SHA512:
-  metadata.gz: a23130f9892dfc34dfa748a50de6de5f768135d9b3bf012da0f648d322be6b11d0b20bbaf7e783d92a189cfc6e459cb7dc885800ea436f10d177f434e084e018
-  data.tar.gz: acbe4ed9caf168c7519881bce74c8ca90d8392bc8ee1f3718af600e64cfdacd3991f8860691db2b2ad9bb0c949aa88f0015f21e847f540030489951ba81f9ef0
+  metadata.gz: '0201838e917434b529250689a5987008f9e8895366b56a78522752fe5b490fb6ab3d0b4c40dc81bcc830af3132c005fc8d136ab719093a16f0c9de5221e3421c'
+  data.tar.gz: 5be0c9c4a47d07f479f49c977d0490aad909a33fbc0c9292594bd51163132ee5229e0131f23d253120732d530930e5474bd31abbef7a5289f3022055799b5525

data/CHANGELOG.md

@@ -1,5 +1,28 @@
 ## [Unreleased]
 
+## [0.3.0] - 2026-04-15
+
+### Refactor
+- `Engine#call` refactorizado: extraídos `step_count`, `step_export`, `step_verify`, `step_purge` como métodos privados con `timed` helper. CC bajó de 13 a 5. Eventos emitidos idénticos al comportamiento anterior. (item 10)
+- Extraído `DataDrain::Observability::Timing` mixin compartido entre Engine y FileIngestor. (item 20)
+- `FileIngestor#call` refactorizado análogo a Engine. (item 20)
+- Eliminados todos los `# rubocop:disable Metrics/*` en `lib/`. (item 20)
+
+### Features
+- `config.vacuum_after_purge = false` (default). Si `true`, ejecuta `VACUUM ANALYZE` post-purga cuando hubo deletes. Emite `engine.vacuum_complete` con dead_tuples antes/después y duración. Errores PG se capturan como `engine.vacuum_error` WARN. (item 5)
+- `config.slow_batch_threshold_s = 30` y `config.slow_batch_alert_after = 5`. Detecta lotes de purga lentos. Emite `engine.slow_batch` WARN por cada lote lento, `engine.purge_degraded` WARN una vez por streak. Incluye hint a docs de tuning. (item 11b)
+
+### Security
+- `Record.connection` aplica `SET lock_configuration=true` post-setup. Congela cualquier SET futuro sobre la conexión (defensa en profundidad). NO afecta secrets ni extensiones ya cargadas. (item 6)
+
+### Telemetry nueva
+- `engine.vacuum_complete`, `engine.vacuum_error`, `engine.slow_batch`, `engine.purge_degraded`.
+
+### Tests
+- Coverage se mantiene ≥ 80%.
+- Nuevo test de equivalencia para Engine (eventos idénticos pre/post refactor).
+- Timecop agregado para tests de timing (item 11b).
+
 ## [0.2.2] - 2026-04-14
 
 ### Security

data/docs/IMPROVEMENT_PLAN.md

@@ -433,12 +433,13 @@ spec/
 
 #### Item 5 — VACUUM ANALYZE opcional post-purga
 
-**Estado:** `[ ]`
+**Estado:** `[x]`
 **Prioridad:** P1
 **Tipo:** `feat` `perf`
 **Compatibilidad:** backward-compatible (default `false`, opt-in)
 **Estimación:** S (2-3h)
 **Release:** v0.2.1
+**Commit:** `93bf8a8`
 
 ##### Contexto
 
@@ -754,12 +755,13 @@ Contenido:
 
 #### Item 6 — Sandboxing de `Record.connection`
 
-**Estado:** `[ ]`
+**Estado:** `[x]`
 **Prioridad:** P1
 **Tipo:** `security`
 **Compatibilidad:** backward-compatible (con risk de breaking si caller hizo workarounds raros)
 **Estimación:** M (3-4h)
 **Release:** v0.3.0
+**Commit:** `f042c56`
 
 ##### Contexto
 
@@ -800,12 +802,13 @@ Reduce blast radius si alguien intenta inyectar SQL malicioso vía `where_clause
 
 #### Item 10 — Refactor `Engine#call` (CC=13 → ~5)
 
-**Estado:** `[ ]`
+**Estado:** `[x]`
 **Prioridad:** P1
 **Tipo:** `refactor`
 **Compatibilidad:** backward-compatible
 **Estimación:** M (4-6h)
 **Release:** v0.3.0
+**Commit:** `6a06850`
 
 ##### Contexto
 
@@ -878,12 +881,13 @@ Reduce blast radius si alguien intenta inyectar SQL malicioso vía `where_clause
 
 #### Item 11b — Warning runtime de purga lenta sin avance
 
-**Estado:** `[ ]`
+**Estado:** `[x]`
 **Prioridad:** P1
 **Tipo:** `feat` `perf`
 **Compatibilidad:** backward-compatible
 **Estimación:** M (3-4h)
 **Release:** v0.3.0
+**Commit:** `d72ec0a`
 
 ##### Contexto
 
@@ -1206,12 +1210,13 @@ v0.2.1 solo corre CI en Ruby 3.4.4. La gema declara `required_ruby_version = ">=
 
 #### Item 20 — Limpiar `rubocop:disable` en `lib/` agregados en v0.2.0
 
-**Estado:** `[ ]`
+**Estado:** `[x]`
 **Prioridad:** P2
 **Tipo:** `refactor`
 **Compatibilidad:** N/A
 **Estimación:** Depende del item 10 (refactor Engine#call)
 **Release sugerido:** v0.3.0 (junto con item 10)
+**Commit:** `f6f4ddc` (FileIngestor), `02d207c` (S3 refactor), `5522c79` (Timing mixin)
 
 ##### Contexto
 

data/lib/data_drain/configuration.rb

@@ -9,7 +9,10 @@ module DataDrain
                   :aws_access_key_id, :aws_secret_access_key,
                   :db_host, :db_port, :db_user, :db_pass, :db_name,
                   :batch_size, :throttle_delay, :logger, :limit_ram, :tmp_directory,
-                  :idle_in_transaction_session_timeout
+                  :idle_in_transaction_session_timeout,
+                  :vacuum_after_purge,
+                  :slow_batch_threshold_s,
+                  :slow_batch_alert_after
 
     def initialize
       @storage_mode   = :local
@@ -20,6 +23,9 @@ module DataDrain
       @limit_ram      = nil # eg 2GB
       @tmp_directory  = nil # eg /tmp/duckdb_work
       @idle_in_transaction_session_timeout = 0
+      @vacuum_after_purge = false
+      @slow_batch_threshold_s = 30
+      @slow_batch_alert_after = 5
       @logger = Logger.new($stdout)
     end
 

data/lib/data_drain/engine.rb

@@ -5,12 +5,12 @@ require "pg"
 
 module DataDrain
   # Motor principal de extracción y purga de datos (DataDrain).
-  # rubocop:disable Metrics/ClassLength, Metrics/AbcSize, Metrics/MethodLength, Naming/AccessorMethodName
   #
   # Orquesta el flujo ETL desde PostgreSQL hacia un Data Lake analítico
   # delegando la interacción del almacenamiento al adaptador configurado.
   class Engine
     include Observability
+    include Observability::Timing
     # Inicializa una nueva instancia del motor de extracción.
     #
     # @param options [Hash] Diccionario de configuración para la extracción.
@@ -50,70 +50,91 @@ module DataDrain
       @duckdb = database.connect
     end
 
-    # Ejecuta el flujo completo del motor: Setup, Conteo, Exportación (opcional), Verificación y Purga.
-    #
-    # @return [Boolean] `true` si el proceso finalizó con éxito, `false` si falló la integridad.
     def call
-      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      safe_log(:info, "engine.start",
-               { table: @table_name, start_date: @start_date.to_date, end_date: @end_date.to_date })
+      @durations = {}
+      start_time = monotonic
+      log_start
 
       setup_duckdb
+      return skip_empty(start_time) if step_count.zero?
 
-      # 1. Conteo inicial en Postgres
-      step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      @pg_count = get_postgres_count
-      db_query_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
-
-      if @pg_count.zero?
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-        safe_log(:info, "engine.skip_empty",
-                 { table: @table_name, duration_s: duration.round(2), db_query_duration_s: db_query_duration.round(2) })
-        return true
-      end
-
-      # 2. Exportación
-      export_duration = 0.0
       if @skip_export
         safe_log(:info, "engine.skip_export", { table: @table_name })
       else
-        safe_log(:info, "engine.export_start", { table: @table_name, count: @pg_count })
-        step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        export_to_parquet
-        export_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+        step_export
       end
+      return integrity_failed(start_time) unless step_verify
 
-      # 3. Verificación de Integridad
-      step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      integrity_ok = verify_integrity
-      integrity_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+      step_purge
+      log_complete(start_time)
+      true
+    end
 
-      if integrity_ok
-        # 4. Purga en Postgres
-        step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        purge_from_postgres
-        purge_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+    private
 
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-        safe_log(:info, "engine.complete", {
-                   table: @table_name,
-                   duration_s: duration.round(2),
-                   db_query_duration_s: db_query_duration.round(2),
-                   export_duration_s: export_duration.round(2),
-                   integrity_duration_s: integrity_duration.round(2),
-                   purge_duration_s: purge_duration.round(2),
-                   count: @pg_count
-                 })
-        true
-      else
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-        safe_log(:error, "engine.integrity_error",
-                 { table: @table_name, duration_s: duration.round(2), count: @pg_count })
-        false
-      end
+    # @api private
+    def log_start
+      safe_log(:info, "engine.start",
+               { table: @table_name, start_date: @start_date.to_date, end_date: @end_date.to_date })
     end
 
-    private
+    # @api private
+    def step_count
+      @pg_count = timed(:db_query) { get_postgres_count }
+      @pg_count
+    end
+
+    # @api private
+    def skip_empty(start_time)
+      duration = monotonic - start_time
+      safe_log(:info, "engine.skip_empty", {
+                 table: @table_name,
+                 duration_s: duration.round(2),
+                 db_query_duration_s: @durations.fetch(:db_query, 0).round(2)
+               })
+      true
+    end
+
+    # @api private
+    def step_export
+      safe_log(:info, "engine.export_start", { table: @table_name, count: @pg_count })
+      timed(:export) { export_to_parquet }
+    end
+
+    # @api private
+    def step_verify
+      timed(:integrity) { verify_integrity }
+    end
+
+    # @api private
+    def step_purge
+      timed(:purge) { purge_from_postgres }
+    end
+
+    # @api private
+    def log_complete(start_time)
+      duration = monotonic - start_time
+      safe_log(:info, "engine.complete", {
+                 table: @table_name,
+                 duration_s: duration.round(2),
+                 db_query_duration_s: @durations.fetch(:db_query, 0).round(2),
+                 export_duration_s: @durations.fetch(:export, 0).round(2),
+                 integrity_duration_s: @durations.fetch(:integrity, 0).round(2),
+                 purge_duration_s: @durations.fetch(:purge, 0).round(2),
+                 count: @pg_count
+               })
+    end
+
+    # @api private
+    def integrity_failed(start_time)
+      duration = monotonic - start_time
+      safe_log(:error, "engine.integrity_error", {
+                 table: @table_name,
+                 duration_s: duration.round(2),
+                 count: @pg_count
+               })
+      false
+    end
 
     # @api private
     # @return [String]
@@ -213,40 +234,129 @@ module DataDrain
         conn.exec("SET idle_in_transaction_session_timeout = #{@config.idle_in_transaction_session_timeout};")
       end
 
+      total_deleted = purge_loop(conn)
+
+      vacuum_if_needed(conn, total_deleted)
+    ensure
+      conn&.close
+    end
+
+    # @api private
+    def vacuum_if_needed(conn, total_deleted)
+      return unless @config.vacuum_after_purge
+      return if total_deleted.zero?
+
+      vacuum_start = monotonic
+      dead_before = fetch_dead_tuple_count(conn)
+
+      begin
+        conn.exec("VACUUM ANALYZE #{@table_name};")
+      rescue PG::Error => e
+        safe_log(:warn, "engine.vacuum_error", {
+          table: @table_name,
+          dead_tuples_before: dead_before,
+          rows_deleted_count: total_deleted,
+          duration_s: (monotonic - vacuum_start).round(2)
+        }.merge(exception_metadata(e)))
+        return
+      end
+
+      dead_after = fetch_dead_tuple_count(conn)
+      vacuum_duration = monotonic - vacuum_start
+
+      safe_log(:info, "engine.vacuum_complete", {
+                 table: @table_name,
+                 duration_s: vacuum_duration.round(2),
+                 dead_tuples_before: dead_before,
+                 dead_tuples_after: dead_after,
+                 rows_deleted_count: total_deleted
+               })
+    end
+
+    # @api private
+    def fetch_dead_tuple_count(conn)
+      result = conn.exec_params(
+        "SELECT n_dead_tup FROM pg_stat_user_tables WHERE relname = $1",
+        [@table_name]
+      )
+      result.first&.dig("n_dead_tup")&.to_i || 0
+    rescue PG::Error
+      -1
+    end
+
+    # @api private
+    # @param conn [PG::Connection]
+    # @return [Integer] total de filas borradas
+    def purge_loop(conn)
       batches_processed = 0
       total_deleted = 0
+      slow_batch_streak = 0
 
       loop do
-        sql = <<~SQL
-          DELETE FROM #{@table_name}
-          WHERE #{@primary_key} IN (
-            SELECT #{@primary_key} FROM #{@table_name}
-            WHERE #{base_where_sql}
-            LIMIT #{@config.batch_size}
-          )
-        SQL
-
-        result = conn.exec(sql)
+        batch_start = monotonic
+        result = conn.exec(build_delete_sql)
+        batch_duration = monotonic - batch_start
         count = result.cmd_tuples
         break if count.zero?
 
         batches_processed += 1
         total_deleted += count
 
-        # Heartbeat cada 100 lotes para monitorear procesos largos de 1TB
-        if (batches_processed % 100).zero?
-          safe_log(:info, "engine.purge_heartbeat", {
+        slow_batch_streak = handle_batch_timing(batch_duration, count, slow_batch_streak)
+        emit_heartbeat_if_due(batches_processed, total_deleted)
+
+        sleep(@config.throttle_delay) if @config.throttle_delay.positive?
+      end
+
+      total_deleted
+    end
+
+    # @api private
+    def handle_batch_timing(batch_duration, count, streak)
+      if batch_duration > @config.slow_batch_threshold_s
+        streak += 1
+        safe_log(:warn, "engine.slow_batch", {
+                   table: @table_name,
+                   batch_duration_s: batch_duration.round(2),
+                   batch_size: count,
+                   streak: streak,
+                   threshold_s: @config.slow_batch_threshold_s
+                 })
+
+        if streak == @config.slow_batch_alert_after
+          safe_log(:warn, "engine.purge_degraded", {
                      table: @table_name,
-                     batches_processed_count: batches_processed,
-                     rows_deleted_count: total_deleted
+                     consecutive_slow_batches: streak,
+                     hint: "considerar índice composite o particionamiento (ver postgres-tuning.md)"
                    })
         end
-
-        sleep(@config.throttle_delay) if @config.throttle_delay.positive?
+        streak
+      else
+        0
       end
-    ensure
-      conn&.close
+    end
+
+    # @api private
+    def emit_heartbeat_if_due(batches_processed, total_deleted)
+      return unless (batches_processed % 100).zero?
+
+      safe_log(:info, "engine.purge_heartbeat", {
+                 table: @table_name,
+                 batches_processed_count: batches_processed,
+                 rows_deleted_count: total_deleted
+               })
+    end
+
+    # @api private
+    def build_delete_sql
+      <<~SQL
+        DELETE FROM #{@table_name}
+        WHERE #{@primary_key} IN (
+          SELECT #{@primary_key} FROM #{@table_name}
+          WHERE #{base_where_sql}
+          LIMIT #{@config.batch_size}
+        )
+      SQL
     end
   end
-  # rubocop:enable Metrics/ClassLength, Metrics/AbcSize, Metrics/MethodLength, Naming/AccessorMethodName
 end

data/lib/data_drain/file_ingestor.rb

@@ -6,8 +6,7 @@ module DataDrain
   # aplicando compresión ZSTD y particionamiento Hive.
   class FileIngestor
     include Observability
-    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity,
-    #   Metrics/MethodLength
+    include Observability::Timing
 
     # @param options [Hash] Opciones de ingestión.
     # @option options [String] :source_path Ruta absoluta al archivo local.
@@ -36,46 +35,77 @@ module DataDrain
     # Ejecuta el flujo de ingestión.
     # @return [Boolean] true si el proceso fue exitoso.
     def call
-      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @durations = {}
+      start_time = monotonic
       safe_log(:info, "file_ingestor.start", { source_path: @source_path })
 
-      unless File.exist?(@source_path)
-        safe_log(:error, "file_ingestor.file_not_found", { source_path: @source_path })
-        return false
-      end
+      return file_not_found(start_time) unless step_validate_file
+
+      step_setup_duckdb
+      @reader_function = determine_reader
+      @source_count = step_count_source
+
+      return skip_empty(start_time) if @source_count.zero?
+
+      step_export
+      log_complete(start_time)
+      cleanup_local_file
+      true
+    rescue DuckDB::Error => e
+      duration = monotonic - start_time
+      safe_log(:error, "file_ingestor.duckdb_error",
+               { source_path: @source_path }.merge(exception_metadata(e)).merge(duration_s: duration.round(2)))
+      false
+    ensure
+      @duckdb&.close
+    end
+
+    private
+
+    # @api private
+    def file_not_found(_start_time)
+      safe_log(:error, "file_ingestor.file_not_found", { source_path: @source_path })
+      false
+    end
+
+    # @api private
+    def step_validate_file
+      File.exist?(@source_path)
+    end
 
+    # @api private
+    def step_setup_duckdb
       @duckdb.query("SET max_memory='#{@config.limit_ram}';") if @config.limit_ram.present?
       @duckdb.query("SET temp_directory='#{@config.tmp_directory}'") if @config.tmp_directory.present?
-
       @adapter.setup_duckdb(@duckdb)
+    end
 
-      # Determinamos la función lectora de DuckDB según la extensión del archivo
-      reader_function = determine_reader
-
-      # 1. Conteo de seguridad
-      step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      source_count = @duckdb.query("SELECT COUNT(*) FROM #{reader_function}").first.first
-      source_query_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+    # @api private
+    def step_count_source
+      source_count = timed(:source_query) { @duckdb.query("SELECT COUNT(*) FROM #{@reader_function}").first.first }
       safe_log(:info, "file_ingestor.count", {
                  source_path: @source_path,
                  count: source_count,
-                 source_query_duration_s: source_query_duration.round(2)
+                 source_query_duration_s: @durations.fetch(:source_query, 0).round(2)
                })
+      source_count
+    end
 
-      if source_count.zero?
-        cleanup_local_file
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-        safe_log(:info, "file_ingestor.skip_empty", { source_path: @source_path, duration_s: duration.round(2) })
-        return true
-      end
+    # @api private
+    def skip_empty(start_time)
+      cleanup_local_file
+      duration = monotonic - start_time
+      safe_log(:info, "file_ingestor.skip_empty", { source_path: @source_path, duration_s: duration.round(2) })
+      true
+    end
 
-      # 2. Exportación / Subida
+    # @api private
+    def step_export
       @adapter.prepare_export_path(@bucket, @folder_name)
       dest_path = if @config.storage_mode.to_sym == :s3
                     "s3://#{@bucket}/#{@folder_name}/"
                   else
-                    File.join(@bucket,
-                              @folder_name, "")
+                    File.join(@bucket, @folder_name, "")
                   end
 
       partition_clause = @partition_keys.any? ? "PARTITION_BY (#{@partition_keys.join(", ")})," : ""
@@ -83,7 +113,7 @@ module DataDrain
       query = <<~SQL
         COPY (
           SELECT #{@select_sql}
-          FROM #{reader_function}
+          FROM #{@reader_function}
         ) TO '#{dest_path}'
         (
           FORMAT PARQUET,
@@ -94,32 +124,21 @@ module DataDrain
       SQL
 
       safe_log(:info, "file_ingestor.export_start", { dest_path: dest_path })
-      step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      @duckdb.query(query)
-      export_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+      timed(:export) { @duckdb.query(query) }
+    end
 
-      duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+    # @api private
+    def log_complete(start_time)
+      duration = monotonic - start_time
       safe_log(:info, "file_ingestor.complete", {
                  source_path: @source_path,
                  duration_s: duration.round(2),
-                 source_query_duration_s: source_query_duration.round(2),
-                 export_duration_s: export_duration.round(2),
-                 count: source_count
+                 source_query_duration_s: @durations.fetch(:source_query, 0).round(2),
+                 export_duration_s: @durations.fetch(:export, 0).round(2),
+                 count: @source_count
                })
-
-      cleanup_local_file
-      true
-    rescue DuckDB::Error => e
-      duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-      safe_log(:error, "file_ingestor.duckdb_error",
-               { source_path: @source_path }.merge(exception_metadata(e)).merge(duration_s: duration.round(2)))
-      false
-    ensure
-      @duckdb&.close
     end
 
-    private
-
     # @api private
     def determine_reader
       case File.extname(@source_path).downcase
@@ -142,6 +161,4 @@ module DataDrain
       safe_log(:info, "file_ingestor.cleanup", { source_path: @source_path })
     end
   end
-  # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity,
-  #   Metrics/MethodLength
 end

data/lib/data_drain/observability/timing.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module DataDrain
+  module Observability
+    # Helper para medición de duración de operaciones.
+    # @api private
+    module Timing
+      private
+
+      def monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def timed(step_name)
+        t = monotonic
+        result = yield
+        @durations ||= {}
+        @durations[step_name] = monotonic - t
+        result
+      end
+    end
+  end
+end

data/lib/data_drain/record.rb

@@ -46,7 +46,6 @@ module DataDrain
     # Esto previene tener que recargar extensiones (como httpfs) en cada consulta.
     #
     # @return [DuckDB::Connection] Conexión activa a DuckDB.
-    # rubocop:disable Metrics/AbcSize
     def self.connection
       Thread.current[:data_drain_duckdb] ||= begin
         db = DuckDB::Database.open(":memory:")
@@ -57,11 +56,13 @@ module DataDrain
         conn.query("SET temp_directory='#{config.tmp_directory}'") if config.tmp_directory.present?
 
         DataDrain::Storage.adapter.setup_duckdb(conn)
+
+        conn.query("SET lock_configuration=true;")
+
         { db: db, conn: conn }
       end
       Thread.current[:data_drain_duckdb][:conn]
     end
-    # rubocop:enable Metrics/AbcSize
 
     # Consulta registros en el Data Lake filtrando por claves de partición.
     #
@@ -138,22 +139,14 @@ module DataDrain
       # @param sql [String]
       # @param columns [Array<String>]
       # @return [Array<DataDrain::Record>]
-      # rubocop:disable Metrics/MethodLength
       def execute_and_instantiate(sql, columns)
         @logger = DataDrain.configuration.logger
-        begin
-          result = connection.query(sql)
-        rescue DuckDB::Error => e
-          safe_log(:warn, "record.parquet_not_found", exception_metadata(e))
-          return []
-        end
-
-        result.map do |row|
-          attributes_hash = columns.zip(row).to_h
-          new(attributes_hash)
-        end
+        result = connection.query(sql)
+        result.map { |row| new(columns.zip(row).to_h) }
+      rescue DuckDB::Error => e
+        safe_log(:warn, "record.parquet_not_found", exception_metadata(e))
+        []
       end
     end
-    # rubocop:enable Metrics/MethodLength
   end
 end

data/lib/data_drain/storage/s3.rb

@@ -3,8 +3,6 @@
 module DataDrain
   module Storage
     class S3 < Base
-      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-
       # Carga la extensión httpfs en DuckDB e inyecta las credenciales de AWS.
       # Si aws_access_key_id y aws_secret_access_key están seteados, usa
       # credenciales explícitas. Si no, usa credential_chain (IAM role, env vars,
@@ -32,34 +30,56 @@ module DataDrain
       # @param partitions [Hash]
       # @return [Integer]
       def destroy_partitions(bucket, folder_name, partition_keys, partitions)
-        client = Aws::S3::Client.new(
+        client = s3_client
+        prefix, pattern_regex = build_destroy_pattern(folder_name, partition_keys, partitions)
+        objects = collect_matching_objects(client, bucket, prefix, pattern_regex)
+        delete_in_batches(client, bucket, objects)
+      end
+
+      private
+
+      # @return [Aws::S3::Client]
+      def s3_client
+        Aws::S3::Client.new(
           region: @config.aws_region,
           access_key_id: @config.aws_access_key_id,
           secret_access_key: @config.aws_secret_access_key
         )
+      end
 
+      # @param folder_name [String]
+      # @param partition_keys [Array<Symbol>]
+      # @param partitions [Hash]
+      # @return [Array(String, Regexp)] prefix y pattern_regex
+      def build_destroy_pattern(folder_name, partition_keys, partitions)
         regex_parts = partition_keys.map do |key|
           val = partitions[key]
           val.nil? || val.to_s.empty? ? "#{key}=[^/]+" : "#{key}=#{val}"
         end
-        pattern_regex = Regexp.new("^#{folder_name}/#{regex_parts.join("/")}")
+        pattern = Regexp.new("^#{folder_name}/#{regex_parts.join("/")}")
 
-        objects_to_delete = []
         prefix = "#{folder_name}/"
         first_key = partition_keys.first
         prefix += "#{first_key}=#{partitions[first_key]}/" if partitions[first_key]
 
+        [prefix, pattern]
+      end
+
+      # @param client [Aws::S3::Client]
+      # @param bucket [String]
+      # @param prefix [String]
+      # @param pattern_regex [Regexp]
+      # @return [Array<Hash>]
+      def collect_matching_objects(client, bucket, prefix, pattern_regex)
+        objects = []
         client.list_objects_v2(bucket: bucket, prefix: prefix).each do |response|
           response.contents.each do |obj|
-            objects_to_delete << { key: obj.key } if obj.key.match?(pattern_regex)
+            objects << { key: obj.key } if obj.key.match?(pattern_regex)
           end
         end
-
-        delete_in_batches(client, bucket, objects_to_delete)
+        objects
       end
 
-      private
-
       # @param connection [DuckDB::Connection]
       # @raise [DataDrain::ConfigurationError]
       def create_s3_secret(connection)
@@ -107,6 +127,5 @@ module DataDrain
         deleted_count
       end
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
   end
 end

data/lib/data_drain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataDrain
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/data_drain.rb

@@ -7,6 +7,7 @@ require_relative "data_drain/configuration"
 require_relative "data_drain/validations"
 require_relative "data_drain/storage"
 require_relative "data_drain/observability"
+require_relative "data_drain/observability/timing"
 require_relative "data_drain/engine"
 require_relative "data_drain/record"
 require_relative "data_drain/file_ingestor"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_drain
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Gabriel
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-14 00:00:00.000000000 Z
+date: 2026-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -109,6 +109,7 @@ files:
 - lib/data_drain/file_ingestor.rb
 - lib/data_drain/glue_runner.rb
 - lib/data_drain/observability.rb
+- lib/data_drain/observability/timing.rb
 - lib/data_drain/record.rb
 - lib/data_drain/storage.rb
 - lib/data_drain/storage/base.rb