data_drain 0.2.2 → 0.3.1

data/lib/data_drain/configuration.rb

@@ -9,7 +9,10 @@ module DataDrain
                   :aws_access_key_id, :aws_secret_access_key,
                   :db_host, :db_port, :db_user, :db_pass, :db_name,
                   :batch_size, :throttle_delay, :logger, :limit_ram, :tmp_directory,
-                  :idle_in_transaction_session_timeout
+                  :idle_in_transaction_session_timeout,
+                  :vacuum_after_purge,
+                  :slow_batch_threshold_s,
+                  :slow_batch_alert_after
 
     def initialize
       @storage_mode   = :local
@@ -20,6 +23,9 @@ module DataDrain
       @limit_ram      = nil # eg 2GB
       @tmp_directory  = nil # eg /tmp/duckdb_work
       @idle_in_transaction_session_timeout = 0
+      @vacuum_after_purge = false
+      @slow_batch_threshold_s = 30
+      @slow_batch_alert_after = 5
       @logger = Logger.new($stdout)
     end
 

data/lib/data_drain/engine.rb

@@ -5,12 +5,12 @@ require "pg"
 
 module DataDrain
   # Motor principal de extracción y purga de datos (DataDrain).
-  # rubocop:disable Metrics/ClassLength, Metrics/AbcSize, Metrics/MethodLength, Naming/AccessorMethodName
   #
   # Orquesta el flujo ETL desde PostgreSQL hacia un Data Lake analítico
   # delegando la interacción del almacenamiento al adaptador configurado.
   class Engine
     include Observability
+    include Observability::Timing
     # Inicializa una nueva instancia del motor de extracción.
     #
     # @param options [Hash] Diccionario de configuración para la extracción.
@@ -50,70 +50,92 @@ module DataDrain
       @duckdb = database.connect
     end
 
-    # Ejecuta el flujo completo del motor: Setup, Conteo, Exportación (opcional), Verificación y Purga.
-    #
-    # @return [Boolean] `true` si el proceso finalizó con éxito, `false` si falló la integridad.
+    # @return [Boolean] true si el flujo completó exitosamente, false si falló
     def call
-      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      safe_log(:info, "engine.start",
-               { table: @table_name, start_date: @start_date.to_date, end_date: @end_date.to_date })
+      @durations = {}
+      start_time = monotonic
+      log_start
 
       setup_duckdb
+      return skip_empty(start_time) if step_count.zero?
 
-      # 1. Conteo inicial en Postgres
-      step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      @pg_count = get_postgres_count
-      db_query_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
-
-      if @pg_count.zero?
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-        safe_log(:info, "engine.skip_empty",
-                 { table: @table_name, duration_s: duration.round(2), db_query_duration_s: db_query_duration.round(2) })
-        return true
-      end
-
-      # 2. Exportación
-      export_duration = 0.0
       if @skip_export
         safe_log(:info, "engine.skip_export", { table: @table_name })
       else
-        safe_log(:info, "engine.export_start", { table: @table_name, count: @pg_count })
-        step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        export_to_parquet
-        export_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+        step_export
       end
+      return integrity_failed(start_time) unless step_verify
 
-      # 3. Verificación de Integridad
-      step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      integrity_ok = verify_integrity
-      integrity_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+      step_purge
+      log_complete(start_time)
+      true
+    end
 
-      if integrity_ok
-        # 4. Purga en Postgres
-        step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-        purge_from_postgres
-        purge_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+    private
 
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-        safe_log(:info, "engine.complete", {
-                   table: @table_name,
-                   duration_s: duration.round(2),
-                   db_query_duration_s: db_query_duration.round(2),
-                   export_duration_s: export_duration.round(2),
-                   integrity_duration_s: integrity_duration.round(2),
-                   purge_duration_s: purge_duration.round(2),
-                   count: @pg_count
-                 })
-        true
-      else
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-        safe_log(:error, "engine.integrity_error",
-                 { table: @table_name, duration_s: duration.round(2), count: @pg_count })
-        false
-      end
+    # @api private
+    def log_start
+      safe_log(:info, "engine.start",
+               { table: @table_name, start_date: @start_date.to_date, end_date: @end_date.to_date })
     end
 
-    private
+    # @api private
+    def step_count
+      @pg_count = timed(:db_query) { get_postgres_count }
+      @pg_count
+    end
+
+    # @api private
+    def skip_empty(start_time)
+      duration = monotonic - start_time
+      safe_log(:info, "engine.skip_empty", {
+                 table: @table_name,
+                 duration_s: duration.round(2),
+                 db_query_duration_s: @durations.fetch(:db_query, 0).round(2)
+               })
+      true
+    end
+
+    # @api private
+    def step_export
+      safe_log(:info, "engine.export_start", { table: @table_name, count: @pg_count })
+      timed(:export) { export_to_parquet }
+    end
+
+    # @api private
+    def step_verify
+      timed(:integrity) { verify_integrity }
+    end
+
+    # @api private
+    def step_purge
+      timed(:purge) { purge_from_postgres }
+    end
+
+    # @api private
+    def log_complete(start_time)
+      duration = monotonic - start_time
+      safe_log(:info, "engine.complete", {
+                 table: @table_name,
+                 duration_s: duration.round(2),
+                 db_query_duration_s: @durations.fetch(:db_query, 0).round(2),
+                 export_duration_s: @durations.fetch(:export, 0).round(2),
+                 integrity_duration_s: @durations.fetch(:integrity, 0).round(2),
+                 purge_duration_s: @durations.fetch(:purge, 0).round(2),
+                 count: @pg_count
+               })
+    end
+
+    # @api private
+    def integrity_failed(start_time)
+      duration = monotonic - start_time
+      safe_log(:error, "engine.integrity_error", {
+                 table: @table_name,
+                 duration_s: duration.round(2),
+                 count: @pg_count
+               })
+      false
+    end
 
     # @api private
     # @return [String]
@@ -137,7 +159,7 @@ module DataDrain
     # @api private
     # @return [Integer]
     def get_postgres_count
-      pg_sql = "SELECT COUNT(*) AS row_count FROM public.#{@table_name} WHERE #{base_where_sql}"
+      pg_sql = "SELECT count() AS row_count FROM public.#{@table_name} WHERE #{base_where_sql}"
       pg_sql = pg_sql.gsub("'", "''")
       query = "SELECT row_count FROM postgres_query('pg_source', '#{pg_sql}')"
       @duckdb.query(query).first.first
@@ -182,7 +204,7 @@ module DataDrain
 
       begin
         query = <<~SQL
-          SELECT COUNT(*)
+          SELECT count()
           FROM read_parquet('#{archive_path}')
           WHERE #{base_where_sql}
         SQL
@@ -213,40 +235,129 @@ module DataDrain
         conn.exec("SET idle_in_transaction_session_timeout = #{@config.idle_in_transaction_session_timeout};")
       end
 
+      total_deleted = purge_loop(conn)
+
+      vacuum_if_needed(conn, total_deleted)
+    ensure
+      conn&.close
+    end
+
+    # @api private
+    def vacuum_if_needed(conn, total_deleted)
+      return unless @config.vacuum_after_purge
+      return if total_deleted.zero?
+
+      vacuum_start = monotonic
+      dead_before = fetch_dead_tuple_count(conn)
+
+      begin
+        conn.exec("VACUUM ANALYZE #{@table_name};")
+      rescue PG::Error => e
+        safe_log(:warn, "engine.vacuum_error", {
+          table: @table_name,
+          dead_tuples_before: dead_before,
+          rows_deleted_count: total_deleted,
+          duration_s: (monotonic - vacuum_start).round(2)
+        }.merge(exception_metadata(e)))
+        return
+      end
+
+      dead_after = fetch_dead_tuple_count(conn)
+      vacuum_duration = monotonic - vacuum_start
+
+      safe_log(:info, "engine.vacuum_complete", {
+                 table: @table_name,
+                 duration_s: vacuum_duration.round(2),
+                 dead_tuples_before: dead_before,
+                 dead_tuples_after: dead_after,
+                 rows_deleted_count: total_deleted
+               })
+    end
+
+    # @api private
+    def fetch_dead_tuple_count(conn)
+      result = conn.exec_params(
+        "SELECT n_dead_tup FROM pg_stat_user_tables WHERE relname = $1",
+        [@table_name]
+      )
+      result.first&.dig("n_dead_tup")&.to_i || 0
+    rescue PG::Error
+      -1
+    end
+
+    # @api private
+    # @param conn [PG::Connection]
+    # @return [Integer] total de filas borradas
+    def purge_loop(conn)
       batches_processed = 0
       total_deleted = 0
+      slow_batch_streak = 0
 
       loop do
-        sql = <<~SQL
-          DELETE FROM #{@table_name}
-          WHERE #{@primary_key} IN (
-            SELECT #{@primary_key} FROM #{@table_name}
-            WHERE #{base_where_sql}
-            LIMIT #{@config.batch_size}
-          )
-        SQL
-
-        result = conn.exec(sql)
+        batch_start = monotonic
+        result = conn.exec(build_delete_sql)
+        batch_duration = monotonic - batch_start
         count = result.cmd_tuples
         break if count.zero?
 
         batches_processed += 1
         total_deleted += count
 
-        # Heartbeat cada 100 lotes para monitorear procesos largos de 1TB
-        if (batches_processed % 100).zero?
-          safe_log(:info, "engine.purge_heartbeat", {
+        slow_batch_streak = handle_batch_timing(batch_duration, count, slow_batch_streak)
+        emit_heartbeat_if_due(batches_processed, total_deleted)
+
+        sleep(@config.throttle_delay) if @config.throttle_delay.positive?
+      end
+
+      total_deleted
+    end
+
+    # @api private
+    def handle_batch_timing(batch_duration, count, streak)
+      if batch_duration > @config.slow_batch_threshold_s
+        streak += 1
+        safe_log(:warn, "engine.slow_batch", {
+                   table: @table_name,
+                   batch_duration_s: batch_duration.round(2),
+                   batch_size: count,
+                   streak: streak,
+                   threshold_s: @config.slow_batch_threshold_s
+                 })
+
+        if streak == @config.slow_batch_alert_after
+          safe_log(:warn, "engine.purge_degraded", {
                      table: @table_name,
-                     batches_processed_count: batches_processed,
-                     rows_deleted_count: total_deleted
+                     consecutive_slow_batches: streak,
+                     hint: "considerar índice composite o particionamiento (ver postgres-tuning.md)"
                    })
         end
-
-        sleep(@config.throttle_delay) if @config.throttle_delay.positive?
+        streak
+      else
+        0
       end
-    ensure
-      conn&.close
+    end
+
+    # @api private
+    def emit_heartbeat_if_due(batches_processed, total_deleted)
+      return unless (batches_processed % 100).zero?
+
+      safe_log(:info, "engine.purge_heartbeat", {
+                 table: @table_name,
+                 batches_processed_count: batches_processed,
+                 rows_deleted_count: total_deleted
+               })
+    end
+
+    # @api private
+    def build_delete_sql
+      <<~SQL
+        DELETE FROM #{@table_name}
+        WHERE #{@primary_key} IN (
+          SELECT #{@primary_key} FROM #{@table_name}
+          WHERE #{base_where_sql}
+          LIMIT #{@config.batch_size}
+        )
+      SQL
     end
   end
-  # rubocop:enable Metrics/ClassLength, Metrics/AbcSize, Metrics/MethodLength, Naming/AccessorMethodName
 end

data/lib/data_drain/file_ingestor.rb

@@ -6,8 +6,7 @@ module DataDrain
   # aplicando compresión ZSTD y particionamiento Hive.
   class FileIngestor
     include Observability
-    # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity,
-    #   Metrics/MethodLength
+    include Observability::Timing
 
     # @param options [Hash] Opciones de ingestión.
     # @option options [String] :source_path Ruta absoluta al archivo local.
@@ -36,46 +35,77 @@ module DataDrain
     # Ejecuta el flujo de ingestión.
     # @return [Boolean] true si el proceso fue exitoso.
     def call
-      start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @durations = {}
+      start_time = monotonic
       safe_log(:info, "file_ingestor.start", { source_path: @source_path })
 
-      unless File.exist?(@source_path)
-        safe_log(:error, "file_ingestor.file_not_found", { source_path: @source_path })
-        return false
-      end
+      return file_not_found(start_time) unless step_validate_file
+
+      step_setup_duckdb
+      @reader_function = determine_reader
+      @source_count = step_count_source
+
+      return skip_empty(start_time) if @source_count.zero?
+
+      step_export
+      log_complete(start_time)
+      cleanup_local_file
+      true
+    rescue DuckDB::Error => e
+      duration = monotonic - start_time
+      safe_log(:error, "file_ingestor.duckdb_error",
+               { source_path: @source_path }.merge(exception_metadata(e)).merge(duration_s: duration.round(2)))
+      false
+    ensure
+      @duckdb&.close
+    end
+
+    private
+
+    # @api private
+    def file_not_found(_start_time)
+      safe_log(:error, "file_ingestor.file_not_found", { source_path: @source_path })
+      false
+    end
+
+    # @api private
+    def step_validate_file
+      File.exist?(@source_path)
+    end
 
+    # @api private
+    def step_setup_duckdb
       @duckdb.query("SET max_memory='#{@config.limit_ram}';") if @config.limit_ram.present?
       @duckdb.query("SET temp_directory='#{@config.tmp_directory}'") if @config.tmp_directory.present?
-
       @adapter.setup_duckdb(@duckdb)
+    end
 
-      # Determinamos la función lectora de DuckDB según la extensión del archivo
-      reader_function = determine_reader
-
-      # 1. Conteo de seguridad
-      step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      source_count = @duckdb.query("SELECT COUNT(*) FROM #{reader_function}").first.first
-      source_query_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+    # @api private
+    def step_count_source
+      source_count = timed(:source_query) { @duckdb.query("SELECT count() FROM #{@reader_function}").first.first }
       safe_log(:info, "file_ingestor.count", {
                  source_path: @source_path,
                  count: source_count,
-                 source_query_duration_s: source_query_duration.round(2)
+                 source_query_duration_s: @durations.fetch(:source_query, 0).round(2)
                })
+      source_count
+    end
 
-      if source_count.zero?
-        cleanup_local_file
-        duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-        safe_log(:info, "file_ingestor.skip_empty", { source_path: @source_path, duration_s: duration.round(2) })
-        return true
-      end
+    # @api private
+    def skip_empty(start_time)
+      cleanup_local_file
+      duration = monotonic - start_time
+      safe_log(:info, "file_ingestor.skip_empty", { source_path: @source_path, duration_s: duration.round(2) })
+      true
+    end
 
-      # 2. Exportación / Subida
+    # @api private
+    def step_export
       @adapter.prepare_export_path(@bucket, @folder_name)
       dest_path = if @config.storage_mode.to_sym == :s3
                     "s3://#{@bucket}/#{@folder_name}/"
                   else
-                    File.join(@bucket,
-                              @folder_name, "")
+                    File.join(@bucket, @folder_name, "")
                   end
 
       partition_clause = @partition_keys.any? ? "PARTITION_BY (#{@partition_keys.join(", ")})," : ""
@@ -83,7 +113,7 @@ module DataDrain
       query = <<~SQL
         COPY (
           SELECT #{@select_sql}
-          FROM #{reader_function}
+          FROM #{@reader_function}
         ) TO '#{dest_path}'
         (
           FORMAT PARQUET,
@@ -94,32 +124,21 @@ module DataDrain
       SQL
 
       safe_log(:info, "file_ingestor.export_start", { dest_path: dest_path })
-      step_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      @duckdb.query(query)
-      export_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - step_start
+      timed(:export) { @duckdb.query(query) }
+    end
 
-      duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
+    # @api private
+    def log_complete(start_time)
+      duration = monotonic - start_time
       safe_log(:info, "file_ingestor.complete", {
                  source_path: @source_path,
                  duration_s: duration.round(2),
-                 source_query_duration_s: source_query_duration.round(2),
-                 export_duration_s: export_duration.round(2),
-                 count: source_count
+                 source_query_duration_s: @durations.fetch(:source_query, 0).round(2),
+                 export_duration_s: @durations.fetch(:export, 0).round(2),
+                 count: @source_count
                })
-
-      cleanup_local_file
-      true
-    rescue DuckDB::Error => e
-      duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-      safe_log(:error, "file_ingestor.duckdb_error",
-               { source_path: @source_path }.merge(exception_metadata(e)).merge(duration_s: duration.round(2)))
-      false
-    ensure
-      @duckdb&.close
     end
 
-    private
-
     # @api private
     def determine_reader
       case File.extname(@source_path).downcase
@@ -142,6 +161,4 @@ module DataDrain
       safe_log(:info, "file_ingestor.cleanup", { source_path: @source_path })
     end
   end
-  # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity,
-  #   Metrics/MethodLength
 end

data/lib/data_drain/observability/timing.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module DataDrain
+  module Observability
+    # Helper para medición de duración de operaciones.
+    # @api private
+    module Timing
+      private
+
+      def monotonic
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def timed(step_name)
+        t = monotonic
+        result = yield
+        @durations ||= {}
+        @durations[step_name] = monotonic - t
+        result
+      end
+    end
+  end
+end

data/lib/data_drain/observability.rb

@@ -7,6 +7,8 @@ module DataDrain
   # Este módulo es genérico y puede ser utilizado en otras gemas.
   # @api private
   module Observability
+    # Regex para detectar claves sensibles en logs y enmascararlas preventivamente.
+    # @!visibility private
     SENSITIVE_KEY_PATTERN = /password|passwd|pass|secret|token|api_key|apikey|auth|credential|private_key/i
 
     private

data/lib/data_drain/record.rb

@@ -46,7 +46,6 @@ module DataDrain
     # Esto previene tener que recargar extensiones (como httpfs) en cada consulta.
     #
     # @return [DuckDB::Connection] Conexión activa a DuckDB.
-    # rubocop:disable Metrics/AbcSize
     def self.connection
       Thread.current[:data_drain_duckdb] ||= begin
         db = DuckDB::Database.open(":memory:")
@@ -57,11 +56,13 @@ module DataDrain
         conn.query("SET temp_directory='#{config.tmp_directory}'") if config.tmp_directory.present?
 
         DataDrain::Storage.adapter.setup_duckdb(conn)
+
+        conn.query("SET lock_configuration=true;")
+
         { db: db, conn: conn }
       end
       Thread.current[:data_drain_duckdb][:conn]
     end
-    # rubocop:enable Metrics/AbcSize
 
     # Consulta registros en el Data Lake filtrando por claves de partición.
     #
@@ -138,22 +139,14 @@ module DataDrain
       # @param sql [String]
       # @param columns [Array<String>]
       # @return [Array<DataDrain::Record>]
-      # rubocop:disable Metrics/MethodLength
       def execute_and_instantiate(sql, columns)
         @logger = DataDrain.configuration.logger
-        begin
-          result = connection.query(sql)
-        rescue DuckDB::Error => e
-          safe_log(:warn, "record.parquet_not_found", exception_metadata(e))
-          return []
-        end
-
-        result.map do |row|
-          attributes_hash = columns.zip(row).to_h
-          new(attributes_hash)
-        end
+        result = connection.query(sql)
+        result.map { |row| new(columns.zip(row).to_h) }
+      rescue DuckDB::Error => e
+        safe_log(:warn, "record.parquet_not_found", exception_metadata(e))
+        []
       end
     end
-    # rubocop:enable Metrics/MethodLength
   end
 end

data/lib/data_drain/storage/base.rb

@@ -54,6 +54,18 @@ module DataDrain
       def destroy_partitions(bucket, folder_name, partition_keys, partitions)
         raise NotImplementedError, "#{self.class} debe implementar #destroy_partitions"
       end
+
+      protected
+
+      # @param bucket [String]
+      # @param folder_name [String]
+      # @param partition_path [String, nil]
+      # @return [String] path sin prefix de protocolo ni sufijo glob
+      def build_path_base(bucket, folder_name, partition_path)
+        base = File.join(bucket, folder_name)
+        base = File.join(base, partition_path) if partition_path && !partition_path.empty?
+        base
+      end
     end
   end
 end

data/lib/data_drain/storage/local.rb

@@ -24,9 +24,7 @@ module DataDrain
       # @param partition_path [String, nil]
       # @return [String]
       def build_path(bucket, folder_name, partition_path)
-        base = File.join(bucket, folder_name)
-        base = File.join(base, partition_path) if partition_path && !partition_path.empty?
-        "#{base}/**/*.parquet"
+        "#{build_path_base(bucket, folder_name, partition_path)}/**/*.parquet"
       end
 
       # @param bucket [String]