data_drain 0.2.2 → 0.3.1

data/lib/data_drain/storage/s3.rb

@@ -1,10 +1,12 @@
 # frozen_string_literal: true
 
+require "aws-sdk-s3"
+
 module DataDrain
   module Storage
+    # Adaptador de almacenamiento para Amazon S3.
+    # Configura credenciales en DuckDB y provee destrucción de particiones vía AWS SDK.
     class S3 < Base
-      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
-
       # Carga la extensión httpfs en DuckDB e inyecta las credenciales de AWS.
       # Si aws_access_key_id y aws_secret_access_key están seteados, usa
       # credenciales explícitas. Si no, usa credential_chain (IAM role, env vars,
@@ -21,9 +23,7 @@ module DataDrain
       # @param partition_path [String, nil]
       # @return [String]
       def build_path(bucket, folder_name, partition_path)
-        base = File.join(bucket, folder_name)
-        base = File.join(base, partition_path) if partition_path && !partition_path.empty?
-        "s3://#{base}/**/*.parquet"
+        "s3://#{build_path_base(bucket, folder_name, partition_path)}/**/*.parquet"
       end
 
       # @param bucket [String]
@@ -32,34 +32,56 @@ module DataDrain
       # @param partitions [Hash]
       # @return [Integer]
       def destroy_partitions(bucket, folder_name, partition_keys, partitions)
-        client = Aws::S3::Client.new(
+        client = s3_client
+        prefix, pattern_regex = build_destroy_pattern(folder_name, partition_keys, partitions)
+        objects = collect_matching_objects(client, bucket, prefix, pattern_regex)
+        delete_in_batches(client, bucket, objects)
+      end
+
+      private
+
+      # @return [Aws::S3::Client]
+      def s3_client
+        Aws::S3::Client.new(
           region: @config.aws_region,
           access_key_id: @config.aws_access_key_id,
           secret_access_key: @config.aws_secret_access_key
         )
+      end
 
+      # @param folder_name [String]
+      # @param partition_keys [Array<Symbol>]
+      # @param partitions [Hash]
+      # @return [Array(String, Regexp)] prefix y pattern_regex
+      def build_destroy_pattern(folder_name, partition_keys, partitions)
         regex_parts = partition_keys.map do |key|
           val = partitions[key]
           val.nil? || val.to_s.empty? ? "#{key}=[^/]+" : "#{key}=#{val}"
         end
-        pattern_regex = Regexp.new("^#{folder_name}/#{regex_parts.join("/")}")
+        pattern = Regexp.new("^#{folder_name}/#{regex_parts.join("/")}")
 
-        objects_to_delete = []
         prefix = "#{folder_name}/"
         first_key = partition_keys.first
         prefix += "#{first_key}=#{partitions[first_key]}/" if partitions[first_key]
 
+        [prefix, pattern]
+      end
+
+      # @param client [Aws::S3::Client]
+      # @param bucket [String]
+      # @param prefix [String]
+      # @param pattern_regex [Regexp]
+      # @return [Array<Hash>]
+      def collect_matching_objects(client, bucket, prefix, pattern_regex)
+        objects = []
         client.list_objects_v2(bucket: bucket, prefix: prefix).each do |response|
           response.contents.each do |obj|
-            objects_to_delete << { key: obj.key } if obj.key.match?(pattern_regex)
+            objects << { key: obj.key } if obj.key.match?(pattern_regex)
           end
         end
-
-        delete_in_batches(client, bucket, objects_to_delete)
+        objects
       end
 
-      private
-
       # @param connection [DuckDB::Connection]
       # @raise [DataDrain::ConfigurationError]
       def create_s3_secret(connection)
@@ -107,6 +129,5 @@ module DataDrain
         deleted_count
       end
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
   end
 end

data/lib/data_drain/types/json_type.rb

@@ -3,6 +3,7 @@
 require "json"
 
 module DataDrain
+  # Tipos personalizados para ActiveModel registrados por DataDrain.
   module Types
     # Tipo personalizado para ActiveModel que maneja la conversión de
     # cadenas JSON de DuckDB hacia Hashes de Ruby.

data/lib/data_drain/validations.rb

@@ -3,6 +3,8 @@
 module DataDrain
   # Módulo de validación de configuración para prevenir errores de uso.
   module Validations
+    # Regex que valida identificadores SQL (tablas, columnas, etc.).
+    # Permite letras, guiones bajos y números (no al inicio).
     IDENTIFIER_REGEX = /\A[a-zA-Z_][a-zA-Z0-9_]*\z/
 
     module_function

data/lib/data_drain/version.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
 module DataDrain
-  VERSION = "0.2.2"
+  # @return [String] versión semver de la gema
+  VERSION = "0.3.1"
 end

data/lib/data_drain.rb

@@ -7,6 +7,7 @@ require_relative "data_drain/configuration"
 require_relative "data_drain/validations"
 require_relative "data_drain/storage"
 require_relative "data_drain/observability"
+require_relative "data_drain/observability/timing"
 require_relative "data_drain/engine"
 require_relative "data_drain/record"
 require_relative "data_drain/file_ingestor"
@@ -30,6 +31,7 @@ module DataDrain
     end
 
     # @api private
+    # @return [void]
     def reset_configuration!
       @configuration = Configuration.new
       DataDrain::Storage.reset_adapter!

data/skill/references/antipatrones.md

@@ -195,6 +195,16 @@ logger.debug("query=#{expensive_serialize(obj)}")  # Siempre evalúa, incluso si
 logger.debug { "query=#{expensive_serialize(obj)}" }
 ```
 
+**Ejemplo real en DataDrain:**
+
+```ruby
+# Incorrecto — el query puede tener MB de datos en partition_by, se serializa siempre:
+logger.debug("export_query=#{query}")
+
+# Correcto — solo se serializa si DEBUG está activo:
+logger.debug { "export_query=#{query}" }
+```
+
 ---
 
 ## 12. Asumir que `Record.connection` se puede cerrar manualmente

data/skill/references/postgres-tuning.md

@@ -121,6 +121,20 @@ Migrar a tabla particionada cambia DataDrain de "DELETE masivo throttled" a
 DataDrain no detecta particiones automáticamente (futuro item). Hoy el
 operador decide.
 
+## Tuning de parámetros DataDrain por tamaño
+
+| Filas tabla | `batch_size` | `throttle_delay` | `vacuum_after_purge` | `slow_batch_threshold_s` |
+|------------|-------------|-----------------|---------------------|-------------------------|
+| <1M | 5000 | 0.1 | false | 30 |
+| 1M-100M | 5000 | 0.5 | true | 30 |
+| 100M-1B | 10000 | 1.0 | true | 60 |
+| >1B | migrar a particionamiento (ver arriba) | | | |
+
+Contexto operacional:
+- **OLTP concurrente**: `throttle_delay` alto (≥0.5s) para no saturar la DB.
+- **Tablas frías** (sin queries de usuarios): `throttle_delay` 0 OK.
+- **`slow_batch_threshold_s`** alto en tablas grandes porque cada batch tarda más legítimamente.
+
 ## Referencias
 
 - Skill: `.agents/skills/postgresql-optimization/SKILL.md`

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_drain
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.1
 platform: ruby
 authors:
 - Gabriel
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-04-14 00:00:00.000000000 Z
+date: 2026-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -100,7 +100,11 @@ files:
 - docs/IMPROVEMENT_PLAN.md
 - docs/execution/archive/v0.2.0.agente-review.md
 - docs/execution/archive/v0.2.0.md
+- docs/execution/archive/v0.3.0-OBSERVACIONES.md
+- docs/execution/archive/v0.3.0.md
 - docs/execution/v0.2.2.md
+- docs/execution/v0.3.1-OBSERVACIONES.md
+- docs/execution/v0.3.1.md
 - docs/glue_pyspark_example.py
 - lib/data_drain.rb
 - lib/data_drain/configuration.rb
@@ -109,6 +113,7 @@ files:
 - lib/data_drain/file_ingestor.rb
 - lib/data_drain/glue_runner.rb
 - lib/data_drain/observability.rb
+- lib/data_drain/observability/timing.rb
 - lib/data_drain/record.rb
 - lib/data_drain/storage.rb
 - lib/data_drain/storage/base.rb
@@ -134,7 +139,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="