data_drain 0.1.13 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c484ac47a5f767491fa8d8e48dbdb53ccdd55d756a6a0eb90d7bbeb0d28f68a
-  data.tar.gz: 18526e071ac821f7c19127cb53dad875108ede9ab9b7bfe40a1d17bde877a6cc
+  metadata.gz: 97d660cb624931d75d6f39e51527c58faf180b7ab727d9c85a7fa44079dc76a0
+  data.tar.gz: 932c85dcf3542e52b0f3981281e6a93a757ac194153c8b0b7080a79857613ed5
 SHA512:
-  metadata.gz: abf18e9f987f24cb2e58fb71be8a15f92f867f6e671b174e6414b7a44a5626a316235c091bd59708c1ddc93c755db87ec92af117573c68213d0f2238165728be
-  data.tar.gz: '00124804ef7f7c9dc2c67d47a1a2304d4dc996b0caff24548acbe913e85f5ae43d410eac6725f264f9b9648d49c7dc8bdc0baed77e5ea958bfa3fc8cea08ee9d'
+  metadata.gz: d30e7aaf152e576821b2b2c9a3a68cba01a4c3db6941209e0d0ad0ffb7f69f763e5cf93bd90ac0964a4a2b9b5a5582e348c6f9f5599a5c3ddb24df45168e6418
+  data.tar.gz: f71de76a5075e99eea50a83d0c0d1831091c011a2a64e17b4f3ea206fe8f50ec4bcd2309dfb3096478995c75b4bbfc384431af0d5a5bf3ff446522fa06857891

data/.claude/settings.local.json

@@ -0,0 +1,24 @@
+{
+  "hooks": {
+    "Notification": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "curl -sf -X POST -H \"Content-Type: application/json\" -H \"X-Emdash-Token: $EMDASH_HOOK_TOKEN\" -H \"X-Emdash-Pty-Id: $EMDASH_PTY_ID\" -H \"X-Emdash-Event-Type: notification\" -d @- \"http://127.0.0.1:$EMDASH_HOOK_PORT/hook\" || true"
+          }
+        ]
+      }
+    ],
+    "Stop": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "curl -sf -X POST -H \"Content-Type: application/json\" -H \"X-Emdash-Token: $EMDASH_HOOK_TOKEN\" -H \"X-Emdash-Pty-Id: $EMDASH_PTY_ID\" -H \"X-Emdash-Event-Type: stop\" -d @- \"http://127.0.0.1:$EMDASH_HOOK_PORT/hook\" || true"
+          }
+        ]
+      }
+    ]
+  }
+}

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.1.14] - 2026-03-17
+
+- Feature: Implementación de **Logging Estructurado** en toda la gema (\`key=value\`) para mejor observabilidad en producción.
+- Optimization: Caching automático de adaptadores de almacenamiento para mejorar el rendimiento de consultas repetidas.
+- Testing: Mejora en la robustez de los tests de \`Engine\` desacoplándolos de cambios menores en el setup de DuckDB.
+
 ## [0.1.13] - 2026-03-17
 
 - Feature: Parametrización total en la orquestación con Glue. Se añadieron \`s3_bucket\`, \`s3_folder\` y \`partition_by\` como argumentos dinámicos, permitiendo que el mismo Job de Glue sirva para múltiples tablas y destinos.

data/lib/data_drain/engine.rb

@@ -49,30 +49,30 @@ module DataDrain
     #
     # @return [Boolean] `true` si el proceso finalizó con éxito, `false` si falló la integridad.
     def call
-      @logger.info "[DataDrain Engine] 🚀 Preparando '#{@table_name}' (#{@start_date.to_date} a #{@end_date.to_date})..."
+      @logger.info "component=data_drain event=engine.start table=#{@table_name} start_date=#{@start_date.to_date} end_date=#{@end_date.to_date}"
 
       setup_duckdb
 
       @pg_count = get_postgres_count
 
       if @pg_count.zero?
-        @logger.info "[DataDrain Engine] ⏭️ No hay registros que cumplan las condiciones."
+        @logger.info "component=data_drain event=engine.skip_empty table=#{@table_name}"
         return true
       end
 
       if @skip_export
-        @logger.info "[DataDrain Engine] ⏭️ Modo 'Skip Export' activo. Saltando paso de exportación..."
+        @logger.info "component=data_drain event=engine.skip_export table=#{@table_name}"
       else
-        @logger.info "[DataDrain Engine] 📦 Exportando #{@pg_count} registros a Parquet..."
+        @logger.info "component=data_drain event=engine.export_start table=#{@table_name} count=#{@pg_count}"
         export_to_parquet
       end
 
       if verify_integrity
         purge_from_postgres
-        @logger.info "[DataDrain Engine] ✅ Proceso completado exitosamente para '#{@table_name}'."
+        @logger.info "component=data_drain event=engine.complete table=#{@table_name}"
         true
       else
-        @logger.error "[DataDrain Engine] ❌ ERROR de integridad en '#{@table_name}'. Abortando purga."
+        @logger.error "component=data_drain event=engine.integrity_error table=#{@table_name}"
         false
       end
     end
@@ -147,17 +147,17 @@ module DataDrain
         SQL
         parquet_result = @duckdb.query(query).first.first
       rescue DuckDB::Error => e
-        @logger.error "[DataDrain Engine] ❌ Error leyendo Parquet: #{e.message}"
+        @logger.error "component=data_drain event=engine.parquet_read_error table=#{@table_name} error=#{e.message}"
         return false
       end
 
-      @logger.info "[DataDrain Engine] 📊 Verificación -> Postgres: #{@pg_count} | Parquet: #{parquet_result}"
+      @logger.info "component=data_drain event=engine.integrity_check table=#{@table_name} pg_count=#{@pg_count} parquet_count=#{parquet_result}"
       @pg_count == parquet_result
     end
 
     # @api private
     def purge_from_postgres
-      @logger.info "[DataDrain Engine] 🗑️ Purgando en base de datos (Lotes de #{@config.batch_size})..."
+      @logger.info "component=data_drain event=engine.purge_start table=#{@table_name} batch_size=#{@config.batch_size}"
 
       conn = PG.connect(
         host:     @config.db_host,

data/lib/data_drain/file_ingestor.rb

@@ -30,10 +30,10 @@ module DataDrain
     # Ejecuta el flujo de ingestión.
     # @return [Boolean] true si el proceso fue exitoso.
     def call
-      @logger.info "[DataDrain FileIngestor] 🚀 Iniciando ingestión de '#{@source_path}'..."
+      @logger.info "component=data_drain event=file_ingestor.start source_path=#{@source_path}"
 
       unless File.exist?(@source_path)
-        @logger.error "[DataDrain FileIngestor] ❌ El archivo origen no existe: #{@source_path}"
+        @logger.error "component=data_drain event=file_ingestor.file_not_found source_path=#{@source_path}"
         return false
       end
 
@@ -47,7 +47,7 @@ module DataDrain
 
       # 1. Conteo de seguridad
       source_count = @duckdb.query("SELECT COUNT(*) FROM #{reader_function}").first.first
-      @logger.info "[DataDrain FileIngestor] 📊 Encontrados #{source_count} registros para procesar."
+      @logger.info "component=data_drain event=file_ingestor.count source_path=#{@source_path} count=#{source_count}"
 
       if source_count.zero?
         cleanup_local_file
@@ -73,15 +73,15 @@ module DataDrain
         );
       SQL
 
-      @logger.info "[DataDrain FileIngestor] ☁️ Escribiendo en el Data Lake..."
+      @logger.info "component=data_drain event=file_ingestor.export_start dest_path=#{dest_path}"
       @duckdb.query(query)
 
-      @logger.info "[DataDrain FileIngestor] ✅ Archivo ingerido y comprimido exitosamente."
+      @logger.info "component=data_drain event=file_ingestor.complete source_path=#{@source_path}"
 
       cleanup_local_file
       true
     rescue DuckDB::Error => e
-      @logger.error "[DataDrain FileIngestor] ❌ Error de DuckDB durante la ingestión: #{e.message}"
+      @logger.error "component=data_drain event=file_ingestor.duckdb_error source_path=#{@source_path} error=#{e.message}"
       false
     ensure
       @duckdb&.close
@@ -107,7 +107,7 @@ module DataDrain
     def cleanup_local_file
       if @delete_after_upload && File.exist?(@source_path)
         File.delete(@source_path)
-        @logger.info "[DataDrain FileIngestor] 🗑️ Archivo temporal local eliminado."
+        @logger.info "component=data_drain event=file_ingestor.cleanup source_path=#{@source_path}"
       end
     end
   end

data/lib/data_drain/glue_runner.rb

@@ -17,7 +17,7 @@ module DataDrain
       config = DataDrain.configuration
       client = Aws::Glue::Client.new(region: config.aws_region)
 
-      config.logger.info "[DataDrain GlueRunner] 🚀 Disparando Job: #{job_name}..."
+      config.logger.info "component=data_drain event=glue_runner.start job=#{job_name}"
       resp = client.start_job_run(job_name: job_name, arguments: arguments)
       run_id = resp.job_run_id
 
@@ -27,14 +27,14 @@ module DataDrain
 
         case status
         when "SUCCEEDED"
-          config.logger.info "[DataDrain GlueRunner] ✅ Job completado con éxito."
+          config.logger.info "component=data_drain event=glue_runner.complete job=#{job_name} run_id=#{run_id}"
           return true
         when "FAILED", "STOPPED", "TIMEOUT"
           error_msg = run_info.error_message || "Sin mensaje de error disponible."
-          config.logger.error "[DataDrain GlueRunner] ❌ ERROR: El Job terminó con estado #{status}: #{error_msg}"
+          config.logger.error "component=data_drain event=glue_runner.failed job=#{job_name} run_id=#{run_id} status=#{status} error=#{error_msg}"
           raise "Glue Job #{job_name} (Run ID: #{run_id}) falló con estado #{status}."
         else
-          config.logger.info "[DataDrain GlueRunner] ⏳ Estado: #{status}. Esperando #{polling_interval}s..."
+          config.logger.info "component=data_drain event=glue_runner.polling job=#{job_name} run_id=#{run_id} status=#{status} next_check_in=#{polling_interval}s"
           sleep polling_interval
         end
       end

data/lib/data_drain/record.rb

@@ -85,7 +85,7 @@ module DataDrain
     # @return [Integer] Cantidad de particiones físicas eliminadas.
     def self.destroy_all(**partitions)
       adapter = DataDrain::Storage.adapter
-      DataDrain.configuration.logger.info "[DataDrain] 🗑️ Ejecutando destroy_all en #{folder_name} con: #{partitions.inspect}"
+      DataDrain.configuration.logger.info "component=data_drain event=record.destroy_all folder=#{folder_name} partitions=#{partitions.inspect}"
 
       adapter.destroy_partitions(bucket, folder_name, partition_keys, partitions)
     end
@@ -118,7 +118,7 @@ module DataDrain
         begin
           result = connection.query(sql)
         rescue DuckDB::Error => e
-          DataDrain.configuration.logger.warn "[DataDrain] ⚠️ Ruta o archivo no encontrado: #{e.message}"
+          DataDrain.configuration.logger.warn "component=data_drain event=record.parquet_not_found error=#{e.message}"
           return []
         end
 

data/lib/data_drain/storage.rb

@@ -11,20 +11,28 @@ module DataDrain
     class InvalidAdapterError < DataDrain::Error; end
 
     # Resuelve e instancia el adaptador de almacenamiento correspondiente
-    # basándose en la configuración actual del framework.
+    # basándose en la configuración actual del framework. La instancia se
+    # cachea para evitar allocations innecesarias entre queries.
     #
     # @return [DataDrain::Storage::Base] Una instancia de Local o S3.
     # @raise [InvalidAdapterError] Si el storage_mode no es válido.
     def self.adapter
-      mode = DataDrain.configuration.storage_mode
-      case mode.to_sym
-      when :local
-        Local.new(DataDrain.configuration)
-      when :s3
-        S3.new(DataDrain.configuration)
-      else
-        raise InvalidAdapterError, "Storage mode '#{mode}' no está soportado."
+      @adapter ||= begin
+        mode = DataDrain.configuration.storage_mode
+        case mode.to_sym
+        when :local
+          Local.new(DataDrain.configuration)
+        when :s3
+          S3.new(DataDrain.configuration)
+        else
+          raise InvalidAdapterError, "Storage mode '#{mode}' no está soportado."
+        end
       end
     end
+
+    # Descarta el adaptador cacheado. Llamar cuando cambia storage_mode.
+    def self.reset_adapter!
+      @adapter = nil
+    end
   end
 end

data/lib/data_drain/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataDrain
-  VERSION = "0.1.13"
+  VERSION = "0.1.14"
 end

data/lib/data_drain.rb

@@ -29,6 +29,7 @@ module DataDrain
     # @api private
     def reset_configuration!
       @configuration = Configuration.new
+      DataDrain::Storage.reset_adapter!
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_drain
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.14
 platform: ruby
 authors:
 - Gabriel
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-03-20 00:00:00.000000000 Z
+date: 2026-03-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -88,6 +88,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".claude/settings.local.json"
 - ".rspec"
 - ".rubocop.yml"
 - CHANGELOG.md