legion-data 1.6.18 → 1.6.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4eef1be3fc0e69e96629e2ffb38904d83215045af3b1b9d6ae411a511b4d429
-  data.tar.gz: f8591ff1b36e7d4c29506b16d6ce1e221093c84e6afba3663af721bda4eacb9f
+  metadata.gz: 65eb28ef61c57c2d1248fe3def497d54253c6f694bb7b5b237b635b382c93487
+  data.tar.gz: 3da968d08c194c396029a0a1ccbf81a02afe390257bfca01aac8bc0d89c2169d
 SHA512:
-  metadata.gz: d002c25b1355b3da5770277e486a566967513d1502a09f3b5c19f8e905b3973fb9f0d94856f20082b79168a6035e1d0e71c51a610667c23661a3c06bc60dd1bf
-  data.tar.gz: fbeb431cc34bd8920d77ded479a7beb579bcdb06f500f6174dbcdaef4f6aa13c726a144b6cba8bd4b581fedc4d9129f45d7883263f55f0ae90bf690baf816ea9
+  metadata.gz: fc7ebee421fa890537dd79fbf23280b8952b3d4145c553965678f7822a1229cc56c38ea812e36ec16b9d88f2d8af7de315507d51a5269c0f45dc43b31d136338
+  data.tar.gz: 3a7f266c4a70f307b86e000d41c52a36ee4ad773071c99fce4d75d47d4ef349fa97497b0bc437df0f6b479f34a3994c9aceb753e58f44f966b31ecf2078d7aa4

data/CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## [Unreleased]
 
+## [1.6.19] - 2026-04-02
+
+### Changed
+- Logging uplift across non-API `lib/` modules to use `Legion::Logging::Helper` and `log.*` instead of direct `Legion::Logging.*` calls
+- Removed direct `log_info` / `log_warn` wrapper usage in partition management and aligned logging with helper-backed tagged loggers
+- Added broader info-level operational logs for archival, retention, spool, extract, storage-tier, and partition workflows
+
+### Fixed
+- Added `handle_exception(...)` coverage to rescue paths across non-API data modules so failures are logged consistently without changing existing fallback behavior
+- Added compatibility fallback for `handle_exception` when older `legion-logging` releases are present in the runtime
+- Included `metadata_json` in EventStore integrity hashes for new events while preserving verification compatibility for legacy rows
+- Fixed encrypted Sequel columns to re-encrypt newly-created rows with their persisted primary key and maintain legacy read compatibility
+- Hardened spool persistence with atomic writes, deterministic replay ordering, and corrupt-file quarantine during read/flush
+- Updated partition manager specs to assert against helper-backed logger behavior
+
 ## [1.6.18] - 2026-03-30
 
 ### Added

data/Gemfile

@@ -3,6 +3,7 @@
 source 'https://rubygems.org'
 
 gemspec
+
 group :test do
   gem 'rake'
   gem 'rspec'

data/legion-data.gemspec

@@ -27,8 +27,8 @@ Gem::Specification.new do |spec|
   }
 
   spec.add_dependency 'csv', '>= 3.2'
-  spec.add_dependency 'legion-logging', '>= 1.2.8'
-  spec.add_dependency 'legion-settings', '>= 1.3.12'
+  spec.add_dependency 'legion-logging', '>= 1.5.0'
+  spec.add_dependency 'legion-settings', '>= 1.3.26'
   spec.add_dependency 'sequel', '>= 5.70'
   spec.add_dependency 'sqlite3', '>= 2.0'
 end

data/lib/legion/data/archival/policy.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
+
 module Legion
   module Data
     module Archival
@@ -37,6 +39,10 @@ module Legion
           Time.now - (cold_after_days * 86_400)
         end
 
+        class << self
+          include Legion::Logging::Helper
+        end
+
         def self.from_settings
           return new unless defined?(Legion::Settings)
 
@@ -46,7 +52,7 @@ module Legion
 
           new(**archival.slice(:warm_after_days, :cold_after_days, :batch_size, :tables))
         rescue StandardError => e
-          Legion::Logging.warn("Policy.from_settings failed: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :policy_from_settings)
           new
         end
       end

data/lib/legion/data/archival.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
 require_relative 'archival/policy'
 
 module Legion
@@ -11,21 +12,28 @@ module Legion
       }.freeze
 
       class << self
+        include Legion::Logging::Helper
+
         def archive!(policy: Policy.new, dry_run: false)
+          log.info "Archival run started dry_run=#{dry_run} tables=#{policy.tables.size}"
           results = {}
           policy.tables.each do |table_name|
             table = table_name.to_sym
             archive_table = ARCHIVE_TABLE_MAP[table]
             next unless archive_table && db_ready?(table) && db_ready?(archive_table)
 
-            Legion::Logging.info "Archiving #{table} -> #{archive_table} (cutoff: #{policy.warm_cutoff}, dry_run: #{dry_run})" if defined?(Legion::Logging)
+            log.info "Archiving #{table} -> #{archive_table} (cutoff: #{policy.warm_cutoff}, dry_run: #{dry_run})"
             count = archive_table!(
               source: table, destination: archive_table,
               cutoff: policy.warm_cutoff, batch_size: policy.batch_size, dry_run: dry_run
             )
             results[table] = count
           end
+          log.info "Archival run completed tables=#{results.keys.join(',')}" unless results.empty?
           results
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :archive!, dry_run: dry_run)
+          raise
         end
 
         def restore(table:, ids:)
@@ -46,8 +54,11 @@ module Legion
             end
             conn[archive_table].where(original_id: ids).delete
           end
-          Legion::Logging.info "Restored #{restored} row(s) from #{archive_table} -> #{source_table}" if defined?(Legion::Logging)
+          log.info "Restored #{restored} row(s) from #{archive_table} -> #{source_table}"
           restored
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :restore, table: source_table, ids: Array(ids))
+          raise
         end
 
         def search(table:, where: {})
@@ -55,10 +66,14 @@ module Legion
           archive_table = ARCHIVE_TABLE_MAP[source_table]
           return [] unless db_ready?(source_table)
 
+          log.info "Archival search table=#{source_table} where_keys=#{where.keys.join(',')}"
           conn = Legion::Data.connection
           hot = conn[source_table].where(where).all
           warm = db_ready?(archive_table) ? conn[archive_table].where(where).all : []
           hot + warm
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :search, table: source_table, where_keys: where.keys)
+          raise
         end
 
         def archive_completed_tasks(days_old: 90, batch_size: 1000)
@@ -92,11 +107,15 @@ module Legion
             end
           end
 
-          Legion::Logging.info "archive_completed_tasks: archived #{count} tasks (cutoff: #{cutoff.iso8601})" if defined?(Legion::Logging)
+          log.info "archive_completed_tasks: archived #{count} tasks (cutoff: #{cutoff.iso8601})"
           { archived: count, cutoff: cutoff.iso8601 }
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :archive_completed_tasks, days_old: days_old, batch_size: batch_size)
+          raise
         end
 
         def run_scheduled_archival
+          log.info 'Running scheduled archival'
           results = {}
           results[:tasks] = archive_completed_tasks
 
@@ -107,7 +126,11 @@ module Legion
             )
           end
 
+          log.info "Scheduled archival completed keys=#{results.keys.join(',')}"
           results
+        rescue StandardError => e
+          handle_exception(e, level: :error, handled: false, operation: :run_scheduled_archival)
+          raise
         end
 
         private
@@ -135,7 +158,7 @@ module Legion
         def db_ready?(table)
           defined?(Legion::Data) && Legion::Data.connection&.table_exists?(table)
         rescue StandardError => e
-          Legion::Logging.debug("Archival#db_ready? check failed for #{table}: #{e.message}") if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :archival_db_ready, table: table)
           false
         end
       end

data/lib/legion/data/archiver.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
 require 'digest'
 require 'fileutils'
 require 'json'
@@ -13,62 +14,41 @@ module Legion
       class UploadError < StandardError; end
 
       class << self
+        include Legion::Logging::Helper
+
         def archive_table(table:, retention_days: 90, batch_size: 1000, storage_backend: nil)
           return { skipped: true, reason: 'not_postgres' } unless postgres?
 
-          Legion::Logging.info "Archiving table #{table} (retention: #{retention_days}d)" if defined?(Legion::Logging)
+          log.info "Archiving table #{table} (retention: #{retention_days}d)"
 
           conn = Legion::Data.connection
           cutoff = Time.now - (retention_days * 86_400)
-          now = Time.now.utc
-
-          batches = 0
-          total_rows = 0
-          paths = []
-          batch_n = 0
-
-          loop do
-            batch_n += 1
-            rows = conn[table].where { created_at < cutoff }.limit(batch_size).all
-            break if rows.empty?
-
-            ids = rows.map { |r| r[:id] }
-            jsonl = serialize_rows(rows)
-            compressed = gzip_compress(jsonl)
-            checksum = Digest::SHA256.hexdigest(compressed)
-            batch_id = SecureRandom.uuid
-
-            path = upload_batch(
-              data:    compressed,
-              table:   table.to_s,
-              year:    now.year,
-              month:   now.month,
-              batch_n: batch_n,
-              backend: storage_backend
-            )
-
-            conn.transaction do
-              conn[:archive_manifest].insert(
-                batch_id:     batch_id,
-                source_table: table.to_s,
-                row_count:    rows.size,
-                checksum:     checksum,
-                storage_path: path,
-                archived_at:  now
-              )
-              conn[table].where(id: ids).delete
-            end
-
-            batches += 1
-            total_rows += rows.size
-            paths << path
-          end
-
-          Legion::Logging.info "Archived #{total_rows} rows from #{table} in #{batches} batch(es)" if defined?(Legion::Logging)
-          { batches: batches, total_rows: total_rows, paths: paths }
+          archive_results = archive_batches(
+            conn:            conn,
+            table:           table,
+            cutoff:          cutoff,
+            batch_size:      batch_size,
+            storage_backend: storage_backend
+          )
+
+          log.info "Archived #{archive_results[:total_rows]} rows from #{table} in #{archive_results[:batches]} batch(es)"
+          archive_results
+        rescue StandardError => e
+          handle_exception(
+            e,
+            level:           :error,
+            handled:         false,
+            operation:       :archive_table,
+            table:           table,
+            retention_days:  retention_days,
+            batch_size:      batch_size,
+            storage_backend: storage_backend
+          )
+          raise
         end
 
         def upload_batch(data:, table:, year:, month:, batch_n:, backend:)
+          log.info "Archiver storing batch table=#{table} backend=#{backend || :tmpdir} year=#{year} month=#{month} batch=#{batch_n}"
           case backend
           when :s3
             upload_s3(data: data, table: table, year: year, month: month, batch_n: batch_n)
@@ -111,6 +91,72 @@ module Legion
           rows.map { |row| json_dump(row) }.join("\n")
         end
 
+        def archive_batches(conn:, table:, cutoff:, batch_size:, storage_backend:)
+          now = Time.now.utc
+          batches = 0
+          total_rows = 0
+          paths = []
+
+          loop do
+            batch_result = archive_batch(
+              conn:            conn,
+              table:           table,
+              cutoff:          cutoff,
+              batch_size:      batch_size,
+              batch_n:         batches + 1,
+              now:             now,
+              storage_backend: storage_backend
+            )
+            break unless batch_result
+
+            batches += 1
+            total_rows += batch_result[:row_count]
+            paths << batch_result[:path]
+          end
+
+          { batches: batches, total_rows: total_rows, paths: paths }
+        end
+
+        def archive_batch(conn:, table:, cutoff:, batch_size:, batch_n:, now:, storage_backend:)
+          rows = conn[table].where { created_at < cutoff }.limit(batch_size).all
+          return if rows.empty?
+
+          compressed = gzip_compress(serialize_rows(rows))
+          path = upload_batch(
+            data:    compressed,
+            table:   table.to_s,
+            year:    now.year,
+            month:   now.month,
+            batch_n: batch_n,
+            backend: storage_backend
+          )
+
+          record_archived_batch(
+            conn:       conn,
+            table:      table,
+            rows:       rows,
+            compressed: compressed,
+            path:       path,
+            now:        now
+          )
+
+          { row_count: rows.size, path: path }
+        end
+
+        def record_archived_batch(conn:, table:, rows:, compressed:, path:, now:)
+          conn.transaction do
+            conn[:archive_manifest].insert(
+              batch_id:     SecureRandom.uuid,
+              source_table: table.to_s,
+              row_count:    rows.size,
+              checksum:     Digest::SHA256.hexdigest(compressed),
+              storage_path: path,
+              archived_at:  now
+            )
+            conn[table].where(id: rows.map { |row| row[:id] }).delete
+          end
+        end
+
         def json_dump(obj)
           if defined?(Legion::JSON)
             Legion::JSON.dump(obj)
@@ -133,11 +179,13 @@ module Legion
 
           key = "legion-archive/#{table}/#{year}/#{month}/batch_#{batch_n}.jsonl.gz"
           Legion::Extensions::S3::Runners::Put.run(key: key, body: data)
+          log.info "Archiver uploaded batch to s3 key=#{key}"
           "s3://#{key}"
-        rescue UploadError
+        rescue UploadError => e
+          handle_exception(e, level: :error, handled: false, operation: :upload_s3, table: table, year: year, month: month, batch_n: batch_n)
           raise
         rescue StandardError => e
-          Legion::Logging.warn "S3 upload failed: #{e.message}" if defined?(Legion::Logging)
+          handle_exception(e, level: :error, handled: true, operation: :upload_s3, table: table, year: year, month: month, batch_n: batch_n)
           raise UploadError, "S3 upload failed: #{e.message}"
         end
 
@@ -148,11 +196,13 @@ module Legion
 
           blob_name = "legion-archive/#{table}/#{year}/#{month}/batch_#{batch_n}.jsonl.gz"
           Legion::Extensions::AzureStorage::Runners::Upload.run(blob_name: blob_name, data: data)
+          log.info "Archiver uploaded batch to azure blob=#{blob_name}"
           "azure://#{blob_name}"
-        rescue UploadError
+        rescue UploadError => e
+          handle_exception(e, level: :error, handled: false, operation: :upload_azure, table: table, year: year, month: month, batch_n: batch_n)
           raise
         rescue StandardError => e
-          Legion::Logging.warn "Azure upload failed: #{e.message}" if defined?(Legion::Logging)
+          handle_exception(e, level: :error, handled: false, operation: :upload_azure, table: table, year: year, month: month, batch_n: batch_n)
           raise UploadError, "Azure upload failed: #{e.message}"
         end
 
@@ -161,8 +211,10 @@ module Legion
           FileUtils.mkdir_p(dir)
           path = File.join(dir, "batch_#{batch_n}.jsonl.gz")
           File.binwrite(path, data)
+          log.info "Archiver stored batch locally path=#{path}"
           "file://#{path}"
         rescue StandardError => e
+          handle_exception(e, level: :error, handled: true, operation: :upload_tmpdir, table: table, year: year, month: month, batch_n: batch_n)
           raise UploadError, "Tmpdir upload failed: #{e.message}"
         end
       end

data/lib/legion/data/audit_record.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'legion/logging/helper'
 require 'digest'
 
 module Legion
@@ -8,6 +9,8 @@ module Legion
       GENESIS_HASH = ('0' * 64).freeze
 
       class << self
+        include Legion::Logging::Helper
+
         # Append a new record to the named chain. Returns the persisted record hash
         # on success, or an error hash when the database is unavailable.
         #
@@ -38,7 +41,7 @@ module Legion
               created_at:   ts
             )
 
-            Legion::Logging.debug "AuditRecord append: chain=#{chain_id} type=#{content_type} id=#{id}" if defined?(Legion::Logging)
+            log.debug "AuditRecord append: chain=#{chain_id} type=#{content_type} id=#{id}"
             { id: id, chain_id: chain_id, chain_hash: ch, parent_hash: parent_hash }
           end
         end
@@ -59,13 +62,13 @@ module Legion
           prev_hash = GENESIS_HASH
           records.each do |r|
             unless r[:parent_hash] == prev_hash
-              Legion::Logging.warn "AuditRecord chain broken: chain=#{chain_id} id=#{r[:id]}" if defined?(Legion::Logging)
+              log.warn "AuditRecord chain broken: chain=#{chain_id} id=#{r[:id]}"
               return { valid: false, broken_at: r[:id], reason: :parent_mismatch }
             end
 
             expected = compute_chain_hash(prev_hash, r[:content_hash], r[:created_at], r[:content_type])
             unless r[:chain_hash] == expected
-              Legion::Logging.warn "AuditRecord hash mismatch: chain=#{chain_id} id=#{r[:id]}" if defined?(Legion::Logging)
+              log.warn "AuditRecord hash mismatch: chain=#{chain_id} id=#{r[:id]}"
               return { valid: false, broken_at: r[:id], reason: :hash_mismatch }
             end
 
@@ -142,7 +145,7 @@ module Legion
 
           Legion::Crypt.sign(chain_hash)
         rescue StandardError => e
-          Legion::Logging.warn "AuditRecord signing failed: #{e.message}" if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :sign_record)
           nil
         end
 
@@ -163,7 +166,7 @@ module Legion
         def db_ready?
           defined?(Legion::Data) && Legion::Data.connection&.table_exists?(:audit_records)
         rescue StandardError => e
-          Legion::Logging.debug "AuditRecord#db_ready? check failed: #{e.message}" if defined?(Legion::Logging)
+          handle_exception(e, level: :warn, handled: true, operation: :audit_record_db_ready?)
           false
         end
       end