legion-data 1.3.7 → 1.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d10f18bacc47360ad1829628bffc116ecbeea74b0b49359f1cee7fba0a7e9d44
-  data.tar.gz: e13f4acfa8992e3a00129322fef794b80877f723e0c59210016f64fee57c5cae
+  metadata.gz: 8b4deeb282c6d4c359d6fa8d46de2f18f3107746536227171ae4b47b5f80d7b0
+  data.tar.gz: 4f445caf46e5258694170c43ea429a3c6de4f25bd25f013617cf987d335077ad
 SHA512:
-  metadata.gz: 0e61649208294598ecdf19ffca469cba3db7b77d31e0e0a065e4c82e47949e4ea87aa291c5fae18970318e0c27ac1dfcd7eb53143bd7b5acc279eb84c0f2b231
-  data.tar.gz: 7692efc8d123ad178c3f3a66531723518e0fac2cb5999d01833bfa472ac8eb5bee06289dd193886fe66c369f3d49ebff689e68267eff4ab030ce746377a6c9a3
+  metadata.gz: '06930c4164c05d58320e1e5a35b9be15b76e647ba06826b05df8bdd4db156283e232a2109f930e31f78cb892f4cd25324ac941810a377760ee6956577c64fa97'
+  data.tar.gz: 1cbfe73b30462c6b3807d15a494e48b7c4cb986e93b472a1357f722f5e13fb597ff2c70fae0f31911f1ae4711855d91a078c16fbf048835d5483aa849da0d252

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Legion::Data Changelog
 
+## v1.3.8
+
+### Added
+- `Legion::Data::Archival`: hot/warm/cold archival pipeline for tasks and metering records
+- `Legion::Data::Archival::Policy`: configurable retention policies (warm_after_days, cold_after_days, batch_size)
+- Archive, restore, and cross-table search operations with dry-run support
+- Migration 021: archive tables for tasks and metering_records
+
 ## v1.3.7
 
 ### Added

data/lib/legion/data/archival/policy.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Archival
+      class Policy
+        DEFAULTS = {
+          warm_after_days: 7,
+          cold_after_days: 90,
+          batch_size:      1000,
+          tables:          %w[tasks metering_records].freeze
+        }.freeze
+
+        attr_reader :warm_after_days, :cold_after_days, :batch_size, :tables
+
+        def initialize(**opts)
+          config = DEFAULTS.merge(opts)
+          @warm_after_days = config[:warm_after_days]
+          @cold_after_days = config[:cold_after_days]
+          @batch_size = config[:batch_size]
+          @tables = config[:tables]
+        end
+
+        def warm_cutoff
+          Time.now - (warm_after_days * 86_400)
+        end
+
+        def cold_cutoff
+          Time.now - (cold_after_days * 86_400)
+        end
+
+        def self.from_settings
+          return new unless defined?(Legion::Settings)
+
+          data_settings = Legion::Settings[:data]
+          archival = data_settings.is_a?(Hash) ? data_settings[:archival] : nil
+          return new unless archival.is_a?(Hash)
+
+          new(**archival.slice(:warm_after_days, :cold_after_days, :batch_size, :tables))
+        rescue StandardError
+          new
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/archival.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative 'archival/policy'
+
+module Legion
+  module Data
+    module Archival
+      ARCHIVE_TABLE_MAP = {
+        tasks:            :tasks_archive,
+        metering_records: :metering_records_archive
+      }.freeze
+
+      class << self
+        def archive!(policy: Policy.new, dry_run: false)
+          results = {}
+          policy.tables.each do |table_name|
+            table = table_name.to_sym
+            archive_table = ARCHIVE_TABLE_MAP[table]
+            next unless archive_table && db_ready?(table) && db_ready?(archive_table)
+
+            count = archive_table!(
+              source: table, destination: archive_table,
+              cutoff: policy.warm_cutoff, batch_size: policy.batch_size, dry_run: dry_run
+            )
+            results[table] = count
+          end
+          results
+        end
+
+        def restore(table:, ids:)
+          source_table = table.to_sym
+          archive_table = ARCHIVE_TABLE_MAP[source_table]
+          return 0 unless archive_table && db_ready?(archive_table)
+
+          conn = Legion::Data.connection
+          restored = 0
+          conn.transaction do
+            conn[archive_table].where(original_id: ids).each do |row|
+              restore_row = row.except(:id, :archived_at, :original_id, :original_created_at, :original_updated_at)
+              restore_row[:id] = row[:original_id]
+              restore_row[:created_at] = row[:original_created_at]
+              restore_row[:updated_at] = row[:original_updated_at]
+              conn[source_table].insert(restore_row)
+              restored += 1
+            end
+            conn[archive_table].where(original_id: ids).delete
+          end
+          restored
+        end
+
+        def search(table:, where: {})
+          source_table = table.to_sym
+          archive_table = ARCHIVE_TABLE_MAP[source_table]
+          return [] unless db_ready?(source_table)
+
+          conn = Legion::Data.connection
+          hot = conn[source_table].where(where).all
+          warm = db_ready?(archive_table) ? conn[archive_table].where(where).all : []
+          hot + warm
+        end
+
+        private
+
+        def archive_table!(source:, destination:, cutoff:, batch_size:, dry_run:)
+          conn = Legion::Data.connection
+          candidates = conn[source].where { created_at < cutoff }.limit(batch_size)
+          count = candidates.count
+          return count if dry_run || count.zero?
+
+          conn.transaction do
+            candidates.each do |row|
+              archive_row = row.dup
+              archive_row[:original_id] = archive_row.delete(:id)
+              archive_row[:original_created_at] = archive_row.delete(:created_at)
+              archive_row[:original_updated_at] = archive_row.delete(:updated_at)
+              archive_row[:archived_at] = Time.now
+              conn[destination].insert(archive_row)
+            end
+            conn[source].where(id: candidates.select(:id)).delete
+          end
+          count
+        end
+
+        def db_ready?(table)
+          defined?(Legion::Data) && Legion::Data.connection&.table_exists?(table)
+        rescue StandardError
+          false
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/migrations/021_add_archive_tables.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    unless table_exists?(:tasks_archive)
+      create_table(:tasks_archive) do
+        primary_key :id
+        Integer :original_id, null: false
+        String :function_name
+        String :status
+        String :runner_class
+        column :args, :text
+        column :result, :text
+        String :queue
+        Integer :relationship_id
+        String :chain_id
+        DateTime :original_created_at
+        DateTime :original_updated_at
+        DateTime :archived_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+        index :original_id
+        index :chain_id
+        index :archived_at
+      end
+    end
+
+    unless table_exists?(:metering_records_archive)
+      create_table(:metering_records_archive) do
+        primary_key :id
+        Integer :original_id, null: false
+        String :worker_id
+        String :event_type
+        String :extension
+        String :runner_function
+        String :status
+        Integer :tokens_in
+        Integer :tokens_out
+        Float :cost_usd
+        Integer :wall_clock_ms
+        Integer :cpu_time_ms
+        Integer :external_api_calls
+        String :model
+        String :tenant_id
+        DateTime :original_created_at
+        DateTime :archived_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+        index :original_id
+        index :worker_id
+        index :tenant_id
+        index :archived_at
+      end
+    end
+  end
+
+  down do
+    drop_table(:metering_records_archive) if table_exists?(:metering_records_archive)
+    drop_table(:tasks_archive) if table_exists?(:tasks_archive)
+  end
+end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.3.7'
+    VERSION = '1.3.8'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.3.7
+  version: 1.3.8
 platform: ruby
 authors:
 - Esity
@@ -86,6 +86,8 @@ files:
 - exe/legionio_migrate
 - legion-data.gemspec
 - lib/legion/data.rb
+- lib/legion/data/archival.rb
+- lib/legion/data/archival/policy.rb
 - lib/legion/data/connection.rb
 - lib/legion/data/encryption/cipher.rb
 - lib/legion/data/encryption/key_provider.rb
@@ -114,6 +116,7 @@ files:
 - lib/legion/data/migrations/018_add_governance_events.rb
 - lib/legion/data/migrations/019_add_audit_hash_chain.rb
 - lib/legion/data/migrations/020_add_webhooks.rb
+- lib/legion/data/migrations/021_add_archive_tables.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo_access_log.rb
 - lib/legion/data/models/apollo_entry.rb