legion-data 1.3.7 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d10f18bacc47360ad1829628bffc116ecbeea74b0b49359f1cee7fba0a7e9d44
-  data.tar.gz: e13f4acfa8992e3a00129322fef794b80877f723e0c59210016f64fee57c5cae
+  metadata.gz: eaf37e68acae5015ef50b01e6a5c7a4806e81b80f5cf74f3c71b72e6799f0bd4
+  data.tar.gz: 81b55083c6e8d59403b51a838935a4146b1d3f39eca8316dfd82ee4d2f6a0b05
 SHA512:
-  metadata.gz: 0e61649208294598ecdf19ffca469cba3db7b77d31e0e0a065e4c82e47949e4ea87aa291c5fae18970318e0c27ac1dfcd7eb53143bd7b5acc279eb84c0f2b231
-  data.tar.gz: 7692efc8d123ad178c3f3a66531723518e0fac2cb5999d01833bfa472ac8eb5bee06289dd193886fe66c369f3d49ebff689e68267eff4ab030ce746377a6c9a3
+  metadata.gz: 4a24d0a86df7d44669ff38d34667c4f0c63036f877d1453df2e1418670b8a8c59ad803f9c245f5895094465d994253e1b4eced1007f0724cd31f53c4ae804d37
+  data.tar.gz: 830b7fc8203eb8a339a5ad84d334e56dabf0e6e02084be99a3b27a3067b104d7b5ca973437f3f8577050983c0126a302903624bdcf6a2916e267a1e6a8558754

data/CHANGELOG.md

@@ -1,5 +1,32 @@
 # Legion::Data Changelog
 
+## v1.4.2
+
+### Fixed
+- Migration 015: use `create_table?` instead of `create_table` for idempotent RBAC table creation
+
+## v1.4.1
+
+### Added
+- Migration 025: tenants table (tenant_id, name, status, quotas, token limits)
+
+## v1.4.0
+
+### Added
+- `Legion::Data::Vector`: reusable pgvector helpers (available?, cosine_search, l2_search, ensure_extension!)
+- `Legion::Data::StorageTiers`: hot/warm/cold archival lifecycle (archive_to_warm, export_to_cold, stats)
+- Migration 022: memory_traces table with optional pgvector embedding column (1536-dim, HNSW index)
+- Migration 023: data_archive table for generic storage tier archival
+- Migration 024: tenant_id partition columns on tasks, digital_workers, audit_log, memory_traces
+
+## v1.3.8
+
+### Added
+- `Legion::Data::Archival`: hot/warm/cold archival pipeline for tasks and metering records
+- `Legion::Data::Archival::Policy`: configurable retention policies (warm_after_days, cold_after_days, batch_size)
+- Archive, restore, and cross-table search operations with dry-run support
+- Migration 021: archive tables for tasks and metering_records
+
 ## v1.3.7
 
 ### Added

data/lib/legion/data/archival/policy.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Archival
+      class Policy
+        DEFAULTS = {
+          warm_after_days: 7,
+          cold_after_days: 90,
+          batch_size:      1000,
+          tables:          %w[tasks metering_records].freeze
+        }.freeze
+
+        attr_reader :warm_after_days, :cold_after_days, :batch_size, :tables
+
+        def initialize(**opts)
+          config = DEFAULTS.merge(opts)
+          @warm_after_days = config[:warm_after_days]
+          @cold_after_days = config[:cold_after_days]
+          @batch_size = config[:batch_size]
+          @tables = config[:tables]
+        end
+
+        def warm_cutoff
+          Time.now - (warm_after_days * 86_400)
+        end
+
+        def cold_cutoff
+          Time.now - (cold_after_days * 86_400)
+        end
+
+        def self.from_settings
+          return new unless defined?(Legion::Settings)
+
+          data_settings = Legion::Settings[:data]
+          archival = data_settings.is_a?(Hash) ? data_settings[:archival] : nil
+          return new unless archival.is_a?(Hash)
+
+          new(**archival.slice(:warm_after_days, :cold_after_days, :batch_size, :tables))
+        rescue StandardError
+          new
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/archival.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require_relative 'archival/policy'
+
+module Legion
+  module Data
+    module Archival
+      ARCHIVE_TABLE_MAP = {
+        tasks:            :tasks_archive,
+        metering_records: :metering_records_archive
+      }.freeze
+
+      class << self
+        def archive!(policy: Policy.new, dry_run: false)
+          results = {}
+          policy.tables.each do |table_name|
+            table = table_name.to_sym
+            archive_table = ARCHIVE_TABLE_MAP[table]
+            next unless archive_table && db_ready?(table) && db_ready?(archive_table)
+
+            count = archive_table!(
+              source: table, destination: archive_table,
+              cutoff: policy.warm_cutoff, batch_size: policy.batch_size, dry_run: dry_run
+            )
+            results[table] = count
+          end
+          results
+        end
+
+        def restore(table:, ids:)
+          source_table = table.to_sym
+          archive_table = ARCHIVE_TABLE_MAP[source_table]
+          return 0 unless archive_table && db_ready?(archive_table)
+
+          conn = Legion::Data.connection
+          restored = 0
+          conn.transaction do
+            conn[archive_table].where(original_id: ids).each do |row|
+              restore_row = row.except(:id, :archived_at, :original_id, :original_created_at, :original_updated_at)
+              restore_row[:id] = row[:original_id]
+              restore_row[:created_at] = row[:original_created_at]
+              restore_row[:updated_at] = row[:original_updated_at]
+              conn[source_table].insert(restore_row)
+              restored += 1
+            end
+            conn[archive_table].where(original_id: ids).delete
+          end
+          restored
+        end
+
+        def search(table:, where: {})
+          source_table = table.to_sym
+          archive_table = ARCHIVE_TABLE_MAP[source_table]
+          return [] unless db_ready?(source_table)
+
+          conn = Legion::Data.connection
+          hot = conn[source_table].where(where).all
+          warm = db_ready?(archive_table) ? conn[archive_table].where(where).all : []
+          hot + warm
+        end
+
+        private
+
+        def archive_table!(source:, destination:, cutoff:, batch_size:, dry_run:)
+          conn = Legion::Data.connection
+          candidates = conn[source].where { created_at < cutoff }.limit(batch_size)
+          count = candidates.count
+          return count if dry_run || count.zero?
+
+          conn.transaction do
+            candidates.each do |row|
+              archive_row = row.dup
+              archive_row[:original_id] = archive_row.delete(:id)
+              archive_row[:original_created_at] = archive_row.delete(:created_at)
+              archive_row[:original_updated_at] = archive_row.delete(:updated_at)
+              archive_row[:archived_at] = Time.now
+              conn[destination].insert(archive_row)
+            end
+            conn[source].where(id: candidates.select(:id)).delete
+          end
+          count
+        end
+
+        def db_ready?(table)
+          defined?(Legion::Data) && Legion::Data.connection&.table_exists?(table)
+        rescue StandardError
+          false
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/migrations/015_add_rbac_tables.rb

@@ -2,7 +2,7 @@
 
 Sequel.migration do
   up do
-    create_table(:rbac_role_assignments) do
+    create_table?(:rbac_role_assignments) do
       primary_key :id
       String :principal_type, null: false, size: 10
       String :principal_id,   null: false, size: 255
@@ -16,7 +16,7 @@ Sequel.migration do
       index :team
     end
 
-    create_table(:rbac_runner_grants) do
+    create_table?(:rbac_runner_grants) do
       primary_key :id
       String :team,            null: false, size: 255
       String :runner_pattern,  null: false, size: 500
@@ -27,7 +27,7 @@ Sequel.migration do
       index :team
     end
 
-    create_table(:rbac_cross_team_grants) do
+    create_table?(:rbac_cross_team_grants) do
       primary_key :id
       String :source_team,     null: false, size: 255
       String :target_team,     null: false, size: 255

data/lib/legion/data/migrations/021_add_archive_tables.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    unless table_exists?(:tasks_archive)
+      create_table(:tasks_archive) do
+        primary_key :id
+        Integer :original_id, null: false
+        String :function_name
+        String :status
+        String :runner_class
+        column :args, :text
+        column :result, :text
+        String :queue
+        Integer :relationship_id
+        String :chain_id
+        DateTime :original_created_at
+        DateTime :original_updated_at
+        DateTime :archived_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+        index :original_id
+        index :chain_id
+        index :archived_at
+      end
+    end
+
+    unless table_exists?(:metering_records_archive)
+      create_table(:metering_records_archive) do
+        primary_key :id
+        Integer :original_id, null: false
+        String :worker_id
+        String :event_type
+        String :extension
+        String :runner_function
+        String :status
+        Integer :tokens_in
+        Integer :tokens_out
+        Float :cost_usd
+        Integer :wall_clock_ms
+        Integer :cpu_time_ms
+        Integer :external_api_calls
+        String :model
+        String :tenant_id
+        DateTime :original_created_at
+        DateTime :archived_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+        index :original_id
+        index :worker_id
+        index :tenant_id
+        index :archived_at
+      end
+    end
+  end
+
+  down do
+    drop_table(:metering_records_archive) if table_exists?(:metering_records_archive)
+    drop_table(:tasks_archive) if table_exists?(:tasks_archive)
+  end
+end

data/lib/legion/data/migrations/022_add_memory_traces.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_table(:memory_traces) do
+      primary_key :id
+      String :agent_id, null: false, size: 64, index: true
+      String :trace_type, null: false, size: 32
+      String :content, text: true, null: false
+      Float :significance, default: 0.5
+      Float :confidence, default: 1.0
+      String :associations, text: true
+      String :metadata, text: true
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :accessed_at
+      DateTime :decayed_at
+      index %i[agent_id trace_type]
+    end
+
+    next unless adapter_scheme == :postgres
+
+    run 'ALTER TABLE memory_traces ADD COLUMN IF NOT EXISTS embedding vector(1536)'
+    run 'CREATE INDEX IF NOT EXISTS idx_memory_traces_embedding ON memory_traces USING hnsw (embedding vector_cosine_ops)'
+  end
+
+  down do
+    drop_table?(:memory_traces)
+  end
+end

data/lib/legion/data/migrations/023_add_data_archive.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_table(:data_archive) do
+      primary_key :id
+      String :source_table, null: false, size: 64, index: true
+      Integer :source_id, null: false
+      String :data, text: true, null: false
+      Integer :tier, default: 1
+      DateTime :archived_at, default: Sequel::CURRENT_TIMESTAMP
+      index %i[source_table source_id]
+      index :tier
+    end
+  end
+
+  down do
+    drop_table?(:data_archive)
+  end
+end

data/lib/legion/data/migrations/024_add_tenant_partition_columns.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    %i[tasks digital_workers audit_log memory_traces].each do |table|
+      next unless table_exists?(table)
+      next if schema(table).any? { |col, _| col == :tenant_id }
+
+      alter_table(table) do
+        add_column :tenant_id, String, size: 64
+        add_index :tenant_id
+      end
+    end
+  end
+
+  down do
+    %i[tasks digital_workers audit_log memory_traces].each do |table|
+      next unless table_exists?(table)
+      next unless schema(table).any? { |col, _| col == :tenant_id }
+
+      alter_table(table) do
+        drop_index :tenant_id
+        drop_column :tenant_id
+      end
+    end
+  end
+end

data/lib/legion/data/migrations/025_add_tenants_table.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_table(:tenants) do
+      primary_key :id
+      String :tenant_id, null: false, unique: true, size: 100
+      String :name, size: 255
+      String :status, default: 'active', size: 20
+      Integer :max_workers, default: 10
+      Integer :max_queue_depth, default: 10_000
+      Float :monthly_token_limit
+      Float :daily_token_limit
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
+      index :status
+    end
+  end
+
+  down do
+    drop_table?(:tenants)
+  end
+end

data/lib/legion/data/storage_tiers.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module StorageTiers
+      TIERS = { hot: 0, warm: 1, cold: 2 }.freeze
+
+      class << self
+        def archive_to_warm(table:, age_days: 90, batch_size: 1000)
+          return { archived: 0, reason: 'no_connection' } unless Legion::Data.connection
+          return { archived: 0, reason: 'no_archive_table' } unless Legion::Data.connection.table_exists?(:data_archive)
+
+          cutoff = Time.now - (age_days * 86_400)
+          records = Legion::Data.connection[table].where { created_at < cutoff }.limit(batch_size).all
+          return { archived: 0 } if records.empty?
+
+          Legion::Data.connection.transaction do
+            records.each do |record|
+              Legion::Data.connection[:data_archive].insert(
+                source_table: table.to_s, source_id: record[:id],
+                data:         Legion::JSON.dump(record),
+                tier:         TIERS[:warm],
+                archived_at:  Time.now.utc
+              )
+            end
+
+            ids = records.map { |r| r[:id] }
+            Legion::Data.connection[table].where(id: ids).delete
+          end
+
+          { archived: records.size, table: table.to_s }
+        end
+
+        def export_to_cold(age_days: 365, batch_size: 5000)
+          return { exported: 0 } unless Legion::Data.connection&.table_exists?(:data_archive)
+
+          cutoff = Time.now - (age_days * 86_400)
+          records = Legion::Data.connection[:data_archive]
+                                .where(tier: TIERS[:warm])
+                                .where { archived_at < cutoff }
+                                .limit(batch_size).all
+          return { exported: 0 } if records.empty?
+
+          ids = records.map { |r| r[:id] }
+          Legion::Data.connection[:data_archive].where(id: ids).update(tier: TIERS[:cold])
+          { exported: records.size, data: records }
+        end
+
+        def stats
+          return {} unless Legion::Data.connection&.table_exists?(:data_archive)
+
+          { warm: count_tier(:warm), cold: count_tier(:cold) }
+        end
+
+        private
+
+        def count_tier(tier)
+          Legion::Data.connection[:data_archive].where(tier: TIERS[tier]).count
+        rescue StandardError
+          0
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/vector.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Vector
+      class << self
+        def available?
+          return false unless Legion::Data.connection
+          return false unless Legion::Data.connection.adapter_scheme == :postgres
+
+          Legion::Data.connection.fetch("SELECT 1 FROM pg_extension WHERE extname = 'vector'").any?
+        rescue StandardError
+          false
+        end
+
+        def ensure_extension!
+          return false unless Legion::Data.connection&.adapter_scheme == :postgres
+
+          Legion::Data.connection.run('CREATE EXTENSION IF NOT EXISTS vector')
+          true
+        rescue StandardError => e
+          Legion::Logging.warn("pgvector extension creation failed: #{e.message}") if defined?(Legion::Logging)
+          false
+        end
+
+        def cosine_search(table:, column:, query_vector:, limit: 10, min_similarity: 0.0)
+          return [] unless available?
+
+          vec_literal = vector_literal(query_vector)
+          ds = Legion::Data.connection[table]
+                           .select_all
+                           .select_append(Sequel.lit("1 - (#{column} <=> ?)", vec_literal).as(:similarity))
+                           .order(Sequel.lit("#{column} <=> ?", vec_literal))
+                           .limit(limit)
+
+          ds = ds.where(Sequel.lit("1 - (#{column} <=> ?) >= ?", vec_literal, min_similarity)) if min_similarity.positive?
+          ds.all
+        end
+
+        def l2_search(table:, column:, query_vector:, limit: 10)
+          return [] unless available?
+
+          vec_literal = vector_literal(query_vector)
+          Legion::Data.connection[table]
+                      .select_all
+                      .select_append(Sequel.lit("#{column} <-> ?", vec_literal).as(:distance))
+                      .order(Sequel.lit("#{column} <-> ?", vec_literal))
+                      .limit(limit)
+                      .all
+        end
+
+        private
+
+        def vector_literal(query_vector)
+          "[#{query_vector.join(',')}]"
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.3.7'
+    VERSION = '1.4.2'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.3.7
+  version: 1.4.2
 platform: ruby
 authors:
 - Esity
@@ -86,6 +86,8 @@ files:
 - exe/legionio_migrate
 - legion-data.gemspec
 - lib/legion/data.rb
+- lib/legion/data/archival.rb
+- lib/legion/data/archival/policy.rb
 - lib/legion/data/connection.rb
 - lib/legion/data/encryption/cipher.rb
 - lib/legion/data/encryption/key_provider.rb
@@ -114,6 +116,11 @@ files:
 - lib/legion/data/migrations/018_add_governance_events.rb
 - lib/legion/data/migrations/019_add_audit_hash_chain.rb
 - lib/legion/data/migrations/020_add_webhooks.rb
+- lib/legion/data/migrations/021_add_archive_tables.rb
+- lib/legion/data/migrations/022_add_memory_traces.rb
+- lib/legion/data/migrations/023_add_data_archive.rb
+- lib/legion/data/migrations/024_add_tenant_partition_columns.rb
+- lib/legion/data/migrations/025_add_tenants_table.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo_access_log.rb
 - lib/legion/data/models/apollo_entry.rb
@@ -133,6 +140,8 @@ files:
 - lib/legion/data/models/task.rb
 - lib/legion/data/models/task_log.rb
 - lib/legion/data/settings.rb
+- lib/legion/data/storage_tiers.rb
+- lib/legion/data/vector.rb
 - lib/legion/data/version.rb
 - sonar-project.properties
 homepage: https://github.com/LegionIO/legion-data