legion-data 1.3.8 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b4deeb282c6d4c359d6fa8d46de2f18f3107746536227171ae4b47b5f80d7b0
-  data.tar.gz: 4f445caf46e5258694170c43ea429a3c6de4f25bd25f013617cf987d335077ad
+  metadata.gz: eaf37e68acae5015ef50b01e6a5c7a4806e81b80f5cf74f3c71b72e6799f0bd4
+  data.tar.gz: 81b55083c6e8d59403b51a838935a4146b1d3f39eca8316dfd82ee4d2f6a0b05
 SHA512:
-  metadata.gz: '06930c4164c05d58320e1e5a35b9be15b76e647ba06826b05df8bdd4db156283e232a2109f930e31f78cb892f4cd25324ac941810a377760ee6956577c64fa97'
-  data.tar.gz: 1cbfe73b30462c6b3807d15a494e48b7c4cb986e93b472a1357f722f5e13fb597ff2c70fae0f31911f1ae4711855d91a078c16fbf048835d5483aa849da0d252
+  metadata.gz: 4a24d0a86df7d44669ff38d34667c4f0c63036f877d1453df2e1418670b8a8c59ad803f9c245f5895094465d994253e1b4eced1007f0724cd31f53c4ae804d37
+  data.tar.gz: 830b7fc8203eb8a339a5ad84d334e56dabf0e6e02084be99a3b27a3067b104d7b5ca973437f3f8577050983c0126a302903624bdcf6a2916e267a1e6a8558754

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Legion::Data Changelog
 
+## v1.4.2
+
+### Fixed
+- Migration 015: use `create_table?` instead of `create_table` for idempotent RBAC table creation
+
+## v1.4.1
+
+### Added
+- Migration 025: tenants table (tenant_id, name, status, quotas, token limits)
+
+## v1.4.0
+
+### Added
+- `Legion::Data::Vector`: reusable pgvector helpers (available?, cosine_search, l2_search, ensure_extension!)
+- `Legion::Data::StorageTiers`: hot/warm/cold archival lifecycle (archive_to_warm, export_to_cold, stats)
+- Migration 022: memory_traces table with optional pgvector embedding column (1536-dim, HNSW index)
+- Migration 023: data_archive table for generic storage tier archival
+- Migration 024: tenant_id partition columns on tasks, digital_workers, audit_log, memory_traces
+
 ## v1.3.8
 
 ### Added

data/lib/legion/data/migrations/015_add_rbac_tables.rb

@@ -2,7 +2,7 @@
 
 Sequel.migration do
   up do
-    create_table(:rbac_role_assignments) do
+    create_table?(:rbac_role_assignments) do
       primary_key :id
       String :principal_type, null: false, size: 10
       String :principal_id,   null: false, size: 255
@@ -16,7 +16,7 @@ Sequel.migration do
       index :team
     end
 
-    create_table(:rbac_runner_grants) do
+    create_table?(:rbac_runner_grants) do
       primary_key :id
       String :team,            null: false, size: 255
       String :runner_pattern,  null: false, size: 500
@@ -27,7 +27,7 @@ Sequel.migration do
       index :team
     end
 
-    create_table(:rbac_cross_team_grants) do
+    create_table?(:rbac_cross_team_grants) do
       primary_key :id
       String :source_team,     null: false, size: 255
       String :target_team,     null: false, size: 255

data/lib/legion/data/migrations/022_add_memory_traces.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_table(:memory_traces) do
+      primary_key :id
+      String :agent_id, null: false, size: 64, index: true
+      String :trace_type, null: false, size: 32
+      String :content, text: true, null: false
+      Float :significance, default: 0.5
+      Float :confidence, default: 1.0
+      String :associations, text: true
+      String :metadata, text: true
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :accessed_at
+      DateTime :decayed_at
+      index %i[agent_id trace_type]
+    end
+
+    next unless adapter_scheme == :postgres
+
+    run 'ALTER TABLE memory_traces ADD COLUMN IF NOT EXISTS embedding vector(1536)'
+    run 'CREATE INDEX IF NOT EXISTS idx_memory_traces_embedding ON memory_traces USING hnsw (embedding vector_cosine_ops)'
+  end
+
+  down do
+    drop_table?(:memory_traces)
+  end
+end

data/lib/legion/data/migrations/023_add_data_archive.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_table(:data_archive) do
+      primary_key :id
+      String :source_table, null: false, size: 64, index: true
+      Integer :source_id, null: false
+      String :data, text: true, null: false
+      Integer :tier, default: 1
+      DateTime :archived_at, default: Sequel::CURRENT_TIMESTAMP
+      index %i[source_table source_id]
+      index :tier
+    end
+  end
+
+  down do
+    drop_table?(:data_archive)
+  end
+end

data/lib/legion/data/migrations/024_add_tenant_partition_columns.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    %i[tasks digital_workers audit_log memory_traces].each do |table|
+      next unless table_exists?(table)
+      next if schema(table).any? { |col, _| col == :tenant_id }
+
+      alter_table(table) do
+        add_column :tenant_id, String, size: 64
+        add_index :tenant_id
+      end
+    end
+  end
+
+  down do
+    %i[tasks digital_workers audit_log memory_traces].each do |table|
+      next unless table_exists?(table)
+      next unless schema(table).any? { |col, _| col == :tenant_id }
+
+      alter_table(table) do
+        drop_index :tenant_id
+        drop_column :tenant_id
+      end
+    end
+  end
+end

data/lib/legion/data/migrations/025_add_tenants_table.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_table(:tenants) do
+      primary_key :id
+      String :tenant_id, null: false, unique: true, size: 100
+      String :name, size: 255
+      String :status, default: 'active', size: 20
+      Integer :max_workers, default: 10
+      Integer :max_queue_depth, default: 10_000
+      Float :monthly_token_limit
+      Float :daily_token_limit
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
+      index :status
+    end
+  end
+
+  down do
+    drop_table?(:tenants)
+  end
+end

data/lib/legion/data/storage_tiers.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module StorageTiers
+      TIERS = { hot: 0, warm: 1, cold: 2 }.freeze
+
+      class << self
+        def archive_to_warm(table:, age_days: 90, batch_size: 1000)
+          return { archived: 0, reason: 'no_connection' } unless Legion::Data.connection
+          return { archived: 0, reason: 'no_archive_table' } unless Legion::Data.connection.table_exists?(:data_archive)
+
+          cutoff = Time.now - (age_days * 86_400)
+          records = Legion::Data.connection[table].where { created_at < cutoff }.limit(batch_size).all
+          return { archived: 0 } if records.empty?
+
+          Legion::Data.connection.transaction do
+            records.each do |record|
+              Legion::Data.connection[:data_archive].insert(
+                source_table: table.to_s, source_id: record[:id],
+                data:         Legion::JSON.dump(record),
+                tier:         TIERS[:warm],
+                archived_at:  Time.now.utc
+              )
+            end
+
+            ids = records.map { |r| r[:id] }
+            Legion::Data.connection[table].where(id: ids).delete
+          end
+
+          { archived: records.size, table: table.to_s }
+        end
+
+        def export_to_cold(age_days: 365, batch_size: 5000)
+          return { exported: 0 } unless Legion::Data.connection&.table_exists?(:data_archive)
+
+          cutoff = Time.now - (age_days * 86_400)
+          records = Legion::Data.connection[:data_archive]
+                                .where(tier: TIERS[:warm])
+                                .where { archived_at < cutoff }
+                                .limit(batch_size).all
+          return { exported: 0 } if records.empty?
+
+          ids = records.map { |r| r[:id] }
+          Legion::Data.connection[:data_archive].where(id: ids).update(tier: TIERS[:cold])
+          { exported: records.size, data: records }
+        end
+
+        def stats
+          return {} unless Legion::Data.connection&.table_exists?(:data_archive)
+
+          { warm: count_tier(:warm), cold: count_tier(:cold) }
+        end
+
+        private
+
+        def count_tier(tier)
+          Legion::Data.connection[:data_archive].where(tier: TIERS[tier]).count
+        rescue StandardError
+          0
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/vector.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Vector
+      class << self
+        def available?
+          return false unless Legion::Data.connection
+          return false unless Legion::Data.connection.adapter_scheme == :postgres
+
+          Legion::Data.connection.fetch("SELECT 1 FROM pg_extension WHERE extname = 'vector'").any?
+        rescue StandardError
+          false
+        end
+
+        def ensure_extension!
+          return false unless Legion::Data.connection&.adapter_scheme == :postgres
+
+          Legion::Data.connection.run('CREATE EXTENSION IF NOT EXISTS vector')
+          true
+        rescue StandardError => e
+          Legion::Logging.warn("pgvector extension creation failed: #{e.message}") if defined?(Legion::Logging)
+          false
+        end
+
+        def cosine_search(table:, column:, query_vector:, limit: 10, min_similarity: 0.0)
+          return [] unless available?
+
+          vec_literal = vector_literal(query_vector)
+          ds = Legion::Data.connection[table]
+                           .select_all
+                           .select_append(Sequel.lit("1 - (#{column} <=> ?)", vec_literal).as(:similarity))
+                           .order(Sequel.lit("#{column} <=> ?", vec_literal))
+                           .limit(limit)
+
+          ds = ds.where(Sequel.lit("1 - (#{column} <=> ?) >= ?", vec_literal, min_similarity)) if min_similarity.positive?
+          ds.all
+        end
+
+        def l2_search(table:, column:, query_vector:, limit: 10)
+          return [] unless available?
+
+          vec_literal = vector_literal(query_vector)
+          Legion::Data.connection[table]
+                      .select_all
+                      .select_append(Sequel.lit("#{column} <-> ?", vec_literal).as(:distance))
+                      .order(Sequel.lit("#{column} <-> ?", vec_literal))
+                      .limit(limit)
+                      .all
+        end
+
+        private
+
+        def vector_literal(query_vector)
+          "[#{query_vector.join(',')}]"
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.3.8'
+    VERSION = '1.4.2'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.3.8
+  version: 1.4.2
 platform: ruby
 authors:
 - Esity
@@ -117,6 +117,10 @@ files:
 - lib/legion/data/migrations/019_add_audit_hash_chain.rb
 - lib/legion/data/migrations/020_add_webhooks.rb
 - lib/legion/data/migrations/021_add_archive_tables.rb
+- lib/legion/data/migrations/022_add_memory_traces.rb
+- lib/legion/data/migrations/023_add_data_archive.rb
+- lib/legion/data/migrations/024_add_tenant_partition_columns.rb
+- lib/legion/data/migrations/025_add_tenants_table.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo_access_log.rb
 - lib/legion/data/models/apollo_entry.rb
@@ -136,6 +140,8 @@ files:
 - lib/legion/data/models/task.rb
 - lib/legion/data/models/task_log.rb
 - lib/legion/data/settings.rb
+- lib/legion/data/storage_tiers.rb
+- lib/legion/data/vector.rb
 - lib/legion/data/version.rb
 - sonar-project.properties
 homepage: https://github.com/LegionIO/legion-data