legion-data 1.6.1 → 1.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '02869f702b7e4ca994edd3fd2b503afa5cca4d450741ab99ed85277315c90ceb'
-  data.tar.gz: 83b8136412ae23993b823e9548f8cf341f66c5f8e37f4002a326ec60e9a8c375
+  metadata.gz: '048cfb8d6ead50ef6ddc554765251d61121dc0f6e99491b9675e410f086cd2fe'
+  data.tar.gz: 58cced043937061ce27fd3c111d5b0b5c5fd3416ab622a3a7700f1ce553628a4
 SHA512:
-  metadata.gz: fbe190c50347a42a28b53627c8a66cf951168c606454672a9f1219cb21373cba8ce0f6da7c557c360c16127ba7cacf317cad2a47b296c72f495581e2253e2128
-  data.tar.gz: 90e3ca22009bff85eccfa9a8803a5fac9563389518b4ae26145561e1d7600b80d432bd8015677c3b64d35f38e9ed42338f82acfa9a549e490201db53342e0a65
+  metadata.gz: 9ba6d97c8cee58a5f41cd5ecc5c2229e75d4880368c43ec44d2f3b969ff144c81c7d6f49fd91d84819b1d3665eab37a7f7ff08f22c0e579109a96ace83d24262
+  data.tar.gz: ba69290675fbf53d9185dcdbc62e46e344b6aad3594c7dc4fd76fdf48af386a2b97d5b1e7fe70606b1c91dacee32e5a4117d6cc5ccf34a184fac89f4ce7b2e26

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Legion::Data Changelog
 
+## [1.6.2] - 2026-03-25
+
+### Changed
+- Migration 041: Resize all pgvector columns from `vector(1536)` to `vector(1024)` for cross-provider embedding compatibility (apollo_entries.embedding, functions.embedding_vector, memory_traces.embedding). Drops and recreates HNSW cosine indexes.
+
 ## [1.6.1] - 2026-03-25
 
 ### Fixed

data/lib/legion/data/archival.rb

@@ -61,6 +61,55 @@ module Legion
           hot + warm
         end
 
+        def archive_completed_tasks(days_old: 90, batch_size: 1000)
+          conn = Legion::Data.connection
+          cutoff = Time.now - (days_old * 86_400)
+
+          return { archived: 0, cutoff: cutoff.iso8601 } unless conn&.table_exists?(:tasks) && conn.table_exists?(:tasks_archive)
+
+          candidates = conn[:tasks]
+                       .where(status: %w[completed failed])
+                       .where(Sequel.lit('created < ?', cutoff))
+                       .limit(batch_size)
+
+          count = candidates.count
+          if count.positive?
+            archive_cols = conn.schema(:tasks_archive).to_set(&:first)
+            conn.transaction do
+              candidates.each do |row|
+                archive_row = {
+                  original_id:         row[:id],
+                  status:              row[:status],
+                  relationship_id:     row[:relationship_id],
+                  original_created_at: row[:created],
+                  original_updated_at: row[:updated],
+                  archived_at:         Time.now
+                }
+                archive_row[:archive_reason] = 'completed_task_archival' if archive_cols.include?(:archive_reason)
+                conn[:tasks_archive].insert(archive_row)
+              end
+              conn[:tasks].where(id: candidates.select(:id)).delete
+            end
+          end
+
+          Legion::Logging.info "archive_completed_tasks: archived #{count} tasks (cutoff: #{cutoff.iso8601})" if defined?(Legion::Logging)
+          { archived: count, cutoff: cutoff.iso8601 }
+        end
+
+        def run_scheduled_archival
+          results = {}
+          results[:tasks] = archive_completed_tasks
+
+          conn = Legion::Data.connection
+          if conn&.table_exists?(:metering_records)
+            results[:metering] = Legion::Data::Retention.archive_old_records(
+              table: :metering_records, date_column: :recorded_at
+            )
+          end
+
+          results
+        end
+
         private
 
         def archive_table!(source:, destination:, cutoff:, batch_size:, dry_run:)

data/lib/legion/data/migrations/041_resize_vector_columns.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    next unless adapter_scheme == :postgres
+
+    # Resize embedding columns from 1536 to 1024 for cross-provider compatibility
+    # (Bedrock Titan v2, OpenAI with dimensions:, Ollama mxbai-embed-large all support 1024)
+    # Knowledge store is empty so no data re-embedding needed.
+
+    if table_exists?(:apollo_entries)
+      run 'DROP INDEX IF EXISTS idx_apollo_entries_embedding'
+      run 'ALTER TABLE apollo_entries ALTER COLUMN embedding TYPE vector(1024)'
+      run 'CREATE INDEX idx_apollo_entries_embedding ON apollo_entries USING hnsw (embedding vector_cosine_ops)'
+    end
+
+    if table_exists?(:functions)
+      run 'DROP INDEX IF EXISTS idx_functions_embedding'
+      run 'ALTER TABLE functions ALTER COLUMN embedding_vector TYPE vector(1024)'
+      run 'CREATE INDEX idx_functions_embedding ON functions USING hnsw (embedding_vector vector_cosine_ops)'
+    end
+
+    if table_exists?(:memory_traces)
+      run 'DROP INDEX IF EXISTS idx_memory_traces_embedding'
+      run 'ALTER TABLE memory_traces ALTER COLUMN embedding TYPE vector(1024)'
+      run 'CREATE INDEX idx_memory_traces_embedding ON memory_traces USING hnsw (embedding vector_cosine_ops)'
+    end
+  end
+
+  down do
+    next unless adapter_scheme == :postgres
+
+    if table_exists?(:apollo_entries)
+      run 'DROP INDEX IF EXISTS idx_apollo_entries_embedding'
+      run 'ALTER TABLE apollo_entries ALTER COLUMN embedding TYPE vector(1536)'
+      run 'CREATE INDEX idx_apollo_entries_embedding ON apollo_entries USING hnsw (embedding vector_cosine_ops)'
+    end
+
+    if table_exists?(:functions)
+      run 'DROP INDEX IF EXISTS idx_functions_embedding'
+      run 'ALTER TABLE functions ALTER COLUMN embedding_vector TYPE vector(1536)'
+      run 'CREATE INDEX idx_functions_embedding ON functions USING hnsw (embedding_vector vector_cosine_ops)'
+    end
+
+    if table_exists?(:memory_traces)
+      run 'DROP INDEX IF EXISTS idx_memory_traces_embedding'
+      run 'ALTER TABLE memory_traces ALTER COLUMN embedding TYPE vector(1536)'
+      run 'CREATE INDEX idx_memory_traces_embedding ON memory_traces USING hnsw (embedding vector_cosine_ops)'
+    end
+  end
+end

data/lib/legion/data/migrations/042_add_tenant_to_registry_tables.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    %i[extensions functions runners nodes settings value_metrics].each do |table|
+      next unless table_exists?(table)
+      next if schema(table).any? { |col, _| col == :tenant_id }
+
+      alter_table(table) do
+        add_column :tenant_id, String, size: 64
+        add_index :tenant_id, name: :"idx_#{table}_tenant_id"
+      end
+    end
+  end
+
+  down do
+    %i[extensions functions runners nodes settings value_metrics].each do |table|
+      next unless table_exists?(table)
+      next unless schema(table).any? { |col, _| col == :tenant_id }
+
+      alter_table(table) do
+        drop_index :tenant_id, name: :"idx_#{table}_tenant_id"
+        drop_column :tenant_id
+      end
+    end
+  end
+end

data/lib/legion/data/migrations/043_add_rls_placeholder.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    next unless adapter_scheme == :postgres
+
+    tables = %i[
+      tasks digital_workers audit_log memory_traces extensions
+      functions runners nodes settings value_metrics
+    ]
+
+    tables.each do |table|
+      next unless table_exists?(table)
+      next unless schema(table).any? { |col, _| col == :tenant_id }
+
+      run "ALTER TABLE #{table} ENABLE ROW LEVEL SECURITY"
+      run <<~SQL
+        DO $$ BEGIN
+          IF NOT EXISTS (
+            SELECT 1 FROM pg_policies WHERE tablename = '#{table}' AND policyname = 'tenant_isolation_#{table}'
+          ) THEN
+            CREATE POLICY tenant_isolation_#{table} ON #{table}
+              USING (tenant_id = current_setting('app.current_tenant', true));
+          END IF;
+        END $$;
+      SQL
+    end
+  end
+
+  down do
+    next unless adapter_scheme == :postgres
+
+    tables = %i[
+      tasks digital_workers audit_log memory_traces extensions
+      functions runners nodes settings value_metrics
+    ]
+
+    tables.each do |table|
+      next unless table_exists?(table)
+
+      run "DROP POLICY IF EXISTS tenant_isolation_#{table} ON #{table}"
+      run "ALTER TABLE #{table} DISABLE ROW LEVEL SECURITY"
+    end
+  end
+end

data/lib/legion/data/migrations/044_expand_memory_traces.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    return unless table_exists?(:memory_traces)
+
+    existing = schema(:memory_traces).map(&:first)
+
+    alter_table(:memory_traces) do
+      add_column :trace_id, String, size: 36 unless existing.include?(:trace_id)
+      add_column :strength, Float, default: 0.5 unless existing.include?(:strength)
+      add_column :peak_strength, Float, default: 0.5 unless existing.include?(:peak_strength)
+      add_column :base_decay_rate, Float, default: 0.05 unless existing.include?(:base_decay_rate)
+      add_column :emotional_valence, Float, default: 0.0 unless existing.include?(:emotional_valence)
+      add_column :emotional_intensity, Float, default: 0.0 unless existing.include?(:emotional_intensity)
+      add_column :domain_tags, :text unless existing.include?(:domain_tags)
+      add_column :origin, String, size: 50 unless existing.include?(:origin)
+      add_column :source_agent_id, String, size: 255 unless existing.include?(:source_agent_id)
+      add_column :storage_tier, String, size: 10, default: 'warm' unless existing.include?(:storage_tier)
+      add_column :last_reinforced, DateTime unless existing.include?(:last_reinforced)
+      add_column :last_decayed, DateTime unless existing.include?(:last_decayed)
+      add_column :reinforcement_count, Integer, default: 0 unless existing.include?(:reinforcement_count)
+      add_column :unresolved, TrueClass, default: false unless existing.include?(:unresolved)
+      add_column :consolidation_candidate, TrueClass, default: false unless existing.include?(:consolidation_candidate)
+      add_column :parent_trace_id, String, size: 36 unless existing.include?(:parent_trace_id)
+      add_column :encryption_key_id, String, size: 255 unless existing.include?(:encryption_key_id)
+      add_column :partition_id, String, size: 255 unless existing.include?(:partition_id)
+    end
+
+    indexes = begin
+      db.indexes(:memory_traces).keys
+    rescue StandardError
+      []
+    end
+
+    add_index :memory_traces, :trace_id, unique: true, name: :idx_memory_traces_trace_id unless existing.include?(:trace_id)
+
+    add_index :memory_traces, :storage_tier, name: :idx_memory_traces_storage_tier unless indexes.include?(:idx_memory_traces_storage_tier)
+    add_index :memory_traces, :partition_id, name: :idx_memory_traces_partition_id unless indexes.include?(:idx_memory_traces_partition_id)
+    add_index :memory_traces, %i[partition_id trace_type], name: :idx_memory_traces_partition_type unless indexes.include?(:idx_memory_traces_partition_type)
+    add_index :memory_traces, :unresolved, name: :idx_memory_traces_unresolved unless indexes.include?(:idx_memory_traces_unresolved)
+  end
+
+  down do
+    return unless table_exists?(:memory_traces)
+
+    existing = schema(:memory_traces).map(&:first)
+
+    %i[trace_id strength peak_strength base_decay_rate emotional_valence emotional_intensity
+       domain_tags origin source_agent_id storage_tier last_reinforced last_decayed
+       reinforcement_count unresolved consolidation_candidate parent_trace_id
+       encryption_key_id partition_id].each do |col|
+      alter_table(:memory_traces) { drop_column col } if existing.include?(col)
+    end
+  end
+end

data/lib/legion/data/migrations/045_add_memory_associations.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    return if table_exists?(:memory_associations)
+
+    create_table(:memory_associations) do
+      primary_key :id
+      String :trace_id_a, size: 36, null: false
+      String :trace_id_b, size: 36, null: false
+      Integer :coactivation_count, default: 1, null: false
+      TrueClass :linked, default: false, null: false
+      String :tenant_id, size: 64
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
+
+      unique %i[trace_id_a trace_id_b]
+      index :trace_id_a
+      index :trace_id_b
+      index :tenant_id
+    end
+  end
+
+  down do
+    drop_table?(:memory_associations)
+  end
+end

data/lib/legion/data/migrations/046_add_metering_hourly_rollup.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    return if table_exists?(:metering_hourly_rollup)
+
+    create_table(:metering_hourly_rollup) do
+      primary_key :id
+      String :worker_id, size: 36, null: false
+      String :provider, size: 100, null: false
+      String :model_id, size: 255, null: false
+      DateTime :hour, null: false
+      Integer :total_input_tokens, default: 0, null: false
+      Integer :total_output_tokens, default: 0, null: false
+      Integer :total_thinking_tokens, default: 0, null: false
+      Integer :total_calls, default: 0, null: false
+      Float :total_cost_usd, default: 0.0, null: false
+      Float :avg_latency_ms, default: 0.0, null: false
+      String :tenant_id, size: 64
+      DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
+
+      unique %i[worker_id provider model_id hour], name: :idx_rollup_unique_hour
+      index :hour
+      index :tenant_id
+      index %i[worker_id hour]
+    end
+  end
+
+  down do
+    drop_table?(:metering_hourly_rollup)
+  end
+end

data/lib/legion/data/rls.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Rls
+      RLS_TABLES = %i[
+        tasks digital_workers audit_log memory_traces extensions
+        functions runners nodes settings value_metrics
+      ].freeze
+
+      module_function
+
+      def rls_enabled?
+        return false unless Legion::Settings[:data][:connected]
+
+        Legion::Data.connection.adapter_scheme == :postgres
+      rescue StandardError
+        false
+      end
+
+      def assign_tenant(tenant_id)
+        return unless rls_enabled?
+
+        Legion::Data.connection.run(
+          Sequel.lit('SET app.current_tenant = ?', tenant_id.to_s)
+        )
+      end
+
+      def current_tenant
+        return nil unless rls_enabled?
+
+        Legion::Data.connection.fetch('SHOW app.current_tenant').first&.values&.first
+      rescue Sequel::DatabaseError
+        nil
+      end
+
+      def reset_tenant
+        return unless rls_enabled?
+
+        Legion::Data.connection.run('RESET app.current_tenant')
+      end
+
+      def with_tenant(tenant_id)
+        previous = current_tenant
+        assign_tenant(tenant_id)
+        yield
+      ensure
+        previous ? assign_tenant(previous) : reset_tenant
+      end
+    end
+  end
+end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.6.1'
+    VERSION = '1.6.3'
   end
 end

data/lib/legion/data.rb

@@ -12,6 +12,7 @@ require_relative 'data/spool'
 require_relative 'data/partition_manager'
 require_relative 'data/archiver'
 require_relative 'data/helper'
+require_relative 'data/rls'
 
 module Legion
   module Data

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.6.3
 platform: ruby
 authors:
 - Esity
@@ -141,6 +141,12 @@ files:
 - lib/legion/data/migrations/038_add_conversations.rb
 - lib/legion/data/migrations/039_add_audit_archive_manifest.rb
 - lib/legion/data/migrations/040_add_slow_query_indexes.rb
+- lib/legion/data/migrations/041_resize_vector_columns.rb
+- lib/legion/data/migrations/042_add_tenant_to_registry_tables.rb
+- lib/legion/data/migrations/043_add_rls_placeholder.rb
+- lib/legion/data/migrations/044_expand_memory_traces.rb
+- lib/legion/data/migrations/045_add_memory_associations.rb
+- lib/legion/data/migrations/046_add_metering_hourly_rollup.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo_access_log.rb
 - lib/legion/data/models/apollo_entry.rb
@@ -161,6 +167,7 @@ files:
 - lib/legion/data/models/task_log.rb
 - lib/legion/data/partition_manager.rb
 - lib/legion/data/retention.rb
+- lib/legion/data/rls.rb
 - lib/legion/data/settings.rb
 - lib/legion/data/spool.rb
 - lib/legion/data/storage_tiers.rb