legion-data 1.6.1 → 1.6.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/lib/legion/data/archival.rb +49 -0
- data/lib/legion/data/migrations/041_resize_vector_columns.rb +51 -0
- data/lib/legion/data/migrations/042_add_tenant_to_registry_tables.rb +27 -0
- data/lib/legion/data/migrations/043_add_rls_placeholder.rb +45 -0
- data/lib/legion/data/migrations/044_expand_memory_traces.rb +56 -0
- data/lib/legion/data/migrations/045_add_memory_associations.rb +27 -0
- data/lib/legion/data/migrations/046_add_metering_hourly_rollup.rb +32 -0
- data/lib/legion/data/migrations/047_apollo_knowledge_capture.rb +152 -0
- data/lib/legion/data/rls.rb +52 -0
- data/lib/legion/data/version.rb +1 -1
- data/lib/legion/data.rb +1 -0
- metadata +9 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4affe9deaa1903b9ff010b805e8f37814bec22ca669186daf69de345a5e8d711
|
|
4
|
+
data.tar.gz: 6e78bf1992c58cd92f9c05395486f1b1c6f211ae3ad653fc385240563bef96ba
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8a16917741977b4bd9c3ed8b873769dc6f3356cdb950a2bef2abd9809cbc340ec2df966bc71700e5a6b116d86453988f3cbd79db5b9f828e74dabba350e106a8
|
|
7
|
+
data.tar.gz: 9f32dc1dd38e9a9181b0f799509c06e1dd58e0661251426a135cf04bad8e64ca39f38f8ec93adc11e5715bbb3d0c95e878d4c82ebd9977425ace69be3b23cd84
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# Legion::Data Changelog
|
|
2
2
|
|
|
3
|
+
## [1.6.4] - 2026-03-25
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Migration 047: Apollo identity columns (submitted_by, submitted_from), content hash dedup, apollo_operations table, apollo_entries_archive table, comprehensive indexes including partial HNSW on active entries only
|
|
7
|
+
|
|
8
|
+
## [1.6.2] - 2026-03-25
|
|
9
|
+
|
|
10
|
+
### Changed
|
|
11
|
+
- Migration 041: Resize all pgvector columns from `vector(1536)` to `vector(1024)` for cross-provider embedding compatibility (apollo_entries.embedding, functions.embedding_vector, memory_traces.embedding). Drops and recreates HNSW cosine indexes.
|
|
12
|
+
|
|
3
13
|
## [1.6.1] - 2026-03-25
|
|
4
14
|
|
|
5
15
|
### Fixed
|
data/lib/legion/data/archival.rb
CHANGED
|
@@ -61,6 +61,55 @@ module Legion
|
|
|
61
61
|
hot + warm
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
+
def archive_completed_tasks(days_old: 90, batch_size: 1000)
|
|
65
|
+
conn = Legion::Data.connection
|
|
66
|
+
cutoff = Time.now - (days_old * 86_400)
|
|
67
|
+
|
|
68
|
+
return { archived: 0, cutoff: cutoff.iso8601 } unless conn&.table_exists?(:tasks) && conn.table_exists?(:tasks_archive)
|
|
69
|
+
|
|
70
|
+
candidates = conn[:tasks]
|
|
71
|
+
.where(status: %w[completed failed])
|
|
72
|
+
.where(Sequel.lit('created < ?', cutoff))
|
|
73
|
+
.limit(batch_size)
|
|
74
|
+
|
|
75
|
+
count = candidates.count
|
|
76
|
+
if count.positive?
|
|
77
|
+
archive_cols = conn.schema(:tasks_archive).to_set(&:first)
|
|
78
|
+
conn.transaction do
|
|
79
|
+
candidates.each do |row|
|
|
80
|
+
archive_row = {
|
|
81
|
+
original_id: row[:id],
|
|
82
|
+
status: row[:status],
|
|
83
|
+
relationship_id: row[:relationship_id],
|
|
84
|
+
original_created_at: row[:created],
|
|
85
|
+
original_updated_at: row[:updated],
|
|
86
|
+
archived_at: Time.now
|
|
87
|
+
}
|
|
88
|
+
archive_row[:archive_reason] = 'completed_task_archival' if archive_cols.include?(:archive_reason)
|
|
89
|
+
conn[:tasks_archive].insert(archive_row)
|
|
90
|
+
end
|
|
91
|
+
conn[:tasks].where(id: candidates.select(:id)).delete
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
Legion::Logging.info "archive_completed_tasks: archived #{count} tasks (cutoff: #{cutoff.iso8601})" if defined?(Legion::Logging)
|
|
96
|
+
{ archived: count, cutoff: cutoff.iso8601 }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def run_scheduled_archival
|
|
100
|
+
results = {}
|
|
101
|
+
results[:tasks] = archive_completed_tasks
|
|
102
|
+
|
|
103
|
+
conn = Legion::Data.connection
|
|
104
|
+
if conn&.table_exists?(:metering_records)
|
|
105
|
+
results[:metering] = Legion::Data::Retention.archive_old_records(
|
|
106
|
+
table: :metering_records, date_column: :recorded_at
|
|
107
|
+
)
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
results
|
|
111
|
+
end
|
|
112
|
+
|
|
64
113
|
private
|
|
65
114
|
|
|
66
115
|
def archive_table!(source:, destination:, cutoff:, batch_size:, dry_run:)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next unless adapter_scheme == :postgres
|
|
6
|
+
|
|
7
|
+
# Resize embedding columns from 1536 to 1024 for cross-provider compatibility
|
|
8
|
+
# (Bedrock Titan v2, OpenAI with dimensions:, Ollama mxbai-embed-large all support 1024)
|
|
9
|
+
# Knowledge store is empty so no data re-embedding needed.
|
|
10
|
+
|
|
11
|
+
if table_exists?(:apollo_entries)
|
|
12
|
+
run 'DROP INDEX IF EXISTS idx_apollo_entries_embedding'
|
|
13
|
+
run 'ALTER TABLE apollo_entries ALTER COLUMN embedding TYPE vector(1024)'
|
|
14
|
+
run 'CREATE INDEX idx_apollo_entries_embedding ON apollo_entries USING hnsw (embedding vector_cosine_ops)'
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
if table_exists?(:functions)
|
|
18
|
+
run 'DROP INDEX IF EXISTS idx_functions_embedding'
|
|
19
|
+
run 'ALTER TABLE functions ALTER COLUMN embedding_vector TYPE vector(1024)'
|
|
20
|
+
run 'CREATE INDEX idx_functions_embedding ON functions USING hnsw (embedding_vector vector_cosine_ops)'
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
if table_exists?(:memory_traces)
|
|
24
|
+
run 'DROP INDEX IF EXISTS idx_memory_traces_embedding'
|
|
25
|
+
run 'ALTER TABLE memory_traces ALTER COLUMN embedding TYPE vector(1024)'
|
|
26
|
+
run 'CREATE INDEX idx_memory_traces_embedding ON memory_traces USING hnsw (embedding vector_cosine_ops)'
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
down do
|
|
31
|
+
next unless adapter_scheme == :postgres
|
|
32
|
+
|
|
33
|
+
if table_exists?(:apollo_entries)
|
|
34
|
+
run 'DROP INDEX IF EXISTS idx_apollo_entries_embedding'
|
|
35
|
+
run 'ALTER TABLE apollo_entries ALTER COLUMN embedding TYPE vector(1536)'
|
|
36
|
+
run 'CREATE INDEX idx_apollo_entries_embedding ON apollo_entries USING hnsw (embedding vector_cosine_ops)'
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
if table_exists?(:functions)
|
|
40
|
+
run 'DROP INDEX IF EXISTS idx_functions_embedding'
|
|
41
|
+
run 'ALTER TABLE functions ALTER COLUMN embedding_vector TYPE vector(1536)'
|
|
42
|
+
run 'CREATE INDEX idx_functions_embedding ON functions USING hnsw (embedding_vector vector_cosine_ops)'
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
if table_exists?(:memory_traces)
|
|
46
|
+
run 'DROP INDEX IF EXISTS idx_memory_traces_embedding'
|
|
47
|
+
run 'ALTER TABLE memory_traces ALTER COLUMN embedding TYPE vector(1536)'
|
|
48
|
+
run 'CREATE INDEX idx_memory_traces_embedding ON memory_traces USING hnsw (embedding vector_cosine_ops)'
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
%i[extensions functions runners nodes settings value_metrics].each do |table|
|
|
6
|
+
next unless table_exists?(table)
|
|
7
|
+
next if schema(table).any? { |col, _| col == :tenant_id }
|
|
8
|
+
|
|
9
|
+
alter_table(table) do
|
|
10
|
+
add_column :tenant_id, String, size: 64
|
|
11
|
+
add_index :tenant_id, name: :"idx_#{table}_tenant_id"
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
down do
|
|
17
|
+
%i[extensions functions runners nodes settings value_metrics].each do |table|
|
|
18
|
+
next unless table_exists?(table)
|
|
19
|
+
next unless schema(table).any? { |col, _| col == :tenant_id }
|
|
20
|
+
|
|
21
|
+
alter_table(table) do
|
|
22
|
+
drop_index :tenant_id, name: :"idx_#{table}_tenant_id"
|
|
23
|
+
drop_column :tenant_id
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next unless adapter_scheme == :postgres
|
|
6
|
+
|
|
7
|
+
tables = %i[
|
|
8
|
+
tasks digital_workers audit_log memory_traces extensions
|
|
9
|
+
functions runners nodes settings value_metrics
|
|
10
|
+
]
|
|
11
|
+
|
|
12
|
+
tables.each do |table|
|
|
13
|
+
next unless table_exists?(table)
|
|
14
|
+
next unless schema(table).any? { |col, _| col == :tenant_id }
|
|
15
|
+
|
|
16
|
+
run "ALTER TABLE #{table} ENABLE ROW LEVEL SECURITY"
|
|
17
|
+
run <<~SQL
|
|
18
|
+
DO $$ BEGIN
|
|
19
|
+
IF NOT EXISTS (
|
|
20
|
+
SELECT 1 FROM pg_policies WHERE tablename = '#{table}' AND policyname = 'tenant_isolation_#{table}'
|
|
21
|
+
) THEN
|
|
22
|
+
CREATE POLICY tenant_isolation_#{table} ON #{table}
|
|
23
|
+
USING (tenant_id = current_setting('app.current_tenant', true));
|
|
24
|
+
END IF;
|
|
25
|
+
END $$;
|
|
26
|
+
SQL
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
down do
|
|
31
|
+
next unless adapter_scheme == :postgres
|
|
32
|
+
|
|
33
|
+
tables = %i[
|
|
34
|
+
tasks digital_workers audit_log memory_traces extensions
|
|
35
|
+
functions runners nodes settings value_metrics
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
tables.each do |table|
|
|
39
|
+
next unless table_exists?(table)
|
|
40
|
+
|
|
41
|
+
run "DROP POLICY IF EXISTS tenant_isolation_#{table} ON #{table}"
|
|
42
|
+
run "ALTER TABLE #{table} DISABLE ROW LEVEL SECURITY"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
return unless table_exists?(:memory_traces)
|
|
6
|
+
|
|
7
|
+
existing = schema(:memory_traces).map(&:first)
|
|
8
|
+
|
|
9
|
+
alter_table(:memory_traces) do
|
|
10
|
+
add_column :trace_id, String, size: 36 unless existing.include?(:trace_id)
|
|
11
|
+
add_column :strength, Float, default: 0.5 unless existing.include?(:strength)
|
|
12
|
+
add_column :peak_strength, Float, default: 0.5 unless existing.include?(:peak_strength)
|
|
13
|
+
add_column :base_decay_rate, Float, default: 0.05 unless existing.include?(:base_decay_rate)
|
|
14
|
+
add_column :emotional_valence, Float, default: 0.0 unless existing.include?(:emotional_valence)
|
|
15
|
+
add_column :emotional_intensity, Float, default: 0.0 unless existing.include?(:emotional_intensity)
|
|
16
|
+
add_column :domain_tags, :text unless existing.include?(:domain_tags)
|
|
17
|
+
add_column :origin, String, size: 50 unless existing.include?(:origin)
|
|
18
|
+
add_column :source_agent_id, String, size: 255 unless existing.include?(:source_agent_id)
|
|
19
|
+
add_column :storage_tier, String, size: 10, default: 'warm' unless existing.include?(:storage_tier)
|
|
20
|
+
add_column :last_reinforced, DateTime unless existing.include?(:last_reinforced)
|
|
21
|
+
add_column :last_decayed, DateTime unless existing.include?(:last_decayed)
|
|
22
|
+
add_column :reinforcement_count, Integer, default: 0 unless existing.include?(:reinforcement_count)
|
|
23
|
+
add_column :unresolved, TrueClass, default: false unless existing.include?(:unresolved)
|
|
24
|
+
add_column :consolidation_candidate, TrueClass, default: false unless existing.include?(:consolidation_candidate)
|
|
25
|
+
add_column :parent_trace_id, String, size: 36 unless existing.include?(:parent_trace_id)
|
|
26
|
+
add_column :encryption_key_id, String, size: 255 unless existing.include?(:encryption_key_id)
|
|
27
|
+
add_column :partition_id, String, size: 255 unless existing.include?(:partition_id)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
indexes = begin
|
|
31
|
+
db.indexes(:memory_traces).keys
|
|
32
|
+
rescue StandardError
|
|
33
|
+
[]
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
add_index :memory_traces, :trace_id, unique: true, name: :idx_memory_traces_trace_id unless existing.include?(:trace_id)
|
|
37
|
+
|
|
38
|
+
add_index :memory_traces, :storage_tier, name: :idx_memory_traces_storage_tier unless indexes.include?(:idx_memory_traces_storage_tier)
|
|
39
|
+
add_index :memory_traces, :partition_id, name: :idx_memory_traces_partition_id unless indexes.include?(:idx_memory_traces_partition_id)
|
|
40
|
+
add_index :memory_traces, %i[partition_id trace_type], name: :idx_memory_traces_partition_type unless indexes.include?(:idx_memory_traces_partition_type)
|
|
41
|
+
add_index :memory_traces, :unresolved, name: :idx_memory_traces_unresolved unless indexes.include?(:idx_memory_traces_unresolved)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
down do
|
|
45
|
+
return unless table_exists?(:memory_traces)
|
|
46
|
+
|
|
47
|
+
existing = schema(:memory_traces).map(&:first)
|
|
48
|
+
|
|
49
|
+
%i[trace_id strength peak_strength base_decay_rate emotional_valence emotional_intensity
|
|
50
|
+
domain_tags origin source_agent_id storage_tier last_reinforced last_decayed
|
|
51
|
+
reinforcement_count unresolved consolidation_candidate parent_trace_id
|
|
52
|
+
encryption_key_id partition_id].each do |col|
|
|
53
|
+
alter_table(:memory_traces) { drop_column col } if existing.include?(col)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
return if table_exists?(:memory_associations)
|
|
6
|
+
|
|
7
|
+
create_table(:memory_associations) do
|
|
8
|
+
primary_key :id
|
|
9
|
+
String :trace_id_a, size: 36, null: false
|
|
10
|
+
String :trace_id_b, size: 36, null: false
|
|
11
|
+
Integer :coactivation_count, default: 1, null: false
|
|
12
|
+
TrueClass :linked, default: false, null: false
|
|
13
|
+
String :tenant_id, size: 64
|
|
14
|
+
DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
|
|
15
|
+
DateTime :updated_at, default: Sequel::CURRENT_TIMESTAMP
|
|
16
|
+
|
|
17
|
+
unique %i[trace_id_a trace_id_b]
|
|
18
|
+
index :trace_id_a
|
|
19
|
+
index :trace_id_b
|
|
20
|
+
index :tenant_id
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
down do
|
|
25
|
+
drop_table?(:memory_associations)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
return if table_exists?(:metering_hourly_rollup)
|
|
6
|
+
|
|
7
|
+
create_table(:metering_hourly_rollup) do
|
|
8
|
+
primary_key :id
|
|
9
|
+
String :worker_id, size: 36, null: false
|
|
10
|
+
String :provider, size: 100, null: false
|
|
11
|
+
String :model_id, size: 255, null: false
|
|
12
|
+
DateTime :hour, null: false
|
|
13
|
+
Integer :total_input_tokens, default: 0, null: false
|
|
14
|
+
Integer :total_output_tokens, default: 0, null: false
|
|
15
|
+
Integer :total_thinking_tokens, default: 0, null: false
|
|
16
|
+
Integer :total_calls, default: 0, null: false
|
|
17
|
+
Float :total_cost_usd, default: 0.0, null: false
|
|
18
|
+
Float :avg_latency_ms, default: 0.0, null: false
|
|
19
|
+
String :tenant_id, size: 64
|
|
20
|
+
DateTime :created_at, default: Sequel::CURRENT_TIMESTAMP
|
|
21
|
+
|
|
22
|
+
unique %i[worker_id provider model_id hour], name: :idx_rollup_unique_hour
|
|
23
|
+
index :hour
|
|
24
|
+
index :tenant_id
|
|
25
|
+
index %i[worker_id hour]
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
down do
|
|
30
|
+
drop_table?(:metering_hourly_rollup)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next unless adapter_scheme == :postgres
|
|
6
|
+
|
|
7
|
+
# --- Identity columns on apollo_entries ---
|
|
8
|
+
alter_table(:apollo_entries) do
|
|
9
|
+
add_column :submitted_by, String, size: 255
|
|
10
|
+
add_column :submitted_from, String, size: 255
|
|
11
|
+
add_column :content_hash, String, fixed: true, size: 32
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# --- apollo_operations table ---
|
|
15
|
+
run <<~SQL
|
|
16
|
+
CREATE TABLE IF NOT EXISTS apollo_operations (
|
|
17
|
+
id BIGSERIAL PRIMARY KEY,
|
|
18
|
+
operation VARCHAR(50) NOT NULL,
|
|
19
|
+
actor VARCHAR(100) NOT NULL,
|
|
20
|
+
target_type VARCHAR(50),
|
|
21
|
+
target_ids INTEGER[],
|
|
22
|
+
summary TEXT,
|
|
23
|
+
detail JSONB,
|
|
24
|
+
old_state JSONB,
|
|
25
|
+
new_state JSONB,
|
|
26
|
+
reason TEXT,
|
|
27
|
+
principal_id VARCHAR(255),
|
|
28
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
29
|
+
);
|
|
30
|
+
SQL
|
|
31
|
+
|
|
32
|
+
# --- apollo_entries_archive table ---
|
|
33
|
+
run <<~SQL
|
|
34
|
+
CREATE TABLE IF NOT EXISTS apollo_entries_archive (
|
|
35
|
+
LIKE apollo_entries INCLUDING ALL,
|
|
36
|
+
archived_at TIMESTAMPTZ DEFAULT NOW(),
|
|
37
|
+
archive_reason TEXT
|
|
38
|
+
);
|
|
39
|
+
SQL
|
|
40
|
+
|
|
41
|
+
# --- Indexes: apollo_entries ---
|
|
42
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_submitted_by ON apollo_entries (submitted_by);'
|
|
43
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_submitted_from ON apollo_entries (submitted_from);'
|
|
44
|
+
|
|
45
|
+
# Content hash dedup (unique among active entries only)
|
|
46
|
+
run <<~SQL
|
|
47
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_apollo_content_hash
|
|
48
|
+
ON apollo_entries (content_hash)
|
|
49
|
+
WHERE status != 'archived';
|
|
50
|
+
SQL
|
|
51
|
+
|
|
52
|
+
# Status filtering (every read query filters on status)
|
|
53
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_status ON apollo_entries (status);'
|
|
54
|
+
|
|
55
|
+
# Partial index: active entries only (hot path)
|
|
56
|
+
run <<~SQL
|
|
57
|
+
CREATE INDEX IF NOT EXISTS idx_apollo_active
|
|
58
|
+
ON apollo_entries (id)
|
|
59
|
+
WHERE status IN ('candidate', 'confirmed', 'disputed');
|
|
60
|
+
SQL
|
|
61
|
+
|
|
62
|
+
# Confidence ranking and decay targeting
|
|
63
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_confidence ON apollo_entries (confidence);'
|
|
64
|
+
|
|
65
|
+
# Time-based: decay age, archival sweep
|
|
66
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_created ON apollo_entries (created_at);'
|
|
67
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_updated ON apollo_entries (updated_at);'
|
|
68
|
+
|
|
69
|
+
# Composite: decay cycle targets
|
|
70
|
+
run <<~SQL
|
|
71
|
+
CREATE INDEX IF NOT EXISTS idx_apollo_decay_target
|
|
72
|
+
ON apollo_entries (updated_at)
|
|
73
|
+
WHERE status != 'archived';
|
|
74
|
+
SQL
|
|
75
|
+
|
|
76
|
+
# Composite: corroboration targets
|
|
77
|
+
run <<~SQL
|
|
78
|
+
CREATE INDEX IF NOT EXISTS idx_apollo_candidates
|
|
79
|
+
ON apollo_entries (status, source_provider, source_channel)
|
|
80
|
+
WHERE status = 'candidate' AND embedding IS NOT NULL;
|
|
81
|
+
SQL
|
|
82
|
+
|
|
83
|
+
# Knowledge domain (expertise, RBAC)
|
|
84
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_domain ON apollo_entries (knowledge_domain);'
|
|
85
|
+
|
|
86
|
+
# Source agent (expertise aggregation)
|
|
87
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_source_agent ON apollo_entries (source_agent);'
|
|
88
|
+
|
|
89
|
+
# Drop existing HNSW index and recreate as partial (active entries only)
|
|
90
|
+
run 'DROP INDEX IF EXISTS apollo_entries_embedding_idx;'
|
|
91
|
+
run <<~SQL
|
|
92
|
+
CREATE INDEX IF NOT EXISTS idx_apollo_embedding_active
|
|
93
|
+
ON apollo_entries USING hnsw (embedding vector_cosine_ops)
|
|
94
|
+
WITH (m = 16, ef_construction = 64)
|
|
95
|
+
WHERE status IN ('candidate', 'confirmed', 'disputed');
|
|
96
|
+
SQL
|
|
97
|
+
|
|
98
|
+
# --- Indexes: apollo_relations ---
|
|
99
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_from ON apollo_relations (from_entry_id);'
|
|
100
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_to ON apollo_relations (to_entry_id);'
|
|
101
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_type ON apollo_relations (relation_type);'
|
|
102
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_rel_composite ON apollo_relations (from_entry_id, relation_type);'
|
|
103
|
+
|
|
104
|
+
# --- Indexes: apollo_expertise ---
|
|
105
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_agent ON apollo_expertise (agent_id);'
|
|
106
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_domain ON apollo_expertise (domain);'
|
|
107
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_exp_composite ON apollo_expertise (agent_id, domain);'
|
|
108
|
+
|
|
109
|
+
# --- Indexes: apollo_operations ---
|
|
110
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_created ON apollo_operations (created_at);'
|
|
111
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_operation ON apollo_operations (operation);'
|
|
112
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_actor ON apollo_operations (actor);'
|
|
113
|
+
run 'CREATE INDEX IF NOT EXISTS idx_apollo_ops_target ON apollo_operations USING GIN (target_ids);'
|
|
114
|
+
|
|
115
|
+
# --- Indexes: apollo_entries_archive ---
|
|
116
|
+
run 'CREATE INDEX IF NOT EXISTS idx_archive_content_hash ON apollo_entries_archive (content_hash);'
|
|
117
|
+
run 'CREATE INDEX IF NOT EXISTS idx_archive_source_agent ON apollo_entries_archive (source_agent);'
|
|
118
|
+
run 'CREATE INDEX IF NOT EXISTS idx_archive_archived_at ON apollo_entries_archive (archived_at);'
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
down do
|
|
122
|
+
next unless adapter_scheme == :postgres
|
|
123
|
+
|
|
124
|
+
# Restore original HNSW index (non-partial)
|
|
125
|
+
run 'DROP INDEX IF EXISTS idx_apollo_embedding_active;'
|
|
126
|
+
run <<~SQL
|
|
127
|
+
CREATE INDEX IF NOT EXISTS apollo_entries_embedding_idx
|
|
128
|
+
ON apollo_entries USING hnsw (embedding vector_cosine_ops);
|
|
129
|
+
SQL
|
|
130
|
+
|
|
131
|
+
drop_table?(:apollo_entries_archive)
|
|
132
|
+
drop_table?(:apollo_operations)
|
|
133
|
+
|
|
134
|
+
# Drop new indexes
|
|
135
|
+
%w[
|
|
136
|
+
idx_apollo_submitted_by idx_apollo_submitted_from idx_apollo_content_hash
|
|
137
|
+
idx_apollo_status idx_apollo_active idx_apollo_confidence
|
|
138
|
+
idx_apollo_created idx_apollo_updated idx_apollo_decay_target
|
|
139
|
+
idx_apollo_candidates idx_apollo_domain idx_apollo_source_agent
|
|
140
|
+
idx_apollo_rel_from idx_apollo_rel_to idx_apollo_rel_type idx_apollo_rel_composite
|
|
141
|
+
idx_apollo_exp_agent idx_apollo_exp_domain idx_apollo_exp_composite
|
|
142
|
+
idx_apollo_ops_created idx_apollo_ops_operation idx_apollo_ops_actor idx_apollo_ops_target
|
|
143
|
+
idx_archive_content_hash idx_archive_source_agent idx_archive_archived_at
|
|
144
|
+
].each { |idx| run "DROP INDEX IF EXISTS #{idx};" }
|
|
145
|
+
|
|
146
|
+
alter_table(:apollo_entries) do
|
|
147
|
+
drop_column :content_hash
|
|
148
|
+
drop_column :submitted_from
|
|
149
|
+
drop_column :submitted_by
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Legion
|
|
4
|
+
module Data
|
|
5
|
+
module Rls
|
|
6
|
+
RLS_TABLES = %i[
|
|
7
|
+
tasks digital_workers audit_log memory_traces extensions
|
|
8
|
+
functions runners nodes settings value_metrics
|
|
9
|
+
].freeze
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
def rls_enabled?
|
|
14
|
+
return false unless Legion::Settings[:data][:connected]
|
|
15
|
+
|
|
16
|
+
Legion::Data.connection.adapter_scheme == :postgres
|
|
17
|
+
rescue StandardError
|
|
18
|
+
false
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def assign_tenant(tenant_id)
|
|
22
|
+
return unless rls_enabled?
|
|
23
|
+
|
|
24
|
+
Legion::Data.connection.run(
|
|
25
|
+
Sequel.lit('SET app.current_tenant = ?', tenant_id.to_s)
|
|
26
|
+
)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def current_tenant
|
|
30
|
+
return nil unless rls_enabled?
|
|
31
|
+
|
|
32
|
+
Legion::Data.connection.fetch('SHOW app.current_tenant').first&.values&.first
|
|
33
|
+
rescue Sequel::DatabaseError
|
|
34
|
+
nil
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def reset_tenant
|
|
38
|
+
return unless rls_enabled?
|
|
39
|
+
|
|
40
|
+
Legion::Data.connection.run('RESET app.current_tenant')
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def with_tenant(tenant_id)
|
|
44
|
+
previous = current_tenant
|
|
45
|
+
assign_tenant(tenant_id)
|
|
46
|
+
yield
|
|
47
|
+
ensure
|
|
48
|
+
previous ? assign_tenant(previous) : reset_tenant
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
data/lib/legion/data/version.rb
CHANGED
data/lib/legion/data.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legion-data
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.6.
|
|
4
|
+
version: 1.6.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Esity
|
|
@@ -141,6 +141,13 @@ files:
|
|
|
141
141
|
- lib/legion/data/migrations/038_add_conversations.rb
|
|
142
142
|
- lib/legion/data/migrations/039_add_audit_archive_manifest.rb
|
|
143
143
|
- lib/legion/data/migrations/040_add_slow_query_indexes.rb
|
|
144
|
+
- lib/legion/data/migrations/041_resize_vector_columns.rb
|
|
145
|
+
- lib/legion/data/migrations/042_add_tenant_to_registry_tables.rb
|
|
146
|
+
- lib/legion/data/migrations/043_add_rls_placeholder.rb
|
|
147
|
+
- lib/legion/data/migrations/044_expand_memory_traces.rb
|
|
148
|
+
- lib/legion/data/migrations/045_add_memory_associations.rb
|
|
149
|
+
- lib/legion/data/migrations/046_add_metering_hourly_rollup.rb
|
|
150
|
+
- lib/legion/data/migrations/047_apollo_knowledge_capture.rb
|
|
144
151
|
- lib/legion/data/model.rb
|
|
145
152
|
- lib/legion/data/models/apollo_access_log.rb
|
|
146
153
|
- lib/legion/data/models/apollo_entry.rb
|
|
@@ -161,6 +168,7 @@ files:
|
|
|
161
168
|
- lib/legion/data/models/task_log.rb
|
|
162
169
|
- lib/legion/data/partition_manager.rb
|
|
163
170
|
- lib/legion/data/retention.rb
|
|
171
|
+
- lib/legion/data/rls.rb
|
|
164
172
|
- lib/legion/data/settings.rb
|
|
165
173
|
- lib/legion/data/spool.rb
|
|
166
174
|
- lib/legion/data/storage_tiers.rb
|