legion-data 1.6.9 → 1.6.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/CLAUDE.md +15 -4
- data/lib/legion/data/archival/policy.rb +8 -0
- data/lib/legion/data/migrations/049_add_remote_invocable_to_functions.rb +21 -0
- data/lib/legion/data/migrations/050_add_missing_indexes.rb +176 -0
- data/lib/legion/data/migrations/051_fix_tasks_created_at.rb +44 -0
- data/lib/legion/data/migrations/052_drop_redundant_apollo_indexes.rb +32 -0
- data/lib/legion/data/migrations/053_add_tasks_relationship_fk.rb +38 -0
- data/lib/legion/data/migrations/054_add_component_type_to_functions.rb +23 -0
- data/lib/legion/data/migrations/055_add_definition_to_functions.rb +21 -0
- data/lib/legion/data/migrations/056_add_absorber_patterns.rb +28 -0
- data/lib/legion/data/migrations/057_add_routing_key_to_runners.rb +23 -0
- data/lib/legion/data/retention.rb +20 -5
- data/lib/legion/data/version.rb +1 -1
- metadata +10 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c0e428fc5d4a92e8fe8b3c24f2277f5842e13b0c9814f712c82cc7cc41a43cfc
|
|
4
|
+
data.tar.gz: b9d1dd28db840e02b4308d2b29b20a1ff89b7932c07952f243b64cad6b45d5c1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: df6941f0d373dbb1d18aadd1b75ee092ec58d8da8fb82b273e6b847f8cf420e999212d0323171bfdf965fe4b105e0e4f13bbe62d3f29ac77f63c35477a4f78c7
|
|
7
|
+
data.tar.gz: d64f91ecafd301f33153e5bc77988014f11ceb2a3cc0eacf0201bcb19250adc7f71888004643d45ab967cf230ca82fb1986b166b389f1b08a4c098890fed98c1
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# Legion::Data Changelog
|
|
2
2
|
|
|
3
|
+
## [1.6.11] - 2026-03-28
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Migration 050: critical missing indexes across 13 tables (runners, tasks, digital_workers, audit_log, webhook_deliveries, webhook_dead_letters, conversations, approval_queue, rbac_role_assignments, rbac_cross_team_grants, memory_traces, agent_cluster_tasks, finlog_executions)
|
|
7
|
+
- Migration 051: fix tasks archival column mismatch — adds `created_at` column (PG: generated from `created`, SQLite/MySQL: backfilled)
|
|
8
|
+
- Migration 052: drop redundant Apollo indexes (PG only) — auto-named duplicates from migration 012 superseded by explicit indexes in migration 047
|
|
9
|
+
- Migration 053: FK constraint for `tasks.relationship_id` (PG only) with orphan cleanup and ON DELETE SET NULL
|
|
10
|
+
- Migration 054: add `component_type` column to functions table (v3.0 naming convention — runner/hook/absorber)
|
|
11
|
+
- Migration 055: add nullable `definition` text column to functions table (v3.0 method contract storage)
|
|
12
|
+
- Migration 056: add `absorber_patterns` table for pattern-matched content acquisition (v3.0)
|
|
13
|
+
- Migration 057: add `routing_key` column to runners table (v3.0 AMQP routing key storage)
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
- `Archival::Policy` now includes `DATE_COLUMN_OVERRIDES` map for legacy tables using non-standard date columns
|
|
17
|
+
|
|
18
|
+
## [1.6.10] - 2026-03-28
|
|
19
|
+
|
|
20
|
+
### Added
|
|
21
|
+
- Migration 049: adds `remote_invocable` boolean column (default: true) to the `functions` table. Allows per-function control over whether a registered function can be dispatched remotely via AMQP from the `LexDispatch` API layer.
|
|
22
|
+
|
|
3
23
|
## [1.6.9] - 2026-03-27
|
|
4
24
|
|
|
5
25
|
### Added
|
data/CLAUDE.md
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
Manages persistent database storage for the LegionIO framework. Supports SQLite (default), MySQL, and PostgreSQL via Sequel ORM. Provides automatic schema migrations and data models for extensions, functions, runners, nodes, tasks, settings, digital workers, task relationships, Apollo shared knowledge tables (PostgreSQL only), tenants, webhooks, audit log, and archive tables. Also provides a parallel local SQLite database (`Legion::Data::Local`) for agentic cognitive state persistence.
|
|
9
9
|
|
|
10
10
|
**GitHub**: https://github.com/LegionIO/legion-data
|
|
11
|
-
**Version**: 1.6.
|
|
11
|
+
**Version**: 1.6.11
|
|
12
12
|
**License**: Apache-2.0
|
|
13
13
|
|
|
14
14
|
## Supported Databases
|
|
@@ -56,7 +56,7 @@ Legion::Data (singleton module)
|
|
|
56
56
|
│ ├── .shutdown # Close local connection
|
|
57
57
|
│ └── .reset! # Clear all state (testing)
|
|
58
58
|
│
|
|
59
|
-
├── Migration # Auto-migration system (
|
|
59
|
+
├── Migration # Auto-migration system (57 migrations, Sequel DSL)
|
|
60
60
|
│ └── migrations/
|
|
61
61
|
│ ├── 001_add_schema_columns
|
|
62
62
|
│ ├── 002_add_nodes
|
|
@@ -104,7 +104,17 @@ Legion::Data (singleton module)
|
|
|
104
104
|
│ ├── 044_expand_memory_traces
|
|
105
105
|
│ ├── 045_add_memory_associations
|
|
106
106
|
│ ├── 046_add_metering_hourly_rollup
|
|
107
|
-
│
|
|
107
|
+
│ ├── 047_apollo_knowledge_capture # identity cols, ops table, archive table, 25+ indexes
|
|
108
|
+
│ ├── 048_add_financial_logging # 7 UAIS cost recovery tables (identity, asset, environment, accounting, execution, tags, usage)
|
|
109
|
+
│ ├── 049_add_remote_invocable_to_functions # remote_invocable boolean on functions (v3.0)
|
|
110
|
+
│ ├── 050_add_missing_indexes # critical indexes across 13 tables
|
|
111
|
+
│ ├── 051_fix_tasks_created_at # created_at alias for archival (PG generated, SQLite backfill)
|
|
112
|
+
│ ├── 052_drop_redundant_apollo_indexes # PG only: remove duplicate auto-named indexes
|
|
113
|
+
│ ├── 053_add_tasks_relationship_fk # PG only: FK constraint on tasks.relationship_id
|
|
114
|
+
│ ├── 054_add_component_type_to_functions # component_type on functions (runner/hook/absorber, v3.0)
|
|
115
|
+
│ ├── 055_add_definition_to_functions # definition text column on functions (v3.0)
|
|
116
|
+
│ ├── 056_add_absorber_patterns # absorber_patterns table for pattern-matched acquisition
|
|
117
|
+
│ └── 057_add_routing_key_to_runners # routing_key on runners (v3.0 AMQP)
|
|
108
118
|
│
|
|
109
119
|
├── Model # Sequel model loader
|
|
110
120
|
│ └── Models/
|
|
@@ -269,7 +279,7 @@ Per-adapter credential defaults are defined in `Settings::CREDS`:
|
|
|
269
279
|
| `lib/legion/data.rb` | Module entry, setup/shutdown lifecycle |
|
|
270
280
|
| `lib/legion/data/connection.rb` | Sequel database connection (adapter selection) |
|
|
271
281
|
| `lib/legion/data/migration.rb` | Migration runner |
|
|
272
|
-
| `lib/legion/data/migrations/` |
|
|
282
|
+
| `lib/legion/data/migrations/` | 48 numbered migration files (Sequel DSL) |
|
|
273
283
|
| `lib/legion/data/model.rb` | Model autoloader |
|
|
274
284
|
| `lib/legion/data/local.rb` | Local SQLite module for agentic cognitive state |
|
|
275
285
|
| `lib/legion/data/models/` | Sequel models (Extension, Function, Runner, Node, Task, TaskLog, Setting, DigitalWorker, Relationship, ApolloEntry, ApolloRelation, ApolloExpertise, ApolloAccessLog, AuditLog, RbacRoleAssignment, RbacRunnerGrant, RbacCrossTeamGrant) |
|
|
@@ -309,6 +319,7 @@ Optional persistent storage initialized during `Legion::Service` startup (after
|
|
|
309
319
|
12. Webhook subscription storage (migration 020)
|
|
310
320
|
13. Archive, memory traces, and tenant partition tables (migrations 021–025)
|
|
311
321
|
14. Function embeddings for semantic runner discovery (migration 026 — description + vector columns on functions table)
|
|
322
|
+
15. Financial logging for UAIS cost recovery (migration 048 — 7 tables: identity, asset, environment, accounting, execution, tags, usage rollup)
|
|
312
323
|
|
|
313
324
|
---
|
|
314
325
|
|
|
@@ -11,6 +11,14 @@ module Legion
|
|
|
11
11
|
tables: %w[tasks metering_records].freeze
|
|
12
12
|
}.freeze
|
|
13
13
|
|
|
14
|
+
# Per-table date column overrides. The Retention module defaults to :created_at,
|
|
15
|
+
# but legacy tables (like tasks) use :created. Migration 051 may add a created_at
|
|
16
|
+
# column/alias for tasks (implementation varies by adapter); this map forces use of
|
|
17
|
+
# :created so behavior is consistent across legacy schemas and adapters.
|
|
18
|
+
DATE_COLUMN_OVERRIDES = {
|
|
19
|
+
'tasks' => :created
|
|
20
|
+
}.freeze
|
|
21
|
+
|
|
14
22
|
attr_reader :warm_after_days, :cold_after_days, :batch_size, :tables
|
|
15
23
|
|
|
16
24
|
def initialize(**opts)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next unless table_exists?(:functions)
|
|
6
|
+
next if schema(:functions).any? { |c, _| c == :remote_invocable }
|
|
7
|
+
|
|
8
|
+
alter_table(:functions) do
|
|
9
|
+
add_column :remote_invocable, TrueClass, default: true, null: false
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
down do
|
|
14
|
+
next unless table_exists?(:functions)
|
|
15
|
+
next unless schema(:functions).any? { |c, _| c == :remote_invocable }
|
|
16
|
+
|
|
17
|
+
alter_table(:functions) do
|
|
18
|
+
drop_column :remote_invocable
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
# runners: FK without index, hot-path lookups, duplicate prevention
|
|
6
|
+
if table_exists?(:runners)
|
|
7
|
+
# Remove any duplicate (extension_id, name) rows before adding the unique index.
|
|
8
|
+
# Keep the active and most recently updated row per pair; use id DESC as tie-breaker.
|
|
9
|
+
run <<~SQL
|
|
10
|
+
WITH ranked AS (
|
|
11
|
+
SELECT
|
|
12
|
+
id,
|
|
13
|
+
ROW_NUMBER() OVER (
|
|
14
|
+
PARTITION BY extension_id, name
|
|
15
|
+
ORDER BY
|
|
16
|
+
active DESC,
|
|
17
|
+
updated DESC,
|
|
18
|
+
id DESC
|
|
19
|
+
) AS rn
|
|
20
|
+
FROM runners
|
|
21
|
+
)
|
|
22
|
+
DELETE FROM runners
|
|
23
|
+
WHERE id IN (
|
|
24
|
+
SELECT id
|
|
25
|
+
FROM ranked
|
|
26
|
+
WHERE rn > 1
|
|
27
|
+
)
|
|
28
|
+
SQL
|
|
29
|
+
|
|
30
|
+
alter_table(:runners) do
|
|
31
|
+
add_index :extension_id, name: :idx_runners_extension_id, if_not_exists: true
|
|
32
|
+
add_index :namespace, name: :idx_runners_namespace, if_not_exists: true
|
|
33
|
+
add_index :name, name: :idx_runners_name, if_not_exists: true
|
|
34
|
+
add_index %i[extension_id name], name: :idx_runners_extension_name, unique: true, if_not_exists: true
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# tasks: plain Integer relationship_id used by ORM association
|
|
39
|
+
if table_exists?(:tasks)
|
|
40
|
+
alter_table(:tasks) do
|
|
41
|
+
add_index :relationship_id, name: :idx_tasks_relationship_id, if_not_exists: true
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# digital_workers: consent/trust-based queries
|
|
46
|
+
if table_exists?(:digital_workers)
|
|
47
|
+
alter_table(:digital_workers) do
|
|
48
|
+
add_index :consent_tier, name: :idx_digital_workers_consent_tier, if_not_exists: true
|
|
49
|
+
add_index :trust_score, name: :idx_digital_workers_trust_score, if_not_exists: true
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# audit_log: composite principal+time query, action/node lookups
|
|
54
|
+
if table_exists?(:audit_log)
|
|
55
|
+
alter_table(:audit_log) do
|
|
56
|
+
add_index %i[principal_id created_at], name: :idx_audit_log_principal_time, if_not_exists: true
|
|
57
|
+
add_index :action, name: :idx_audit_log_action, if_not_exists: true
|
|
58
|
+
add_index :node, name: :idx_audit_log_node, if_not_exists: true
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# webhook_deliveries: event/time/success filtering
|
|
63
|
+
if table_exists?(:webhook_deliveries)
|
|
64
|
+
alter_table(:webhook_deliveries) do
|
|
65
|
+
add_index :event_name, name: :idx_webhook_deliveries_event_name, if_not_exists: true
|
|
66
|
+
add_index :delivered_at, name: :idx_webhook_deliveries_delivered_at, if_not_exists: true
|
|
67
|
+
add_index :success, name: :idx_webhook_deliveries_success, if_not_exists: true
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# webhook_dead_letters: event/time filtering
|
|
72
|
+
if table_exists?(:webhook_dead_letters)
|
|
73
|
+
alter_table(:webhook_dead_letters) do
|
|
74
|
+
add_index :event_name, name: :idx_webhook_dead_letters_event_name, if_not_exists: true
|
|
75
|
+
add_index :created_at, name: :idx_webhook_dead_letters_created_at, if_not_exists: true
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# conversations: identity and recency lookups
|
|
80
|
+
if table_exists?(:conversations)
|
|
81
|
+
alter_table(:conversations) do
|
|
82
|
+
add_index :caller_identity, name: :idx_conversations_caller_identity, if_not_exists: true
|
|
83
|
+
add_index :updated_at, name: :idx_conversations_updated_at, if_not_exists: true
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# approval_queue: requester/reviewer lookups
|
|
88
|
+
if table_exists?(:approval_queue)
|
|
89
|
+
alter_table(:approval_queue) do
|
|
90
|
+
add_index :requester_id, name: :idx_approval_queue_requester_id, if_not_exists: true
|
|
91
|
+
add_index :reviewer_id, name: :idx_approval_queue_reviewer_id, if_not_exists: true
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# rbac_role_assignments: role and expiry lookups
|
|
96
|
+
if table_exists?(:rbac_role_assignments)
|
|
97
|
+
alter_table(:rbac_role_assignments) do
|
|
98
|
+
add_index :role, name: :idx_rbac_role_assignments_role, if_not_exists: true
|
|
99
|
+
add_index :expires_at, name: :idx_rbac_role_assignments_expires_at, if_not_exists: true
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# rbac_cross_team_grants: target team and expiry lookups
|
|
104
|
+
if table_exists?(:rbac_cross_team_grants)
|
|
105
|
+
alter_table(:rbac_cross_team_grants) do
|
|
106
|
+
add_index :target_team, name: :idx_rbac_cross_team_grants_target_team, if_not_exists: true
|
|
107
|
+
add_index :expires_at, name: :idx_rbac_cross_team_grants_expires_at, if_not_exists: true
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# memory_traces: consolidation and source agent lookups
|
|
112
|
+
if table_exists?(:memory_traces)
|
|
113
|
+
existing_cols = schema(:memory_traces).map(&:first)
|
|
114
|
+
|
|
115
|
+
if existing_cols.include?(:consolidation_candidate)
|
|
116
|
+
alter_table(:memory_traces) do
|
|
117
|
+
add_index :consolidation_candidate, name: :idx_memory_traces_consolidation, if_not_exists: true
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
if existing_cols.include?(:source_agent_id)
|
|
122
|
+
alter_table(:memory_traces) do
|
|
123
|
+
add_index :source_agent_id, name: :idx_memory_traces_source_agent_id, if_not_exists: true
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# agent_cluster_tasks: time-based querying
|
|
129
|
+
if table_exists?(:agent_cluster_tasks)
|
|
130
|
+
alter_table(:agent_cluster_tasks) do
|
|
131
|
+
add_index :created_at, name: :idx_agent_cluster_tasks_created_at, if_not_exists: true
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# finlog_executions: environment_id FK without index
|
|
136
|
+
if table_exists?(:finlog_executions)
|
|
137
|
+
alter_table(:finlog_executions) do
|
|
138
|
+
add_index :environment_id, name: :idx_finlog_exec_environment_id, if_not_exists: true
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
down do
|
|
144
|
+
[
|
|
145
|
+
[:runners, %i[
|
|
146
|
+
idx_runners_extension_id idx_runners_namespace idx_runners_name idx_runners_extension_name
|
|
147
|
+
]],
|
|
148
|
+
[:tasks, %i[idx_tasks_relationship_id]],
|
|
149
|
+
[:digital_workers, %i[idx_digital_workers_consent_tier idx_digital_workers_trust_score]],
|
|
150
|
+
[:audit_log, %i[idx_audit_log_principal_time idx_audit_log_action idx_audit_log_node]],
|
|
151
|
+
[:webhook_deliveries, %i[
|
|
152
|
+
idx_webhook_deliveries_event_name idx_webhook_deliveries_delivered_at idx_webhook_deliveries_success
|
|
153
|
+
]],
|
|
154
|
+
[:webhook_dead_letters, %i[
|
|
155
|
+
idx_webhook_dead_letters_event_name idx_webhook_dead_letters_created_at
|
|
156
|
+
]],
|
|
157
|
+
[:conversations, %i[idx_conversations_caller_identity idx_conversations_updated_at]],
|
|
158
|
+
[:approval_queue, %i[idx_approval_queue_requester_id idx_approval_queue_reviewer_id]],
|
|
159
|
+
[:rbac_role_assignments, %i[idx_rbac_role_assignments_role idx_rbac_role_assignments_expires_at]],
|
|
160
|
+
[:rbac_cross_team_grants, %i[
|
|
161
|
+
idx_rbac_cross_team_grants_target_team idx_rbac_cross_team_grants_expires_at
|
|
162
|
+
]],
|
|
163
|
+
[:memory_traces, %i[idx_memory_traces_consolidation idx_memory_traces_source_agent_id]],
|
|
164
|
+
[:agent_cluster_tasks, %i[idx_agent_cluster_tasks_created_at]],
|
|
165
|
+
[:finlog_executions, %i[idx_finlog_exec_environment_id]]
|
|
166
|
+
].each do |table, indexes|
|
|
167
|
+
next unless table_exists?(table)
|
|
168
|
+
|
|
169
|
+
alter_table(table) do
|
|
170
|
+
indexes.each do |idx_name|
|
|
171
|
+
drop_index nil, name: idx_name, if_exists: true
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
end
|
|
176
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next unless table_exists?(:tasks)
|
|
6
|
+
|
|
7
|
+
existing_cols = schema(:tasks).map(&:first)
|
|
8
|
+
next if existing_cols.include?(:created_at)
|
|
9
|
+
|
|
10
|
+
if adapter_scheme == :postgres
|
|
11
|
+
# Add a generated column so retention/archival queries using created_at work transparently
|
|
12
|
+
run 'ALTER TABLE tasks ADD COLUMN created_at TIMESTAMPTZ GENERATED ALWAYS AS (created) STORED'
|
|
13
|
+
run 'CREATE INDEX IF NOT EXISTS idx_tasks_created_at ON tasks (created_at)'
|
|
14
|
+
else
|
|
15
|
+
# SQLite/MySQL: add real column and backfill from created
|
|
16
|
+
alter_table(:tasks) do
|
|
17
|
+
add_column :created_at, DateTime, default: Sequel::CURRENT_TIMESTAMP
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
run 'UPDATE tasks SET created_at = created WHERE created_at IS NULL'
|
|
21
|
+
|
|
22
|
+
alter_table(:tasks) do
|
|
23
|
+
add_index :created_at, name: :idx_tasks_created_at, if_not_exists: true
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
down do
|
|
29
|
+
next unless table_exists?(:tasks)
|
|
30
|
+
|
|
31
|
+
existing_cols = schema(:tasks).map(&:first)
|
|
32
|
+
next unless existing_cols.include?(:created_at)
|
|
33
|
+
|
|
34
|
+
if adapter_scheme == :postgres
|
|
35
|
+
run 'DROP INDEX IF EXISTS idx_tasks_created_at'
|
|
36
|
+
run 'ALTER TABLE tasks DROP COLUMN IF EXISTS created_at'
|
|
37
|
+
else
|
|
38
|
+
alter_table(:tasks) do
|
|
39
|
+
drop_index :created_at, name: :idx_tasks_created_at, if_exists: true
|
|
40
|
+
drop_column :created_at
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
# PostgreSQL only — these auto-named indexes from migration 012 are exact duplicates
|
|
6
|
+
# of explicitly named indexes added in migration 047.
|
|
7
|
+
next unless adapter_scheme == :postgres
|
|
8
|
+
|
|
9
|
+
run 'DROP INDEX IF EXISTS apollo_entries_status_index'
|
|
10
|
+
run 'DROP INDEX IF EXISTS apollo_relations_from_entry_id_index'
|
|
11
|
+
run 'DROP INDEX IF EXISTS apollo_relations_to_entry_id_index'
|
|
12
|
+
run 'DROP INDEX IF EXISTS apollo_expertise_agent_id_index'
|
|
13
|
+
run 'DROP INDEX IF EXISTS apollo_expertise_domain_index'
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
down do
|
|
17
|
+
next unless adapter_scheme == :postgres
|
|
18
|
+
|
|
19
|
+
# Recreate the auto-named indexes that migration 012 created inline.
|
|
20
|
+
# idx_apollo_status, idx_apollo_rel_from, etc. from migration 047 remain in place.
|
|
21
|
+
run 'CREATE INDEX IF NOT EXISTS apollo_entries_status_index ON apollo_entries (status)' \
|
|
22
|
+
if table_exists?(:apollo_entries)
|
|
23
|
+
run 'CREATE INDEX IF NOT EXISTS apollo_relations_from_entry_id_index ON apollo_relations (from_entry_id)' \
|
|
24
|
+
if table_exists?(:apollo_relations)
|
|
25
|
+
run 'CREATE INDEX IF NOT EXISTS apollo_relations_to_entry_id_index ON apollo_relations (to_entry_id)' \
|
|
26
|
+
if table_exists?(:apollo_relations)
|
|
27
|
+
run 'CREATE INDEX IF NOT EXISTS apollo_expertise_agent_id_index ON apollo_expertise (agent_id)' \
|
|
28
|
+
if table_exists?(:apollo_expertise)
|
|
29
|
+
run 'CREATE INDEX IF NOT EXISTS apollo_expertise_domain_index ON apollo_expertise (domain)' \
|
|
30
|
+
if table_exists?(:apollo_expertise)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
# PostgreSQL only — add FK constraint for tasks.relationship_id with ON DELETE SET NULL.
|
|
6
|
+
# Orphaned values must be cleaned first.
|
|
7
|
+
next unless adapter_scheme == :postgres
|
|
8
|
+
next unless table_exists?(:tasks)
|
|
9
|
+
next unless table_exists?(:relationships)
|
|
10
|
+
|
|
11
|
+
# Clean orphaned relationship_id values before adding constraint
|
|
12
|
+
run <<~SQL
|
|
13
|
+
UPDATE tasks
|
|
14
|
+
SET relationship_id = NULL
|
|
15
|
+
WHERE relationship_id IS NOT NULL
|
|
16
|
+
AND relationship_id NOT IN (SELECT id FROM relationships);
|
|
17
|
+
SQL
|
|
18
|
+
|
|
19
|
+
# Skip if constraint already exists (idempotency guard)
|
|
20
|
+
constraint_exists = self[:pg_constraint].where(conname: 'fk_tasks_relationship_id').any?
|
|
21
|
+
|
|
22
|
+
unless constraint_exists
|
|
23
|
+
run <<~SQL
|
|
24
|
+
ALTER TABLE tasks
|
|
25
|
+
ADD CONSTRAINT fk_tasks_relationship_id
|
|
26
|
+
FOREIGN KEY (relationship_id) REFERENCES relationships(id)
|
|
27
|
+
ON DELETE SET NULL;
|
|
28
|
+
SQL
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
down do
|
|
33
|
+
next unless adapter_scheme == :postgres
|
|
34
|
+
next unless table_exists?(:tasks)
|
|
35
|
+
|
|
36
|
+
run 'ALTER TABLE tasks DROP CONSTRAINT IF EXISTS fk_tasks_relationship_id'
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next unless table_exists?(:functions)
|
|
6
|
+
next if schema(:functions).any? { |c, _| c == :component_type }
|
|
7
|
+
|
|
8
|
+
alter_table(:functions) do
|
|
9
|
+
add_column :component_type, String, size: 32, null: false, default: 'runner'
|
|
10
|
+
add_index :component_type, name: :idx_functions_component_type, if_not_exists: true
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
down do
|
|
15
|
+
next unless table_exists?(:functions)
|
|
16
|
+
next unless schema(:functions).any? { |c, _| c == :component_type }
|
|
17
|
+
|
|
18
|
+
alter_table(:functions) do
|
|
19
|
+
drop_index :component_type, name: :idx_functions_component_type, if_exists: true
|
|
20
|
+
drop_column :component_type
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next unless table_exists?(:functions)
|
|
6
|
+
next if schema(:functions).any? { |c, _| c == :definition }
|
|
7
|
+
|
|
8
|
+
alter_table(:functions) do
|
|
9
|
+
add_column :definition, String, text: true, null: true
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
down do
|
|
14
|
+
next unless table_exists?(:functions)
|
|
15
|
+
next unless schema(:functions).any? { |c, _| c == :definition }
|
|
16
|
+
|
|
17
|
+
alter_table(:functions) do
|
|
18
|
+
drop_column :definition
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next if table_exists?(:absorber_patterns)
|
|
6
|
+
|
|
7
|
+
create_table(:absorber_patterns) do
|
|
8
|
+
primary_key :id
|
|
9
|
+
foreign_key :function_id, :functions, null: false, on_delete: :cascade, index: true
|
|
10
|
+
String :pattern_type, size: 32, null: false, default: 'url'
|
|
11
|
+
String :pattern, size: 1024, null: false
|
|
12
|
+
Integer :priority, null: false, default: 0
|
|
13
|
+
TrueClass :active, null: false, default: true
|
|
14
|
+
String :tenant_id, size: 64, null: true
|
|
15
|
+
DateTime :created_at, null: false, default: Sequel::CURRENT_TIMESTAMP
|
|
16
|
+
DateTime :updated_at, null: true
|
|
17
|
+
|
|
18
|
+
index :pattern_type, name: :idx_absorber_patterns_pattern_type
|
|
19
|
+
index :active, name: :idx_absorber_patterns_active
|
|
20
|
+
index :tenant_id, name: :idx_absorber_patterns_tenant_id
|
|
21
|
+
index %i[pattern_type active], name: :idx_absorber_patterns_type_active
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
down do
|
|
26
|
+
drop_table?(:absorber_patterns)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
Sequel.migration do
|
|
4
|
+
up do
|
|
5
|
+
next unless table_exists?(:runners)
|
|
6
|
+
next if schema(:runners).any? { |c, _| c == :routing_key }
|
|
7
|
+
|
|
8
|
+
alter_table(:runners) do
|
|
9
|
+
add_column :routing_key, String, size: 512, null: true
|
|
10
|
+
add_index :routing_key, name: :idx_runners_routing_key, if_not_exists: true
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
down do
|
|
15
|
+
next unless table_exists?(:runners)
|
|
16
|
+
next unless schema(:runners).any? { |c, _| c == :routing_key }
|
|
17
|
+
|
|
18
|
+
alter_table(:runners) do
|
|
19
|
+
drop_index :routing_key, name: :idx_runners_routing_key, if_exists: true
|
|
20
|
+
drop_column :routing_key
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'archival/policy'
|
|
4
|
+
|
|
3
5
|
module Legion
|
|
4
6
|
module Data
|
|
5
7
|
module Retention
|
|
@@ -7,10 +9,11 @@ module Legion
|
|
|
7
9
|
DEFAULT_ARCHIVE_AFTER_DAYS = 90
|
|
8
10
|
|
|
9
11
|
class << self
|
|
10
|
-
def archive_old_records(table:, date_column:
|
|
12
|
+
def archive_old_records(table:, date_column: nil, archive_after_days: DEFAULT_ARCHIVE_AFTER_DAYS)
|
|
11
13
|
db = Legion::Data.connection
|
|
12
14
|
return { archived: 0, table: table } unless db
|
|
13
15
|
|
|
16
|
+
date_column = resolve_date_column(table, date_column)
|
|
14
17
|
cutoff = Time.now - (archive_after_days * 86_400)
|
|
15
18
|
archive_table = archive_table_name(table)
|
|
16
19
|
|
|
@@ -18,7 +21,7 @@ module Legion
|
|
|
18
21
|
|
|
19
22
|
count = 0
|
|
20
23
|
db.transaction do
|
|
21
|
-
records = db[table].where(Sequel.
|
|
24
|
+
records = db[table].where(Sequel.identifier(date_column) < cutoff)
|
|
22
25
|
count = records.count
|
|
23
26
|
if count.positive?
|
|
24
27
|
db[archive_table].multi_insert(records.all)
|
|
@@ -30,13 +33,14 @@ module Legion
|
|
|
30
33
|
{ archived: count, table: table }
|
|
31
34
|
end
|
|
32
35
|
|
|
33
|
-
def purge_expired_records(table:, date_column:
|
|
36
|
+
def purge_expired_records(table:, date_column: nil, retention_years: DEFAULT_RETENTION_YEARS)
|
|
34
37
|
db = Legion::Data.connection
|
|
35
38
|
archive_table = archive_table_name(table)
|
|
36
39
|
return { purged: 0, table: table } unless db&.table_exists?(archive_table)
|
|
37
40
|
|
|
41
|
+
date_column = resolve_date_column(table, date_column)
|
|
38
42
|
cutoff = Time.now - (retention_years * 365 * 86_400)
|
|
39
|
-
expired = db[archive_table].where(Sequel.
|
|
43
|
+
expired = db[archive_table].where(Sequel.identifier(date_column) < cutoff)
|
|
40
44
|
count = expired.count
|
|
41
45
|
expired.delete if count.positive?
|
|
42
46
|
Legion::Logging.info "Purged #{count} expired row(s) from #{archive_table}" if defined?(Legion::Logging) && count.positive?
|
|
@@ -44,9 +48,10 @@ module Legion
|
|
|
44
48
|
{ purged: count, table: table }
|
|
45
49
|
end
|
|
46
50
|
|
|
47
|
-
def retention_status(table:, date_column:
|
|
51
|
+
def retention_status(table:, date_column: nil)
|
|
48
52
|
db = Legion::Data.connection
|
|
49
53
|
archive_table = archive_table_name(table)
|
|
54
|
+
date_column = resolve_date_column(table, date_column)
|
|
50
55
|
|
|
51
56
|
active_count = db&.table_exists?(table) ? db[table].count : 0
|
|
52
57
|
archived_count = db&.table_exists?(archive_table) ? db[archive_table].count : 0
|
|
@@ -70,6 +75,16 @@ module Legion
|
|
|
70
75
|
|
|
71
76
|
private
|
|
72
77
|
|
|
78
|
+
def resolve_date_column(table, date_column)
|
|
79
|
+
return date_column if date_column
|
|
80
|
+
|
|
81
|
+
if defined?(Legion::Data::Archival::Policy::DATE_COLUMN_OVERRIDES)
|
|
82
|
+
Legion::Data::Archival::Policy::DATE_COLUMN_OVERRIDES[table.to_s] || :created_at
|
|
83
|
+
else
|
|
84
|
+
:created_at
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
73
88
|
def ensure_archive_table!(db, source_table, archive_table)
|
|
74
89
|
return if db.table_exists?(archive_table)
|
|
75
90
|
|
data/lib/legion/data/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: legion-data
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.6.
|
|
4
|
+
version: 1.6.11
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Esity
|
|
@@ -176,6 +176,15 @@ files:
|
|
|
176
176
|
- lib/legion/data/migrations/046_add_metering_hourly_rollup.rb
|
|
177
177
|
- lib/legion/data/migrations/047_apollo_knowledge_capture.rb
|
|
178
178
|
- lib/legion/data/migrations/048_add_financial_logging.rb
|
|
179
|
+
- lib/legion/data/migrations/049_add_remote_invocable_to_functions.rb
|
|
180
|
+
- lib/legion/data/migrations/050_add_missing_indexes.rb
|
|
181
|
+
- lib/legion/data/migrations/051_fix_tasks_created_at.rb
|
|
182
|
+
- lib/legion/data/migrations/052_drop_redundant_apollo_indexes.rb
|
|
183
|
+
- lib/legion/data/migrations/053_add_tasks_relationship_fk.rb
|
|
184
|
+
- lib/legion/data/migrations/054_add_component_type_to_functions.rb
|
|
185
|
+
- lib/legion/data/migrations/055_add_definition_to_functions.rb
|
|
186
|
+
- lib/legion/data/migrations/056_add_absorber_patterns.rb
|
|
187
|
+
- lib/legion/data/migrations/057_add_routing_key_to_runners.rb
|
|
179
188
|
- lib/legion/data/model.rb
|
|
180
189
|
- lib/legion/data/models/apollo_access_log.rb
|
|
181
190
|
- lib/legion/data/models/apollo_entry.rb
|