legion-data 1.2.0 → 1.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +16 -0
  3. data/.gitignore +5 -0
  4. data/.rubocop.yml +41 -12
  5. data/CHANGELOG.md +80 -1
  6. data/CLAUDE.md +199 -0
  7. data/Gemfile +4 -0
  8. data/LICENSE +1 -1
  9. data/README.md +157 -50
  10. data/exe/legionio_migrate +0 -0
  11. data/legion-data.gemspec +13 -13
  12. data/lib/legion/data/connection.rb +39 -25
  13. data/lib/legion/data/encryption/cipher.rb +49 -0
  14. data/lib/legion/data/encryption/key_provider.rb +45 -0
  15. data/lib/legion/data/encryption/sequel_plugin.rb +54 -0
  16. data/lib/legion/data/event_store/projection.rb +56 -0
  17. data/lib/legion/data/event_store.rb +112 -0
  18. data/lib/legion/data/local.rb +77 -0
  19. data/lib/legion/data/migration.rb +5 -3
  20. data/lib/legion/data/migrations/001_add_schema_columns.rb +9 -3
  21. data/lib/legion/data/migrations/002_add_nodes.rb +10 -12
  22. data/lib/legion/data/migrations/003_add_settings.rb +10 -10
  23. data/lib/legion/data/migrations/004_add_extensions.rb +15 -17
  24. data/lib/legion/data/migrations/005_add_runners.rb +13 -13
  25. data/lib/legion/data/migrations/006_add_functions.rb +13 -15
  26. data/lib/legion/data/migrations/007_add_default_extensions.rb +2 -0
  27. data/lib/legion/data/migrations/008_add_tasks.rb +15 -21
  28. data/lib/legion/data/migrations/009_add_digital_workers.rb +45 -0
  29. data/lib/legion/data/migrations/010_add_value_metrics.rb +19 -0
  30. data/lib/legion/data/migrations/011_add_extensions_registry.rb +30 -0
  31. data/lib/legion/data/migrations/012_add_apollo_tables.rb +66 -0
  32. data/lib/legion/data/migrations/013_add_relationships.rb +21 -0
  33. data/lib/legion/data/migrations/014_add_relationship_columns.rb +27 -0
  34. data/lib/legion/data/migrations/015_add_rbac_tables.rb +49 -0
  35. data/lib/legion/data/migrations/016_add_worker_health.rb +33 -0
  36. data/lib/legion/data/migrations/017_add_audit_log.rb +30 -0
  37. data/lib/legion/data/migrations/018_add_governance_events.rb +21 -0
  38. data/lib/legion/data/migrations/019_add_audit_hash_chain.rb +29 -0
  39. data/lib/legion/data/migrations/020_add_webhooks.rb +37 -0
  40. data/lib/legion/data/model.rb +4 -1
  41. data/lib/legion/data/models/apollo_access_log.rb +13 -0
  42. data/lib/legion/data/models/apollo_entry.rb +18 -0
  43. data/lib/legion/data/models/apollo_expertise.rb +12 -0
  44. data/lib/legion/data/models/apollo_relation.rb +14 -0
  45. data/lib/legion/data/models/audit_log.rb +34 -0
  46. data/lib/legion/data/models/digital_worker.rb +44 -0
  47. data/lib/legion/data/models/function.rb +2 -2
  48. data/lib/legion/data/models/node.rb +16 -0
  49. data/lib/legion/data/models/rbac_cross_team_grant.rb +33 -0
  50. data/lib/legion/data/models/rbac_role_assignment.rb +29 -0
  51. data/lib/legion/data/models/rbac_runner_grant.rb +21 -0
  52. data/lib/legion/data/models/relationship.rb +13 -0
  53. data/lib/legion/data/settings.rb +54 -25
  54. data/lib/legion/data/version.rb +3 -1
  55. data/lib/legion/data.rb +20 -1
  56. metadata +51 -32
  57. data/.github/workflows/rubocop-analysis.yml +0 -28
  58. data/.github/workflows/sourcehawk-scan.yml +0 -20
  59. data/CODE_OF_CONDUCT.md +0 -75
  60. data/CONTRIBUTING.md +0 -55
  61. data/INDIVIDUAL_CONTRIBUTOR_LICENSE.md +0 -30
  62. data/NOTICE.txt +0 -9
  63. data/SECURITY.md +0 -9
  64. data/attribution.txt +0 -1
  65. data/sourcehawk.yml +0 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '09d8c21be1d2410f76565f4fabd1fee21160f35bd58912cc98bcc8ad63fa52a3'
4
- data.tar.gz: 86524c9bfad846debf51cdd225e2e021af50649803ebf18d5c92392bf8a10ec2
3
+ metadata.gz: d10f18bacc47360ad1829628bffc116ecbeea74b0b49359f1cee7fba0a7e9d44
4
+ data.tar.gz: e13f4acfa8992e3a00129322fef794b80877f723e0c59210016f64fee57c5cae
5
5
  SHA512:
6
- metadata.gz: 7d59b15c21ad79a121342bd58c794efb6a4805bc3b2e57f11c095794c69d5f03ff5152429b48f78c9484444085be91b3f320ccd58c5072257a70bb27809c019c
7
- data.tar.gz: 5218ff04d17b9b8a0e4a9ad8fb5e968d8421f869e58abf4ddf8356d76d377d2a4a027c938627a667011d503f6206e07ea4e1b7963a65391c8ddef698ccfcb53a
6
+ metadata.gz: 0e61649208294598ecdf19ffca469cba3db7b77d31e0e0a065e4c82e47949e4ea87aa291c5fae18970318e0c27ac1dfcd7eb53143bd7b5acc279eb84c0f2b231
7
+ data.tar.gz: 7692efc8d123ad178c3f3a66531723518e0fac2cb5999d01833bfa472ac8eb5bee06289dd193886fe66c369f3d49ebff689e68267eff4ab030ce746377a6c9a3
@@ -0,0 +1,16 @@
1
+ name: CI
2
+ on:
3
+ push:
4
+ branches: [main]
5
+ pull_request:
6
+
7
+ jobs:
8
+ ci:
9
+ uses: LegionIO/.github/.github/workflows/ci.yml@main
10
+
11
+ release:
12
+ needs: ci
13
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
14
+ uses: LegionIO/.github/.github/workflows/release.yml@main
15
+ secrets:
16
+ rubygems-api-key: ${{ secrets.RUBYGEMS_API_KEY }}
data/.gitignore CHANGED
@@ -13,3 +13,8 @@
13
13
  # rspec failure tracking
14
14
  .rspec_status
15
15
  legionio.key
16
+ # logs and OS artifacts
17
+ legion.log
18
+ .DS_Store
19
+ # SQLite database files
20
+ *.db
data/.rubocop.yml CHANGED
@@ -1,22 +1,51 @@
1
+ AllCops:
2
+ TargetRubyVersion: 3.4
3
+ NewCops: enable
4
+ SuggestExtensions: false
5
+
1
6
  Layout/LineLength:
2
- Max: 120
3
- Exclude:
4
- - 'lib/legion/data/migrations/*.rb'
7
+ Max: 160
8
+
9
+ Layout/SpaceAroundEqualsInParameterDefault:
10
+ EnforcedStyle: space
11
+
12
+ Layout/HashAlignment:
13
+ EnforcedHashRocketStyle: table
14
+ EnforcedColonStyle: table
15
+
5
16
  Metrics/MethodLength:
6
- Max: 30
17
+ Max: 50
18
+
7
19
  Metrics/ClassLength:
8
20
  Max: 1500
9
- Metrics/AbcSize:
10
- Max: 34
21
+
22
+ Metrics/ModuleLength:
23
+ Max: 1500
24
+
11
25
  Metrics/BlockLength:
12
- Max: 50
26
+ Max: 40
13
27
  Exclude:
14
- - 'lib/legion/data/migrations/*'
28
+ - 'spec/**/*'
29
+ - 'lib/legion/data/migrations/**/*'
30
+
31
+ Metrics/AbcSize:
32
+ Max: 60
33
+
34
+ Metrics/CyclomaticComplexity:
35
+ Max: 15
36
+
37
+ Metrics/PerceivedComplexity:
38
+ Max: 17
39
+
15
40
  Style/Documentation:
16
41
  Enabled: false
17
- AllCops:
18
- TargetRubyVersion: 2.5
19
- NewCops: enable
20
- SuggestExtensions: false
42
+
43
+ Style/SymbolArray:
44
+ Enabled: true
45
+
21
46
  Style/FrozenStringLiteralComment:
47
+ Enabled: true
48
+ EnforcedStyle: always
49
+
50
+ Naming/FileName:
22
51
  Enabled: false
data/CHANGELOG.md CHANGED
@@ -1,4 +1,83 @@
1
1
  # Legion::Data Changelog
2
2
 
3
+ ## v1.3.7
4
+
5
+ ### Added
6
+ - Migration 020: `webhooks`, `webhook_deliveries`, `webhook_dead_letters` tables
7
+
8
+ ### Fixed
9
+ - Migration 019: guard against duplicate column adds when `record_hash` already exists from migration 017
10
+
11
+ ## v1.3.6
12
+
13
+ ### Added
14
+ - Migration 019: adds `record_hash`, `previous_hash`, `retention_tier` columns to `audit_log`
15
+
16
+ ## v1.3.5
17
+
18
+ ### Added
19
+ - `Legion::Data::EventStore`: append-only governance event store with stream semantics
20
+ - Hash chain integrity verification for tamper detection
21
+ - `EventStore::Projection` base class with `build_from` stream replay
22
+ - `ConsentState` projection: rebuild consent state from event history
23
+ - `GovernanceTimeline` projection: chronological governance event timeline
24
+ - Migration 018: governance_events table with stream/sequence indexing
25
+
26
+ ## v1.3.4
27
+
28
+ ### Added
29
+ - `Legion::Data::Encryption::Cipher`: AES-256-GCM with versioned binary format, random IV, and AAD
30
+ - `Legion::Data::Encryption::KeyProvider`: Vault-backed key derivation with local fallback for dev mode
31
+ - `Legion::Data::Encryption::SequelPlugin`: transparent `encrypted_column` DSL for Sequel models
32
+ - Per-tenant key scope support for cryptographic erasure compliance
33
+
34
+ ## v1.3.3
35
+
36
+ ### Added
37
+ - Migration 017: `audit_log` table with SHA-256 hash chain columns (`record_hash`, `prev_hash`)
38
+ - `Legion::Data::Model::AuditLog` immutable Sequel model with event type/status validation
39
+ - Indexes on `event_type`, `principal_id`, and `created_at` for audit query performance
40
+
41
+ ## v1.3.2
42
+
43
+ ### Added
44
+ - Migration 016: worker health columns (`health_status`, `last_heartbeat_at`, `health_node` on digital_workers; `metrics`, `hosted_worker_ids`, `version` on nodes)
45
+ - `DigitalWorker#health_status` validation against `HEALTH_STATUSES` (`online`, `offline`, `unknown`)
46
+ - `DigitalWorker#online?` and `DigitalWorker#offline?` convenience methods
47
+ - `Node#parsed_metrics` and `Node#parsed_hosted_worker_ids` JSON deserialization helpers
48
+
49
+ ## v1.3.1
50
+
51
+ ### Added
52
+ - Migration 015: RBAC tables (rbac_role_assignments, rbac_runner_grants, rbac_cross_team_grants)
53
+ - `Legion::Data::Model::RbacRoleAssignment` Sequel model with expiry and validation
54
+ - `Legion::Data::Model::RbacRunnerGrant` Sequel model with actions_list helper
55
+ - `Legion::Data::Model::RbacCrossTeamGrant` Sequel model with cross-team validation
56
+
57
+ ## v1.3.0
58
+
59
+ ### Added
60
+ - `Legion::Data::Local` module — parallel local SQLite database for agentic cognitive state persistence
61
+ - TimestampMigrator-based migration registration for per-extension local schemas
62
+ - `Legion::Data::Local.model(:table)` helper for local-bound Sequel models
63
+ - Dev mode fallback: shared DB falls back to SQLite when `dev_mode: true` and network DB unreachable
64
+ - New settings: `data.local.enabled`, `data.local.database`, `data.dev_mode`, `data.dev_fallback`
65
+ - `Legion::Data.local` accessor for the Local module
66
+ - Local connection lifecycle wired into `Legion::Data.setup` / `.shutdown`
67
+ - 13 new specs (62 total)
68
+
69
+ ## v1.2.2
70
+
71
+ ### Added
72
+ - Migration 014: add missing columns to `relationships` table (`delay`, `chain_id`, `debug`, `allow_new_chains`, `conditions`, `transformation`, `active`) required by lex-tasker query helpers
73
+
74
+ ## v1.2.1
75
+
76
+ ### Added
77
+ - Migration 013: `relationships` table with trigger/action foreign keys to functions
78
+ - `Legion::Data::Model::Relationship` Sequel model with trigger/action associations
79
+ - Relationship model registered in model loader (loaded before Task for association resolution)
80
+ - Uncommented `trigger_relationships` and `action_relationships` associations on Function model
81
+
3
82
  ## v1.2.0
4
- Moving from BitBucket to GitHub inside the Optum org. All git history is reset from this point on
83
+ Moving from BitBucket to GitHub. All git history is reset from this point on
data/CLAUDE.md ADDED
@@ -0,0 +1,199 @@
1
+ # legion-data: Persistent Storage for LegionIO
2
+
3
+ **Repository Level 3 Documentation**
4
+ - **Parent**: `/Users/miverso2/rubymine/legion/CLAUDE.md`
5
+
6
+ ## Purpose
7
+
8
+ Manages persistent database storage for the LegionIO framework. Supports SQLite (default), MySQL, and PostgreSQL via Sequel ORM. Provides automatic schema migrations and data models for extensions, functions, runners, nodes, tasks, settings, digital workers, task relationships, and Apollo shared knowledge tables (PostgreSQL only). Also provides a parallel local SQLite database (`Legion::Data::Local`) for agentic cognitive state persistence.
9
+
10
+ **GitHub**: https://github.com/LegionIO/legion-data
11
+ **License**: Apache-2.0
12
+
13
+ ## Supported Databases
14
+
15
+ | Database | Adapter | Gem | Use Case |
16
+ |----------|---------|-----|----------|
17
+ | SQLite | `sqlite` | `sqlite3` (bundled) | Default, dev/test, single-node |
18
+ | MySQL | `mysql2` | `mysql2` (optional) | Production |
19
+ | PostgreSQL | `postgres` | `pg` (optional) | Production |
20
+
21
+ Adapter is set via `Legion::Settings[:data][:adapter]`. All migrations use Sequel DSL for cross-database compatibility.
22
+
23
+ ## Architecture
24
+
25
+ ```
26
+ Legion::Data (singleton module)
27
+ ├── .setup # Connect, migrate, load models, setup cache, setup local
28
+ ├── .connection # Sequel database handle (shared/central)
29
+ ├── .local # Legion::Data::Local accessor
30
+ ├── .shutdown # Close both connections
31
+
32
+ ├── Connection # Sequel database connection management (shared)
33
+ │ ├── .adapter # Reads from settings (sqlite, mysql2, postgres)
34
+ │ ├── .setup # Establish connection (dev_mode fallback to SQLite if network DB unreachable)
35
+ │ ├── .sequel # Raw Sequel::Database accessor
36
+ │ └── .shutdown # Close connection
37
+
38
+ ├── Local # Local SQLite database for agentic cognitive state
39
+ │ ├── .setup # Lazy init — creates legionio_local.db on first access
40
+ │ ├── .connection # Sequel::SQLite::Database handle
41
+ │ ├── .connected? # Whether local DB is active
42
+ │ ├── .db_path # Path to the local SQLite file
43
+ │ ├── .model(:table) # Create Sequel::Model bound to local connection
44
+ │ ├── .register_migrations(name:, path:) # Extensions register their migration dirs
45
+ │ ├── .shutdown # Close local connection
46
+ │ └── .reset! # Clear all state (testing)
47
+
48
+ ├── Migration # Auto-migration system (14 migrations, Sequel DSL)
49
+ │ └── migrations/
50
+ │ ├── 001_add_schema_columns
51
+ │ ├── 002_add_nodes
52
+ │ ├── 003_add_settings
53
+ │ ├── 004_add_extensions
54
+ │ ├── 005_add_runners
55
+ │ ├── 006_add_functions
56
+ │ ├── 007_add_default_extensions
57
+ │ ├── 008_add_tasks
58
+ │ ├── 009_add_digital_workers
59
+ │ ├── 010_add_value_metrics
60
+ │ ├── 011_add_extensions_registry
61
+ │ ├── 012_add_apollo_tables # postgres-only: pgvector, uuid-ossp, 4 apollo tables
62
+ │ ├── 013_add_relationships # relationships table with trigger/action FK to functions
63
+ │ ├── 014_add_relationship_columns # delay, chain_id, debug, conditions, transformation, active, allow_new_chains
64
+ │ ├── 015_add_rbac_tables
65
+ │ ├── 016_add_worker_health
66
+ │ ├── 017_add_audit_log
67
+ │ └── 018_add_governance_events # append-only event store with hash chain
68
+
69
+ ├── Model # Sequel model loader
70
+ │ └── Models/
71
+ │ ├── Extension # Installed LEX extensions
72
+ │ ├── Function # Available functions per extension (with trigger/action relationship associations)
73
+ │ ├── Runner # Runner definitions (extension + function bindings)
74
+ │ ├── Node # Cluster node registry
75
+ │ ├── Task # Task instances (belongs_to Relationship, belongs_to DigitalWorker)
76
+ │ ├── TaskLog # Task execution logs
77
+ │ ├── Setting # Persistent settings store
78
+ │ ├── DigitalWorker # Digital worker registry (lifecycle: bootstrap/active/paused/retired/terminated)
79
+ │ ├── Relationship # Task trigger/action relationships between functions (migration 013/014)
80
+ │ ├── ApolloEntry # Apollo knowledge entries — postgres only (pgvector embedding, confidence lifecycle)
81
+ │ ├── ApolloRelation # Weighted relations between Apollo entries — postgres only
82
+ │ ├── ApolloExpertise # Per-agent domain expertise tracking — postgres only
83
+ │ └── ApolloAccessLog # Apollo entry access audit log — postgres only
84
+ │ Note: value_metrics table (migration 010) is accessed via raw Sequel dataset,
85
+ │ not via a named Sequel::Model subclass.
86
+ │ Note: Apollo models are guarded with `return unless adapter == :postgres` at load time.
87
+
88
+ ├── Settings # Default DB config with per-adapter credential presets
89
+ └── Version
90
+ ```
91
+
92
+ ### Key Design Patterns
93
+
94
+ - **Two-Database Architecture**: Shared (MySQL/PG/SQLite) for control plane data + Local (always SQLite) for agentic cognitive state. Two files, always separate, no cross-database joins.
95
+ - **Adapter-Driven**: `Connection.adapter` reads from settings; SQLite uses `Sequel.sqlite(path)`, others use `Sequel.connect`
96
+ - **Dev Mode Fallback**: When `dev_mode: true` and network DB unreachable, shared connection falls back to SQLite (`legionio.db`) with warning log
97
+ - **Cross-DB Migrations**: Shared migrations use IntegerMigrator (Sequel DSL), local migrations use TimestampMigrator (per-extension registration)
98
+ - **Auto-Migration**: Runs Sequel migrations on startup (`auto_migrate: true` by default)
99
+ - **Sequel ORM**: Shared models are `Sequel::Model` subclasses (inherit global connection). Local models use `Legion::Data::Local.model(:table)` (explicit connection binding).
100
+ - **Optional Caching**: `setup_cache` checks for `Legion::Cache` presence but Sequel model caching is currently disabled (code is commented out, pending implementation)
101
+ - **Cryptographic Erasure**: Deleting `legionio_local.db` is a hard guarantee — no residual data. Used by `lex-privatecore`.
102
+ - **CLI Executable**: Ships with `legionio_migrate` executable in `exe/` for running database migrations standalone
103
+
104
+ ## Default Settings
105
+
106
+ ```json
107
+ {
108
+ "adapter": "sqlite",
109
+ "connected": false,
110
+ "dev_mode": false,
111
+ "dev_fallback": true,
112
+ "connect_on_start": true,
113
+ "connection": {
114
+ "log": false,
115
+ "log_connection_info": false,
116
+ "log_warn_duration": 1,
117
+ "sql_log_level": "debug",
118
+ "max_connections": 10,
119
+ "preconnect": false
120
+ },
121
+ "creds": {
122
+ "database": "legionio.db"
123
+ },
124
+ "migrations": {
125
+ "continue_on_fail": false,
126
+ "auto_migrate": true,
127
+ "ran": false,
128
+ "version": null
129
+ },
130
+ "models": {
131
+ "continue_on_load_fail": false,
132
+ "autoload": true
133
+ },
134
+ "local": {
135
+ "enabled": true,
136
+ "database": "legionio_local.db",
137
+ "migrations": {
138
+ "auto_migrate": true
139
+ }
140
+ },
141
+ "cache": {
142
+ "connected": false,
143
+ "auto_enable": false,
144
+ "ttl": 60
145
+ }
146
+ }
147
+ ```
148
+
149
+ Per-adapter credential defaults are defined in `Settings::CREDS`:
150
+ - **sqlite**: `{ database: "legionio.db" }`
151
+ - **mysql2**: `{ username: "legion", password: "legion", database: "legionio", host: "127.0.0.1", port: 3306 }`
152
+ - **postgres**: `{ user: "legion", password: "legion", database: "legionio", host: "127.0.0.1", port: 5432 }`
153
+
154
+ ## Dependencies
155
+
156
+ | Gem | Purpose |
157
+ |-----|---------|
158
+ | `sequel` (>= 5.70) | ORM and migration framework |
159
+ | `sqlite3` (>= 2.0) | SQLite adapter (default, bundled) |
160
+ | `mysql2` (>= 0.5.5) | MySQL adapter (optional) |
161
+ | `pg` (>= 1.5) | PostgreSQL adapter (optional) |
162
+ | `legion-logging` | Logging |
163
+ | `legion-settings` | Configuration |
164
+
165
+ ## File Map
166
+
167
+ | Path | Purpose |
168
+ |------|---------|
169
+ | `lib/legion/data.rb` | Module entry, setup/shutdown lifecycle |
170
+ | `lib/legion/data/connection.rb` | Sequel database connection (adapter selection) |
171
+ | `lib/legion/data/migration.rb` | Migration runner |
172
+ | `lib/legion/data/migrations/` | 14 numbered migration files (Sequel DSL) |
173
+ | `lib/legion/data/model.rb` | Model autoloader |
174
+ | `lib/legion/data/local.rb` | Local SQLite module for agentic cognitive state |
175
+ | `lib/legion/data/models/` | Sequel models (Extension, Function, Runner, Node, Task, TaskLog, Setting, DigitalWorker, Relationship, ApolloEntry, ApolloRelation, ApolloExpertise, ApolloAccessLog) |
176
+ | `lib/legion/data/encryption/cipher.rb` | AES-256-GCM encrypt/decrypt with versioned binary format and AAD |
177
+ | `lib/legion/data/encryption/key_provider.rb` | Vault-backed key derivation with per-tenant scope and local fallback |
178
+ | `lib/legion/data/encryption/sequel_plugin.rb` | Transparent `encrypted_column` DSL for Sequel models |
179
+ | `lib/legion/data/event_store.rb` | Append-only governance event store with hash chain integrity |
180
+ | `lib/legion/data/event_store/projection.rb` | Projection base class, ConsentState, GovernanceTimeline |
181
+ | `lib/legion/data/settings.rb` | Default configuration with per-adapter credential presets |
182
+ | `lib/legion/data/version.rb` | VERSION constant |
183
+ | `exe/legionio_migrate` | CLI executable for running database migrations standalone |
184
+
185
+ ## Role in LegionIO
186
+
187
+ Optional persistent storage initialized during `Legion::Service` startup (after transport). Provides:
188
+ 1. Extension and function registry (which LEXs are installed, what functions they expose)
189
+ 2. Task scheduling and logging
190
+ 3. Node cluster membership tracking
191
+ 4. Persistent settings storage
192
+ 5. Digital worker registry (AI-as-labor platform)
193
+ 6. Task relationship graph (trigger/action chains)
194
+ 7. Apollo shared knowledge store (PostgreSQL + pgvector only, used by lex-apollo)
195
+ 8. Local SQLite for agentic cognitive state (memory traces, trust scores, dream journals) — always on-node, independent of shared DB
196
+
197
+ ---
198
+
199
+ **Maintained By**: Matthew Iverson (@Esity)
data/Gemfile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source 'https://rubygems.org'
2
4
 
3
5
  gemspec
@@ -8,3 +10,5 @@ group :test do
8
10
  gem 'rubocop'
9
11
  gem 'simplecov'
10
12
  end
13
+ gem 'mysql2', '>= 0.5.5'
14
+ gem 'pg', '>= 1.5'
data/LICENSE CHANGED
@@ -186,7 +186,7 @@
186
186
  same "printed page" as the copyright notice for easier
187
187
  identification within third-party archives.
188
188
 
189
- Copyright 2021 Optum
189
+ Copyright 2021 Esity
190
190
 
191
191
  Licensed under the Apache License, Version 2.0 (the "License");
192
192
  you may not use this file except in compliance with the License.
data/README.md CHANGED
@@ -1,74 +1,181 @@
1
- Legion::Data
2
- =====
1
+ # legion-data
3
2
 
4
- Legion::Data is a gem for the LegionIO framework to use persistent storage. Currently only MySQL is supported
3
+ Persistent database storage for the [LegionIO](https://github.com/LegionIO/LegionIO) framework. Provides database connectivity via Sequel ORM, automatic schema migrations, and data models for extensions, functions, runners, nodes, tasks, settings, digital workers, task relationships, and Apollo shared knowledge tables.
5
4
 
6
- Supported Ruby versions and implementations
7
- ------------------------------------------------
5
+ Version: 1.3.0
8
6
 
9
- Legion::Json should work identically on:
7
+ ## Supported Databases
10
8
 
11
- * Ruby 2.5+
9
+ | Database | Adapter | Gem | Default |
10
+ |----------|---------|-----|---------|
11
+ | SQLite | `sqlite` | `sqlite3` (included) | Yes |
12
+ | MySQL | `mysql2` | `mysql2` | No |
13
+ | PostgreSQL | `postgres` | `pg` | No |
12
14
 
15
+ SQLite is the default adapter and requires no external database server. For MySQL or PostgreSQL, install the corresponding gem and set the adapter in your configuration.
13
16
 
14
- Installation and Usage
15
- ------------------------
16
-
17
- You can verify your installation using this piece of code:
17
+ ## Installation
18
18
 
19
19
  ```bash
20
20
  gem install legion-data
21
21
  ```
22
22
 
23
+ Or add to your Gemfile:
24
+
25
+ ```ruby
26
+ gem 'legion-data'
27
+
28
+ # Add one of these for production databases:
29
+ # gem 'mysql2', '>= 0.5.5'
30
+ # gem 'pg', '>= 1.5'
31
+ ```
32
+
33
+ ## Data Models
34
+
35
+ | Model | Table | Description |
36
+ |-------|-------|-------------|
37
+ | `Extension` | `extensions` | Installed LEX extensions |
38
+ | `Function` | `functions` | Available functions per extension |
39
+ | `Runner` | `runners` | Runner definitions (extension + function bindings) |
40
+ | `Node` | `nodes` | Cluster node registry |
41
+ | `Task` | `tasks` | Task instances |
42
+ | `TaskLog` | `task_logs` | Task execution logs |
43
+ | `Setting` | `settings` | Persistent settings store |
44
+ | `DigitalWorker` | `digital_workers` | Digital worker registry (AI-as-labor platform) |
45
+ | `Relationship` | `relationships` | Task trigger/action relationships between functions |
46
+ | `ApolloEntry` | `apollo_entries` | Apollo shared knowledge entries (PostgreSQL only) |
47
+ | `ApolloRelation` | `apollo_relations` | Relations between Apollo knowledge entries (PostgreSQL only) |
48
+ | `ApolloExpertise` | `apollo_expertise` | Per-agent domain expertise tracking (PostgreSQL only) |
49
+ | `ApolloAccessLog` | `apollo_access_log` | Apollo entry access audit log (PostgreSQL only) |
50
+
51
+ Apollo models require PostgreSQL with the `pgvector` extension. They are skipped silently on SQLite and MySQL.
52
+
53
+ ## Usage
54
+
23
55
  ```ruby
24
56
  require 'legion/data'
25
57
 
58
+ # Standard setup (shared DB + local SQLite)
26
59
  Legion::Data.setup
27
- Legion::Data.connected? # => true
28
- Legion::Data::Model::Extension.all # Sequel::Dataset
60
+ Legion::Data.connection # => Sequel::Database (shared)
61
+ Legion::Data.local.connection # => Sequel::SQLite::Database (local cognitive state)
62
+ Legion::Data::Model::Extension.all # => Sequel::Dataset
63
+ ```
64
+
65
+ ### Local Database
66
+
67
+ v1.3.0 introduces `Legion::Data::Local`, a parallel SQLite database always stored locally on the node. It is used for agentic cognitive state persistence (memory traces, trust scores, dream journals, etc.) and is independent of the shared database.
68
+
69
+ ```ruby
70
+ # Local DB is set up automatically during Legion::Data.setup
71
+ # Extensions register their own migration directories
72
+ Legion::Data::Local.register_migrations(name: :memory, path: '/path/to/migrations')
73
+
74
+ # Create a model bound to the local connection
75
+ MyModel = Legion::Data::Local.model(:my_table)
76
+
77
+ # Check status
78
+ Legion::Data::Local.connected? # => true
79
+ Legion::Data::Local.db_path # => "legionio_local.db"
80
+ ```
81
+
82
+ The local database file (`legionio_local.db` by default) can be deleted for cryptographic erasure — no residual data. This is used by `lex-privatecore`.
83
+
84
+ ## Configuration
85
+
86
+ ### SQLite (default)
87
+
88
+ ```json
89
+ {
90
+ "data": {
91
+ "adapter": "sqlite",
92
+ "creds": {
93
+ "database": "legionio.db"
94
+ }
95
+ }
96
+ }
97
+ ```
98
+
99
+ ### MySQL
100
+
101
+ ```json
102
+ {
103
+ "data": {
104
+ "adapter": "mysql2",
105
+ "creds": {
106
+ "username": "legion",
107
+ "password": "legion",
108
+ "database": "legionio",
109
+ "host": "127.0.0.1",
110
+ "port": 3306
111
+ }
112
+ }
113
+ }
114
+ ```
115
+
116
+ ### PostgreSQL
117
+
118
+ ```json
119
+ {
120
+ "data": {
121
+ "adapter": "postgres",
122
+ "creds": {
123
+ "user": "legion",
124
+ "password": "legion",
125
+ "database": "legionio",
126
+ "host": "127.0.0.1",
127
+ "port": 5432
128
+ }
129
+ }
130
+ }
131
+ ```
132
+
133
+ PostgreSQL with `pgvector` is required for Apollo models. Install the extension in your database before running migrations:
134
+
135
+ ```sql
136
+ CREATE EXTENSION IF NOT EXISTS vector;
137
+ CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
29
138
  ```
30
139
 
31
- Settings
32
- ----------
140
+ ### Local Database
33
141
 
34
142
  ```json
35
143
  {
36
- "connected": false,
37
- "cache": {
38
- "connected": false,
39
- "auto_enable": null,
40
- "ttl": 60
41
- },
42
- "connection": {
43
- "log": false,
44
- "log_connection_info": false,
45
- "log_warn_duration": 1,
46
- "log_warn_duration": "debug",
47
- "max_connections": 10,
48
- "preconnect": false
49
- },
50
- "creds": {
51
- "username": "legion",
52
- "password": "legion",
53
- "database": "legionio",
54
- "host": "127.0.0.1",
55
- "port": 3306
56
- },
57
- "migrations": {
58
- "continue_on_fail": false,
59
- "auto_migrate": true,
60
- "ran": false,
61
- "version": null
62
- },
63
- "models": {
64
- "continue_on_load_fail": false,
65
- "autoload": true
66
- },
67
- "connect_on_start": true
144
+ "data": {
145
+ "local": {
146
+ "enabled": true,
147
+ "database": "legionio_local.db",
148
+ "migrations": {
149
+ "auto_migrate": true
150
+ }
151
+ }
152
+ }
68
153
  }
69
154
  ```
70
155
 
71
- Authors
72
- ----------
156
+ Set `enabled: false` to disable local SQLite entirely.
157
+
158
+ ### Dev Mode Fallback
159
+
160
+ When `dev_mode: true` and a network database (MySQL/PostgreSQL) is unreachable, the shared connection falls back to SQLite automatically instead of raising.
161
+
162
+ ```json
163
+ {
164
+ "data": {
165
+ "dev_mode": true,
166
+ "dev_fallback": true
167
+ }
168
+ }
169
+ ```
170
+
171
+ ### HashiCorp Vault Integration
172
+
173
+ When Vault is connected and a `database/creds/legion` secret path exists, credentials are fetched dynamically from Vault at connection time, overriding any static `creds` configuration.
174
+
175
+ ## Requirements
176
+
177
+ - Ruby >= 3.4
178
+
179
+ ## License
73
180
 
74
- * [Matthew Iverson](https://github.com/Esity) - current maintainer
181
+ Apache-2.0
File without changes