legion-data 1.2.0 → 1.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +16 -0
- data/.gitignore +5 -0
- data/.rubocop.yml +41 -12
- data/CHANGELOG.md +80 -1
- data/CLAUDE.md +199 -0
- data/Gemfile +4 -0
- data/LICENSE +1 -1
- data/README.md +157 -50
- data/exe/legionio_migrate +0 -0
- data/legion-data.gemspec +13 -13
- data/lib/legion/data/connection.rb +39 -25
- data/lib/legion/data/encryption/cipher.rb +49 -0
- data/lib/legion/data/encryption/key_provider.rb +45 -0
- data/lib/legion/data/encryption/sequel_plugin.rb +54 -0
- data/lib/legion/data/event_store/projection.rb +56 -0
- data/lib/legion/data/event_store.rb +112 -0
- data/lib/legion/data/local.rb +77 -0
- data/lib/legion/data/migration.rb +5 -3
- data/lib/legion/data/migrations/001_add_schema_columns.rb +9 -3
- data/lib/legion/data/migrations/002_add_nodes.rb +10 -12
- data/lib/legion/data/migrations/003_add_settings.rb +10 -10
- data/lib/legion/data/migrations/004_add_extensions.rb +15 -17
- data/lib/legion/data/migrations/005_add_runners.rb +13 -13
- data/lib/legion/data/migrations/006_add_functions.rb +13 -15
- data/lib/legion/data/migrations/007_add_default_extensions.rb +2 -0
- data/lib/legion/data/migrations/008_add_tasks.rb +15 -21
- data/lib/legion/data/migrations/009_add_digital_workers.rb +45 -0
- data/lib/legion/data/migrations/010_add_value_metrics.rb +19 -0
- data/lib/legion/data/migrations/011_add_extensions_registry.rb +30 -0
- data/lib/legion/data/migrations/012_add_apollo_tables.rb +66 -0
- data/lib/legion/data/migrations/013_add_relationships.rb +21 -0
- data/lib/legion/data/migrations/014_add_relationship_columns.rb +27 -0
- data/lib/legion/data/migrations/015_add_rbac_tables.rb +49 -0
- data/lib/legion/data/migrations/016_add_worker_health.rb +33 -0
- data/lib/legion/data/migrations/017_add_audit_log.rb +30 -0
- data/lib/legion/data/migrations/018_add_governance_events.rb +21 -0
- data/lib/legion/data/migrations/019_add_audit_hash_chain.rb +29 -0
- data/lib/legion/data/migrations/020_add_webhooks.rb +37 -0
- data/lib/legion/data/model.rb +4 -1
- data/lib/legion/data/models/apollo_access_log.rb +13 -0
- data/lib/legion/data/models/apollo_entry.rb +18 -0
- data/lib/legion/data/models/apollo_expertise.rb +12 -0
- data/lib/legion/data/models/apollo_relation.rb +14 -0
- data/lib/legion/data/models/audit_log.rb +34 -0
- data/lib/legion/data/models/digital_worker.rb +44 -0
- data/lib/legion/data/models/function.rb +2 -2
- data/lib/legion/data/models/node.rb +16 -0
- data/lib/legion/data/models/rbac_cross_team_grant.rb +33 -0
- data/lib/legion/data/models/rbac_role_assignment.rb +29 -0
- data/lib/legion/data/models/rbac_runner_grant.rb +21 -0
- data/lib/legion/data/models/relationship.rb +13 -0
- data/lib/legion/data/settings.rb +54 -25
- data/lib/legion/data/version.rb +3 -1
- data/lib/legion/data.rb +20 -1
- metadata +51 -32
- data/.github/workflows/rubocop-analysis.yml +0 -28
- data/.github/workflows/sourcehawk-scan.yml +0 -20
- data/CODE_OF_CONDUCT.md +0 -75
- data/CONTRIBUTING.md +0 -55
- data/INDIVIDUAL_CONTRIBUTOR_LICENSE.md +0 -30
- data/NOTICE.txt +0 -9
- data/SECURITY.md +0 -9
- data/attribution.txt +0 -1
- data/sourcehawk.yml +0 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d10f18bacc47360ad1829628bffc116ecbeea74b0b49359f1cee7fba0a7e9d44
|
|
4
|
+
data.tar.gz: e13f4acfa8992e3a00129322fef794b80877f723e0c59210016f64fee57c5cae
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0e61649208294598ecdf19ffca469cba3db7b77d31e0e0a065e4c82e47949e4ea87aa291c5fae18970318e0c27ac1dfcd7eb53143bd7b5acc279eb84c0f2b231
|
|
7
|
+
data.tar.gz: 7692efc8d123ad178c3f3a66531723518e0fac2cb5999d01833bfa472ac8eb5bee06289dd193886fe66c369f3d49ebff689e68267eff4ab030ce746377a6c9a3
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches: [main]
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
ci:
|
|
9
|
+
uses: LegionIO/.github/.github/workflows/ci.yml@main
|
|
10
|
+
|
|
11
|
+
release:
|
|
12
|
+
needs: ci
|
|
13
|
+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
|
14
|
+
uses: LegionIO/.github/.github/workflows/release.yml@main
|
|
15
|
+
secrets:
|
|
16
|
+
rubygems-api-key: ${{ secrets.RUBYGEMS_API_KEY }}
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -1,22 +1,51 @@
|
|
|
1
|
+
AllCops:
|
|
2
|
+
TargetRubyVersion: 3.4
|
|
3
|
+
NewCops: enable
|
|
4
|
+
SuggestExtensions: false
|
|
5
|
+
|
|
1
6
|
Layout/LineLength:
|
|
2
|
-
Max:
|
|
3
|
-
|
|
4
|
-
|
|
7
|
+
Max: 160
|
|
8
|
+
|
|
9
|
+
Layout/SpaceAroundEqualsInParameterDefault:
|
|
10
|
+
EnforcedStyle: space
|
|
11
|
+
|
|
12
|
+
Layout/HashAlignment:
|
|
13
|
+
EnforcedHashRocketStyle: table
|
|
14
|
+
EnforcedColonStyle: table
|
|
15
|
+
|
|
5
16
|
Metrics/MethodLength:
|
|
6
|
-
Max:
|
|
17
|
+
Max: 50
|
|
18
|
+
|
|
7
19
|
Metrics/ClassLength:
|
|
8
20
|
Max: 1500
|
|
9
|
-
|
|
10
|
-
|
|
21
|
+
|
|
22
|
+
Metrics/ModuleLength:
|
|
23
|
+
Max: 1500
|
|
24
|
+
|
|
11
25
|
Metrics/BlockLength:
|
|
12
|
-
Max:
|
|
26
|
+
Max: 40
|
|
13
27
|
Exclude:
|
|
14
|
-
- '
|
|
28
|
+
- 'spec/**/*'
|
|
29
|
+
- 'lib/legion/data/migrations/**/*'
|
|
30
|
+
|
|
31
|
+
Metrics/AbcSize:
|
|
32
|
+
Max: 60
|
|
33
|
+
|
|
34
|
+
Metrics/CyclomaticComplexity:
|
|
35
|
+
Max: 15
|
|
36
|
+
|
|
37
|
+
Metrics/PerceivedComplexity:
|
|
38
|
+
Max: 17
|
|
39
|
+
|
|
15
40
|
Style/Documentation:
|
|
16
41
|
Enabled: false
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
42
|
+
|
|
43
|
+
Style/SymbolArray:
|
|
44
|
+
Enabled: true
|
|
45
|
+
|
|
21
46
|
Style/FrozenStringLiteralComment:
|
|
47
|
+
Enabled: true
|
|
48
|
+
EnforcedStyle: always
|
|
49
|
+
|
|
50
|
+
Naming/FileName:
|
|
22
51
|
Enabled: false
|
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,83 @@
|
|
|
1
1
|
# Legion::Data Changelog
|
|
2
2
|
|
|
3
|
+
## v1.3.7
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- Migration 020: `webhooks`, `webhook_deliveries`, `webhook_dead_letters` tables
|
|
7
|
+
|
|
8
|
+
### Fixed
|
|
9
|
+
- Migration 019: guard against duplicate column adds when `record_hash` already exists from migration 017
|
|
10
|
+
|
|
11
|
+
## v1.3.6
|
|
12
|
+
|
|
13
|
+
### Added
|
|
14
|
+
- Migration 019: adds `record_hash`, `previous_hash`, `retention_tier` columns to `audit_log`
|
|
15
|
+
|
|
16
|
+
## v1.3.5
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
- `Legion::Data::EventStore`: append-only governance event store with stream semantics
|
|
20
|
+
- Hash chain integrity verification for tamper detection
|
|
21
|
+
- `EventStore::Projection` base class with `build_from` stream replay
|
|
22
|
+
- `ConsentState` projection: rebuild consent state from event history
|
|
23
|
+
- `GovernanceTimeline` projection: chronological governance event timeline
|
|
24
|
+
- Migration 018: governance_events table with stream/sequence indexing
|
|
25
|
+
|
|
26
|
+
## v1.3.4
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
- `Legion::Data::Encryption::Cipher`: AES-256-GCM with versioned binary format, random IV, and AAD
|
|
30
|
+
- `Legion::Data::Encryption::KeyProvider`: Vault-backed key derivation with local fallback for dev mode
|
|
31
|
+
- `Legion::Data::Encryption::SequelPlugin`: transparent `encrypted_column` DSL for Sequel models
|
|
32
|
+
- Per-tenant key scope support for cryptographic erasure compliance
|
|
33
|
+
|
|
34
|
+
## v1.3.3
|
|
35
|
+
|
|
36
|
+
### Added
|
|
37
|
+
- Migration 017: `audit_log` table with SHA-256 hash chain columns (`record_hash`, `prev_hash`)
|
|
38
|
+
- `Legion::Data::Model::AuditLog` immutable Sequel model with event type/status validation
|
|
39
|
+
- Indexes on `event_type`, `principal_id`, and `created_at` for audit query performance
|
|
40
|
+
|
|
41
|
+
## v1.3.2
|
|
42
|
+
|
|
43
|
+
### Added
|
|
44
|
+
- Migration 016: worker health columns (`health_status`, `last_heartbeat_at`, `health_node` on digital_workers; `metrics`, `hosted_worker_ids`, `version` on nodes)
|
|
45
|
+
- `DigitalWorker#health_status` validation against `HEALTH_STATUSES` (`online`, `offline`, `unknown`)
|
|
46
|
+
- `DigitalWorker#online?` and `DigitalWorker#offline?` convenience methods
|
|
47
|
+
- `Node#parsed_metrics` and `Node#parsed_hosted_worker_ids` JSON deserialization helpers
|
|
48
|
+
|
|
49
|
+
## v1.3.1
|
|
50
|
+
|
|
51
|
+
### Added
|
|
52
|
+
- Migration 015: RBAC tables (rbac_role_assignments, rbac_runner_grants, rbac_cross_team_grants)
|
|
53
|
+
- `Legion::Data::Model::RbacRoleAssignment` Sequel model with expiry and validation
|
|
54
|
+
- `Legion::Data::Model::RbacRunnerGrant` Sequel model with actions_list helper
|
|
55
|
+
- `Legion::Data::Model::RbacCrossTeamGrant` Sequel model with cross-team validation
|
|
56
|
+
|
|
57
|
+
## v1.3.0
|
|
58
|
+
|
|
59
|
+
### Added
|
|
60
|
+
- `Legion::Data::Local` module — parallel local SQLite database for agentic cognitive state persistence
|
|
61
|
+
- TimestampMigrator-based migration registration for per-extension local schemas
|
|
62
|
+
- `Legion::Data::Local.model(:table)` helper for local-bound Sequel models
|
|
63
|
+
- Dev mode fallback: shared DB falls back to SQLite when `dev_mode: true` and network DB unreachable
|
|
64
|
+
- New settings: `data.local.enabled`, `data.local.database`, `data.dev_mode`, `data.dev_fallback`
|
|
65
|
+
- `Legion::Data.local` accessor for the Local module
|
|
66
|
+
- Local connection lifecycle wired into `Legion::Data.setup` / `.shutdown`
|
|
67
|
+
- 13 new specs (62 total)
|
|
68
|
+
|
|
69
|
+
## v1.2.2
|
|
70
|
+
|
|
71
|
+
### Added
|
|
72
|
+
- Migration 014: add missing columns to `relationships` table (`delay`, `chain_id`, `debug`, `allow_new_chains`, `conditions`, `transformation`, `active`) required by lex-tasker query helpers
|
|
73
|
+
|
|
74
|
+
## v1.2.1
|
|
75
|
+
|
|
76
|
+
### Added
|
|
77
|
+
- Migration 013: `relationships` table with trigger/action foreign keys to functions
|
|
78
|
+
- `Legion::Data::Model::Relationship` Sequel model with trigger/action associations
|
|
79
|
+
- Relationship model registered in model loader (loaded before Task for association resolution)
|
|
80
|
+
- Uncommented `trigger_relationships` and `action_relationships` associations on Function model
|
|
81
|
+
|
|
3
82
|
## v1.2.0
|
|
4
|
-
Moving from BitBucket to GitHub
|
|
83
|
+
Moving from BitBucket to GitHub. All git history is reset from this point on
|
data/CLAUDE.md
ADDED
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
# legion-data: Persistent Storage for LegionIO
|
|
2
|
+
|
|
3
|
+
**Repository Level 3 Documentation**
|
|
4
|
+
- **Parent**: `/Users/miverso2/rubymine/legion/CLAUDE.md`
|
|
5
|
+
|
|
6
|
+
## Purpose
|
|
7
|
+
|
|
8
|
+
Manages persistent database storage for the LegionIO framework. Supports SQLite (default), MySQL, and PostgreSQL via Sequel ORM. Provides automatic schema migrations and data models for extensions, functions, runners, nodes, tasks, settings, digital workers, task relationships, and Apollo shared knowledge tables (PostgreSQL only). Also provides a parallel local SQLite database (`Legion::Data::Local`) for agentic cognitive state persistence.
|
|
9
|
+
|
|
10
|
+
**GitHub**: https://github.com/LegionIO/legion-data
|
|
11
|
+
**License**: Apache-2.0
|
|
12
|
+
|
|
13
|
+
## Supported Databases
|
|
14
|
+
|
|
15
|
+
| Database | Adapter | Gem | Use Case |
|
|
16
|
+
|----------|---------|-----|----------|
|
|
17
|
+
| SQLite | `sqlite` | `sqlite3` (bundled) | Default, dev/test, single-node |
|
|
18
|
+
| MySQL | `mysql2` | `mysql2` (optional) | Production |
|
|
19
|
+
| PostgreSQL | `postgres` | `pg` (optional) | Production |
|
|
20
|
+
|
|
21
|
+
Adapter is set via `Legion::Settings[:data][:adapter]`. All migrations use Sequel DSL for cross-database compatibility.
|
|
22
|
+
|
|
23
|
+
## Architecture
|
|
24
|
+
|
|
25
|
+
```
|
|
26
|
+
Legion::Data (singleton module)
|
|
27
|
+
├── .setup # Connect, migrate, load models, setup cache, setup local
|
|
28
|
+
├── .connection # Sequel database handle (shared/central)
|
|
29
|
+
├── .local # Legion::Data::Local accessor
|
|
30
|
+
├── .shutdown # Close both connections
|
|
31
|
+
│
|
|
32
|
+
├── Connection # Sequel database connection management (shared)
|
|
33
|
+
│ ├── .adapter # Reads from settings (sqlite, mysql2, postgres)
|
|
34
|
+
│ ├── .setup # Establish connection (dev_mode fallback to SQLite if network DB unreachable)
|
|
35
|
+
│ ├── .sequel # Raw Sequel::Database accessor
|
|
36
|
+
│ └── .shutdown # Close connection
|
|
37
|
+
│
|
|
38
|
+
├── Local # Local SQLite database for agentic cognitive state
|
|
39
|
+
│ ├── .setup # Lazy init — creates legionio_local.db on first access
|
|
40
|
+
│ ├── .connection # Sequel::SQLite::Database handle
|
|
41
|
+
│ ├── .connected? # Whether local DB is active
|
|
42
|
+
│ ├── .db_path # Path to the local SQLite file
|
|
43
|
+
│ ├── .model(:table) # Create Sequel::Model bound to local connection
|
|
44
|
+
│ ├── .register_migrations(name:, path:) # Extensions register their migration dirs
|
|
45
|
+
│ ├── .shutdown # Close local connection
|
|
46
|
+
│ └── .reset! # Clear all state (testing)
|
|
47
|
+
│
|
|
48
|
+
├── Migration # Auto-migration system (14 migrations, Sequel DSL)
|
|
49
|
+
│ └── migrations/
|
|
50
|
+
│ ├── 001_add_schema_columns
|
|
51
|
+
│ ├── 002_add_nodes
|
|
52
|
+
│ ├── 003_add_settings
|
|
53
|
+
│ ├── 004_add_extensions
|
|
54
|
+
│ ├── 005_add_runners
|
|
55
|
+
│ ├── 006_add_functions
|
|
56
|
+
│ ├── 007_add_default_extensions
|
|
57
|
+
│ ├── 008_add_tasks
|
|
58
|
+
│ ├── 009_add_digital_workers
|
|
59
|
+
│ ├── 010_add_value_metrics
|
|
60
|
+
│ ├── 011_add_extensions_registry
|
|
61
|
+
│ ├── 012_add_apollo_tables # postgres-only: pgvector, uuid-ossp, 4 apollo tables
|
|
62
|
+
│ ├── 013_add_relationships # relationships table with trigger/action FK to functions
|
|
63
|
+
│ ├── 014_add_relationship_columns # delay, chain_id, debug, conditions, transformation, active, allow_new_chains
|
|
64
|
+
│ ├── 015_add_rbac_tables
|
|
65
|
+
│ ├── 016_add_worker_health
|
|
66
|
+
│ ├── 017_add_audit_log
|
|
67
|
+
│ └── 018_add_governance_events # append-only event store with hash chain
|
|
68
|
+
│
|
|
69
|
+
├── Model # Sequel model loader
|
|
70
|
+
│ └── Models/
|
|
71
|
+
│ ├── Extension # Installed LEX extensions
|
|
72
|
+
│ ├── Function # Available functions per extension (with trigger/action relationship associations)
|
|
73
|
+
│ ├── Runner # Runner definitions (extension + function bindings)
|
|
74
|
+
│ ├── Node # Cluster node registry
|
|
75
|
+
│ ├── Task # Task instances (belongs_to Relationship, belongs_to DigitalWorker)
|
|
76
|
+
│ ├── TaskLog # Task execution logs
|
|
77
|
+
│ ├── Setting # Persistent settings store
|
|
78
|
+
│ ├── DigitalWorker # Digital worker registry (lifecycle: bootstrap/active/paused/retired/terminated)
|
|
79
|
+
│ ├── Relationship # Task trigger/action relationships between functions (migration 013/014)
|
|
80
|
+
│ ├── ApolloEntry # Apollo knowledge entries — postgres only (pgvector embedding, confidence lifecycle)
|
|
81
|
+
│ ├── ApolloRelation # Weighted relations between Apollo entries — postgres only
|
|
82
|
+
│ ├── ApolloExpertise # Per-agent domain expertise tracking — postgres only
|
|
83
|
+
│ └── ApolloAccessLog # Apollo entry access audit log — postgres only
|
|
84
|
+
│ Note: value_metrics table (migration 010) is accessed via raw Sequel dataset,
|
|
85
|
+
│ not via a named Sequel::Model subclass.
|
|
86
|
+
│ Note: Apollo models are guarded with `return unless adapter == :postgres` at load time.
|
|
87
|
+
│
|
|
88
|
+
├── Settings # Default DB config with per-adapter credential presets
|
|
89
|
+
└── Version
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
### Key Design Patterns
|
|
93
|
+
|
|
94
|
+
- **Two-Database Architecture**: Shared (MySQL/PG/SQLite) for control plane data + Local (always SQLite) for agentic cognitive state. Two files, always separate, no cross-database joins.
|
|
95
|
+
- **Adapter-Driven**: `Connection.adapter` reads from settings; SQLite uses `Sequel.sqlite(path)`, others use `Sequel.connect`
|
|
96
|
+
- **Dev Mode Fallback**: When `dev_mode: true` and network DB unreachable, shared connection falls back to SQLite (`legionio.db`) with warning log
|
|
97
|
+
- **Cross-DB Migrations**: Shared migrations use IntegerMigrator (Sequel DSL), local migrations use TimestampMigrator (per-extension registration)
|
|
98
|
+
- **Auto-Migration**: Runs Sequel migrations on startup (`auto_migrate: true` by default)
|
|
99
|
+
- **Sequel ORM**: Shared models are `Sequel::Model` subclasses (inherit global connection). Local models use `Legion::Data::Local.model(:table)` (explicit connection binding).
|
|
100
|
+
- **Optional Caching**: `setup_cache` checks for `Legion::Cache` presence but Sequel model caching is currently disabled (code is commented out, pending implementation)
|
|
101
|
+
- **Cryptographic Erasure**: Deleting `legionio_local.db` is a hard guarantee — no residual data. Used by `lex-privatecore`.
|
|
102
|
+
- **CLI Executable**: Ships with `legionio_migrate` executable in `exe/` for running database migrations standalone
|
|
103
|
+
|
|
104
|
+
## Default Settings
|
|
105
|
+
|
|
106
|
+
```json
|
|
107
|
+
{
|
|
108
|
+
"adapter": "sqlite",
|
|
109
|
+
"connected": false,
|
|
110
|
+
"dev_mode": false,
|
|
111
|
+
"dev_fallback": true,
|
|
112
|
+
"connect_on_start": true,
|
|
113
|
+
"connection": {
|
|
114
|
+
"log": false,
|
|
115
|
+
"log_connection_info": false,
|
|
116
|
+
"log_warn_duration": 1,
|
|
117
|
+
"sql_log_level": "debug",
|
|
118
|
+
"max_connections": 10,
|
|
119
|
+
"preconnect": false
|
|
120
|
+
},
|
|
121
|
+
"creds": {
|
|
122
|
+
"database": "legionio.db"
|
|
123
|
+
},
|
|
124
|
+
"migrations": {
|
|
125
|
+
"continue_on_fail": false,
|
|
126
|
+
"auto_migrate": true,
|
|
127
|
+
"ran": false,
|
|
128
|
+
"version": null
|
|
129
|
+
},
|
|
130
|
+
"models": {
|
|
131
|
+
"continue_on_load_fail": false,
|
|
132
|
+
"autoload": true
|
|
133
|
+
},
|
|
134
|
+
"local": {
|
|
135
|
+
"enabled": true,
|
|
136
|
+
"database": "legionio_local.db",
|
|
137
|
+
"migrations": {
|
|
138
|
+
"auto_migrate": true
|
|
139
|
+
}
|
|
140
|
+
},
|
|
141
|
+
"cache": {
|
|
142
|
+
"connected": false,
|
|
143
|
+
"auto_enable": false,
|
|
144
|
+
"ttl": 60
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Per-adapter credential defaults are defined in `Settings::CREDS`:
|
|
150
|
+
- **sqlite**: `{ database: "legionio.db" }`
|
|
151
|
+
- **mysql2**: `{ username: "legion", password: "legion", database: "legionio", host: "127.0.0.1", port: 3306 }`
|
|
152
|
+
- **postgres**: `{ user: "legion", password: "legion", database: "legionio", host: "127.0.0.1", port: 5432 }`
|
|
153
|
+
|
|
154
|
+
## Dependencies
|
|
155
|
+
|
|
156
|
+
| Gem | Purpose |
|
|
157
|
+
|-----|---------|
|
|
158
|
+
| `sequel` (>= 5.70) | ORM and migration framework |
|
|
159
|
+
| `sqlite3` (>= 2.0) | SQLite adapter (default, bundled) |
|
|
160
|
+
| `mysql2` (>= 0.5.5) | MySQL adapter (optional) |
|
|
161
|
+
| `pg` (>= 1.5) | PostgreSQL adapter (optional) |
|
|
162
|
+
| `legion-logging` | Logging |
|
|
163
|
+
| `legion-settings` | Configuration |
|
|
164
|
+
|
|
165
|
+
## File Map
|
|
166
|
+
|
|
167
|
+
| Path | Purpose |
|
|
168
|
+
|------|---------|
|
|
169
|
+
| `lib/legion/data.rb` | Module entry, setup/shutdown lifecycle |
|
|
170
|
+
| `lib/legion/data/connection.rb` | Sequel database connection (adapter selection) |
|
|
171
|
+
| `lib/legion/data/migration.rb` | Migration runner |
|
|
172
|
+
| `lib/legion/data/migrations/` | 14 numbered migration files (Sequel DSL) |
|
|
173
|
+
| `lib/legion/data/model.rb` | Model autoloader |
|
|
174
|
+
| `lib/legion/data/local.rb` | Local SQLite module for agentic cognitive state |
|
|
175
|
+
| `lib/legion/data/models/` | Sequel models (Extension, Function, Runner, Node, Task, TaskLog, Setting, DigitalWorker, Relationship, ApolloEntry, ApolloRelation, ApolloExpertise, ApolloAccessLog) |
|
|
176
|
+
| `lib/legion/data/encryption/cipher.rb` | AES-256-GCM encrypt/decrypt with versioned binary format and AAD |
|
|
177
|
+
| `lib/legion/data/encryption/key_provider.rb` | Vault-backed key derivation with per-tenant scope and local fallback |
|
|
178
|
+
| `lib/legion/data/encryption/sequel_plugin.rb` | Transparent `encrypted_column` DSL for Sequel models |
|
|
179
|
+
| `lib/legion/data/event_store.rb` | Append-only governance event store with hash chain integrity |
|
|
180
|
+
| `lib/legion/data/event_store/projection.rb` | Projection base class, ConsentState, GovernanceTimeline |
|
|
181
|
+
| `lib/legion/data/settings.rb` | Default configuration with per-adapter credential presets |
|
|
182
|
+
| `lib/legion/data/version.rb` | VERSION constant |
|
|
183
|
+
| `exe/legionio_migrate` | CLI executable for running database migrations standalone |
|
|
184
|
+
|
|
185
|
+
## Role in LegionIO
|
|
186
|
+
|
|
187
|
+
Optional persistent storage initialized during `Legion::Service` startup (after transport). Provides:
|
|
188
|
+
1. Extension and function registry (which LEXs are installed, what functions they expose)
|
|
189
|
+
2. Task scheduling and logging
|
|
190
|
+
3. Node cluster membership tracking
|
|
191
|
+
4. Persistent settings storage
|
|
192
|
+
5. Digital worker registry (AI-as-labor platform)
|
|
193
|
+
6. Task relationship graph (trigger/action chains)
|
|
194
|
+
7. Apollo shared knowledge store (PostgreSQL + pgvector only, used by lex-apollo)
|
|
195
|
+
8. Local SQLite for agentic cognitive state (memory traces, trust scores, dream journals) — always on-node, independent of shared DB
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
|
|
199
|
+
**Maintained By**: Matthew Iverson (@Esity)
|
data/Gemfile
CHANGED
data/LICENSE
CHANGED
|
@@ -186,7 +186,7 @@
|
|
|
186
186
|
same "printed page" as the copyright notice for easier
|
|
187
187
|
identification within third-party archives.
|
|
188
188
|
|
|
189
|
-
Copyright 2021
|
|
189
|
+
Copyright 2021 Esity
|
|
190
190
|
|
|
191
191
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
192
192
|
you may not use this file except in compliance with the License.
|
data/README.md
CHANGED
|
@@ -1,74 +1,181 @@
|
|
|
1
|
-
|
|
2
|
-
=====
|
|
1
|
+
# legion-data
|
|
3
2
|
|
|
4
|
-
|
|
3
|
+
Persistent database storage for the [LegionIO](https://github.com/LegionIO/LegionIO) framework. Provides database connectivity via Sequel ORM, automatic schema migrations, and data models for extensions, functions, runners, nodes, tasks, settings, digital workers, task relationships, and Apollo shared knowledge tables.
|
|
5
4
|
|
|
6
|
-
|
|
7
|
-
------------------------------------------------
|
|
5
|
+
Version: 1.3.0
|
|
8
6
|
|
|
9
|
-
|
|
7
|
+
## Supported Databases
|
|
10
8
|
|
|
11
|
-
|
|
9
|
+
| Database | Adapter | Gem | Default |
|
|
10
|
+
|----------|---------|-----|---------|
|
|
11
|
+
| SQLite | `sqlite` | `sqlite3` (included) | Yes |
|
|
12
|
+
| MySQL | `mysql2` | `mysql2` | No |
|
|
13
|
+
| PostgreSQL | `postgres` | `pg` | No |
|
|
12
14
|
|
|
15
|
+
SQLite is the default adapter and requires no external database server. For MySQL or PostgreSQL, install the corresponding gem and set the adapter in your configuration.
|
|
13
16
|
|
|
14
|
-
Installation
|
|
15
|
-
------------------------
|
|
16
|
-
|
|
17
|
-
You can verify your installation using this piece of code:
|
|
17
|
+
## Installation
|
|
18
18
|
|
|
19
19
|
```bash
|
|
20
20
|
gem install legion-data
|
|
21
21
|
```
|
|
22
22
|
|
|
23
|
+
Or add to your Gemfile:
|
|
24
|
+
|
|
25
|
+
```ruby
|
|
26
|
+
gem 'legion-data'
|
|
27
|
+
|
|
28
|
+
# Add one of these for production databases:
|
|
29
|
+
# gem 'mysql2', '>= 0.5.5'
|
|
30
|
+
# gem 'pg', '>= 1.5'
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Data Models
|
|
34
|
+
|
|
35
|
+
| Model | Table | Description |
|
|
36
|
+
|-------|-------|-------------|
|
|
37
|
+
| `Extension` | `extensions` | Installed LEX extensions |
|
|
38
|
+
| `Function` | `functions` | Available functions per extension |
|
|
39
|
+
| `Runner` | `runners` | Runner definitions (extension + function bindings) |
|
|
40
|
+
| `Node` | `nodes` | Cluster node registry |
|
|
41
|
+
| `Task` | `tasks` | Task instances |
|
|
42
|
+
| `TaskLog` | `task_logs` | Task execution logs |
|
|
43
|
+
| `Setting` | `settings` | Persistent settings store |
|
|
44
|
+
| `DigitalWorker` | `digital_workers` | Digital worker registry (AI-as-labor platform) |
|
|
45
|
+
| `Relationship` | `relationships` | Task trigger/action relationships between functions |
|
|
46
|
+
| `ApolloEntry` | `apollo_entries` | Apollo shared knowledge entries (PostgreSQL only) |
|
|
47
|
+
| `ApolloRelation` | `apollo_relations` | Relations between Apollo knowledge entries (PostgreSQL only) |
|
|
48
|
+
| `ApolloExpertise` | `apollo_expertise` | Per-agent domain expertise tracking (PostgreSQL only) |
|
|
49
|
+
| `ApolloAccessLog` | `apollo_access_log` | Apollo entry access audit log (PostgreSQL only) |
|
|
50
|
+
|
|
51
|
+
Apollo models require PostgreSQL with the `pgvector` extension. They are skipped silently on SQLite and MySQL.
|
|
52
|
+
|
|
53
|
+
## Usage
|
|
54
|
+
|
|
23
55
|
```ruby
|
|
24
56
|
require 'legion/data'
|
|
25
57
|
|
|
58
|
+
# Standard setup (shared DB + local SQLite)
|
|
26
59
|
Legion::Data.setup
|
|
27
|
-
Legion::Data.
|
|
28
|
-
Legion::Data
|
|
60
|
+
Legion::Data.connection # => Sequel::Database (shared)
|
|
61
|
+
Legion::Data.local.connection # => Sequel::SQLite::Database (local cognitive state)
|
|
62
|
+
Legion::Data::Model::Extension.all # => Sequel::Dataset
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Local Database
|
|
66
|
+
|
|
67
|
+
v1.3.0 introduces `Legion::Data::Local`, a parallel SQLite database always stored locally on the node. It is used for agentic cognitive state persistence (memory traces, trust scores, dream journals, etc.) and is independent of the shared database.
|
|
68
|
+
|
|
69
|
+
```ruby
|
|
70
|
+
# Local DB is set up automatically during Legion::Data.setup
|
|
71
|
+
# Extensions register their own migration directories
|
|
72
|
+
Legion::Data::Local.register_migrations(name: :memory, path: '/path/to/migrations')
|
|
73
|
+
|
|
74
|
+
# Create a model bound to the local connection
|
|
75
|
+
MyModel = Legion::Data::Local.model(:my_table)
|
|
76
|
+
|
|
77
|
+
# Check status
|
|
78
|
+
Legion::Data::Local.connected? # => true
|
|
79
|
+
Legion::Data::Local.db_path # => "legionio_local.db"
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The local database file (`legionio_local.db` by default) can be deleted for cryptographic erasure — no residual data. This is used by `lex-privatecore`.
|
|
83
|
+
|
|
84
|
+
## Configuration
|
|
85
|
+
|
|
86
|
+
### SQLite (default)
|
|
87
|
+
|
|
88
|
+
```json
|
|
89
|
+
{
|
|
90
|
+
"data": {
|
|
91
|
+
"adapter": "sqlite",
|
|
92
|
+
"creds": {
|
|
93
|
+
"database": "legionio.db"
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
### MySQL
|
|
100
|
+
|
|
101
|
+
```json
|
|
102
|
+
{
|
|
103
|
+
"data": {
|
|
104
|
+
"adapter": "mysql2",
|
|
105
|
+
"creds": {
|
|
106
|
+
"username": "legion",
|
|
107
|
+
"password": "legion",
|
|
108
|
+
"database": "legionio",
|
|
109
|
+
"host": "127.0.0.1",
|
|
110
|
+
"port": 3306
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
### PostgreSQL
|
|
117
|
+
|
|
118
|
+
```json
|
|
119
|
+
{
|
|
120
|
+
"data": {
|
|
121
|
+
"adapter": "postgres",
|
|
122
|
+
"creds": {
|
|
123
|
+
"user": "legion",
|
|
124
|
+
"password": "legion",
|
|
125
|
+
"database": "legionio",
|
|
126
|
+
"host": "127.0.0.1",
|
|
127
|
+
"port": 5432
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
PostgreSQL with `pgvector` is required for Apollo models. Install the extension in your database before running migrations:
|
|
134
|
+
|
|
135
|
+
```sql
|
|
136
|
+
CREATE EXTENSION IF NOT EXISTS vector;
|
|
137
|
+
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
|
|
29
138
|
```
|
|
30
139
|
|
|
31
|
-
|
|
32
|
-
----------
|
|
140
|
+
### Local Database
|
|
33
141
|
|
|
34
142
|
```json
|
|
35
143
|
{
|
|
36
|
-
"
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
"log_warn_duration": 1,
|
|
46
|
-
"log_warn_duration": "debug",
|
|
47
|
-
"max_connections": 10,
|
|
48
|
-
"preconnect": false
|
|
49
|
-
},
|
|
50
|
-
"creds": {
|
|
51
|
-
"username": "legion",
|
|
52
|
-
"password": "legion",
|
|
53
|
-
"database": "legionio",
|
|
54
|
-
"host": "127.0.0.1",
|
|
55
|
-
"port": 3306
|
|
56
|
-
},
|
|
57
|
-
"migrations": {
|
|
58
|
-
"continue_on_fail": false,
|
|
59
|
-
"auto_migrate": true,
|
|
60
|
-
"ran": false,
|
|
61
|
-
"version": null
|
|
62
|
-
},
|
|
63
|
-
"models": {
|
|
64
|
-
"continue_on_load_fail": false,
|
|
65
|
-
"autoload": true
|
|
66
|
-
},
|
|
67
|
-
"connect_on_start": true
|
|
144
|
+
"data": {
|
|
145
|
+
"local": {
|
|
146
|
+
"enabled": true,
|
|
147
|
+
"database": "legionio_local.db",
|
|
148
|
+
"migrations": {
|
|
149
|
+
"auto_migrate": true
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
68
153
|
}
|
|
69
154
|
```
|
|
70
155
|
|
|
71
|
-
|
|
72
|
-
|
|
156
|
+
Set `enabled: false` to disable local SQLite entirely.
|
|
157
|
+
|
|
158
|
+
### Dev Mode Fallback
|
|
159
|
+
|
|
160
|
+
When `dev_mode: true` and a network database (MySQL/PostgreSQL) is unreachable, the shared connection falls back to SQLite automatically instead of raising.
|
|
161
|
+
|
|
162
|
+
```json
|
|
163
|
+
{
|
|
164
|
+
"data": {
|
|
165
|
+
"dev_mode": true,
|
|
166
|
+
"dev_fallback": true
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
### HashiCorp Vault Integration
|
|
172
|
+
|
|
173
|
+
When Vault is connected and a `database/creds/legion` secret path exists, credentials are fetched dynamically from Vault at connection time, overriding any static `creds` configuration.
|
|
174
|
+
|
|
175
|
+
## Requirements
|
|
176
|
+
|
|
177
|
+
- Ruby >= 3.4
|
|
178
|
+
|
|
179
|
+
## License
|
|
73
180
|
|
|
74
|
-
|
|
181
|
+
Apache-2.0
|
|
File without changes
|