legion-data 1.6.7 → 1.6.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '086c96d0cbcc3b76593dd560ac03bd117030c9b4eba8a46267c06dea52bbdd65'
-  data.tar.gz: 3fbea1b46dc309a31eaca4037943d247b9f8d2d5f8f7aa9bbaee496c3f4c445b
+  metadata.gz: 49c73e289f9b262a372502c338850d55671323b3c7911f5e4f06f9f27ec090de
+  data.tar.gz: cdccf57ec5b2810d5113f5e28cea5ba54856160fc0ac9c2d5258c0c30f64d8ab
 SHA512:
-  metadata.gz: 4b90cc9cd4bf154bea5b38bfee693897c48dd03494617e129eae547772833cebf5283d7360c52712d2d234956033a825f490dc1a1543aa15aea2ccd859960784
-  data.tar.gz: ab0f9fc400ec8a1a047da73e7128386f8b3fdc03ab751a4e0ec456515e48a50b1f36e9d18c0051839922c3ee6fdf05bf3553183dc282d9a6437ac2193001a481
+  metadata.gz: a65b9366d797d2e47d5846a428f7619314528f63265c2ca091043433fb52923cd0e2c53fc68552db569645fdd3c752831ae8a2c4141a5479abba44f1d4714355
+  data.tar.gz: fbb37ce91bc9555e2460b45382a005e2fda63cf2fedc2962aee6ae95108f272745c8f1b858ea6e569b9aa0b918054b4c0c508dcaa9dac0fb267fe9960bee8cb8

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Legion::Data Changelog
 
+## [1.6.8] - 2026-03-27
+
+### Changed
+- Documentation updates (CLAUDE.md, README.md)
+
 ## [1.6.7] - 2026-03-26
 
 ### Removed

data/CLAUDE.md

@@ -8,7 +8,7 @@
 Manages persistent database storage for the LegionIO framework. Supports SQLite (default), MySQL, and PostgreSQL via Sequel ORM. Provides automatic schema migrations and data models for extensions, functions, runners, nodes, tasks, settings, digital workers, task relationships, Apollo shared knowledge tables (PostgreSQL only), tenants, webhooks, audit log, and archive tables. Also provides a parallel local SQLite database (`Legion::Data::Local`) for agentic cognitive state persistence.
 
 **GitHub**: https://github.com/LegionIO/legion-data
-**Version**: 1.6.0
+**Version**: 1.6.6
 **License**: Apache-2.0
 
 ## Supported Databases
@@ -56,7 +56,7 @@ Legion::Data (singleton module)
 │   ├── .shutdown      # Close local connection
 │   └── .reset!        # Clear all state (testing)
 │
-├── Migration          # Auto-migration system (26 migrations, Sequel DSL)
+├── Migration          # Auto-migration system (47 migrations, Sequel DSL)
 │   └── migrations/
 │       ├── 001_add_schema_columns
 │       ├── 002_add_nodes
@@ -83,7 +83,28 @@ Legion::Data (singleton module)
 │       ├── 023_add_data_archive
 │       ├── 024_add_tenant_partition_columns
 │       ├── 025_add_tenants_table
-│       └── 026_add_function_embeddings  # description + embedding (TEXT) on functions; postgres: embedding_vector vector(1536) with HNSW cosine index
+│       ├── 026_add_function_embeddings  # description + embedding (TEXT) on functions; postgres: embedding_vector vector(1536) with HNSW cosine index
+│       ├── 027_add_apollo_source_provider
+│       ├── 028_add_agent_cluster
+│       ├── 029_add_agent_cluster_tasks
+│       ├── 030_add_approval_queue
+│       ├── 031_add_task_depth
+│       ├── 032_add_task_cancelled_at
+│       ├── 033_add_task_delay
+│       ├── 034_add_archive_manifest
+│       ├── 035_add_apollo_source_channel
+│       ├── 036_add_audit_context_snapshot
+│       ├── 037_add_apollo_knowledge_domain
+│       ├── 038_add_conversations
+│       ├── 039_add_audit_archive_manifest  # 7-year tiered audit retention
+│       ├── 040_add_slow_query_indexes       # tasks table performance indexes
+│       ├── 041_resize_vector_columns
+│       ├── 042_add_tenant_to_registry_tables
+│       ├── 043_add_rls_placeholder          # PostgreSQL row-level security
+│       ├── 044_expand_memory_traces
+│       ├── 045_add_memory_associations
+│       ├── 046_add_metering_hourly_rollup
+│       └── 047_apollo_knowledge_capture     # identity cols, ops table, archive table, 25+ indexes
 │
 ├── Model              # Sequel model loader
 │   └── Models/
@@ -248,7 +269,7 @@ Per-adapter credential defaults are defined in `Settings::CREDS`:
 | `lib/legion/data.rb` | Module entry, setup/shutdown lifecycle |
 | `lib/legion/data/connection.rb` | Sequel database connection (adapter selection) |
 | `lib/legion/data/migration.rb` | Migration runner |
-| `lib/legion/data/migrations/` | 26 numbered migration files (Sequel DSL) |
+| `lib/legion/data/migrations/` | 47 numbered migration files (Sequel DSL) |
 | `lib/legion/data/model.rb` | Model autoloader |
 | `lib/legion/data/local.rb` | Local SQLite module for agentic cognitive state |
 | `lib/legion/data/models/` | Sequel models (Extension, Function, Runner, Node, Task, TaskLog, Setting, DigitalWorker, Relationship, ApolloEntry, ApolloRelation, ApolloExpertise, ApolloAccessLog, AuditLog, RbacRoleAssignment, RbacRunnerGrant, RbacCrossTeamGrant) |
@@ -261,6 +282,12 @@ Per-adapter credential defaults are defined in `Settings::CREDS`:
 | `lib/legion/data/storage_tiers.rb` | Hot/warm/cold archival lifecycle: `archive_to_warm`, `export_to_cold`, `stats` |
 | `lib/legion/data/archival.rb` | Archival module entry point and configuration |
 | `lib/legion/data/archival/` | Archival strategy implementations |
+| `lib/legion/data/extract.rb` | 10-handler text extraction registry (txt/md/csv/json/jsonl/html/xlsx/docx/pdf/pptx) |
+| `lib/legion/data/extract/handlers/` | Per-format extraction handlers (base, csv, docx, html, json, jsonl, markdown, pdf, pptx, text, xlsx) |
+| `lib/legion/data/extract/type_detector.rb` | MIME type detection for extract registry |
+| `lib/legion/data/rls.rb` | PostgreSQL row-level security helpers (tenant isolation, session variable) |
+| `lib/legion/data/partition_manager.rb` | Tenant partition management |
+| `lib/legion/data/retention.rb` | Audit retention and archival lifecycle |
 | `lib/legion/data/settings.rb` | Default configuration with per-adapter credential presets |
 | `lib/legion/data/version.rb` | VERSION constant |
 | `exe/legionio_migrate` | CLI executable for running database migrations standalone |

data/README.md

@@ -1,8 +1,8 @@
 # legion-data
 
-Persistent database storage for the [LegionIO](https://github.com/LegionIO/LegionIO) framework. Provides database connectivity via Sequel ORM, automatic schema migrations, and data models for extensions, functions, runners, nodes, tasks, settings, digital workers, task relationships, and Apollo shared knowledge tables.
+Persistent database storage for the [LegionIO](https://github.com/LegionIO/LegionIO) framework. Provides database connectivity via Sequel ORM, automatic schema migrations (47 numbered migrations), and data models for extensions, functions, runners, nodes, tasks, settings, digital workers, task relationships, Apollo shared knowledge tables (PostgreSQL only), tenants, audit log, and archive tables.
 
-**Version**: 1.4.12
+**Version**: 1.6.6
 
 ## Supported Databases
 
@@ -12,7 +12,7 @@ Persistent database storage for the [LegionIO](https://github.com/LegionIO/Legio
 | MySQL | `mysql2` | `mysql2` | No |
 | PostgreSQL | `postgres` | `pg` | No |
 
-SQLite is the default adapter and requires no external database server. For MySQL or PostgreSQL, install the corresponding gem and set the adapter in your configuration.
+SQLite is the default adapter. For MySQL or PostgreSQL, install the corresponding gem and set the adapter in your configuration.
 
 ## Installation
 
@@ -36,22 +36,24 @@ gem 'legion-data'
 |-------|-------|-------------|
 | `Extension` | `extensions` | Installed LEX extensions |
 | `Function` | `functions` | Available functions per extension |
-| `Runner` | `runners` | Runner definitions (extension + function bindings) |
+| `Runner` | `runners` | Runner definitions |
 | `Node` | `nodes` | Cluster node registry |
 | `Task` | `tasks` | Task instances |
 | `TaskLog` | `task_logs` | Task execution logs |
 | `Setting` | `settings` | Persistent settings store |
-| `DigitalWorker` | `digital_workers` | Digital worker registry (AI-as-labor platform) |
+| `DigitalWorker` | `digital_workers` | Digital worker registry |
 | `Relationship` | `relationships` | Task trigger/action relationships between functions |
-| `ApolloEntry` | `apollo_entries` | Apollo shared knowledge entries (PostgreSQL only) |
-| `ApolloRelation` | `apollo_relations` | Relations between Apollo knowledge entries (PostgreSQL only) |
-| `ApolloExpertise` | `apollo_expertise` | Per-agent domain expertise tracking (PostgreSQL only) |
-| `ApolloAccessLog` | `apollo_access_log` | Apollo entry access audit log (PostgreSQL only) |
+| `AuditLog` | `audit_log` | Tamper-evident audit trail with hash chain |
+| `RbacRoleAssignment` | `rbac_role_assignments` | RBAC principal -> role mappings |
+| `RbacRunnerGrant` | `rbac_runner_grants` | Per-runner permission grants |
+| `RbacCrossTeamGrant` | `rbac_cross_team_grants` | Cross-team access grants |
+| `ApolloEntry` | `apollo_entries` | Apollo knowledge entries — PostgreSQL only (pgvector) |
+| `ApolloRelation` | `apollo_relations` | Relations between Apollo entries — PostgreSQL only |
+| `ApolloExpertise` | `apollo_expertise` | Per-agent domain expertise — PostgreSQL only |
+| `ApolloAccessLog` | `apollo_access_log` | Apollo access audit log — PostgreSQL only |
 
 Apollo models require PostgreSQL with the `pgvector` extension. They are skipped silently on SQLite and MySQL.
 
-Migration 026 adds `description` (TEXT) and `embedding` (TEXT, JSON-serialized vector) columns to the `functions` table, plus a `embedding_vector vector(1536)` column with HNSW cosine index on PostgreSQL for semantic similarity search of runner functions.
-
 ## Usage
 
 ```ruby
@@ -66,7 +68,7 @@ Legion::Data::Model::Extension.all  # => Sequel::Dataset
 
 ### Local Database
 
-v1.3.0 introduces `Legion::Data::Local`, a parallel SQLite database always stored locally on the node. It is used for agentic cognitive state persistence (memory traces, trust scores, dream journals, etc.) and is independent of the shared database.
+`Legion::Data::Local` is a parallel SQLite database always stored locally on the node. Used for agentic cognitive state persistence (memory traces, trust scores, dream journals) and is independent of the shared database.
 
 ```ruby
 # Local DB is set up automatically during Legion::Data.setup
@@ -81,7 +83,29 @@ Legion::Data::Local.connected?   # => true
 Legion::Data::Local.db_path      # => "legionio_local.db"
 ```
 
-The local database file (`legionio_local.db` by default) can be deleted for cryptographic erasure — no residual data. This is used by `lex-privatecore`.
+Deleting `legionio_local.db` provides cryptographic erasure — no residual data.
+
+### Text Extraction
+
+`Legion::Data::Extract` provides a 10-handler registry for extracting text from documents. Supports: `.txt`, `.md`, `.csv`, `.json`, `.jsonl`, `.html`, `.xlsx`, `.docx`, `.pdf`, `.pptx`. Used by `lex-knowledge` for corpus ingestion.
+
+```ruby
+text = Legion::Data::Extract.extract('/path/to/document.pdf')
+```
+
+### Row-Level Security
+
+`Legion::Data::Rls` provides tenant isolation helpers for PostgreSQL (migration 043). Sets `app.current_tenant_id` session variable before queries and resets it after.
+
+### Spool (Filesystem Buffer)
+
+`Legion::Data::Spool` provides a filesystem-backed write buffer. When the database is unavailable, data is written to `~/.legionio/data/spool/` and replayed once the connection is restored.
+
+```ruby
+spool = Legion::Data::Spool.for(Legion::Extensions::MyLex)
+spool.write({ task_id: SecureRandom.uuid, data: payload })
+spool.drain { |entry| process(entry) }
+```
 
 ## Configuration
 
@@ -132,7 +156,7 @@ The local database file (`legionio_local.db` by default) can be deleted for cryp
 }
 ```
 
-PostgreSQL with `pgvector` is required for Apollo models. Install the extension in your database before running migrations:
+PostgreSQL with `pgvector` is required for Apollo models:
 
 ```sql
 CREATE EXTENSION IF NOT EXISTS vector;
@@ -155,21 +179,9 @@ CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
 }
 ```
 
-Set `enabled: false` to disable local SQLite entirely.
-
-### Spool (Filesystem Buffer)
-
-`Legion::Data::Spool` provides a filesystem-backed write buffer for extensions. When the database is unavailable, task data can be written to `~/.legionio/data/spool/` and replayed once the connection is restored.
-
-```ruby
-spool = Legion::Data::Spool.for(Legion::Extensions::MyLex)
-spool.write({ task_id: SecureRandom.uuid, data: payload })
-spool.drain { |entry| process(entry) }
-```
-
 ### Dev Mode Fallback
 
-When `dev_mode: true` and a network database (MySQL/PostgreSQL) is unreachable, the shared connection falls back to SQLite automatically instead of raising.
+When `dev_mode: true` and a network database is unreachable, the shared connection falls back to SQLite automatically:
 
 ```json
 {
@@ -182,7 +194,7 @@ When `dev_mode: true` and a network database (MySQL/PostgreSQL) is unreachable,
 
 ### HashiCorp Vault Integration
 
-When Vault is connected and a `database/creds/legion` secret path exists, credentials are fetched dynamically from Vault at connection time, overriding any static `creds` configuration.
+When Vault is connected, credentials are fetched dynamically from `database/creds/legion`, overriding any static `creds` configuration.
 
 ## Requirements
 

data/lib/legion/data/connection.rb

@@ -123,18 +123,7 @@ module Legion
                       end
                     end
           Legion::Settings[:data][:connected] = true
-          if defined?(Legion::Logging)
-            if adapter == :sqlite
-              Legion::Logging.info "Connected to SQLite at #{sqlite_path}"
-            else
-              creds = Legion::Data::Settings.creds(adapter)
-              user = creds[:user] || creds[:username] || 'unknown'
-              host = creds[:host] || '127.0.0.1'
-              port = creds[:port]
-              db   = creds[:database] || creds[:db]
-              Legion::Logging.info "Connected to #{adapter}://#{user}@#{host}:#{port}/#{db}"
-            end
-          end
+          log_connection_info if defined?(Legion::Logging)
           configure_extensions
           connect_with_replicas
         end
@@ -273,6 +262,19 @@ module Legion
           {}
         end
 
+        def log_connection_info
+          if adapter == :sqlite
+            Legion::Logging.info "Connected to SQLite at #{sqlite_path}"
+          else
+            actual = Legion::Settings[:data][:creds] || {}
+            user = actual[:user] || actual[:username] || 'unknown'
+            host = actual[:host] || '127.0.0.1'
+            port = actual[:port]
+            db   = actual[:database] || actual[:db]
+            Legion::Logging.info "Connected to #{adapter}://#{user}@#{host}:#{port}/#{db}"
+          end
+        end
+
         def dev_fallback?
           data_settings = Legion::Settings[:data]
           data_settings[:dev_mode] == true && data_settings[:dev_fallback] != false

data/lib/legion/data/migration.rb

@@ -10,6 +10,13 @@ module Legion
           Legion::Settings[:data][:migrations][:version] = Sequel::Migrator.run(connection, path, **)
           Legion::Logging.info("Legion::Data::Migration ran successfully to version #{Legion::Settings[:data][:migrations][:version]}")
           Legion::Settings[:data][:migrations][:ran] = true
+        rescue Sequel::DatabaseError => e
+          if e.message.include?('InsufficientPrivilege') || e.message.include?('permission denied')
+            raise Sequel::DatabaseError,
+                  "#{e.message}\n  Hint: the database user lacks CREATE on schema public " \
+                  '(required for PG 15+). Grant via: GRANT CREATE ON SCHEMA public TO <user>;'
+          end
+          raise
         end
       end
     end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.6.7'
+    VERSION = '1.6.8'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.6.7
+  version: 1.6.8
 platform: ruby
 authors:
 - Esity