legion-data 1.7.0 → 1.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8f45999e6d1c3b0727dc351630f7c76da902a40513e9cbcd35dbce97d5c90304
-  data.tar.gz: b3efe7c69e3ccb9540f44fc4e7a464428898abaa74bcddbab9c4b39f2eefc3b4
+  metadata.gz: b562a2c2ce70e81d1b71038255dd8a23a34fc78805acadc34da983dc995b2690
+  data.tar.gz: ae6cb3c93f9498b9f16df5294c3a9ce871a47723da87933c7ae5ac48b2fcf4cb
 SHA512:
-  metadata.gz: 6449f9218be46e31329571cca1d9d3233860b8dd3e307eb35fcc824d3385969dd25a3493a1b0e4261786e622fe754c3d20f152bbf3bf9bfb920c0f0041103df9
-  data.tar.gz: b24487e1c279cda679fac9f27b3b4d7d5af00be5652782065e0f9b3a2b78502dce1d45c0db45b3f5f3632da9fd68f18b75714825d83dc0c0cd05953a26ae6c03
+  metadata.gz: ec953bd837aa49a737584ea20c8668eb35156953593584297fc80edca0d9c7b7594a55b91976006a572b7521b797e44310d7244c4055683b5cf6c673bcddd8fd
+  data.tar.gz: 750707075046c2b09edbd5a0f8bca49059e3b60bf6707fe0ffdb694b54df868e358a308d7e86fbbaf1b1e5c005a797517edde17cfdcf0782d9338532c067d460

data/.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+# Standard LegionIO pre-commit configuration
+# Install: pre-commit install
+# Manual:  pre-commit run --all-files
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-json
+        exclude: Gemfile\.lock
+      - id: check-merge-conflict
+
+  - repo: local
+    hooks:
+      - id: rubocop
+        name: RuboCop (autofix)
+        entry: scripts/pre-commit-rubocop.sh
+        language: script
+        types: [ruby]
+        pass_filenames: true
+
+      - id: ruby-syntax
+        name: Ruby syntax check
+        entry: bash -c 'status=0; for file in "$@"; do ruby -c "$file" || status=$?; done; exit $status' --
+        language: system
+        types: [ruby]
+        pass_filenames: true

data/CHANGELOG.md

@@ -2,6 +2,43 @@
 
 ## [Unreleased]
 
+## [1.7.4] - 2026-04-28
+
+### Fixed
+- Pre-commit RuboCop hook now distinguishes missing tools from real RuboCop failures and propagates failures instead of silently passing.
+- Ruby syntax pre-commit hook now checks every staged Ruby file instead of only the first argument.
+- Connection setup now refreshes the configured adapter before each setup call and clears fallback state on shutdown so fallback health checks do not stay stale across reconnects.
+
+### Changed
+- README refreshed for the current migration count, version line, fallback diagnostics, pre-commit workflow, and recent model surface.
+
+## [1.7.3] - 2026-04-27
+
+### Added
+- Migration 074: widens Apollo `content_hash` to 64 fixed characters and `knowledge_domain` / `source_provider` / `source_agent` to 255 characters so SHA-256 hashes and real-world identifiers fit without ingestion truncation failures. (Fixes #33, #34)
+- Migration 075: adds task `idempotency_key` and `idempotency_expires_at` columns plus indexes for SHA-256 payload deduplication windows. (Fixes #14)
+- Migration 076: adds `extract_step_timings` for per-step Extract pipeline timing visibility. (Fixes #15)
+- `Task.idempotency_key_for`, `Task.find_active_by_idempotency_key`, and `Task.create_idempotent` for stable content-addressed task dispatch deduplication. (Fixes #14)
+- Extract results now include `extract_id` and `step_timings`, and persist timing rows when the migration is present. (Fixes #15)
+- `AuditLogHashChain` plus `AuditLog.compute_hash` / `AuditLog.verify_chain` as the canonical data-side audit log hash-chain implementation for standard write paths to share. (Refs #13)
+
+### Fixed
+- Migration 051 now adds SQLite/MySQL `tasks.created_at` without a non-constant default before backfilling from `created`, allowing later migration specs and fresh SQLite databases to migrate cleanly.
+
+## [1.7.2] - 2026-04-27
+
+### Fixed
+- Dev-fallback to SQLite now logs at `:error` level with explicit warnings that data written to SQLite will not be visible when the configured network database reconnects.
+
+### Added
+- `Connection.connection_info` — returns adapter, connection state, and fallback status for health checks and diagnostics
+- `Connection.fallback_active?` — returns true when the data layer fell back to SQLite from a configured network database; Apollo and other services can check this to detect degraded mode and log appropriate warnings
+
+## [1.7.1] - 2026-04-27
+
+### Fixed
+- `QueryFileLogger` now treats writes after `close` as no-ops, preventing repeated `IOError: closed stream` warnings from late Sequel query callbacks during shutdown. (Fixes #35)
+
 ## [1.7.0] - 2026-04-24
 
 ### Added

data/README.md

@@ -1,8 +1,8 @@
 # legion-data
 
-Persistent database storage for the [LegionIO](https://github.com/LegionIO/LegionIO) async job engine and AI coding assistant platform. Provides database connectivity via the [Sequel ORM](https://sequel.jeremyevans.net/), automatic schema migrations (71 numbered migrations), Sequel models for the full LegionIO control plane, and a parallel local SQLite database for on-node agentic cognitive state.
+Persistent database storage for the [LegionIO](https://github.com/LegionIO/LegionIO) async job engine and AI coding assistant platform. Provides database connectivity via the [Sequel ORM](https://sequel.jeremyevans.net/), automatic schema migrations (76 numbered migrations), Sequel models for the full LegionIO control plane, and a parallel local SQLite database for on-node agentic cognitive state.
 
-**Version**: 1.6.25 | **Ruby**: >= 3.4 | **License**: Apache-2.0
+**Version**: 1.7.4 | **Ruby**: >= 3.4 | **License**: Apache-2.0
 
 ---
 
@@ -51,17 +51,19 @@ Legion::Data (singleton module)
 │   ├── .adapter        # Reads adapter from settings (:sqlite, :mysql2, :postgres)
 │   ├── .setup          # Establish connection (dev_mode fallback to SQLite if unreachable)
 │   ├── .sequel         # Raw Sequel::Database accessor
+│   ├── .connection_info  # Adapter, liveness, and fallback diagnostics
+│   ├── .fallback_active? # True when dev fallback moved a network DB to SQLite
 │   ├── .stats          # Pool metrics, tuning snapshot, adapter-specific DB stats
 │   └── .shutdown       # Disconnect and close query file logger
 │
-├── Migration           # Auto-migration system (71 numbered Sequel DSL migrations)
+├── Migration           # Auto-migration system (76 numbered Sequel DSL migrations)
 │
 ├── Model               # Sequel model autoloader
 │   └── Models: Extension, Function, Runner, Node, Task, TaskLog, Setting,
 │               DigitalWorker, Relationship, AuditLog, AuditRecord, Chain,
 │               RbacRoleAssignment, RbacRunnerGrant, RbacCrossTeamGrant,
 │               IdentityProvider, Principal, Identity, IdentityGroup,
-│               IdentityGroupMembership,
+│               IdentityGroupMembership, IdentityAuditLog,
 │               ApolloEntry, ApolloRelation, ApolloExpertise, ApolloAccessLog (PG only)
 │
 ├── Local               # Parallel local SQLite for agentic cognitive state
@@ -117,6 +119,10 @@ Legion::Data.local.db_path           # => "legionio_local.db"
 Legion::Data.connected?              # => true
 Legion::Data.stats                   # => { shared: {...}, local: {...} }
 
+# Inspect shared DB diagnostics, including dev fallback state
+Legion::Data::Connection.connection_info
+# => { adapter: :sqlite, connected: true, fallback_active: false, ... }
+
 # Shut down both connections
 Legion::Data.shutdown
 ```
@@ -139,12 +145,30 @@ MyMemoryTrace.all  # queries legionio_local.db, never the shared DB
 `Legion::Data::Extract` provides a handler registry for extracting text from documents, used by `lex-knowledge` for corpus ingestion:
 
 ```ruby
-text = Legion::Data::Extract.extract('/path/to/document.pdf')
-text = Legion::Data::Extract.extract('/path/to/data.csv')
+result = Legion::Data::Extract.extract('/path/to/document.pdf')
+text = result[:text]
+result[:step_timings] # per-step name, start_time, end_time, status, error, duration_ms
 ```
 
 Supported formats: `.txt`, `.md`, `.csv`, `.json`, `.jsonl`, `.html`, `.xlsx`, `.docx`, `.pdf`, `.pptx`, `.vtt`
 
+When migration 076 is present, Extract also persists the same per-step timing rows to `extract_step_timings`
+under the returned `extract_id`.
+
+### Task Idempotency
+
+`Task.idempotency_key_for` computes a stable SHA-256 key from canonical JSON payloads. `Task.create_idempotent`
+returns an existing non-terminal task for the same key inside the optional TTL window, or creates a new task
+with `idempotency_key` and `idempotency_expires_at` populated:
+
+```ruby
+task = Legion::Data::Model::Task.create_idempotent(
+  { status: 'pending', payload: Legion::JSON.dump(payload) },
+  payload: payload,
+  ttl: 300
+)
+```
+
 ### Filesystem Spool (Write Buffer)
 
 When the database is unavailable, `Legion::Data::Spool` buffers writes to `~/.legionio/data/spool/` and replays once the connection is restored:
@@ -296,6 +320,8 @@ When `dev_mode: true` and a network database is unreachable, the shared connecti
 { "data": { "dev_mode": true, "dev_fallback": true } }
 ```
 
+Fallback is intentionally loud. `Connection.setup` logs the degraded mode at error level, `Connection.fallback_active?` returns `true`, and `Connection.connection_info` reports the configured adapter, actual adapter, connection state, and Sequel liveness. Data written during fallback is local-only SQLite data and will not appear in the configured network database after reconnect.
+
 ### HashiCorp Vault Integration
 
 When Vault is connected, credentials are fetched dynamically from `database/creds/legion`, overriding any static `creds` block.
@@ -343,6 +369,7 @@ Legion::Data.reload_static_cache
 | `Chain` | `chains` | Task execution chains |
 | `AuditLog` | `audit_log` | Tamper-evident audit trail with hash chain |
 | `AuditRecord` | `audit_records` | Structured audit records |
+| `ExtractStepTiming` | `extract_step_timings` | Per-step Extract pipeline timing metadata |
 | `RbacRoleAssignment` | `rbac_role_assignments` | RBAC principal -> role mappings |
 | `RbacRunnerGrant` | `rbac_runner_grants` | Per-runner permission grants |
 | `RbacCrossTeamGrant` | `rbac_cross_team_grants` | Cross-team access grants |
@@ -377,7 +404,7 @@ Apollo models require PostgreSQL with the `pgvector` extension. They are skipped
 
 ## Migrations
 
-71 numbered Sequel DSL migrations run automatically on startup (`auto_migrate: true`). Key milestones:
+76 numbered Sequel DSL migrations run automatically on startup (`auto_migrate: true`). Key milestones:
 
 | Range | What was added |
 |-------|---------------|
@@ -393,6 +420,8 @@ Apollo models require PostgreSQL with the `pgvector` extension. They are skipped
 | 050 | Critical indexes across 13 tables |
 | 058–067 | Audit records, chains, knowledge tiers, tool embedding cache, identity system (providers, principals, identities, groups) |
 | 068–071 | Entity type on audit records, principal on nodes, approval queue resume, engine on relationships |
+| 072–073 | Identity audit log and multi-instance identity columns |
+| 074–076 | Apollo field width fixes, task idempotency columns, and Extract step timing rows |
 
 Run migrations standalone:
 
@@ -442,6 +471,15 @@ bundle exec rspec        # all tests must pass
 bundle exec rubocop -A   # zero offenses expected
 ```
 
+This repo also includes a pre-commit configuration:
+
+```bash
+pre-commit install
+pre-commit run --all-files
+```
+
+The local RuboCop hook auto-corrects staged Ruby files when RuboCop is available and fails the commit when RuboCop reports real offenses. The Ruby syntax hook checks every staged Ruby file.
+
 Follow the [LegionIO contribution guide](https://github.com/LegionIO/.github/blob/main/CONTRIBUTING.md). Open a PR against `main`.
 
 ---

data/lib/legion/data/audit_log_hash_chain.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'legion/json'
+require 'time'
+
+module Legion
+  module Data
+    module AuditLogHashChain
+      GENESIS_HASH = ('0' * 64).freeze
+      CANONICAL_FIELDS = %i[
+        principal_id action resource source status detail created_at previous_hash
+      ].freeze
+
+      class << self
+        def compute_hash(record)
+          Digest::SHA256.hexdigest(canonical_payload(record))
+        end
+
+        def verify(records)
+          previous_hash = GENESIS_HASH
+          records.each do |record|
+            return invalid(record, :parent_mismatch) unless value_for(record, :previous_hash).to_s == previous_hash
+
+            expected = compute_hash(record)
+            return invalid(record, :hash_mismatch) unless value_for(record, :record_hash).to_s == expected
+
+            previous_hash = expected
+          end
+
+          { valid: true, length: records.size }
+        end
+
+        def canonical_payload(record)
+          CANONICAL_FIELDS.map do |field|
+            "#{field}:#{canonical_value(value_for(record, field))}"
+          end.join('|')
+        end
+
+        private
+
+        def invalid(record, reason)
+          { valid: false, broken_at: value_for(record, :id), reason: reason }
+        end
+
+        def canonical_value(value)
+          case value
+          when Time
+            value.utc.iso8601(6)
+          when DateTime
+            value.to_time.utc.iso8601(6)
+          when Hash
+            Legion::JSON.dump(canonical_hash(value))
+          when Array
+            Legion::JSON.dump(value.map { |item| canonical_json_value(item) })
+          else
+            value.to_s
+          end
+        end
+
+        def canonical_json_value(value)
+          case value
+          when Hash then canonical_hash(value)
+          when Array then value.map { |item| canonical_json_value(item) }
+          else value
+          end
+        end
+
+        def canonical_hash(hash)
+          hash.keys.map(&:to_s).sort.to_h do |key|
+            [key, canonical_json_value(hash.fetch(key) { hash.fetch(key.to_sym) })]
+          end
+        end
+
+        def value_for(record, field)
+          return record[field] if record.respond_to?(:[]) && !record[field].nil?
+          return record[field.to_s] if record.respond_to?(:[]) && !record[field.to_s].nil?
+          return record.public_send(field) if record.respond_to?(field)
+
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/legion/data/connection.rb

@@ -103,12 +103,13 @@ module Legion
 
         def initialize(path)
           @path = path
+          @closed = false
+          @mutex = Mutex.new
           dir = File.dirname(path)
           FileUtils.mkdir_p(dir)
           FileUtils.chmod(0o700, dir) if File.directory?(dir)
           @file = File.open(path, File::WRONLY | File::APPEND | File::CREAT, 0o600)
           @file.sync = true
-          @mutex = Mutex.new
         end
 
         def debug(message)
@@ -128,16 +129,23 @@ module Legion
         end
 
         def close
-          @mutex.synchronize { @file.close unless @file.closed? }
+          @mutex.synchronize do
+            @closed = true
+            @file.close unless @file.closed?
+          end
         end
 
         private
 
         def write(level, message)
           @mutex.synchronize do
+            return if @closed || @file.closed?
+
             @file.puts "[#{Time.now.strftime('%Y-%m-%d %H:%M:%S.%L')}] #{level} #{message}"
           end
         rescue IOError => e
+          return nil if @closed || @file.closed?
+
           handle_exception(e, level: :warn, handled: true, operation: :query_file_write, path: @path)
           nil
         end
@@ -153,18 +161,25 @@ module Legion
         end
 
         def setup
+          @adapter = Legion::Settings[:data][:adapter]&.to_sym || :sqlite
           opts = sequel_opts
           log.info("Legion::Data::Connection setup adapter=#{adapter}")
+          @fallback_active = false
           @sequel = if adapter == :sqlite
                       ::Sequel.connect(opts.merge(adapter: :sqlite, database: sqlite_path))
                     else
+                      attempted_adapter = adapter
                       begin
-                        ::Sequel.connect(connection_opts_for(adapter: adapter, opts: opts))
+                        ::Sequel.connect(connection_opts_for(adapter: attempted_adapter, opts: opts))
                       rescue StandardError => e
                         raise unless dev_fallback?
 
-                        handle_exception(e, level: :warn, handled: true, operation: :shared_connect, fallback: :sqlite)
+                        log.error("Legion::Data FALLING BACK TO SQLITE — #{attempted_adapter} network DB connection failed: #{e.message}")
+                        log.error("Legion::Data WARNING: Data written to SQLite will NOT be visible when #{attempted_adapter} reconnects. " \
+                                  'Apollo knowledge, audit logs, and other DB-backed services will use a local-only store.')
+                        handle_exception(e, level: :error, handled: true, operation: :shared_connect, fallback: :sqlite)
                         @adapter = :sqlite
+                        @fallback_active = true
                         sqlite_opts = sequel_opts
                         ::Sequel.connect(sqlite_opts.merge(adapter: :sqlite, database: sqlite_path))
                       end
@@ -175,6 +190,25 @@ module Legion
           connect_with_replicas
         end
 
+        # Returns connection metadata for health checks and diagnostics.
+        # Apollo and other services can use this to detect silent fallback.
+        def connection_info
+          {
+            adapter:            adapter,
+            connected:          Legion::Settings[:data][:connected],
+            fallback_active:    @fallback_active || false,
+            configured_adapter: Legion::Settings[:data][:adapter]&.to_sym || :sqlite,
+            sequel_alive:       (begin; !@sequel&.test_connection.nil?; rescue StandardError; false; end)
+          }
+        end
+
+        # Returns true if the data layer fell back to SQLite from a configured
+        # network database (PostgreSQL/MySQL). Services should check this and
+        # log warnings when operating in degraded mode.
+        def fallback_active?
+          @fallback_active == true
+        end
+
         def stats
           return { connected: false } unless @sequel
 
@@ -228,6 +262,7 @@ module Legion
           @sequel&.disconnect
           @query_file_logger&.close
           @query_file_logger = nil
+          @fallback_active = false
           Legion::Settings[:data][:connected] = false
           log.info 'Legion::Data connection closed'
         end

data/lib/legion/data/extract.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'legion/logging/helper'
+require 'securerandom'
 require_relative 'extract/type_detector'
 require_relative 'extract/handlers/base'
 
@@ -11,29 +12,49 @@ module Legion
         include Legion::Logging::Helper
 
         def extract(source, type: :auto)
-          detected_type = type == :auto ? TypeDetector.detect(source) : type&.to_sym
-          return { success: false, text: nil, error: :unknown_type } unless detected_type
+          extract_id = SecureRandom.uuid
+          timings = []
+          detected_type = timed_step(:detect_type, timings) do
+            type == :auto ? TypeDetector.detect(source) : type&.to_sym
+          end
+          unless detected_type
+            result = { success: false, text: nil, error: :unknown_type, extract_id: extract_id,
+                       step_timings: timings }
+            persist_step_timings(extract_id, timings)
+            return result
+          end
 
-          handler = Handlers::Base.for_type(detected_type)
-          return { success: false, text: nil, error: :no_handler, type: detected_type } unless handler
+          handler = timed_step(:resolve_handler, timings) { Handlers::Base.for_type(detected_type) }
+          unless handler
+            result = { success: false, text: nil, error: :no_handler, type: detected_type, extract_id: extract_id,
+                       step_timings: timings }
+            persist_step_timings(extract_id, timings)
+            return result
+          end
 
-          unless handler.available?
+          available = timed_step(:check_availability, timings) { handler.available? }
+          unless available
             return { success: false, text: nil, error: :gem_not_installed,
-                     gem: handler.gem_name, type: detected_type }
+                     gem: handler.gem_name, type: detected_type, extract_id: extract_id,
+                     step_timings: timings }.tap { persist_step_timings(extract_id, timings) }
           end
 
           log.info "Extract starting type=#{detected_type} handler=#{handler.name}"
-          result = handler.extract(source)
+          result = timed_step(:handler_extract, timings) { handler.extract(source) }
           if result[:text]
             log.info "Extract succeeded type=#{detected_type}"
-            { success: true, text: result[:text], metadata: result[:metadata], type: detected_type }
+            { success: true, text: result[:text], metadata: result[:metadata], type: detected_type,
+              extract_id: extract_id, step_timings: timings }
           else
             log.warn "Extract failed type=#{detected_type} error=#{result[:error]}"
-            { success: false, text: nil, error: result[:error], type: detected_type }
-          end
+            { success: false, text: nil, error: result[:error], type: detected_type,
+              extract_id: extract_id, step_timings: timings }
+          end.tap { persist_step_timings(extract_id, timings) }
         rescue StandardError => e
           handle_exception(e, level: :error, handled: true, operation: :extract, type: detected_type)
-          { success: false, text: nil, error: e.message, type: detected_type }
+          persist_step_timings(extract_id, timings) if extract_id
+          { success: false, text: nil, error: e.message, type: detected_type, extract_id: extract_id,
+            step_timings: timings }
         end
 
         def supported_types
@@ -54,6 +75,48 @@ module Legion
 
         private
 
+        def timed_step(name, timings)
+          monotonic_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          start_time = Time.now.utc
+          result = yield
+          record_step_timing(timings, name: name, start_time: start_time, monotonic_start: monotonic_start,
+                                      status: :success)
+          result
+        rescue StandardError => e
+          record_step_timing(timings, name: name, start_time: start_time, monotonic_start: monotonic_start,
+                                      status: :error, error: "#{e.class}: #{e.message}")
+          raise
+        end
+
+        def record_step_timing(timings, name:, start_time:, monotonic_start:, status:, error: nil)
+          end_time = Time.now.utc
+          duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - monotonic_start) * 1000).round
+          timings << {
+            name:        name.to_s,
+            start_time:  start_time,
+            end_time:    end_time,
+            status:      status.to_s,
+            error:       error,
+            duration_ms: duration_ms
+          }
+        end
+
+        def persist_step_timings(extract_id, timings)
+          return unless defined?(Legion::Data)
+
+          connection = Legion::Data.connection
+          return unless connection&.table_exists?(:extract_step_timings)
+
+          existing_steps = connection[:extract_step_timings].where(extract_id: extract_id).select_map(:name)
+          rows = timings.reject { |timing| existing_steps.include?(timing[:name]) }.map do |timing|
+            timing.merge(extract_id: extract_id)
+          end
+          connection[:extract_step_timings].multi_insert(rows) unless rows.empty?
+        rescue StandardError => e
+          handle_exception(e, level: :warn, handled: true, operation: :persist_extract_step_timings,
+                            extract_id: extract_id)
+        end
+
         def load_all_handlers
           return if @handlers_loaded
 

data/lib/legion/data/migrations/051_fix_tasks_created_at.rb

@@ -14,7 +14,7 @@ Sequel.migration do
     else
       # SQLite/MySQL: add real column and backfill from created
       alter_table(:tasks) do
-        add_column :created_at, DateTime, default: Sequel::CURRENT_TIMESTAMP
+        add_column :created_at, DateTime
       end
 
       run 'UPDATE tasks SET created_at = created WHERE created_at IS NULL'

data/lib/legion/data/migrations/074_widen_apollo_entry_identifiers.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    next unless adapter_scheme == :postgres
+    next unless table_exists?(:apollo_entries)
+
+    apollo_columns = schema(:apollo_entries).map(&:first)
+    alter_table(:apollo_entries) do
+      set_column_type :content_hash, String, fixed: true, size: 64 if apollo_columns.include?(:content_hash)
+      set_column_type :knowledge_domain, String, size: 255 if apollo_columns.include?(:knowledge_domain)
+      set_column_type :source_provider, String, size: 255 if apollo_columns.include?(:source_provider)
+      set_column_type :source_agent, String, size: 255 if apollo_columns.include?(:source_agent)
+    end
+
+    next unless table_exists?(:apollo_entries_archive)
+
+    archive_columns = schema(:apollo_entries_archive).map(&:first)
+    alter_table(:apollo_entries_archive) do
+      set_column_type :content_hash, String, fixed: true, size: 64 if archive_columns.include?(:content_hash)
+      set_column_type :knowledge_domain, String, size: 255 if archive_columns.include?(:knowledge_domain)
+      set_column_type :source_provider, String, size: 255 if archive_columns.include?(:source_provider)
+      set_column_type :source_agent, String, size: 255 if archive_columns.include?(:source_agent)
+    end
+  end
+
+  down do
+    next unless adapter_scheme == :postgres
+    next unless table_exists?(:apollo_entries)
+
+    apollo_columns = schema(:apollo_entries).map(&:first)
+    alter_table(:apollo_entries) do
+      set_column_type :content_hash, String, fixed: true, size: 32 if apollo_columns.include?(:content_hash)
+      set_column_type :knowledge_domain, String, size: 50 if apollo_columns.include?(:knowledge_domain)
+      set_column_type :source_provider, String, size: 50 if apollo_columns.include?(:source_provider)
+      set_column_type :source_agent, String, size: 50 if apollo_columns.include?(:source_agent)
+    end
+
+    next unless table_exists?(:apollo_entries_archive)
+
+    archive_columns = schema(:apollo_entries_archive).map(&:first)
+    alter_table(:apollo_entries_archive) do
+      set_column_type :content_hash, String, fixed: true, size: 32 if archive_columns.include?(:content_hash)
+      set_column_type :knowledge_domain, String, size: 50 if archive_columns.include?(:knowledge_domain)
+      set_column_type :source_provider, String, size: 50 if archive_columns.include?(:source_provider)
+      set_column_type :source_agent, String, size: 50 if archive_columns.include?(:source_agent)
+    end
+  end
+end

data/lib/legion/data/migrations/075_add_task_idempotency.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    next unless table_exists?(:tasks)
+
+    existing_columns = schema(:tasks).map(&:first)
+    alter_table(:tasks) do
+      add_column :idempotency_key, String, size: 64 unless existing_columns.include?(:idempotency_key)
+      add_column :idempotency_expires_at, DateTime unless existing_columns.include?(:idempotency_expires_at)
+    end
+
+    add_index :tasks, :idempotency_key, name: :idx_tasks_idempotency_key, if_not_exists: true
+    add_index :tasks, :idempotency_expires_at, name: :idx_tasks_idempotency_expires_at, if_not_exists: true
+  end
+
+  down do
+    next unless table_exists?(:tasks)
+
+    existing_columns = schema(:tasks).map(&:first)
+    alter_table(:tasks) do
+      drop_index :idempotency_key, name: :idx_tasks_idempotency_key, if_exists: true
+      drop_index :idempotency_expires_at, name: :idx_tasks_idempotency_expires_at, if_exists: true
+      drop_column :idempotency_expires_at if existing_columns.include?(:idempotency_expires_at)
+      drop_column :idempotency_key if existing_columns.include?(:idempotency_key)
+    end
+  end
+end

data/lib/legion/data/migrations/076_create_extract_step_timings.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    create_table?(:extract_step_timings) do
+      primary_key :id
+      String :extract_id, size: 36, null: false
+      String :name, size: 100, null: false
+      DateTime :start_time, null: false
+      DateTime :end_time, null: false
+      String :status, size: 20, null: false
+      String :error, text: true
+      Integer :duration_ms, null: false, default: 0
+
+      index :extract_id, name: :idx_extract_step_timings_extract_id
+      index %i[extract_id name], name: :idx_extract_step_timings_extract_name
+      index :status, name: :idx_extract_step_timings_status
+    end
+  end
+
+  down do
+    drop_table?(:extract_step_timings)
+  end
+end

data/lib/legion/data/model.rb

@@ -14,7 +14,7 @@ module Legion
           %w[extension function relationship chain task runner node setting digital_worker
              apollo_entry apollo_relation apollo_expertise apollo_access_log audit_log
              audit_record identity_provider principal identity identity_group
-             identity_group_membership identity_audit_log]
+             identity_group_membership identity_audit_log extract_step_timing]
         end
 
         def load

data/lib/legion/data/models/audit_log.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'legion/logging/helper'
+require 'legion/data/audit_log_hash_chain'
 
 module Legion
   module Data
@@ -33,6 +34,14 @@ module Legion
         def before_destroy
           raise 'audit_log records are immutable and cannot be deleted'
         end
+
+        def self.compute_hash(record)
+          Legion::Data::AuditLogHashChain.compute_hash(record)
+        end
+
+        def self.verify_chain(records = order(:created_at, :id).all)
+          Legion::Data::AuditLogHashChain.verify(records)
+        end
       end
     end
   end

data/lib/legion/data/models/extract_step_timing.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Legion
+  module Data
+    module Model
+      class ExtractStepTiming < Sequel::Model(:extract_step_timings)
+      end
+    end
+  end
+end

data/lib/legion/data/models/task.rb

@@ -1,9 +1,17 @@
 # frozen_string_literal: true
 
+require 'digest'
+require 'legion/json'
+require 'time'
+
 module Legion
   module Data
     module Model
       class Task < Sequel::Model
+        TERMINAL_STATUSES = %w[
+          completed complete failed error cancelled canceled timeout timed_out
+        ].freeze
+
         many_to_one :relationship
         one_to_many :task_log
         many_to_one :parent, class: self
@@ -11,9 +19,51 @@ module Legion
         many_to_one :master, class: self
         one_to_many :slave, key: :master_id, class: self
 
+        def self.idempotency_key_for(payload)
+          Digest::SHA256.hexdigest(Legion::JSON.dump(canonical_payload(payload)))
+        end
+
+        def self.find_active_by_idempotency_key(key, now: Time.now)
+          return nil if key.to_s.empty?
+          return nil unless columns.include?(:idempotency_key)
+
+          where(idempotency_key: key)
+            .exclude(status: TERMINAL_STATUSES)
+            .where { (idempotency_expires_at =~ nil) | (idempotency_expires_at > now) }
+            .reverse_order(:created, :id)
+            .first
+        end
+
+        def self.create_idempotent(values, payload: nil, idempotency_key: nil, ttl: nil)
+          key = idempotency_key || idempotency_key_for(payload || values)
+          existing = find_active_by_idempotency_key(key)
+          return existing if existing
+
+          expires_at = ttl ? Time.now + ttl : nil
+          create(values.merge(idempotency_key: key, idempotency_expires_at: expires_at))
+        end
+
         def cancelled?
           !cancelled_at.nil?
         end
+
+        def self.canonical_payload(value)
+          case value
+          when Hash
+            value.keys.map(&:to_s).sort.to_h do |key|
+              [key, canonical_payload(value.fetch(key) { value.fetch(key.to_sym) })]
+            end
+          when Array
+            value.map { |item| canonical_payload(item) }
+          when Time
+            value.utc.iso8601(6)
+          when DateTime
+            value.to_time.utc.iso8601(6)
+          else
+            value
+          end
+        end
+        private_class_method :canonical_payload
       end
     end
   end

data/lib/legion/data/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Data
-    VERSION = '1.7.0'
+    VERSION = '1.7.4'
   end
 end

data/lib/legion/data.rb

@@ -16,6 +16,7 @@ require_relative 'data/helper'
 require_relative 'data/rls'
 require_relative 'data/extract'
 require_relative 'data/audit_record'
+require_relative 'data/audit_log_hash_chain'
 
 unless Legion::Logging::Helper.method_defined?(:handle_exception)
   module Legion

data/scripts/pre-commit-rubocop.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Pre-commit hook: run RuboCop with autofix on staged Ruby files.
+set -uo pipefail
+
+FILES=("$@")
+
+if command -v rubocop >/dev/null 2>&1; then
+  exec rubocop -A --force-exclusion "${FILES[@]}"
+fi
+
+if bundle exec rubocop -v >/dev/null 2>&1; then
+  exec bundle exec rubocop -A --force-exclusion "${FILES[@]}"
+fi
+
+echo "RuboCop is not available locally; CI will enforce RuboCop."
+exit 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-data
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.7.4
 platform: ruby
 authors:
 - Esity
@@ -107,6 +107,7 @@ files:
 - ".github/dependabot.yml"
 - ".github/workflows/ci.yml"
 - ".gitignore"
+- ".pre-commit-config.yaml"
 - ".rubocop.yml"
 - AGENTS.md
 - CHANGELOG.md
@@ -121,6 +122,7 @@ files:
 - lib/legion/data/archival.rb
 - lib/legion/data/archival/policy.rb
 - lib/legion/data/archiver.rb
+- lib/legion/data/audit_log_hash_chain.rb
 - lib/legion/data/audit_record.rb
 - lib/legion/data/connection.rb
 - lib/legion/data/encryption/cipher.rb
@@ -218,6 +220,9 @@ files:
 - lib/legion/data/migrations/071_add_engine_to_relationships.rb
 - lib/legion/data/migrations/072_create_identity_audit_log.rb
 - lib/legion/data/migrations/073_add_identity_multi_instance_columns.rb
+- lib/legion/data/migrations/074_widen_apollo_entry_identifiers.rb
+- lib/legion/data/migrations/075_add_task_idempotency.rb
+- lib/legion/data/migrations/076_create_extract_step_timings.rb
 - lib/legion/data/model.rb
 - lib/legion/data/models/apollo_access_log.rb
 - lib/legion/data/models/apollo_entry.rb
@@ -228,6 +233,7 @@ files:
 - lib/legion/data/models/chain.rb
 - lib/legion/data/models/digital_worker.rb
 - lib/legion/data/models/extension.rb
+- lib/legion/data/models/extract_step_timing.rb
 - lib/legion/data/models/function.rb
 - lib/legion/data/models/identity.rb
 - lib/legion/data/models/identity_audit_log.rb
@@ -252,6 +258,7 @@ files:
 - lib/legion/data/storage_tiers.rb
 - lib/legion/data/vector.rb
 - lib/legion/data/version.rb
+- scripts/pre-commit-rubocop.sh
 homepage: https://github.com/LegionIO/legion-data
 licenses:
 - Apache-2.0