legion-apollo 0.4.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7468f022e582731ace40da083285569d57c98e38ef375e5745f43d93e9c9d171
-  data.tar.gz: 3d2bb59a17a6060e690c7321fc914d2dc2e2bdd7bb1ec108917bf86fc7f26c72
+  metadata.gz: 9320cef3392467f9819e47c9cacc3893f45ffab9570825cb5d6b270ff3ce6a67
+  data.tar.gz: d38343aa96c9ba0b1d634f6e4d3850e22c899fe81c13550dce19809cec290a94
 SHA512:
-  metadata.gz: a47cb6e18c47432ec0f3a15872f52dc193d7253fda94833da1c63d6046ad39bba8894f10dacc1c3472d7ef538157aad39cf1cf9894061623a9f1a162f69a5c16
-  data.tar.gz: 3ca7dc35c483089a4b2c6bb0af57b524afe3a3cce571b893a14a4aa01f74fbb01b51b5eb09040389db09a6b5f2516d0bd6d97fc499bf8c912aadc6c3b622a587
+  metadata.gz: 2dcaf3fc8fbf087b8e59c079e71921aebf8c53620e32d9cd8939a00981474747b78e978412d28a5c3ff4f204df01b9538f23a81c82a76a0d94afab8080219e46
+  data.tar.gz: 8d078c7e1c727bd23b08bfdb09af0abe7581296132d715b7ed7c4c5205d4645c004f1f980af6546239961c6649e9433e66a725341e2275e3cdb2e1de7f1687a3

data/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## [0.5.2] - 2026-04-27
+
+### Added
+- Store `raw_content` alongside indexed `content` in Apollo Local so callers can preserve verbatim source text separately from retrieval text (#25, #26)
+- Add `valid_from`/`valid_to` temporal windows and `as_of:` query filtering for local knowledge entries (#27)
+
+### Fixed
+- Sanitize Apollo ingest and query text by scrubbing invalid UTF-8 and removing null bytes before routing to local or global backends (#29)
+
+## [0.5.1] - 2026-04-27
+
+### Fixed
+- Guard Apollo Local tag queries and promotion against nil, shutdown, or unavailable local DB connections before SQL and Ruby fallback paths (#30)
+
+## [0.5.0] - 2026-04-18
+
+### Added
+- Migration 004: versioning, tiers, inference, expiry metadata, and source linkage columns on `local_knowledge`; `local_source_links` table (#6-#11)
+- Inference tagging: `is_inference` flag on ingest, `include_inferences:` filter on query, `INITIAL_INFERENCE_CONFIDENCE` (0.35) for LLM-derived entries (#9)
+- Temporal expiry metadata: `forget_reason` and custom `expires_at` on ingest (#8)
+- Versioned knowledge: `parent_knowledge_id`/`is_latest`/`supersession_type` on ingest, automatic parent supersession, `version_chain` traversal, `include_history:` query filter (#7)
+- L0/L1/L2 tiered retrieval: `tier:` parameter on query with summary projection and truncation fallback (#6)
+- Source-to-fact linkage: `source_uri`/`source_hash`/`relevance_score`/`extraction_method` on ingest, `source_links_for` query method, `local_source_links` table (#10)
+- `SUPERSEDES` relation type in `Local::Graph` (#11)
+- Versioning and expiry settings defaults
+
+### Fixed
+- FTS5 search crashes on punctuation (`.`, `:`, `-`, `+`, etc.) by tokenizing input into quoted alphanumeric terms with implicit AND semantics; ILIKE fallback now escapes `%` and `_` wildcards (#22)
+- Apollo query returns HTTP 500 on non-Postgres backends: `direct_query` exceptions normalized to `:backend_query_failed` symbol, `apollo_status_code` maps known unavailability symbols to 503 (#23)
+
 ## [0.4.0] - 2026-04-02
 
 ### Changed

data/README.md

@@ -1,10 +1,10 @@
 # legion-apollo
 
-Apollo client library for the LegionIO framework.
+Apollo is the LegionIO knowledge client. It gives extensions one API for writing, retrieving, and merging knowledge across the global Apollo service and the node-local SQLite store.
 
-**Version**: 0.3.2
+**Version**: 0.5.2
 
-Provides `query`, `ingest`, and `retrieve` with smart routing: co-located lex-apollo service, RabbitMQ transport, or graceful failure. Supports a node-local SQLite knowledge store (`Apollo::Local`) that mirrors the same API without requiring any remote infrastructure.
+`legion-apollo` provides `query`, `ingest`, and `retrieve` with smart routing: co-located `lex-apollo`, RabbitMQ transport, node-local SQLite, or graceful failure. `Apollo::Local` mirrors the same public API for offline and low-latency retrieval without requiring remote infrastructure.
 
 ## Usage
 
@@ -21,6 +21,24 @@ results = Legion::Apollo.query(text: 'local note', scope: :local)
 
 # Query both and merge (deduped by content hash, ranked by confidence)
 results = Legion::Apollo.query(text: 'ruby', scope: :all)
+
+# Preserve verbatim source text separately from indexed retrieval content
+Legion::Apollo.ingest(
+  content: 'Summarized policy note for search',
+  raw_content: 'Exact source text from the original record',
+  tags: %w[policy source],
+  scope: :local
+)
+
+# Query the local store as it was valid at a point in time
+Legion::Apollo.ingest(
+  content: 'Policy version active in Q2',
+  tags: %w[policy],
+  valid_from: '2026-04-01T00:00:00.000Z',
+  valid_to: '2026-06-30T23:59:59.999Z',
+  scope: :local
+)
+results = Legion::Apollo.query(text: 'policy', scope: :local, as_of: '2026-05-01T00:00:00.000Z')
 ```
 
 ## Scopes
@@ -37,9 +55,12 @@ results = Legion::Apollo.query(text: 'ruby', scope: :all)
 
 Features:
 - Content-hash dedup (MD5 of normalized content)
+- `raw_content` preservation for verbatim source text
+- `valid_from` / `valid_to` temporal windows with `as_of:` query filtering
 - Optional LLM embeddings (1024-dim) with cosine rerank when `Legion::LLM.can_embed?`
 - TTL expiry (default 5-year retention)
 - FTS5 full-text search with `ILIKE` fallback
+- Null-byte removal and invalid UTF-8 scrubbing before persistence or backend routing
 
 ## Configuration
 

data/lib/legion/apollo/helpers/confidence.rb

@@ -17,6 +17,7 @@ module Legion
         WRITE_GATE_THRESHOLD   = 0.3
         HIGH_CONFIDENCE        = 0.8
         ARCHIVE_THRESHOLD      = 0.1
+        INITIAL_INFERENCE_CONFIDENCE = 0.35
 
         STATUSES = %i[pending confirmed disputed deprecated archived].freeze
 

data/lib/legion/apollo/local/graph.rb

@@ -11,7 +11,7 @@ module Legion
       # Relationships are directional typed edges between two entities.
       # Graph traversal expands one frontier batch per depth to avoid per-node queries.
       module Graph # rubocop:disable Metrics/ModuleLength
-        VALID_RELATION_TYPES = %w[AFFECTS OWNED_BY DEPENDS_ON RELATED_TO].freeze
+        VALID_RELATION_TYPES = %w[AFFECTS OWNED_BY DEPENDS_ON RELATED_TO SUPERSEDES].freeze
 
         class << self # rubocop:disable Metrics/ClassLength
           include Legion::Logging::Helper

data/lib/legion/apollo/local/migrations/004_add_versioning_tiers_inference.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+Sequel.migration do # rubocop:disable Metrics/BlockLength
+  up do # rubocop:disable Metrics/BlockLength
+    alter_table(:local_knowledge) do
+      add_column :is_inference, :boolean, default: false, null: false
+      add_column :parent_knowledge_id, Integer, null: true
+      add_column :is_latest, :boolean, default: true, null: false
+      add_column :supersession_type, String, size: 32, null: true
+      add_column :forget_reason, String, size: 128, null: true
+      add_column :summary_l0, String, size: 500, null: true
+      add_column :summary_l1, :text, null: true
+      add_column :knowledge_tier, String, size: 4, null: false, default: 'L2'
+      add_column :l0_generated_at, String, null: true
+      add_column :l1_generated_at, String, null: true
+
+      add_index :is_latest, name: :idx_local_knowledge_is_latest
+      add_index :is_inference, name: :idx_local_knowledge_is_inference
+      add_index :knowledge_tier, name: :idx_local_knowledge_tier
+      add_index :parent_knowledge_id, name: :idx_local_knowledge_parent
+    end
+
+    create_table(:local_source_links) do
+      primary_key :id
+      Integer :entry_id, null: false
+      String  :source_uri, text: true
+      String  :source_hash, size: 64
+      Float   :relevance_score, default: 1.0
+      String  :extraction_method, size: 64
+      String  :created_at, null: false
+
+      index :entry_id, name: :idx_source_links_entry
+      index :source_hash, name: :idx_source_links_hash
+    end
+  end
+
+  down do
+    drop_table(:local_source_links) if table_exists?(:local_source_links)
+
+    alter_table(:local_knowledge) do
+      drop_column :is_inference
+      drop_column :parent_knowledge_id
+      drop_column :is_latest
+      drop_column :supersession_type
+      drop_column :forget_reason
+      drop_column :summary_l0
+      drop_column :summary_l1
+      drop_column :knowledge_tier
+      drop_column :l0_generated_at
+      drop_column :l1_generated_at
+    end
+  end
+end

data/lib/legion/apollo/local/migrations/005_add_raw_content_temporal_windows.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:local_knowledge) do
+      add_column :raw_content, :text, null: true
+      add_column :valid_from, String, null: true
+      add_column :valid_to, String, null: true
+
+      add_index :valid_from, name: :idx_local_knowledge_valid_from
+      add_index :valid_to, name: :idx_local_knowledge_valid_to
+    end
+  end
+
+  down do
+    alter_table(:local_knowledge) do
+      drop_column :raw_content
+      drop_column :valid_from
+      drop_column :valid_to
+    end
+  end
+end

data/lib/legion/apollo/local.rb

@@ -5,6 +5,7 @@ require 'legion/logging'
 require 'socket'
 require 'time'
 require_relative 'local/graph'
+require_relative 'helpers/confidence'
 require_relative 'helpers/similarity'
 require_relative 'helpers/tag_normalizer'
 
@@ -90,7 +91,7 @@ module Legion
           { success: false, error: e.message }
         end
 
-        def query(text:, limit: nil, min_confidence: nil, tags: nil, **) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        def query(text:, limit: nil, min_confidence: nil, tags: nil, **opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity
           return not_started_error unless started?
 
           text = normalize_text_input(text)
@@ -98,19 +99,27 @@ module Legion
           limit ||= local_setting(:default_limit, 5)
           min_confidence ||= local_setting(:min_confidence, 0.3)
           multiplier = local_setting(:fts_candidate_multiplier, 3)
+          as_of = normalize_temporal_value(opts[:as_of])
           log.info do
             "Apollo::Local query executing text_length=#{text.to_s.length} " \
               "limit=#{limit} min_confidence=#{min_confidence} tag_count=#{Array(tags).size}"
           end
           log.debug { "Apollo::Local query limit=#{limit} min_confidence=#{min_confidence} tags=#{Array(tags).size}" }
 
-          candidates = fts_search(text, limit: limit * multiplier)
-          candidates = filter_candidates(candidates, min_confidence: min_confidence, tags: tags)
+          candidates = fts_search(text, limit: limit * multiplier, as_of: as_of)
+          include_inferences = opts.fetch(:include_inferences, true)
+          include_history = opts.fetch(:include_history, false)
+          candidates = filter_candidates(candidates, min_confidence: min_confidence, tags: tags,
+                                                     options: { include_inferences: include_inferences,
+                                                                include_history: include_history, as_of: as_of })
           candidates = cosine_rerank(text, candidates) if can_rerank?
           results = candidates.first(limit)
 
+          tier = opts[:tier]
+          results = results.map { |r| project_tier(r, tier) } if tier
+
           log.info { "Apollo::Local query completed count=#{results.size}" }
-          { success: true, results: results, count: results.size, mode: :local }
+          { success: true, results: results, count: results.size, mode: :local, tier: tier }
         rescue StandardError => e
           handle_exception(
             e,
@@ -151,10 +160,11 @@ module Legion
         end
 
         def query_by_tags(tags:, limit: 50) # rubocop:disable Metrics/MethodLength
-          return { success: false, error: :not_started } unless started?
-
+          connection = local_db_connection
           tags = normalize_tags_input(tags)
-          results = query_by_tags_via_sql(tags: tags, limit: limit)
+          return { success: false, error: :not_started } unless local_db_usable?(connection)
+
+          results = query_by_tags_via_sql(connection, tags: tags, limit: limit)
 
           log.info { "Apollo::Local query_by_tags completed tag_count=#{tags.size} count=#{results.size}" }
           { success: true, results: results, count: results.size }
@@ -170,11 +180,13 @@ module Legion
         end
 
         def promote_to_global(tags:, min_confidence: 0.6) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-          return { success: false, error: :not_started } unless started?
+          return { success: false, error: :not_started } unless local_db_usable?(local_db_connection)
 
           tags = normalize_tags_input(tags)
           entries = query_by_tags(tags: tags)
-          unless entries[:success] && entries[:results]&.any?
+          return entries unless entries[:success]
+
+          unless entries[:results]&.any?
             log.info { "Apollo::Local promote_to_global skipped tag_count=#{tags.size} reason=no_entries" }
             return { success: true, promoted: 0 }
           end
@@ -192,6 +204,7 @@ module Legion
             end
             result = Legion::Apollo.ingest(
               content:        entry[:content],
+              raw_content:    entry[:raw_content] || entry[:content],
               tags:           entry_tags + ['promoted_from_local'],
               source_channel: 'local_promotion',
               submitted_by:   "node:#{hostname}",
@@ -223,6 +236,41 @@ module Legion
           { success: false, error: e.message }
         end
 
+        def version_chain(entry_id:, max_depth: 50) # rubocop:disable Metrics/MethodLength
+          return not_started_error unless started?
+
+          chain = []
+          current_id = entry_id
+          seen = Set.new
+
+          max_depth.times do
+            break unless current_id
+            break if seen.include?(current_id)
+
+            seen.add(current_id)
+            row = db[:local_knowledge].where(id: current_id).first
+            break unless row
+
+            chain << row
+            current_id = row[:parent_knowledge_id]
+          end
+
+          { success: true, chain: chain, count: chain.size }
+        rescue StandardError => e
+          handle_exception(e, level: :error, operation: 'apollo.local.version_chain', entry_id: entry_id)
+          { success: false, error: e.message }
+        end
+
+        def source_links_for(entry_id:)
+          return not_started_error unless started?
+
+          links = db[:local_source_links].where(entry_id: entry_id).all
+          { success: true, links: links, count: links.size }
+        rescue StandardError => e
+          handle_exception(e, level: :error, operation: 'apollo.local.source_links_for', entry_id: entry_id)
+          { success: false, error: e.message }
+        end
+
         private
 
         def self_knowledge_files
@@ -294,6 +342,26 @@ module Legion
           Legion::Data::Local.connection
         end
 
+        def local_db_connection
+          return nil unless started? && data_local_available?
+
+          db
+        rescue StandardError => e
+          handle_exception(e, level: :debug, operation: 'apollo.local.local_db_connection')
+          nil
+        end
+
+        def local_db_usable?(connection)
+          return false unless started? && connection
+          return false if connection.respond_to?(:closed?) && connection.closed?
+
+          connection.test_connection if connection.respond_to?(:test_connection)
+          true
+        rescue StandardError => e
+          handle_exception(e, level: :debug, operation: 'apollo.local.local_db_usable')
+          false
+        end
+
         def content_hash(content)
           normalized = content.to_s.strip.downcase.gsub(/\s+/, ' ')
           Digest::MD5.hexdigest(normalized)
@@ -353,9 +421,12 @@ module Legion
 
             result = ingest(
               content:        entry[:content],
+              raw_content:    entry[:raw_content] || entry[:content],
               tags:           clean_tags,
               confidence:     ((entry[:confidence] || 0.5) * 0.9).round(10),
-              source_channel: 'global_hydration'
+              source_channel: 'global_hydration',
+              valid_from:     entry[:valid_from],
+              valid_to:       entry[:valid_to]
             )
             hydrated += 1 if result[:success]
           end
@@ -365,6 +436,8 @@ module Legion
         end
 
         def ingest_without_lock(content:, tags:, **opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+          content = normalize_text_input(content)
+          raw_content = normalize_text_input(opts.key?(:raw_content) ? opts[:raw_content] : content)
           hash = content_hash(content)
           return deduplicated_ingest(hash) if duplicate?(hash)
 
@@ -374,8 +447,11 @@ module Legion
           end
           log.debug { "Apollo::Local ingest hash=#{hash} tags=#{Array(tags).size} source_channel=#{opts[:source_channel]}" }
 
-          row = build_ingest_row(content: content, hash: hash, tags: tags, **opts)
-          id = persist_ingest_row(row)
+          metadata = opts.dup
+          metadata.delete(:raw_content)
+          row = build_ingest_row(content: content, raw_content: raw_content, hash: hash, tags: tags, **metadata)
+          id = persist_ingest_row(row, metadata)
+          mark_parent_superseded(metadata[:parent_knowledge_id]) if metadata[:parent_knowledge_id]
 
           log.info { "Apollo::Local ingest stored id=#{id} hash=#{hash}" }
           { success: true, mode: :local, id: id }
@@ -385,31 +461,94 @@ module Legion
           deduplicated_ingest(hash)
         end
 
-        def build_ingest_row(content:, hash:, tags:, **opts)
+        def build_ingest_row(content:, raw_content:, hash:, tags:, **opts) # rubocop:disable Metrics/MethodLength
+          is_inference = opts[:is_inference] == true
+          default_confidence = is_inference ? Legion::Apollo::Helpers::Confidence::INITIAL_INFERENCE_CONFIDENCE : 1.0
+          ingest_metadata_columns(
+            content:            content,
+            raw_content:        raw_content,
+            hash:               hash,
+            tags:               tags,
+            opts:               opts,
+            is_inference:       is_inference,
+            default_confidence: default_confidence
+          ).merge(embedding_columns(content, opts)).merge(timestamp_columns)
+        end
+
+        def ingest_metadata_columns(context)
+          ingest_base_columns(context)
+            .merge(ingest_lineage_columns(context[:opts]))
+            .merge(ingest_temporal_columns(context[:opts]))
+        end
+
+        def ingest_base_columns(context)
+          opts = context[:opts]
           {
-            content:        content,
-            content_hash:   hash,
-            tags:           serialized_tags(tags),
-            source_channel: opts[:source_channel],
-            source_agent:   opts[:source_agent],
-            submitted_by:   opts[:submitted_by],
-            confidence:     opts[:confidence] || 1.0
-          }.merge(embedding_columns(content)).merge(timestamp_columns)
+            content:      context[:content],
+            raw_content:  context[:raw_content],
+            content_hash: context[:hash],
+            tags:         serialized_tags(context[:tags]),
+            confidence:   opts[:confidence] || context[:default_confidence],
+            is_inference: context[:is_inference]
+          }.merge(ingest_source_columns(opts))
+        end
+
+        def ingest_source_columns(opts)
+          { source_channel: opts[:source_channel], source_agent: opts[:source_agent],
+            submitted_by: opts[:submitted_by] }
+        end
+
+        def ingest_lineage_columns(opts)
+          {
+            forget_reason:       opts[:forget_reason],
+            parent_knowledge_id: opts[:parent_knowledge_id],
+            supersession_type:   opts[:supersession_type]
+          }
         end
 
-        def persist_ingest_row(row)
+        def ingest_temporal_columns(opts)
+          {
+            valid_from: normalize_temporal_value(opts[:valid_from]),
+            valid_to:   normalize_temporal_value(opts[:valid_to])
+          }
+        end
+
+        def persist_ingest_row(row, opts = {})
           db.transaction do
             id = db[:local_knowledge].insert(row)
             sync_fts!(id, row[:content], row[:tags])
+            create_source_link(id, opts) if opts[:source_uri]
             id
           end
         end
 
+        def create_source_link(entry_id, opts)
+          db[:local_source_links].insert(
+            entry_id:          entry_id,
+            source_uri:        opts[:source_uri],
+            source_hash:       opts[:source_hash],
+            relevance_score:   opts[:relevance_score] || 1.0,
+            extraction_method: opts[:extraction_method],
+            created_at:        Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+          )
+        rescue StandardError => e
+          handle_exception(e, level: :warn, operation: 'apollo.local.create_source_link', entry_id: entry_id)
+        end
+
         def deduplicated_ingest(hash)
           log.info { "Apollo::Local ingest deduplicated hash=#{hash}" }
           { success: true, mode: :deduplicated }
         end
 
+        def mark_parent_superseded(parent_id)
+          return unless parent_id
+
+          db[:local_knowledge].where(id: parent_id).update(is_latest: false)
+          log.info { "Apollo::Local marked entry id=#{parent_id} as superseded" }
+        rescue StandardError => e
+          handle_exception(e, level: :warn, operation: 'apollo.local.mark_parent_superseded', parent_id: parent_id)
+        end
+
         def generate_embedding(content) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
           unless defined?(Legion::LLM) && Legion::LLM.respond_to?(:can_embed?) && Legion::LLM.can_embed?
             log.debug 'Apollo::Local embedding skipped because embeddings are unavailable'
@@ -447,13 +586,13 @@ module Legion
           log.debug { "Apollo::Local FTS synced id=#{id}" }
         end
 
-        def embedding_columns(content)
+        def embedding_columns(content, opts = {})
           embedding, embedded_at = generate_embedding(content)
 
           {
             embedding:   embedding ? Legion::JSON.dump(embedding) : nil,
             embedded_at: embedded_at,
-            expires_at:  compute_expires_at
+            expires_at:  opts[:expires_at] || compute_expires_at
           }
         end
 
@@ -466,33 +605,45 @@ module Legion
           Legion::JSON.dump(normalize_tags_input(tags))
         end
 
-        def fts_search(text, limit:) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
-          if text.to_s.strip.empty?
-            return db[:local_knowledge]
-                   .where(Sequel.lit('expires_at > ?', Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')))
-                   .limit(limit)
-                   .all
-          end
-
-          escaped = text.to_s.gsub('"', '""')
+        def fts_search(text, limit:, as_of: nil) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
           now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+          return active_knowledge_dataset(now: now, as_of: as_of).limit(limit).all if text.to_s.strip.empty?
+
+          tokens = text.to_s.scan(/[\p{L}\p{N}_]+/)
+          return ilike_search(text, now: now, limit: limit, as_of: as_of) if tokens.empty?
+
+          escaped = tokens.map { |t| %("#{t}") }.join(' ')
+          temporal_sql, temporal_params = temporal_window_sql(as_of, table_alias: 'lk')
           db.fetch(
             'SELECT lk.* FROM local_knowledge lk ' \
             'INNER JOIN local_knowledge_fts fts ON lk.id = fts.rowid ' \
-            'WHERE local_knowledge_fts MATCH ? AND lk.expires_at > ? ORDER BY fts.rank LIMIT ?',
-            escaped, now, limit
+            "WHERE local_knowledge_fts MATCH ? AND lk.expires_at > ?#{temporal_sql} " \
+            'ORDER BY fts.rank LIMIT ?',
+            escaped, now, *temporal_params, limit
           ).all
         rescue StandardError => e
           handle_exception(e, level: :debug, operation: 'apollo.local.fts_search', limit: limit, fallback: :ilike)
-          db[:local_knowledge]
-            .where(Sequel.lit('expires_at > ?', Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')))
-            .where(Sequel.ilike(:content, "%#{text}%"))
+          ilike_search(text, now: Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ'), limit: limit, as_of: as_of)
+        end
+
+        def ilike_search(text, now:, limit:, as_of: nil)
+          safe_text = text.to_s.gsub('\\', '\\\\\\\\').gsub('%', '\%').gsub('_', '\_')
+          active_knowledge_dataset(now: now, as_of: as_of)
+            .where(Sequel.lit("content LIKE ? ESCAPE '\\' COLLATE NOCASE", "%#{safe_text}%"))
             .limit(limit)
             .all
         end
 
-        def filter_candidates(candidates, min_confidence:, tags:)
+        def filter_candidates(candidates, min_confidence:, tags:, options: {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/AbcSize
+          include_inferences = options.fetch(:include_inferences, true)
+          include_history = options.fetch(:include_history, false)
+          as_of = options[:as_of]
           candidates = candidates.select { |c| (c[:confidence] || 0) >= min_confidence }
+          candidates = candidates.select { |c| temporally_valid?(c, as_of) }
+          candidates = candidates.reject { |c| [1, true].include?(c[:is_inference]) } unless include_inferences
+          unless include_history
+            candidates = candidates.select { |c| c[:is_latest].nil? || c[:is_latest] == 1 || c[:is_latest] == true }
+          end
           if tags && !tags.empty?
             tag_set = Array(tags).map(&:to_s)
             candidates = candidates.select do |c|
@@ -503,6 +654,36 @@ module Legion
           candidates
         end
 
+        def active_knowledge_dataset(now:, as_of: nil)
+          apply_temporal_window(db[:local_knowledge].where(Sequel.lit('expires_at > ?', now)), as_of)
+        end
+
+        def apply_temporal_window(dataset, as_of)
+          return dataset if as_of.to_s.empty?
+
+          dataset.where(
+            Sequel.lit('(valid_from IS NULL OR valid_from <= ?) AND (valid_to IS NULL OR valid_to >= ?)', as_of, as_of)
+          )
+        end
+
+        def temporal_window_sql(as_of, table_alias:)
+          return ['', []] if as_of.to_s.empty?
+
+          [
+            " AND (#{table_alias}.valid_from IS NULL OR #{table_alias}.valid_from <= ?) " \
+            "AND (#{table_alias}.valid_to IS NULL OR #{table_alias}.valid_to >= ?)",
+            [as_of, as_of]
+          ]
+        end
+
+        def temporally_valid?(row, as_of)
+          return true if as_of.to_s.empty?
+
+          valid_from = row[:valid_from]
+          valid_to = row[:valid_to]
+          (valid_from.nil? || valid_from <= as_of) && (valid_to.nil? || valid_to >= as_of)
+        end
+
         def parse_tags(tags_json)
           return [] if tags_json.nil? || tags_json.empty?
 
@@ -571,6 +752,17 @@ module Legion
           value.to_s
         end
 
+        def normalize_temporal_value(value)
+          return nil if value.nil?
+
+          text = value.respond_to?(:utc) ? value.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ') : value.to_s.strip
+          return nil if text.empty?
+
+          Time.parse(text).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+        rescue StandardError
+          text
+        end
+
         def normalize_tags_input(tags)
           Legion::Apollo::Helpers::TagNormalizer.normalize(Array(tags)).first(max_tags_limit)
         rescue StandardError => e
@@ -589,9 +781,9 @@ module Legion
           Legion::Apollo::Helpers::TagNormalizer::MAX_TAGS
         end
 
-        def query_by_tags_via_sql(tags:, limit:) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        def query_by_tags_via_sql(connection, tags:, limit:) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
           now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
-          dataset = db[:local_knowledge].where(Sequel.lit('expires_at > ?', now))
+          dataset = connection[:local_knowledge].where(Sequel.lit('expires_at > ?', now))
 
           Array(tags).map(&:to_s).each do |tag|
             dataset = dataset.where(
@@ -611,11 +803,15 @@ module Legion
             tag_count: Array(tags).size,
             limit:     limit
           )
-          query_by_tags_via_ruby(tags: tags, limit: limit)
+          raise unless local_db_usable?(connection)
+
+          query_by_tags_via_ruby(connection, tags: tags, limit: limit)
         end
 
-        def query_by_tags_via_ruby(tags:, limit:)
-          candidates = db[:local_knowledge]
+        def query_by_tags_via_ruby(connection, tags:, limit:)
+          raise Sequel::DatabaseConnectionError, 'local database unavailable' unless local_db_usable?(connection)
+
+          candidates = connection[:local_knowledge]
                        .where(Sequel.lit('expires_at > ?', Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')))
                        .all
 
@@ -626,7 +822,7 @@ module Legion
         end
 
         def update_upsert_entry(existing, content, tags_json, opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
-          content = content.to_s
+          content = normalize_text_input(content)
           new_hash = content_hash(content)
           embedding, embedded_at = generate_embedding(content)
           now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
@@ -658,6 +854,21 @@ module Legion
           log.debug { "Apollo::Local FTS rebuilt id=#{id}" }
         end
 
+        def project_tier(entry, tier) # rubocop:disable Metrics/MethodLength
+          case tier
+          when :l0
+            entry.slice(:id, :content_hash, :confidence, :tags, :source_channel, :is_inference, :is_latest).merge(
+              summary: entry[:summary_l0] || entry[:content]&.slice(0, 200)
+            )
+          when :l1
+            entry.slice(:id, :content_hash, :confidence, :tags, :source_channel, :is_inference, :is_latest).merge(
+              summary: entry[:summary_l1] || entry[:content]&.slice(0, 1000)
+            )
+          else
+            entry
+          end
+        end
+
         def not_started_error
           { success: false, error: :not_started }
         end

data/lib/legion/apollo/routes.rb

@@ -50,7 +50,7 @@ module Legion
         end
       end
 
-      def self.register_query_route(app) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity
+      def self.register_query_route(app) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
         app.post '/api/apollo/query' do
           unless apollo_api_available?
             halt 503, json_error('apollo_unavailable', 'apollo is not available', status_code: 503)
@@ -59,21 +59,25 @@ module Legion
           body = parse_request_body
           default_limit = defined?(Legion::Settings) ? (Legion::Settings[:apollo]&.dig(:default_limit) || 5) : 5
           result = Legion::Apollo.query(
-            text:           body[:query],
-            limit:          body[:limit] || default_limit,
-            min_confidence: body[:min_confidence],
-            status:         body[:status] || [:confirmed],
-            tags:           body[:tags],
-            domain:         body[:domain],
-            agent_id:       body[:agent_id] || 'api',
-            scope:          normalize_scope(body[:scope])
+            text:               body[:query],
+            limit:              body[:limit] || default_limit,
+            min_confidence:     body[:min_confidence],
+            status:             body[:status] || [:confirmed],
+            tags:               body[:tags],
+            domain:             body[:domain],
+            agent_id:           body[:agent_id] || 'api',
+            scope:              normalize_scope(body[:scope]),
+            tier:               body[:tier]&.to_sym,
+            as_of:              body[:as_of],
+            include_inferences: body.fetch(:include_inferences, true),
+            include_history:    body.fetch(:include_history, false)
           )
           json_response(result, status_code: apollo_status_code(result))
         end
       end
 
       def self.register_ingest_route(app) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-        app.post '/api/apollo/ingest' do
+        app.post '/api/apollo/ingest' do # rubocop:disable Metrics/BlockLength
           unless apollo_api_available?
             halt 503, json_error('apollo_unavailable', 'apollo is not available', status_code: 503)
           end
@@ -84,15 +88,27 @@ module Legion
           effective_max_tags = [max_tags, Legion::Apollo::Helpers::TagNormalizer::MAX_TAGS].min
           tags = Legion::Apollo::Helpers::TagNormalizer.normalize(Array(body[:tags])).first(effective_max_tags)
           result = Legion::Apollo.ingest(
-            content:          body[:content],
-            content_type:     body[:content_type] || :observation,
-            tags:             tags,
-            source_agent:     body[:source_agent] || 'api',
-            source_provider:  body[:source_provider],
-            source_channel:   body[:source_channel] || 'rest_api',
-            knowledge_domain: body[:knowledge_domain],
-            context:          body[:context] || {},
-            scope:            normalize_scope(body[:scope])
+            content:             body[:content],
+            raw_content:         body[:raw_content],
+            content_type:        body[:content_type] || :observation,
+            tags:                tags,
+            source_agent:        body[:source_agent] || 'api',
+            source_provider:     body[:source_provider],
+            source_channel:      body[:source_channel] || 'rest_api',
+            knowledge_domain:    body[:knowledge_domain],
+            context:             body[:context] || {},
+            scope:               normalize_scope(body[:scope]),
+            is_inference:        body[:is_inference] == true,
+            forget_reason:       body[:forget_reason],
+            expires_at:          body[:expires_at],
+            valid_from:          body[:valid_from],
+            valid_to:            body[:valid_to],
+            parent_knowledge_id: body[:parent_knowledge_id],
+            supersession_type:   body[:supersession_type],
+            source_uri:          body[:source_uri],
+            source_hash:         body[:source_hash],
+            relevance_score:     body[:relevance_score],
+            extraction_method:   body[:extraction_method]
           )
           json_response(result, status_code: apollo_status_code(result, success_status: 201))
         end
@@ -197,13 +213,16 @@ module Legion
           :global
         end
 
-        def apollo_status_code(result, success_status: 200)
+        def apollo_status_code(result, success_status: 200) # rubocop:disable Metrics/MethodLength
           return 202 if result[:success] && result[:mode] == :async
           return success_status if result[:success]
 
           case result[:error]
-          when :no_path_available, :not_started then 503
-          else 500
+          when :no_path_available, :not_started, :local_not_started,
+               :upstream_query_failed, :backend_query_failed
+            503
+          else
+            500
           end
         rescue StandardError => e
           handle_exception(e, level: :debug, operation: :apollo_status_code)

data/lib/legion/apollo/settings.rb

@@ -10,7 +10,9 @@ module Legion
           max_tags:       20,
           default_limit:  5,
           min_confidence: 0.3,
-          local:          local_defaults
+          local:          local_defaults,
+          versioning:     versioning_defaults,
+          expiry:         expiry_defaults
         }
       end
 
@@ -24,6 +26,22 @@ module Legion
           default_limit:            5
         }
       end
+
+      def self.versioning_defaults
+        {
+          enabled:                true,
+          supersession_threshold: 0.85,
+          max_chain_depth:        50
+        }
+      end
+
+      def self.expiry_defaults
+        {
+          enabled:            true,
+          sweep_interval:     3600,
+          warn_before_expiry: 86_400
+        }
+      end
     end
   end
 end

data/lib/legion/apollo/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Apollo
-    VERSION = '0.4.0'
+    VERSION = '0.5.2'
   end
 end

data/lib/legion/apollo.rb

@@ -97,9 +97,11 @@ module Legion
         return not_started_error unless started?
 
         normalized_tags = normalize_tags_input(tags)
-        payload = { content: content, tags: normalized_tags, **opts }
+        normalized_content = normalize_text_input(content)
+        normalized_raw_content = normalize_text_input(opts.key?(:raw_content) ? opts[:raw_content] : content)
+        payload = { **opts, content: normalized_content, raw_content: normalized_raw_content, tags: normalized_tags }
         log.info do
-          "Apollo ingest requested scope=#{scope} content_length=#{content.to_s.length} " \
+          "Apollo ingest requested scope=#{scope} content_length=#{payload[:content].to_s.length} " \
             "tags=#{payload[:tags].size} source_channel=#{payload[:source_channel]}"
         end
         log.debug do
@@ -247,7 +249,7 @@ module Legion
         Legion::Extensions::Apollo::Runners::Knowledge.handle_query(**normalize_query_payload(payload))
       rescue StandardError => e
         handle_exception(e, level: :error, operation: 'apollo.direct_query', payload_keys: payload.keys)
-        { success: false, error: e.message }
+        { success: false, error: :backend_query_failed, detail: e.message }
       end
 
       def direct_ingest(payload)
@@ -288,7 +290,9 @@ module Legion
           "Apollo query using local store text_length=#{payload[:text].to_s.length} " \
             "limit=#{payload[:limit]}"
         end
-        result = Legion::Apollo::Local.query(**payload.slice(:text, :limit, :min_confidence, :tags))
+        result = Legion::Apollo::Local.query(**payload.slice(:text, :limit, :min_confidence, :tags,
+                                                             :tier, :include_inferences, :include_history,
+                                                             :as_of))
         return result unless result[:success]
 
         entries = normalize_local_entries(Array(result[:results]))
@@ -322,7 +326,9 @@ module Legion
 
         if Legion::Apollo::Local.started?
           attempted = true
-          local = Legion::Apollo::Local.query(**payload.slice(:text, :limit, :min_confidence, :tags))
+          local = Legion::Apollo::Local.query(**payload.slice(:text, :limit, :min_confidence, :tags,
+                                                              :tier, :include_inferences, :include_history,
+                                                              :as_of))
           if local[:success]
             any_success = true
             entries.concat(normalize_local_entries(Array(local[:results]))) if local[:results]
@@ -341,6 +347,9 @@ module Legion
         return { success: false, error: :no_path_available } unless attempted
 
         unless any_success
+          symbol_errors = errors.compact.grep(Symbol).uniq
+          return { success: false, error: symbol_errors.first } if symbol_errors.size == 1
+
           combined_error = errors.compact.map(&:to_s).reject(&:empty?).join('; ')
           combined_error = :upstream_query_failed if combined_error.empty?
           return { success: false, error: combined_error }
@@ -372,18 +381,29 @@ module Legion
                  else
                    Array(e[:tags])
                  end
-          { id: e[:id], content: e[:content], content_hash: hash,
-            confidence: e[:confidence] || 0.5, content_type: 'fact', tags: tags, source: :local }
+          { id: e[:id], content: e[:content], raw_content: e[:raw_content] || e[:content], content_hash: hash,
+            confidence: e[:confidence] || 0.5, content_type: 'fact', tags: tags, source: :local,
+            valid_from: e[:valid_from], valid_to: e[:valid_to] }
         end
       end
 
       def normalize_global_entries(entries)
-        entries.map do |e|
-          hash = e[:content_hash] || Digest::MD5.hexdigest(e[:content].to_s.strip.downcase.gsub(/\s+/, ' '))
-          { id: e[:id], content: e[:content], content_hash: hash,
-            confidence: e[:confidence] || 0.5, content_type: e[:content_type] || 'fact',
-            tags: Array(e[:tags]), source: :global }
-        end
+        entries.map { |entry| normalize_global_entry(entry) }
+      end
+
+      def normalize_global_entry(entry)
+        { id: entry[:id], content: entry[:content], raw_content: normalized_raw_content(entry),
+          content_hash: normalized_content_hash(entry), confidence: entry[:confidence] || 0.5,
+          content_type: entry[:content_type] || 'fact', tags: Array(entry[:tags]), source: :global,
+          valid_from: entry[:valid_from], valid_to: entry[:valid_to] }
+      end
+
+      def normalized_raw_content(entry)
+        entry[:raw_content] || entry[:content]
+      end
+
+      def normalized_content_hash(entry)
+        entry[:content_hash] || Digest::MD5.hexdigest(entry[:content].to_s.strip.downcase.gsub(/\s+/, ' '))
       end
 
       def dedup_and_rank(entries, limit:)
@@ -465,20 +485,28 @@ module Legion
       end
 
       def normalize_text_input(value) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength
-        case value
-        when String
-          value
-        when Array
-          parts = value.filter_map { |entry| extract_text_fragment(entry) }
-          joined = parts.map(&:to_s).map(&:strip).reject(&:empty?).join("\n")
-          joined.empty? ? value.to_s : joined
-        when Hash
-          extract_text_fragment(value).to_s
-        when nil
-          ''
-        else
-          value.to_s
-        end
+        text = case value
+               when String
+                 value
+               when Array
+                 parts = value.filter_map { |entry| extract_text_fragment(entry) }
+                 joined = parts.map(&:to_s).map(&:strip).reject(&:empty?).join("\n")
+                 joined.empty? ? value.to_s : joined
+               when Hash
+                 extract_text_fragment(value).to_s
+               when nil
+                 ''
+               else
+                 value.to_s
+               end
+        sanitize_text_input(text)
+      end
+
+      def sanitize_text_input(value)
+        text = value.to_s.dup
+        text = text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+        text = text.scrub('') unless text.valid_encoding?
+        text.delete("\u0000")
       end
 
       def normalize_tags_input(tags)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-apollo
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.2
 platform: ruby
 authors:
 - Esity
@@ -86,6 +86,8 @@ files:
 - lib/legion/apollo/local/migrations/001_create_local_knowledge.rb
 - lib/legion/apollo/local/migrations/002_create_graph_tables.rb
 - lib/legion/apollo/local/migrations/003_harden_graph_relationships.rb
+- lib/legion/apollo/local/migrations/004_add_versioning_tiers_inference.rb
+- lib/legion/apollo/local/migrations/005_add_raw_content_temporal_windows.rb
 - lib/legion/apollo/messages/access_boost.rb
 - lib/legion/apollo/messages/ingest.rb
 - lib/legion/apollo/messages/query.rb