legion-apollo 0.5.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 37c9cddf7269f0a38272eeedb3fe3f3abb50440271b4ec9686820307233fe12f
-  data.tar.gz: 28962732f0a157e4e6f5abbf2ad494cd4d9fdb56f9d3723de3d074710143118e
+  metadata.gz: 9320cef3392467f9819e47c9cacc3893f45ffab9570825cb5d6b270ff3ce6a67
+  data.tar.gz: d38343aa96c9ba0b1d634f6e4d3850e22c899fe81c13550dce19809cec290a94
 SHA512:
-  metadata.gz: f1a1999ff0d3731d67af120efc58c215faeccc6a278ba02c11bfbc84f45dfb50520556e7966bdc6ee9a29a7744cdd61ac3cbc31f99b5112a0c7c3a82e2011141
-  data.tar.gz: f4421e4ace8b3e735ae78012e8579a052ee054f2f1a72160cf401f47fcf74fef3ab96925d567378804fadcd84f290c774cdaf73ecc24d94f8fdf23427b9b76bf
+  metadata.gz: 2dcaf3fc8fbf087b8e59c079e71921aebf8c53620e32d9cd8939a00981474747b78e978412d28a5c3ff4f204df01b9538f23a81c82a76a0d94afab8080219e46
+  data.tar.gz: 8d078c7e1c727bd23b08bfdb09af0abe7581296132d715b7ed7c4c5205d4645c004f1f980af6546239961c6649e9433e66a725341e2275e3cdb2e1de7f1687a3

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Changelog
 
+## [0.5.2] - 2026-04-27
+
+### Added
+- Store `raw_content` alongside indexed `content` in Apollo Local so callers can preserve verbatim source text separately from retrieval text (#25, #26)
+- Add `valid_from`/`valid_to` temporal windows and `as_of:` query filtering for local knowledge entries (#27)
+
+### Fixed
+- Sanitize Apollo ingest and query text by scrubbing invalid UTF-8 and removing null bytes before routing to local or global backends (#29)
+
+## [0.5.1] - 2026-04-27
+
+### Fixed
+- Guard Apollo Local tag queries and promotion against nil, shutdown, or unavailable local DB connections before SQL and Ruby fallback paths (#30)
+
 ## [0.5.0] - 2026-04-18
 
 ### Added

data/README.md

@@ -1,10 +1,10 @@
 # legion-apollo
 
-Apollo client library for the LegionIO framework.
+Apollo is the LegionIO knowledge client. It gives extensions one API for writing, retrieving, and merging knowledge across the global Apollo service and the node-local SQLite store.
 
-**Version**: 0.3.2
+**Version**: 0.5.2
 
-Provides `query`, `ingest`, and `retrieve` with smart routing: co-located lex-apollo service, RabbitMQ transport, or graceful failure. Supports a node-local SQLite knowledge store (`Apollo::Local`) that mirrors the same API without requiring any remote infrastructure.
+`legion-apollo` provides `query`, `ingest`, and `retrieve` with smart routing: co-located `lex-apollo`, RabbitMQ transport, node-local SQLite, or graceful failure. `Apollo::Local` mirrors the same public API for offline and low-latency retrieval without requiring remote infrastructure.
 
 ## Usage
 
@@ -21,6 +21,24 @@ results = Legion::Apollo.query(text: 'local note', scope: :local)
 
 # Query both and merge (deduped by content hash, ranked by confidence)
 results = Legion::Apollo.query(text: 'ruby', scope: :all)
+
+# Preserve verbatim source text separately from indexed retrieval content
+Legion::Apollo.ingest(
+  content: 'Summarized policy note for search',
+  raw_content: 'Exact source text from the original record',
+  tags: %w[policy source],
+  scope: :local
+)
+
+# Query the local store as it was valid at a point in time
+Legion::Apollo.ingest(
+  content: 'Policy version active in Q2',
+  tags: %w[policy],
+  valid_from: '2026-04-01T00:00:00.000Z',
+  valid_to: '2026-06-30T23:59:59.999Z',
+  scope: :local
+)
+results = Legion::Apollo.query(text: 'policy', scope: :local, as_of: '2026-05-01T00:00:00.000Z')
 ```
 
 ## Scopes
@@ -37,9 +55,12 @@ results = Legion::Apollo.query(text: 'ruby', scope: :all)
 
 Features:
 - Content-hash dedup (MD5 of normalized content)
+- `raw_content` preservation for verbatim source text
+- `valid_from` / `valid_to` temporal windows with `as_of:` query filtering
 - Optional LLM embeddings (1024-dim) with cosine rerank when `Legion::LLM.can_embed?`
 - TTL expiry (default 5-year retention)
 - FTS5 full-text search with `ILIKE` fallback
+- Null-byte removal and invalid UTF-8 scrubbing before persistence or backend routing
 
 ## Configuration
 

data/lib/legion/apollo/local/migrations/005_add_raw_content_temporal_windows.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    alter_table(:local_knowledge) do
+      add_column :raw_content, :text, null: true
+      add_column :valid_from, String, null: true
+      add_column :valid_to, String, null: true
+
+      add_index :valid_from, name: :idx_local_knowledge_valid_from
+      add_index :valid_to, name: :idx_local_knowledge_valid_to
+    end
+  end
+
+  down do
+    alter_table(:local_knowledge) do
+      drop_column :raw_content
+      drop_column :valid_from
+      drop_column :valid_to
+    end
+  end
+end

data/lib/legion/apollo/local.rb

@@ -99,18 +99,19 @@ module Legion
           limit ||= local_setting(:default_limit, 5)
           min_confidence ||= local_setting(:min_confidence, 0.3)
           multiplier = local_setting(:fts_candidate_multiplier, 3)
+          as_of = normalize_temporal_value(opts[:as_of])
           log.info do
             "Apollo::Local query executing text_length=#{text.to_s.length} " \
               "limit=#{limit} min_confidence=#{min_confidence} tag_count=#{Array(tags).size}"
           end
           log.debug { "Apollo::Local query limit=#{limit} min_confidence=#{min_confidence} tags=#{Array(tags).size}" }
 
-          candidates = fts_search(text, limit: limit * multiplier)
+          candidates = fts_search(text, limit: limit * multiplier, as_of: as_of)
           include_inferences = opts.fetch(:include_inferences, true)
           include_history = opts.fetch(:include_history, false)
           candidates = filter_candidates(candidates, min_confidence: min_confidence, tags: tags,
-                                                     include_inferences: include_inferences,
-                                                     include_history: include_history)
+                                                     options: { include_inferences: include_inferences,
+                                                                include_history: include_history, as_of: as_of })
           candidates = cosine_rerank(text, candidates) if can_rerank?
           results = candidates.first(limit)
 
@@ -159,10 +160,11 @@ module Legion
         end
 
         def query_by_tags(tags:, limit: 50) # rubocop:disable Metrics/MethodLength
-          return { success: false, error: :not_started } unless started?
-
+          connection = local_db_connection
           tags = normalize_tags_input(tags)
-          results = query_by_tags_via_sql(tags: tags, limit: limit)
+          return { success: false, error: :not_started } unless local_db_usable?(connection)
+
+          results = query_by_tags_via_sql(connection, tags: tags, limit: limit)
 
           log.info { "Apollo::Local query_by_tags completed tag_count=#{tags.size} count=#{results.size}" }
           { success: true, results: results, count: results.size }
@@ -178,11 +180,13 @@ module Legion
         end
 
         def promote_to_global(tags:, min_confidence: 0.6) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
-          return { success: false, error: :not_started } unless started?
+          return { success: false, error: :not_started } unless local_db_usable?(local_db_connection)
 
           tags = normalize_tags_input(tags)
           entries = query_by_tags(tags: tags)
-          unless entries[:success] && entries[:results]&.any?
+          return entries unless entries[:success]
+
+          unless entries[:results]&.any?
             log.info { "Apollo::Local promote_to_global skipped tag_count=#{tags.size} reason=no_entries" }
             return { success: true, promoted: 0 }
           end
@@ -200,6 +204,7 @@ module Legion
             end
             result = Legion::Apollo.ingest(
               content:        entry[:content],
+              raw_content:    entry[:raw_content] || entry[:content],
               tags:           entry_tags + ['promoted_from_local'],
               source_channel: 'local_promotion',
               submitted_by:   "node:#{hostname}",
@@ -337,6 +342,26 @@ module Legion
           Legion::Data::Local.connection
         end
 
+        def local_db_connection
+          return nil unless started? && data_local_available?
+
+          db
+        rescue StandardError => e
+          handle_exception(e, level: :debug, operation: 'apollo.local.local_db_connection')
+          nil
+        end
+
+        def local_db_usable?(connection)
+          return false unless started? && connection
+          return false if connection.respond_to?(:closed?) && connection.closed?
+
+          connection.test_connection if connection.respond_to?(:test_connection)
+          true
+        rescue StandardError => e
+          handle_exception(e, level: :debug, operation: 'apollo.local.local_db_usable')
+          false
+        end
+
         def content_hash(content)
           normalized = content.to_s.strip.downcase.gsub(/\s+/, ' ')
           Digest::MD5.hexdigest(normalized)
@@ -396,9 +421,12 @@ module Legion
 
             result = ingest(
               content:        entry[:content],
+              raw_content:    entry[:raw_content] || entry[:content],
               tags:           clean_tags,
               confidence:     ((entry[:confidence] || 0.5) * 0.9).round(10),
-              source_channel: 'global_hydration'
+              source_channel: 'global_hydration',
+              valid_from:     entry[:valid_from],
+              valid_to:       entry[:valid_to]
             )
             hydrated += 1 if result[:success]
           end
@@ -408,6 +436,8 @@ module Legion
         end
 
         def ingest_without_lock(content:, tags:, **opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+          content = normalize_text_input(content)
+          raw_content = normalize_text_input(opts.key?(:raw_content) ? opts[:raw_content] : content)
           hash = content_hash(content)
           return deduplicated_ingest(hash) if duplicate?(hash)
 
@@ -417,9 +447,11 @@ module Legion
           end
           log.debug { "Apollo::Local ingest hash=#{hash} tags=#{Array(tags).size} source_channel=#{opts[:source_channel]}" }
 
-          row = build_ingest_row(content: content, hash: hash, tags: tags, **opts)
-          id = persist_ingest_row(row, opts)
-          mark_parent_superseded(opts[:parent_knowledge_id]) if opts[:parent_knowledge_id]
+          metadata = opts.dup
+          metadata.delete(:raw_content)
+          row = build_ingest_row(content: content, raw_content: raw_content, hash: hash, tags: tags, **metadata)
+          id = persist_ingest_row(row, metadata)
+          mark_parent_superseded(metadata[:parent_knowledge_id]) if metadata[:parent_knowledge_id]
 
           log.info { "Apollo::Local ingest stored id=#{id} hash=#{hash}" }
           { success: true, mode: :local, id: id }
@@ -429,22 +461,56 @@ module Legion
           deduplicated_ingest(hash)
         end
 
-        def build_ingest_row(content:, hash:, tags:, **opts) # rubocop:disable Metrics/MethodLength
+        def build_ingest_row(content:, raw_content:, hash:, tags:, **opts) # rubocop:disable Metrics/MethodLength
           is_inference = opts[:is_inference] == true
           default_confidence = is_inference ? Legion::Apollo::Helpers::Confidence::INITIAL_INFERENCE_CONFIDENCE : 1.0
+          ingest_metadata_columns(
+            content:            content,
+            raw_content:        raw_content,
+            hash:               hash,
+            tags:               tags,
+            opts:               opts,
+            is_inference:       is_inference,
+            default_confidence: default_confidence
+          ).merge(embedding_columns(content, opts)).merge(timestamp_columns)
+        end
+
+        def ingest_metadata_columns(context)
+          ingest_base_columns(context)
+            .merge(ingest_lineage_columns(context[:opts]))
+            .merge(ingest_temporal_columns(context[:opts]))
+        end
+
+        def ingest_base_columns(context)
+          opts = context[:opts]
+          {
+            content:      context[:content],
+            raw_content:  context[:raw_content],
+            content_hash: context[:hash],
+            tags:         serialized_tags(context[:tags]),
+            confidence:   opts[:confidence] || context[:default_confidence],
+            is_inference: context[:is_inference]
+          }.merge(ingest_source_columns(opts))
+        end
+
+        def ingest_source_columns(opts)
+          { source_channel: opts[:source_channel], source_agent: opts[:source_agent],
+            submitted_by: opts[:submitted_by] }
+        end
+
+        def ingest_lineage_columns(opts)
           {
-            content:             content,
-            content_hash:        hash,
-            tags:                serialized_tags(tags),
-            source_channel:      opts[:source_channel],
-            source_agent:        opts[:source_agent],
-            submitted_by:        opts[:submitted_by],
-            confidence:          opts[:confidence] || default_confidence,
-            is_inference:        is_inference,
             forget_reason:       opts[:forget_reason],
             parent_knowledge_id: opts[:parent_knowledge_id],
             supersession_type:   opts[:supersession_type]
-          }.merge(embedding_columns(content, opts)).merge(timestamp_columns)
+          }
+        end
+
+        def ingest_temporal_columns(opts)
+          {
+            valid_from: normalize_temporal_value(opts[:valid_from]),
+            valid_to:   normalize_temporal_value(opts[:valid_to])
+          }
         end
 
         def persist_ingest_row(row, opts = {})
@@ -539,41 +605,41 @@ module Legion
           Legion::JSON.dump(normalize_tags_input(tags))
         end
 
-        def fts_search(text, limit:) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+        def fts_search(text, limit:, as_of: nil) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
           now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
-          if text.to_s.strip.empty?
-            return db[:local_knowledge]
-                   .where(Sequel.lit('expires_at > ?', now))
-                   .limit(limit)
-                   .all
-          end
+          return active_knowledge_dataset(now: now, as_of: as_of).limit(limit).all if text.to_s.strip.empty?
 
           tokens = text.to_s.scan(/[\p{L}\p{N}_]+/)
-          return ilike_search(text, now: now, limit: limit) if tokens.empty?
+          return ilike_search(text, now: now, limit: limit, as_of: as_of) if tokens.empty?
 
           escaped = tokens.map { |t| %("#{t}") }.join(' ')
+          temporal_sql, temporal_params = temporal_window_sql(as_of, table_alias: 'lk')
           db.fetch(
             'SELECT lk.* FROM local_knowledge lk ' \
             'INNER JOIN local_knowledge_fts fts ON lk.id = fts.rowid ' \
-            'WHERE local_knowledge_fts MATCH ? AND lk.expires_at > ? ORDER BY fts.rank LIMIT ?',
-            escaped, now, limit
+            "WHERE local_knowledge_fts MATCH ? AND lk.expires_at > ?#{temporal_sql} " \
+            'ORDER BY fts.rank LIMIT ?',
+            escaped, now, *temporal_params, limit
           ).all
         rescue StandardError => e
           handle_exception(e, level: :debug, operation: 'apollo.local.fts_search', limit: limit, fallback: :ilike)
-          ilike_search(text, now: Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ'), limit: limit)
+          ilike_search(text, now: Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ'), limit: limit, as_of: as_of)
         end
 
-        def ilike_search(text, now:, limit:)
+        def ilike_search(text, now:, limit:, as_of: nil)
           safe_text = text.to_s.gsub('\\', '\\\\\\\\').gsub('%', '\%').gsub('_', '\_')
-          db[:local_knowledge]
-            .where(Sequel.lit('expires_at > ?', now))
+          active_knowledge_dataset(now: now, as_of: as_of)
             .where(Sequel.lit("content LIKE ? ESCAPE '\\' COLLATE NOCASE", "%#{safe_text}%"))
             .limit(limit)
             .all
         end
 
-        def filter_candidates(candidates, min_confidence:, tags:, include_inferences: true, include_history: false) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/AbcSize
+        def filter_candidates(candidates, min_confidence:, tags:, options: {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/AbcSize
+          include_inferences = options.fetch(:include_inferences, true)
+          include_history = options.fetch(:include_history, false)
+          as_of = options[:as_of]
           candidates = candidates.select { |c| (c[:confidence] || 0) >= min_confidence }
+          candidates = candidates.select { |c| temporally_valid?(c, as_of) }
           candidates = candidates.reject { |c| [1, true].include?(c[:is_inference]) } unless include_inferences
           unless include_history
             candidates = candidates.select { |c| c[:is_latest].nil? || c[:is_latest] == 1 || c[:is_latest] == true }
@@ -588,6 +654,36 @@ module Legion
           candidates
         end
 
+        def active_knowledge_dataset(now:, as_of: nil)
+          apply_temporal_window(db[:local_knowledge].where(Sequel.lit('expires_at > ?', now)), as_of)
+        end
+
+        def apply_temporal_window(dataset, as_of)
+          return dataset if as_of.to_s.empty?
+
+          dataset.where(
+            Sequel.lit('(valid_from IS NULL OR valid_from <= ?) AND (valid_to IS NULL OR valid_to >= ?)', as_of, as_of)
+          )
+        end
+
+        def temporal_window_sql(as_of, table_alias:)
+          return ['', []] if as_of.to_s.empty?
+
+          [
+            " AND (#{table_alias}.valid_from IS NULL OR #{table_alias}.valid_from <= ?) " \
+            "AND (#{table_alias}.valid_to IS NULL OR #{table_alias}.valid_to >= ?)",
+            [as_of, as_of]
+          ]
+        end
+
+        def temporally_valid?(row, as_of)
+          return true if as_of.to_s.empty?
+
+          valid_from = row[:valid_from]
+          valid_to = row[:valid_to]
+          (valid_from.nil? || valid_from <= as_of) && (valid_to.nil? || valid_to >= as_of)
+        end
+
         def parse_tags(tags_json)
           return [] if tags_json.nil? || tags_json.empty?
 
@@ -656,6 +752,17 @@ module Legion
           value.to_s
         end
 
+        def normalize_temporal_value(value)
+          return nil if value.nil?
+
+          text = value.respond_to?(:utc) ? value.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ') : value.to_s.strip
+          return nil if text.empty?
+
+          Time.parse(text).utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
+        rescue StandardError
+          text
+        end
+
         def normalize_tags_input(tags)
           Legion::Apollo::Helpers::TagNormalizer.normalize(Array(tags)).first(max_tags_limit)
         rescue StandardError => e
@@ -674,9 +781,9 @@ module Legion
           Legion::Apollo::Helpers::TagNormalizer::MAX_TAGS
         end
 
-        def query_by_tags_via_sql(tags:, limit:) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        def query_by_tags_via_sql(connection, tags:, limit:) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
           now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
-          dataset = db[:local_knowledge].where(Sequel.lit('expires_at > ?', now))
+          dataset = connection[:local_knowledge].where(Sequel.lit('expires_at > ?', now))
 
           Array(tags).map(&:to_s).each do |tag|
             dataset = dataset.where(
@@ -696,11 +803,15 @@ module Legion
             tag_count: Array(tags).size,
             limit:     limit
           )
-          query_by_tags_via_ruby(tags: tags, limit: limit)
+          raise unless local_db_usable?(connection)
+
+          query_by_tags_via_ruby(connection, tags: tags, limit: limit)
         end
 
-        def query_by_tags_via_ruby(tags:, limit:)
-          candidates = db[:local_knowledge]
+        def query_by_tags_via_ruby(connection, tags:, limit:)
+          raise Sequel::DatabaseConnectionError, 'local database unavailable' unless local_db_usable?(connection)
+
+          candidates = connection[:local_knowledge]
                        .where(Sequel.lit('expires_at > ?', Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')))
                        .all
 
@@ -711,7 +822,7 @@ module Legion
         end
 
         def update_upsert_entry(existing, content, tags_json, opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
-          content = content.to_s
+          content = normalize_text_input(content)
           new_hash = content_hash(content)
           embedding, embedded_at = generate_embedding(content)
           now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')

data/lib/legion/apollo/routes.rb

@@ -68,6 +68,7 @@ module Legion
             agent_id:           body[:agent_id] || 'api',
             scope:              normalize_scope(body[:scope]),
             tier:               body[:tier]&.to_sym,
+            as_of:              body[:as_of],
             include_inferences: body.fetch(:include_inferences, true),
             include_history:    body.fetch(:include_history, false)
           )
@@ -88,6 +89,7 @@ module Legion
           tags = Legion::Apollo::Helpers::TagNormalizer.normalize(Array(body[:tags])).first(effective_max_tags)
           result = Legion::Apollo.ingest(
             content:             body[:content],
+            raw_content:         body[:raw_content],
             content_type:        body[:content_type] || :observation,
             tags:                tags,
             source_agent:        body[:source_agent] || 'api',
@@ -99,6 +101,8 @@ module Legion
             is_inference:        body[:is_inference] == true,
             forget_reason:       body[:forget_reason],
             expires_at:          body[:expires_at],
+            valid_from:          body[:valid_from],
+            valid_to:            body[:valid_to],
             parent_knowledge_id: body[:parent_knowledge_id],
             supersession_type:   body[:supersession_type],
             source_uri:          body[:source_uri],

data/lib/legion/apollo/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module Apollo
-    VERSION = '0.5.0'
+    VERSION = '0.5.2'
   end
 end

data/lib/legion/apollo.rb

@@ -97,9 +97,11 @@ module Legion
         return not_started_error unless started?
 
         normalized_tags = normalize_tags_input(tags)
-        payload = { content: content, tags: normalized_tags, **opts }
+        normalized_content = normalize_text_input(content)
+        normalized_raw_content = normalize_text_input(opts.key?(:raw_content) ? opts[:raw_content] : content)
+        payload = { **opts, content: normalized_content, raw_content: normalized_raw_content, tags: normalized_tags }
         log.info do
-          "Apollo ingest requested scope=#{scope} content_length=#{content.to_s.length} " \
+          "Apollo ingest requested scope=#{scope} content_length=#{payload[:content].to_s.length} " \
             "tags=#{payload[:tags].size} source_channel=#{payload[:source_channel]}"
         end
         log.debug do
@@ -289,7 +291,8 @@ module Legion
             "limit=#{payload[:limit]}"
         end
         result = Legion::Apollo::Local.query(**payload.slice(:text, :limit, :min_confidence, :tags,
-                                                             :tier, :include_inferences, :include_history))
+                                                             :tier, :include_inferences, :include_history,
+                                                             :as_of))
         return result unless result[:success]
 
         entries = normalize_local_entries(Array(result[:results]))
@@ -324,7 +327,8 @@ module Legion
         if Legion::Apollo::Local.started?
           attempted = true
           local = Legion::Apollo::Local.query(**payload.slice(:text, :limit, :min_confidence, :tags,
-                                                              :tier, :include_inferences, :include_history))
+                                                              :tier, :include_inferences, :include_history,
+                                                              :as_of))
           if local[:success]
             any_success = true
             entries.concat(normalize_local_entries(Array(local[:results]))) if local[:results]
@@ -377,18 +381,29 @@ module Legion
                  else
                    Array(e[:tags])
                  end
-          { id: e[:id], content: e[:content], content_hash: hash,
-            confidence: e[:confidence] || 0.5, content_type: 'fact', tags: tags, source: :local }
+          { id: e[:id], content: e[:content], raw_content: e[:raw_content] || e[:content], content_hash: hash,
+            confidence: e[:confidence] || 0.5, content_type: 'fact', tags: tags, source: :local,
+            valid_from: e[:valid_from], valid_to: e[:valid_to] }
         end
       end
 
       def normalize_global_entries(entries)
-        entries.map do |e|
-          hash = e[:content_hash] || Digest::MD5.hexdigest(e[:content].to_s.strip.downcase.gsub(/\s+/, ' '))
-          { id: e[:id], content: e[:content], content_hash: hash,
-            confidence: e[:confidence] || 0.5, content_type: e[:content_type] || 'fact',
-            tags: Array(e[:tags]), source: :global }
-        end
+        entries.map { |entry| normalize_global_entry(entry) }
+      end
+
+      def normalize_global_entry(entry)
+        { id: entry[:id], content: entry[:content], raw_content: normalized_raw_content(entry),
+          content_hash: normalized_content_hash(entry), confidence: entry[:confidence] || 0.5,
+          content_type: entry[:content_type] || 'fact', tags: Array(entry[:tags]), source: :global,
+          valid_from: entry[:valid_from], valid_to: entry[:valid_to] }
+      end
+
+      def normalized_raw_content(entry)
+        entry[:raw_content] || entry[:content]
+      end
+
+      def normalized_content_hash(entry)
+        entry[:content_hash] || Digest::MD5.hexdigest(entry[:content].to_s.strip.downcase.gsub(/\s+/, ' '))
       end
 
       def dedup_and_rank(entries, limit:)
@@ -470,20 +485,28 @@ module Legion
       end
 
       def normalize_text_input(value) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/MethodLength
-        case value
-        when String
-          value
-        when Array
-          parts = value.filter_map { |entry| extract_text_fragment(entry) }
-          joined = parts.map(&:to_s).map(&:strip).reject(&:empty?).join("\n")
-          joined.empty? ? value.to_s : joined
-        when Hash
-          extract_text_fragment(value).to_s
-        when nil
-          ''
-        else
-          value.to_s
-        end
+        text = case value
+               when String
+                 value
+               when Array
+                 parts = value.filter_map { |entry| extract_text_fragment(entry) }
+                 joined = parts.map(&:to_s).map(&:strip).reject(&:empty?).join("\n")
+                 joined.empty? ? value.to_s : joined
+               when Hash
+                 extract_text_fragment(value).to_s
+               when nil
+                 ''
+               else
+                 value.to_s
+               end
+        sanitize_text_input(text)
+      end
+
+      def sanitize_text_input(value)
+        text = value.to_s.dup
+        text = text.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+        text = text.scrub('') unless text.valid_encoding?
+        text.delete("\u0000")
       end
 
       def normalize_tags_input(tags)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-apollo
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
 platform: ruby
 authors:
 - Esity
@@ -87,6 +87,7 @@ files:
 - lib/legion/apollo/local/migrations/002_create_graph_tables.rb
 - lib/legion/apollo/local/migrations/003_harden_graph_relationships.rb
 - lib/legion/apollo/local/migrations/004_add_versioning_tiers_inference.rb
+- lib/legion/apollo/local/migrations/005_add_raw_content_temporal_windows.rb
 - lib/legion/apollo/messages/access_boost.rb
 - lib/legion/apollo/messages/ingest.rb
 - lib/legion/apollo/messages/query.rb