lex-agentic-memory 0.1.38 → 0.1.39

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69de94b29aa73fa12a7adb17e3c114d5d01dd6cfd89f076d77ed053ee1c0e663
-  data.tar.gz: dc9a817d778c429e27b1225daf790f5aaf2c87c2ad11ce3237b34c24730717ce
+  metadata.gz: 89cf392450f3ff2b43d9ff1430c9cb4e8b108d0221b46aa4c2e3e064514c6200
+  data.tar.gz: 0343f1b89ae601f477939c5890fca5bcb2e8e4961f411049612731edf16ae43e
 SHA512:
-  metadata.gz: 68a898cb81e1e16dfbe001a150fd1c5254eac34dedd7477d17c13d360c19117ac4e8d82a402538b44d3d6f7f67bd67d6f87d4fc41d71005010c63d6cc7e58506
-  data.tar.gz: a52951edc85a09cb417eefafd057acd7be4ab723ced5b0a0fd65c0c89b8a9e6af5f1eebd9e5bdcc62b9154c421d311d3c0d1d4ec05d5934fcab5486eab21c928
+  metadata.gz: ca0b83053d099fd5e445427105023210371b4e4048961bfa4d3446b8ca1412902c8c6dbbcfaa0881d787d55ba5ec70fc5321023374c3a9c35b6718d99e9af8b8
+  data.tar.gz: 5c6e4acf09c5cafae4694253381e963316fe898e9c738bb9ae0143e9788e242e01b5dd442f3187ebb8f997b1a3ef095fdb1c1fd9a77c0ac2072c0e255247b860

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [0.1.39] - 2026-06-01
+### Fixed
+- `ErrorTracer` now guards against infinite recursion — if downstream trace storage triggers error/fatal logging, the `tracing?` thread-local flag prevents re-entry
+- `ErrorTracer` background worker thread now terminates cleanly — `setup` registers an `at_exit` hook, and a new `shutdown` method pushes the `:stop` sentinel to the queue (previously the thread ran forever with no shutdown path)
+- `HotTier.serialize_trace` now preserves all trace fields (was only serializing 9 out of 25+) — adds `base_decay_rate`, `emotional_valence`, `emotional_intensity`, `origin`, `source_agent_id`, `domain_tags`, `associated_traces`, `child_trace_ids`, `reinforcement_count`, `unresolved`, `consolidation_candidate`, `last_decayed`, `created_at`, `encryption_key_id`, `parent_trace_id`
+- `HotTier.deserialize_trace` reconstructs all fields with proper type coercion (arrays via JSON, booleans, integers, symbols, timestamps)
+- `PostgresStore#deserialize_trace` now reads `child_trace_ids` from the DB column instead of hardcoding `[]` — children are no longer silently lost on retrieval
+- `PostgresStore#serialize_trace` now writes `child_trace_ids` to the DB row
+- `PostgresStore#map_update_fields` now maps `child_trace_ids` to the `:child_trace_ids` column (was `nil`, silently dropping partial updates)
+- `PostgresStore#retrieve_by_domain` uses PostgreSQL JSONB `@>` containment operator for exact array-member matching instead of `LIKE` — eliminates full table scans and substring false positives (e.g., `%auth%` matching `oauth`)
+- `Consolidation#erase_by_type` now uses `batch_delete_by_type` on PostgresStore, avoiding loading all matching traces into Ruby memory before deleting one-by-one (O(100k) heap allocation → O(1) SQL delete)
+
 ## [0.1.38] - 2026-05-27
 ### Fixed
 - `PostgresStore#parse_json_or_raw` now requires matching start/end delimiters (`{}`/`[]`) before attempting JSON parse — eliminates 100k+/day error log spam from bracket-prefixed text content (e.g. `[trace_persistence] ...`) that triggered a self-amplifying feedback loop via RabbitMQ logging

data/lib/legion/extensions/agentic/memory/archaeology/helpers/archaeology_engine.rb

@@ -161,12 +161,13 @@ module Legion
               end
 
               def site_summary(site)
+                artifact_types = site.artifacts_found.each_with_object(Hash.new(0)) do |a, h|
+                  h[a.artifact_type] += 1
+                end
+
                 site.survey.merge(
                   depth_progress: depth_progress(site),
-                  artifact_types: site.artifacts_found
-                                      .each_with_object(Hash.new(0)) do |a, h|
-                                        h[a.artifact_type] += 1
-                                      end
+                  artifact_types: artifact_types
                 )
               end
 

data/lib/legion/extensions/agentic/memory/trace/helpers/error_tracer.rb

@@ -25,6 +25,7 @@ module Legion
                   @worker.name  = 'legion-error-tracer'
                   wrap_logging_methods
                   @active = true
+                  register_shutdown_hook
                   Legion::Logging.info '[memory] ErrorTracer active — errors/fatals will become episodic traces'
                 end
 
@@ -32,8 +33,25 @@ module Legion
                   @active == true
                 end
 
+                # Gracefully shut down the background worker and unregister the at_exit hook.
+                def shutdown
+                  return unless @active
+
+                  @write_queue&.push(:stop)
+                  @worker&.join(5) # wait up to 5 seconds for clean exit
+                  @active = false
+                  Legion::Logging.info '[memory] ErrorTracer shut down'
+                rescue StandardError
+                  @active = false
+                end
+
                 private
 
+                def register_shutdown_hook
+                  @shutdown_hook = proc { ErrorTracer.shutdown }
+                  at_exit(&@shutdown_hook)
+                end
+
                 def drain_queue
                   loop do
                     payload = @write_queue.pop
@@ -54,43 +72,55 @@ module Legion
                   Legion::Logging.define_singleton_method(:error) do |message = nil, &block|
                     message = block.call if message.nil? && block
                     original_error.call(message)
-                    ErrorTracer.send(:record_trace, message, :error) if message.is_a?(String)
+                    ErrorTracer.send(:record_trace, message, :error) if message.is_a?(String) && !ErrorTracer.send(:tracing?)
                   end
 
                   Legion::Logging.define_singleton_method(:fatal) do |message = nil, &block|
                     message = block.call if message.nil? && block
                     original_fatal.call(message)
-                    ErrorTracer.send(:record_trace, message, :fatal) if message.is_a?(String)
+                    ErrorTracer.send(:record_trace, message, :fatal) if message.is_a?(String) && !ErrorTracer.send(:tracing?)
                   end
                 end
 
+                def tracing?
+                  Thread.current[:error_tracer_active]
+                end
+
                 def record_trace(message, level)
                   return unless message.is_a?(String) && !message.empty?
 
-                  now = Time.now.utc
-                  key = "#{level}:#{message[0..100]}"
-
-                  @recent_mutex.synchronize do
-                    return if @recent[key] && (now - @recent[key]) < DEBOUNCE_WINDOW
-
-                    @recent[key] = now
-                    @recent.delete_if { |_, t| (now - t) > DEBOUNCE_WINDOW } if @recent.size > 500
+                  # Guard against infinite recursion if downstream logging triggers error/fatal
+                  return if tracing?
+
+                  Thread.current[:error_tracer_active] = true
+                  begin
+                    now = Time.now.utc
+                    key = "#{level}:#{message[0..100]}"
+
+                    @recent_mutex.synchronize do
+                      return if @recent[key] && (now - @recent[key]) < DEBOUNCE_WINDOW
+
+                      @recent[key] = now
+                      @recent.delete_if { |_, t| (now - t) > DEBOUNCE_WINDOW } if @recent.size > 500
+                    end
+
+                    component = message.match(/\A\[([^\]]+)\]/)&.captures&.first || 'unknown'
+                    valence   = level == :fatal ? FATAL_VALENCE : ERROR_VALENCE
+                    intensity = level == :fatal ? FATAL_INTENSITY : ERROR_INTENSITY
+
+                    @write_queue.push(
+                      type:                :episodic,
+                      content_payload:     message,
+                      domain_tags:         ['error', component.downcase],
+                      origin:              :direct_experience,
+                      emotional_valence:   valence,
+                      emotional_intensity: intensity,
+                      unresolved:          true,
+                      confidence:          0.9
+                    )
+                  ensure
+                    Thread.current[:error_tracer_active] = false
                   end
-
-                  component = message.match(/\A\[([^\]]+)\]/)&.captures&.first || 'unknown'
-                  valence   = level == :fatal ? FATAL_VALENCE : ERROR_VALENCE
-                  intensity = level == :fatal ? FATAL_INTENSITY : ERROR_INTENSITY
-
-                  @write_queue.push(
-                    type:                :episodic,
-                    content_payload:     message,
-                    domain_tags:         ['error', component.downcase],
-                    origin:              :direct_experience,
-                    emotional_valence:   valence,
-                    emotional_intensity: intensity,
-                    unresolved:          true,
-                    confidence:          0.9
-                  )
                 rescue StandardError
                   nil
                 end

data/lib/legion/extensions/agentic/memory/trace/helpers/hot_tier.rb

@@ -81,34 +81,101 @@ module Legion
               end
 
               # Serialize a trace hash to a string-only flat hash suitable for Redis HSET.
+              # All fields are preserved as strings; arrays/hashes are JSON-encoded.
               def serialize_trace(trace)
+                payload = trace[:content_payload] || trace[:content]
                 {
-                  'trace_id'        => trace[:trace_id].to_s,
-                  'trace_type'      => trace[:trace_type].to_s,
-                  'content_payload' => trace[:content_payload].to_s,
-                  'strength'        => trace[:strength].to_s,
-                  'peak_strength'   => trace[:peak_strength].to_s,
-                  'confidence'      => trace[:confidence].to_s,
-                  'storage_tier'    => 'hot',
-                  'partition_id'    => trace[:partition_id].to_s,
-                  'last_reinforced' => (trace[:last_reinforced] || Time.now).to_s
+                  'trace_id'                => trace[:trace_id].to_s,
+                  'trace_type'              => trace[:trace_type].to_s,
+                  'content_payload'         => payload.is_a?(Hash) || payload.is_a?(Array) ? Legion::JSON.dump(payload) : payload.to_s,
+                  'strength'                => trace[:strength].to_s,
+                  'peak_strength'           => trace[:peak_strength].to_s,
+                  'base_decay_rate'         => trace[:base_decay_rate].to_s,
+                  'confidence'              => trace[:confidence].to_s,
+                  'emotional_valence'       => trace[:emotional_valence].to_s,
+                  'emotional_intensity'     => trace[:emotional_intensity].to_s,
+                  'storage_tier'            => 'hot',
+                  'partition_id'            => trace[:partition_id].to_s,
+                  'origin'                  => trace[:origin].to_s,
+                  'source_agent_id'         => trace[:source_agent_id].to_s,
+                  'encryption_key_id'       => trace[:encryption_key_id].to_s,
+                  'parent_trace_id'         => trace[:parent_trace_id].to_s,
+                  'domain_tags'             => trace[:domain_tags].is_a?(Array) ? Legion::JSON.dump(trace[:domain_tags]) : '[]',
+                  'associated_traces'       => trace[:associated_traces].is_a?(Array) ? Legion::JSON.dump(trace[:associated_traces]) : '[]',
+                  'child_trace_ids'         => trace[:child_trace_ids].is_a?(Array) ? Legion::JSON.dump(trace[:child_trace_ids]) : '[]',
+                  'reinforcement_count'     => trace[:reinforcement_count].to_s,
+                  'unresolved'              => trace[:unresolved].to_s,
+                  'consolidation_candidate' => trace[:consolidation_candidate].to_s,
+                  'last_reinforced'         => (trace[:last_reinforced] || Time.now).to_s,
+                  'last_decayed'            => trace[:last_decayed].to_s,
+                  'created_at'              => trace[:created_at].to_s
                 }
               end
 
               # Deserialize a Redis string-hash back to a typed trace hash.
               def deserialize_trace(data)
                 {
-                  trace_id:        data['trace_id'],
-                  trace_type:      data['trace_type']&.to_sym,
-                  content_payload: data['content_payload'],
-                  strength:        data['strength']&.to_f,
-                  peak_strength:   data['peak_strength']&.to_f,
-                  confidence:      data['confidence']&.to_f,
-                  storage_tier:    :hot,
-                  partition_id:    data['partition_id'],
-                  last_reinforced: data['last_reinforced']
+                  trace_id:                data['trace_id'],
+                  trace_type:              data['trace_type']&.to_sym,
+                  content_payload:         parse_json_or_string(data['content_payload']),
+                  content:                 parse_json_or_string(data['content_payload']),
+                  strength:                data['strength']&.to_f,
+                  peak_strength:           data['peak_strength']&.to_f,
+                  base_decay_rate:         data['base_decay_rate']&.to_f,
+                  confidence:              data['confidence']&.to_f,
+                  emotional_valence:       data['emotional_valence'].to_f,
+                  emotional_intensity:     data['emotional_intensity'].to_f,
+                  storage_tier:            :hot,
+                  partition_id:            presence(data['partition_id']),
+                  origin:                  presence(data['origin'])&.to_sym,
+                  source_agent_id:         presence(data['source_agent_id']),
+                  encryption_key_id:       presence(data['encryption_key_id']),
+                  parent_trace_id:         presence(data['parent_trace_id']),
+                  domain_tags:             parse_json_array(data['domain_tags']),
+                  associated_traces:       parse_json_array(data['associated_traces']),
+                  child_trace_ids:         parse_json_array(data['child_trace_ids']),
+                  reinforcement_count:     data['reinforcement_count'].to_i,
+                  unresolved:              data['unresolved'] == 'true',
+                  consolidation_candidate: data['consolidation_candidate'] == 'true',
+                  last_reinforced:         data['last_reinforced'],
+                  last_decayed:            presence(data['last_decayed']),
+                  created_at:              presence(data['created_at'])
                 }
               end
+
+              # Parse a JSON array string safely; returns [] on failure.
+              def parse_json_array(raw)
+                return [] if raw.nil? || !raw.is_a?(String) || raw.strip.empty?
+
+                parsed = Legion::JSON.load(raw)
+                parsed.is_a?(Array) ? parsed : []
+              rescue StandardError => e
+                log.debug "[trace_persistence] parse_json_array: #{e.message}"
+                []
+              end
+
+              # Attempt to parse JSON, fall back to raw string.
+              def parse_json_or_string(raw)
+                return raw unless raw.is_a?(String)
+
+                stripped = raw.strip
+                return raw unless (stripped.start_with?('{') && stripped.end_with?('}')) ||
+                                  (stripped.start_with?('[') && stripped.end_with?(']'))
+
+                parsed = Legion::JSON.load(stripped)
+                parsed.is_a?(Hash) || parsed.is_a?(Array) ? parsed : raw
+              rescue StandardError => e
+                log.debug "[trace_persistence] parse_json_or_string: #{e.message}"
+                raw
+              end
+
+              # Return value only if it is a non-empty string.
+              def presence(value)
+                return nil unless value.is_a?(String)
+
+                stripped = value.strip
+                stripped.empty? ? nil : stripped
+              end
             end
           end
         end

data/lib/legion/extensions/agentic/memory/trace/helpers/postgres_store.rb

@@ -80,12 +80,22 @@ module Legion
                 []
               end
 
-              # Retrieve traces whose domain_tags column contains the given tag string.
+              # Retrieve traces whose domain_tags JSON array contains the given tag.
+              # Uses PostgreSQL JSON containment operator (@>) when available for exact array-member
+              # matching; falls back to LIKE for other adapters.
               def retrieve_by_domain(tag, min_strength: 0.0, limit: 50)
                 return [] unless db_ready?
 
-                rows = traces_ds
-                       .where(Sequel.like(:domain_tags, "%#{tag}%"))
+                ds = traces_ds
+                if db.adapter_scheme == :postgres
+                  # JSONB @> operator matches exact array elements, not substrings
+                  json_array = ::JSON.dump([tag])
+                  ds = ds.where(Sequel.lit("domain_tags::jsonb @> '#{json_array}'::jsonb"))
+                else
+                  # Fallback: substring match (imprecise but broadly compatible)
+                  ds = ds.where(Sequel.like(:domain_tags, "%#{tag}%"))
+                end
+                rows = ds
                        .where { strength >= min_strength }
                        .order(Sequel.desc(:strength))
                        .limit(limit)
@@ -330,6 +340,7 @@ module Legion
                   parent_trace_id:         sanitize_pg_string(trace[:parent_trace_id]),
                   encryption_key_id:       sanitize_pg_string(trace[:encryption_key_id]),
                   partition_id:            sanitize_pg_string(trace[:partition_id]),
+                  child_trace_ids:         sanitize_pg_string((trace[:child_trace_ids] || []).is_a?(Array) ? Legion::JSON.dump(trace[:child_trace_ids]) : '[]'),
                   created_at:              trace[:created_at] || Time.now.utc,
                   accessed_at:             Time.now.utc
                 }
@@ -360,7 +371,7 @@ module Legion
                   encryption_key_id:       row[:encryption_key_id],
                   associated_traces:       parse_json_array(row[:associations]),
                   parent_trace_id:         row[:parent_trace_id],
-                  child_trace_ids:         [],
+                  child_trace_ids:         parse_json_array(row[:child_trace_ids]),
                   unresolved:              row[:unresolved]              || false,
                   consolidation_candidate: row[:consolidation_candidate] || false
                 }
@@ -372,7 +383,7 @@ module Legion
                   content_payload:   :content,
                   associated_traces: :associations,
                   parent_trace_id:   :parent_trace_id,
-                  child_trace_ids:   nil # not stored as a column
+                  child_trace_ids:   :child_trace_ids
                 }
 
                 fields.each_with_object({}) do |(k, v), row|
@@ -382,7 +393,7 @@ module Legion
                   row[col] = case col
                              when :content
                                sanitize_pg_string(v.is_a?(Hash) ? Legion::JSON.dump(v) : v.to_s)
-                             when :associations
+                             when :associations, :child_trace_ids
                                sanitize_pg_string(v.is_a?(Array) ? Legion::JSON.dump(v) : '[]')
                              when :domain_tags
                                sanitize_pg_string(v.is_a?(Array) ? Legion::JSON.dump(v) : nil)
@@ -394,6 +405,22 @@ module Legion
                 end
               end
 
+              # Delete all traces of a given type in a single SQL statement,
+              # avoiding loading rows into Ruby memory.
+              def batch_delete_by_type(trace_type)
+                return 0 unless db_ready?
+
+                ids = traces_ds
+                      .where(trace_type: trace_type.to_s)
+                      .select_map(:trace_id)
+
+                ids.each { |tid| delete(tid) }
+                ids.size
+              rescue StandardError => e
+                log_warn("batch_delete_by_type failed: #{e.message}")
+                0
+              end
+
               def parse_json_or_raw(raw)
                 return raw unless raw.is_a?(String)
 

data/lib/legion/extensions/agentic/memory/trace/runners/consolidation.rb

@@ -117,9 +117,15 @@ module Legion
               def erase_by_type(type:, store: nil, **)
                 store ||= default_store
                 type = type.to_sym
-                traces = store.retrieve_by_type(type, min_strength: 0.0, limit: 100_000)
-                count = traces.size
-                traces.each { |t| store.delete(t[:trace_id]) }
+
+                count = if store.respond_to?(:batch_delete_by_type)
+                          store.batch_delete_by_type(type)
+                        else
+                          traces = store.retrieve_by_type(type, min_strength: 0.0, limit: 100_000)
+                          traces.each { |t| store.delete(t[:trace_id]) }
+                          traces.size
+                        end
+
                 persist_store(store) if count.positive?
                 log.info("[memory] erased #{count} traces of type=#{type}")
                 { erased: count, type: type }

data/lib/legion/extensions/agentic/memory/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Agentic
       module Memory
-        VERSION = '0.1.38'
+        VERSION = '0.1.39'
       end
     end
   end

data/spec/legion/extensions/agentic/memory/trace/helpers/postgres_store_spec.rb

@@ -36,6 +36,7 @@ RSpec.describe Legion::Extensions::Agentic::Memory::Trace::Helpers::PostgresStor
       String  :parent_trace_id, size: 36
       String  :encryption_key_id
       String  :partition_id
+      String  :child_trace_ids, text: true
       DateTime :created_at
       DateTime :accessed_at
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-agentic-memory
 version: !ruby/object:Gem::Version
-  version: 0.1.38
+  version: 0.1.39
 platform: ruby
 authors:
 - Esity