lex-agentic-memory 0.1.9 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e7d4306cac20ea6106de6c5961c23651a3a891d1891ae986869904141e8374c
-  data.tar.gz: 77a5a10d16f09dd6c9f3057219435c676909529740e2d6d429932f542d601e46
+  metadata.gz: 44a4999991835bdd28a7189717f6a53f38928644a86c7f653517331687659901
+  data.tar.gz: b3aeb50fddc4b82580430829ec924549690b8dd6171be74ff9213db40b135888
 SHA512:
-  metadata.gz: d1d4ad0cb84988d8d76965f45017429caa2d3e77d6bf184673d4353fc249547d61da10853c297891c496c56862d93fa010130f53c28fbd054b61cff91bfb6c0f
-  data.tar.gz: 86ff6e678ebc1d9f3690966cbef2a3e6f90fb1635eaa78b0911734cf2a3dd230b63672365c9c1e87cf4a3081513cd05d4e2cf74258b969978f9d8512a4347bc2
+  metadata.gz: 32930b8b8092925f0793eef8ae8b1a6764d50cbc9692bb2952c46fb969137b41609eed96ecb455fdc7d0518373ebd912075eff8ac47c4deb8e97e1fc9176962b
+  data.tar.gz: a142cbf70545c4f80717ddda3d81abab26a1cc2cf5d8e40584625a416629d1551991d716fc6a9592fff194b523b0ebed2963be24fa414039cf11bdb3c934e60a

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## [0.1.11] - 2026-03-25
+
+### Added
+- `Helpers::HotTier` module: Redis hot-tier cache in front of PostgresStore using `Legion::Cache::RedisHash`. Stores traces as Redis hashes with 24-hour TTL, maintains a sorted-set index per tenant, and provides `cache_trace`, `fetch_trace`, and `evict_trace` operations
+- `PostgresStore#retrieve` checks hot tier first; falls through to DB on miss and populates hot tier on DB hit
+- `PostgresStore#store` writes through to hot tier after successful DB write
+- `PostgresStore#delete` evicts from hot tier before DB delete
+- `PostgresStore#update` evicts stale hot-tier entry after DB update
+- 32 new specs covering HotTier interface, serialize/deserialize round-trip, availability guard, and all four PostgresStore integration points
+
+## [0.1.10] - 2026-03-25
+
+### Added
+- `Helpers::PostgresStore`: write-through durable store backed by Legion::Data (PostgreSQL or MySQL), scoped by tenant_id. Implements full store interface: store, retrieve, retrieve_by_type, retrieve_by_domain, all_traces, delete, update, record_coactivation, associations_for, walk_associations, delete_lowest_confidence, delete_least_recently_used, firmware_traces, flush (no-op), db_ready?
+- `create_store` in `Trace` module now selects PostgresStore when a PostgreSQL or MySQL connection is available with both required tables; falls back to CacheStore or in-memory Store
+- `postgres_available?` and `resolve_tenant_id` private helpers on `Trace` module
+- 46 new specs covering all PostgresStore methods using an in-memory SQLite DB
+
 ## [0.1.9] - 2026-03-25
 
 ### Added

data/lib/legion/extensions/agentic/memory/trace/helpers/hot_tier.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Agentic
+      module Memory
+        module Trace
+          module Helpers
+            module HotTier
+              HOT_TTL = 86_400 # 24 hours
+
+              module_function
+
+              # Cache a trace in the Redis hot tier.
+              def cache_trace(trace, tenant_id: nil)
+                return unless available?
+
+                tid = tenant_id || trace[:partition_id]
+                key = trace_key(tid, trace[:trace_id])
+                data = serialize_trace(trace)
+                Legion::Cache::RedisHash.hset(key, data)
+                Legion::Cache::RedisHash.expire(key, HOT_TTL)
+
+                index_key = "legion:tier:hot:#{tid}"
+                Legion::Cache::RedisHash.zadd(index_key, Time.now.to_f, trace[:trace_id])
+              end
+
+              # Fetch a trace from the hot tier. Returns a deserialized trace hash or nil on miss.
+              def fetch_trace(trace_id, tenant_id: nil)
+                return nil unless available?
+
+                key = trace_key(tenant_id, trace_id)
+                data = Legion::Cache::RedisHash.hgetall(key)
+                return nil if data.nil? || data.empty?
+
+                deserialize_trace(data)
+              end
+
+              # Evict a trace from the hot tier and remove it from the sorted-set index.
+              def evict_trace(trace_id, tenant_id: nil)
+                return unless available?
+
+                key = trace_key(tenant_id, trace_id)
+                Legion::Cache.delete(key)
+
+                index_key = "legion:tier:hot:#{tenant_id}"
+                Legion::Cache::RedisHash.zrem(index_key, trace_id)
+              end
+
+              # Returns true when the RedisHash module is loaded and Redis is reachable.
+              def available?
+                defined?(Legion::Cache::RedisHash) &&
+                  Legion::Cache::RedisHash.redis_available?
+              rescue StandardError
+                false
+              end
+
+              # Build the namespaced Redis key for a trace.
+              def trace_key(tenant_id, trace_id)
+                "legion:trace:#{tenant_id}:#{trace_id}"
+              end
+
+              # Serialize a trace hash to a string-only flat hash suitable for Redis HSET.
+              def serialize_trace(trace)
+                {
+                  'trace_id'        => trace[:trace_id].to_s,
+                  'trace_type'      => trace[:trace_type].to_s,
+                  'content_payload' => trace[:content_payload].to_s,
+                  'strength'        => trace[:strength].to_s,
+                  'peak_strength'   => trace[:peak_strength].to_s,
+                  'confidence'      => trace[:confidence].to_s,
+                  'storage_tier'    => 'hot',
+                  'partition_id'    => trace[:partition_id].to_s,
+                  'last_reinforced' => (trace[:last_reinforced] || Time.now).to_s
+                }
+              end
+
+              # Deserialize a Redis string-hash back to a typed trace hash.
+              def deserialize_trace(data)
+                {
+                  trace_id:        data['trace_id'],
+                  trace_type:      data['trace_type']&.to_sym,
+                  content_payload: data['content_payload'],
+                  strength:        data['strength']&.to_f,
+                  peak_strength:   data['peak_strength']&.to_f,
+                  confidence:      data['confidence']&.to_f,
+                  storage_tier:    :hot,
+                  partition_id:    data['partition_id'],
+                  last_reinforced: data['last_reinforced']
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/agentic/memory/trace/helpers/postgres_store.rb

@@ -0,0 +1,393 @@
+# frozen_string_literal: true
+
+require_relative 'hot_tier'
+
+module Legion
+  module Extensions
+    module Agentic
+      module Memory
+        module Trace
+          module Helpers
+            # Write-through durable store backed by Legion::Data (PostgreSQL or MySQL).
+            # All writes go directly to the database — no in-memory dirty tracking, no flush.
+            # Scoped by tenant_id so multiple agents can share the same DB tables safely.
+            class PostgresStore
+              TRACES_TABLE = :memory_traces
+              ASSOCIATIONS_TABLE = :memory_associations
+
+              def initialize(tenant_id: nil)
+                @tenant_id = tenant_id
+              end
+
+              # Store (upsert) a trace by trace_id.
+              # Returns the trace_id on success, nil if the DB is not ready.
+              def store(trace)
+                return nil unless db_ready?
+
+                row = serialize_trace(trace)
+                begin
+                  db[TRACES_TABLE].insert_conflict(:replace).insert(row)
+                rescue Sequel::UniqueConstraintViolation
+                  db[TRACES_TABLE].where(trace_id: trace[:trace_id]).update(row.except(:trace_id))
+                end
+                HotTier.cache_trace(trace, tenant_id: @tenant_id) if HotTier.available?
+                trace[:trace_id]
+              rescue StandardError => e
+                log_warn("store failed: #{e.message}")
+                nil
+              end
+
+              # Retrieve a single trace by trace_id (tenant-scoped).
+              # Checks the Redis hot tier first; falls through to DB on a miss and caches the result.
+              # Returns a trace hash or nil.
+              def retrieve(trace_id)
+                if HotTier.available?
+                  cached = HotTier.fetch_trace(trace_id, tenant_id: @tenant_id)
+                  return cached if cached
+                end
+
+                return nil unless db_ready?
+
+                row = traces_ds.where(trace_id: trace_id).first
+                trace = row ? deserialize_trace(row) : nil
+                HotTier.cache_trace(trace, tenant_id: @tenant_id) if HotTier.available? && trace
+                trace
+              rescue StandardError => e
+                log_warn("retrieve failed: #{e.message}")
+                nil
+              end
+
+              # Retrieve traces by type, ordered by strength descending.
+              def retrieve_by_type(type, limit: 100, min_strength: 0.0)
+                return [] unless db_ready?
+
+                rows = traces_ds
+                       .where(trace_type: type.to_s)
+                       .where { strength >= min_strength }
+                       .order(Sequel.desc(:strength))
+                       .limit(limit)
+                       .all
+                rows.map { |r| deserialize_trace(r) }
+              rescue StandardError => e
+                log_warn("retrieve_by_type failed: #{e.message}")
+                []
+              end
+
+              # Retrieve traces whose domain_tags column contains the given tag string.
+              def retrieve_by_domain(tag, limit: 50)
+                return [] unless db_ready?
+
+                rows = traces_ds
+                       .where(Sequel.like(:domain_tags, "%#{tag}%"))
+                       .order(Sequel.desc(:strength))
+                       .limit(limit)
+                       .all
+                rows.map { |r| deserialize_trace(r) }
+              rescue StandardError => e
+                log_warn("retrieve_by_domain failed: #{e.message}")
+                []
+              end
+
+              # Return all traces for this tenant.
+              def all_traces
+                return [] unless db_ready?
+
+                traces_ds.all.map { |r| deserialize_trace(r) }
+              rescue StandardError => e
+                log_warn("all_traces failed: #{e.message}")
+                []
+              end
+
+              # Delete a trace and its association rows.
+              def delete(trace_id)
+                HotTier.evict_trace(trace_id, tenant_id: @tenant_id) if HotTier.available?
+                return unless db_ready?
+
+                db[ASSOCIATIONS_TABLE].where(trace_id_a: trace_id).delete
+                db[ASSOCIATIONS_TABLE].where(trace_id_b: trace_id).delete
+                db[TRACES_TABLE].where(trace_id: trace_id).delete
+              rescue StandardError => e
+                log_warn("delete failed: #{e.message}")
+              end
+
+              # Partial update of a trace by trace_id.
+              # Evicts the hot-tier entry so a stale cached version cannot be served.
+              def update(trace_id, **fields)
+                return unless db_ready?
+
+                db[TRACES_TABLE].where(trace_id: trace_id).update(map_update_fields(fields))
+                HotTier.evict_trace(trace_id, tenant_id: @tenant_id) if HotTier.available?
+              rescue StandardError => e
+                log_warn("update failed: #{e.message}")
+              end
+
+              # Create or increment a coactivation association between two traces.
+              def record_coactivation(id_a, id_b)
+                return unless db_ready?
+                return if id_a == id_b
+
+                now = Time.now.utc
+                existing = db[ASSOCIATIONS_TABLE]
+                           .where(trace_id_a: id_a, trace_id_b: id_b)
+                           .first
+
+                if existing
+                  db[ASSOCIATIONS_TABLE]
+                    .where(id: existing[:id])
+                    .update(
+                      coactivation_count: existing[:coactivation_count] + 1,
+                      updated_at:         now
+                    )
+                else
+                  db[ASSOCIATIONS_TABLE].insert(
+                    trace_id_a:         id_a,
+                    trace_id_b:         id_b,
+                    coactivation_count: 1,
+                    linked:             false,
+                    tenant_id:          @tenant_id,
+                    created_at:         now,
+                    updated_at:         now
+                  )
+                end
+              rescue StandardError => e
+                log_warn("record_coactivation failed: #{e.message}")
+              end
+
+              # Return the set of trace IDs associated with a given trace (bidirectional).
+              def associations_for(trace_id)
+                return [] unless db_ready?
+
+                a_side = db[ASSOCIATIONS_TABLE]
+                         .where(trace_id_a: trace_id)
+                         .select_map(:trace_id_b)
+                b_side = db[ASSOCIATIONS_TABLE]
+                         .where(trace_id_b: trace_id)
+                         .select_map(:trace_id_a)
+                (a_side + b_side).uniq
+              rescue StandardError => e
+                log_warn("associations_for failed: #{e.message}")
+                []
+              end
+
+              # BFS traversal starting from start_id.
+              # Returns an array of { trace_id:, depth:, path: } hashes.
+              def walk_associations(start_id:, max_hops: 12, min_strength: 0.1)
+                return [] unless db_ready?
+
+                start_row = traces_ds.where(trace_id: start_id).first
+                return [] unless start_row
+
+                results = []
+                visited = Set.new([start_id])
+                queue   = [[start_id, 0, [start_id]]]
+
+                until queue.empty?
+                  current_id, depth, path = queue.shift
+                  neighbor_ids = associations_for(current_id)
+
+                  neighbor_ids.each do |nid|
+                    next if visited.include?(nid)
+
+                    neighbor_row = traces_ds
+                                   .where(trace_id: nid)
+                                   .where { strength >= min_strength }
+                                   .first
+                    next unless neighbor_row
+
+                    visited << nid
+                    neighbor_path = path + [nid]
+                    results << { trace_id: nid, depth: depth + 1, path: neighbor_path }
+                    queue << [nid, depth + 1, neighbor_path] if depth + 1 < max_hops
+                  end
+                end
+
+                results
+              rescue StandardError => e
+                log_warn("walk_associations failed: #{e.message}")
+                []
+              end
+
+              # Delete the N traces with the lowest confidence for a given type (quota enforcement).
+              def delete_lowest_confidence(trace_type:, count:)
+                return unless db_ready?
+
+                ids = traces_ds
+                      .where(trace_type: trace_type.to_s)
+                      .order(:confidence)
+                      .limit(count)
+                      .select_map(:trace_id)
+
+                ids.each { |tid| delete(tid) }
+              rescue StandardError => e
+                log_warn("delete_lowest_confidence failed: #{e.message}")
+              end
+
+              # Delete the N least-recently-used traces for a given type (quota enforcement).
+              def delete_least_recently_used(trace_type:, count:)
+                return unless db_ready?
+
+                ids = traces_ds
+                      .where(trace_type: trace_type.to_s)
+                      .order(:last_reinforced)
+                      .limit(count)
+                      .select_map(:trace_id)
+
+                ids.each { |tid| delete(tid) }
+              rescue StandardError => e
+                log_warn("delete_least_recently_used failed: #{e.message}")
+              end
+
+              # Convenience: retrieve firmware-type traces.
+              def firmware_traces
+                retrieve_by_type(:firmware)
+              end
+
+              # No-op — this store is write-through; nothing to flush.
+              def flush; end
+
+              # Returns true when both required tables exist in the connected DB.
+              def db_ready?
+                defined?(Legion::Data) &&
+                  Legion::Data.respond_to?(:connection) &&
+                  Legion::Data.connection&.table_exists?(TRACES_TABLE) &&
+                  Legion::Data.connection.table_exists?(ASSOCIATIONS_TABLE)
+              rescue StandardError
+                false
+              end
+
+              private
+
+              def db
+                Legion::Data.connection
+              end
+
+              # Dataset for memory_traces scoped by tenant_id (if set).
+              def traces_ds
+                ds = db[TRACES_TABLE]
+                @tenant_id ? ds.where(tenant_id: @tenant_id) : ds
+              end
+
+              def serialize_trace(trace)
+                payload = trace[:content_payload] || trace[:content]
+                tags    = trace[:domain_tags]
+                assocs  = trace[:associated_traces]
+                conf    = trace[:confidence]
+                ev      = trace[:emotional_valence]
+
+                {
+                  trace_id:                trace[:trace_id],
+                  tenant_id:               @tenant_id,
+                  trace_type:              trace[:trace_type].to_s,
+                  content:                 payload.is_a?(Hash) ? Legion::JSON.dump(payload) : payload.to_s,
+                  significance:            conf,
+                  confidence:              conf,
+                  associations:            assocs.is_a?(Array) ? Legion::JSON.dump(assocs) : '[]',
+                  domain_tags:             tags.is_a?(Array)   ? Legion::JSON.dump(tags)   : nil,
+                  strength:                trace[:strength],
+                  peak_strength:           trace[:peak_strength],
+                  base_decay_rate:         trace[:base_decay_rate],
+                  emotional_valence:       ev.is_a?(Numeric) ? ev.to_f : 0.0,
+                  emotional_intensity:     trace[:emotional_intensity],
+                  origin:                  trace[:origin].to_s,
+                  source_agent_id:         trace[:source_agent_id],
+                  storage_tier:            trace[:storage_tier].to_s,
+                  last_reinforced:         trace[:last_reinforced],
+                  last_decayed:            trace[:last_decayed],
+                  reinforcement_count:     trace[:reinforcement_count],
+                  unresolved:              trace[:unresolved]              || false,
+                  consolidation_candidate: trace[:consolidation_candidate] || false,
+                  parent_trace_id:         trace[:parent_trace_id],
+                  encryption_key_id:       trace[:encryption_key_id],
+                  partition_id:            trace[:partition_id],
+                  created_at:              trace[:created_at] || Time.now.utc,
+                  accessed_at:             Time.now.utc
+                }
+              end
+
+              def deserialize_trace(row)
+                content = parse_json_or_raw(row[:content])
+                {
+                  trace_id:                row[:trace_id],
+                  trace_type:              row[:trace_type]&.to_sym,
+                  content_payload:         content,
+                  content:                 content,
+                  strength:                row[:strength],
+                  peak_strength:           row[:peak_strength],
+                  base_decay_rate:         row[:base_decay_rate],
+                  emotional_valence:       row[:emotional_valence].to_f,
+                  emotional_intensity:     row[:emotional_intensity],
+                  domain_tags:             parse_json_array(row[:domain_tags]),
+                  origin:                  row[:origin]&.to_sym,
+                  source_agent_id:         row[:source_agent_id],
+                  created_at:              row[:created_at],
+                  last_reinforced:         row[:last_reinforced],
+                  last_decayed:            row[:last_decayed],
+                  reinforcement_count:     row[:reinforcement_count],
+                  confidence:              row[:confidence],
+                  storage_tier:            row[:storage_tier]&.to_sym,
+                  partition_id:            row[:partition_id],
+                  encryption_key_id:       row[:encryption_key_id],
+                  associated_traces:       parse_json_array(row[:associations]),
+                  parent_trace_id:         row[:parent_trace_id],
+                  child_trace_ids:         [],
+                  unresolved:              row[:unresolved]              || false,
+                  consolidation_candidate: row[:consolidation_candidate] || false
+                }
+              end
+
+              # Map keyword fields for partial updates, translating to DB column names.
+              def map_update_fields(fields)
+                mapping = {
+                  content_payload:   :content,
+                  associated_traces: :associations,
+                  parent_trace_id:   :parent_trace_id,
+                  child_trace_ids:   nil # not stored as a column
+                }
+
+                fields.each_with_object({}) do |(k, v), row|
+                  col = mapping.key?(k) ? mapping[k] : k
+                  next if col.nil?
+
+                  row[col] = case col
+                             when :content
+                               v.is_a?(Hash) ? Legion::JSON.dump(v) : v.to_s
+                             when :associations
+                               v.is_a?(Array) ? Legion::JSON.dump(v) : '[]'
+                             when :domain_tags
+                               v.is_a?(Array) ? Legion::JSON.dump(v) : nil
+                             when :trace_type, :origin, :storage_tier
+                               v.to_s
+                             else
+                               v
+                             end
+                end
+              end
+
+              def parse_json_or_raw(raw)
+                return raw unless raw.is_a?(String)
+
+                parsed = Legion::JSON.load(raw)
+                parsed.is_a?(Hash) ? parsed : raw
+              rescue StandardError
+                raw
+              end
+
+              def parse_json_array(raw)
+                return [] unless raw.is_a?(String)
+
+                result = Legion::JSON.load(raw)
+                result.is_a?(Array) ? result : []
+              rescue StandardError
+                []
+              end
+
+              def log_warn(message)
+                Legion::Logging.warn "[memory:postgres_store] #{message}" if defined?(Legion::Logging)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/agentic/memory/trace.rb

@@ -5,6 +5,7 @@ require 'legion/extensions/agentic/memory/trace/helpers/trace'
 require 'legion/extensions/agentic/memory/trace/helpers/decay'
 require 'legion/extensions/agentic/memory/trace/helpers/store'
 require 'legion/extensions/agentic/memory/trace/helpers/cache_store'
+require 'legion/extensions/agentic/memory/trace/helpers/postgres_store'
 require 'legion/extensions/agentic/memory/trace/helpers/error_tracer'
 require 'legion/extensions/agentic/memory/trace/runners/traces'
 require 'legion/extensions/agentic/memory/trace/runners/consolidation'
@@ -31,7 +32,10 @@ module Legion
             private
 
             def create_store
-              if defined?(Legion::Cache) && Legion::Cache.respond_to?(:connected?) && Legion::Cache.connected?
+              if postgres_available?
+                Legion::Logging.debug '[memory] Using shared PostgresStore (write-through)'
+                Helpers::PostgresStore.new(tenant_id: resolve_tenant_id)
+              elsif defined?(Legion::Cache) && Legion::Cache.respond_to?(:connected?) && Legion::Cache.connected?
                 Legion::Logging.debug '[memory] Using shared CacheStore (memcached)'
                 Helpers::CacheStore.new
               else
@@ -39,6 +43,23 @@ module Legion
                 Helpers::Store.new
               end
             end
+
+            def postgres_available?
+              defined?(Legion::Data) &&
+                Legion::Data.respond_to?(:connection) &&
+                Legion::Data.connection &&
+                %i[postgres mysql2].include?(Legion::Data.connection.adapter_scheme) &&
+                Legion::Data.connection.table_exists?(:memory_traces) &&
+                Legion::Data.connection.table_exists?(:memory_associations)
+            rescue StandardError
+              false
+            end
+
+            def resolve_tenant_id
+              Legion::Settings[:data]&.dig(:tenant_id)
+            rescue StandardError
+              nil
+            end
           end
         end
       end

data/lib/legion/extensions/agentic/memory/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module Agentic
       module Memory
-        VERSION = '0.1.9'
+        VERSION = '0.1.11'
       end
     end
   end