llmemory 0.2.10 → 0.3.0

data/lib/llmemory/zero_mem/evidence_calibrator.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Llmemory
+  module ZeroMem
+    class EvidenceCalibrator
+      CONFLICT_PATTERNS = [
+        /\btalla\s+es\s+([A-Z])\b/i,
+        /\bsize\s+is\s+([A-Z])\b/i
+      ].freeze
+
+      def initialize(storage)
+        @storage = storage
+      end
+
+      def calibrate(user_id:, traces:, profile:, current_trace_id: nil, as_of: Time.now,
+                    fusion_rows: nil, seed_ids: nil, closure_ids: nil)
+        excluded = []
+        kept = []
+
+        Array(traces).each do |t|
+          reason = exclusion_reason(user_id, t, profile, current_trace_id, as_of)
+          if reason
+            excluded << { id: t.id, reason: reason }
+            next
+          end
+
+          kept << t
+        end
+
+        deduped, dup_excluded = dedupe_by_content(kept, fusion_rows)
+        excluded.concat(dup_excluded)
+
+        conflict_ids = detect_conflicts(deduped)
+        ordered = sort_traces(deduped, profile, as_of)
+
+        {
+          traces: ordered,
+          excluded: excluded,
+          conflict_trace_ids: conflict_ids,
+          seed_ids: Array(seed_ids).map(&:to_s),
+          closure_ids: Array(closure_ids).map(&:to_s)
+        }
+      end
+
+      private
+
+      def exclusion_reason(user_id, trace, profile, current_trace_id, as_of)
+        return :wrong_tenant unless trace.user_id == user_id.to_s
+        return :archived unless trace.active?
+        return :current_query if current_trace_id && trace.id == current_trace_id.to_s
+        return :outside_boundary unless boundary_match?(trace, profile.boundary)
+        return :hash_mismatch unless sha_ok?(trace)
+        return :stale_state unless current_state_ok?(user_id, trace, profile, as_of)
+
+        nil
+      end
+
+      def boundary_match?(trace, boundary)
+        return true unless boundary
+
+        ok = true
+        ok &&= (trace.session_id == boundary[:session_id].to_s) if boundary[:session_id]
+        ok &&= (trace.boundary_id == boundary[:boundary_id].to_s) if boundary[:boundary_id]
+        ok
+      end
+
+      def sha_ok?(trace)
+        Digest::SHA256.hexdigest(trace.content) == trace.content_sha256
+      end
+
+      def current_state_ok?(user_id, trace, profile, as_of)
+        return true unless profile.workload_class == :current_state
+
+        links = @storage.state_links_for(user_id, state_attribute_key(trace))
+        return true if links.empty?
+
+        current_id = @storage.current_state_trace_id(user_id, state_attribute_key(trace), as_of: as_of)
+        return true if current_id.nil?
+
+        trace.id == current_id.to_s || links.any? { |l| l.trace_id == trace.id }
+      end
+
+      def state_attribute_key(trace)
+        down = trace.content.downcase
+        return "talla" if down.include?("talla")
+        return "size" if down.include?("size")
+
+        "state:#{trace.session_id}"
+      end
+
+      def dedupe_by_content(traces, fusion_rows)
+        by_hash = traces.group_by(&:content_sha256)
+        kept = []
+        excluded = []
+        by_hash.each do |hash, group|
+          if group.size == 1
+            kept << group.first
+            next
+          end
+
+          winner = pick_winner(group, fusion_rows)
+          kept << winner
+          group.each do |t|
+            next if t.id == winner.id
+
+            excluded << { id: t.id, reason: :duplicate_content, content_sha256: hash }
+          end
+        end
+        [kept, excluded]
+      end
+
+      def pick_winner(group, fusion_rows)
+        scores = fusion_rows&.index_by { |r| r[:trace_id].to_s } || {}
+        group.max_by do |t|
+          row = scores[t.id]
+          [row ? row[:final].to_f : 0.0, t.occurred_at.to_f]
+        end
+      end
+
+      def detect_conflicts(traces)
+        buckets = Hash.new { |h, k| h[k] = [] }
+        traces.each do |t|
+          CONFLICT_PATTERNS.each do |pat|
+            m = t.content.match(pat)
+            next unless m
+
+            buckets[pat.source] << { id: t.id, value: m[1].upcase }
+          end
+        end
+
+        conflict_ids = []
+        buckets.each_value do |entries|
+          values = entries.map { |e| e[:value] }.uniq
+          next if values.size <= 1
+
+          conflict_ids.concat(entries.map { |e| e[:id] })
+        end
+        conflict_ids.uniq
+      end
+
+      def sort_traces(traces, profile, as_of)
+        if profile.freshness_requirement || profile.workload_class == :current_state
+          traces.sort_by { |t| [-t.occurred_at.to_f, subject_rank(t, profile)] }
+        else
+          traces.sort_by { |t| [subject_rank(t, profile), -t.occurred_at.to_f] }
+        end
+      end
+
+      def subject_rank(trace, profile)
+        down = trace.content.downcase
+        hits = profile.subject_entities.count { |e| down.include?(e.downcase) }
+        -hits
+      end
+    end
+  end
+end

data/lib/llmemory/zero_mem/evidence_closure.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ZeroMem
+    class EvidenceClosure
+      DEFAULT_BRIDGE_BUDGET = 4
+      DEFAULT_NEIGHBOR_BUDGET = 4
+
+      def initialize(storage, config: Llmemory.configuration)
+        @storage = storage
+        @config = config
+      end
+
+      # Expands fused seeds with relational bridges and local neighbors inside the boundary.
+      def expand(user_id:, profile:, seed_trace_ids:, fused_ranking:)
+        seed_trace_ids = Array(seed_trace_ids).map(&:to_s).uniq
+        return empty(seed_trace_ids) if seed_trace_ids.empty?
+
+        bridge_budget = [seed_trace_ids.size, DEFAULT_BRIDGE_BUDGET].min
+        neighbor_budget = DEFAULT_NEIGHBOR_BUDGET
+        per_seed = [neighbor_budget / [seed_trace_ids.size, 1].max, 1].max
+
+        closure_ids = []
+        bridges = []
+        groups = []
+
+        seed_trace_ids.each do |seed_id|
+          group = [seed_id]
+          added = 0
+          prev, nxt = @storage.adjacent_traces(user_id, seed_id)
+          [prev, nxt].compact.each do |tid|
+            break if added >= per_seed
+            next if seed_trace_ids.include?(tid)
+            next unless trace_ok?(user_id, tid, profile)
+
+            closure_ids << tid
+            bridges << [seed_id, tid]
+            group << tid
+            added += 1
+          end
+          groups << group if group.size > 1
+        end
+
+        span = @config.zero_mem_local_span.to_i
+        span = 2 if span <= 0
+        seed_trace_ids.each do |seed_id|
+          trace = @storage.get_trace(user_id, seed_id)
+          next unless trace
+
+          session_traces = @storage.list_traces(user_id, session_id: trace.session_id)
+          by_seq = session_traces.each_with_object({}) { |t, acc| acc[t.sequence] = t.id }
+          ((trace.sequence - span)..(trace.sequence + span)).each do |seq|
+            tid = by_seq[seq]
+            next if tid.nil? || seed_trace_ids.include?(tid) || closure_ids.include?(tid)
+            next unless trace_ok?(user_id, tid, profile)
+
+            closure_ids << tid
+          end
+        end
+
+        closure_ids = closure_ids.uniq.first(bridge_budget + neighbor_budget)
+        {
+          seed_trace_ids: seed_trace_ids,
+          closure_trace_ids: closure_ids,
+          bridges: bridges.first(bridge_budget),
+          atomic_groups: groups.map { |g| g.uniq.sort }
+        }
+      end
+
+      private
+
+      def empty(seed_trace_ids)
+        {
+          seed_trace_ids: seed_trace_ids,
+          closure_trace_ids: [],
+          bridges: [],
+          atomic_groups: []
+        }
+      end
+
+      def trace_ok?(user_id, trace_id, profile)
+        trace = @storage.get_trace(user_id, trace_id)
+        return false unless trace&.active?
+
+        boundary_match?(trace, profile.boundary)
+      end
+
+      def boundary_match?(trace, boundary)
+        return true unless boundary
+
+        ok = true
+        ok &&= (trace.session_id == boundary[:session_id].to_s) if boundary[:session_id]
+        ok &&= (trace.boundary_id == boundary[:boundary_id].to_s) if boundary[:boundary_id]
+        ok
+      end
+    end
+  end
+end

data/lib/llmemory/zero_mem/evidence_fusion.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ZeroMem
+    class EvidenceFusion
+      # When all raw scores are equal, min-max yields this normalized value (see benchmarks/zero_mem/profile.yml).
+      EQUAL_SCORE_NORMALIZED = 0.5
+
+      def fuse(graph_scores:, hierarchy_scores:, weights:)
+        g_norm = min_max(graph_scores)
+        h_norm = min_max(hierarchy_scores)
+        ids = (g_norm.keys + h_norm.keys).uniq
+
+        fused = ids.map do |trace_id|
+          graph = g_norm.fetch(trace_id, 0.0)
+          hierarchy = h_norm.fetch(trace_id, 0.0)
+          graph_w = weights.fetch(:graph, 0.5).to_f
+          hierarchy_w = weights.fetch(:hierarchy, 0.5).to_f
+          total = graph_w + hierarchy_w
+          graph_w /= total
+          hierarchy_w /= total
+          final = (graph_w * graph) + (hierarchy_w * hierarchy)
+          sources = []
+          sources << :graph if graph_scores.key?(trace_id)
+          sources << :hierarchy if hierarchy_scores.key?(trace_id)
+
+          {
+            trace_id: trace_id,
+            graph: graph,
+            hierarchy: hierarchy,
+            graph_raw: graph_scores[trace_id],
+            hierarchy_raw: hierarchy_scores[trace_id],
+            final: final,
+            sources: sources
+          }
+        end
+
+        fused.sort_by { |row| [-row[:final], row[:trace_id]] }
+      end
+
+      def min_max(scores)
+        return {} if scores.nil? || scores.empty?
+
+        values = scores.values.map(&:to_f)
+        min = values.min
+        max = values.max
+        if max == min
+          return scores.keys.to_h { |id| [id, EQUAL_SCORE_NORMALIZED] }
+        end
+
+        scores.transform_values do |v|
+          ((v.to_f - min) / (max - min)).clamp(0.0, 1.0)
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/zero_mem/evidence_set.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ZeroMem
+    class EvidenceSet
+      attr_reader :route, :profile, :evidence, :metrics, :explain, :degraded,
+                  :relational_trace_scores, :hierarchy_traces, :relational_traces, :warnings,
+                  :seed_evidence, :closure_evidence
+
+      def initialize(route:, profile:, evidence:, metrics:, explain: nil, degraded: [],
+                     relational_trace_scores: {}, hierarchy_traces: [], relational_traces: [],
+                     warnings: [], seed_evidence: [], closure_evidence: [])
+        @route = route
+        @profile = profile
+        @evidence = evidence
+        @metrics = metrics
+        @explain = explain
+        @degraded = Array(degraded)
+        @relational_trace_scores = relational_trace_scores
+        @hierarchy_traces = hierarchy_traces
+        @relational_traces = relational_traces
+        @warnings = Array(warnings)
+        @seed_evidence = Array(seed_evidence)
+        @closure_evidence = Array(closure_evidence)
+      end
+
+      def recall_at_k(gold_trace_ids, k: nil)
+        k ||= evidence.size
+        gold = Array(gold_trace_ids).map(&:to_s)
+        ranked = evidence.map(&:trace_id).first(k)
+        hit = (gold & ranked).size
+        {
+          hit: hit,
+          gold: gold.size,
+          recall: gold.empty? ? 0.0 : hit.to_f / gold.size,
+          ranked: ranked
+        }
+      end
+
+      def to_context
+        return "" if evidence.empty?
+
+        lines = ["=== ZERO-MEM EVIDENCE ===", ""]
+        evidence.each do |ev|
+          ts = ev.occurred_at.is_a?(Time) ? ev.occurred_at.utc.iso8601 : ev.occurred_at.to_s
+          lines << "[#{ev.trace_id}] #{ts} #{ev.session_id} #{ev.role}: #{ev.content}"
+        end
+        lines << ""
+        lines << "=== END ZERO-MEM EVIDENCE ==="
+        lines.join("\n")
+      end
+    end
+  end
+end

data/lib/llmemory/zero_mem/extractors/heuristic.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative "../entity_extractor"
+
+module Llmemory
+  module ZeroMem
+    module Extractors
+      class Heuristic < EntityExtractor
+        CAP_PHRASE = /\b[\p{Lu}][\p{Ll}]+(?:\s+[\p{Lu}][\p{Ll}]+)*\b/u
+        ALL_CAPS = /\b[\p{Lu}]{2,}\b/u
+
+        def extract(text)
+          normalized_text = Llmemory::Tokenizer.normalize(text)
+          spans = []
+          text.to_s.scan(CAP_PHRASE) { |m| spans << [Regexp.last_match.begin(0), m] }
+          text.to_s.scan(ALL_CAPS) { |m| spans << [Regexp.last_match.begin(0), m] }
+
+          spans.map do |start, raw|
+            norm = normalize_key(raw)
+            next if norm.length < 2
+
+            {
+              text: raw,
+              normalized: norm,
+              type: :entity,
+              start: start,
+              end: start + raw.length
+            }
+          end.compact.uniq { |h| h[:normalized] }
+        end
+
+        def name
+          "heuristic"
+        end
+
+        private
+
+        def normalize_key(raw)
+          Llmemory::Tokenizer.normalize(raw).gsub(/\s+/, " ").strip
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/zero_mem/extractors/http.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "json"
+require "faraday"
+require_relative "../entity_extractor"
+
+module Llmemory
+  module ZeroMem
+    module Extractors
+      class Http < EntityExtractor
+        def initialize(base_url:, timeout: 5, client: nil)
+          @base_url = base_url.to_s.sub(%r{/\z}, "")
+          @timeout = timeout
+          @client = client
+        end
+
+        def extract(text)
+          body = sidecar.post_json("/ner", { text: text.to_s })
+          Array(body["entities"]).map do |ent|
+            {
+              text: ent["text"],
+              normalized: ent["normalized"] || ent["text"].to_s.downcase,
+              type: (ent["type"] || "entity").to_sym,
+              start: ent["start"] || 0,
+              end: ent["end"] || 0
+            }
+          end
+        end
+
+        def name
+          "http"
+        end
+
+        def version
+          health["version"] || "unknown"
+        rescue StandardError
+          "unknown"
+        end
+
+        def health
+          sidecar.get_json("/health")
+        rescue StoreError
+          { "status" => "down" }
+        end
+
+        def degraded?
+          sidecar.degraded?
+        end
+
+        private
+
+        def sidecar
+          @client ||= SidecarClient.new(base_url: @base_url, timeout: @timeout)
+        end
+      end
+    end
+  end
+end

data/lib/llmemory/zero_mem/graph_retriever.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ZeroMem
+    class GraphRetriever
+      MAX_NODES = 200
+
+      def initialize(storage, entity_index: nil, embedding_provider: nil, config: Llmemory.configuration)
+        @storage = storage
+        @entity_index = entity_index || EntityIndex.new(storage)
+        @embedding_provider = embedding_provider
+        @config = config
+        @pagerank = PageRank.new(gamma: config.zero_mem_pagerank_damping)
+      end
+
+      def retrieve(user_id:, query:, profile:, top_k:)
+        warnings = []
+        degraded = []
+        query_entities = @entity_index.extract_entities(query)
+        degraded << :ner if query_entities.empty?
+
+        seeds = match_entities(user_id, query, query_entities, degraded: degraded)
+        return empty_result(degraded, warnings) if seeds.empty?
+
+        trace_ids = seeds.flat_map { |key| @storage.trace_ids_for_entity_key(user_id, key) }.uniq
+        trace_ids = trace_ids.first(MAX_NODES)
+        warnings << :pagerank_truncated if trace_ids.size >= MAX_NODES
+
+        graph, reset = build_graph(user_id, trace_ids, seeds)
+        ranks = @pagerank.run(graph, reset)
+        trace_scores = ranks.select { |node, _| node.start_with?("tr:") }
+                            .transform_keys { |k| k.sub("tr:", "") }
+        trace_ids.each { |tid| trace_scores[tid] ||= 0.5 }
+        trace_scores = trace_scores.sort_by { |_, score| -score }
+
+        traces = trace_scores.first(top_k).map do |trace_id, score|
+          t = @storage.get_trace(user_id, trace_id)
+          next unless t
+
+          { trace: t, score: score }
+        end.compact
+
+        {
+          traces: traces.map { |h| h[:trace] },
+          trace_scores: trace_scores.to_h,
+          degraded: degraded.uniq,
+          warnings: warnings
+        }
+      end
+
+      private
+
+      def empty_result(degraded, warnings)
+        { traces: [], trace_scores: {}, degraded: degraded.uniq, warnings: warnings }
+      end
+
+      def match_entities(user_id, query, query_entities, degraded:)
+        keys = query_entities.map { |e| e[:normalized] }.compact
+        tokens = Llmemory::Tokenizer.tokenize(query)
+        keys = expand_entity_keys(user_id, keys) if keys.any?
+        if keys.empty?
+          keys = @storage.all_entity_keys(user_id).select do |key|
+            tokens.any? { |t| key.include?(t) || t.include?(key) }
+          end
+        end
+        degraded << :dense if @embedding_provider.nil?
+        keys.select { |k| @storage.entity_exists?(user_id, k) }
+      end
+
+      def expand_entity_keys(user_id, keys)
+        stored = @storage.all_entity_keys(user_id)
+        keys.flat_map do |k|
+          next [] if k.to_s.length < 2
+
+          stored.select { |s| s == k || s.include?(k) || k.include?(s) }
+        end.uniq
+      end
+
+      def build_graph(user_id, trace_ids, seed_keys)
+        graph = {}
+        reset = {}
+
+        trace_ids.each do |tid|
+          node = "tr:#{tid}"
+          neighbors = ["tr:#{tid}"]
+          @storage.entity_keys_for_trace(user_id, tid).each do |key|
+            ent_node = "ent:#{key}"
+            graph[ent_node] ||= []
+            graph[ent_node] << node
+            graph[node] ||= []
+            graph[node] << ent_node
+            neighbors << ent_node
+            reset[ent_node] = 1.0 if seed_keys.include?(key)
+          end
+          prev, nxt = @storage.adjacent_traces(user_id, tid)
+          [prev, nxt].compact.each do |other|
+            graph[node] ||= []
+            graph[node] << "tr:#{other}"
+          end
+          graph[node] = (graph[node] || neighbors).uniq
+        end
+
+        [graph, reset]
+      end
+    end
+  end
+end

data/lib/llmemory/zero_mem/hierarchy_builder.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Llmemory
+  module ZeroMem
+    class HierarchyBuilder
+      def initialize(storage, config: Llmemory.configuration)
+        @storage = storage
+        @config = config
+      end
+
+      def index_trace(trace, index_version:)
+        affected = 0
+        affected += write_turn_unit(trace, index_version)
+        affected += update_tail_windows(trace, index_version)
+        affected += update_episode(trace, index_version)
+        affected
+      end
+
+      private
+
+      def window_size
+        @config.zero_mem_window_size.to_i.positive? ? @config.zero_mem_window_size.to_i : 8
+      end
+
+      def window_overlap
+        ov = @config.zero_mem_window_overlap.to_i
+        ov = 2 if ov.negative?
+        [ov, window_size - 1].min
+      end
+
+      def episode_gap_seconds
+        gap = @config.zero_mem_episode_gap_seconds.to_i
+        gap.positive? ? gap : 86_400
+      end
+
+      def write_turn_unit(trace, index_version)
+        unit = TraceUnit.build(
+          id: "turn_#{trace.id}",
+          user_id: trace.user_id,
+          kind: :turn,
+          session_id: trace.session_id,
+          boundary_id: trace.boundary_id,
+          start_sequence: trace.sequence,
+          end_sequence: trace.sequence,
+          occurred_from: trace.occurred_at,
+          occurred_to: trace.occurred_at,
+          member_trace_ids: [trace.id],
+          index_version: index_version
+        )
+        @storage.write_unit(unit)
+        1
+      end
+
+      def update_tail_windows(trace, index_version)
+        size = window_size
+        seq = trace.sequence
+        start_seq = [1, seq - size + 1].max
+        traces = @storage.list_traces(trace.user_id, session_id: trace.session_id).select do |t|
+          t.sequence >= start_seq && t.sequence <= seq
+        end
+        return 0 if traces.empty?
+
+        unit = TraceUnit.build(
+          id: "window_#{trace.session_id}_#{traces.first.sequence}_#{traces.last.sequence}",
+          user_id: trace.user_id,
+          kind: :window,
+          session_id: trace.session_id,
+          boundary_id: trace.boundary_id,
+          start_sequence: traces.first.sequence,
+          end_sequence: traces.last.sequence,
+          occurred_from: traces.first.occurred_at,
+          occurred_to: traces.last.occurred_at,
+          member_trace_ids: traces.map(&:id),
+          index_version: index_version
+        )
+        @storage.write_unit(unit)
+        1
+      end
+
+      def update_episode(trace, index_version)
+        key = trace.boundary_id || trace.session_id
+        open_episode = @storage.open_episode(trace.user_id, key)
+        gap = episode_gap_seconds
+
+        if open_episode
+          last_at = open_episode.occurred_to
+          if (trace.occurred_at - last_at).to_f > gap
+            @storage.close_episode(open_episode.id)
+            open_episode = nil
+          end
+        end
+
+        if open_episode
+          ids = (open_episode.member_trace_ids + [trace.id]).uniq
+          traces = ids.map { |id| @storage.get_trace(trace.user_id, id) }.compact
+          updated = open_episode.with(
+            end_sequence: trace.sequence,
+            occurred_to: trace.occurred_at,
+            member_trace_ids: ids,
+            index_version: index_version
+          )
+          @storage.write_unit(updated)
+        else
+          unit = TraceUnit.build(
+            user_id: trace.user_id,
+            kind: :episode,
+            session_id: trace.session_id,
+            boundary_id: trace.boundary_id,
+            start_sequence: trace.sequence,
+            end_sequence: trace.sequence,
+            occurred_from: trace.occurred_at,
+            occurred_to: trace.occurred_at,
+            member_trace_ids: [trace.id],
+            index_version: index_version
+          )
+          @storage.write_unit(unit)
+          @storage.set_open_episode(trace.user_id, key, unit.id)
+        end
+        1
+      end
+    end
+  end
+end