rubino-agent 0.3.0

data/lib/rubino/memory/deduplicator.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Memory
+    # Prevents duplicate memories from being stored.
+    # Uses content similarity to detect duplicates.
+    class Deduplicator
+      # Similarity threshold (0.0 to 1.0) - above this is considered duplicate
+      SIMILARITY_THRESHOLD = 0.85
+
+      def initialize(store: nil)
+        @store = store || Store.new
+      end
+
+      # Returns true if a similar memory already exists
+      def duplicate?(kind:, content:)
+        existing = @store.by_kind(kind, limit: 100)
+        existing.any? { |m| similar?(m[:content], content) }
+      end
+
+      # Removes duplicate memories, keeping the highest confidence version
+      def deduplicate_all!
+        removed = 0
+        Store::VALID_KINDS.each do |kind|
+          removed += deduplicate_kind(kind)
+        end
+        removed
+      end
+
+      private
+
+      def similar?(text_a, text_b)
+        return true if text_a == text_b
+
+        # Simple Jaccard similarity on word sets
+        words_a = text_a.downcase.split(/\W+/).to_set
+        words_b = text_b.downcase.split(/\W+/).to_set
+
+        return false if words_a.empty? || words_b.empty?
+
+        intersection = (words_a & words_b).size
+        union = (words_a | words_b).size
+
+        (intersection.to_f / union) >= SIMILARITY_THRESHOLD
+      end
+
+      def deduplicate_kind(kind)
+        memories = @store.by_kind(kind, limit: 500)
+        to_remove = []
+
+        memories.each_with_index do |mem, i|
+          next if to_remove.include?(mem[:id])
+
+          memories[(i + 1)..].each do |other|
+            next if to_remove.include?(other[:id])
+
+            if similar?(mem[:content], other[:content])
+              # Keep the one with higher confidence
+              if mem[:confidence] >= (other[:confidence] || 0)
+                to_remove << other[:id]
+              else
+                to_remove << mem[:id]
+                break
+              end
+            end
+          end
+        end
+
+        to_remove.each { |id| @store.delete(id) }
+        to_remove.size
+      end
+    end
+  end
+end

data/lib/rubino/memory/extractor.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Memory
+    # Extracts potential memories from conversation history.
+    # Identifies facts, preferences, decisions, and other memorable items.
+    class Extractor
+      # Patterns that suggest extractable memories
+      PREFERENCE_PATTERNS = [
+        /(?:I prefer|I like|I always|I never|I usually|my preferred)/i,
+        /(?:please always|please never|don't ever|always use|never use)/i
+      ].freeze
+
+      DECISION_PATTERNS = [
+        /(?:we decided|the decision is|let's go with|I chose|we'll use)/i,
+        /(?:the approach is|the strategy is|we agreed on)/i
+      ].freeze
+
+      def initialize(store: nil)
+        @store = store || Store.new
+      end
+
+      # Extracts memories from a session's messages
+      def extract_from_session(session_id)
+        message_store = Session::Store.new
+        messages = message_store.for_session(session_id)
+        extracted = []
+
+        messages.each do |msg|
+          next unless %w[user assistant].include?(msg.role)
+          next if msg.content.nil? || msg.content.empty?
+
+          memories = extract_from_content(msg.content, session_id)
+          extracted.concat(memories)
+        end
+
+        extracted
+      end
+
+      # Extracts memories from a single content string
+      def extract_from_content(content, session_id = nil)
+        memories = []
+
+        # Check for preferences
+        if matches_patterns?(content, PREFERENCE_PATTERNS)
+          memories << save_memory(
+            kind: "preference",
+            content: content.strip[0..500],
+            session_id: session_id
+          )
+        end
+
+        # Check for technical decisions
+        if matches_patterns?(content, DECISION_PATTERNS)
+          memories << save_memory(
+            kind: "technical_decision",
+            content: content.strip[0..500],
+            session_id: session_id
+          )
+        end
+
+        memories.compact
+      end
+
+      private
+
+      def matches_patterns?(content, patterns)
+        patterns.any? { |p| content.match?(p) }
+      end
+
+      def save_memory(kind:, content:, session_id:)
+        # Check for duplicates before saving
+        deduplicator = Deduplicator.new(store: @store)
+        return nil if deduplicator.duplicate?(kind: kind, content: content)
+
+        @store.create(
+          kind: kind,
+          content: content,
+          source_session_id: session_id,
+          confidence: 0.8
+        )
+      end
+    end
+  end
+end

data/lib/rubino/memory/flusher.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Memory
+    # Flushes working memory to persistent storage before compaction.
+    # Ensures no important information is lost when context is compressed.
+    class Flusher
+      def initialize(backend: nil)
+        @backend = backend
+      end
+
+      # Flushes all pending memories for a session before compaction.
+      # Routes through the configured backend's extract path so compaction
+      # mines facts with the same backend the rest of the gem uses.
+      def flush_before_compaction!(session_id)
+        extracted = backend.extract(session_id)
+
+        {
+          flushed_count: extracted.size,
+          session_id: session_id
+        }
+      end
+
+      private
+
+      def backend
+        @backend ||= Backends.build
+      end
+    end
+  end
+end

data/lib/rubino/memory/retriever.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Memory
+    # Retrieves relevant memories for inclusion in prompts.
+    # Handles user profile, project context, and session-relevant memories.
+    class Retriever
+      def initialize(store: nil, config: nil)
+        @store = store || Store.new
+        @config = config || Rubino.configuration
+      end
+
+      # Returns the user profile text (concatenated user_profile memories)
+      def user_profile
+        return nil unless @config.dig("memory", "user_profile_enabled")
+
+        char_limit = @config.memory_user_char_limit
+        memories = @store.by_kind("user_profile")
+
+        text = memories.map { |m| m[:content] }.join("\n")
+        text.length > char_limit ? text[0...char_limit] : text
+      end
+
+      # Returns project context memories
+      def project_context
+        return nil unless @config.dig("memory", "project_context_enabled")
+
+        memories = @store.by_kind("project_context", limit: 10)
+        return nil if memories.empty?
+
+        memories.map { |m| m[:content] }.join("\n")
+      end
+
+      # Returns memories relevant to the current session context
+      def relevant_for_session(_session_id)
+        char_limit = @config.memory_char_limit
+        @store.within_limit(char_limit: char_limit)
+      end
+
+      # Returns all memories formatted for prompt inclusion
+      def for_prompt
+        {
+          user_profile: user_profile,
+          project_context: project_context,
+          general: @store.within_limit(char_limit: @config.memory_char_limit)
+        }
+      end
+    end
+  end
+end

data/lib/rubino/memory/sqlite_extraction_prompt.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Rubino
+  module Memory
+    # The single aux-LLM extraction prompt for the Sqlite backend. Collapses
+    # Zep's six-step ingestion (entity/fact/temporal extraction + invalidation)
+    # into ONE structured call: given the latest turn and the currently-live
+    # facts, the model returns durable atomic facts to `add` and contradicted
+    # facts to `supersede`. The doctrine ("durable declarative facts, not
+    # imperatives, not stale artifacts") is lifted from the reference MEMORY_GUIDANCE.
+    module SqliteExtractionPrompt
+      KINDS = %w[user_profile preference project fact env].freeze
+
+      SYSTEM = <<~PROMPT
+        You maintain a long-term memory of durable facts about the user and their project.
+        You will see the latest conversation turn and the facts already in memory.
+
+        Extract only DURABLE facts worth remembering across sessions:
+          - user identity, preferences, and recurring corrections (highest value — they reduce future steering)
+          - stable project/environment conventions and tool quirks
+        Write each as ONE atomic declarative fact in the third person, present tense.
+          GOOD: "User prefers concise answers without preamble."
+          GOOD: "Project uses pytest with the xdist plugin."
+          BAD (imperative): "Always answer concisely."        BAD (procedure): "Run tests with pytest -n 4."
+          BAD (stale artifact): "Fixed bug #4821."  "Opened PR 90."  "Phase 2 done."
+        If a fact will be stale within a week (PR/issue/commit numbers, task progress, TODO state), DO NOT save it.
+        Procedures and how-to workflows are NOT memory — skip them.
+
+        SUPERSEDE: if a new fact CONTRADICTS an existing one (same subject, changed value),
+        emit it under "supersede" with the id of the fact it replaces. Prefer the newer information.
+        Tag each fact with 1-4 lowercase entity keywords (people, tools, projects) for retrieval.
+
+        EDGES (optional, light): if the turn states a clear RELATIONSHIP between two of the
+        entities you tagged, emit it under "edges" as {"src","relation","dst"} with a short
+        lowercase relation (e.g. uses, deploys_to, written_in, runs_on, depends_on). Keep it to
+        the few obvious relations the turn actually asserts — do not invent links. These let a
+        later query like "what does X use for Y" reach the connected fact. Omit when unsure.
+
+        Return STRICT JSON, no prose:
+        { "add":       [ {"text": "...", "kind": "preference|user_profile|project|fact|env",
+                          "entities": ["..."], "valid_from": "<ISO8601 or null>"} ],
+          "supersede": [ {"id": "<existing fact id>", "by_text": "...", "kind": "...",
+                          "entities": ["..."]} ],
+          "edges":     [ {"src": "...", "relation": "...", "dst": "..."} ] }
+        If nothing is worth saving, return {"add": [], "supersede": [], "edges": []}.
+      PROMPT
+
+      module_function
+
+      # Builds the USER message: reference timestamp, the live fact set (already
+      # char-capped by the caller), and the latest turn rendered as a transcript.
+      def user_message(now:, live_facts:, turn:)
+        live = if live_facts.empty?
+                 "(none)"
+               else
+                 live_facts.map { |f| "#{f[:id]} | #{f[:kind]} | #{f[:text]}" }.join("\n")
+               end
+
+        <<~MSG
+          Reference timestamp: #{now}
+          Existing live facts:
+          #{live}
+
+          Latest turn:
+          #{turn}
+        MSG
+      end
+    end
+  end
+end

data/lib/rubino/memory/sqlite_graph.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "time"
+
+module Rubino
+  module Memory
+    # Graph-lite layer for the Sqlite backend (Memory Phase 3b).
+    #
+    # A thin mixin over two tables (memory_entities + memory_edges) that turns
+    # the per-fact entity tags into a tiny knowledge graph and blends a bounded
+    # 1-hop traversal into retrieval. NOT a graph DB — just entity resolution by
+    # normalized name and a bounded join over edges.
+    #
+    # Edges are populated two ways, both cheap (no extra LLM call beyond the
+    # single extraction call the backend already makes):
+    #   * DETERMINISTIC co-occurrence — every pair of entities tagged on the
+    #     same fact gets a `co_occurs` edge (free, derived from `entities_json`).
+    #   * TYPED relations — the extraction LLM optionally returns `edges:
+    #     [{src, relation, dst}]` in the SAME structured call, so the typed
+    #     graph costs 0 additional calls/turn.
+    #
+    # Edges are bi-temporal like facts: a contradicting relation soft-retires
+    # the old edge (valid_to set), it is not deleted.
+    module SqliteGraph
+      ENTITIES = :memory_entities
+      EDGES    = :memory_edges
+      CO_OCCURS = "co_occurs"
+
+      # ---- node resolution ----
+
+      # Resolve (find-or-create) an entity node by normalized name, returning
+      # its id. Same name from different facts collapses to one node.
+      def resolve_entity(name, kind: nil)
+        norm = normalize_entity_name(name)
+        return nil if norm.empty?
+
+        existing = @db[ENTITIES].where(name_norm: norm).first
+        return existing[:id] if existing
+
+        now = Time.now.utc.iso8601
+        id = SecureRandom.uuid
+        @db[ENTITIES].insert(
+          id: id, name: name.to_s.strip, name_norm: norm, kind: kind,
+          created_at: now, updated_at: now
+        )
+        id
+      rescue Sequel::UniqueConstraintViolation
+        # Concurrent insert: re-read the winner.
+        @db[ENTITIES].where(name_norm: norm).get(:id)
+      end
+
+      def normalize_entity_name(name)
+        name.to_s.strip.downcase.gsub(/\s+/, " ")
+      end
+
+      # ---- edge population ----
+
+      # Wire the graph for a freshly-inserted fact: upsert its entity nodes,
+      # connect every co-occurring pair with a co_occurs edge, and add any
+      # typed relations the extractor emitted for this fact. Bounded and free
+      # of extra LLM calls. `typed` is an array of {src, relation, dst} hashes.
+      def index_fact_graph(fact_id, entities, typed: [])
+        ids = Array(entities).filter_map { |e| resolve_entity(e) }.uniq
+        ids.combination(2).each { |a, b| upsert_edge(a, b, CO_OCCURS, fact_id) }
+
+        Array(typed).each do |edge|
+          src = resolve_entity(edge["src"] || edge[:src])
+          dst = resolve_entity(edge["dst"] || edge[:dst])
+          rel = (edge["relation"] || edge[:relation]).to_s.strip.downcase
+          next if src.nil? || dst.nil? || src == dst || rel.empty?
+
+          # A changed typed relation between the SAME pair supersedes the old
+          # one (e.g. "uses postgres" -> "uses sqlite" is handled at the fact
+          # level; here we keep the latest relation label live).
+          supersede_edge(src, dst, rel)
+          upsert_edge(src, dst, rel, fact_id)
+        end
+      end
+
+      # Insert a live edge unless an identical live edge already exists
+      # (idempotent). Co_occurs edges are undirected in effect: we store the
+      # canonical ordering for the pair so the de-dup works both ways.
+      def upsert_edge(src, dst, relation, source_fact_id)
+        a, b = relation == CO_OCCURS ? [src, dst].minmax : [src, dst]
+        return if @db[EDGES].where(
+          src_entity_id: a, dst_entity_id: b, relation: relation, valid_to: nil
+        ).count.positive?
+
+        now = Time.now.utc.iso8601
+        @db[EDGES].insert(
+          id: SecureRandom.uuid, src_entity_id: a, dst_entity_id: b,
+          relation: relation, source_fact_id: source_fact_id,
+          valid_from: now, valid_to: nil, superseded_by: nil,
+          created_at: now, updated_at: now
+        )
+      end
+
+      # Soft-retire any live typed edge between src->dst whose relation differs,
+      # so a contradicting relation supersedes the old one (history kept).
+      def supersede_edge(src, dst, _relation)
+        @db[EDGES].where(src_entity_id: src, dst_entity_id: dst, valid_to: nil)
+                  .exclude(relation: CO_OCCURS)
+                  .update(valid_to: Time.now.utc.iso8601, updated_at: Time.now.utc.iso8601)
+      end
+
+      # ---- 1-hop traversal ----
+
+      # Given query text, find seed entities whose name appears in the query,
+      # walk LIVE edges out one hop to neighbor entities, and return the ids of
+      # LIVE facts tagged with any seed-or-neighbor entity. This surfaces facts
+      # connected through a relation that pure FTS on the probe would miss.
+      # Bounded: capped seeds, single hop, capped fact scan.
+      def graph_neighbors(query, limit)
+        seeds = seed_entities(query)
+        return [] if seeds.empty?
+
+        # 1-hop: neighbors reachable via a live edge in either direction.
+        neighbor_ids = @db[EDGES]
+                       .where(valid_to: nil)
+                       .where(Sequel.|({ src_entity_id: seeds }, { dst_entity_id: seeds }))
+                       .select_map(%i[src_entity_id dst_entity_id])
+                       .flatten.uniq
+
+        entity_ids = (seeds + neighbor_ids).uniq
+        return [] if entity_ids.empty?
+
+        names = @db[ENTITIES].where(id: entity_ids).select_map(:name_norm)
+        facts_tagged_with(names, limit)
+      end
+
+      # Entities whose normalized name (or a token of it) appears in the query.
+      def seed_entities(query)
+        tokens = query.to_s.downcase.scan(/[\p{L}\p{N}]+/).reject { |w| w.length < 2 }.uniq
+        return [] if tokens.empty?
+
+        @db[ENTITIES].where(name_norm: tokens).select_map(:id).first(8)
+      end
+
+      # Live fact ids whose entities_json contains any of the given normalized
+      # entity names. Bounded scan over the live set (small in practice).
+      def facts_tagged_with(norm_names, limit)
+        wanted = norm_names.to_set
+        return [] if wanted.empty?
+
+        live_dataset.exclude(entities_json: nil).order(Sequel.desc(:created_at))
+                    .limit(limit * 6).all.filter_map do |row|
+          ents = parse_entities(row[:entities_json]).map { |e| e.to_s.downcase }
+          row[:id] if ents.any? { |e| wanted.include?(e) }
+        end.first(limit)
+      end
+    end
+  end
+end

data/lib/rubino/memory/store.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "json"
+
+module Rubino
+  module Memory
+    # Primary storage interface for persistent memories.
+    # Handles CRUD operations on the memories table.
+    class Store
+      VALID_KINDS = %w[
+        user_profile
+        preference
+        project_context
+        technical_decision
+        fact
+        task_state
+        tool_result
+      ].freeze
+
+      # Raised when ThreatScanner flags content destined for the store.
+      # Carries the threat label so callers can branch on it without
+      # parsing the human-facing message.
+      class ThreatDetectedError < Rubino::Error
+        attr_reader :threat
+
+        def initialize(threat)
+          @threat = threat
+          super("memory threat detected: #{threat}")
+        end
+      end
+
+      # Raised when adding `content` to `kind`'s group would push the
+      # group's total characters past the configured budget. Group rules
+      # live in Store#group_for_kind — user_profile is its own group,
+      # everything else shares the general "memory" budget.
+      class BudgetExceededError < Rubino::Error
+        attr_reader :group, :limit, :current, :requested
+
+        def initialize(group:, limit:, current:, requested:)
+          @group     = group
+          @limit     = limit
+          @current   = current
+          @requested = requested
+          super("memory budget exceeded for #{group}: " \
+                "#{current}+#{requested} > #{limit}")
+        end
+      end
+
+      def initialize(db: nil, config: nil)
+        @db = db || Rubino.database.db
+        @config = config
+      end
+
+      # Creates a new memory entry.
+      #
+      # Two boundary checks run *before* the row is inserted:
+      #   1. ThreatScanner — refuses prompt-injection, exfil, invisible
+      #      unicode, etc. Memory persists across sessions, so a tainted
+      #      write would keep biasing every future prompt.
+      #   2. char-budget — refuses writes that would push the group's
+      #      total past memory_char_limit / memory_user_char_limit. Lets
+      #      callers (Tools::MemoryTool) surface a "delete or replace
+      #      older entries first" message instead of silently truncating
+      #      at read-time.
+      def create(kind:, content:, source_session_id: nil, confidence: 1.0, metadata: {})
+        validate_kind!(kind)
+        enforce_threat_scan!(content)
+        enforce_char_budget!(kind, content)
+
+        now = Time.now.utc.iso8601
+        id = SecureRandom.uuid
+
+        @db[:memories].insert(
+          id: id,
+          kind: kind,
+          content: content,
+          source_session_id: source_session_id,
+          confidence: confidence,
+          metadata_json: metadata.empty? ? nil : JSON.generate(metadata),
+          created_at: now,
+          updated_at: now
+        )
+
+        find(id)
+      end
+
+      # Finds a memory by ID (supports prefix matching)
+      def find(id)
+        @db[:memories].where(Sequel.like(:id, "#{id}%")).first
+      end
+
+      # Lists memories with optional filters
+      def list(kind: nil, limit: 20)
+        dataset = @db[:memories].order(Sequel.desc(:created_at)).limit(limit)
+        dataset = dataset.where(kind: kind) if kind
+        dataset.all
+      end
+
+      # Updates a memory's content.
+      #
+      # Same two boundary checks as create — the replace path was a hole that
+      # let an agent rewrite a benign entry with prompt-injection / exfil
+      # content without going through ThreatScanner, and let a chain of
+      # replaces grow a group past its char budget one byte at a time. The
+      # budget check subtracts the old row's length before re-adding the new,
+      # otherwise a same-size edit would be flagged as over budget when it
+      # isn't.
+      def update(id, content:, confidence: nil)
+        existing = find(id)
+        enforce_threat_scan!(content)
+        enforce_char_budget_for_update!(existing, content) if existing
+
+        attrs = { content: content, updated_at: Time.now.utc.iso8601 }
+        attrs[:confidence] = confidence if confidence
+        @db[:memories].where(id: id).update(attrs)
+      end
+
+      # Deletes a memory
+      def delete(id)
+        count = @db[:memories].where(Sequel.like(:id, "#{id}%")).delete
+        count > 0
+      end
+
+      # Returns memories of a specific kind
+      def by_kind(kind, limit: 50)
+        @db[:memories]
+          .where(kind: kind)
+          .order(Sequel.desc(:confidence), Sequel.desc(:created_at))
+          .limit(limit)
+          .all
+      end
+
+      # Returns all memories within the character limit
+      def within_limit(char_limit:)
+        memories = @db[:memories]
+                   .order(Sequel.desc(:confidence), Sequel.desc(:updated_at))
+                   .all
+
+        selected = []
+        total_chars = 0
+
+        memories.each do |m|
+          break if total_chars + m[:content].length > char_limit
+
+          selected << m
+          total_chars += m[:content].length
+        end
+
+        selected
+      end
+
+      # Returns the total count of stored memories
+      def count
+        @db[:memories].count
+      end
+
+      # Returns the budget group a kind belongs to:
+      # - "user"   → user_profile (its own dedicated budget)
+      # - "memory" → everything else (shared general-memory budget)
+      def self.group_for_kind(kind)
+        kind == "user_profile" ? "user" : "memory"
+      end
+
+      # Sum of content length across every row in the given group.
+      def total_chars_for_group(group)
+        if group == "user"
+          @db[:memories].where(kind: "user_profile").sum(Sequel.function(:length, :content)).to_i
+        else
+          @db[:memories].exclude(kind: "user_profile").sum(Sequel.function(:length, :content)).to_i
+        end
+      end
+
+      private
+
+      def validate_kind!(kind)
+        return if VALID_KINDS.include?(kind)
+
+        raise Error, "Invalid memory kind: #{kind}. Valid: #{VALID_KINDS.join(", ")}"
+      end
+
+      def enforce_threat_scan!(content)
+        threat = ThreatScanner.scan(content)
+        return unless threat
+
+        begin
+          Rubino.logger.warn(event: "memory.threat_detected", threat: threat)
+        rescue StandardError
+          # logging must never block the refusal path
+        end
+        raise ThreatDetectedError.new(threat)
+      end
+
+      def enforce_char_budget!(kind, content)
+        cfg = @config || Rubino.configuration
+        group = self.class.group_for_kind(kind)
+        limit = group == "user" ? cfg.memory_user_char_limit : cfg.memory_char_limit
+        return unless limit && limit > 0
+
+        current = total_chars_for_group(group)
+        requested = content.to_s.length
+        return if current + requested <= limit
+
+        raise BudgetExceededError.new(
+          group: group, limit: limit, current: current, requested: requested
+        )
+      end
+
+      # Update variant: subtract the row's current content length from the
+      # group total before checking the new one, so a same-size or smaller
+      # edit always passes even when the group is already at the limit.
+      def enforce_char_budget_for_update!(existing, new_content)
+        cfg = @config || Rubino.configuration
+        group = self.class.group_for_kind(existing[:kind])
+        limit = group == "user" ? cfg.memory_user_char_limit : cfg.memory_char_limit
+        return unless limit && limit > 0
+
+        current = total_chars_for_group(group) - existing[:content].to_s.length
+        requested = new_content.to_s.length
+        return if current + requested <= limit
+
+        raise BudgetExceededError.new(
+          group: group, limit: limit, current: current, requested: requested
+        )
+      end
+    end
+  end
+end