claude_memory 0.9.1 → 0.10.0

data/lib/claude_memory/dashboard/server.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "webrick"
+require "json"
+
+module ClaudeMemory
+  module Dashboard
+    class Server
+      DEFAULT_PORT = 3377
+
+      def initialize(manager:, port: DEFAULT_PORT, open_browser: true)
+        @manager = manager
+        @port = port
+        @open_browser = open_browser
+        @server = nil
+      end
+
+      def start
+        @server = WEBrick::HTTPServer.new(
+          Port: @port,
+          Logger: WEBrick::Log.new(File::NULL),
+          AccessLog: []
+        )
+
+        mount_routes
+
+        trap("INT") { @server.shutdown }
+        trap("TERM") { @server.shutdown }
+
+        open_browser if @open_browser
+        @server.start
+      end
+
+      def stop
+        @server&.shutdown
+      end
+
+      private
+
+      def mount_routes
+        api = API.new(@manager)
+
+        @server.mount_proc("/") { |_req, res| serve_html(res) }
+        @server.mount_proc("/api/health") { |_req, res| with_fresh_connections { json_response(res, api.health) } }
+        @server.mount_proc("/api/stats") { |_req, res| with_fresh_connections { json_response(res, api.stats) } }
+        @server.mount_proc("/api/activity") { |req, res|
+          with_fresh_connections {
+            if (id = activity_id_from_path(req.path))
+              json_response(res, api.activity_detail(id))
+            else
+              json_response(res, api.activity(req.query))
+            end
+          }
+        }
+        @server.mount_proc("/api/facts") { |req, res| with_fresh_connections { handle_facts(api, req, res) } }
+        @server.mount_proc("/api/efficacy") { |req, res| with_fresh_connections { json_response(res, api.efficacy(req.query)) } }
+        @server.mount_proc("/api/session") { |req, res|
+          with_fresh_connections {
+            session_id = req.query["session_id"]
+            json_response(res, api.session_summary(session_id))
+          }
+        }
+        @server.mount_proc("/api/timeline") { |_req, res| with_fresh_connections { json_response(res, api.timeline) } }
+        @server.mount_proc("/api/recall") { |req, res| with_fresh_connections { json_response(res, api.recall(req.query)) } }
+        @server.mount_proc("/api/conflicts") { |req, res| with_fresh_connections { handle_conflicts(api, req, res) } }
+        @server.mount_proc("/api/moments") { |req, res| with_fresh_connections { handle_moments(api, req, res) } }
+        @server.mount_proc("/api/trust") { |_req, res| with_fresh_connections { json_response(res, api.trust) } }
+        @server.mount_proc("/api/knowledge") { |req, res| with_fresh_connections { json_response(res, api.knowledge(req.query)) } }
+        @server.mount_proc("/api/reuse") { |req, res| with_fresh_connections { json_response(res, api.reuse(req.query)) } }
+      end
+
+      # WAL-mode SQLite caches pages on reader connections; when the MCP
+      # server (or hooks, or any other writer) modifies the same DB
+      # concurrently, long-lived dashboard connections can see stale pages
+      # and surface "database disk image is malformed" errors even though
+      # PRAGMA integrity_check reports ok. Releasing connections after each
+      # HTTP request forces a fresh connection on the next read, matching
+      # what MCP::Server#release_connections does per tool call.
+      def with_fresh_connections
+        yield
+      ensure
+        release_connections
+      end
+
+      def release_connections
+        return unless @manager
+        @manager.global_store&.db&.disconnect
+        @manager.project_store&.db&.disconnect
+      rescue Sequel::DatabaseError, Extralite::Error
+        # Best-effort; next call will reopen.
+      end
+
+      def handle_moments(api, req, res)
+        feedback_id = moment_feedback_id_from_path(req.path)
+
+        if feedback_id && req.request_method == "POST"
+          body = parse_json_body(req)
+          json_response(res, api.moment_feedback(feedback_id, verdict: body["verdict"], note: body["note"]))
+        elsif feedback_id && req.request_method == "DELETE"
+          json_response(res, api.clear_moment_feedback(feedback_id))
+        else
+          json_response(res, api.moments(req.query))
+        end
+      end
+
+      def moment_feedback_id_from_path(path)
+        match = path.match(%r{\A/api/moments/(\d+)/feedback\z})
+        match && match[1]
+      end
+
+      def handle_conflicts(api, req, res)
+        reject_id = conflict_reject_id_from_path(req.path)
+        detail_id = conflict_id_from_path(req.path)
+        is_reject_similar = req.path == "/api/conflicts/reject_similar"
+
+        if req.request_method == "POST" && is_reject_similar
+          body = parse_json_body(req)
+          keeper_id = body["keeper_fact_id"]
+          reason = body["reason"]
+          scope = body["scope"] || req.query["scope"] || "project"
+          json_response(res, api.reject_similar_conflicts(keeper_id, reason: reason, scope: scope))
+        elsif req.request_method == "POST" && reject_id
+          body = parse_json_body(req)
+          side = body["side"]
+          reason = body["reason"]
+          scope = body["scope"] || req.query["scope"] || "project"
+          json_response(res, api.reject_conflict_fact(reject_id, side: side, reason: reason, scope: scope))
+        elsif detail_id
+          scope = req.query["scope"] || "project"
+          json_response(res, api.conflict_detail(detail_id, scope))
+        else
+          json_response(res, api.conflicts(req.query))
+        end
+      end
+
+      def parse_json_body(req)
+        return {} if req.body.nil? || req.body.empty?
+        JSON.parse(req.body)
+      rescue JSON::ParserError
+        {}
+      end
+
+      def serve_html(res)
+        html_path = File.expand_path("index.html", __dir__)
+        res["Content-Type"] = "text/html; charset=utf-8"
+        res.body = File.read(html_path)
+      end
+
+      def activity_id_from_path(path)
+        match = path.match(%r{\A/api/activity/(\d+)\z})
+        match && match[1]
+      end
+
+      def fact_id_from_path(path)
+        match = path.match(%r{\A/api/facts/(\d+)\z})
+        match && match[1]
+      end
+
+      def fact_action_from_path(path)
+        match = path.match(%r{\A/api/facts/(\d+)/(reject|promote)\z})
+        match ? [match[1], match[2]] : nil
+      end
+
+      def handle_facts(api, req, res)
+        action = fact_action_from_path(req.path)
+        detail_id = fact_id_from_path(req.path)
+
+        if req.request_method == "POST" && action
+          fact_id, verb = action
+          body = parse_json_body(req)
+          scope = body["scope"] || req.query["scope"] || "project"
+          case verb
+          when "reject"
+            json_response(res, api.reject_fact(fact_id, reason: body["reason"], scope: scope))
+          when "promote"
+            json_response(res, api.promote_fact(fact_id))
+          end
+        elsif detail_id
+          scope = req.query["scope"] || "project"
+          json_response(res, api.fact_detail(detail_id, scope))
+        else
+          json_response(res, api.facts(req.query))
+        end
+      end
+
+      def conflict_id_from_path(path)
+        match = path.match(%r{\A/api/conflicts/(\d+)\z})
+        match && match[1]
+      end
+
+      def conflict_reject_id_from_path(path)
+        match = path.match(%r{\A/api/conflicts/(\d+)/reject\z})
+        match && match[1]
+      end
+
+      def json_response(res, data)
+        res["Content-Type"] = "application/json; charset=utf-8"
+        res["Access-Control-Allow-Origin"] = "*"
+        res.body = JSON.generate(data)
+      end
+
+      def open_browser
+        url = "http://localhost:#{@port}"
+        Thread.new do
+          sleep 0.5
+          system("open", url) || system("xdg-open", url) || system("start", url)
+        end
+      end
+    end
+  end
+end

data/lib/claude_memory/dashboard/timeline.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Dashboard
+    # Daily activity rollup for the dashboard timeline view. Aggregates three
+    # event sources (fact creation, content ingestion, activity events) into
+    # per-day buckets covering the last 30 days. Returns the empty shape
+    # ({days: []}) when no project store is available so the dashboard can
+    # render before the first ingest.
+    class Timeline
+      LOOKBACK_DAYS = 30
+
+      def initialize(manager)
+        @manager = manager
+      end
+
+      def days
+        store = @manager.default_store(prefer: :project)
+        return {days: []} unless store
+
+        cutoff = (Time.now - LOOKBACK_DAYS * 86_400).utc.iso8601
+        {days: build_days(store, cutoff)}
+      end
+
+      private
+
+      def build_days(store, cutoff)
+        fact_rows = group_count(store.facts, cutoff_field: :created_at, cutoff: cutoff)
+        content_rows = group_count(store.content_items, cutoff_field: :ingested_at, cutoff: cutoff)
+        event_rows = activity_event_rows(store, cutoff)
+
+        all_days = (fact_rows + content_rows + event_rows).map { |r| r[:day] }.uniq.sort
+        all_days.map { |day| compose_day(day, fact_rows, content_rows, event_rows) }
+      end
+
+      def group_count(dataset, cutoff_field:, cutoff:)
+        dataset
+          .where { Sequel[cutoff_field] >= cutoff }
+          .select_group(Sequel.lit("DATE(#{cutoff_field})").as(:day))
+          .select_append { count(id).as(:count) }
+          .order(:day)
+          .all
+      end
+
+      def activity_event_rows(store, cutoff)
+        return [] unless store.db.table_exists?(:activity_events)
+
+        store.activity_events
+          .where { occurred_at >= cutoff }
+          .select_group(Sequel.lit("DATE(occurred_at)").as(:day), :event_type)
+          .select_append { count(id).as(:count) }
+          .order(:day)
+          .all
+      end
+
+      def compose_day(day, fact_rows, content_rows, event_rows)
+        day_events = event_rows.select { |r| r[:day] == day }
+        {
+          date: day,
+          facts_created: fact_rows.find { |r| r[:day] == day }&.dig(:count) || 0,
+          content_ingested: content_rows.find { |r| r[:day] == day }&.dig(:count) || 0,
+          hook_events: day_events.sum { |r| r[:count] },
+          recalls: day_events.select { |r| r[:event_type] == "recall" }.sum { |r| r[:count] }
+        }
+      end
+    end
+  end
+end

data/lib/claude_memory/dashboard/trust.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Dashboard
+    # Sidebar data for the feed-first dashboard. Three things:
+    #
+    # 1. Moments this week + week-over-week delta — the headline value number.
+    #    A moment is any meaningful activity event (recall hit, extraction,
+    #    context injection, conflict detected). Ingest-only events don't count
+    #    because they're not directly user-visible value.
+    #
+    # 2. "What memory knows about you" — up to 5 global facts rendered as
+    #    plain English. This is the trust panel's most compelling surface:
+    #    users can sanity-check what's being injected into their sessions.
+    #
+    # 3. Needs review — open conflicts plus facts that have gone stale
+    #    (active but never recalled in the last N days). A single actionable
+    #    count; the feed surfaces the individual items.
+    class Trust
+      WEEK_SECONDS = 7 * 86_400
+      UTILIZATION_DAYS = 30
+      VALUE_EVENT_TYPES = %w[hook_context recall store_extraction].freeze
+
+      def initialize(manager)
+        @manager = manager
+      end
+
+      def snapshot
+        {
+          weekly_moments: weekly_moments,
+          fingerprint: fingerprint,
+          needs_review: needs_review,
+          utilization: utilization,
+          feedback: feedback_summary
+        }
+      end
+
+      private
+
+      def weekly_moments
+        store = @manager.default_store(prefer: :project)
+        return {this_week: 0, last_week: 0, delta: 0, by_kind: {}} unless store
+
+        now = Time.now.utc
+        this_week_since = (now - WEEK_SECONDS).iso8601
+        last_week_since = (now - 2 * WEEK_SECONDS).iso8601
+
+        this_rows = valuable_events(store, this_week_since)
+        last_rows = valuable_events(store, last_week_since, before: this_week_since)
+
+        by_kind = this_rows.group_by { |r| r[:event_type] }.transform_values(&:size)
+
+        {
+          this_week: this_rows.size,
+          last_week: last_rows.size,
+          delta: this_rows.size - last_rows.size,
+          by_kind: by_kind
+        }
+      rescue Sequel::DatabaseError => e
+        ClaudeMemory.logger.debug("Trust#weekly_moments failed: #{e.message}")
+        {this_week: 0, last_week: 0, delta: 0, by_kind: {}}
+      end
+
+      def valuable_events(store, since, before: nil)
+        dataset = store.activity_events
+          .where(event_type: VALUE_EVENT_TYPES)
+          .where(status: "success")
+          .where { occurred_at >= since }
+        dataset = dataset.where { occurred_at < before } if before
+        dataset.all
+      end
+
+      # Up to 5 global facts rendered as plain-English sentences so a skeptical
+      # user can verify at-a-glance what's being injected into their Claude
+      # sessions. Prefers high-signal predicates (convention, decision,
+      # uses_framework, uses_database) and falls back to most-recent active.
+      def fingerprint
+        store = @manager.store_if_exists("global")
+        return [] unless store
+
+        preferred_predicates = %w[convention decision uses_framework uses_database uses_language]
+        rows = store.facts
+          .where(status: "active", scope: "global")
+          .where(predicate: preferred_predicates)
+          .order(Sequel.desc(:confidence), Sequel.desc(:created_at))
+          .limit(5)
+          .all
+
+        if rows.size < 5
+          extra = store.facts
+            .where(status: "active", scope: "global")
+            .exclude(id: rows.map { |r| r[:id] })
+            .order(Sequel.desc(:created_at))
+            .limit(5 - rows.size)
+            .all
+          rows += extra
+        end
+
+        presenter = FactPresenter.new(store)
+        presenter.list_summary(rows).map { |f| render_sentence(f) }
+      rescue Sequel::DatabaseError => e
+        ClaudeMemory.logger.debug("Trust#fingerprint failed: #{e.message}")
+        []
+      end
+
+      def render_sentence(fact)
+        predicate = fact[:predicate]
+        object = fact[:object]
+        subject = fact[:subject]
+
+        sentence = case predicate
+        when "convention"
+          object
+        when "decision"
+          object
+        when "uses_framework", "uses_language"
+          "Uses #{object}"
+        when "uses_database"
+          "Uses #{object} for storage"
+        when "deployment_platform"
+          "Deploys to #{object}"
+        when "auth_method"
+          "Auth via #{object}"
+        else
+          "#{subject} #{predicate.tr("_", " ")} #{object}"
+        end
+
+        {
+          id: fact[:id],
+          docid: fact[:docid],
+          sentence: sentence.to_s.strip,
+          predicate: predicate,
+          confidence: fact[:confidence]
+        }
+      end
+
+      def needs_review
+        {
+          open_conflicts: count_open_conflicts,
+          stale_facts: count_stale_facts,
+          empty_recalls: count_empty_recalls
+        }
+      end
+
+      def count_open_conflicts
+        Conflicts.new(@manager).distinct_open_counts
+      rescue Sequel::DatabaseError
+        {project: 0, global: 0, total: 0}
+      end
+
+      # User-supplied thumbs on feed moments. The ratio answers "when Claude
+      # surfaces something from memory, is the user signaling it was helpful?"
+      # Only moments recorded in the last UTILIZATION_DAYS count toward the
+      # ratio so old clicks don't distort an active week's signal.
+      #
+      # Shape: {up: Int, down: Int, net: Int, ratio_pct: Int, window_days: Int}
+      # ratio_pct = up / (up + down) × 100, or nil when there's no feedback.
+      def feedback_summary
+        store = @manager.default_store(prefer: :project)
+        return feedback_zero unless store
+
+        cutoff = (Time.now.utc - UTILIZATION_DAYS * 86_400).iso8601
+        rows = store.moment_feedback.where { recorded_at >= cutoff }.all
+        up = rows.count { |r| r[:verdict] == "up" }
+        down = rows.count { |r| r[:verdict] == "down" }
+        total = up + down
+        ratio_pct = total.zero? ? nil : ((up.to_f / total) * 100).round
+
+        {up: up, down: down, net: up - down, ratio_pct: ratio_pct, window_days: UTILIZATION_DAYS}
+      rescue Sequel::DatabaseError
+        feedback_zero
+      end
+
+      def feedback_zero
+        {up: 0, down: 0, net: 0, ratio_pct: nil, window_days: UTILIZATION_DAYS}
+      end
+
+      # "Stale" = active facts whose last_recalled_at is older than the
+      # configured threshold (or never set, with a grace window so freshly
+      # extracted facts don't show up as stale on day one).
+      #
+      # Backed by Recall::StaleDetector, which reads the column populated by
+      # Sweep::RecallTimestampRefresher. Replaces the older "active facts
+      # minus seen-in-recalls" approximation, which couldn't distinguish a
+      # never-touched 6-month-old fact from a freshly stored one.
+      def count_stale_facts
+        threshold = Configuration.new.stale_days
+        Recall::StaleDetector.stale_count(@manager, threshold_days: threshold)
+      rescue Sequel::DatabaseError, JSON::ParserError => e
+        ClaudeMemory.logger.debug("Trust#count_stale_facts failed: #{e.message}")
+        0
+      end
+
+      # The ROI signal: of the facts Claude has extracted into memory over the
+      # last UTILIZATION_DAYS, how many has Claude actually *used* (appeared
+      # in any recall or context injection's top_fact_ids)? Low ratios are
+      # themselves a signal — it means memory is accumulating knowledge but
+      # Claude isn't reaching for it. Anomalies worth surfacing honestly.
+      #
+      # Shape: {extracted: Int, used: Int, ratio_pct: Int, window_days: Int}
+      # Both counts are scope-union (project + global) so the headline number
+      # reflects everything memory did, not just one store.
+      def utilization
+        cutoff = (Time.now.utc - UTILIZATION_DAYS * 86_400).iso8601
+        extracted_pairs = extracted_fact_pairs(cutoff)
+        used_pairs = used_fact_pairs(cutoff)
+
+        extracted = extracted_pairs.size
+        # "Used" counted against the extracted set — a fact used but not
+        # extracted in this window (taught earlier, used now) is still
+        # re-use worth recognizing; count it too.
+        used_from_extracted = (used_pairs & extracted_pairs).size
+        used_total = used_pairs.size
+
+        ratio_pct = extracted.zero? ? 0 : ((used_from_extracted.to_f / extracted) * 100).round
+
+        {
+          extracted: extracted,
+          used: used_total,
+          used_from_extracted: used_from_extracted,
+          ratio_pct: ratio_pct,
+          window_days: UTILIZATION_DAYS
+        }
+      rescue Sequel::DatabaseError, JSON::ParserError => e
+        ClaudeMemory.logger.debug("Trust#utilization failed: #{e.message}")
+        {extracted: 0, used: 0, used_from_extracted: 0, ratio_pct: 0, window_days: UTILIZATION_DAYS}
+      end
+      public :utilization
+
+      # Facts that were extracted (distilled + stored) within the window.
+      # Returns (scope, id) pairs across both stores.
+      def extracted_fact_pairs(cutoff)
+        pairs = Set.new
+        %w[project global].each do |scope|
+          store = @manager.store_if_exists(scope)
+          next unless store
+          store.facts
+            .where(status: "active")
+            .where { created_at >= cutoff }
+            .select(:id)
+            .all
+            .each { |r| pairs << [scope, r[:id]] }
+        end
+        pairs
+      end
+
+      # Facts that appeared as top_fact_ids in any recall or context injection
+      # within the window. Returns (scope, id) pairs.
+      def used_fact_pairs(cutoff)
+        store = @manager.default_store(prefer: :project)
+        return Set.new unless store
+        pairs = Set.new
+        store.activity_events
+          .where(event_type: %w[recall hook_context], status: "success")
+          .where { occurred_at >= cutoff }
+          .select(:detail_json)
+          .all
+          .each do |row|
+            details = row[:detail_json] ? JSON.parse(row[:detail_json]) : {}
+            scoped = ScopedFactResolver.scoped_ids_from_details(details)
+            ScopedFactResolver.flat_pairs(scoped).each { |pair| pairs << pair }
+          end
+        pairs
+      end
+
+      def count_empty_recalls
+        store = @manager.default_store(prefer: :project)
+        return 0 unless store
+
+        cutoff = (Time.now.utc - WEEK_SECONDS).iso8601
+        store.activity_events
+          .where(event_type: "recall")
+          .where(status: "success")
+          .where { occurred_at >= cutoff }
+          .all
+          .count do |row|
+            details = row[:detail_json] ? JSON.parse(row[:detail_json]) : {}
+            (details["result_count"] || 0).zero?
+          end
+      rescue Sequel::DatabaseError, JSON::ParserError
+        0
+      end
+    end
+  end
+end

data/lib/claude_memory/distill/reference_material_detector.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module ClaudeMemory
+  module Distill
+    # Guards against the LLM distiller mislabeling reference material as
+    # `convention`. Audited in production data on 2026-04-24: project facts
+    # labeled `predicate=convention` with objects like "Cloud-backed Claude
+    # Code plugin (~1,195 LOC JavaScript) using Supermemory API…" and
+    # "Claude Code plugin with marketplace.json, 5,700+ stars, by Tobi Lütke."
+    # These are descriptions of external projects, not conventions the user
+    # applies. Leaving them under `convention` pollutes the Knowledge-base
+    # sidebar and the `memory.conventions` MCP tool.
+    #
+    # Heuristic: only conventions are re-examined (decisions and architecture
+    # notes about external projects are legitimately those predicates). A
+    # convention is retagged to `reference` when its object text matches any
+    # of the descriptive patterns below. Kept deliberately conservative —
+    # false-positive retagging is worse than occasionally missing a case, so
+    # the patterns target telltale numeric/attribution phrases that rarely
+    # appear in real conventions.
+    class ReferenceMaterialDetector
+      # Strong signals — any one of these on its own justifies reclassification.
+      # Kept tight to avoid false positives on real conventions that happen
+      # to quote external project names.
+      STRONG_PATTERNS = [
+        # Line-of-code counts: "~1,195 LOC", "1200 lines of code"
+        /~?\d+[,.]?\d*\s*(?:LOC|lines of code)/i,
+        # Star counts: "5,700+ stars", "3.2k stars"
+        /\d[\d,.]*\+?\s*(?:k\s+)?stars?\b/i,
+        # "X is a (plugin|library|tool|gem|service|framework|extension) …"
+        /\b(?:is\s+an?|are)\s+(?:cloud-backed\s+)?(?:plugin|library|tool|gem|service|framework|extension|cli|mcp\s+server)\b/i,
+        # Leading descriptor: "Plugin that…", "Library for…"
+        /\A(?:cloud-backed\s+)?(?:plugin|library|tool|gem|service|framework|extension|cli|mcp\s+server)(?:\s+(?:with|using|for|that))/i
+      ].freeze
+
+      # Weak signals — only fire in combination with a strong signal.
+      # Author attribution ("by Jane Doe") was originally a standalone
+      # trigger, but production text like "MCP launched by Claude Code run
+      # from PATH" contains the same surface pattern inside a legitimate
+      # convention. Requiring a co-occurring strong signal keeps the guard
+      # conservative.
+      WEAK_PATTERNS = [
+        /\bby\s+[[:upper:]][[:alpha:]'-]+\s+[[:upper:]][[:alpha:]'-]+/
+      ].freeze
+
+      # Predicates we inspect. Decisions stay decisions even when they cite
+      # external projects ("From QMD restudy: adopt X"); the guard targets
+      # only `convention`, where misclassification is most common.
+      GUARDED_PREDICATES = %w[convention].freeze
+
+      def reclassify(extraction)
+        return extraction if extraction.facts.nil? || extraction.facts.empty?
+
+        new_facts = extraction.facts.map do |fact|
+          if reference_material?(fact)
+            fact.merge(predicate: "reference")
+          else
+            fact
+          end
+        end
+
+        Distill::Extraction.new(
+          entities: extraction.entities,
+          facts: new_facts,
+          decisions: extraction.decisions,
+          signals: extraction.signals
+        )
+      end
+
+      def reference_material?(fact)
+        return false unless GUARDED_PREDICATES.include?(fact[:predicate].to_s)
+        object = fact[:object].to_s
+        return false if object.empty?
+        STRONG_PATTERNS.any? { |re| object.match?(re) }
+      end
+    end
+  end
+end