@tznthou/ccrecall 0.5.6 → 0.7.0

package/hooks/README.md

@@ -23,6 +23,7 @@ If the daemon is not running, hooks log a warning to stderr and exit cleanly —
 |--------|------------------|--------|
 | `session-start.mjs` | `SessionStart` | GET `/memory/startup` with the project name (3-tier: cold + recent-confidence + FTS fallback), write matching memories to stdout (Claude prepends them to context) |
 | `session-end.mjs` | `SessionEnd` | POST `/session/end` to harvest the just-ended session into a memory |
+| `user-prompt-submit.mjs` | `UserPromptSubmit` | GET `/memory/prompt` with the prompt's topics, write related memories to stdout. Runs on **every** prompt — see the restraint notes below |
 
 ## Finding the Hook Scripts
 
@@ -70,11 +71,24 @@ Add to `~/.claude/settings.json` (or a project-scoped `.claude/settings.json`).
           }
         ]
       }
+    ],
+    "UserPromptSubmit": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node {HOOKS_DIR}/user-prompt-submit.mjs"
+          }
+        ]
+      }
     ]
   }
 }
 ```
 
+Omit the `UserPromptSubmit` block if you only want memories at session start —
+it is the one hook that runs on every prompt.
+
 Restart any running Claude Code sessions — settings changes don't hot-reload.
 
 ## Environment Variables
@@ -124,4 +138,5 @@ Check the daemon side: `~/Library/Logs/ccrecall/ccrecall.err.log` (macOS) or the
   - SessionStart: `source === 'resume'` means the context is already loaded
 - **SessionStart query strategy**: uses the last path segment of `cwd` as a keyword. Default strategy `startup-v1` calls `/memory/startup` (3-tier: cold + recent-confidence + FTS fallback); legacy strategy calls `/memory/query` (project-name FTS only). Controlled by `CCRECALL_SESSION_START_STRATEGY` env var
 - **SessionStart stdout = context injection**: only memories are written to stdout; all errors and diagnostics go to stderr to avoid polluting Claude's context
-- **Timeouts**: SessionEnd 5s, SessionStart 2s (tighter because it sits on the pre-prompt critical path)
+- **Timeouts**: SessionEnd 5s, SessionStart 2s (tighter because it sits on the pre-prompt critical path), UserPromptSubmit 300ms (tightest — it blocks every prompt)
+- **UserPromptSubmit restraint**: this hook fires on every prompt, and injected context accumulates in the conversation rather than being replaced ([claude-code#40216](https://github.com/anthropics/claude-code/issues/40216)), so each injection is permanent weight. It therefore skips short prompts and slash commands without any network call, suppresses memories already surfaced in the session, and stops entirely after 8 memories per session. Set `CCRECALL_PROMPT_RECALL=off` to disable it without editing `settings.json`

package/hooks/session-start.mjs

@@ -18,6 +18,10 @@ const HOST = '127.0.0.1'
 const TIMEOUT_MS = 2000
 const MEMORY_LIMIT = 5
 const MAX_TOKENS = 300
+// A key longer than this is not rendered at all rather than truncated: a clipped
+// key is a handle that silently resolves to nothing. Observed keys run to 52
+// characters, while the write schema permits 100.
+const MAX_KEY_CHARS = 60
 const STRATEGY = process.env.CCRECALL_SESSION_START_STRATEGY ?? 'startup-v1'
 const TELEMETRY_OFF = process.env.CCRECALL_TELEMETRY === 'off'
 const TELEMETRY_PATH = path.join(os.homedir(), '.ccrecall', 'startup-recall.log.jsonl')
@@ -132,18 +136,35 @@ function formatMemories(memories, query) {
   ].join('\n')
 }
 
-function formatStartupV1(memories, query, totalMemoryCount) {
+function formatStartupV1(memories, totalMemoryCount) {
   if (memories.length === 0 && totalMemoryCount === 0) return ''
   const lines = memories.map((m) => {
     const conf = m.confidence != null && m.confidence !== 1
       ? ` (conf ${Number(m.confidence).toFixed(2)})`
       : ''
-    return `- ${m.content}${conf}`
+    // #77: each line is a ~150-char excerpt of a memory averaging ~600, and the
+    // cut typically lands before the conclusion. The key is the only handle that
+    // makes the rest reachable — without it the reader can see that a line was
+    // clipped (the ellipsis) but has no way to ask for the remainder. Appended
+    // rather than prefixed so the content still leads.
+    const handle = m.key && m.key.length <= MAX_KEY_CHARS ? ` [key: ${m.key}]` : ''
+    return `- ${m.content}${conf}${handle}`
   })
   const parts = ['[ccRecall memory recall]', '']
-  if (lines.length > 0) parts.push(...lines, '')
-  if (totalMemoryCount > memories.length) {
-    parts.push(`(${totalMemoryCount} memories available — use recall_query to search more)`)
+  if (lines.length > 0) {
+    // The old footer only reported how many rows existed, which implied the rows
+    // shown were whole. Say plainly that they are excerpts, and name the way out.
+    const available = totalMemoryCount > memories.length
+      ? ` ${totalMemoryCount} memories available.`
+      : ''
+    parts.push(
+      ...lines,
+      '',
+      `(Each line is a ~150-char excerpt — conclusions are often past the cut.`
+      + ` recall_query a [key] for the full text.${available})`,
+    )
+  } else if (totalMemoryCount > 0) {
+    parts.push(`(${totalMemoryCount} memories available — use recall_query to search.)`)
   }
   return parts.join('\n')
 }
@@ -182,7 +203,7 @@ async function main() {
       emittedIds: result.emittedIds,
       droppedCount: result.droppedCount,
     })
-    text = formatStartupV1(memories, query, totalMemoryCount)
+    text = formatStartupV1(memories, totalMemoryCount)
   } else {
     const memories = await queryLegacy(query, projectId)
     text = formatMemories(memories, query)

package/hooks/user-prompt-submit.mjs

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+// ccRecall UserPromptSubmit hook — mid-conversation recall (L1).
+// Wired via ~/.claude/settings.json hooks.UserPromptSubmit. Stdout is added to
+// Claude's context for this turn; errors go to stderr only.
+//
+// Why this exists: SessionStart injects once, at second zero. Everything after
+// it — however long the session runs, whatever subject it turns to — had no
+// memory access at all, which is why recall_query accounted for under 1% of all
+// memory surfacing. This is the second trigger point.
+//
+// Why it is deliberately timid:
+//   - It runs on EVERY prompt, blocking the user's path to a response. A slow
+//     or dead daemon must cost nothing, so the timeout is short and every
+//     failure path exits 0 with no output.
+//   - additionalContext accumulates in history rather than being request-scoped
+//     (anthropics/claude-code#40216), so each injection is permanent context
+//     weight. Per-session dedup and a hard ceiling live in the daemon; the
+//     cheap skips (short prompts, slash commands) live here so they cost no
+//     round trip at all.
+//
+// Plain stdout is used rather than hookSpecificOutput JSON: the docs support
+// both for this event, and the JSON form errors on the first message of a new
+// session (anthropics/claude-code#17550).
+import http from 'node:http'
+
+const PORT = Number(process.env.CCRECALL_PORT ?? 7749)
+const HOST = '127.0.0.1'
+// This hook runs on every prompt, so it needs a kill switch that does not
+// require editing settings.json — same shape as CCRECALL_SESSION_START_STRATEGY.
+const DISABLED = process.env.CCRECALL_PROMPT_RECALL === 'off'
+// 300ms: a localhost topic lookup over an indexed table answers in well under
+// 50ms. This is generous headroom, and still short enough that a hung daemon is
+// imperceptible rather than a stall the user feels on every prompt.
+const TIMEOUT_MS = 300
+// Below this a prompt is almost always a continuation that introduces no new
+// subject, so there is nothing to recall against and no reason to pay for a
+// round trip. Two thresholds because one code-point count cannot serve both
+// scripts: 「這個 bug 怎麼修」 carries a full question in 10 characters while
+// "ok go ahead now" carries none in 15. The codebase already draws this
+// distinction (MIN_TOPIC_LENGTH 3 vs MIN_TOPIC_LENGTH_CJK 2 in topic-extractor).
+const MIN_PROMPT_CHARS = 12
+const MIN_PROMPT_CHARS_CJK = 6
+const CJK_REGEX = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u
+// Prompt text is only a topic-extraction seed; the opening sentences carry the
+// subject, and a pasted file would otherwise push a huge query string.
+const MAX_QUERY_CHARS = 300
+// A key longer than this is not rendered at all rather than truncated: a clipped
+// key is a handle that silently resolves to nothing. Observed keys run to 52
+// characters, while the write schema permits 100.
+const MAX_KEY_CHARS = 60
+
+async function readStdin() {
+  const chunks = []
+  for await (const chunk of process.stdin) chunks.push(chunk)
+  return Buffer.concat(chunks).toString('utf8')
+}
+
+function httpGet(pathWithQuery) {
+  return new Promise((resolve) => {
+    const req = http.request({
+      hostname: HOST,
+      port: PORT,
+      path: pathWithQuery,
+      method: 'GET',
+      timeout: TIMEOUT_MS,
+    }, (res) => {
+      let body = ''
+      res.on('data', (c) => { body += c })
+      res.on('end', () => {
+        if (res.statusCode !== 200) {
+          console.error(`[ccRecall] prompt recall failed ${res.statusCode}`)
+          resolve(null)
+          return
+        }
+        try {
+          resolve(JSON.parse(body))
+        } catch {
+          resolve(null)
+        }
+      })
+    })
+    // Diagnostics go to stderr, never stdout: stdout is context. Without these
+    // a dead daemon is indistinguishable from "nothing relevant" — the same
+    // silent-failure shape this release fixes on the extraction side.
+    req.on('error', (err) => {
+      console.error(`[ccRecall] prompt recall error: ${err.message} (service on :${PORT}?)`)
+      resolve(null)
+    })
+    req.on('timeout', () => {
+      req.destroy()
+      console.error(`[ccRecall] prompt recall timeout after ${TIMEOUT_MS}ms`)
+      resolve(null)
+    })
+    req.end()
+  })
+}
+
+// Matches Claude Code's project-directory encoding under ~/.claude/projects/
+function projectIdFromCwd(cwd) {
+  if (!cwd || typeof cwd !== 'string') return ''
+  return cwd.replace(/\//g, '-')
+}
+
+function formatRecall(memories) {
+  if (!Array.isArray(memories) || memories.length === 0) return ''
+  const lines = memories.map((m) => {
+    const handle = m.key && m.key.length <= MAX_KEY_CHARS ? ` [key: ${m.key}]` : ''
+    return `- ${m.content}${handle}`
+  })
+  return [
+    '[ccRecall — related to what you just asked]',
+    '',
+    ...lines,
+    '',
+    '(Excerpts — recall_query a [key] for the full text.)',
+  ].join('\n')
+}
+
+async function main() {
+  if (DISABLED) return
+  let input
+  try {
+    const raw = await readStdin()
+    input = raw ? JSON.parse(raw) : {}
+  } catch {
+    return
+  }
+
+  const prompt = typeof input.prompt === 'string' ? input.prompt.trim() : ''
+  if (!prompt) return
+  // Slash commands address the harness, not the subject matter.
+  if (prompt.startsWith('/')) return
+  const minChars = CJK_REGEX.test(prompt) ? MIN_PROMPT_CHARS_CJK : MIN_PROMPT_CHARS
+  if (Array.from(prompt).length < minChars) return
+
+  const projectId = projectIdFromCwd(input.cwd)
+  if (!projectId) return
+
+  // Neither limit nor maxTokens is sent: the daemon owns budget policy, and it
+  // can be changed by upgrading the service rather than by rewriting a hook
+  // file already installed in the user's settings.
+  const params = new URLSearchParams({
+    project: projectId,
+    q: Array.from(prompt).slice(0, MAX_QUERY_CHARS).join(''),
+  })
+  if (input.session_id) params.set('sessionId', input.session_id)
+
+  const result = await httpGet(`/memory/prompt?${params.toString()}`)
+  if (!result || result.throttled) return
+
+  const text = formatRecall(result.memories)
+  if (text) process.stdout.write(text + '\n')
+}
+
+main().catch((err) => {
+  // Never let a recall failure interfere with the user's prompt.
+  console.error(`[ccRecall] prompt recall failed: ${err.message}`)
+})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tznthou/ccrecall",
-  "version": "0.5.6",
+  "version": "0.7.0",
   "description": "AI memory service for Claude Code — on-demand context injection with metacognition",
   "type": "module",
   "main": "dist/index.js",

package/scripts/backfill-memory-topics.ts

@@ -1,29 +1,86 @@
 // SPDX-License-Identifier: Apache-2.0
 //
-// One-off script: backfill memory_topics for existing memories + clean orphans.
-// Phase 3.0 prerequisite — Tier 0 cross-project queries need memory_topics.
+// Maintenance script for memory_topics.
+//
+// Two modes:
+//   backfill (default) — topics for memories that have none. Phase 3.0
+//     prerequisite: Tier 0 cross-project queries need memory_topics.
+//   --rebuild          — recompute topics for EVERY memory. Required whenever
+//     extractTopicsFromContent changes (e.g. #80's Intl.Segmenter switch),
+//     because backfill skips any memory that already has rows and therefore
+//     leaves stale topics in place.
 //
 // Usage:
 //   pnpm tsx scripts/backfill-memory-topics.ts
+//   pnpm tsx scripts/backfill-memory-topics.ts --rebuild --dry-run
+//   pnpm tsx scripts/backfill-memory-topics.ts --rebuild
 //   pnpm tsx scripts/backfill-memory-topics.ts --db <path>
+//
+// After a --rebuild, run rebuildKnowledgeMap for each affected project so the
+// knowledge_map aggregate reflects the new topic keys.
 
 import path from 'node:path'
 import os from 'node:os'
 import { Database } from '../src/core/database.js'
 import { extractTopicsFromContent } from '../src/core/topic-extractor.js'
 
-const dbPath = process.argv.includes('--db')
-  ? process.argv[process.argv.indexOf('--db') + 1]
+const argv = process.argv
+const dbPath = argv.includes('--db')
+  ? argv[argv.indexOf('--db') + 1]
   : path.join(os.homedir(), '.ccrecall', 'ccrecall.db')
+const rebuild = argv.includes('--rebuild')
+const dryRun = argv.includes('--dry-run')
+
+// backfillMemoryTopics has no dry-run path, so the flag cannot be honoured in
+// default mode. Refuse rather than write while the user believes nothing is
+// being written.
+if (dryRun && !rebuild) {
+  console.error('--dry-run is only supported with --rebuild.')
+  console.error('Backfill mode has no preview path; re-run with --rebuild --dry-run,')
+  console.error('or drop --dry-run to backfill topic-less memories for real.')
+  process.exit(1)
+}
 
 console.log(`DB: ${dbPath}`)
+console.log(`Mode: ${rebuild ? 'rebuild (all memories)' : 'backfill (topic-less only)'}${dryRun ? ' [dry run]' : ''}`)
 
 const db = new Database(dbPath)
 
-const orphansRemoved = db.cleanOrphanedMemoryTopics()
-console.log(`Orphaned memory_topics removed: ${orphansRemoved}`)
+const orphansRemoved = dryRun ? 0 : db.cleanOrphanedMemoryTopics()
+if (!dryRun) console.log(`Orphaned memory_topics removed: ${orphansRemoved}`)
+
+if (rebuild) {
+  const cjkBefore = db.rawAll<{ c: number }>(
+    "SELECT COUNT(*) AS c FROM memory_topics WHERE topic_key GLOB '*[一-龥]*'",
+  )[0].c
+
+  const r = db.rebuildMemoryTopics(extractTopicsFromContent, { dryRun })
+
+  console.log(`Memories scanned            : ${r.scanned}`)
+  console.log(`Memories with changed topics: ${r.changed}`)
+  console.log(`memory_topics rows before   : ${r.topicsBefore}`)
+  console.log(`memory_topics rows after    : ${r.topicsAfter}`)
+  console.log(`  ...of which CJK before    : ${cjkBefore}`)
 
-const backfilled = db.backfillMemoryTopics(extractTopicsFromContent)
-console.log(`Memories backfilled with topics: ${backfilled}`)
+  if (!dryRun) {
+    const cjkAfter = db.rawAll<{ c: number }>(
+      "SELECT COUNT(*) AS c FROM memory_topics WHERE topic_key GLOB '*[一-龥]*'",
+    )[0].c
+    console.log(`  ...of which CJK after     : ${cjkAfter}`)
+    console.log('')
+    console.log('Next: rebuild knowledge_map for affected projects.')
+  } else {
+    console.log('')
+    console.log('Dry run — nothing written.')
+  }
+} else {
+  const backfilled = db.backfillMemoryTopics(extractTopicsFromContent)
+  console.log(`Memories backfilled with topics: ${backfilled}`)
+  if (backfilled === 0) {
+    console.log('')
+    console.log('0 processed. If the extractor changed, this mode cannot help —')
+    console.log('it only sees memories with no topics at all. Use --rebuild.')
+  }
+}
 
 db.close()

package/scripts/post-session-extract.sh

@@ -251,6 +251,29 @@ ${prompt}"
   # phrase (guaranteed to occur in this repo's own sessions).
   local stdout_marker
   stdout_marker=$(grep -m1 -oE '^Error: Reached max turns \([0-9]+\)$' "$stdout_tmp" 2>/dev/null)
+  # Second marker (#75): the model sometimes PRINTS `recall_save(...)` instead of
+  # invoking the MCP tool. That run exits 0 with empty stderr and writes nothing,
+  # which in telemetry is byte-identical to a session that genuinely had nothing
+  # worth saving — the reason this failure mode stayed invisible for months.
+  #
+  # Anchored at line start (leading indentation allowed, since the observed
+  # captures were fenced code blocks) so prose that merely mentions the tool
+  # — guaranteed to occur in this repo's own sessions — does not match. The
+  # anchoring bar is deliberately lower than the max-turns marker above: that
+  # one decides whether a run was a total loss, so a forged match would corrupt
+  # the verdict. This one only flags a row for follow-up and is meant to be read
+  # against the origin session's actual memory count (marker hit + zero writes =
+  # silent miss; marker hit + writes present = the model narrated as well as
+  # called). A false positive costs one row to check; a false negative restores
+  # the status quo of total invisibility.
+  #
+  # -c yields a bare count and never the matched line, so no session content can
+  # reach the telemetry log — the same constraint that governs the stdout capture.
+  local recall_save_text_count
+  recall_save_text_count=$(grep -cE '^[[:space:]]*recall_save[[:space:]]*\(' "$stdout_tmp" 2>/dev/null)
+  # grep exits 1 with empty output on no match (and $stdout_tmp is /dev/null when
+  # mktemp failed); --argjson would abort the whole telemetry write on a non-number.
+  [[ "$recall_save_text_count" =~ ^[0-9]+$ ]] || recall_save_text_count=0
   rm -f -- "$stdout_tmp" 2>/dev/null
   # Scrub common credential formats before stderr reaches the telemetry log.
   # claude echoes its own key (sk-ant-) in auth errors, and any MCP server
@@ -266,7 +289,14 @@ ${prompt}"
   local extract_duration=$(( extract_end - extract_start ))
 
   if [[ $extract_exit -eq 0 ]]; then
-    printf '✅ ccRecall: extraction complete (%ds).\n' "$extract_duration"
+    # A clean exit that printed call syntax is the silent-miss shape (#75): say so
+    # at the terminal, where it can still be acted on, instead of only in the log.
+    if [[ $recall_save_text_count -gt 0 ]]; then
+      printf '⚠️  ccRecall: extraction exited cleanly (%ds) but printed %d recall_save call(s) as text instead of invoking the tool — memories from this session may not have been saved.\n' \
+        "$extract_duration" "$recall_save_text_count"
+    else
+      printf '✅ ccRecall: extraction complete (%ds).\n' "$extract_duration"
+    fi
   else
     printf '⚠️  ccRecall: extraction exited with code %d (%ds)%s.\n' \
       "$extract_exit" "$extract_duration" "${stdout_marker:+ — $stdout_marker}"
@@ -275,6 +305,10 @@ ${prompt}"
   # Telemetry log (-c = one compact JSON object per line = valid JSONL).
   # stdoutMarker: "Error: Reached max turns (N)" when the turn budget was hit,
   # else "" — splits exit-1 rows into total-loss vs unclean-finish post-hoc (#56).
+  # recallSaveTextCount: how many recall_save calls the model printed as text
+  # rather than invoking — splits exit-0 zero-write rows into silent miss vs
+  # genuine no-op (#75). Absent on rows written before this marker shipped, so
+  # analysis must treat missing as unknown, not as 0.
   jq -n -c \
     --arg ts "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
     --arg sid "$session_id" \
@@ -283,7 +317,8 @@ ${prompt}"
     --arg marker "$stdout_marker" \
     --argjson exit "$extract_exit" \
     --argjson dur "$extract_duration" \
-    '{ts:$ts,sessionId:$sid,projectId:$pid,mode:"jsonl",exitCode:$exit,durationSec:$dur,stderr:$stderr,stdoutMarker:$marker}' \
+    --argjson textSaves "$recall_save_text_count" \
+    '{ts:$ts,sessionId:$sid,projectId:$pid,mode:"jsonl",exitCode:$exit,durationSec:$dur,stderr:$stderr,stdoutMarker:$marker,recallSaveTextCount:$textSaves}' \
     >> "$CCRECALL_EXTRACT_LOG"
 
   return $claude_exit