@gswangg/duncan-cc 0.1.0

package/README.md

@@ -0,0 +1,110 @@
+# duncan-cc
+
+Query dormant Claude Code sessions. The [Duncan Idaho approach](https://gswangg.net/posts/duncan-idaho-agent-memory) to agent memory, for CC.
+
+When CC sessions end or get compacted, their conversation history is still on disk. Duncan loads that history into a fresh LLM call and asks it your question — leveraging the model's native attention mechanism instead of summaries or search.
+
+## Install
+
+```bash
+npm install -g @gswangg/duncan-cc
+```
+
+Or from source:
+
+```bash
+git clone https://github.com/gswangg/duncan-cc.git
+cd duncan-cc
+npm install
+```
+
+## Configure CC
+
+```bash
+# If installed globally via npm:
+claude mcp add duncan -- npx @gswangg/duncan-cc
+
+# If installed from source:
+claude mcp add duncan -- npx tsx /path/to/duncan-cc/src/mcp-server.ts
+```
+
+## Authentication
+
+Duncan resolves auth automatically:
+
+1. Explicit apiKey/token parameter
+2. CC OAuth credentials (`~/.claude/.credentials.json`)
+3. `ANTHROPIC_API_KEY` environment variable
+
+## Tools
+
+### `duncan_query`
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `question` | string | ✓ | The question to ask |
+| `mode` | string | ✓ | `project`, `global`, `session`, `self`, `ancestors` |
+| `projectDir` | string | | For project mode |
+| `sessionId` | string | | For session mode |
+| `cwd` | string | | Working directory for context resolution |
+| `limit` | number | | Max sessions/windows (default: 10) |
+| `offset` | number | | Pagination offset |
+| `copies` | number | | For self mode: sample count (default: 3) |
+| `includeSubagents` | boolean | | Include subagent transcripts |
+
+### `duncan_list_sessions`
+
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `mode` | string | ✓ | `project` or `global` |
+| `projectDir` | string | | For project mode |
+| `cwd` | string | | Working directory |
+| `limit` | number | | Max sessions (default: 20) |
+
+## Routing modes
+
+| Mode | Target |
+|------|--------|
+| `project` | Sessions from same project dir (self-excluded) |
+| `global` | All sessions across all projects (self-excluded) |
+| `session` | Specific session by ID or path |
+| `self` | Own active window, queried N times for sampling diversity |
+| `ancestors` | Own prior compaction windows (excluding active) |
+
+## How it works
+
+Duncan replicates CC's full session-to-API pipeline, then substitutes its own query:
+
+1. Parse JSONL session file
+2. Relink preserved segments (compaction tree surgery)
+3. Walk parentUuid chain from leaf to root
+4. Post-process (merge split assistants, fix orphan tool results)
+5. Normalize messages (filter, convert types, merge, 8 post-transforms)
+6. Apply content replacements (persisted outputs from disk)
+7. Microcompact (truncate old tool results)
+8. Inject userContext (CLAUDE.md + date)
+9. Build system prompt (full parity with CC's static sections + dynamic context from project dir)
+10. Convert to API format
+11. Add prompt caching breakpoints
+12. Query with `duncan_response` structured output tool
+
+Self-exclusion: the calling session is identified by scanning for the MCP `toolUseId` in session file tails — deterministic, zero config, swarm-safe.
+
+## Known gaps
+
+- **MCP server instructions** — not available for dormant sessions (fetched live, not persisted)
+- **Tool schemas** — only `duncan_response` is sent; session's original tools aren't callable
+- **Compaction test coverage** — synthetic tests only; no real compacted sessions in test corpus
+
+## Tests
+
+```bash
+npm test
+```
+
+Corpus-dependent tests skip gracefully when `testdata/` is absent.
+
+## Related
+
+- [duncan-pi](https://github.com/gswangg/duncan-pi) — duncan for the [pi](https://github.com/badlogic/pi-mono) coding agent
+- [The Duncan Idaho Approach to Agent Memory](https://gswangg.net/posts/duncan-idaho-agent-memory) — design writeup

package/SPEC.md

@@ -0,0 +1,195 @@
+# Duncan for Claude Code — Spec
+
+## Overview
+
+Duncan-cc replicates CC's full message pipeline to hydrate dormant CC sessions,
+then queries them with questions via the Anthropic API. Exposed as an MCP server
+(stdio transport) with two tools: `duncan_query` and `duncan_list_sessions`.
+
+## Pipeline: Disk → API
+
+```
+Session file (.jsonl)
+  │
+  ▼
+Parse JSONL — separate transcript from metadata
+  │
+  ▼
+Preserved segment relinking (compaction tree surgery)
+  │
+  ▼
+Walk parentUuid chain from leaf to root
+  │
+  ▼
+Post-process: handle orphan tool results, deduplicate assistant splits
+  │
+  ▼
+Strip internal fields (isSidechain, parentUuid)
+  │
+  ▼
+Slice from last compact boundary onward
+  │
+  ▼
+Content replacements (persisted-output resolution from tool-results/)
+  │
+  ▼
+Microcompact (truncate old tool results)
+  │
+  ▼
+Normalize messages:
+  ├── Reorder attachments adjacent to referencing messages
+  ├── Filter: progress, non-local system, API error messages
+  ├── System messages → user messages (system-reminder wrapper)
+  ├── Strip tool_references from user messages
+  ├── Merge consecutive same-role messages
+  ├── Merge split assistant messages (same message.id)
+  ├── Convert attachment messages to user messages
+  │
+  ├── Post-transform 1: Relocate deferred tool_reference text
+  ├── Post-transform 2: Filter orphaned thinking-only assistant messages
+  ├── Post-transform 3: Remove trailing thinking from last assistant
+  ├── Post-transform 4: Remove whitespace-only assistants + re-merge users
+  ├── Post-transform 5: Fix empty assistant content (placeholder)
+  ├── Post-transform 6: Reorder system-reminder within tool_results
+  ├── Post-transform 7: Flatten error tool_results (text-only)
+  └── Post-transform 8: Fix orphaned tool_use (synthetic tool_result)
+  │
+  ▼
+Inject userContext (<system-reminder> with CLAUDE.md + date)
+  │
+  ▼
+Build system prompt (full parity with CC):
+  ├── Identity/intro
+  ├── System rules
+  ├── Coding instructions
+  ├── Careful actions guidelines
+  ├── Tool usage
+  ├── Tone and style
+  ├── Output efficiency
+  ├── Environment info (cwd, platform, model)
+  ├── CLAUDE.md (from session's original cwd)
+  ├── Memory (from project dir MEMORY.md)
+  └── Language preference
+  │
+  ▼
+Convert to API format: {role, content} only
+  │
+  ▼
+Add cache_control breakpoints:
+  ├── System prompt blocks: ephemeral cache
+  └── Penultimate message: ephemeral cache (session context boundary)
+  │
+  ▼
+Append duncan query as final user message
+  │
+  ▼
+messages.create() with duncan_response tool
+```
+
+## Routing Modes
+
+| Mode | Target | Self-exclusion |
+|------|--------|----------------|
+| `project` | All sessions in same project dir | ✅ via toolUseId |
+| `global` | All sessions across all projects | ✅ via toolUseId |
+| `session` | Specific session by ID/path | — |
+| `self` | Own active window, N copies (sampling diversity) | — (queries self intentionally) |
+| `ancestors` | Own prior compaction windows (excluding active) | Active window excluded |
+
+### Self-exclusion
+
+CC passes `toolUseId` in MCP request `_meta` as `"claudecode/toolUseId"`.
+The assistant message containing that tool_use is written to the session JSONL
+before the tool is invoked (`appendFileSync`). We scan the last 32KB of candidate
+session files for the ID to deterministically identify the calling session.
+
+### Self mode
+
+Sends the question to N copies of the active window for sampling diversity.
+Two-wave cache strategy:
+1. Wave 1: 1 query primes the cache (full input cost)
+2. Wave 2: remaining N-1 queries in batches (hit cached prefix)
+
+### Ancestors mode
+
+Queries compaction windows of the calling session excluding the active window.
+Returns nothing if the session has no compaction boundaries. In CC (no dfork
+lineage), "ancestors" = the compacted-away context from the current session.
+
+## Authentication
+
+Resolution order:
+1. Explicit apiKey/token parameter
+2. CC OAuth credentials (`~/.claude/.credentials.json`)
+3. `ANTHROPIC_API_KEY` environment variable
+
+## Prompt Caching
+
+Cache breakpoints placed on:
+- **System prompt**: each text block gets `cache_control: { type: "ephemeral" }`
+- **Messages**: breakpoint on last content block of penultimate message
+
+This caches the session context (stable across queries) while letting the duncan
+query question (last message) vary without invalidating cache.
+
+## System Prompt Reconstruction
+
+Static sections embedded verbatim from CC source:
+- Identity/intro, system rules, coding instructions, careful actions,
+  tool usage (conditionally includes per-tool instructions like "use Read
+  instead of cat" based on which tools appear in the session), tone/style,
+  output efficiency
+
+Dynamic sections reconstructed from session context:
+- **Environment**: from session JSONL metadata (cwd, model) + local filesystem
+- **CLAUDE.md**: from session's original cwd hierarchy (if paths exist)
+- **Memory**: from CC project dir (`~/.claude/projects/<hash>/memory/MEMORY.md`)
+- **Language**: configurable
+
+This matches CC's own resume behavior: rebuild system prompt from current state.
+
+Note: tool schemas are NOT included — duncan sends only its own `duncan_response`
+tool. The session's original tools are not callable during a duncan query.
+
+## Known Gaps
+
+### MCP Server Instructions
+CC injects MCP server `instructions` from the initialize handshake into the system
+prompt. Cannot reconstruct for dormant sessions — instructions are fetched live and
+not persisted to disk. Equivalent to resuming a CC session with tools disconnected.
+
+### Compaction Test Coverage
+No real CC sessions with compaction boundaries in the test corpus (CC's 30-day
+`cleanupPeriodDays` default purged older sessions before the corpus was captured).
+Compaction logic is tested with synthetic fixtures only.
+
+## Session Storage
+
+- **Config dir**: `~/.claude/`
+- **Projects dir**: `~/.claude/projects/`
+- **Project dir**: `~/.claude/projects/<hashed-cwd>/` (cwd with `/` → `-`)
+- **Session file**: `<project-dir>/<session-id>.jsonl`
+- **Subagent transcripts**: `<project-dir>/<session-id>/subagents/<subdir>/agent-<id>.jsonl`
+- **Tool results**: `<project-dir>/<session-id>/tool-results/<id>.txt`
+- **Memory**: `<project-dir>/memory/MEMORY.md`
+
+## MCP Server
+
+Two tools exposed via stdio transport:
+
+### duncan_query
+Query dormant sessions. Parameters:
+- `question` (required): the question to ask
+- `mode` (required): `project`, `global`, `session`, `self`, `ancestors`
+- `projectDir`: for project mode
+- `sessionId`: for session mode
+- `cwd`: working directory context
+- `limit`: max sessions/windows (default: 10)
+- `offset`: pagination offset
+- `copies`: for self mode, number of samples (default: 3)
+- `includeSubagents`: include subagent transcripts (default: false)
+
+### duncan_list_sessions
+List available sessions. Parameters:
+- `mode` (required): `project`, `global`
+- `projectDir`, `cwd`, `limit`

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@gswangg/duncan-cc",
+  "version": "0.1.0",
+  "description": "Query dormant Claude Code sessions — the Duncan Idaho approach to agent memory, for CC.",
+  "type": "module",
+  "bin": {
+    "duncan-cc": "./src/mcp-server.ts"
+  },
+  "scripts": {
+    "start": "npx tsx src/mcp-server.ts",
+    "test": "for t in tests/*.test.ts; do npx tsx \"$t\" || exit 1; done",
+    "test:parser": "npx tsx tests/parser-tree.test.ts",
+    "test:normalize": "npx tsx tests/normalize.test.ts",
+    "test:replacements": "npx tsx tests/content-replacements.test.ts",
+    "test:system": "npx tsx tests/system-prompt.test.ts",
+    "test:pipeline": "npx tsx tests/pipeline.test.ts",
+    "test:discovery": "npx tsx tests/discovery.test.ts",
+    "test:self-exclusion": "npx tsx tests/self-exclusion.test.ts"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.52.0",
+    "@modelcontextprotocol/sdk": "^1.12.1",
+    "tsx": "^4.21.0"
+  },
+  "devDependencies": {
+    "@babel/generator": "^7.29.1",
+    "@babel/parser": "^7.29.2",
+    "@babel/traverse": "^7.29.0",
+    "@babel/types": "^7.29.0"
+  },
+  "engines": {
+    "node": ">=22"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/gswangg/duncan-cc.git"
+  },
+  "license": "MIT"
+}

package/src/content-replacements.ts

@@ -0,0 +1,185 @@
+/**
+ * Content Replacements + Microcompact
+ * 
+ * Replicates CC's content replacement and microcompact transforms.
+ * 
+ * Content replacements: replace large tool_result content with persisted
+ * output references. The persisted outputs live in tool-results/ dirs.
+ * 
+ * Microcompact: on session resume after time gap, truncate old tool results.
+ */
+
+import { readFileSync, existsSync } from "node:fs";
+import { join, dirname } from "node:path";
+import type { CCMessage, ParsedSession } from "./parser.js";
+
+// ============================================================================
+// Content Replacements
+// ============================================================================
+
+const PERSISTED_OUTPUT_MARKER = "<persisted-output>";
+
+/**
+ * Apply content replacements to messages.
+ * 
+ * Two sources:
+ * 1. content-replacement entries from session metadata
+ * 2. persisted output files on disk (tool-results/ directory)
+ * 
+ * @param messages - messages to process
+ * @param parsed - parsed session with contentReplacements map
+ * @param sessionFile - path to session file (for resolving tool-results/ dir)
+ */
+export function applyContentReplacements(
+  messages: CCMessage[],
+  parsed: ParsedSession,
+  sessionFile?: string,
+): CCMessage[] {
+  // Build replacement map from session metadata
+  const replacements = new Map<string, string>();
+  for (const [, repls] of parsed.contentReplacements) {
+    for (const r of repls) {
+      if (r.kind === "tool-result" && r.toolUseId && r.replacement) {
+        replacements.set(r.toolUseId, r.replacement);
+      }
+    }
+  }
+
+  // Also check for persisted output files on disk
+  const toolResultsDir = sessionFile
+    ? join(dirname(sessionFile), basename(sessionFile), "tool-results")
+    : null;
+
+  if (replacements.size === 0 && !toolResultsDir) return messages;
+
+  return messages.map((msg) => {
+    if (msg.type !== "user") return msg;
+    const content = msg.message.content;
+    if (!Array.isArray(content)) return msg;
+
+    let changed = false;
+    const newContent = content.map((block) => {
+      if (block.type !== "tool_result") return block;
+
+      const toolUseId = block.tool_use_id;
+      if (!toolUseId) return block;
+
+      // Check metadata replacements first
+      const replacement = replacements.get(toolUseId);
+      if (replacement) {
+        changed = true;
+        return { ...block, content: replacement };
+      }
+
+      // Check if content is a persisted-output reference that we can resolve
+      const blockContent = typeof block.content === "string" ? block.content : "";
+      if (blockContent.includes(PERSISTED_OUTPUT_MARKER) && toolResultsDir) {
+        const resolved = resolvePersistedOutput(toolUseId, toolResultsDir);
+        if (resolved) {
+          changed = true;
+          return { ...block, content: resolved };
+        }
+      }
+
+      return block;
+    });
+
+    if (!changed) return msg;
+    return { ...msg, message: { ...msg.message, content: newContent } };
+  });
+}
+
+/**
+ * Try to resolve a persisted output from the tool-results directory.
+ * Files are named by tool_use_id or a hash.
+ */
+function resolvePersistedOutput(toolUseId: string, toolResultsDir: string): string | null {
+  if (!existsSync(toolResultsDir)) return null;
+
+  // Try exact match first
+  const exactPath = join(toolResultsDir, `${toolUseId}.txt`);
+  if (existsSync(exactPath)) {
+    try {
+      return readFileSync(exactPath, "utf-8");
+    } catch {
+      return null;
+    }
+  }
+
+  return null;
+}
+
+function basename(path: string): string {
+  return path.replace(/\.jsonl$/, "");
+}
+
+// ============================================================================
+// Microcompact — CC's Kp() / Oe9()
+// ============================================================================
+
+const MICROCOMPACT_PLACEHOLDER = "[content truncated — tool result from previous session segment]";
+
+/**
+ * Microcompact: truncate old tool results when there's a time gap.
+ * 
+ * CC does this on session resume after a gap > threshold minutes.
+ * For duncan, we apply it based on the time gap between the last
+ * assistant message and the current time (or a specified reference time).
+ * 
+ * @param messages - messages to process (post-normalization)
+ * @param gapThresholdMinutes - minutes of gap to trigger microcompact (default: 30)
+ * @param keepRecentTurns - number of recent turns to keep intact (default: 1)
+ */
+export function microcompact(
+  messages: CCMessage[],
+  gapThresholdMinutes: number = 30,
+  keepRecentTurns: number = 1,
+): CCMessage[] {
+  // Find the last assistant message
+  const lastAssistant = [...messages].reverse().find((m) => m.type === "assistant");
+  if (!lastAssistant) return messages;
+
+  const lastTime = Date.parse(lastAssistant.timestamp);
+  const now = Date.now();
+  const gapMinutes = (now - lastTime) / 60000;
+
+  if (!Number.isFinite(gapMinutes) || gapMinutes < gapThresholdMinutes) {
+    return messages;
+  }
+
+  // Identify tool_use IDs from recent turns to keep
+  const recentToolUseIds = new Set<string>();
+  const assistantMessages = messages.filter((m) => m.type === "assistant");
+  const recentAssistants = assistantMessages.slice(-keepRecentTurns);
+
+  for (const msg of recentAssistants) {
+    const content = msg.message.content;
+    if (!Array.isArray(content)) continue;
+    for (const block of content) {
+      if (block.type === "tool_use" && block.id) {
+        recentToolUseIds.add(block.id);
+      }
+    }
+  }
+
+  // Truncate old tool results
+  return messages.map((msg) => {
+    if (msg.type !== "user") return msg;
+    const content = msg.message.content;
+    if (!Array.isArray(content)) return msg;
+
+    let changed = false;
+    const newContent = content.map((block) => {
+      if (block.type !== "tool_result") return block;
+      if (recentToolUseIds.has(block.tool_use_id)) return block;
+      // Already truncated
+      if (block.content === MICROCOMPACT_PLACEHOLDER) return block;
+
+      changed = true;
+      return { ...block, content: MICROCOMPACT_PLACEHOLDER };
+    });
+
+    if (!changed) return msg;
+    return { ...msg, message: { ...msg.message, content: newContent } };
+  });
+}