@residue/cli 0.0.3 → 0.0.4

package/src/lib/search-text.ts

@@ -0,0 +1,257 @@
+/**
+ * Lightweight text extractors for search indexing.
+ *
+ * These are NOT full conversation mappers. They produce a simple text
+ * representation optimized for embedding/search -- stripping thinking
+ * blocks, token metadata, tool output, signatures, cache data, etc.
+ */
+
+type SearchTextMetadata = {
+	sessionId: string;
+	agent: string;
+	commits: string[];
+	branch: string;
+	repo: string;
+};
+
+type SearchLine = {
+	role: "human" | "assistant" | "tool";
+	text: string;
+};
+
+/**
+ * Build the final search document from metadata + extracted lines.
+ */
+function buildSearchText(opts: {
+	metadata: SearchTextMetadata;
+	lines: SearchLine[];
+}): string {
+	const header = [
+		`Session: ${opts.metadata.sessionId}`,
+		`Agent: ${opts.metadata.agent}`,
+		opts.metadata.commits.length > 0
+			? `Commits: ${opts.metadata.commits.join(", ")}`
+			: null,
+		opts.metadata.branch ? `Branch: ${opts.metadata.branch}` : null,
+		opts.metadata.repo ? `Repo: ${opts.metadata.repo}` : null,
+	]
+		.filter(Boolean)
+		.join("\n");
+
+	const body = opts.lines
+		.map((line) => `[${line.role}] ${line.text}`)
+		.join("\n");
+
+	return `${header}\n\n${body}\n`;
+}
+
+// -- Claude Code extractor --------------------------------------------------
+
+type ClaudeContentBlock = {
+	type: string;
+	text?: string;
+	thinking?: string;
+	name?: string;
+	input?: Record<string, unknown>;
+	tool_use_id?: string;
+	content?: string | ClaudeContentBlock[];
+};
+
+type ClaudeEntry = {
+	type: string;
+	isMeta?: boolean;
+	isSidechain?: boolean;
+	message?: {
+		role?: string;
+		content?: string | ClaudeContentBlock[];
+	};
+};
+
+function extractClaudeCode(raw: string): SearchLine[] {
+	const lines: SearchLine[] = [];
+	if (!raw.trim()) return lines;
+
+	const entries: ClaudeEntry[] = [];
+	for (const line of raw.split("\n")) {
+		if (!line.trim()) continue;
+		try {
+			entries.push(JSON.parse(line) as ClaudeEntry);
+		} catch {
+			// skip malformed
+		}
+	}
+
+	for (const entry of entries) {
+		if (entry.isMeta || entry.isSidechain) continue;
+
+		if (entry.type === "user") {
+			const content = entry.message?.content;
+			if (!content) continue;
+
+			if (typeof content === "string") {
+				const trimmed = content.trim();
+				if (trimmed) lines.push({ role: "human", text: trimmed });
+			} else if (Array.isArray(content)) {
+				// Extract text blocks only, skip tool_result blocks
+				const hasToolResult = content.some((b) => b.type === "tool_result");
+				if (hasToolResult) continue;
+
+				const text = content
+					.filter((b) => b.type === "text" && b.text)
+					.map((b) => b.text!)
+					.join("\n")
+					.trim();
+				if (text) lines.push({ role: "human", text });
+			}
+		} else if (entry.type === "assistant") {
+			const content = entry.message?.content;
+			if (!Array.isArray(content)) continue;
+
+			for (const block of content) {
+				if (block.type === "text" && block.text) {
+					const trimmed = block.text.trim();
+					if (trimmed) lines.push({ role: "assistant", text: trimmed });
+				} else if (block.type === "tool_use" && block.name) {
+					// Extract tool name + short descriptor from input
+					const desc = summarizeToolInput(block.name, block.input);
+					lines.push({ role: "tool", text: desc });
+				}
+				// Skip thinking blocks, signatures, etc.
+			}
+		}
+		// Skip system, summary, progress, etc.
+	}
+
+	return lines;
+}
+
+// -- Pi extractor ------------------------------------------------------------
+
+type PiContentBlock = {
+	type: string;
+	text?: string;
+	thinking?: string;
+	name?: string;
+	arguments?: Record<string, unknown>;
+};
+
+type PiEntry = {
+	type: string;
+	message?: {
+		role?: string;
+		content?: string | PiContentBlock[];
+	};
+};
+
+function extractPi(raw: string): SearchLine[] {
+	const lines: SearchLine[] = [];
+	if (!raw.trim()) return lines;
+
+	const entries: PiEntry[] = [];
+	for (const line of raw.split("\n")) {
+		if (!line.trim()) continue;
+		try {
+			entries.push(JSON.parse(line) as PiEntry);
+		} catch {
+			// skip malformed
+		}
+	}
+
+	for (const entry of entries) {
+		if (entry.type !== "message") continue;
+		const msg = entry.message;
+		if (!msg) continue;
+
+		if (msg.role === "user") {
+			const content = msg.content;
+			if (!content) continue;
+
+			if (typeof content === "string") {
+				const trimmed = content.trim();
+				if (trimmed) lines.push({ role: "human", text: trimmed });
+			} else if (Array.isArray(content)) {
+				const text = content
+					.filter((b) => b.type === "text" && b.text)
+					.map((b) => b.text!)
+					.join("\n")
+					.trim();
+				if (text) lines.push({ role: "human", text });
+			}
+		} else if (msg.role === "assistant") {
+			const content = msg.content;
+			if (!Array.isArray(content)) {
+				if (typeof content === "string" && content.trim()) {
+					lines.push({ role: "assistant", text: content.trim() });
+				}
+				continue;
+			}
+
+			for (const block of content) {
+				if (block.type === "text" && block.text) {
+					const trimmed = block.text.trim();
+					if (trimmed) lines.push({ role: "assistant", text: trimmed });
+				} else if (block.type === "toolCall" && block.name) {
+					const desc = summarizeToolInput(block.name, block.arguments);
+					lines.push({ role: "tool", text: desc });
+				}
+				// Skip thinking blocks
+			}
+		}
+		// Skip toolResult, session, compaction, etc.
+	}
+
+	return lines;
+}
+
+// -- Shared helpers ----------------------------------------------------------
+
+/**
+ * Produce a short one-line summary of a tool invocation.
+ * Extracts file paths and commands when recognizable.
+ */
+function summarizeToolInput(
+	name: string,
+	input: Record<string, unknown> | undefined,
+): string {
+	if (!input) return name;
+
+	// Common tool patterns
+	const path =
+		input.path ?? input.file_path ?? input.filePath ?? input.filename;
+	if (typeof path === "string") return `${name} ${path}`;
+
+	const command = input.command ?? input.cmd;
+	if (typeof command === "string") {
+		// Truncate long commands
+		const short =
+			command.length > 120 ? command.slice(0, 120) + "..." : command;
+		return `${name} ${short}`;
+	}
+
+	const query = input.query ?? input.search ?? input.pattern;
+	if (typeof query === "string") return `${name} ${query}`;
+
+	return name;
+}
+
+// -- Public API --------------------------------------------------------------
+
+type ExtractorName = "claude-code" | "pi";
+
+const extractors: Record<ExtractorName, (raw: string) => SearchLine[]> = {
+	"claude-code": extractClaudeCode,
+	pi: extractPi,
+};
+
+function getExtractor(agent: string): ((raw: string) => SearchLine[]) | null {
+	return extractors[agent as ExtractorName] ?? null;
+}
+
+export {
+	buildSearchText,
+	extractClaudeCode,
+	extractPi,
+	getExtractor,
+	summarizeToolInput,
+};
+export type { SearchLine, SearchTextMetadata, ExtractorName };

package/test/lib/search-text.test.ts

@@ -0,0 +1,366 @@
+import { describe, expect, test } from "bun:test";
+import {
+	buildSearchText,
+	extractClaudeCode,
+	extractPi,
+	getExtractor,
+	summarizeToolInput,
+} from "@/lib/search-text";
+
+describe("search text extractors", () => {
+	describe("extractClaudeCode", () => {
+		test("extracts human and assistant messages from JSONL", () => {
+			const raw = [
+				JSON.stringify({
+					type: "user",
+					message: { role: "user", content: "fix the auth redirect" },
+				}),
+				JSON.stringify({
+					type: "assistant",
+					message: {
+						role: "assistant",
+						content: [{ type: "text", text: "I will update the middleware." }],
+					},
+				}),
+			].join("\n");
+
+			const lines = extractClaudeCode(raw);
+			expect(lines).toHaveLength(2);
+			expect(lines[0]).toEqual({
+				role: "human",
+				text: "fix the auth redirect",
+			});
+			expect(lines[1]).toEqual({
+				role: "assistant",
+				text: "I will update the middleware.",
+			});
+		});
+
+		test("extracts tool_use as tool lines with file path", () => {
+			const raw = JSON.stringify({
+				type: "assistant",
+				message: {
+					role: "assistant",
+					content: [
+						{
+							type: "tool_use",
+							name: "edit",
+							input: { path: "src/auth.ts" },
+						},
+					],
+				},
+			});
+
+			const lines = extractClaudeCode(raw);
+			expect(lines).toHaveLength(1);
+			expect(lines[0]).toEqual({ role: "tool", text: "edit src/auth.ts" });
+		});
+
+		test("skips thinking blocks", () => {
+			const raw = JSON.stringify({
+				type: "assistant",
+				message: {
+					role: "assistant",
+					content: [
+						{
+							type: "thinking",
+							thinking: "Let me think about this...",
+							signature: "abc123",
+						},
+						{ type: "text", text: "Here is my answer." },
+					],
+				},
+			});
+
+			const lines = extractClaudeCode(raw);
+			expect(lines).toHaveLength(1);
+			expect(lines[0].role).toBe("assistant");
+			expect(lines[0].text).toBe("Here is my answer.");
+		});
+
+		test("skips meta entries", () => {
+			const raw = [
+				JSON.stringify({
+					type: "user",
+					isMeta: true,
+					message: { role: "user", content: "auto-injected system stuff" },
+				}),
+				JSON.stringify({
+					type: "user",
+					message: { role: "user", content: "real user message" },
+				}),
+			].join("\n");
+
+			const lines = extractClaudeCode(raw);
+			expect(lines).toHaveLength(1);
+			expect(lines[0].text).toBe("real user message");
+		});
+
+		test("skips sidechain entries", () => {
+			const raw = JSON.stringify({
+				type: "assistant",
+				isSidechain: true,
+				message: {
+					role: "assistant",
+					content: [{ type: "text", text: "sidechain response" }],
+				},
+			});
+
+			const lines = extractClaudeCode(raw);
+			expect(lines).toHaveLength(0);
+		});
+
+		test("skips tool_result user entries", () => {
+			const raw = JSON.stringify({
+				type: "user",
+				message: {
+					role: "user",
+					content: [
+						{
+							type: "tool_result",
+							tool_use_id: "abc",
+							content: "some output",
+						},
+					],
+				},
+			});
+
+			const lines = extractClaudeCode(raw);
+			expect(lines).toHaveLength(0);
+		});
+
+		test("skips system and summary entries", () => {
+			const raw = [
+				JSON.stringify({ type: "system", message: { content: "init" } }),
+				JSON.stringify({ type: "summary", summary: "something" }),
+				JSON.stringify({ type: "progress" }),
+			].join("\n");
+
+			const lines = extractClaudeCode(raw);
+			expect(lines).toHaveLength(0);
+		});
+
+		test("handles empty input", () => {
+			expect(extractClaudeCode("")).toHaveLength(0);
+			expect(extractClaudeCode("  \n  ")).toHaveLength(0);
+		});
+
+		test("handles malformed JSON lines gracefully", () => {
+			const raw = [
+				"not json at all",
+				JSON.stringify({
+					type: "user",
+					message: { content: "valid line" },
+				}),
+				"{ broken json",
+			].join("\n");
+
+			const lines = extractClaudeCode(raw);
+			expect(lines).toHaveLength(1);
+		});
+	});
+
+	describe("extractPi", () => {
+		test("extracts human and assistant messages", () => {
+			const raw = [
+				JSON.stringify({
+					type: "message",
+					message: { role: "user", content: "add a search feature" },
+				}),
+				JSON.stringify({
+					type: "message",
+					message: {
+						role: "assistant",
+						content: [
+							{ type: "text", text: "I will add the search endpoint." },
+						],
+					},
+				}),
+			].join("\n");
+
+			const lines = extractPi(raw);
+			expect(lines).toHaveLength(2);
+			expect(lines[0]).toEqual({
+				role: "human",
+				text: "add a search feature",
+			});
+			expect(lines[1]).toEqual({
+				role: "assistant",
+				text: "I will add the search endpoint.",
+			});
+		});
+
+		test("extracts toolCall blocks", () => {
+			const raw = JSON.stringify({
+				type: "message",
+				message: {
+					role: "assistant",
+					content: [
+						{
+							type: "toolCall",
+							name: "bash",
+							arguments: { command: "git diff --staged" },
+						},
+					],
+				},
+			});
+
+			const lines = extractPi(raw);
+			expect(lines).toHaveLength(1);
+			expect(lines[0]).toEqual({
+				role: "tool",
+				text: "bash git diff --staged",
+			});
+		});
+
+		test("skips non-message entry types", () => {
+			const raw = [
+				JSON.stringify({ type: "session", message: {} }),
+				JSON.stringify({ type: "compaction" }),
+				JSON.stringify({
+					type: "message",
+					message: { role: "user", content: "hello" },
+				}),
+			].join("\n");
+
+			const lines = extractPi(raw);
+			expect(lines).toHaveLength(1);
+		});
+
+		test("skips toolResult entries", () => {
+			const raw = JSON.stringify({
+				type: "message",
+				message: {
+					role: "toolResult",
+					content: "some output text",
+				},
+			});
+
+			const lines = extractPi(raw);
+			expect(lines).toHaveLength(0);
+		});
+
+		test("handles assistant string content", () => {
+			const raw = JSON.stringify({
+				type: "message",
+				message: {
+					role: "assistant",
+					content: "simple string response",
+				},
+			});
+
+			const lines = extractPi(raw);
+			expect(lines).toHaveLength(1);
+			expect(lines[0]).toEqual({
+				role: "assistant",
+				text: "simple string response",
+			});
+		});
+
+		test("handles empty input", () => {
+			expect(extractPi("")).toHaveLength(0);
+		});
+	});
+
+	describe("summarizeToolInput", () => {
+		test("returns tool name when no input", () => {
+			expect(summarizeToolInput("read", undefined)).toBe("read");
+		});
+
+		test("extracts path from input", () => {
+			expect(summarizeToolInput("edit", { path: "src/index.ts" })).toBe(
+				"edit src/index.ts",
+			);
+		});
+
+		test("extracts file_path variant", () => {
+			expect(summarizeToolInput("read", { file_path: "README.md" })).toBe(
+				"read README.md",
+			);
+		});
+
+		test("extracts command from input", () => {
+			expect(summarizeToolInput("bash", { command: "ls -la" })).toBe(
+				"bash ls -la",
+			);
+		});
+
+		test("truncates long commands", () => {
+			const longCmd = "a".repeat(200);
+			const result = summarizeToolInput("bash", { command: longCmd });
+			expect(result.length).toBeLessThan(200);
+			expect(result).toContain("...");
+		});
+
+		test("extracts query from input", () => {
+			expect(summarizeToolInput("search", { query: "auth middleware" })).toBe(
+				"search auth middleware",
+			);
+		});
+
+		test("falls back to tool name for unknown input shape", () => {
+			expect(summarizeToolInput("custom_tool", { foo: 42, bar: true })).toBe(
+				"custom_tool",
+			);
+		});
+	});
+
+	describe("getExtractor", () => {
+		test("returns extractor for claude-code", () => {
+			expect(getExtractor("claude-code")).not.toBeNull();
+		});
+
+		test("returns extractor for pi", () => {
+			expect(getExtractor("pi")).not.toBeNull();
+		});
+
+		test("returns null for unknown agent", () => {
+			expect(getExtractor("unknown-agent")).toBeNull();
+		});
+	});
+
+	describe("buildSearchText", () => {
+		test("builds header + body from metadata and lines", () => {
+			const result = buildSearchText({
+				metadata: {
+					sessionId: "abc-123",
+					agent: "claude-code",
+					commits: ["abc1234", "def5678"],
+					branch: "feature-auth",
+					repo: "my-team/my-app",
+				},
+				lines: [
+					{ role: "human", text: "fix the bug" },
+					{ role: "assistant", text: "I will fix it." },
+					{ role: "tool", text: "edit src/fix.ts" },
+				],
+			});
+
+			expect(result).toContain("Session: abc-123");
+			expect(result).toContain("Agent: claude-code");
+			expect(result).toContain("Commits: abc1234, def5678");
+			expect(result).toContain("Branch: feature-auth");
+			expect(result).toContain("Repo: my-team/my-app");
+			expect(result).toContain("[human] fix the bug");
+			expect(result).toContain("[assistant] I will fix it.");
+			expect(result).toContain("[tool] edit src/fix.ts");
+		});
+
+		test("omits empty metadata fields", () => {
+			const result = buildSearchText({
+				metadata: {
+					sessionId: "abc",
+					agent: "pi",
+					commits: [],
+					branch: "",
+					repo: "",
+				},
+				lines: [{ role: "human", text: "hello" }],
+			});
+
+			expect(result).not.toContain("Commits:");
+			expect(result).not.toContain("Branch:");
+			expect(result).not.toContain("Repo:");
+		});
+	});
+});