@research-copilot/plugin 1.1.17 → 1.1.19

package/dist/skills/llm-wiki/workbench/server/src/digest/batch.ts

@@ -0,0 +1,189 @@
+import { mkdir, readFile, realpath, stat, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+import type { ModelRef } from "../config.js";
+import {
+	inspectKnowledgeBasePath,
+	validateInspectedSourceFiles,
+} from "../knowledge-bases.js";
+import { mapWithConcurrencyLimit } from "./concurrency.js";
+import { digestFileWithSubagent } from "./subagent.js";
+
+export type BatchDigestEvent =
+	| { type: "start"; total: number; concurrency: number; outputDir: string }
+	| { type: "file_start"; index: number; filePath: string }
+	| { type: "file_progress"; index: number; filePath: string; chars: number }
+	| { type: "file_complete"; index: number; filePath: string; outputPath: string }
+	| { type: "file_error"; index: number; filePath: string; error: string }
+	| { type: "done"; total: number; completed: number; failed: number; outputDir: string };
+
+export interface BatchDigestInput {
+	kbPath: string;
+	filePaths: string[];
+	concurrency?: number;
+	sourceScanId?: string;
+	digestModel?: ModelRef | null;
+}
+
+const ALLOWED_EXTS = new Set([".md", ".txt", ".pdf"]);
+
+export async function runBatchDigest(
+	input: BatchDigestInput,
+	emit: (event: BatchDigestEvent) => Promise<void>,
+): Promise<void> {
+	const validated = await validateBatchDigestInput(input);
+	const purpose = await readPurpose(validated.kbPath);
+	const outputDir = path.join(validated.kbPath, "wiki", "synthesis", "sessions");
+	await mkdir(outputDir, { recursive: true });
+
+	await emit({
+		type: "start",
+		total: validated.filePaths.length,
+		concurrency: validated.concurrency,
+		outputDir,
+	});
+
+	const outcomes = await mapWithConcurrencyLimit(
+		validated.filePaths,
+		validated.concurrency,
+		async (filePath, index) => {
+			await emit({ type: "file_start", index, filePath });
+			try {
+				const resolvedFilePath = await resolveDigestFile(
+					filePath,
+					validated.kbPath,
+					validated.sourceScanId,
+				);
+				const content = await digestFileWithSubagent(
+					{
+						kbPath: validated.kbPath,
+						filePath: resolvedFilePath,
+						purpose,
+						model: input.digestModel ?? null,
+					},
+					async (chars) => {
+						await emit({ type: "file_progress", index, filePath, chars });
+					},
+				);
+				const outputPath = await writeDigestOutput(outputDir, resolvedFilePath, index, content);
+				await emit({ type: "file_complete", index, filePath, outputPath });
+				return outputPath;
+			} catch (err) {
+				const error = err instanceof Error ? err.message : String(err);
+				await emit({ type: "file_error", index, filePath, error });
+				throw err;
+			}
+		},
+	);
+
+	const completed = outcomes.filter((outcome) => outcome.ok).length;
+	await emit({
+		type: "done",
+		total: validated.filePaths.length,
+		completed,
+		failed: validated.filePaths.length - completed,
+		outputDir,
+	});
+}
+
+async function validateBatchDigestInput(input: BatchDigestInput): Promise<{
+	kbPath: string;
+	filePaths: string[];
+	concurrency: 1 | 3 | 5;
+	sourceScanId?: string;
+}> {
+	const kbPath = await realpath(path.resolve(input.kbPath));
+	const kbCheck = await inspectKnowledgeBasePath(kbPath);
+	if (!kbCheck.valid) throw new Error(`不是合法知识库（${kbCheck.reason}）：${kbPath}`);
+
+	const concurrency = input.concurrency ?? 3;
+	if (![1, 3, 5].includes(concurrency)) {
+		throw new Error("concurrency 只能是 1、3 或 5");
+	}
+	if (!Array.isArray(input.filePaths) || input.filePaths.length === 0) {
+		throw new Error("filePaths 不能为空");
+	}
+
+	const filePaths: string[] = [];
+	for (const filePath of input.filePaths) {
+		if (typeof filePath !== "string" || !path.isAbsolute(filePath)) {
+			throw new Error("filePaths 必须是绝对路径");
+		}
+		filePaths.push(path.resolve(filePath));
+	}
+
+	return {
+		kbPath,
+		filePaths: Array.from(new Set(filePaths)),
+		concurrency: concurrency as 1 | 3 | 5,
+		...(input.sourceScanId ? { sourceScanId: input.sourceScanId } : {}),
+	};
+}
+
+async function resolveDigestFile(
+	filePath: string,
+	kbPath: string,
+	sourceScanId?: string,
+): Promise<string> {
+	const resolved = await realpath(filePath);
+	assertReadableDigestFile(resolved);
+	const info = await stat(resolved);
+	if (!info.isFile()) throw new Error(`不是文件：${resolved}`);
+	if (isInside(resolved, kbPath)) return resolved;
+	const sourceCheck = validateInspectedSourceFiles(sourceScanId, [filePath]);
+	if (sourceCheck.ok) return resolved;
+	throw new Error(sourceCheck.error);
+}
+
+function assertReadableDigestFile(filePath: string): void {
+	const ext = path.extname(filePath).toLowerCase();
+	if (!ALLOWED_EXTS.has(ext)) {
+		throw new Error(`不支持的文件类型：${filePath}`);
+	}
+}
+
+async function readPurpose(kbPath: string): Promise<string> {
+	for (const rel of ["purpose.md", ".wiki-schema.md"]) {
+		const content = await readFile(path.join(kbPath, rel), "utf8").catch(() => "");
+		if (content.trim()) return content.trim().slice(0, 2000);
+	}
+	return "";
+}
+
+async function writeDigestOutput(
+	outputDir: string,
+	sourcePath: string,
+	index: number,
+	content: string,
+): Promise<string> {
+	const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+	const base = path.basename(sourcePath, path.extname(sourcePath));
+	const slug = slugify(base) || `material-${index + 1}`;
+	const outputPath = path.join(outputDir, `${slug}-${timestamp}.md`);
+	const body = [
+		"---",
+		`source: ${JSON.stringify(sourcePath)}`,
+		`createdAt: ${JSON.stringify(new Date().toISOString())}`,
+		"stage: 3.5-batch-digest",
+		"---",
+		"",
+		content.trim(),
+		"",
+	].join("\n");
+	await writeFile(outputPath, body, "utf8");
+	return outputPath;
+}
+
+function isInside(target: string, root: string): boolean {
+	const relative = path.relative(root, target);
+	return relative === "" || (!!relative && !relative.startsWith("..") && !path.isAbsolute(relative));
+}
+
+function slugify(value: string): string {
+	return value
+		.normalize("NFKD")
+		.replace(/[^\w\u4e00-\u9fa5-]+/g, "-")
+		.replace(/-+/g, "-")
+		.replace(/^-|-$/g, "")
+		.slice(0, 60);
+}

package/dist/skills/llm-wiki/workbench/server/src/digest/concurrency.test.ts

@@ -0,0 +1,36 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import { mapWithConcurrencyLimit } from "./concurrency.js";
+
+test("limits active work to requested concurrency", async () => {
+	let active = 0;
+	let maxActive = 0;
+	const items = Array.from({ length: 10 }, (_, i) => i);
+	const outcomes = await mapWithConcurrencyLimit(items, 3, async (item) => {
+		active++;
+		maxActive = Math.max(maxActive, active);
+		await new Promise((resolve) => setTimeout(resolve, 5));
+		active--;
+		return item * 2;
+	});
+
+	assert.equal(maxActive, 3);
+	assert.deepEqual(
+		outcomes.map((outcome) => outcome.value),
+		items.map((item) => item * 2),
+	);
+	assert.ok(outcomes.every((outcome) => outcome.ok));
+});
+
+test("handles empty and smaller-than-concurrency inputs", async () => {
+	assert.deepEqual(await mapWithConcurrencyLimit([], 3, async (item) => item), []);
+	const outcomes = await mapWithConcurrencyLimit([1, 2], 5, async (item) => item + 1);
+	assert.deepEqual(outcomes.map((outcome) => outcome.value), [2, 3]);
+});
+
+test("rejects invalid concurrency", async () => {
+	await assert.rejects(() => mapWithConcurrencyLimit([1], 0, async (item) => item));
+	await assert.rejects(() => mapWithConcurrencyLimit([1], -1, async (item) => item));
+	await assert.rejects(() => mapWithConcurrencyLimit([1], Number.NaN, async (item) => item));
+});

package/dist/skills/llm-wiki/workbench/server/src/digest/concurrency.ts

@@ -0,0 +1,31 @@
+export interface Outcome<R> {
+	ok: boolean;
+	value?: R;
+	error?: string;
+}
+
+export async function mapWithConcurrencyLimit<T, R>(
+	items: T[],
+	concurrency: number,
+	fn: (item: T, index: number) => Promise<R>,
+): Promise<Outcome<R>[]> {
+	if (!Number.isInteger(concurrency) || concurrency < 1) {
+		throw new Error("concurrency must be a positive integer");
+	}
+	const results: Outcome<R>[] = new Array(items.length);
+	let cursor = 0;
+
+	async function worker() {
+		while (cursor < items.length) {
+			const i = cursor++;
+			try {
+				results[i] = { ok: true, value: await fn(items[i] as T, i) };
+			} catch (err) {
+				results[i] = { ok: false, error: err instanceof Error ? err.message : String(err) };
+			}
+		}
+	}
+
+	await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, worker));
+	return results;
+}

package/dist/skills/llm-wiki/workbench/server/src/digest/subagent.ts

@@ -0,0 +1,91 @@
+import { createAgentSession, SessionManager } from "@earendil-works/pi-coding-agent";
+
+import {
+	authStorage,
+	getConfiguredModel,
+	getResourceLoader,
+	getRoleModel,
+	modelRegistry,
+} from "../agent.js";
+import type { ModelRef } from "../config.js";
+
+export interface DigestFileInput {
+	kbPath: string;
+	filePath: string;
+	purpose: string;
+	model?: ModelRef | null;
+}
+
+export async function digestFileWithSubagent(
+	input: DigestFileInput,
+	onProgress?: (chars: number) => void | Promise<void>,
+): Promise<string> {
+	const loader = await getResourceLoader();
+	const model = input.model
+		? getConfiguredModel(input.model) ?? (await getRoleModel("digest"))
+		: await getRoleModel("digest");
+	const { session, modelFallbackMessage } = await createAgentSession({
+		cwd: input.kbPath,
+		resourceLoader: loader,
+		sessionManager: SessionManager.inMemory(input.kbPath),
+		authStorage,
+		modelRegistry,
+		...(model ? { model } : {}),
+		thinkingLevel: "off",
+		tools: ["read"],
+	});
+	if (modelFallbackMessage) console.log(`[digest] ${modelFallbackMessage}`);
+
+	let output = "";
+	let lastEmittedChars = 0;
+	let lastEmittedAt = 0;
+	const unsubscribe = session.subscribe((event) => {
+		if (event.type === "message_update" && event.assistantMessageEvent.type === "text_delta") {
+			output += event.assistantMessageEvent.delta;
+			const now = Date.now();
+			if (onProgress && (output.length - lastEmittedChars >= 500 || now - lastEmittedAt >= 300)) {
+				lastEmittedChars = output.length;
+				lastEmittedAt = now;
+				void onProgress(output.length);
+			}
+		}
+	});
+
+	try {
+		await session.prompt(buildDigestPrompt(input));
+		const trimmed = output.trim();
+		if (!trimmed) throw new Error("子代理没有返回内容");
+		await onProgress?.(trimmed.length);
+		return trimmed;
+	} finally {
+		unsubscribe();
+		session.dispose();
+	}
+}
+
+function buildDigestPrompt(input: DigestFileInput): string {
+	return [
+		"你是 llm-wiki 的素材消化子代理。",
+		"只处理一个文件，读完后输出一篇可直接保存到 wiki 的中文 Markdown 笔记。",
+		"不要改写文件，不要调用写入工具，不要输出过程说明。",
+		"",
+		`知识库路径：${input.kbPath}`,
+		`研究方向：${input.purpose || "未提供"}`,
+		`素材文件：${input.filePath}`,
+		"",
+		"输出结构：",
+		"# 标题",
+		"",
+		"## 核心摘要",
+		"用 3-5 条 bullet 说明最重要的信息。",
+		"",
+		"## 关键细节",
+		"整理事实、概念、数字、论点和可复用片段。",
+		"",
+		"## 与研究方向的关系",
+		"说明这份素材为什么值得放进当前知识库。",
+		"",
+		"## 后续问题",
+		"列出 2-5 个值得继续追问的问题。",
+	].join("\n");
+}

package/dist/skills/llm-wiki/workbench/server/src/extensions/artifacts.ts

@@ -0,0 +1,81 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+
+import type { ActiveContext } from "../agent.js";
+import {
+	type ArtifactKind,
+	finalizeArtifact,
+	prepareArtifact,
+} from "../artifacts.js";
+
+interface PrepareParams {
+	kind: ArtifactKind;
+	title: string;
+	sourceSkill?: string;
+}
+
+interface FinalizeParams {
+	id: string;
+	primaryFile: string;
+	sourceSkill?: string;
+}
+
+type ToolContent = { type: "text"; text: string };
+
+function toolResult(text: string, details: Record<string, unknown> = {}) {
+	return { content: [{ type: "text", text } satisfies ToolContent], details };
+}
+
+export function createArtifactsExtension(getActive: () => ActiveContext | null) {
+	return function artifactsExtension(pi: ExtensionAPI) {
+		pi.registerTool({
+			name: "prepare_artifact",
+			label: "准备产物目录",
+			description:
+				"为 HTML/PDF/Word/PPT/Excel 产物创建工作目录。生成文件前先调用它，返回 id 和 workspacePath。",
+			parameters: Type.Object({
+				kind: Type.Union([
+					Type.Literal("html"),
+					Type.Literal("pdf"),
+					Type.Literal("docx"),
+					Type.Literal("pptx"),
+					Type.Literal("xlsx"),
+				]),
+				title: Type.String(),
+				sourceSkill: Type.Optional(Type.String()),
+			}),
+			async execute(_toolCallId, params: PrepareParams) {
+				const active = getActive();
+				if (!active) return toolResult("当前没有活跃知识库。请先选择一个知识库。");
+				const prepared = await prepareArtifact({
+					kind: params.kind,
+					title: params.title,
+					sourceConversationId: active.conversationId,
+					sourceKbPath: active.kb.path,
+					sourceSkill: params.sourceSkill,
+				});
+				return toolResult(`已准备产物目录：${prepared.workspacePath}`, prepared);
+			},
+		});
+
+		pi.registerTool({
+			name: "finalize_artifact",
+			label: "登记产物",
+			description:
+				"文件生成完成后调用，登记 manifest.json 并通知前端打开右抽屉。primaryFile 必须是产物目录里的文件名。",
+			parameters: Type.Object({
+				id: Type.String(),
+				primaryFile: Type.String(),
+				sourceSkill: Type.Optional(Type.String()),
+			}),
+			async execute(_toolCallId, params: FinalizeParams) {
+				try {
+					const manifest = await finalizeArtifact(params);
+					return toolResult(`产物已登记：${manifest.id} / ${manifest.primaryFile}`, { manifest });
+				} catch (err) {
+					return toolResult(`登记产物失败：${err instanceof Error ? err.message : String(err)}`);
+				}
+			},
+		});
+	};
+}

package/dist/skills/llm-wiki/workbench/server/src/extensions/knowledge-base.ts

@@ -0,0 +1,263 @@
+/**
+ * 知识库上下文 Extension
+ *
+ * 阶段一 step 5：向 agent 注入"当前在哪个知识库"的概念。
+ * 不通过拼 system prompt 实现（ADR-7），而是注册工具让 agent 主动查询。
+ *
+ * 工具：
+ *   current_knowledge_base()        → 返回当前库路径 + 元数据
+ *   list_knowledge_base_pages()     → 列出 wiki/ 下所有 .md 文件
+ *
+ * 模块级状态：currentKnowledgeBase
+ *   HTTP 层通过 setCurrentKnowledgeBase() 改它，Extension 工具读它。
+ *   Extension 是单例（一个 process 一份），所以 module state 等价于应用状态。
+ */
+
+import type { Dirent } from "node:fs";
+import { readdir, readFile } from "node:fs/promises";
+import { join, relative } from "node:path";
+
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { Type } from "typebox";
+
+import { inspectKnowledgeBasePath } from "../knowledge-bases.js";
+import {
+	formatSearchResultsForTool,
+	searchKnowledgeBase,
+} from "../retrieval.js";
+
+// ============= 状态 =============
+
+interface KnowledgeBaseState {
+	path: string;
+	name: string;
+}
+
+let current: KnowledgeBaseState | null = null;
+let pendingKnowledgeContext: string | null = null;
+
+export function getCurrentKnowledgeBase(): KnowledgeBaseState | null {
+	return current;
+}
+
+export async function setCurrentKnowledgeBase(absolutePath: string): Promise<KnowledgeBaseState> {
+	// 共享的有效性检查（PRODUCT.md §6.2 约定）
+	const check = await inspectKnowledgeBasePath(absolutePath);
+	if (!check.valid) {
+		throw new Error(`不是合法知识库（${check.reason}）：${absolutePath}`);
+	}
+
+	const name = absolutePath.split("/").filter(Boolean).pop() ?? absolutePath;
+	current = { path: absolutePath, name };
+	return current;
+}
+
+export function clearCurrentKnowledgeBase(): void {
+	current = null;
+}
+
+export function setPendingKnowledgeContext(context: string): void {
+	pendingKnowledgeContext = context;
+}
+
+export function clearPendingKnowledgeContext(): void {
+	pendingKnowledgeContext = null;
+}
+
+// ============= Extension =============
+
+/**
+ * agent 不读这些目录 / 文件（PRODUCT.md §6.4 Obsidian 共存规则）
+ */
+const IGNORE_NAMES = new Set([
+	".obsidian",
+	".DS_Store",
+	".wiki-tmp",
+	".git",
+	"node_modules",
+	".venv",
+	"venv",
+	".cache",
+]);
+
+const IGNORE_SUFFIXES = [".base", ".canvas"];
+
+function shouldIgnore(name: string): boolean {
+	if (IGNORE_NAMES.has(name)) return true;
+	return IGNORE_SUFFIXES.some((suffix) => name.endsWith(suffix));
+}
+
+async function listMarkdownFiles(rootDir: string, base: string): Promise<string[]> {
+	const results: string[] = [];
+
+	async function walk(dir: string): Promise<void> {
+		let entries: Dirent[];
+		try {
+			entries = await readdir(dir, { withFileTypes: true });
+		} catch {
+			return;
+		}
+		for (const entry of entries) {
+			if (shouldIgnore(entry.name)) continue;
+			const full = join(dir, entry.name);
+			if (entry.isDirectory()) {
+				await walk(full);
+			} else if (entry.isFile() && entry.name.endsWith(".md")) {
+				results.push(relative(base, full));
+			}
+		}
+	}
+
+	await walk(rootDir);
+	results.sort();
+	return results;
+}
+
+export default function knowledgeBaseExtension(pi: ExtensionAPI) {
+	pi.on("before_agent_start", async (event) => {
+		const context = pendingKnowledgeContext;
+		if (!context) return undefined;
+		pendingKnowledgeContext = null;
+		return {
+			systemPrompt: `${event.systemPrompt}\n\n${context}`,
+		};
+	});
+
+	pi.registerTool({
+		name: "current_knowledge_base",
+		label: "当前知识库",
+		description:
+			"Get the absolute filesystem path and basic metadata of the user's currently active knowledge base. ALWAYS call this FIRST before reading, writing, or listing files in the user's knowledge base, since file paths depend on which library the user has selected. Returns an error message if no knowledge base is selected.",
+		parameters: Type.Object({}),
+		async execute(): Promise<{
+			content: { type: "text"; text: string }[];
+			details: Record<string, unknown>;
+		}> {
+			if (!current) {
+				return {
+					content: [
+						{
+							type: "text",
+							text: "No knowledge base is currently selected. Ask the user to pick one.",
+						},
+					],
+					details: { selected: false },
+				};
+			}
+
+			// 读 purpose.md 给 agent 一段"这是什么库"的提示
+			let purpose = "";
+			try {
+				purpose = (await readFile(join(current.path, "purpose.md"), "utf8")).slice(0, 500);
+			} catch {
+				// 没 purpose.md 也无所谓
+			}
+
+			const text = [
+				`Current knowledge base:`,
+				`  name: ${current.name}`,
+				`  path: ${current.path}`,
+				purpose ? `\nResearch direction (from purpose.md):\n${purpose}` : "",
+			]
+				.filter(Boolean)
+				.join("\n");
+
+			return {
+				content: [{ type: "text", text }],
+				details: {
+					selected: true,
+					path: current.path,
+					name: current.name,
+				},
+			};
+		},
+	});
+
+	pi.registerTool({
+		name: "list_knowledge_base_pages",
+		label: "列出知识库页面",
+		description:
+			"List all markdown wiki pages in the user's currently active knowledge base (under raw/ and wiki/, excluding Obsidian / dev / temp files). Returns relative paths from the knowledge base root. Use this to discover what content exists before answering questions about the user's wiki.",
+		parameters: Type.Object({}),
+		async execute(): Promise<{
+			content: { type: "text"; text: string }[];
+			details: Record<string, unknown>;
+		}> {
+			if (!current) {
+				return {
+					content: [
+						{
+							type: "text",
+							text: "No knowledge base is currently selected.",
+						},
+					],
+					details: { selected: false },
+				};
+			}
+
+			try {
+				const files = await listMarkdownFiles(current.path, current.path);
+				return {
+					content: [
+						{
+							type: "text",
+							text:
+								files.length === 0
+									? "(Knowledge base is empty)"
+									: `${files.length} markdown files:\n${files.join("\n")}`,
+						},
+					],
+					details: { selected: true, count: files.length, files, path: current.path },
+				};
+			} catch (err) {
+				const msg = err instanceof Error ? err.message : String(err);
+				return {
+					content: [{ type: "text", text: `Error listing files: ${msg}` }],
+					details: { selected: true, error: msg },
+				};
+			}
+		},
+	});
+
+	pi.registerTool({
+		name: "query_knowledge_base",
+		label: "检索知识库",
+		description:
+			"Search the user's current knowledge base for pages relevant to a query. Use this when the user asks a follow-up question that may require pulling additional context from the knowledge base beyond what's already in the conversation.",
+		parameters: Type.Object({
+			query: Type.String(),
+		}),
+		async execute(
+			_toolCallId: string,
+			params: { query?: string },
+		): Promise<{
+			content: { type: "text"; text: string }[];
+			details: Record<string, unknown>;
+		}> {
+			if (!current) {
+				return {
+					content: [{ type: "text", text: "No knowledge base is currently selected." }],
+					details: { selected: false },
+				};
+			}
+			const query = params.query?.trim() ?? "";
+			if (!query) {
+				return {
+					content: [{ type: "text", text: "query 不能为空。" }],
+					details: { selected: true, count: 0, files: [] },
+				};
+			}
+			const search = await searchKnowledgeBase(current.path, query, {
+				totalBudgetChars: 4000,
+			});
+			return {
+				content: [{ type: "text", text: formatSearchResultsForTool(search) }],
+				details: {
+					selected: true,
+					count: search.results.length,
+					files: search.results.map((result) => result.path),
+				},
+			};
+		},
+	});
+}