@zosmaai/pi-llm-wiki 0.10.9 → 0.11.1

package/dist/extensions/llm-wiki/lib/ingest-worker.js

@@ -0,0 +1,410 @@
+import { existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import { Type } from "typebox";
+import { createKnowledgeDocument, patchKnowledgeDocument, readKnowledgeDocumentFile, serializeKnowledgeDocument, writeKnowledgeDocumentFile, } from "./knowledge-document.js";
+import { appendEvent, rebuildMetadataLight } from "./metadata.js";
+import { runSubAgent } from "./subagent.js";
+import { fmtDate, slugify } from "./utils.js";
+import { VaultWriteError, assertWritableVault } from "./vault-format.js";
+/**
+ * Background ingest synthesis (issue #65, part of epic #63).
+ *
+ * Moves the work the main agent used to do during `wiki_ingest` — reading a
+ * captured source's extracted text and writing the source page + entity /
+ * concept pages — onto a background sub-agent, so capturing/ingesting never
+ * stalls the user.
+ *
+ * Design: the sub-agent produces ONE structured `commit_synthesis` call; the
+ * persistence (`commitSynthesis`) is fully deterministic and unit-testable
+ * without an LLM. This mirrors pi-observational-memory's single-structured-tool
+ * pattern and keeps the file-writing logic verifiable in isolation.
+ */
+// ── structured synthesis schema ───────────────────────────
+export const CommitSynthesisSchema = Type.Object({
+    summary: Type.String({
+        minLength: 1,
+        description: "2-3 paragraph summary of the source's key content.",
+    }),
+    key_takeaways: Type.Array(Type.String({ minLength: 1 }), {
+        description: "The most important points, one per item.",
+    }),
+    entities: Type.Array(Type.Object({
+        title: Type.String({
+            minLength: 1,
+            description: "Entity name (person, org, tool, product).",
+        }),
+        description: Type.String({ description: "One-line description of the entity." }),
+    }), { description: "Named entities mentioned in the source." }),
+    concepts: Type.Array(Type.Object({
+        title: Type.String({ minLength: 1, description: "Concept name (idea, pattern, framework)." }),
+        definition: Type.String({ description: "One-line definition of the concept." }),
+    }), { description: "Concepts discussed in the source." }),
+    quotes: Type.Optional(Type.Array(Type.Object({
+        text: Type.String({ minLength: 1 }),
+        attribution: Type.Optional(Type.String()),
+    }), { description: "Notable verbatim quotes." })),
+    contradictions: Type.Optional(Type.Array(Type.String({ minLength: 1 }), {
+        description: "Tensions/contradictions with existing wiki content, if any.",
+    })),
+});
+// ── deterministic persistence (no LLM) ────────────────────
+/** Localized headings for generated pages (issue #124). */
+const HEADINGS = {
+    ru: {
+        summary: "Резюме",
+        keyTakeaways: "Ключевые выводы",
+        entitiesMentioned: "Упомянутые сущности",
+        conceptsMentioned: "Упомянутые концепции",
+        notableQuotes: "Заметные цитаты",
+        contradictions: "Противоречия",
+        sourcePacket: "Пакет источника",
+        overview: "Обзор",
+        definition: "Определение",
+        noneRecorded: "[Не записано]",
+        none: "[Нет]",
+        id: "ID",
+        extracted: "Извлечено",
+        manifest: "Манифест",
+        contradiction: "⚠️ **Противоречие**",
+    },
+    fr: {
+        summary: "Résumé",
+        keyTakeaways: "Points clés",
+        entitiesMentioned: "Entités mentionnées",
+        conceptsMentioned: "Concepts mentionnés",
+        notableQuotes: "Citations notables",
+        contradictions: "Contradictions",
+        sourcePacket: "Paquet source",
+        overview: "Aperçu",
+        definition: "Définition",
+        noneRecorded: "[Aucun enregistré]",
+        none: "[Aucun]",
+        id: "ID",
+        extracted: "Extrait",
+        manifest: "Manifeste",
+        contradiction: "⚠️ **Contradiction**",
+    },
+    de: {
+        summary: "Zusammenfassung",
+        keyTakeaways: "Wichtige Erkenntnisse",
+        entitiesMentioned: "Erwähnte Entitäten",
+        conceptsMentioned: "Erwähnte Konzepte",
+        notableQuotes: "Bemerkenswerte Zitate",
+        contradictions: "Widersprüche",
+        sourcePacket: "Quellenpaket",
+        overview: "Übersicht",
+        definition: "Definition",
+        noneRecorded: "[Keine aufgezeichnet]",
+        none: "[Keine]",
+        id: "ID",
+        extracted: "Extrahiert",
+        manifest: "Manifest",
+        contradiction: "⚠️ **Widerspruch**",
+    },
+    ja: {
+        summary: "要約",
+        keyTakeaways: "主な要点",
+        entitiesMentioned: "言及されたエンティティ",
+        conceptsMentioned: "言及された概念",
+        notableQuotes: "注目すべき引用",
+        contradictions: "矛盾",
+        sourcePacket: "ソースパケット",
+        overview: "概要",
+        definition: "定義",
+        noneRecorded: "[記録なし]",
+        none: "[なし]",
+        id: "ID",
+        extracted: "抽出済み",
+        manifest: "マニフェスト",
+        contradiction: "⚠️ **矛盾**",
+    },
+};
+function getHeadings(lang) {
+    if (lang && HEADINGS[lang])
+        return HEADINGS[lang];
+    // English defaults
+    return {
+        summary: "Summary",
+        keyTakeaways: "Key Takeaways",
+        entitiesMentioned: "Entities Mentioned",
+        conceptsMentioned: "Concepts Mentioned",
+        notableQuotes: "Notable Quotes",
+        contradictions: "Contradictions",
+        sourcePacket: "Source Packet",
+        overview: "Overview",
+        definition: "Definition",
+        noneRecorded: "[None recorded]",
+        none: "[None]",
+        id: "ID",
+        extracted: "Extracted",
+        manifest: "Manifest",
+        contradiction: "⚠️ **Contradiction**",
+    };
+}
+function buildEntityPageBody(title, description, sourceId, lang) {
+    const desc = description.trim() || "One-line description.";
+    const h = getHeadings(lang);
+    return `# ${title}
+
+${desc}
+
+## ${h.overview}
+
+[Key facts]
+
+## Links
+
+- [${sourceId}](/sources/${sourceId}.md)`;
+}
+function buildConceptPageBody(title, definition, sourceId, lang) {
+    const def = definition.trim() || "One-line definition.";
+    const h = getHeadings(lang);
+    return `# ${title}
+
+${def}
+
+## ${h.definition}
+
+[Clear explanation]
+
+## Links
+
+- [${sourceId}](/sources/${sourceId}.md)`;
+}
+/** Rebuild the source page body from synthesis data, marking it ingested. */
+export function buildIngestedSourcePageBody(manifest, data, _date, lang) {
+    const id = String(manifest.id);
+    const title = String(manifest.title || id);
+    const url = manifest.url ? `\n> _Original: [${manifest.url}](${manifest.url})_` : "";
+    const h = getHeadings(lang);
+    const takeaways = data.key_takeaways.length > 0
+        ? data.key_takeaways.map((t) => `- ${t.trim()}`).join("\n")
+        : `- ${h.noneRecorded}`;
+    const entities = data.entities.length > 0
+        ? data.entities.map((e) => `- [${e.title}](/entities/${slugify(e.title)}.md)`).join("\n")
+        : `- ${h.none}`;
+    const concepts = data.concepts.length > 0
+        ? data.concepts.map((c) => `- [${c.title}](/concepts/${slugify(c.title)}.md)`).join("\n")
+        : `- ${h.none}`;
+    const quotes = data.quotes && data.quotes.length > 0
+        ? data.quotes
+            .map((q) => `> ${q.text.trim()}${q.attribution ? ` — ${q.attribution}` : ""}`)
+            .join("\n\n")
+        : `> ${h.noneRecorded}`;
+    const contradictions = data.contradictions && data.contradictions.length > 0
+        ? `\n## ${h.contradictions}\n\n${data.contradictions.map((c) => `${h.contradiction}: ${c.trim()}`).join("\n")}\n`
+        : "";
+    return `# ${title}${url}
+
+## ${h.summary}
+
+${data.summary.trim()}
+
+## ${h.keyTakeaways}
+
+${takeaways}
+
+## ${h.entitiesMentioned}
+
+${entities}
+
+## ${h.conceptsMentioned}
+
+${concepts}
+
+## ${h.notableQuotes}
+
+${quotes}
+${contradictions}## ${h.sourcePacket}
+
+- **${h.id}:** \`sources/${id}\`
+- **${h.extracted}:** \`raw/sources/${id}/extracted.md\`
+- **${h.manifest}:** \`raw/sources/${id}/manifest.json\`
+`;
+}
+/** Rebuild the source page from synthesis data, marking it ingested. */
+export function buildIngestedSourcePage(manifest, data, date, lang) {
+    const id = String(manifest.id);
+    const title = String(manifest.title || id);
+    const format = String(manifest.format || "unknown");
+    const captured = String(manifest.captured || date);
+    const body = buildIngestedSourcePageBody(manifest, data, date, lang);
+    const doc = createKnowledgeDocument(`sources/${id}.md`, {
+        type: "source",
+        title,
+        format,
+        source_id: id,
+        raw_path: `raw/sources/${id}/extracted.md`,
+        captured,
+        status: "ingested",
+        updated: date,
+    }, body);
+    return serializeKnowledgeDocument(doc);
+}
+/**
+ * Persist a synthesis deterministically: rewrite the source page (status →
+ * ingested), create missing entity/concept pages (existing pages are linked,
+ * never overwritten), and log the event. Pure file I/O — no LLM, no network.
+ */
+export function commitSynthesis(paths, sourceId, manifest, data, date = fmtDate(), lang) {
+    const result = {
+        sourceId,
+        sourcePage: join(paths.wiki, "sources", `${sourceId}.md`),
+        entitiesCreated: [],
+        conceptsCreated: [],
+        entitiesLinked: [],
+        conceptsLinked: [],
+        contradictions: data.contradictions?.length ?? 0,
+    };
+    try {
+        assertWritableVault(paths);
+    }
+    catch (error) {
+        if (error instanceof VaultWriteError) {
+            return { ok: false, sourceId, diagnostics: error.diagnostics };
+        }
+        throw error;
+    }
+    // Patch existing documents so unknown fields, legacy sources, and titles survive.
+    let sourceDocument;
+    if (existsSync(result.sourcePage)) {
+        const parsed = readKnowledgeDocumentFile(result.sourcePage, `sources/${sourceId}.md`);
+        if (!parsed.ok)
+            return { ok: false, sourceId, diagnostics: parsed.diagnostics };
+        sourceDocument = patchKnowledgeDocument(parsed.document, {
+            fields: { status: "ingested", updated: date },
+            body: buildIngestedSourcePageBody(manifest, data, date, lang),
+        });
+    }
+    else {
+        sourceDocument = createKnowledgeDocument(`sources/${sourceId}.md`, {
+            type: "source",
+            title: String(manifest.title || sourceId),
+            format: String(manifest.format || "unknown"),
+            source_id: sourceId,
+            raw_path: `raw/sources/${sourceId}/extracted.md`,
+            captured: String(manifest.captured || date),
+            status: "ingested",
+            updated: date,
+        }, buildIngestedSourcePageBody(manifest, data, date, lang));
+    }
+    mkdirSync(join(paths.wiki, "sources"), { recursive: true });
+    writeKnowledgeDocumentFile(result.sourcePage, sourceDocument);
+    // Entity pages — create if absent, link if present.
+    mkdirSync(join(paths.wiki, "entities"), { recursive: true });
+    for (const e of data.entities) {
+        const slug = slugify(e.title);
+        if (!slug)
+            continue;
+        const pagePath = join(paths.wiki, "entities", `${slug}.md`);
+        if (existsSync(pagePath)) {
+            result.entitiesLinked.push(slug);
+        }
+        else {
+            const entityDoc = createKnowledgeDocument(`entities/${slug}.md`, {
+                type: "entity",
+                title: e.title,
+                description: e.description.trim() || "One-line description.",
+                created: date,
+                updated: date,
+            }, buildEntityPageBody(e.title, e.description, sourceId, lang), [{ id: sourceId, resource: `/sources/${sourceId}.md` }]);
+            writeKnowledgeDocumentFile(pagePath, entityDoc);
+            result.entitiesCreated.push(slug);
+        }
+    }
+    // Concept pages — create if absent, link if present.
+    mkdirSync(join(paths.wiki, "concepts"), { recursive: true });
+    for (const c of data.concepts) {
+        const slug = slugify(c.title);
+        if (!slug)
+            continue;
+        const pagePath = join(paths.wiki, "concepts", `${slug}.md`);
+        if (existsSync(pagePath)) {
+            result.conceptsLinked.push(slug);
+        }
+        else {
+            const conceptDoc = createKnowledgeDocument(`concepts/${slug}.md`, {
+                type: "concept",
+                title: c.title,
+                description: c.definition.trim() || "One-line definition.",
+                created: date,
+                updated: date,
+            }, buildConceptPageBody(c.title, c.definition, sourceId, lang), [{ id: sourceId, resource: `/sources/${sourceId}.md` }]);
+            writeKnowledgeDocumentFile(pagePath, conceptDoc);
+            result.conceptsCreated.push(slug);
+        }
+    }
+    appendEvent(paths, {
+        kind: "ingest",
+        source_id: sourceId,
+        entities_created: result.entitiesCreated.length,
+        concepts_created: result.conceptsCreated.length,
+        contradictions: result.contradictions,
+        background: true,
+    });
+    return { ok: true, ...result };
+}
+// ── sub-agent synthesis (LLM) ─────────────────────────────
+export const INGEST_SYSTEM = `You are the LLM Wiki ingestion synthesizer. You turn a single captured source's extracted text into structured wiki knowledge.
+
+Read the source content, then call \`commit_synthesis\` EXACTLY ONCE with:
+- summary: a faithful 2-3 paragraph summary (no fabrication).
+- key_takeaways: the most important points.
+- entities: named people, organizations, tools, products actually mentioned.
+- concepts: ideas, patterns, frameworks actually discussed.
+- quotes: notable verbatim quotes (optional).
+- contradictions: tensions with general knowledge or noted in the text (optional).
+
+Rules:
+- Never fabricate. Only include entities/concepts present in the source.
+- Keep descriptions to one line.
+- After calling commit_synthesis once, reply with a one-line confirmation and stop.`;
+/**
+ * Run the synthesis sub-agent for a single source, then commit + rebuild
+ * metadata. Returns the commit result, or undefined if the model produced no
+ * synthesis.
+ */
+export async function runIngestSynthesis(args) {
+    const { model, apiKey, headers, paths, sourceId, manifest, extracted, maxChars, signal, synthesisLanguage, } = args;
+    const content = extracted.slice(0, maxChars ?? 24_000);
+    if (!content.trim())
+        return undefined;
+    const languageInstruction = synthesisLanguage
+        ? `\n\nWrite all generated content in ${synthesisLanguage}, including titles, headings, summaries, descriptions, and concept/entity names. Only preserve code, API names, file paths, commands, exact technical identifiers, and verbatim quotations in their original form.`
+        : "";
+    const systemPrompt = INGEST_SYSTEM + languageInstruction;
+    let committed;
+    const commitTool = {
+        name: "commit_synthesis",
+        label: "Commit synthesis",
+        description: "Persist the structured synthesis of this source into wiki pages. Call exactly once.",
+        parameters: CommitSynthesisSchema,
+        execute: async (_id, params) => {
+            const outcome = commitSynthesis(paths, sourceId, manifest, params, undefined, synthesisLanguage);
+            if (!outcome.ok) {
+                return {
+                    content: [{ type: "text", text: `Failed: ${outcome.diagnostics[0].message}` }],
+                    details: { sourceId },
+                    isError: true,
+                };
+            }
+            committed = outcome;
+            const ack = `Committed: source page + ${committed.entitiesCreated.length} new entit${committed.entitiesCreated.length === 1 ? "y" : "ies"}, ${committed.conceptsCreated.length} new concept${committed.conceptsCreated.length === 1 ? "" : "s"}. Reply with a one-line confirmation and stop.`;
+            return { content: [{ type: "text", text: ack }], details: { sourceId } };
+        },
+    };
+    const title = String(manifest.title || sourceId);
+    const userPrompt = `Synthesize this captured source into wiki knowledge by calling commit_synthesis once.\n\nSOURCE: ${title} (${sourceId})\n\nEXTRACTED CONTENT:\n${content}`;
+    await runSubAgent({
+        model,
+        apiKey,
+        headers,
+        systemPrompt,
+        userPrompt,
+        tools: [commitTool],
+        signal,
+    });
+    if (committed)
+        rebuildMetadataLight(paths);
+    return committed;
+}

package/dist/extensions/llm-wiki/lib/inject.js

@@ -0,0 +1,65 @@
+/**
+ * System-prompt context injection primitives (issues #87, #92).
+ *
+ * The `before_agent_start` hook augments the chained system prompt with a
+ * visible wiki-status footer. This module isolates that append so it can be
+ * unit-tested for idempotency — a turn that aborts (network error / ESC) and is
+ * retried can carry the prior injection forward, and a naive append stacks the
+ * footer 2x, 3x, ...
+ *
+ * It also owns the cache-safety split (issue #92): VOLATILE per-turn context
+ * (recall results, one-time topic-inference directive) must never enter the
+ * system prompt — that is the provider's primary cache prefix, so per-turn
+ * variation there forces a full cache miss every turn. Volatile content is
+ * routed into a tail conversation message instead. `buildAgentStartInjection`
+ * is the single, pure decision point for that split.
+ */
+/** The always-injected wiki-status footer (sans surrounding whitespace). */
+export const WIKI_STATUS_BLOCK = "<wiki_status>LLM Wiki active — use wiki_recall for deeper search, wiki_observe to record observations, wiki_retro to save insights.</wiki_status>";
+/**
+ * Append the wiki-status footer to a system prompt — idempotently (issue #87).
+ *
+ * Strips any already-present footer (with its leading blank line) before
+ * appending exactly one. This makes the injection safe across aborted/retried
+ * agent starts that carry the prior injection forward in the chained system
+ * prompt, so the footer never stacks (2x, 3x, ...).
+ * See test/inject-idempotent.test.ts.
+ */
+export function appendWikiStatus(systemPrompt) {
+    const base = systemPrompt.split(`\n\n${WIKI_STATUS_BLOCK}`).join("");
+    return `${base}\n\n${WIKI_STATUS_BLOCK}`;
+}
+/** Normalize upstream Pi and OMP system-prompt representations. */
+export function normalizeSystemPrompt(systemPrompt) {
+    if (Array.isArray(systemPrompt))
+        return systemPrompt.join("\n\n");
+    return typeof systemPrompt === "string" ? systemPrompt : "";
+}
+/** customType of the hidden tail message carrying volatile per-turn context. */
+export const WIKI_RECALL_MESSAGE_TYPE = "wiki-recall-context";
+/**
+ * Split a turn's injection into a cache-stable system prompt and a volatile
+ * tail message (issue #92).
+ *
+ * - `systemPrompt` is always `appendWikiStatus(baseSystemPrompt)` — the static
+ *   footer only. It carries NONE of `dynamicBlocks`, so it does not vary with
+ *   recall results and never breaks the provider cache prefix.
+ * - `dynamicBlocks` (recall context, topic-inference directive, ...) are
+ *   trimmed, emptied entries dropped, and joined with a blank line into the
+ *   message body. When nothing survives, no message is emitted.
+ *
+ * Pure and side-effect free — see test/agent-start-injection.test.ts.
+ */
+export function buildAgentStartInjection(baseSystemPrompt, dynamicBlocks) {
+    const systemPrompt = appendWikiStatus(normalizeSystemPrompt(baseSystemPrompt));
+    const content = dynamicBlocks
+        .map((b) => b?.trim())
+        .filter((b) => Boolean(b))
+        .join("\n\n");
+    if (!content)
+        return { systemPrompt };
+    return {
+        systemPrompt,
+        message: { customType: WIKI_RECALL_MESSAGE_TYPE, content, display: false },
+    };
+}