@zosmaai/pi-llm-wiki 0.10.9 → 0.11.1

package/extensions/llm-wiki/lib/ingest-worker.ts

@@ -4,9 +4,19 @@ import type { AgentTool } from "@mariozechner/pi-agent-core";
 import type { Api, Model } from "@mariozechner/pi-ai";
 import { Type } from "typebox";
 import type { Static } from "typebox";
+import {
+  type KnowledgeDiagnostic,
+  type KnowledgeDocument,
+  createKnowledgeDocument,
+  patchKnowledgeDocument,
+  readKnowledgeDocumentFile,
+  serializeKnowledgeDocument,
+  writeKnowledgeDocumentFile,
+} from "./knowledge-document.js";
 import { appendEvent, rebuildMetadataLight } from "./metadata.js";
 import { runSubAgent } from "./subagent.js";
 import { type VaultPaths, fmtDate, slugify } from "./utils.js";
+import { VaultWriteError, assertWritableVault } from "./vault-format.js";
 
 /**
  * Background ingest synthesis (issue #65, part of epic #63).
@@ -76,64 +86,239 @@ export interface CommitResult {
   contradictions: number;
 }
 
+export type CommitSynthesisOutcome =
+  | ({ ok: true } & CommitResult)
+  | { ok: false; sourceId: string; diagnostics: KnowledgeDiagnostic[] };
+
 // ── deterministic persistence (no LLM) ────────────────────
 
-function buildEntityPage(
+/** Localized headings for generated pages (issue #124). */
+const HEADINGS: Record<string, Record<string, string>> = {
+  ru: {
+    summary: "Резюме",
+    keyTakeaways: "Ключевые выводы",
+    entitiesMentioned: "Упомянутые сущности",
+    conceptsMentioned: "Упомянутые концепции",
+    notableQuotes: "Заметные цитаты",
+    contradictions: "Противоречия",
+    sourcePacket: "Пакет источника",
+    overview: "Обзор",
+    definition: "Определение",
+    noneRecorded: "[Не записано]",
+    none: "[Нет]",
+    id: "ID",
+    extracted: "Извлечено",
+    manifest: "Манифест",
+    contradiction: "⚠️ **Противоречие**",
+  },
+  fr: {
+    summary: "Résumé",
+    keyTakeaways: "Points clés",
+    entitiesMentioned: "Entités mentionnées",
+    conceptsMentioned: "Concepts mentionnés",
+    notableQuotes: "Citations notables",
+    contradictions: "Contradictions",
+    sourcePacket: "Paquet source",
+    overview: "Aperçu",
+    definition: "Définition",
+    noneRecorded: "[Aucun enregistré]",
+    none: "[Aucun]",
+    id: "ID",
+    extracted: "Extrait",
+    manifest: "Manifeste",
+    contradiction: "⚠️ **Contradiction**",
+  },
+  de: {
+    summary: "Zusammenfassung",
+    keyTakeaways: "Wichtige Erkenntnisse",
+    entitiesMentioned: "Erwähnte Entitäten",
+    conceptsMentioned: "Erwähnte Konzepte",
+    notableQuotes: "Bemerkenswerte Zitate",
+    contradictions: "Widersprüche",
+    sourcePacket: "Quellenpaket",
+    overview: "Übersicht",
+    definition: "Definition",
+    noneRecorded: "[Keine aufgezeichnet]",
+    none: "[Keine]",
+    id: "ID",
+    extracted: "Extrahiert",
+    manifest: "Manifest",
+    contradiction: "⚠️ **Widerspruch**",
+  },
+  ja: {
+    summary: "要約",
+    keyTakeaways: "主な要点",
+    entitiesMentioned: "言及されたエンティティ",
+    conceptsMentioned: "言及された概念",
+    notableQuotes: "注目すべき引用",
+    contradictions: "矛盾",
+    sourcePacket: "ソースパケット",
+    overview: "概要",
+    definition: "定義",
+    noneRecorded: "[記録なし]",
+    none: "[なし]",
+    id: "ID",
+    extracted: "抽出済み",
+    manifest: "マニフェスト",
+    contradiction: "⚠️ **矛盾**",
+  },
+};
+
+function getHeadings(lang?: string): Record<string, string> {
+  if (lang && HEADINGS[lang]) return HEADINGS[lang];
+  // English defaults
+  return {
+    summary: "Summary",
+    keyTakeaways: "Key Takeaways",
+    entitiesMentioned: "Entities Mentioned",
+    conceptsMentioned: "Concepts Mentioned",
+    notableQuotes: "Notable Quotes",
+    contradictions: "Contradictions",
+    sourcePacket: "Source Packet",
+    overview: "Overview",
+    definition: "Definition",
+    noneRecorded: "[None recorded]",
+    none: "[None]",
+    id: "ID",
+    extracted: "Extracted",
+    manifest: "Manifest",
+    contradiction: "⚠️ **Contradiction**",
+  };
+}
+
+function buildEntityPageBody(
   title: string,
   description: string,
-  date: string,
   sourceId: string,
+  lang?: string,
 ): string {
   const desc = description.trim() || "One-line description.";
-  return `---\ntype: entity\ncreated: ${date}\nupdated: ${date}\nsources: [[[sources/${sourceId}]]]\n---\n\n# ${title}\n\n${desc}\n\n## Overview\n\n[Key facts]\n\n## Links\n\n- [[sources/${sourceId}]]\n`;
+  const h = getHeadings(lang);
+  return `# ${title}
+
+${desc}
+
+## ${h.overview}
+
+[Key facts]
+
+## Links
+
+- [${sourceId}](/sources/${sourceId}.md)`;
 }
 
-function buildConceptPage(
+function buildConceptPageBody(
   title: string,
   definition: string,
-  date: string,
   sourceId: string,
+  lang?: string,
 ): string {
   const def = definition.trim() || "One-line definition.";
-  return `---\ntype: concept\ncreated: ${date}\nupdated: ${date}\nsources: [[[sources/${sourceId}]]]\n---\n\n# ${title}\n\n${def}\n\n## Definition\n\n[Clear explanation]\n\n## Links\n\n- [[sources/${sourceId}]]\n`;
+  const h = getHeadings(lang);
+  return `# ${title}
+
+${def}
+
+## ${h.definition}
+
+[Clear explanation]
+
+## Links
+
+- [${sourceId}](/sources/${sourceId}.md)`;
 }
 
-/** Rebuild the source page from synthesis data, marking it ingested. */
-export function buildIngestedSourcePage(
+/** Rebuild the source page body from synthesis data, marking it ingested. */
+export function buildIngestedSourcePageBody(
   manifest: Record<string, unknown>,
   data: SynthesisData,
-  date: string,
+  _date: string,
+  lang?: string,
 ): string {
   const id = String(manifest.id);
   const title = String(manifest.title || id);
   const url = manifest.url ? `\n> _Original: [${manifest.url}](${manifest.url})_` : "";
-  const format = String(manifest.format || "unknown");
-  const captured = String(manifest.captured || date);
+  const h = getHeadings(lang);
 
   const takeaways =
     data.key_takeaways.length > 0
       ? data.key_takeaways.map((t) => `- ${t.trim()}`).join("\n")
-      : "- [None recorded]";
+      : `- ${h.noneRecorded}`;
   const entities =
     data.entities.length > 0
-      ? data.entities.map((e) => `- [[entities/${slugify(e.title)}]]`).join("\n")
-      : "- [None]";
+      ? data.entities.map((e) => `- [${e.title}](/entities/${slugify(e.title)}.md)`).join("\n")
+      : `- ${h.none}`;
   const concepts =
     data.concepts.length > 0
-      ? data.concepts.map((c) => `- [[concepts/${slugify(c.title)}]]`).join("\n")
-      : "- [None]";
+      ? data.concepts.map((c) => `- [${c.title}](/concepts/${slugify(c.title)}.md)`).join("\n")
+      : `- ${h.none}`;
   const quotes =
     data.quotes && data.quotes.length > 0
       ? data.quotes
           .map((q) => `> ${q.text.trim()}${q.attribution ? ` — ${q.attribution}` : ""}`)
           .join("\n\n")
-      : "> [None recorded]";
+      : `> ${h.noneRecorded}`;
   const contradictions =
     data.contradictions && data.contradictions.length > 0
-      ? `\n## Contradictions\n\n${data.contradictions.map((c) => `⚠️ **Contradiction**: ${c.trim()}`).join("\n")}\n`
+      ? `\n## ${h.contradictions}\n\n${data.contradictions.map((c) => `${h.contradiction}: ${c.trim()}`).join("\n")}\n`
       : "";
 
-  return `---\ntype: source\nformat: ${format}\nsource_id: ${id}\nraw_path: raw/sources/${id}/extracted.md\ncaptured: ${captured}\nstatus: ingested\nupdated: ${date}\n---\n\n# ${title}${url}\n\n## Summary\n\n${data.summary.trim()}\n\n## Key Takeaways\n\n${takeaways}\n\n## Entities Mentioned\n\n${entities}\n\n## Concepts Mentioned\n\n${concepts}\n\n## Notable Quotes\n\n${quotes}\n${contradictions}\n## Source Packet\n\n- **ID:** \`[[sources/${id}]]\`\n- **Extracted:** [raw/sources/${id}/extracted.md](../raw/sources/${id}/extracted.md)\n- **Manifest:** [raw/sources/${id}/manifest.json](../raw/sources/${id}/manifest.json)\n`;
+  return `# ${title}${url}
+
+## ${h.summary}
+
+${data.summary.trim()}
+
+## ${h.keyTakeaways}
+
+${takeaways}
+
+## ${h.entitiesMentioned}
+
+${entities}
+
+## ${h.conceptsMentioned}
+
+${concepts}
+
+## ${h.notableQuotes}
+
+${quotes}
+${contradictions}## ${h.sourcePacket}
+
+- **${h.id}:** \`sources/${id}\`
+- **${h.extracted}:** \`raw/sources/${id}/extracted.md\`
+- **${h.manifest}:** \`raw/sources/${id}/manifest.json\`
+`;
+}
+
+/** Rebuild the source page from synthesis data, marking it ingested. */
+export function buildIngestedSourcePage(
+  manifest: Record<string, unknown>,
+  data: SynthesisData,
+  date: string,
+  lang?: string,
+): string {
+  const id = String(manifest.id);
+  const title = String(manifest.title || id);
+  const format = String(manifest.format || "unknown");
+  const captured = String(manifest.captured || date);
+  const body = buildIngestedSourcePageBody(manifest, data, date, lang);
+  const doc = createKnowledgeDocument(
+    `sources/${id}.md`,
+    {
+      type: "source",
+      title,
+      format,
+      source_id: id,
+      raw_path: `raw/sources/${id}/extracted.md`,
+      captured,
+      status: "ingested",
+      updated: date,
+    },
+    body,
+  );
+  return serializeKnowledgeDocument(doc);
 }
 
 /**
@@ -147,7 +332,8 @@ export function commitSynthesis(
   manifest: Record<string, unknown>,
   data: SynthesisData,
   date: string = fmtDate(),
-): CommitResult {
+  lang?: string,
+): CommitSynthesisOutcome {
   const result: CommitResult = {
     sourceId,
     sourcePage: join(paths.wiki, "sources", `${sourceId}.md`),
@@ -158,9 +344,42 @@ export function commitSynthesis(
     contradictions: data.contradictions?.length ?? 0,
   };
 
-  // Source page (always rewritten from skeleton → ingested).
+  try {
+    assertWritableVault(paths);
+  } catch (error: unknown) {
+    if (error instanceof VaultWriteError) {
+      return { ok: false, sourceId, diagnostics: error.diagnostics };
+    }
+    throw error;
+  }
+
+  // Patch existing documents so unknown fields, legacy sources, and titles survive.
+  let sourceDocument: KnowledgeDocument;
+  if (existsSync(result.sourcePage)) {
+    const parsed = readKnowledgeDocumentFile(result.sourcePage, `sources/${sourceId}.md`);
+    if (!parsed.ok) return { ok: false, sourceId, diagnostics: parsed.diagnostics };
+    sourceDocument = patchKnowledgeDocument(parsed.document, {
+      fields: { status: "ingested", updated: date },
+      body: buildIngestedSourcePageBody(manifest, data, date, lang),
+    });
+  } else {
+    sourceDocument = createKnowledgeDocument(
+      `sources/${sourceId}.md`,
+      {
+        type: "source",
+        title: String(manifest.title || sourceId),
+        format: String(manifest.format || "unknown"),
+        source_id: sourceId,
+        raw_path: `raw/sources/${sourceId}/extracted.md`,
+        captured: String(manifest.captured || date),
+        status: "ingested",
+        updated: date,
+      },
+      buildIngestedSourcePageBody(manifest, data, date, lang),
+    );
+  }
   mkdirSync(join(paths.wiki, "sources"), { recursive: true });
-  writeFileSync(result.sourcePage, buildIngestedSourcePage(manifest, data, date), "utf-8");
+  writeKnowledgeDocumentFile(result.sourcePage, sourceDocument);
 
   // Entity pages — create if absent, link if present.
   mkdirSync(join(paths.wiki, "entities"), { recursive: true });
@@ -171,7 +390,19 @@ export function commitSynthesis(
     if (existsSync(pagePath)) {
       result.entitiesLinked.push(slug);
     } else {
-      writeFileSync(pagePath, buildEntityPage(e.title, e.description, date, sourceId), "utf-8");
+      const entityDoc = createKnowledgeDocument(
+        `entities/${slug}.md`,
+        {
+          type: "entity",
+          title: e.title,
+          description: e.description.trim() || "One-line description.",
+          created: date,
+          updated: date,
+        },
+        buildEntityPageBody(e.title, e.description, sourceId, lang),
+        [{ id: sourceId, resource: `/sources/${sourceId}.md` }],
+      );
+      writeKnowledgeDocumentFile(pagePath, entityDoc);
       result.entitiesCreated.push(slug);
     }
   }
@@ -185,7 +416,19 @@ export function commitSynthesis(
     if (existsSync(pagePath)) {
       result.conceptsLinked.push(slug);
     } else {
-      writeFileSync(pagePath, buildConceptPage(c.title, c.definition, date, sourceId), "utf-8");
+      const conceptDoc = createKnowledgeDocument(
+        `concepts/${slug}.md`,
+        {
+          type: "concept",
+          title: c.title,
+          description: c.definition.trim() || "One-line definition.",
+          created: date,
+          updated: date,
+        },
+        buildConceptPageBody(c.title, c.definition, sourceId, lang),
+        [{ id: sourceId, resource: `/sources/${sourceId}.md` }],
+      );
+      writeKnowledgeDocumentFile(pagePath, conceptDoc);
       result.conceptsCreated.push(slug);
     }
   }
@@ -199,7 +442,7 @@ export function commitSynthesis(
     background: true,
   });
 
-  return result;
+  return { ok: true, ...result };
 }
 
 // ── sub-agent synthesis (LLM) ─────────────────────────────
@@ -230,6 +473,8 @@ export interface RunIngestSynthesisArgs {
   /** Cap on extracted chars fed to the model (avoid huge prompts). Default 24k. */
   maxChars?: number;
   signal?: AbortSignal;
+  /** BCP 47 language tag for narrative content (issue #124). */
+  synthesisLanguage?: string;
 }
 
 /**
@@ -240,10 +485,26 @@ export interface RunIngestSynthesisArgs {
 export async function runIngestSynthesis(
   args: RunIngestSynthesisArgs,
 ): Promise<CommitResult | undefined> {
-  const { model, apiKey, headers, paths, sourceId, manifest, extracted, maxChars, signal } = args;
+  const {
+    model,
+    apiKey,
+    headers,
+    paths,
+    sourceId,
+    manifest,
+    extracted,
+    maxChars,
+    signal,
+    synthesisLanguage,
+  } = args;
   const content = extracted.slice(0, maxChars ?? 24_000);
   if (!content.trim()) return undefined;
 
+  const languageInstruction = synthesisLanguage
+    ? `\n\nWrite all generated content in ${synthesisLanguage}, including titles, headings, summaries, descriptions, and concept/entity names. Only preserve code, API names, file paths, commands, exact technical identifiers, and verbatim quotations in their original form.`
+    : "";
+  const systemPrompt = INGEST_SYSTEM + languageInstruction;
+
   let committed: CommitResult | undefined;
 
   const commitTool: AgentTool<typeof CommitSynthesisSchema> = {
@@ -253,7 +514,22 @@ export async function runIngestSynthesis(
       "Persist the structured synthesis of this source into wiki pages. Call exactly once.",
     parameters: CommitSynthesisSchema,
     execute: async (_id, params) => {
-      committed = commitSynthesis(paths, sourceId, manifest, params);
+      const outcome = commitSynthesis(
+        paths,
+        sourceId,
+        manifest,
+        params,
+        undefined,
+        synthesisLanguage,
+      );
+      if (!outcome.ok) {
+        return {
+          content: [{ type: "text", text: `Failed: ${outcome.diagnostics[0].message}` }],
+          details: { sourceId },
+          isError: true,
+        };
+      }
+      committed = outcome;
       const ack = `Committed: source page + ${committed.entitiesCreated.length} new entit${
         committed.entitiesCreated.length === 1 ? "y" : "ies"
       }, ${committed.conceptsCreated.length} new concept${
@@ -270,7 +546,7 @@ export async function runIngestSynthesis(
     model,
     apiKey,
     headers,
-    systemPrompt: INGEST_SYSTEM,
+    systemPrompt,
     userPrompt,
     tools: [commitTool as AgentTool],
     signal,