@zosmaai/pi-llm-wiki 0.1.2 → 0.2.0

package/docs/configuration.md

@@ -0,0 +1,38 @@
+# Configuration
+
+Wiki configuration lives in `.wiki/config.json`.
+
+## Modes
+
+### Personal
+
+- Extra folders: `wiki/journal/`, `wiki/goals/`
+- Track: learning, books, health, reflections
+
+### Company
+
+- Extra folders: `wiki/changes/`, `wiki/decisions/`
+- Track: competitors, market, strategy
+- Frontmatter: `confidence: high | medium | low`
+
+## Settings
+
+| Setting                    | Default | Description                        |
+| -------------------------- | ------- | ---------------------------------- |
+| `max_sources_per_discover` | 8       | Sources fetched per discovery run  |
+| `auto_fix_lint`            | false   | Auto-fix lint issues               |
+| `batch_ingest_size`        | 3       | Sources processed per ingest batch |
+
+## Page Frontmatter
+
+```yaml
+---
+type: entity | concept | source | synthesis | analysis
+created: YYYY-MM-DD
+updated: YYYY-MM-DD
+sources: [sources/SRC-YYYY-MM-DD-NNN]
+---
+```
+
+Entity: add `category: person | organization | tool | project | product`
+Concept: add `domain: ai | engineering | business | product | design | personal`

package/docs/obsidian.md

@@ -0,0 +1,21 @@
+# Obsidian Integration
+
+## Setup
+
+1. Open `wiki/` as an Obsidian vault
+2. The extension generates `meta/index.md` as a browsable catalog
+3. `meta/backlinks.json` is available for graph plugins
+
+## Recommended Plugins
+
+- [Dataview](https://github.com/blacksmithgu/obsidian-dataview) — Query pages by frontmatter
+- [Graph View](https://obsidian.md) (built-in) — Visualize `[[wikilink]]` connections
+- [Backlinks](https://obsidian.md) (built-in) — See inbound links
+
+## Web Clipper
+
+Use [Obsidian Web Clipper](https://obsidian.md/clipper) to save articles directly into `raw/articles/`.
+
+## Dataview Dashboard
+
+The extension creates `meta/index.md` with page listings. For custom dashboards, use Dataview queries against frontmatter fields like `type`, `domain`, `category`, `sources`.

package/extensions/llm-wiki/index.ts

@@ -0,0 +1,53 @@
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { installGuardrails } from "./lib/guardrails.js";
+import {
+  registerWikiBootstrap,
+  registerWikiCaptureSource,
+  registerWikiEnsurePage,
+  registerWikiIngest,
+  registerWikiLint,
+  registerWikiLogEvent,
+  registerWikiRebuildMeta,
+  registerWikiSearch,
+  registerWikiStatus,
+  registerWikiWatch,
+} from "./lib/tools.js";
+
+/**
+ * @zosmaai/pi-llm-wiki — LLM Wiki extension for Pi
+ *
+ * Registers 10 custom tools and installs guardrails:
+ * - wiki_bootstrap      Initialize a new vault
+ * - wiki_capture_source Capture URL/file/text into source packet
+ * - wiki_ingest         Get batch of sources needing synthesis
+ * - wiki_ensure_page    Create canonical page from template
+ * - wiki_search         Search generated registry
+ * - wiki_lint           Health check with auto-fix
+ * - wiki_status         Instant stats from registry
+ * - wiki_rebuild_meta   Force metadata rebuild
+ * - wiki_log_event      Append event and regenerate log
+ * - wiki_watch          Schedule auto-updates
+ *
+ * Guardrails:
+ * - Blocks direct edits to raw/** and meta/**
+ * - Auto-rebuilds metadata after wiki/** edits
+ */
+
+export default function (pi: ExtensionAPI) {
+  registerWikiBootstrap(pi);
+  registerWikiCaptureSource(pi);
+  registerWikiIngest(pi);
+  registerWikiEnsurePage(pi);
+  registerWikiSearch(pi);
+  registerWikiLint(pi);
+  registerWikiStatus(pi);
+  registerWikiRebuildMeta(pi);
+  registerWikiLogEvent(pi);
+  registerWikiWatch(pi);
+
+  installGuardrails(pi);
+
+  pi.on("session_start", async (_event, ctx) => {
+    ctx.ui.setStatus("llm-wiki", "🧠 LLM Wiki (10 tools, guardrails active)");
+  });
+}

package/extensions/llm-wiki/lib/guardrails.ts

@@ -0,0 +1,69 @@
+import { isToolCallEventType } from "@mariozechner/pi-coding-agent";
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { rebuildMetadataLight } from "./metadata.js";
+import { isProtectedPath, resolveVaultRoot } from "./utils.js";
+
+/**
+ * Guardrails and auto-rebuild hooks for the LLM Wiki extension.
+ */
+
+let pendingRebuild = false;
+
+/** Install guardrails on the extension API. */
+export function installGuardrails(pi: ExtensionAPI): void {
+  // Block direct edits to raw/ and meta/
+  pi.on("tool_call", async (event) => {
+    if (isToolCallEventType("write", event)) {
+      const path = event.input.path as string;
+      const root = resolveVaultRoot(process.cwd());
+      const check = isProtectedPath(path, root);
+      if (check.protected) {
+        return { block: true, reason: check.reason };
+      }
+    }
+
+    if (isToolCallEventType("edit", event)) {
+      const path = event.input.path as string;
+      const root = resolveVaultRoot(process.cwd());
+      const check = isProtectedPath(path, root);
+      if (check.protected) {
+        return { block: true, reason: check.reason };
+      }
+    }
+  });
+
+  // Track wiki edits for auto-rebuild
+  pi.on("tool_result", async (event) => {
+    if (event.toolName === "write" || event.toolName === "edit") {
+      const path = event.input.path as string;
+      const root = resolveVaultRoot(process.cwd());
+      const wikiPath = `${root}/wiki/`;
+      if (path?.startsWith(wikiPath)) {
+        pendingRebuild = true;
+      }
+    }
+  });
+
+  // Rebuild metadata at end of turn if wiki was modified
+  pi.on("turn_end", async (_event) => {
+    if (pendingRebuild) {
+      pendingRebuild = false;
+      try {
+        const root = resolveVaultRoot(process.cwd());
+        const paths = {
+          root,
+          raw: `${root}/raw`,
+          rawSources: `${root}/raw/sources`,
+          wiki: `${root}/wiki`,
+          meta: `${root}/meta`,
+          dotWiki: `${root}/.wiki`,
+          outputs: `${root}/outputs`,
+          discoveries: `${root}/.discoveries`,
+        };
+        rebuildMetadataLight(paths);
+      } catch {
+        // Silently fail — metadata rebuild is best-effort
+      }
+    }
+  });
+}

package/extensions/llm-wiki/lib/metadata.ts

@@ -0,0 +1,218 @@
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import {
+  type VaultPaths,
+  extractWikilinks,
+  findWikiPages,
+  fmtDate,
+  parseFrontmatter,
+  readJson,
+  readText,
+  writeJson,
+} from "./utils.js";
+
+/**
+ * Metadata generation for the LLM Wiki.
+ *
+ * Rebuilds registry.json, backlinks.json, index.md, log.md, and lint-report.md
+ * deterministically from the current state of raw/ and wiki/.
+ */
+
+export interface RegistryEntry {
+  type: "source" | "entity" | "concept" | "synthesis" | "analysis";
+  title: string;
+  created: string;
+  updated: string;
+  [key: string]: unknown;
+}
+
+export interface Registry {
+  version: string;
+  last_updated: string;
+  pages: Record<string, RegistryEntry>;
+}
+
+export interface Backlinks {
+  [pageId: string]: string[];
+}
+
+export interface WikiEvent {
+  timestamp: string;
+  kind: string;
+  [key: string]: unknown;
+}
+
+/** Rebuild the complete metadata layer. */
+export function rebuildMetadata(paths: VaultPaths): void {
+  mkdirSync(paths.meta, { recursive: true });
+
+  const registry = buildRegistry(paths);
+  const backlinks = buildBacklinks(paths, registry);
+
+  writeJson(join(paths.meta, "registry.json"), registry);
+  writeJson(join(paths.meta, "backlinks.json"), backlinks);
+  writeFileSync(join(paths.meta, "index.md"), buildIndexMarkdown(registry), "utf-8");
+
+  const log = buildLogMarkdown(paths);
+  writeFileSync(join(paths.meta, "log.md"), log, "utf-8");
+}
+
+/** Build registry from wiki/ and raw/ state. */
+export function buildRegistry(paths: VaultPaths): Registry {
+  const pages: Record<string, RegistryEntry> = {};
+
+  // Scan wiki pages
+  for (const page of findWikiPages(paths.wiki)) {
+    const { frontmatter } = parseFrontmatter(page.content);
+    const type = String(frontmatter.type || "page") as RegistryEntry["type"];
+    const title = String(frontmatter.title || page.relative.split("/").pop() || "Untitled");
+
+    pages[page.relative] = {
+      type,
+      title,
+      created: String(frontmatter.created || fmtDate()),
+      updated: String(frontmatter.updated || frontmatter.created || fmtDate()),
+      ...frontmatter,
+    };
+  }
+
+  // Scan raw source packets
+  if (existsSync(paths.rawSources)) {
+    for (const entry of readdirSync(paths.rawSources)) {
+      const manifestPath = join(paths.rawSources, entry, "manifest.json");
+      if (!existsSync(manifestPath)) continue;
+
+      const manifest = readJson<Record<string, unknown>>(manifestPath, {});
+      const id = String(manifest.id || entry);
+      const sourcePage = `sources/${id}`;
+
+      if (!pages[sourcePage]) {
+        pages[sourcePage] = {
+          type: "source",
+          title: String(manifest.title || id),
+          created: String(manifest.captured || fmtDate()),
+          updated: String(manifest.captured || fmtDate()),
+          ...manifest,
+        };
+      }
+    }
+  }
+
+  return {
+    version: "1.0",
+    last_updated: new Date().toISOString(),
+    pages,
+  };
+}
+
+/** Build backlinks map from all wiki pages. */
+export function buildBacklinks(paths: VaultPaths, registry: Registry): Backlinks {
+  const inbound: Backlinks = {};
+
+  // Initialize all pages with empty arrays
+  for (const id of Object.keys(registry.pages)) {
+    inbound[id] = [];
+  }
+
+  // Count inbound links
+  for (const page of findWikiPages(paths.wiki)) {
+    const links = extractWikilinks(page.content);
+    for (const link of links) {
+      if (inbound[link] && !inbound[link].includes(page.relative)) {
+        inbound[link].push(page.relative);
+      }
+    }
+  }
+
+  return inbound;
+}
+
+/** Build index markdown from registry. */
+export function buildIndexMarkdown(registry: Registry): string {
+  const byType: Record<string, Array<{ id: string; entry: RegistryEntry }>> = {};
+
+  for (const [id, entry] of Object.entries(registry.pages)) {
+    const t = entry.type;
+    if (!byType[t]) byType[t] = [];
+    byType[t].push({ id, entry });
+  }
+
+  const sections: string[] = [];
+  sections.push(
+    "# Wiki Index\n\n> Auto-generated from meta/registry.json. Do not edit manually.\n",
+  );
+
+  for (const [type, items] of Object.entries(byType).sort()) {
+    const label = `${type.charAt(0).toUpperCase() + type.slice(1)}s`;
+    sections.push(`## ${label}\n`);
+    for (const { id, entry } of items.sort((a, b) => a.id.localeCompare(b.id))) {
+      sections.push(`- [[${id}]] — ${entry.title} *(created: ${entry.created})*`);
+    }
+    sections.push("");
+  }
+
+  sections.push(
+    `---\n*Last updated: ${registry.last_updated}* | *Total pages: ${Object.keys(registry.pages).length}*`,
+  );
+  return `${sections.join("\n")}\n`;
+}
+
+/** Build log markdown from events.jsonl. */
+export function buildLogMarkdown(paths: VaultPaths): string {
+  const eventsPath = join(paths.meta, "events.jsonl");
+  const events: WikiEvent[] = [];
+
+  if (existsSync(eventsPath)) {
+    const raw = readFileSync(eventsPath, "utf-8").trim();
+    for (const line of raw.split("\n")) {
+      if (!line.trim()) continue;
+      try {
+        events.push(JSON.parse(line) as WikiEvent);
+      } catch {
+        // skip malformed
+      }
+    }
+  }
+
+  const lines: string[] = [];
+  lines.push("# Activity Log\n\n> Auto-generated from meta/events.jsonl. Do not edit manually.\n");
+
+  for (const ev of events) {
+    const ts = ev.timestamp || "unknown";
+    const kind = ev.kind || "event";
+    const details = Object.entries(ev)
+      .filter(([k]) => k !== "timestamp" && k !== "kind")
+      .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
+      .join(", ");
+
+    lines.push(`## [${ts}] ${kind}`);
+    if (details) lines.push(`- ${details}`);
+    lines.push("");
+  }
+
+  if (events.length === 0) {
+    lines.push("_No events recorded yet._\n");
+  }
+
+  return `${lines.join("\n")}\n`;
+}
+
+/** Append an event to events.jsonl. */
+export function appendEvent(paths: VaultPaths, event: Omit<WikiEvent, "timestamp">): void {
+  mkdirSync(paths.meta, { recursive: true });
+  const eventsPath = join(paths.meta, "events.jsonl");
+  const line = JSON.stringify({ timestamp: new Date().toISOString(), ...event });
+  writeFileSync(eventsPath, `${line}\n`, { flag: "a", encoding: "utf-8" });
+}
+
+/** Quick lightweight metadata rebuild (backlinks + index + log only). */
+export function rebuildMetadataLight(paths: VaultPaths): void {
+  const registry = buildRegistry(paths);
+  const backlinks = buildBacklinks(paths, registry);
+  writeJson(join(paths.meta, "registry.json"), registry);
+  writeJson(join(paths.meta, "backlinks.json"), backlinks);
+  writeFileSync(join(paths.meta, "index.md"), buildIndexMarkdown(registry), "utf-8");
+
+  const log = buildLogMarkdown(paths);
+  writeFileSync(join(paths.meta, "log.md"), log, "utf-8");
+}

package/extensions/llm-wiki/lib/source-packet.ts

@@ -0,0 +1,292 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { appendEvent } from "./metadata.js";
+import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
+
+/**
+ * Source packet capture and management.
+ *
+ * Each source is stored as a structured packet:
+ *   raw/sources/SRC-YYYY-MM-DD-NNN/
+ *     manifest.json    — capture metadata
+ *     original/        — original artifact (if file/URL)
+ *     extracted.md     — normalized markdown text
+ *     attachments/     — downloaded images, PDFs, etc.
+ */
+
+export interface CaptureResult {
+  sourceId: string;
+  packetPath: string;
+  sourcePagePath: string;
+  extracted: string;
+}
+
+/** Capture a URL into a source packet. */
+export async function captureUrl(
+  pi: ExtensionAPI,
+  paths: VaultPaths,
+  url: string,
+  signal?: AbortSignal,
+): Promise<CaptureResult> {
+  const sourceId = nextSourceId(paths);
+  const packetPath = join(paths.rawSources, sourceId);
+  mkdirSync(packetPath, { recursive: true });
+  mkdirSync(join(packetPath, "original"), { recursive: true });
+  mkdirSync(join(packetPath, "attachments"), { recursive: true });
+
+  // Try to fetch and extract content
+  let extracted = "";
+  let title = url;
+
+  // Try markitdown first
+  const markitdown = await exec(
+    pi,
+    "sh",
+    ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
+    { signal },
+  );
+
+  if (markitdown.stdout.trim() === "yes") {
+    try {
+      const mdResult = await exec(
+        pi,
+        "sh",
+        ["-c", `uvx --from 'markitdown[pdf]' markitdown "${url}" 2>/dev/null || echo ""`],
+        { signal, timeout: 30_000 },
+      );
+      if (mdResult.stdout.trim()) {
+        extracted = mdResult.stdout;
+        // Try to extract title from first h1
+        const h1Match = extracted.match(/^#\s+(.+)$/m);
+        if (h1Match) title = h1Match[1].trim();
+      }
+    } catch {
+      // markitdown failed, fall through
+    }
+  }
+
+  // Fallback: try fetch_content equivalent via curl
+  if (!extracted) {
+    try {
+      const curlResult = await exec(pi, "curl", ["-sL", "--max-time", "30", url], {
+        signal,
+        timeout: 35_000,
+      });
+      if (curlResult.stdout) {
+        extracted = curlResult.stdout;
+        // Try to extract title from HTML
+        const titleMatch = extracted.match(/<title>([^<]*)<\/title>/i);
+        if (titleMatch) title = titleMatch[1].trim();
+      }
+    } catch {
+      // curl failed too
+    }
+  }
+
+  // Write extracted text
+  writeFileSync(
+    join(packetPath, "extracted.md"),
+    extracted || `_Content could not be extracted from ${url}_\n`,
+    "utf-8",
+  );
+
+  // Write manifest
+  const manifest = {
+    id: sourceId,
+    title,
+    url,
+    captured: fmtDate(),
+    format: "web",
+    packet_version: "1.0",
+  };
+  writeJson(join(packetPath, "manifest.json"), manifest);
+
+  // Create skeleton source page in wiki
+  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
+  const sourcePageContent = buildSourcePageSkeleton(manifest, extracted);
+  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+
+  // Log event
+  appendEvent(paths, { kind: "capture", source_id: sourceId, url, format: "web" });
+
+  return { sourceId, packetPath, sourcePagePath, extracted };
+}
+
+/** Capture a local file into a source packet. */
+export async function captureFile(
+  pi: ExtensionAPI,
+  paths: VaultPaths,
+  filePath: string,
+  signal?: AbortSignal,
+): Promise<CaptureResult> {
+  const sourceId = nextSourceId(paths);
+  const packetPath = join(paths.rawSources, sourceId);
+  mkdirSync(packetPath, { recursive: true });
+  mkdirSync(join(packetPath, "original"), { recursive: true });
+  mkdirSync(join(packetPath, "attachments"), { recursive: true });
+
+  const content = readText(filePath);
+  const fileName = filePath.split("/").pop() || "unknown";
+
+  // Try markitdown for PDFs
+  let extracted = content;
+  if (filePath.toLowerCase().endsWith(".pdf")) {
+    const markitdown = await exec(
+      pi,
+      "sh",
+      ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
+      { signal },
+    );
+
+    if (markitdown.stdout.trim() === "yes") {
+      try {
+        const mdResult = await exec(
+          pi,
+          "sh",
+          ["-c", `uvx --from 'markitdown[pdf]' markitdown "${filePath}" 2>/dev/null || echo ""`],
+          { signal, timeout: 30_000 },
+        );
+        if (mdResult.stdout.trim()) extracted = mdResult.stdout;
+      } catch {
+        // fallback to original
+      }
+    }
+  }
+
+  // Copy original to packet
+  try {
+    await exec(pi, "cp", [filePath, join(packetPath, "original", fileName)], { signal });
+  } catch {
+    // If cp fails, just write the content
+    writeFileSync(join(packetPath, "original", fileName), content, "utf-8");
+  }
+
+  // Write extracted text
+  writeFileSync(join(packetPath, "extracted.md"), extracted, "utf-8");
+
+  // Write manifest
+  const manifest = {
+    id: sourceId,
+    title: fileName,
+    file_path: filePath,
+    captured: fmtDate(),
+    format: guessFormat(filePath),
+    packet_version: "1.0",
+  };
+  writeJson(join(packetPath, "manifest.json"), manifest);
+
+  // Create skeleton source page
+  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
+  const sourcePageContent = buildSourcePageSkeleton(manifest, extracted);
+  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+
+  // Log event
+  appendEvent(paths, {
+    kind: "capture",
+    source_id: sourceId,
+    file_path: filePath,
+    format: manifest.format,
+  });
+
+  return { sourceId, packetPath, sourcePagePath, extracted };
+}
+
+/** Capture pasted text into a source packet. */
+export function captureText(paths: VaultPaths, text: string, title?: string): CaptureResult {
+  const sourceId = nextSourceId(paths);
+  const packetPath = join(paths.rawSources, sourceId);
+  mkdirSync(packetPath, { recursive: true });
+  mkdirSync(join(packetPath, "attachments"), { recursive: true });
+
+  // Write extracted text
+  writeFileSync(join(packetPath, "extracted.md"), text, "utf-8");
+
+  // Write manifest
+  const manifest = {
+    id: sourceId,
+    title: title || `Pasted text — ${fmtDate()}`,
+    captured: fmtDate(),
+    format: "text",
+    packet_version: "1.0",
+  };
+  writeJson(join(packetPath, "manifest.json"), manifest);
+
+  // Create skeleton source page
+  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
+  const sourcePageContent = buildSourcePageSkeleton(manifest, text);
+  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+
+  // Log event
+  appendEvent(paths, { kind: "capture", source_id: sourceId, format: "text" });
+
+  return { sourceId, packetPath, sourcePagePath, extracted: text };
+}
+
+/** Build a skeleton source page from manifest and extracted text. */
+function buildSourcePageSkeleton(manifest: Record<string, unknown>, extracted: string): string {
+  const id = String(manifest.id);
+  const title = String(manifest.title || id);
+  const url = manifest.url ? `\n> _Original: ${manifest.url}_` : "";
+  const format = String(manifest.format || "unknown");
+  const captured = String(manifest.captured || fmtDate());
+
+  // Generate a brief auto-summary (first 500 chars)
+  const preview = extracted
+    .replace(/[#*_`]/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+    .slice(0, 500);
+
+  return `---
+type: source
+format: ${format}
+source_id: ${id}
+raw_path: raw/sources/${id}/extracted.md
+captured: ${captured}
+status: skeleton
+---
+
+# ${title}${url}
+
+## Summary
+
+[LLM: Replace with 2-3 paragraph summary of key content]
+
+> _Auto-preview: ${preview}${extracted.length > 500 ? "..." : ""}_
+
+## Key Takeaways
+
+- [LLM: Most important point]
+- [LLM: Second important point]
+- [LLM: Third important point]
+
+## Entities Mentioned
+
+- [[entity-name]]
+
+## Concepts Mentioned
+
+- [[concept-name]]
+
+## Notable Quotes
+
+> [LLM: Important quote] — attribution
+
+## Source Packet
+
+- **ID:** \`[[sources/${id}]]\`
+- **Extracted:** [raw/sources/${id}/extracted.md](../raw/sources/${id}/extracted.md)
+- **Manifest:** [raw/sources/${id}/manifest.json](../raw/sources/${id}/manifest.json)
+`;
+}
+
+function guessFormat(filePath: string): string {
+  const lower = filePath.toLowerCase();
+  if (lower.endsWith(".pdf")) return "pdf";
+  if (lower.endsWith(".md")) return "markdown";
+  if (lower.endsWith(".txt")) return "text";
+  if (lower.endsWith(".html") || lower.endsWith(".htm")) return "html";
+  if (lower.endsWith(".docx")) return "docx";
+  return "file";
+}