@zosmaai/pi-llm-wiki 0.1.6 → 0.2.1

package/docs/api.md

@@ -0,0 +1,105 @@
+# API Reference
+
+## Extension Tools
+
+### wiki_bootstrap
+
+Initialize a new LLM Wiki vault.
+
+```
+wiki_bootstrap(topic: string, mode?: "personal" | "company", root?: string)
+```
+
+### wiki_capture_source
+
+Capture a URL, file, or text into an immutable source packet.
+
+```
+wiki_capture_source(
+  inputType: "url" | "file" | "text",
+  value: string,
+  title?: string,
+  kind?: string,
+  tags?: string[],
+  createSourcePage?: boolean
+)
+```
+
+### wiki_ingest
+
+Get a batch of sources needing synthesis.
+
+```
+wiki_ingest(batch_size?: number)
+```
+
+### wiki_ensure_page
+
+Create or resolve a canonical page.
+
+```
+wiki_ensure_page(
+  type: "concept" | "entity" | "synthesis" | "analysis",
+  title: string,
+  aliases?: string[],
+  tags?: string[],
+  summary?: string,
+  createIfMissing?: boolean
+)
+```
+
+### wiki_search
+
+Search the wiki registry.
+
+```
+wiki_search(query: string, type?: string, limit?: number)
+```
+
+### wiki_lint
+
+Health check the wiki.
+
+```
+wiki_lint(mode?: string, writeReport?: boolean, limit?: number)
+```
+
+### wiki_status
+
+Show wiki statistics.
+
+```
+wiki_status()
+```
+
+### wiki_rebuild_meta
+
+Force metadata rebuild.
+
+```
+wiki_rebuild_meta()
+```
+
+### wiki_log_event
+
+Record a structured event.
+
+```
+wiki_log_event(
+  kind: string,
+  title: string,
+  summary?: string,
+  sourceIds?: string[],
+  pagePaths?: string[],
+  notes?: string[],
+  actor?: "agent" | "user" | "extension"
+)
+```
+
+### wiki_watch
+
+Schedule auto-updates.
+
+```
+wiki_watch(schedule: string, command: string)
+```

package/docs/architecture.md

@@ -0,0 +1,65 @@
+# Architecture
+
+## Four Layers
+
+```
+WIKI_ROOT/
+├── raw/sources/SRC-*/       # Immutable source packets (extension-owned)
+│   ├── manifest.json         # Capture metadata
+│   ├── original/             # Original artifact
+│   ├── extracted.md          # Normalized markdown
+│   └── attachments/          # Downloaded images, PDFs
+├── wiki/                     # Editable knowledge pages (you + LLM)
+│   ├── sources/              # One summary per source
+│   ├── entities/             # People, orgs, tools, products
+│   ├── concepts/             # Ideas, patterns, frameworks
+│   ├── syntheses/            # Cross-cutting analyses
+│   └── analyses/             # Durable query answers
+├── meta/                     # Auto-generated (extension-owned)
+│   ├── registry.json         # Master page catalog
+│   ├── backlinks.json        # Inbound link map
+│   ├── index.md              # Human-readable catalog
+│   ├── log.md                # Activity log
+│   └── events.jsonl          # Structured event stream
+└── .wiki/                    # Config and templates
+    ├── config.json
+    └── templates/
+```
+
+## Ownership Rules
+
+| Path      | Owner                    | Rule                     |
+| --------- | ------------------------ | ------------------------ |
+| `raw/**`  | Extension                | Immutable after capture  |
+| `wiki/**` | Model + user             | Editable knowledge pages |
+| `meta/*`  | Extension                | Auto-generated           |
+| `.wiki/*` | Human + explicit request | Operating rules          |
+
+## Source Packet Format
+
+Each captured source becomes a packet:
+
+```
+raw/sources/SRC-YYYY-MM-DD-NNN/
+  manifest.json
+  original/
+  extracted.md
+  attachments/
+```
+
+## Page Types
+
+- **source** — what this specific source says
+- **entity** — people, orgs, tools, products
+- **concept** — ideas, patterns, frameworks
+- **synthesis** — cross-source theses and tensions
+- **analysis** — durable filed answers from queries
+
+## Linking Style
+
+- Internal: `[[folder/page-name]]`
+- Citation: `[[sources/SRC-YYYY-MM-DD-NNN]]`
+
+## Guardrails
+
+The extension blocks direct edits to `raw/**` and `meta/**`. Metadata rebuilds automatically after `wiki/**` edits.

package/docs/commands.md

@@ -0,0 +1,51 @@
+# Commands
+
+## Slash Commands
+
+| Command          | Description                           |
+| ---------------- | ------------------------------------- |
+| `/wiki-init`     | Create a new wiki vault               |
+| `/wiki-ingest`   | Process new sources                   |
+| `/wiki-query`    | Ask questions against the wiki        |
+| `/wiki-lint`     | Health check                          |
+| `/wiki-discover` | Auto-discover sources                 |
+| `/wiki-run`      | Full cycle (discover → ingest → lint) |
+| `/wiki-status`   | Show wiki health                      |
+| `/wiki-digest`   | Daily/weekly summary                  |
+
+## Extension Tools
+
+The extension registers 10 tools the LLM can call directly:
+
+| Tool                  | Purpose                                     |
+| --------------------- | ------------------------------------------- |
+| `wiki_bootstrap`      | Initialize a new vault                      |
+| `wiki_capture_source` | Capture URL/file/text into immutable packet |
+| `wiki_ingest`         | Get batch of uningested sources             |
+| `wiki_ensure_page`    | Create canonical page from template         |
+| `wiki_search`         | Search the wiki registry                    |
+| `wiki_lint`           | Health check with auto-fix                  |
+| `wiki_status`         | Instant stats                               |
+| `wiki_rebuild_meta`   | Force metadata rebuild                      |
+| `wiki_log_event`      | Record custom event                         |
+| `wiki_watch`          | Schedule auto-updates                       |
+
+## Workflows
+
+### Capture → Ingest → Synthesize
+
+1. `wiki_capture_source(url="...")` — creates packet + skeleton
+2. `wiki_ingest()` — get batch of sources needing synthesis
+3. Read `raw/sources/SRC-*/extracted.md`
+4. Update skeleton source page with summary, entities, concepts
+5. `wiki_ensure_page(type="entity", title="...")` for each entity
+6. Add `[[wikilinks]]` between related pages
+7. Extension auto-rebuilds metadata
+
+### Query → Answer → File
+
+1. `wiki_search(query="...")` to find relevant pages
+2. Read those pages
+3. Synthesize answer with `[[wikilink]]` citations
+4. If novel: create analysis page via `wiki_ensure_page(type="analysis")`
+5. Extension auto-updates metadata

package/docs/configuration.md

@@ -0,0 +1,38 @@
+# Configuration
+
+Wiki configuration lives in `.wiki/config.json`.
+
+## Modes
+
+### Personal
+
+- Extra folders: `wiki/journal/`, `wiki/goals/`
+- Track: learning, books, health, reflections
+
+### Company
+
+- Extra folders: `wiki/changes/`, `wiki/decisions/`
+- Track: competitors, market, strategy
+- Frontmatter: `confidence: high | medium | low`
+
+## Settings
+
+| Setting                    | Default | Description                        |
+| -------------------------- | ------- | ---------------------------------- |
+| `max_sources_per_discover` | 8       | Sources fetched per discovery run  |
+| `auto_fix_lint`            | false   | Auto-fix lint issues               |
+| `batch_ingest_size`        | 3       | Sources processed per ingest batch |
+
+## Page Frontmatter
+
+```yaml
+---
+type: entity | concept | source | synthesis | analysis
+created: YYYY-MM-DD
+updated: YYYY-MM-DD
+sources: [sources/SRC-YYYY-MM-DD-NNN]
+---
+```
+
+Entity: add `category: person | organization | tool | project | product`
+Concept: add `domain: ai | engineering | business | product | design | personal`

package/docs/obsidian.md

@@ -0,0 +1,21 @@
+# Obsidian Integration
+
+## Setup
+
+1. Open `wiki/` as an Obsidian vault
+2. The extension generates `meta/index.md` as a browsable catalog
+3. `meta/backlinks.json` is available for graph plugins
+
+## Recommended Plugins
+
+- [Dataview](https://github.com/blacksmithgu/obsidian-dataview) — Query pages by frontmatter
+- [Graph View](https://obsidian.md) (built-in) — Visualize `[[wikilink]]` connections
+- [Backlinks](https://obsidian.md) (built-in) — See inbound links
+
+## Web Clipper
+
+Use [Obsidian Web Clipper](https://obsidian.md/clipper) to save articles directly into `raw/articles/`.
+
+## Dataview Dashboard
+
+The extension creates `meta/index.md` with page listings. For custom dashboards, use Dataview queries against frontmatter fields like `type`, `domain`, `category`, `sources`.

package/extensions/llm-wiki/index.ts

@@ -0,0 +1,53 @@
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { installGuardrails } from "./lib/guardrails.js";
+import {
+  registerWikiBootstrap,
+  registerWikiCaptureSource,
+  registerWikiEnsurePage,
+  registerWikiIngest,
+  registerWikiLint,
+  registerWikiLogEvent,
+  registerWikiRebuildMeta,
+  registerWikiSearch,
+  registerWikiStatus,
+  registerWikiWatch,
+} from "./lib/tools.js";
+
+/**
+ * @zosmaai/pi-llm-wiki — LLM Wiki extension for Pi
+ *
+ * Registers 10 custom tools and installs guardrails:
+ * - wiki_bootstrap      Initialize a new vault
+ * - wiki_capture_source Capture URL/file/text into source packet
+ * - wiki_ingest         Get batch of sources needing synthesis
+ * - wiki_ensure_page    Create canonical page from template
+ * - wiki_search         Search generated registry
+ * - wiki_lint           Health check with auto-fix
+ * - wiki_status         Instant stats from registry
+ * - wiki_rebuild_meta   Force metadata rebuild
+ * - wiki_log_event      Append event and regenerate log
+ * - wiki_watch          Schedule auto-updates
+ *
+ * Guardrails:
+ * - Blocks direct edits to raw/** and meta/**
+ * - Auto-rebuilds metadata after wiki/** edits
+ */
+
+export default function (pi: ExtensionAPI) {
+  registerWikiBootstrap(pi);
+  registerWikiCaptureSource(pi);
+  registerWikiIngest(pi);
+  registerWikiEnsurePage(pi);
+  registerWikiSearch(pi);
+  registerWikiLint(pi);
+  registerWikiStatus(pi);
+  registerWikiRebuildMeta(pi);
+  registerWikiLogEvent(pi);
+  registerWikiWatch(pi);
+
+  installGuardrails(pi);
+
+  pi.on("session_start", async (_event, ctx) => {
+    ctx.ui.setStatus("llm-wiki", "🧠 LLM Wiki (10 tools, guardrails active)");
+  });
+}

package/extensions/llm-wiki/lib/guardrails.ts

@@ -0,0 +1,69 @@
+import { isToolCallEventType } from "@mariozechner/pi-coding-agent";
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { rebuildMetadataLight } from "./metadata.js";
+import { isProtectedPath, resolveVaultRoot } from "./utils.js";
+
+/**
+ * Guardrails and auto-rebuild hooks for the LLM Wiki extension.
+ */
+
+let pendingRebuild = false;
+
+/** Install guardrails on the extension API. */
+export function installGuardrails(pi: ExtensionAPI): void {
+  // Block direct edits to raw/ and meta/
+  pi.on("tool_call", async (event) => {
+    if (isToolCallEventType("write", event)) {
+      const path = event.input.path as string;
+      const root = resolveVaultRoot(process.cwd());
+      const check = isProtectedPath(path, root);
+      if (check.protected) {
+        return { block: true, reason: check.reason };
+      }
+    }
+
+    if (isToolCallEventType("edit", event)) {
+      const path = event.input.path as string;
+      const root = resolveVaultRoot(process.cwd());
+      const check = isProtectedPath(path, root);
+      if (check.protected) {
+        return { block: true, reason: check.reason };
+      }
+    }
+  });
+
+  // Track wiki edits for auto-rebuild
+  pi.on("tool_result", async (event) => {
+    if (event.toolName === "write" || event.toolName === "edit") {
+      const path = event.input.path as string;
+      const root = resolveVaultRoot(process.cwd());
+      const wikiPath = `${root}/wiki/`;
+      if (path?.startsWith(wikiPath)) {
+        pendingRebuild = true;
+      }
+    }
+  });
+
+  // Rebuild metadata at end of turn if wiki was modified
+  pi.on("turn_end", async (_event) => {
+    if (pendingRebuild) {
+      pendingRebuild = false;
+      try {
+        const root = resolveVaultRoot(process.cwd());
+        const paths = {
+          root,
+          raw: `${root}/raw`,
+          rawSources: `${root}/raw/sources`,
+          wiki: `${root}/wiki`,
+          meta: `${root}/meta`,
+          dotWiki: `${root}/.wiki`,
+          outputs: `${root}/outputs`,
+          discoveries: `${root}/.discoveries`,
+        };
+        rebuildMetadataLight(paths);
+      } catch {
+        // Silently fail — metadata rebuild is best-effort
+      }
+    }
+  });
+}

package/extensions/llm-wiki/lib/metadata.ts

@@ -0,0 +1,218 @@
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import {
+  type VaultPaths,
+  extractWikilinks,
+  findWikiPages,
+  fmtDate,
+  parseFrontmatter,
+  readJson,
+  readText,
+  writeJson,
+} from "./utils.js";
+
+/**
+ * Metadata generation for the LLM Wiki.
+ *
+ * Rebuilds registry.json, backlinks.json, index.md, log.md, and lint-report.md
+ * deterministically from the current state of raw/ and wiki/.
+ */
+
+export interface RegistryEntry {
+  type: "source" | "entity" | "concept" | "synthesis" | "analysis";
+  title: string;
+  created: string;
+  updated: string;
+  [key: string]: unknown;
+}
+
+export interface Registry {
+  version: string;
+  last_updated: string;
+  pages: Record<string, RegistryEntry>;
+}
+
+export interface Backlinks {
+  [pageId: string]: string[];
+}
+
+export interface WikiEvent {
+  timestamp: string;
+  kind: string;
+  [key: string]: unknown;
+}
+
+/** Rebuild the complete metadata layer. */
+export function rebuildMetadata(paths: VaultPaths): void {
+  mkdirSync(paths.meta, { recursive: true });
+
+  const registry = buildRegistry(paths);
+  const backlinks = buildBacklinks(paths, registry);
+
+  writeJson(join(paths.meta, "registry.json"), registry);
+  writeJson(join(paths.meta, "backlinks.json"), backlinks);
+  writeFileSync(join(paths.meta, "index.md"), buildIndexMarkdown(registry), "utf-8");
+
+  const log = buildLogMarkdown(paths);
+  writeFileSync(join(paths.meta, "log.md"), log, "utf-8");
+}
+
+/** Build registry from wiki/ and raw/ state. */
+export function buildRegistry(paths: VaultPaths): Registry {
+  const pages: Record<string, RegistryEntry> = {};
+
+  // Scan wiki pages
+  for (const page of findWikiPages(paths.wiki)) {
+    const { frontmatter } = parseFrontmatter(page.content);
+    const type = String(frontmatter.type || "page") as RegistryEntry["type"];
+    const title = String(frontmatter.title || page.relative.split("/").pop() || "Untitled");
+
+    pages[page.relative] = {
+      type,
+      title,
+      created: String(frontmatter.created || fmtDate()),
+      updated: String(frontmatter.updated || frontmatter.created || fmtDate()),
+      ...frontmatter,
+    };
+  }
+
+  // Scan raw source packets
+  if (existsSync(paths.rawSources)) {
+    for (const entry of readdirSync(paths.rawSources)) {
+      const manifestPath = join(paths.rawSources, entry, "manifest.json");
+      if (!existsSync(manifestPath)) continue;
+
+      const manifest = readJson<Record<string, unknown>>(manifestPath, {});
+      const id = String(manifest.id || entry);
+      const sourcePage = `sources/${id}`;
+
+      if (!pages[sourcePage]) {
+        pages[sourcePage] = {
+          type: "source",
+          title: String(manifest.title || id),
+          created: String(manifest.captured || fmtDate()),
+          updated: String(manifest.captured || fmtDate()),
+          ...manifest,
+        };
+      }
+    }
+  }
+
+  return {
+    version: "1.0",
+    last_updated: new Date().toISOString(),
+    pages,
+  };
+}
+
+/** Build backlinks map from all wiki pages. */
+export function buildBacklinks(paths: VaultPaths, registry: Registry): Backlinks {
+  const inbound: Backlinks = {};
+
+  // Initialize all pages with empty arrays
+  for (const id of Object.keys(registry.pages)) {
+    inbound[id] = [];
+  }
+
+  // Count inbound links
+  for (const page of findWikiPages(paths.wiki)) {
+    const links = extractWikilinks(page.content);
+    for (const link of links) {
+      if (inbound[link] && !inbound[link].includes(page.relative)) {
+        inbound[link].push(page.relative);
+      }
+    }
+  }
+
+  return inbound;
+}
+
+/** Build index markdown from registry. */
+export function buildIndexMarkdown(registry: Registry): string {
+  const byType: Record<string, Array<{ id: string; entry: RegistryEntry }>> = {};
+
+  for (const [id, entry] of Object.entries(registry.pages)) {
+    const t = entry.type;
+    if (!byType[t]) byType[t] = [];
+    byType[t].push({ id, entry });
+  }
+
+  const sections: string[] = [];
+  sections.push(
+    "# Wiki Index\n\n> Auto-generated from meta/registry.json. Do not edit manually.\n",
+  );
+
+  for (const [type, items] of Object.entries(byType).sort()) {
+    const label = `${type.charAt(0).toUpperCase() + type.slice(1)}s`;
+    sections.push(`## ${label}\n`);
+    for (const { id, entry } of items.sort((a, b) => a.id.localeCompare(b.id))) {
+      sections.push(`- [[${id}]] — ${entry.title} *(created: ${entry.created})*`);
+    }
+    sections.push("");
+  }
+
+  sections.push(
+    `---\n*Last updated: ${registry.last_updated}* | *Total pages: ${Object.keys(registry.pages).length}*`,
+  );
+  return `${sections.join("\n")}\n`;
+}
+
+/** Build log markdown from events.jsonl. */
+export function buildLogMarkdown(paths: VaultPaths): string {
+  const eventsPath = join(paths.meta, "events.jsonl");
+  const events: WikiEvent[] = [];
+
+  if (existsSync(eventsPath)) {
+    const raw = readFileSync(eventsPath, "utf-8").trim();
+    for (const line of raw.split("\n")) {
+      if (!line.trim()) continue;
+      try {
+        events.push(JSON.parse(line) as WikiEvent);
+      } catch {
+        // skip malformed
+      }
+    }
+  }
+
+  const lines: string[] = [];
+  lines.push("# Activity Log\n\n> Auto-generated from meta/events.jsonl. Do not edit manually.\n");
+
+  for (const ev of events) {
+    const ts = ev.timestamp || "unknown";
+    const kind = ev.kind || "event";
+    const details = Object.entries(ev)
+      .filter(([k]) => k !== "timestamp" && k !== "kind")
+      .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
+      .join(", ");
+
+    lines.push(`## [${ts}] ${kind}`);
+    if (details) lines.push(`- ${details}`);
+    lines.push("");
+  }
+
+  if (events.length === 0) {
+    lines.push("_No events recorded yet._\n");
+  }
+
+  return `${lines.join("\n")}\n`;
+}
+
+/** Append an event to events.jsonl. */
+export function appendEvent(paths: VaultPaths, event: Omit<WikiEvent, "timestamp">): void {
+  mkdirSync(paths.meta, { recursive: true });
+  const eventsPath = join(paths.meta, "events.jsonl");
+  const line = JSON.stringify({ timestamp: new Date().toISOString(), ...event });
+  writeFileSync(eventsPath, `${line}\n`, { flag: "a", encoding: "utf-8" });
+}
+
+/** Quick lightweight metadata rebuild (backlinks + index + log only). */
+export function rebuildMetadataLight(paths: VaultPaths): void {
+  const registry = buildRegistry(paths);
+  const backlinks = buildBacklinks(paths, registry);
+  writeJson(join(paths.meta, "registry.json"), registry);
+  writeJson(join(paths.meta, "backlinks.json"), backlinks);
+  writeFileSync(join(paths.meta, "index.md"), buildIndexMarkdown(registry), "utf-8");
+
+  const log = buildLogMarkdown(paths);
+  writeFileSync(join(paths.meta, "log.md"), log, "utf-8");
+}