@zosmaai/pi-llm-wiki 0.3.0 → 0.5.0

package/extensions/llm-wiki/lib/source-packet.ts

@@ -1,7 +1,8 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { extname, join } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { appendEvent } from "./metadata.js";
+import { type ExtractedContent, extractUrlContent, fileExtractorFor } from "./source-extractors.js";
 import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
 
 /**
@@ -22,295 +23,206 @@ export interface CaptureResult {
   extracted: string;
 }
 
-const DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000;
-const DEFAULT_CURL_TIMEOUT_SECONDS = 30;
-
-function markitdownTimeoutMs(): number {
-  return positiveIntegerFromEnv("WIKI_MARKITDOWN_TIMEOUT_MS", DEFAULT_MARKITDOWN_TIMEOUT_MS);
-}
-
-function positiveIntegerFromEnv(name: string, fallback: number): number {
-  const raw = process.env[name];
-  if (!raw) return fallback;
-  const parsed = Number.parseInt(raw, 10);
-  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
-}
-
-function isPdfUrl(url: string): boolean {
-  try {
-    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
-  } catch {
-    return url.toLowerCase().split(/[?#]/, 1)[0].endsWith(".pdf");
-  }
-}
-
-function looksLikePdf(content: string): boolean {
-  return content.trimStart().startsWith("%PDF-");
+interface SourcePacket {
+  sourceId: string;
+  packetPath: string;
 }
 
-function pdfExtractionFailureMessage(source: string): string {
-  return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
+interface CaptureSource {
+  needsOriginalDir: boolean;
+  fallbackText: string;
+  preserveOriginal?(packetPath: string): Promise<void>;
+  extract(): Promise<ExtractedContent> | ExtractedContent;
+  manifest(content: ExtractedContent): Record<string, unknown>;
+  event(content: ExtractedContent): Record<string, unknown>;
 }
 
 const URL_ORIGINAL_EXTENSIONS = new Set([".html", ".htm", ".md", ".pdf", ".txt", ".xml", ".json"]);
 
-function originalFileNameForUrl(url: string): string {
-  try {
-    const parsed = new URL(url);
-    const ext = extname(parsed.pathname).toLowerCase();
-    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
-  } catch {
-    const path = url.split(/[?#]/, 1)[0] ?? "";
-    const ext = extname(path).toLowerCase();
-    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
-  }
-
-  return "source.html";
-}
-
-async function preserveUrlOriginal(
+/** Capture a URL into a source packet. */
+export async function captureUrl(
   pi: ExtensionAPI,
-  packetPath: string,
+  paths: VaultPaths,
   url: string,
   signal?: AbortSignal,
-): Promise<void> {
-  const originalPath = join(packetPath, "original", originalFileNameForUrl(url));
-  try {
-    await exec(pi, "curl", ["-sL", "--max-time", "30", "-o", originalPath, url], {
-      signal,
-      timeout: 35_000,
-    });
-  } catch {
-    // Preserve best-effort extraction behavior even when the original artifact cannot be saved.
-  }
+): Promise<CaptureResult> {
+  return captureSource(paths, urlCaptureSource(pi, url, signal));
 }
 
-/** Capture a URL into a source packet. */
-export async function captureUrl(
+/** Capture a local file into a source packet. */
+export async function captureFile(
   pi: ExtensionAPI,
   paths: VaultPaths,
-  url: string,
+  filePath: string,
   signal?: AbortSignal,
 ): Promise<CaptureResult> {
-  const sourceId = nextSourceId(paths);
-  const packetPath = join(paths.rawSources, sourceId);
-  mkdirSync(packetPath, { recursive: true });
-  mkdirSync(join(packetPath, "original"), { recursive: true });
-  mkdirSync(join(packetPath, "attachments"), { recursive: true });
+  return captureSource(paths, fileCaptureSource(pi, filePath, signal));
+}
 
-  await preserveUrlOriginal(pi, packetPath, url, signal);
-
-  // Try to fetch and extract content
-  let extracted = "";
-  let title = url;
-  const isPdf = isPdfUrl(url);
-
-  // Try MarkItDown first.
-  const markitdown = await exec(
-    pi,
-    "sh",
-    ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
-    { signal },
-  );
-
-  if (markitdown.stdout.trim() === "yes") {
-    try {
-      const mdResult = await exec(
-        pi,
-        "sh",
-        ["-c", `uvx --from 'markitdown[pdf]' markitdown "${url}" 2>/dev/null || echo ""`],
-        { signal, timeout: markitdownTimeoutMs() },
-      );
-      if (mdResult.stdout.trim()) {
-        extracted = mdResult.stdout;
-        // Try to extract title from first h1
-        const h1Match = extracted.match(/^#\s+(.+)$/m);
-        if (h1Match) title = h1Match[1].trim();
-      }
-    } catch {
-      // markitdown failed, fall through
-    }
-  }
+/** Capture pasted text into a source packet. */
+export function captureText(paths: VaultPaths, text: string, title?: string): CaptureResult {
+  return captureSourceSync(paths, textCaptureSource(text, title));
+}
 
-  // Fallback: try fetch_content equivalent via curl for text/html sources.
-  // Do not write binary PDF bytes into extracted.md when PDF conversion fails.
-  if (!extracted) {
-    if (isPdf) {
-      extracted = pdfExtractionFailureMessage(url);
-    } else {
-      try {
-        const curlResult = await exec(
-          pi,
-          "curl",
-          ["-sL", "--max-time", String(DEFAULT_CURL_TIMEOUT_SECONDS), url],
-          {
-            signal,
-            timeout: (DEFAULT_CURL_TIMEOUT_SECONDS + 5) * 1_000,
-          },
-        );
-        if (curlResult.stdout) {
-          if (looksLikePdf(curlResult.stdout)) {
-            extracted = pdfExtractionFailureMessage(url);
-          } else {
-            extracted = curlResult.stdout;
-            // Try to extract title from HTML
-            const titleMatch = extracted.match(/<title>([^<]*)<\/title>/i);
-            if (titleMatch) title = titleMatch[1].trim();
-          }
-        }
-      } catch {
-        // curl failed too
-      }
-    }
-  }
+async function captureSource(paths: VaultPaths, source: CaptureSource): Promise<CaptureResult> {
+  const packet = createSourcePacket(paths, source.needsOriginalDir);
+  await source.preserveOriginal?.(packet.packetPath);
+  const content = await source.extract();
+  return finalizeCapture(paths, packet, source, content);
+}
 
-  // Write extracted text
-  writeFileSync(
-    join(packetPath, "extracted.md"),
-    extracted || `_Content could not be extracted from ${url}_\n`,
-    "utf-8",
-  );
+function captureSourceSync(paths: VaultPaths, source: CaptureSource): CaptureResult {
+  const packet = createSourcePacket(paths, source.needsOriginalDir);
+  const content = source.extract() as ExtractedContent;
+  return finalizeCapture(paths, packet, source, content);
+}
 
-  // Write manifest
-  const manifest = {
-    id: sourceId,
-    title,
-    url,
-    captured: fmtDate(),
-    format: "web",
-    packet_version: "1.0",
+function urlCaptureSource(pi: ExtensionAPI, url: string, signal?: AbortSignal): CaptureSource {
+  return {
+    needsOriginalDir: true,
+    fallbackText: contentExtractionFailureMessage(url),
+    preserveOriginal: (packetPath) => preserveUrlOriginal(pi, packetPath, url, signal),
+    extract: () => extractUrlContent(pi, url, signal),
+    manifest: (content) => ({
+      title: content.title || url,
+      url,
+      format: "web",
+    }),
+    event: () => ({ url, format: "web" }),
   };
-  writeJson(join(packetPath, "manifest.json"), manifest);
-
-  // Create skeleton source page in wiki
-  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
-  const sourcePageContent = buildSourcePageSkeleton(manifest, extracted);
-  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
-
-  // Log event
-  appendEvent(paths, { kind: "capture", source_id: sourceId, url, format: "web" });
-
-  return { sourceId, packetPath, sourcePagePath, extracted };
 }
 
-/** Capture a local file into a source packet. */
-export async function captureFile(
+function fileCaptureSource(
   pi: ExtensionAPI,
-  paths: VaultPaths,
   filePath: string,
   signal?: AbortSignal,
-): Promise<CaptureResult> {
+): CaptureSource {
+  const fileName = filePath.split("/").pop() || "unknown";
+  const extractor = fileExtractorFor(filePath);
+  const content = extractor.shouldReadText ? readText(filePath) : "";
+
+  return {
+    needsOriginalDir: true,
+    fallbackText: "",
+    preserveOriginal: (packetPath) =>
+      preserveFileOriginal(pi, packetPath, filePath, fileName, content, signal),
+    extract: async () => ({
+      extracted: await extractor.extract({ pi, filePath, content, signal }),
+    }),
+    manifest: () => ({
+      title: fileName,
+      file_path: filePath,
+      format: extractor.format,
+    }),
+    event: () => ({ file_path: filePath, format: extractor.format }),
+  };
+}
+
+function textCaptureSource(text: string, title?: string): CaptureSource {
+  return {
+    needsOriginalDir: false,
+    fallbackText: "",
+    extract: () => ({ extracted: text }),
+    manifest: () => ({
+      title: title || `Pasted text — ${fmtDate()}`,
+      format: "text",
+    }),
+    event: () => ({ format: "text" }),
+  };
+}
+
+function createSourcePacket(paths: VaultPaths, needsOriginalDir: boolean): SourcePacket {
   const sourceId = nextSourceId(paths);
   const packetPath = join(paths.rawSources, sourceId);
   mkdirSync(packetPath, { recursive: true });
-  mkdirSync(join(packetPath, "original"), { recursive: true });
   mkdirSync(join(packetPath, "attachments"), { recursive: true });
+  if (needsOriginalDir) mkdirSync(join(packetPath, "original"), { recursive: true });
+  return { sourceId, packetPath };
+}
 
-  const lowerPath = filePath.toLowerCase();
-  const isPdf = lowerPath.endsWith(".pdf");
-  const isXml = lowerPath.endsWith(".xml");
-  const content = isPdf ? "" : readText(filePath);
-  const fileName = filePath.split("/").pop() || "unknown";
-
-  let extracted = content;
-
-  // Convert XML to markdown
-  if (isXml && content) {
-    extracted = xmlToMarkdown(content);
-  }
-
-  // Try MarkItDown for PDFs.
-  if (isPdf) {
-    const markitdown = await exec(
-      pi,
-      "sh",
-      ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
-      { signal },
-    );
-
-    if (markitdown.stdout.trim() === "yes") {
-      try {
-        const mdResult = await exec(
-          pi,
-          "sh",
-          ["-c", `uvx --from 'markitdown[pdf]' markitdown "${filePath}" 2>/dev/null || echo ""`],
-          { signal, timeout: markitdownTimeoutMs() },
-        );
-        if (mdResult.stdout.trim()) extracted = mdResult.stdout;
-      } catch {
-        extracted = pdfExtractionFailureMessage(filePath);
-      }
-    }
-    if (!extracted) extracted = pdfExtractionFailureMessage(filePath);
-  }
-
-  // Copy original to packet
-  try {
-    await exec(pi, "cp", [filePath, join(packetPath, "original", fileName)], { signal });
-  } catch {
-    // If cp fails, just write the content
-    writeFileSync(join(packetPath, "original", fileName), content, "utf-8");
-  }
-
-  // Write extracted text
-  writeFileSync(join(packetPath, "extracted.md"), extracted, "utf-8");
-
-  // Write manifest
+function finalizeCapture(
+  paths: VaultPaths,
+  packet: SourcePacket,
+  source: CaptureSource,
+  content: ExtractedContent,
+): CaptureResult {
+  const extracted = content.extracted || source.fallbackText;
   const manifest = {
-    id: sourceId,
-    title: fileName,
-    file_path: filePath,
+    id: packet.sourceId,
     captured: fmtDate(),
-    format: guessFormat(filePath),
     packet_version: "1.0",
+    ...source.manifest({ ...content, extracted }),
   };
-  writeJson(join(packetPath, "manifest.json"), manifest);
 
-  // Create skeleton source page
-  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
-  const sourcePageContent = buildSourcePageSkeleton(manifest, extracted);
-  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+  writeFileSync(join(packet.packetPath, "extracted.md"), extracted, "utf-8");
+  writeJson(join(packet.packetPath, "manifest.json"), manifest);
+
+  const sourcePagePath = join(paths.wiki, "sources", `${packet.sourceId}.md`);
+  writeFileSync(sourcePagePath, buildSourcePageSkeleton(manifest, extracted), "utf-8");
 
-  // Log event
   appendEvent(paths, {
     kind: "capture",
-    source_id: sourceId,
-    file_path: filePath,
-    format: manifest.format,
+    source_id: packet.sourceId,
+    ...source.event({ ...content, extracted }),
   });
 
-  return { sourceId, packetPath, sourcePagePath, extracted };
+  return {
+    sourceId: packet.sourceId,
+    packetPath: packet.packetPath,
+    sourcePagePath,
+    extracted,
+  };
 }
 
-/** Capture pasted text into a source packet. */
-export function captureText(paths: VaultPaths, text: string, title?: string): CaptureResult {
-  const sourceId = nextSourceId(paths);
-  const packetPath = join(paths.rawSources, sourceId);
-  mkdirSync(packetPath, { recursive: true });
-  mkdirSync(join(packetPath, "attachments"), { recursive: true });
-
-  // Write extracted text
-  writeFileSync(join(packetPath, "extracted.md"), text, "utf-8");
+async function preserveFileOriginal(
+  pi: ExtensionAPI,
+  packetPath: string,
+  filePath: string,
+  fileName: string,
+  fallbackContent: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  try {
+    await exec(pi, "cp", [filePath, join(packetPath, "original", fileName)], { signal });
+  } catch {
+    // If cp fails, preserve whatever text content was available.
+    writeFileSync(join(packetPath, "original", fileName), fallbackContent, "utf-8");
+  }
+}
 
-  // Write manifest
-  const manifest = {
-    id: sourceId,
-    title: title || `Pasted text — ${fmtDate()}`,
-    captured: fmtDate(),
-    format: "text",
-    packet_version: "1.0",
-  };
-  writeJson(join(packetPath, "manifest.json"), manifest);
+async function preserveUrlOriginal(
+  pi: ExtensionAPI,
+  packetPath: string,
+  url: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  const originalPath = join(packetPath, "original", originalFileNameForUrl(url));
+  try {
+    await exec(pi, "curl", ["-sL", "--max-time", "30", "-o", originalPath, url], {
+      signal,
+      timeout: 35_000,
+    });
+  } catch {
+    // Preserve best-effort extraction behavior even when the original artifact cannot be saved.
+  }
+}
 
-  // Create skeleton source page
-  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
-  const sourcePageContent = buildSourcePageSkeleton(manifest, text);
-  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+function originalFileNameForUrl(url: string): string {
+  try {
+    const parsed = new URL(url);
+    const ext = extname(parsed.pathname).toLowerCase();
+    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
+  } catch {
+    const path = url.split(/[?#]/, 1)[0] ?? "";
+    const ext = extname(path).toLowerCase();
+    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
+  }
 
-  // Log event
-  appendEvent(paths, { kind: "capture", source_id: sourceId, format: "text" });
+  return "source.html";
+}
 
-  return { sourceId, packetPath, sourcePagePath, extracted: text };
+function contentExtractionFailureMessage(source: string): string {
+  return `_Content could not be extracted from ${source}_\n`;
 }
 
 /** Build a skeleton source page from manifest and extracted text. */
@@ -370,59 +282,3 @@ status: skeleton
 - **Manifest:** [raw/sources/${id}/manifest.json](../raw/sources/${id}/manifest.json)
 `;
 }
-
-/** Basic XML to markdown conversion: strip tags while preserving text structure. */
-function xmlToMarkdown(xml: string): string {
-  // Extract title from first <title> or root element
-  let title = "";
-  const titleMatch = xml.match(/<title[^>]*>([^<]*)<\/title>/i);
-  if (titleMatch) title = titleMatch[1].trim();
-
-  // Strip XML declaration and doctype
-  let text = xml.replace(/<\?xml[^>]*\?>\s*/gi, "");
-  text = text.replace(/<!DOCTYPE[^>]*>\s*/gi, "");
-
-  // Replace block-level tags with newlines
-  text = text.replace(/<\/(p|div|section|article|li|h\d|tr|blockquote|pre)>/gi, "\n");
-  text = text.replace(/<br\s*\/?>/gi, "\n");
-
-  // Strip remaining tags — match < followed by tag name characters to >
-  // Using a loop to handle malformed/broken tags that lack a closing >
-  let prev = "";
-  while (prev !== text) {
-    prev = text;
-    text = text.replace(/<[a-zA-Z\/!?][^>]*>/g, "");
-  }
-  // Remove any stray < that didn't form a complete tag
-  text = text.replace(/</g, "");
-
-  // Decode XML entities in a single pass to avoid double-unescaping
-  text = text.replace(/&(?:amp|lt|gt|quot|#\d+);/gi, (entity) => {
-    const map: Record<string, string> = { "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"' };
-    const lower = entity.toLowerCase();
-    if (map[lower]) return map[lower];
-    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
-    return entity;
-  });
-
-  // Clean up excessive blank lines
-  text = text.replace(/\n{3,}/g, "\n\n").trim();
-
-  if (!text) return xml; // fallback: return raw if stripping produced nothing
-
-  const lines = [];
-  if (title) lines.push(`# ${title}\n`);
-  lines.push(text);
-  return lines.join("\n\n");
-}
-
-function guessFormat(filePath: string): string {
-  const lower = filePath.toLowerCase();
-  if (lower.endsWith(".pdf")) return "pdf";
-  if (lower.endsWith(".md")) return "markdown";
-  if (lower.endsWith(".txt")) return "text";
-  if (lower.endsWith(".html") || lower.endsWith(".htm")) return "html";
-  if (lower.endsWith(".xml")) return "xml";
-  if (lower.endsWith(".docx")) return "docx";
-  return "file";
-}

package/package.json

@@ -1,17 +1,26 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.3.0",
-  "description": "LLM Wiki for Pi — self-maintaining knowledge base following Karpathy's pattern. Obsidian-friendly, auto-updating, personal & company wiki.",
+  "version": "0.5.0",
+  "description": "Self-maintaining LLM Wiki for Pi — Karpathy-pattern knowledge base with immutable source capture, automated ingestion, search, linting, and Obsidian-compatible vault. auto-updating personal & company wiki.",
   "keywords": [
+    "pi",
     "pi-package",
-    "pi-skill",
     "pi-extension",
+    "pi-skill",
     "llm-wiki",
     "karpathy",
     "knowledge-base",
     "obsidian",
     "second-brain",
     "wiki",
+    "memory",
+    "markdown",
+    "pkm",
+    "personal-knowledge-management",
+    "research",
+    "vault",
+    "zettelkasten",
+    "rag",
     "zosmaai"
   ],
   "license": "MIT",
@@ -24,6 +33,19 @@
     "url": "https://github.com/zosmaai/pi-llm-wiki/issues"
   },
   "homepage": "https://github.com/zosmaai/pi-llm-wiki#readme",
+  "files": [
+    "extensions",
+    "skills",
+    "prompts",
+    "docs",
+    "assets",
+    "README.md",
+    "CHANGELOG.md",
+    "LICENSE"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
   "scripts": {
     "test": "vitest run",
     "test:watch": "vitest",

package/.coderabbit.yaml

@@ -1,43 +0,0 @@
-# CodeRabbit AI Code Review Configuration
-# Docs: https://docs.coderabbit.ai/guides/configure-coderabbit
-
-language: en-US
-early_access: false
-reviews:
-  profile: assertive
-  request_changes_workflow: true
-  high_level_summary: true
-  poem: false
-  review_status: true
-  collapse_walkthrough: false
-  auto_review:
-    enabled: true
-    ignore_title_keywords:
-      - "chore"
-      - "WIP"
-      - "wip"
-    drafts: false
-    base_branches:
-      - main
-  path_instructions:
-    - path: "extensions/**/*.ts"
-      instructions: |
-        Review TypeScript extension files for pi coding agent. Check that:
-        1. Tools use pi.registerTool() with TypeBox schemas
-        2. Error handling uses try/catch with proper messages
-        3. Unused parameters use underscore prefix or are removed
-        4. File exports a default function receiving ExtensionAPI
-    - path: "skills/**/SKILL.md"
-      instructions: |
-        Review SKILL.md for pi. Verify:
-        1. Frontmatter follows Agent Skills spec (name, description)
-        2. Workflows are clearly documented with steps
-        3. Golden rules and conventions are explicit
-    - path: "prompts/**/*.md"
-      instructions: |
-        Review prompt templates. Verify:
-        1. Frontmatter has description, args, section, topLevelCli
-        2. Instructions are actionable and clear
-
-chat:
-  auto_reply: true

package/.github/codeql/codeql-config.yml

@@ -1,12 +0,0 @@
-name: "CodeQL Config"
-
-queries:
-  - uses: security-and-quality
-
-# Suppress false positives for temporary file creation.
-# This is a single-user CLI tool — TOCTOU race conditions on temp directories
-# are not a valid threat model. The alerts only fire because tests pass
-# os.tmpdir() paths into production capture functions.
-query-filters:
-  - exclude:
-      id: js/insecure-temporary-file

package/.github/workflows/ci.yml

@@ -1,45 +0,0 @@
-name: CI
-
-on:
-  push:
-    branches: [main]
-  pull_request:
-    branches: [main]
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  quality:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [20, 22, 23, 24, 25]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-node@v4
-        with:
-          node-version: ${{ matrix.node-version }}
-          cache: npm
-
-      - run: npm ci
-
-      - name: Type Check
-        run: npm run typecheck
-
-      - name: Lint
-        run: npm run lint
-
-      - name: Run Tests with Coverage
-        run: npm run test:coverage
-
-      - name: Upload Coverage to Codecov
-        if: ${{ matrix.node-version == 22 && github.event_name == 'push' }}
-        uses: codecov/codecov-action@v5
-        with:
-          files: ./coverage/lcov.info
-          fail_ci_if_error: false
-          verbose: true