@zosmaai/pi-llm-wiki 0.3.0 → 0.4.0

package/extensions/llm-wiki/lib/source-extractors.ts

@@ -0,0 +1,369 @@
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { exec } from "./utils.js";
+
+export interface ExtractedContent {
+  extracted: string;
+  title?: string;
+}
+
+export interface FileExtractor {
+  format: string;
+  shouldReadText: boolean;
+  matches(filePath: string): boolean;
+  extract(args: FileExtractArgs): Promise<string> | string;
+}
+
+interface FileExtractArgs {
+  pi: ExtensionAPI;
+  filePath: string;
+  content: string;
+  signal?: AbortSignal;
+}
+
+interface UrlExtractor {
+  matches(url: string): boolean;
+  extract(args: UrlExtractArgs): Promise<ExtractedContent>;
+}
+
+interface UrlExtractArgs {
+  pi: ExtensionAPI;
+  url: string;
+  signal?: AbortSignal;
+}
+
+const DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000;
+const DEFAULT_CURL_TIMEOUT_SECONDS = 30;
+
+const FILE_EXTRACTORS: FileExtractor[] = [
+  {
+    format: "pdf",
+    shouldReadText: false,
+    matches: hasExtension(".pdf"),
+    extract: ({ pi, filePath, signal }) => extractPdf(pi, filePath, signal),
+  },
+  textFileExtractor("markdown", [".md"]),
+  textFileExtractor("text", [".txt"]),
+  textFileExtractor("html", [".html", ".htm"]),
+  {
+    format: "xml",
+    shouldReadText: true,
+    matches: hasExtension(".xml"),
+    extract: ({ content }) => xmlToMarkdown(content),
+  },
+  {
+    format: "json",
+    shouldReadText: true,
+    matches: hasExtension(".json"),
+    extract: ({ content }) => jsonToMarkdown(content),
+  },
+  textFileExtractor("docx", [".docx"]),
+  textFileExtractor("file", []),
+];
+
+const URL_EXTRACTORS: UrlExtractor[] = [
+  {
+    matches: isPdfUrl,
+    extract: ({ pi, url, signal }) => extractPdfUrl(pi, url, signal),
+  },
+  {
+    matches: () => true,
+    extract: ({ pi, url, signal }) => extractTextUrl(pi, url, signal),
+  },
+];
+
+export function fileExtractorFor(filePath: string): FileExtractor {
+  return (
+    FILE_EXTRACTORS.find((extractor) => extractor.matches(filePath)) ?? FILE_EXTRACTORS.at(-1)!
+  );
+}
+
+export function extractUrlContent(
+  pi: ExtensionAPI,
+  url: string,
+  signal?: AbortSignal,
+): Promise<ExtractedContent> {
+  const extractor =
+    URL_EXTRACTORS.find((candidate) => candidate.matches(url)) ?? URL_EXTRACTORS.at(-1)!;
+  return extractor.extract({ pi, url, signal });
+}
+
+export function pdfExtractionFailureMessage(source: string): string {
+  return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
+}
+
+function textFileExtractor(format: string, extensions: string[]): FileExtractor {
+  return {
+    format,
+    shouldReadText: true,
+    matches: extensions.length ? hasAnyExtension(extensions) : () => true,
+    extract: ({ content }) => content,
+  };
+}
+
+function hasExtension(extension: string): (path: string) => boolean {
+  return (path) => path.toLowerCase().endsWith(extension);
+}
+
+function hasAnyExtension(extensions: string[]): (path: string) => boolean {
+  return (path) => extensions.some((extension) => hasExtension(extension)(path));
+}
+
+async function extractPdf(pi: ExtensionAPI, source: string, signal?: AbortSignal): Promise<string> {
+  const extracted = await extractWithMarkItDown(pi, source, signal);
+  return extracted || pdfExtractionFailureMessage(source);
+}
+
+async function extractPdfUrl(
+  pi: ExtensionAPI,
+  url: string,
+  signal?: AbortSignal,
+): Promise<ExtractedContent> {
+  const extracted = await extractPdf(pi, url, signal);
+  return { extracted, title: titleFromMarkdown(extracted) };
+}
+
+async function extractTextUrl(
+  pi: ExtensionAPI,
+  url: string,
+  signal?: AbortSignal,
+): Promise<ExtractedContent> {
+  const markitdownExtracted = await extractWithMarkItDown(pi, url, signal);
+  if (markitdownExtracted) {
+    return { extracted: markitdownExtracted, title: titleFromMarkdown(markitdownExtracted) };
+  }
+
+  const curlExtracted = await fetchTextUrl(pi, url, signal);
+  if (!curlExtracted) return { extracted: "" };
+  if (looksLikePdf(curlExtracted)) return { extracted: pdfExtractionFailureMessage(url) };
+  return { extracted: curlExtracted, title: titleFromHtml(curlExtracted) };
+}
+
+async function extractWithMarkItDown(
+  pi: ExtensionAPI,
+  source: string,
+  signal?: AbortSignal,
+): Promise<string> {
+  if (!(await hasMarkItDown(pi, signal))) return "";
+
+  try {
+    const mdResult = await exec(
+      pi,
+      "sh",
+      ["-c", `uvx --from 'markitdown[pdf]' markitdown "${source}" 2>/dev/null || echo ""`],
+      { signal, timeout: markitdownTimeoutMs() },
+    );
+    return mdResult.stdout.trim() ? mdResult.stdout : "";
+  } catch {
+    return "";
+  }
+}
+
+async function hasMarkItDown(pi: ExtensionAPI, signal?: AbortSignal): Promise<boolean> {
+  const markitdown = await exec(
+    pi,
+    "sh",
+    ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
+    { signal },
+  );
+  return markitdown.stdout.trim() === "yes";
+}
+
+async function fetchTextUrl(pi: ExtensionAPI, url: string, signal?: AbortSignal): Promise<string> {
+  try {
+    const curlResult = await exec(
+      pi,
+      "curl",
+      ["-sL", "--max-time", String(DEFAULT_CURL_TIMEOUT_SECONDS), url],
+      {
+        signal,
+        timeout: (DEFAULT_CURL_TIMEOUT_SECONDS + 5) * 1_000,
+      },
+    );
+    return curlResult.stdout || "";
+  } catch {
+    return "";
+  }
+}
+
+function markitdownTimeoutMs(): number {
+  return positiveIntegerFromEnv("WIKI_MARKITDOWN_TIMEOUT_MS", DEFAULT_MARKITDOWN_TIMEOUT_MS);
+}
+
+function positiveIntegerFromEnv(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function isPdfUrl(url: string): boolean {
+  try {
+    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return url.toLowerCase().split(/[?#]/, 1)[0].endsWith(".pdf");
+  }
+}
+
+function looksLikePdf(content: string): boolean {
+  return content.trimStart().startsWith("%PDF-");
+}
+
+function titleFromMarkdown(markdown: string): string | undefined {
+  return markdown.match(/^#\s+(.+)$/m)?.[1]?.trim();
+}
+
+function titleFromHtml(html: string): string | undefined {
+  return html.match(/<title>([^<]*)<\/title>/i)?.[1]?.trim();
+}
+
+/** Basic XML to markdown conversion: strip tags while preserving text structure. */
+function xmlToMarkdown(xml: string): string {
+  let title = "";
+  const titleMatch = xml.match(/<title[^>]*>([^<]*)<\/title>/i);
+  if (titleMatch) title = titleMatch[1].trim();
+
+  let text = xml.replace(/<\?xml[^>]*\?>\s*/gi, "");
+  text = text.replace(/<!DOCTYPE[^>]*>\s*/gi, "");
+  text = text.replace(/<\/(p|div|section|article|li|h\d|tr|blockquote|pre)>/gi, "\n");
+  text = text.replace(/<br\s*\/?>/gi, "\n");
+
+  let prev = "";
+  while (prev !== text) {
+    prev = text;
+    text = text.replace(/<[a-zA-Z\/!?][^>]*>/g, "");
+  }
+  text = text.replace(/</g, "");
+
+  text = text.replace(/&(?:amp|lt|gt|quot|#\d+);/gi, (entity) => {
+    const map: Record<string, string> = { "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"' };
+    const lower = entity.toLowerCase();
+    if (map[lower]) return map[lower];
+    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
+    return entity;
+  });
+
+  text = text.replace(/\n{3,}/g, "\n\n").trim();
+  if (!text) return xml;
+
+  const lines = [];
+  if (title) lines.push(`# ${title}\n`);
+  lines.push(text);
+  return lines.join("\n\n");
+}
+
+function jsonToMarkdown(json: string): string {
+  let value: unknown;
+  try {
+    value = JSON.parse(json);
+  } catch {
+    return json;
+  }
+
+  const lines: string[] = [];
+  const title = titleFromValue(value) || "JSON Extract";
+  lines.push(`# ${title}`, "");
+  renderJsonValue(value, lines, 0);
+
+  const markdown = lines
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+  return markdown || json;
+}
+
+function titleFromValue(value: unknown): string | undefined {
+  if (!isRecord(value)) return undefined;
+  for (const key of ["title", "name", "id"]) {
+    const candidate = value[key];
+    if (typeof candidate === "string" && candidate.trim()) return candidate.trim();
+  }
+  return undefined;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function renderJsonValue(value: unknown, lines: string[], depth: number, label?: string): void {
+  if (Array.isArray(value)) {
+    renderJsonArray(value, lines, depth, label);
+    return;
+  }
+
+  if (isRecord(value)) {
+    renderJsonObject(value, lines, depth, label);
+    return;
+  }
+
+  if (label) lines.push(`${indent(depth)}- **${humanizeKey(label)}:** ${formatJsonScalar(value)}`);
+  else lines.push(`${indent(depth)}- ${formatJsonScalar(value)}`);
+}
+
+function renderJsonObject(
+  object: Record<string, unknown>,
+  lines: string[],
+  depth: number,
+  label?: string,
+): void {
+  if (label) {
+    lines.push(`${heading(depth)} ${humanizeKey(label)}`, "");
+  }
+
+  for (const [key, value] of Object.entries(object)) {
+    if (Array.isArray(value) || isRecord(value)) {
+      const childDepth = label ? depth + 1 : depth;
+      renderJsonValue(value, lines, childDepth, key);
+    } else {
+      lines.push(`${indent(depth)}- **${humanizeKey(key)}:** ${formatJsonScalar(value)}`);
+    }
+  }
+  lines.push("");
+}
+
+function renderJsonArray(array: unknown[], lines: string[], depth: number, label?: string): void {
+  if (label) lines.push(`${heading(depth)} ${humanizeKey(label)}`, "");
+
+  if (array.length === 0) {
+    lines.push(`${indent(depth)}- _(empty)_`, "");
+    return;
+  }
+
+  for (const [index, item] of array.entries()) {
+    if (isRecord(item)) {
+      const itemTitle = titleFromValue(item) || `Item ${index + 1}`;
+      const itemDepth = label ? depth + 1 : depth;
+      lines.push(`${heading(itemDepth)} ${itemTitle}`, "");
+      renderJsonObject(item, lines, itemDepth);
+    } else if (Array.isArray(item)) {
+      lines.push(`${indent(depth)}- Item ${index + 1}:`);
+      renderJsonArray(item, lines, depth + 1);
+    } else {
+      lines.push(`${indent(depth)}- ${formatJsonScalar(item)}`);
+    }
+  }
+  lines.push("");
+}
+
+function formatJsonScalar(value: unknown): string {
+  if (value === null) return "null";
+  if (typeof value === "string") return value;
+  if (typeof value === "number" || typeof value === "boolean") return String(value);
+  return String(value);
+}
+
+function humanizeKey(key: string): string {
+  return key
+    .replace(/[_-]+/g, " ")
+    .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+    .replace(/\s+/g, " ")
+    .trim()
+    .replace(/^./, (char) => char.toUpperCase());
+}
+
+function heading(depth: number): string {
+  return "#".repeat(Math.min(depth + 2, 6));
+}
+
+function indent(depth: number): string {
+  return "  ".repeat(Math.max(0, depth));
+}

package/extensions/llm-wiki/lib/source-packet.ts

@@ -1,7 +1,8 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { mkdirSync, writeFileSync } from "node:fs";
 import { extname, join } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { appendEvent } from "./metadata.js";
+import { type ExtractedContent, extractUrlContent, fileExtractorFor } from "./source-extractors.js";
 import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
 
 /**
@@ -22,295 +23,206 @@ export interface CaptureResult {
   extracted: string;
 }
 
-const DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000;
-const DEFAULT_CURL_TIMEOUT_SECONDS = 30;
-
-function markitdownTimeoutMs(): number {
-  return positiveIntegerFromEnv("WIKI_MARKITDOWN_TIMEOUT_MS", DEFAULT_MARKITDOWN_TIMEOUT_MS);
-}
-
-function positiveIntegerFromEnv(name: string, fallback: number): number {
-  const raw = process.env[name];
-  if (!raw) return fallback;
-  const parsed = Number.parseInt(raw, 10);
-  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
-}
-
-function isPdfUrl(url: string): boolean {
-  try {
-    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
-  } catch {
-    return url.toLowerCase().split(/[?#]/, 1)[0].endsWith(".pdf");
-  }
-}
-
-function looksLikePdf(content: string): boolean {
-  return content.trimStart().startsWith("%PDF-");
+interface SourcePacket {
+  sourceId: string;
+  packetPath: string;
 }
 
-function pdfExtractionFailureMessage(source: string): string {
-  return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
+interface CaptureSource {
+  needsOriginalDir: boolean;
+  fallbackText: string;
+  preserveOriginal?(packetPath: string): Promise<void>;
+  extract(): Promise<ExtractedContent> | ExtractedContent;
+  manifest(content: ExtractedContent): Record<string, unknown>;
+  event(content: ExtractedContent): Record<string, unknown>;
 }
 
 const URL_ORIGINAL_EXTENSIONS = new Set([".html", ".htm", ".md", ".pdf", ".txt", ".xml", ".json"]);
 
-function originalFileNameForUrl(url: string): string {
-  try {
-    const parsed = new URL(url);
-    const ext = extname(parsed.pathname).toLowerCase();
-    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
-  } catch {
-    const path = url.split(/[?#]/, 1)[0] ?? "";
-    const ext = extname(path).toLowerCase();
-    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
-  }
-
-  return "source.html";
-}
-
-async function preserveUrlOriginal(
+/** Capture a URL into a source packet. */
+export async function captureUrl(
   pi: ExtensionAPI,
-  packetPath: string,
+  paths: VaultPaths,
   url: string,
   signal?: AbortSignal,
-): Promise<void> {
-  const originalPath = join(packetPath, "original", originalFileNameForUrl(url));
-  try {
-    await exec(pi, "curl", ["-sL", "--max-time", "30", "-o", originalPath, url], {
-      signal,
-      timeout: 35_000,
-    });
-  } catch {
-    // Preserve best-effort extraction behavior even when the original artifact cannot be saved.
-  }
+): Promise<CaptureResult> {
+  return captureSource(paths, urlCaptureSource(pi, url, signal));
 }
 
-/** Capture a URL into a source packet. */
-export async function captureUrl(
+/** Capture a local file into a source packet. */
+export async function captureFile(
   pi: ExtensionAPI,
   paths: VaultPaths,
-  url: string,
+  filePath: string,
   signal?: AbortSignal,
 ): Promise<CaptureResult> {
-  const sourceId = nextSourceId(paths);
-  const packetPath = join(paths.rawSources, sourceId);
-  mkdirSync(packetPath, { recursive: true });
-  mkdirSync(join(packetPath, "original"), { recursive: true });
-  mkdirSync(join(packetPath, "attachments"), { recursive: true });
+  return captureSource(paths, fileCaptureSource(pi, filePath, signal));
+}
 
-  await preserveUrlOriginal(pi, packetPath, url, signal);
-
-  // Try to fetch and extract content
-  let extracted = "";
-  let title = url;
-  const isPdf = isPdfUrl(url);
-
-  // Try MarkItDown first.
-  const markitdown = await exec(
-    pi,
-    "sh",
-    ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
-    { signal },
-  );
-
-  if (markitdown.stdout.trim() === "yes") {
-    try {
-      const mdResult = await exec(
-        pi,
-        "sh",
-        ["-c", `uvx --from 'markitdown[pdf]' markitdown "${url}" 2>/dev/null || echo ""`],
-        { signal, timeout: markitdownTimeoutMs() },
-      );
-      if (mdResult.stdout.trim()) {
-        extracted = mdResult.stdout;
-        // Try to extract title from first h1
-        const h1Match = extracted.match(/^#\s+(.+)$/m);
-        if (h1Match) title = h1Match[1].trim();
-      }
-    } catch {
-      // markitdown failed, fall through
-    }
-  }
+/** Capture pasted text into a source packet. */
+export function captureText(paths: VaultPaths, text: string, title?: string): CaptureResult {
+  return captureSourceSync(paths, textCaptureSource(text, title));
+}
 
-  // Fallback: try fetch_content equivalent via curl for text/html sources.
-  // Do not write binary PDF bytes into extracted.md when PDF conversion fails.
-  if (!extracted) {
-    if (isPdf) {
-      extracted = pdfExtractionFailureMessage(url);
-    } else {
-      try {
-        const curlResult = await exec(
-          pi,
-          "curl",
-          ["-sL", "--max-time", String(DEFAULT_CURL_TIMEOUT_SECONDS), url],
-          {
-            signal,
-            timeout: (DEFAULT_CURL_TIMEOUT_SECONDS + 5) * 1_000,
-          },
-        );
-        if (curlResult.stdout) {
-          if (looksLikePdf(curlResult.stdout)) {
-            extracted = pdfExtractionFailureMessage(url);
-          } else {
-            extracted = curlResult.stdout;
-            // Try to extract title from HTML
-            const titleMatch = extracted.match(/<title>([^<]*)<\/title>/i);
-            if (titleMatch) title = titleMatch[1].trim();
-          }
-        }
-      } catch {
-        // curl failed too
-      }
-    }
-  }
+async function captureSource(paths: VaultPaths, source: CaptureSource): Promise<CaptureResult> {
+  const packet = createSourcePacket(paths, source.needsOriginalDir);
+  await source.preserveOriginal?.(packet.packetPath);
+  const content = await source.extract();
+  return finalizeCapture(paths, packet, source, content);
+}
 
-  // Write extracted text
-  writeFileSync(
-    join(packetPath, "extracted.md"),
-    extracted || `_Content could not be extracted from ${url}_\n`,
-    "utf-8",
-  );
+function captureSourceSync(paths: VaultPaths, source: CaptureSource): CaptureResult {
+  const packet = createSourcePacket(paths, source.needsOriginalDir);
+  const content = source.extract() as ExtractedContent;
+  return finalizeCapture(paths, packet, source, content);
+}
 
-  // Write manifest
-  const manifest = {
-    id: sourceId,
-    title,
-    url,
-    captured: fmtDate(),
-    format: "web",
-    packet_version: "1.0",
+function urlCaptureSource(pi: ExtensionAPI, url: string, signal?: AbortSignal): CaptureSource {
+  return {
+    needsOriginalDir: true,
+    fallbackText: contentExtractionFailureMessage(url),
+    preserveOriginal: (packetPath) => preserveUrlOriginal(pi, packetPath, url, signal),
+    extract: () => extractUrlContent(pi, url, signal),
+    manifest: (content) => ({
+      title: content.title || url,
+      url,
+      format: "web",
+    }),
+    event: () => ({ url, format: "web" }),
   };
-  writeJson(join(packetPath, "manifest.json"), manifest);
-
-  // Create skeleton source page in wiki
-  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
-  const sourcePageContent = buildSourcePageSkeleton(manifest, extracted);
-  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
-
-  // Log event
-  appendEvent(paths, { kind: "capture", source_id: sourceId, url, format: "web" });
-
-  return { sourceId, packetPath, sourcePagePath, extracted };
 }
 
-/** Capture a local file into a source packet. */
-export async function captureFile(
+function fileCaptureSource(
   pi: ExtensionAPI,
-  paths: VaultPaths,
   filePath: string,
   signal?: AbortSignal,
-): Promise<CaptureResult> {
+): CaptureSource {
+  const fileName = filePath.split("/").pop() || "unknown";
+  const extractor = fileExtractorFor(filePath);
+  const content = extractor.shouldReadText ? readText(filePath) : "";
+
+  return {
+    needsOriginalDir: true,
+    fallbackText: "",
+    preserveOriginal: (packetPath) =>
+      preserveFileOriginal(pi, packetPath, filePath, fileName, content, signal),
+    extract: async () => ({
+      extracted: await extractor.extract({ pi, filePath, content, signal }),
+    }),
+    manifest: () => ({
+      title: fileName,
+      file_path: filePath,
+      format: extractor.format,
+    }),
+    event: () => ({ file_path: filePath, format: extractor.format }),
+  };
+}
+
+function textCaptureSource(text: string, title?: string): CaptureSource {
+  return {
+    needsOriginalDir: false,
+    fallbackText: "",
+    extract: () => ({ extracted: text }),
+    manifest: () => ({
+      title: title || `Pasted text — ${fmtDate()}`,
+      format: "text",
+    }),
+    event: () => ({ format: "text" }),
+  };
+}
+
+function createSourcePacket(paths: VaultPaths, needsOriginalDir: boolean): SourcePacket {
   const sourceId = nextSourceId(paths);
   const packetPath = join(paths.rawSources, sourceId);
   mkdirSync(packetPath, { recursive: true });
-  mkdirSync(join(packetPath, "original"), { recursive: true });
   mkdirSync(join(packetPath, "attachments"), { recursive: true });
+  if (needsOriginalDir) mkdirSync(join(packetPath, "original"), { recursive: true });
+  return { sourceId, packetPath };
+}
 
-  const lowerPath = filePath.toLowerCase();
-  const isPdf = lowerPath.endsWith(".pdf");
-  const isXml = lowerPath.endsWith(".xml");
-  const content = isPdf ? "" : readText(filePath);
-  const fileName = filePath.split("/").pop() || "unknown";
-
-  let extracted = content;
-
-  // Convert XML to markdown
-  if (isXml && content) {
-    extracted = xmlToMarkdown(content);
-  }
-
-  // Try MarkItDown for PDFs.
-  if (isPdf) {
-    const markitdown = await exec(
-      pi,
-      "sh",
-      ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
-      { signal },
-    );
-
-    if (markitdown.stdout.trim() === "yes") {
-      try {
-        const mdResult = await exec(
-          pi,
-          "sh",
-          ["-c", `uvx --from 'markitdown[pdf]' markitdown "${filePath}" 2>/dev/null || echo ""`],
-          { signal, timeout: markitdownTimeoutMs() },
-        );
-        if (mdResult.stdout.trim()) extracted = mdResult.stdout;
-      } catch {
-        extracted = pdfExtractionFailureMessage(filePath);
-      }
-    }
-    if (!extracted) extracted = pdfExtractionFailureMessage(filePath);
-  }
-
-  // Copy original to packet
-  try {
-    await exec(pi, "cp", [filePath, join(packetPath, "original", fileName)], { signal });
-  } catch {
-    // If cp fails, just write the content
-    writeFileSync(join(packetPath, "original", fileName), content, "utf-8");
-  }
-
-  // Write extracted text
-  writeFileSync(join(packetPath, "extracted.md"), extracted, "utf-8");
-
-  // Write manifest
+function finalizeCapture(
+  paths: VaultPaths,
+  packet: SourcePacket,
+  source: CaptureSource,
+  content: ExtractedContent,
+): CaptureResult {
+  const extracted = content.extracted || source.fallbackText;
   const manifest = {
-    id: sourceId,
-    title: fileName,
-    file_path: filePath,
+    id: packet.sourceId,
     captured: fmtDate(),
-    format: guessFormat(filePath),
     packet_version: "1.0",
+    ...source.manifest({ ...content, extracted }),
   };
-  writeJson(join(packetPath, "manifest.json"), manifest);
 
-  // Create skeleton source page
-  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
-  const sourcePageContent = buildSourcePageSkeleton(manifest, extracted);
-  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+  writeFileSync(join(packet.packetPath, "extracted.md"), extracted, "utf-8");
+  writeJson(join(packet.packetPath, "manifest.json"), manifest);
+
+  const sourcePagePath = join(paths.wiki, "sources", `${packet.sourceId}.md`);
+  writeFileSync(sourcePagePath, buildSourcePageSkeleton(manifest, extracted), "utf-8");
 
-  // Log event
   appendEvent(paths, {
     kind: "capture",
-    source_id: sourceId,
-    file_path: filePath,
-    format: manifest.format,
+    source_id: packet.sourceId,
+    ...source.event({ ...content, extracted }),
   });
 
-  return { sourceId, packetPath, sourcePagePath, extracted };
+  return {
+    sourceId: packet.sourceId,
+    packetPath: packet.packetPath,
+    sourcePagePath,
+    extracted,
+  };
 }
 
-/** Capture pasted text into a source packet. */
-export function captureText(paths: VaultPaths, text: string, title?: string): CaptureResult {
-  const sourceId = nextSourceId(paths);
-  const packetPath = join(paths.rawSources, sourceId);
-  mkdirSync(packetPath, { recursive: true });
-  mkdirSync(join(packetPath, "attachments"), { recursive: true });
-
-  // Write extracted text
-  writeFileSync(join(packetPath, "extracted.md"), text, "utf-8");
+async function preserveFileOriginal(
+  pi: ExtensionAPI,
+  packetPath: string,
+  filePath: string,
+  fileName: string,
+  fallbackContent: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  try {
+    await exec(pi, "cp", [filePath, join(packetPath, "original", fileName)], { signal });
+  } catch {
+    // If cp fails, preserve whatever text content was available.
+    writeFileSync(join(packetPath, "original", fileName), fallbackContent, "utf-8");
+  }
+}
 
-  // Write manifest
-  const manifest = {
-    id: sourceId,
-    title: title || `Pasted text — ${fmtDate()}`,
-    captured: fmtDate(),
-    format: "text",
-    packet_version: "1.0",
-  };
-  writeJson(join(packetPath, "manifest.json"), manifest);
+async function preserveUrlOriginal(
+  pi: ExtensionAPI,
+  packetPath: string,
+  url: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  const originalPath = join(packetPath, "original", originalFileNameForUrl(url));
+  try {
+    await exec(pi, "curl", ["-sL", "--max-time", "30", "-o", originalPath, url], {
+      signal,
+      timeout: 35_000,
+    });
+  } catch {
+    // Preserve best-effort extraction behavior even when the original artifact cannot be saved.
+  }
+}
 
-  // Create skeleton source page
-  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
-  const sourcePageContent = buildSourcePageSkeleton(manifest, text);
-  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+function originalFileNameForUrl(url: string): string {
+  try {
+    const parsed = new URL(url);
+    const ext = extname(parsed.pathname).toLowerCase();
+    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
+  } catch {
+    const path = url.split(/[?#]/, 1)[0] ?? "";
+    const ext = extname(path).toLowerCase();
+    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
+  }
 
-  // Log event
-  appendEvent(paths, { kind: "capture", source_id: sourceId, format: "text" });
+  return "source.html";
+}
 
-  return { sourceId, packetPath, sourcePagePath, extracted: text };
+function contentExtractionFailureMessage(source: string): string {
+  return `_Content could not be extracted from ${source}_\n`;
 }
 
 /** Build a skeleton source page from manifest and extracted text. */
@@ -370,59 +282,3 @@ status: skeleton
 - **Manifest:** [raw/sources/${id}/manifest.json](../raw/sources/${id}/manifest.json)
 `;
 }
-
-/** Basic XML to markdown conversion: strip tags while preserving text structure. */
-function xmlToMarkdown(xml: string): string {
-  // Extract title from first <title> or root element
-  let title = "";
-  const titleMatch = xml.match(/<title[^>]*>([^<]*)<\/title>/i);
-  if (titleMatch) title = titleMatch[1].trim();
-
-  // Strip XML declaration and doctype
-  let text = xml.replace(/<\?xml[^>]*\?>\s*/gi, "");
-  text = text.replace(/<!DOCTYPE[^>]*>\s*/gi, "");
-
-  // Replace block-level tags with newlines
-  text = text.replace(/<\/(p|div|section|article|li|h\d|tr|blockquote|pre)>/gi, "\n");
-  text = text.replace(/<br\s*\/?>/gi, "\n");
-
-  // Strip remaining tags — match < followed by tag name characters to >
-  // Using a loop to handle malformed/broken tags that lack a closing >
-  let prev = "";
-  while (prev !== text) {
-    prev = text;
-    text = text.replace(/<[a-zA-Z\/!?][^>]*>/g, "");
-  }
-  // Remove any stray < that didn't form a complete tag
-  text = text.replace(/</g, "");
-
-  // Decode XML entities in a single pass to avoid double-unescaping
-  text = text.replace(/&(?:amp|lt|gt|quot|#\d+);/gi, (entity) => {
-    const map: Record<string, string> = { "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"' };
-    const lower = entity.toLowerCase();
-    if (map[lower]) return map[lower];
-    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
-    return entity;
-  });
-
-  // Clean up excessive blank lines
-  text = text.replace(/\n{3,}/g, "\n\n").trim();
-
-  if (!text) return xml; // fallback: return raw if stripping produced nothing
-
-  const lines = [];
-  if (title) lines.push(`# ${title}\n`);
-  lines.push(text);
-  return lines.join("\n\n");
-}
-
-function guessFormat(filePath: string): string {
-  const lower = filePath.toLowerCase();
-  if (lower.endsWith(".pdf")) return "pdf";
-  if (lower.endsWith(".md")) return "markdown";
-  if (lower.endsWith(".txt")) return "text";
-  if (lower.endsWith(".html") || lower.endsWith(".htm")) return "html";
-  if (lower.endsWith(".xml")) return "xml";
-  if (lower.endsWith(".docx")) return "docx";
-  return "file";
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "LLM Wiki for Pi — self-maintaining knowledge base following Karpathy's pattern. Obsidian-friendly, auto-updating, personal & company wiki.",
   "keywords": [
     "pi-package",

package/test/llm-wiki.test.ts

@@ -183,13 +183,27 @@ describe("package structure", () => {
 
   it("should keep MarkItDown timeout configurable and avoid PDF byte fallbacks", () => {
     const sourcePacketPath = join(rootDir, "extensions", "llm-wiki", "lib", "source-packet.ts");
+    const sourceExtractorsPath = join(
+      rootDir,
+      "extensions",
+      "llm-wiki",
+      "lib",
+      "source-extractors.ts",
+    );
     expect(existsSync(sourcePacketPath)).toBe(true);
-    const content = readFile(sourcePacketPath);
-    expect(content).toContain("WIKI_MARKITDOWN_TIMEOUT_MS");
-    expect(content).toContain("DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000");
-    expect(content).toContain("isPdfUrl(url)");
-    expect(content).toContain("looksLikePdf(curlResult.stdout)");
-    expect(content).toContain("pdfExtractionFailureMessage");
+    expect(existsSync(sourceExtractorsPath)).toBe(true);
+
+    const sourcePacket = readFile(sourcePacketPath);
+    const sourceExtractors = readFile(sourceExtractorsPath);
+    expect(sourcePacket).toContain("captureSource");
+    expect(sourcePacket).toContain("fileExtractorFor");
+    expect(sourcePacket).toContain("extractUrlContent");
+    expect(sourceExtractors).toContain("WIKI_MARKITDOWN_TIMEOUT_MS");
+    expect(sourceExtractors).toContain("DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000");
+    expect(sourceExtractors).toContain("URL_EXTRACTORS");
+    expect(sourceExtractors).toContain("matches: isPdfUrl");
+    expect(sourceExtractors).toContain("looksLikePdf(curlExtracted)");
+    expect(sourceExtractors).toContain("pdfExtractionFailureMessage");
   });
 
   it("should have a comprehensive README with install instructions", () => {
@@ -438,6 +452,62 @@ describe("source packet capture", () => {
     // Should have the text content at minimum
     expect(extracted).toContain("Hello");
   });
+
+  it("should convert JSON files to readable markdown in extracted.md", async () => {
+    const paths = makePaths();
+    const jsonContent = JSON.stringify(
+      {
+        title: "Project Roadmap",
+        scope: "Improve the client portal and project record.",
+        assumptions: ["Routes already exist", "Use generated API types"],
+        tasks: [
+          {
+            id: "client-portal",
+            title: "Client portal hardening",
+            acceptance: ["Shows open actions", "Build passes"],
+          },
+        ],
+      },
+      null,
+      2,
+    );
+    const jsonPath = join(tempDir, "roadmap.json");
+    writeFileSync(jsonPath, jsonContent, "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, jsonPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).toContain("# Project Roadmap");
+    expect(extracted).toContain("**Scope:** Improve the client portal and project record.");
+    expect(extracted).toMatch(/^## Assumptions$/m);
+    expect(extracted).not.toMatch(/^### Assumptions$/m);
+    expect(extracted).toContain("- Routes already exist");
+    expect(extracted).toMatch(/^## Tasks$/m);
+    expect(extracted).not.toMatch(/^### Tasks$/m);
+    expect(extracted).toMatch(/^### Client portal hardening$/m);
+    expect(extracted).toContain("Shows open actions");
+    expect(extracted).not.toContain('"tasks"');
+    expect(extracted).not.toContain("{");
+
+    expect(existsSync(join(result.packetPath, "original", "roadmap.json"))).toBe(true);
+
+    const manifest = JSON.parse(readFile(join(result.packetPath, "manifest.json")));
+    expect(manifest.format).toBe("json");
+  });
+
+  it("should fall back to raw JSON content when parsing fails", async () => {
+    const paths = makePaths();
+    const jsonContent = `{ "title": "Broken", `;
+    const jsonPath = join(tempDir, "broken.json");
+    writeFileSync(jsonPath, jsonContent, "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, jsonPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).toBe(jsonContent);
+  });
 });
 
 describe("wiki directory structure", () => {