@zosmaai/pi-llm-wiki 0.1.6 → 0.2.1

package/extensions/llm-wiki/lib/source-packet.ts

@@ -0,0 +1,339 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { appendEvent } from "./metadata.js";
+import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
+
+/**
+ * Source packet capture and management.
+ *
+ * Each source is stored as a structured packet:
+ *   raw/sources/SRC-YYYY-MM-DD-NNN/
+ *     manifest.json    — capture metadata
+ *     original/        — original artifact (if file/URL)
+ *     extracted.md     — normalized markdown text
+ *     attachments/     — downloaded images, PDFs, etc.
+ */
+
+export interface CaptureResult {
+  sourceId: string;
+  packetPath: string;
+  sourcePagePath: string;
+  extracted: string;
+}
+
+const DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000;
+const DEFAULT_CURL_TIMEOUT_SECONDS = 30;
+
+function markitdownTimeoutMs(): number {
+  return positiveIntegerFromEnv("WIKI_MARKITDOWN_TIMEOUT_MS", DEFAULT_MARKITDOWN_TIMEOUT_MS);
+}
+
+function positiveIntegerFromEnv(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function isPdfUrl(url: string): boolean {
+  try {
+    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return url.toLowerCase().split(/[?#]/, 1)[0].endsWith(".pdf");
+  }
+}
+
+function looksLikePdf(content: string): boolean {
+  return content.trimStart().startsWith("%PDF-");
+}
+
+function pdfExtractionFailureMessage(source: string): string {
+  return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
+}
+
+/** Capture a URL into a source packet. */
+export async function captureUrl(
+  pi: ExtensionAPI,
+  paths: VaultPaths,
+  url: string,
+  signal?: AbortSignal,
+): Promise<CaptureResult> {
+  const sourceId = nextSourceId(paths);
+  const packetPath = join(paths.rawSources, sourceId);
+  mkdirSync(packetPath, { recursive: true });
+  mkdirSync(join(packetPath, "original"), { recursive: true });
+  mkdirSync(join(packetPath, "attachments"), { recursive: true });
+
+  // Try to fetch and extract content
+  let extracted = "";
+  let title = url;
+  const isPdf = isPdfUrl(url);
+
+  // Try MarkItDown first.
+  const markitdown = await exec(
+    pi,
+    "sh",
+    ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
+    { signal },
+  );
+
+  if (markitdown.stdout.trim() === "yes") {
+    try {
+      const mdResult = await exec(
+        pi,
+        "sh",
+        ["-c", `uvx --from 'markitdown[pdf]' markitdown "${url}" 2>/dev/null || echo ""`],
+        { signal, timeout: markitdownTimeoutMs() },
+      );
+      if (mdResult.stdout.trim()) {
+        extracted = mdResult.stdout;
+        // Try to extract title from first h1
+        const h1Match = extracted.match(/^#\s+(.+)$/m);
+        if (h1Match) title = h1Match[1].trim();
+      }
+    } catch {
+      // markitdown failed, fall through
+    }
+  }
+
+  // Fallback: try fetch_content equivalent via curl for text/html sources.
+  // Do not write binary PDF bytes into extracted.md when PDF conversion fails.
+  if (!extracted) {
+    if (isPdf) {
+      extracted = pdfExtractionFailureMessage(url);
+    } else {
+      try {
+        const curlResult = await exec(
+          pi,
+          "curl",
+          ["-sL", "--max-time", String(DEFAULT_CURL_TIMEOUT_SECONDS), url],
+          {
+            signal,
+            timeout: (DEFAULT_CURL_TIMEOUT_SECONDS + 5) * 1_000,
+          },
+        );
+        if (curlResult.stdout) {
+          if (looksLikePdf(curlResult.stdout)) {
+            extracted = pdfExtractionFailureMessage(url);
+          } else {
+            extracted = curlResult.stdout;
+            // Try to extract title from HTML
+            const titleMatch = extracted.match(/<title>([^<]*)<\/title>/i);
+            if (titleMatch) title = titleMatch[1].trim();
+          }
+        }
+      } catch {
+        // curl failed too
+      }
+    }
+  }
+
+  // Write extracted text
+  writeFileSync(
+    join(packetPath, "extracted.md"),
+    extracted || `_Content could not be extracted from ${url}_\n`,
+    "utf-8",
+  );
+
+  // Write manifest
+  const manifest = {
+    id: sourceId,
+    title,
+    url,
+    captured: fmtDate(),
+    format: "web",
+    packet_version: "1.0",
+  };
+  writeJson(join(packetPath, "manifest.json"), manifest);
+
+  // Create skeleton source page in wiki
+  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
+  const sourcePageContent = buildSourcePageSkeleton(manifest, extracted);
+  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+
+  // Log event
+  appendEvent(paths, { kind: "capture", source_id: sourceId, url, format: "web" });
+
+  return { sourceId, packetPath, sourcePagePath, extracted };
+}
+
+/** Capture a local file into a source packet. */
+export async function captureFile(
+  pi: ExtensionAPI,
+  paths: VaultPaths,
+  filePath: string,
+  signal?: AbortSignal,
+): Promise<CaptureResult> {
+  const sourceId = nextSourceId(paths);
+  const packetPath = join(paths.rawSources, sourceId);
+  mkdirSync(packetPath, { recursive: true });
+  mkdirSync(join(packetPath, "original"), { recursive: true });
+  mkdirSync(join(packetPath, "attachments"), { recursive: true });
+
+  const isPdf = filePath.toLowerCase().endsWith(".pdf");
+  const content = isPdf ? "" : readText(filePath);
+  const fileName = filePath.split("/").pop() || "unknown";
+
+  // Try MarkItDown for PDFs.
+  let extracted = content;
+  if (isPdf) {
+    const markitdown = await exec(
+      pi,
+      "sh",
+      ["-c", `which uvx >/dev/null 2>&1 && echo "yes" || echo "no"`],
+      { signal },
+    );
+
+    if (markitdown.stdout.trim() === "yes") {
+      try {
+        const mdResult = await exec(
+          pi,
+          "sh",
+          ["-c", `uvx --from 'markitdown[pdf]' markitdown "${filePath}" 2>/dev/null || echo ""`],
+          { signal, timeout: markitdownTimeoutMs() },
+        );
+        if (mdResult.stdout.trim()) extracted = mdResult.stdout;
+      } catch {
+        extracted = pdfExtractionFailureMessage(filePath);
+      }
+    }
+    if (!extracted) extracted = pdfExtractionFailureMessage(filePath);
+  }
+
+  // Copy original to packet
+  try {
+    await exec(pi, "cp", [filePath, join(packetPath, "original", fileName)], { signal });
+  } catch {
+    // If cp fails, just write the content
+    writeFileSync(join(packetPath, "original", fileName), content, "utf-8");
+  }
+
+  // Write extracted text
+  writeFileSync(join(packetPath, "extracted.md"), extracted, "utf-8");
+
+  // Write manifest
+  const manifest = {
+    id: sourceId,
+    title: fileName,
+    file_path: filePath,
+    captured: fmtDate(),
+    format: guessFormat(filePath),
+    packet_version: "1.0",
+  };
+  writeJson(join(packetPath, "manifest.json"), manifest);
+
+  // Create skeleton source page
+  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
+  const sourcePageContent = buildSourcePageSkeleton(manifest, extracted);
+  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+
+  // Log event
+  appendEvent(paths, {
+    kind: "capture",
+    source_id: sourceId,
+    file_path: filePath,
+    format: manifest.format,
+  });
+
+  return { sourceId, packetPath, sourcePagePath, extracted };
+}
+
+/** Capture pasted text into a source packet. */
+export function captureText(paths: VaultPaths, text: string, title?: string): CaptureResult {
+  const sourceId = nextSourceId(paths);
+  const packetPath = join(paths.rawSources, sourceId);
+  mkdirSync(packetPath, { recursive: true });
+  mkdirSync(join(packetPath, "attachments"), { recursive: true });
+
+  // Write extracted text
+  writeFileSync(join(packetPath, "extracted.md"), text, "utf-8");
+
+  // Write manifest
+  const manifest = {
+    id: sourceId,
+    title: title || `Pasted text — ${fmtDate()}`,
+    captured: fmtDate(),
+    format: "text",
+    packet_version: "1.0",
+  };
+  writeJson(join(packetPath, "manifest.json"), manifest);
+
+  // Create skeleton source page
+  const sourcePagePath = join(paths.wiki, "sources", `${sourceId}.md`);
+  const sourcePageContent = buildSourcePageSkeleton(manifest, text);
+  writeFileSync(sourcePagePath, sourcePageContent, "utf-8");
+
+  // Log event
+  appendEvent(paths, { kind: "capture", source_id: sourceId, format: "text" });
+
+  return { sourceId, packetPath, sourcePagePath, extracted: text };
+}
+
+/** Build a skeleton source page from manifest and extracted text. */
+function buildSourcePageSkeleton(manifest: Record<string, unknown>, extracted: string): string {
+  const id = String(manifest.id);
+  const title = String(manifest.title || id);
+  const url = manifest.url ? `\n> _Original: ${manifest.url}_` : "";
+  const format = String(manifest.format || "unknown");
+  const captured = String(manifest.captured || fmtDate());
+
+  // Generate a brief auto-summary (first 500 chars)
+  const preview = extracted
+    .replace(/[#*_`]/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+    .slice(0, 500);
+
+  return `---
+type: source
+format: ${format}
+source_id: ${id}
+raw_path: raw/sources/${id}/extracted.md
+captured: ${captured}
+status: skeleton
+---
+
+# ${title}${url}
+
+## Summary
+
+[LLM: Replace with 2-3 paragraph summary of key content]
+
+> _Auto-preview: ${preview}${extracted.length > 500 ? "..." : ""}_
+
+## Key Takeaways
+
+- [LLM: Most important point]
+- [LLM: Second important point]
+- [LLM: Third important point]
+
+## Entities Mentioned
+
+- [[entity-name]]
+
+## Concepts Mentioned
+
+- [[concept-name]]
+
+## Notable Quotes
+
+> [LLM: Important quote] — attribution
+
+## Source Packet
+
+- **ID:** \`[[sources/${id}]]\`
+- **Extracted:** [raw/sources/${id}/extracted.md](../raw/sources/${id}/extracted.md)
+- **Manifest:** [raw/sources/${id}/manifest.json](../raw/sources/${id}/manifest.json)
+`;
+}
+
+function guessFormat(filePath: string): string {
+  const lower = filePath.toLowerCase();
+  if (lower.endsWith(".pdf")) return "pdf";
+  if (lower.endsWith(".md")) return "markdown";
+  if (lower.endsWith(".txt")) return "text";
+  if (lower.endsWith(".html") || lower.endsWith(".htm")) return "html";
+  if (lower.endsWith(".docx")) return "docx";
+  return "file";
+}