@zosmaai/pi-llm-wiki 0.2.0 → 0.2.1

package/extensions/llm-wiki/lib/source-packet.ts

@@ -22,6 +22,36 @@ export interface CaptureResult {
   extracted: string;
 }
 
+const DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000;
+const DEFAULT_CURL_TIMEOUT_SECONDS = 30;
+
+function markitdownTimeoutMs(): number {
+  return positiveIntegerFromEnv("WIKI_MARKITDOWN_TIMEOUT_MS", DEFAULT_MARKITDOWN_TIMEOUT_MS);
+}
+
+function positiveIntegerFromEnv(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+function isPdfUrl(url: string): boolean {
+  try {
+    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return url.toLowerCase().split(/[?#]/, 1)[0].endsWith(".pdf");
+  }
+}
+
+function looksLikePdf(content: string): boolean {
+  return content.trimStart().startsWith("%PDF-");
+}
+
+function pdfExtractionFailureMessage(source: string): string {
+  return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
+}
+
 /** Capture a URL into a source packet. */
 export async function captureUrl(
   pi: ExtensionAPI,
@@ -38,8 +68,9 @@ export async function captureUrl(
   // Try to fetch and extract content
   let extracted = "";
   let title = url;
+  const isPdf = isPdfUrl(url);
 
-  // Try markitdown first
+  // Try MarkItDown first.
   const markitdown = await exec(
     pi,
     "sh",
@@ -53,7 +84,7 @@ export async function captureUrl(
         pi,
         "sh",
         ["-c", `uvx --from 'markitdown[pdf]' markitdown "${url}" 2>/dev/null || echo ""`],
-        { signal, timeout: 30_000 },
+        { signal, timeout: markitdownTimeoutMs() },
       );
       if (mdResult.stdout.trim()) {
         extracted = mdResult.stdout;
@@ -66,21 +97,35 @@ export async function captureUrl(
     }
   }
 
-  // Fallback: try fetch_content equivalent via curl
+  // Fallback: try fetch_content equivalent via curl for text/html sources.
+  // Do not write binary PDF bytes into extracted.md when PDF conversion fails.
   if (!extracted) {
-    try {
-      const curlResult = await exec(pi, "curl", ["-sL", "--max-time", "30", url], {
-        signal,
-        timeout: 35_000,
-      });
-      if (curlResult.stdout) {
-        extracted = curlResult.stdout;
-        // Try to extract title from HTML
-        const titleMatch = extracted.match(/<title>([^<]*)<\/title>/i);
-        if (titleMatch) title = titleMatch[1].trim();
+    if (isPdf) {
+      extracted = pdfExtractionFailureMessage(url);
+    } else {
+      try {
+        const curlResult = await exec(
+          pi,
+          "curl",
+          ["-sL", "--max-time", String(DEFAULT_CURL_TIMEOUT_SECONDS), url],
+          {
+            signal,
+            timeout: (DEFAULT_CURL_TIMEOUT_SECONDS + 5) * 1_000,
+          },
+        );
+        if (curlResult.stdout) {
+          if (looksLikePdf(curlResult.stdout)) {
+            extracted = pdfExtractionFailureMessage(url);
+          } else {
+            extracted = curlResult.stdout;
+            // Try to extract title from HTML
+            const titleMatch = extracted.match(/<title>([^<]*)<\/title>/i);
+            if (titleMatch) title = titleMatch[1].trim();
+          }
+        }
+      } catch {
+        // curl failed too
       }
-    } catch {
-      // curl failed too
     }
   }
 
@@ -126,12 +171,13 @@ export async function captureFile(
   mkdirSync(join(packetPath, "original"), { recursive: true });
   mkdirSync(join(packetPath, "attachments"), { recursive: true });
 
-  const content = readText(filePath);
+  const isPdf = filePath.toLowerCase().endsWith(".pdf");
+  const content = isPdf ? "" : readText(filePath);
   const fileName = filePath.split("/").pop() || "unknown";
 
-  // Try markitdown for PDFs
+  // Try MarkItDown for PDFs.
   let extracted = content;
-  if (filePath.toLowerCase().endsWith(".pdf")) {
+  if (isPdf) {
     const markitdown = await exec(
       pi,
       "sh",
@@ -145,13 +191,14 @@ export async function captureFile(
           pi,
           "sh",
           ["-c", `uvx --from 'markitdown[pdf]' markitdown "${filePath}" 2>/dev/null || echo ""`],
-          { signal, timeout: 30_000 },
+          { signal, timeout: markitdownTimeoutMs() },
         );
         if (mdResult.stdout.trim()) extracted = mdResult.stdout;
       } catch {
-        // fallback to original
+        extracted = pdfExtractionFailureMessage(filePath);
       }
     }
+    if (!extracted) extracted = pdfExtractionFailureMessage(filePath);
   }
 
   // Copy original to packet

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "LLM Wiki for Pi — self-maintaining knowledge base following Karpathy's pattern. Obsidian-friendly, auto-updating, personal & company wiki.",
   "keywords": [
     "pi-package",

package/prompts/wiki-digest.md

@@ -1,6 +1,6 @@
 ---
 description: Generate a daily or weekly digest of wiki changes — new sources, pages, insights, and gaps.
-args: [--period daily|weekly]
+argument-hint: "[--period daily|weekly]"
 section: LLM Wiki
 topLevelCli: true
 ---
@@ -9,6 +9,10 @@ topLevelCli: true
 
 Generate a digest of recent wiki activity.
 
+## User Arguments
+
+$ARGUMENTS
+
 ## Steps
 
 1. Read `wiki/LOG.md` — filter entries since last digest

package/prompts/wiki-discover.md

@@ -1,6 +1,6 @@
 ---
 description: Auto-discover new sources from the web. Searches based on config topics and known knowledge gaps.
-args: [--topic <topic>]
+argument-hint: "[--topic <topic>]"
 section: LLM Wiki
 topLevelCli: true
 ---
@@ -9,6 +9,10 @@ topLevelCli: true
 
 Find new source material for the wiki by searching the web.
 
+## User Arguments
+
+$ARGUMENTS
+
 Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first. Also read `config.yaml` for topics and feeds.
 
 ## Steps

package/prompts/wiki-ingest.md

@@ -1,6 +1,6 @@
 ---
 description: Process new source files in raw/ and update the wiki. Creates summaries, entities, concepts, and cross-references.
-args: [path]
+argument-hint: "[path]"
 section: LLM Wiki
 topLevelCli: true
 ---
@@ -9,6 +9,10 @@ topLevelCli: true
 
 Process new files in `raw/` and integrate them into the wiki.
 
+## User Arguments
+
+$ARGUMENTS
+
 Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first to understand the full schema, page formats, and conventions.
 
 ## Steps

package/prompts/wiki-init.md

@@ -1,6 +1,6 @@
 ---
 description: Initialize a new LLM Wiki in the current directory. Creates the full directory structure, config, and template files.
-args: <topic> [--mode personal|company]
+argument-hint: "<topic> [--mode personal|company]"
 section: LLM Wiki
 topLevelCli: true
 ---
@@ -9,6 +9,10 @@ topLevelCli: true
 
 Initialize a new LLM Wiki in the current directory.
 
+## User Arguments
+
+$ARGUMENTS
+
 Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` (or wherever the skill is installed) first to understand the full schema and conventions.
 
 ## Steps

package/prompts/wiki-lint.md

@@ -1,6 +1,6 @@
 ---
 description: Health check the wiki. Detects contradictions, orphans, missing pages, stale claims, and knowledge gaps.
-args: [--fix]
+argument-hint: "[--fix]"
 section: LLM Wiki
 topLevelCli: true
 ---
@@ -9,6 +9,10 @@ topLevelCli: true
 
 Run a comprehensive health check on the wiki.
 
+## User Arguments
+
+$ARGUMENTS
+
 Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first to understand the full schema and conventions.
 
 ## Steps

package/prompts/wiki-query.md

@@ -1,6 +1,6 @@
 ---
 description: Ask questions against the wiki. Synthesizes answers from wiki pages with cross-reference citations.
-args: <question>
+argument-hint: "<question>"
 section: LLM Wiki
 topLevelCli: true
 ---
@@ -9,6 +9,10 @@ topLevelCli: true
 
 Ask a question and get an answer synthesized from wiki content.
 
+## User Question
+
+$ARGUMENTS
+
 Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first to understand the full schema and conventions.
 
 ## Steps

package/prompts/wiki-run.md

@@ -1,6 +1,6 @@
 ---
 description: Run the full wiki cycle: discover → ingest → lint. Optionally schedule for auto-updates.
-args: [--schedule daily|weekly]
+argument-hint: "[--schedule daily|weekly]"
 section: LLM Wiki
 topLevelCli: true
 ---
@@ -9,6 +9,10 @@ topLevelCli: true
 
 Run the complete wiki maintenance cycle: discover new sources, ingest them, and lint for health.
 
+## User Arguments
+
+$ARGUMENTS
+
 Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first.
 
 ## Steps

package/prompts/wiki-status.md

@@ -1,6 +1,6 @@
 ---
 description: Show wiki health overview — source count, page stats, orphan count, last activity dates.
-args: []
+argument-hint: ""
 section: LLM Wiki
 topLevelCli: true
 ---

package/test/llm-wiki.test.ts

@@ -111,11 +111,33 @@ describe("package structure", () => {
       expect(existsSync(path)).toBe(true);
       const content = readFile(path);
       expect(content).toContain("description:");
+      expect(content).toContain("argument-hint:");
       expect(content).toContain("section: LLM Wiki");
       expect(content).toContain("topLevelCli: true");
+      expect(content).not.toContain("\nargs:");
     }
   });
 
+  it("should include prompt arguments in templates that accept them", () => {
+    const promptsWithArgs = [
+      "wiki-init.md",
+      "wiki-ingest.md",
+      "wiki-query.md",
+      "wiki-lint.md",
+      "wiki-discover.md",
+      "wiki-run.md",
+      "wiki-digest.md",
+    ];
+    for (const prompt of promptsWithArgs) {
+      const content = readFile(join(rootDir, "prompts", prompt));
+      expect(content).toContain("$ARGUMENTS");
+    }
+
+    const query = readFile(join(rootDir, "prompts", "wiki-query.md"));
+    expect(query).toContain("## User Question");
+    expect(query).toContain("$ARGUMENTS");
+  });
+
   it("should have all wiki template files", () => {
     const t = join(rootDir, "skills", "llm-wiki", "templates");
     expect(existsSync(join(t, "INDEX.md"))).toBe(true);
@@ -157,6 +179,17 @@ describe("package structure", () => {
     }
   });
 
+  it("should keep MarkItDown timeout configurable and avoid PDF byte fallbacks", () => {
+    const sourcePacketPath = join(rootDir, "extensions", "llm-wiki", "lib", "source-packet.ts");
+    expect(existsSync(sourcePacketPath)).toBe(true);
+    const content = readFile(sourcePacketPath);
+    expect(content).toContain("WIKI_MARKITDOWN_TIMEOUT_MS");
+    expect(content).toContain("DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000");
+    expect(content).toContain("isPdfUrl(url)");
+    expect(content).toContain("looksLikePdf(curlResult.stdout)");
+    expect(content).toContain("pdfExtractionFailureMessage");
+  });
+
   it("should have a comprehensive README with install instructions", () => {
     const readme = readFile(join(rootDir, "README.md"));
     expect(readme).toContain("@zosmaai/pi-llm-wiki");