@zosmaai/pi-llm-wiki 0.2.1 → 0.2.2

package/.github/codeql/codeql-config.yml

@@ -0,0 +1,12 @@
+name: "CodeQL Config"
+
+queries:
+  - uses: security-and-quality
+
+# Suppress false positives for temporary file creation.
+# This is a single-user CLI tool — TOCTOU race conditions on temp directories
+# are not a valid threat model. The alerts only fire because tests pass
+# os.tmpdir() paths into production capture functions.
+query-filters:
+  - exclude:
+      id: js/insecure-temporary-file

package/.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [20, 22, 23]
+        node-version: [20, 22, 23, 24, 25]
 
     steps:
       - uses: actions/checkout@v4

package/.github/workflows/codeql.yml

@@ -32,7 +32,7 @@ jobs:
         with:
           languages: ${{ matrix.language }}
           build-mode: ${{ matrix.build-mode }}
-          queries: security-and-quality
+          config-file: ./.github/codeql/codeql-config.yml
 
       - uses: github/codeql-action/analyze@v3
         with:

package/.github/workflows/release.yml

@@ -43,6 +43,14 @@ jobs:
           node -e "const p=require('./package.json'); p.version='${{ steps.version.outputs.VERSION }}'; require('fs').writeFileSync('./package.json', JSON.stringify(p, null, 2)+'\n')"
           npx biome check --fix package.json 2>&1
 
+      - name: Commit version bump back to main
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add package.json
+          git commit -m "chore(release): v${{ steps.version.outputs.VERSION }}"
+          git push origin HEAD:main
+
       - name: Generate Release Notes
         id: notes
         run: |

package/README.md

@@ -42,6 +42,14 @@ Drop sources into `raw/`, then:
 | `wiki_log_event`      | Record custom event                           |
 | `wiki_watch`          | Schedule auto-updates                         |
 
+## Features
+
+- **Configurable PDF extraction** — MarkItDown timeout adjustable via `WIKI_MARKITDOWN_TIMEOUT_MS` env var
+- **Smart content detection** — PDF bytes sniffed even from non-`.pdf` URLs, never written as markdown
+- **Original artifacts preserved** — URL captures save the fetched payload under `original/source.*`
+- **Clickable source links** — Captured URLs render as clickable Markdown links in source pages
+- **Reliable prompt forwarding** — Slash commands properly forward user arguments to the model
+
 ## Architecture
 
 Four layers with clear ownership:
@@ -67,6 +75,22 @@ Read [docs/architecture.md](docs/architecture.md) for details.
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).
 
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=zosmaai/pi-llm-wiki&type=Date)](https://star-history.com/#zosmaai/pi-llm-wiki&Date)
+
+## Contributors
+
+<a href="https://github.com/zosmaai/pi-llm-wiki/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=zosmaai/pi-llm-wiki" alt="Contributors" />
+</a>
+
+---
+
+<div align="center">
+  <sub>Built with ❤️ by <a href="https://github.com/zosmaai">zosmaai</a></sub>
+</div>
+
 ## License
 
 MIT

package/docs/configuration.md

@@ -23,6 +23,12 @@ Wiki configuration lives in `.wiki/config.json`.
 | `auto_fix_lint`            | false   | Auto-fix lint issues               |
 | `batch_ingest_size`        | 3       | Sources processed per ingest batch |
 
+## Environment Variables
+
+| Variable                      | Default | Description                                     |
+| ----------------------------- | ------- | ----------------------------------------------- |
+| `WIKI_MARKITDOWN_TIMEOUT_MS` | 180000  | Timeout (ms) for MarkItDown PDF/text extraction |
+
 ## Page Frontmatter
 
 ```yaml

package/extensions/llm-wiki/lib/source-packet.ts

@@ -1,5 +1,5 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
+import { extname, join } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { appendEvent } from "./metadata.js";
 import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
@@ -52,6 +52,39 @@ function pdfExtractionFailureMessage(source: string): string {
   return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
 }
 
+const URL_ORIGINAL_EXTENSIONS = new Set([".html", ".htm", ".md", ".pdf", ".txt", ".xml", ".json"]);
+
+function originalFileNameForUrl(url: string): string {
+  try {
+    const parsed = new URL(url);
+    const ext = extname(parsed.pathname).toLowerCase();
+    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
+  } catch {
+    const path = url.split(/[?#]/, 1)[0] ?? "";
+    const ext = extname(path).toLowerCase();
+    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
+  }
+
+  return "source.html";
+}
+
+async function preserveUrlOriginal(
+  pi: ExtensionAPI,
+  packetPath: string,
+  url: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  const originalPath = join(packetPath, "original", originalFileNameForUrl(url));
+  try {
+    await exec(pi, "curl", ["-sL", "--max-time", "30", "-o", originalPath, url], {
+      signal,
+      timeout: 35_000,
+    });
+  } catch {
+    // Preserve best-effort extraction behavior even when the original artifact cannot be saved.
+  }
+}
+
 /** Capture a URL into a source packet. */
 export async function captureUrl(
   pi: ExtensionAPI,
@@ -65,6 +98,8 @@ export async function captureUrl(
   mkdirSync(join(packetPath, "original"), { recursive: true });
   mkdirSync(join(packetPath, "attachments"), { recursive: true });
 
+  await preserveUrlOriginal(pi, packetPath, url, signal);
+
   // Try to fetch and extract content
   let extracted = "";
   let title = url;
@@ -274,7 +309,7 @@ export function captureText(paths: VaultPaths, text: string, title?: string): Ca
 function buildSourcePageSkeleton(manifest: Record<string, unknown>, extracted: string): string {
   const id = String(manifest.id);
   const title = String(manifest.title || id);
-  const url = manifest.url ? `\n> _Original: ${manifest.url}_` : "";
+  const url = manifest.url ? `\n> _Original: [${manifest.url}](${manifest.url})_` : "";
   const format = String(manifest.format || "unknown");
   const captured = String(manifest.captured || fmtDate());
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "LLM Wiki for Pi — self-maintaining knowledge base following Karpathy's pattern. Obsidian-friendly, auto-updating, personal & company wiki.",
   "keywords": [
     "pi-package",

package/test/llm-wiki.test.ts

@@ -3,6 +3,8 @@ import { tmpdir } from "node:os";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { captureFile, captureText, captureUrl } from "../extensions/llm-wiki/lib/source-packet.js";
+import { ensureVaultStructure, getVaultPaths } from "../extensions/llm-wiki/lib/utils.js";
 
 // ─── Helpers ────────────────────────────────────────────
 
@@ -232,6 +234,162 @@ describe("skill frontmatter validation", () => {
 
 // ─── Wiki Directory Structure Tests ─────────────────────
 
+describe("source packet capture", () => {
+  const html = "<html><head><title>Example Page</title></head><body>Hello</body></html>";
+  const pdfBytes = "%PDF-1.7\n1 0 obj\n<</Type/Catalog>>\nendobj\n";
+
+  beforeEach(() => {
+    tempDir = join(tmpdir(), `pi-llm-wiki-capture-${Date.now()}`);
+    mkdirSync(tempDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  /** Build a mock pi.exec that optionally handles curl -o file writes. */
+  function mockPi(stdout?: string, writeOriginal = true) {
+    return {
+      exec: async (command: string, args: string[]) => {
+        if (command === "sh") return { stdout: "no\n", stderr: "", code: 0 };
+        if (command === "curl" && args.includes("-o")) {
+          if (writeOriginal) {
+            const outputPath = args[args.indexOf("-o") + 1];
+            writeFileSync(outputPath, stdout ?? html, "utf-8");
+          }
+          return { stdout: "", stderr: "", code: 0 };
+        }
+        if (command === "curl") return { stdout: stdout ?? html, stderr: "", code: 0 };
+        throw new Error(`Unexpected command: ${command}`);
+      },
+    };
+  }
+
+  function makePaths() {
+    const p = getVaultPaths(join(tempDir, `wiki-${Math.random().toString(36).slice(2)}`));
+    ensureVaultStructure(p);
+    return p;
+  }
+
+  it("should preserve the original artifact for URL captures and render clickable links", async () => {
+    const paths = makePaths();
+    const pi = mockPi();
+
+    const url = "https://example.com/article";
+    const result = await captureUrl(pi as never, paths, url);
+
+    // Original artifact preserved
+    expect(existsSync(join(result.packetPath, "original", "source.html"))).toBe(true);
+    expect(readFile(join(result.packetPath, "original", "source.html"))).toBe(html);
+    expect(readFile(join(result.packetPath, "extracted.md"))).toBe(html);
+
+    // Clickable URL in source page
+    const sourcePage = readFile(result.sourcePagePath);
+    expect(sourcePage).toContain(`> _Original: [${url}](${url})_`);
+  });
+
+  it("should write PDF extraction failure message for .pdf URLs when MarkItDown is unavailable", async () => {
+    const paths = makePaths();
+    const pi = mockPi();
+
+    const url = "https://example.com/report.pdf";
+    const result = await captureUrl(pi as never, paths, url);
+
+    // Should NOT have written raw original for .pdf (no uvx)
+    expect(existsSync(join(result.packetPath, "original", "source.pdf"))).toBe(true);
+
+    // extracted.md should contain a failure message, not raw bytes
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).not.toContain("%PDF-");
+    expect(extracted).toContain("PDF content could not be converted");
+    expect(extracted).toContain(url);
+  });
+
+  it("should sniff %PDF- bytes from non-.pdf URLs and write a failure message instead", async () => {
+    const paths = makePaths();
+    // curl returns PDF bytes from a non-.pdf URL
+    const pi = mockPi(pdfBytes);
+
+    const url = "https://example.com/download?format=pdf";
+    const result = await captureUrl(pi as never, paths, url);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).not.toContain("%PDF-");
+    expect(extracted).toContain("PDF content could not be converted");
+    expect(extracted).toContain(url);
+  });
+
+  it("should name original artifacts based on URL extension", async () => {
+    const cases: Array<{ url: string; expected: string }> = [
+      { url: "https://example.com/article.html", expected: "source.html" },
+      { url: "https://example.com/doc.pdf", expected: "source.pdf" },
+      { url: "https://example.com/notes.md", expected: "source.md" },
+      { url: "https://example.com/data.xml", expected: "source.xml" },
+      { url: "https://example.com/readme.txt", expected: "source.txt" },
+      { url: "https://example.com/page", expected: "source.html" },
+      { url: "https://example.com/page?format=pdf", expected: "source.html" },
+    ];
+
+    for (const { url, expected } of cases) {
+      const paths = makePaths();
+      const pi = mockPi();
+      const result = await captureUrl(pi as never, paths, url);
+      const originalFile = join(result.packetPath, "original", expected);
+      expect(existsSync(originalFile)).toBe(true);
+    }
+  });
+
+  it("should render source page without an Original: line for text captures", async () => {
+    const paths = makePaths();
+    const result = captureText(paths, "Some text content", "My Note");
+
+    const sourcePage = readFile(result.sourcePagePath);
+    expect(sourcePage).toContain("# My Note");
+    expect(sourcePage).not.toContain("Original:");
+    expect(sourcePage).toContain("_Auto-preview: Some text content_");
+  });
+
+  it("should truncate auto-preview to 500 characters", async () => {
+    const paths = makePaths();
+    const longText = "A".repeat(1000);
+    const result = captureText(paths, longText, "Long Note");
+
+    const sourcePage = readFile(result.sourcePagePath);
+    // Preview should be 500 chars + "..."
+    expect(sourcePage).toContain(`_Auto-preview: ${"A".repeat(500)}..._`);
+  });
+
+  it("should handle local PDF file capture failure message when MarkItDown is unavailable", async () => {
+    const paths = makePaths();
+    // Create a temporary PDF file
+    const pdfPath = join(tempDir, "test.pdf");
+    writeFileSync(pdfPath, pdfBytes, "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, pdfPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).not.toContain("%PDF-");
+    expect(extracted).toContain("PDF content could not be converted");
+  });
+
+  it("should copy local non-PDF file content to extracted.md", async () => {
+    const paths = makePaths();
+    const mdPath = join(tempDir, "notes.md");
+    writeFileSync(mdPath, "# My Notes\n\nHello world.", "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, mdPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).toContain("# My Notes");
+    expect(extracted).toContain("Hello world.");
+
+    // Original file should be preserved
+    expect(existsSync(join(result.packetPath, "original", "notes.md"))).toBe(true);
+  });
+});
+
 describe("wiki directory structure", () => {
   let wikiDir: string;