@zosmaai/pi-llm-wiki 0.2.1 → 0.3.0

package/.github/codeql/codeql-config.yml

@@ -0,0 +1,12 @@
+name: "CodeQL Config"
+
+queries:
+  - uses: security-and-quality
+
+# Suppress false positives for temporary file creation.
+# This is a single-user CLI tool — TOCTOU race conditions on temp directories
+# are not a valid threat model. The alerts only fire because tests pass
+# os.tmpdir() paths into production capture functions.
+query-filters:
+  - exclude:
+      id: js/insecure-temporary-file

package/.github/workflows/ci.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: [20, 22, 23]
+        node-version: [20, 22, 23, 24, 25]
 
     steps:
       - uses: actions/checkout@v4

package/.github/workflows/codeql.yml

@@ -32,7 +32,7 @@ jobs:
         with:
           languages: ${{ matrix.language }}
           build-mode: ${{ matrix.build-mode }}
-          queries: security-and-quality
+          config-file: ./.github/codeql/codeql-config.yml
 
       - uses: github/codeql-action/analyze@v3
         with:

package/.github/workflows/release.yml

@@ -43,6 +43,14 @@ jobs:
           node -e "const p=require('./package.json'); p.version='${{ steps.version.outputs.VERSION }}'; require('fs').writeFileSync('./package.json', JSON.stringify(p, null, 2)+'\n')"
           npx biome check --fix package.json 2>&1
 
+      - name: Commit version bump back to main
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git add package.json
+          git commit -m "chore(release): v${{ steps.version.outputs.VERSION }}"
+          git push origin HEAD:main
+
       - name: Generate Release Notes
         id: notes
         run: |

package/README.md

@@ -42,6 +42,14 @@ Drop sources into `raw/`, then:
 | `wiki_log_event`      | Record custom event                           |
 | `wiki_watch`          | Schedule auto-updates                         |
 
+## Features
+
+- **Configurable PDF extraction** — MarkItDown timeout adjustable via `WIKI_MARKITDOWN_TIMEOUT_MS` env var
+- **Smart content detection** — PDF bytes sniffed even from non-`.pdf` URLs, never written as markdown
+- **Original artifacts preserved** — URL captures save the fetched payload under `original/source.*`
+- **Clickable source links** — Captured URLs render as clickable Markdown links in source pages
+- **Reliable prompt forwarding** — Slash commands properly forward user arguments to the model
+
 ## Architecture
 
 Four layers with clear ownership:
@@ -67,6 +75,22 @@ Read [docs/architecture.md](docs/architecture.md) for details.
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).
 
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=zosmaai/pi-llm-wiki&type=Date)](https://star-history.com/#zosmaai/pi-llm-wiki&Date)
+
+## Contributors
+
+<a href="https://github.com/zosmaai/pi-llm-wiki/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=zosmaai/pi-llm-wiki" alt="Contributors" />
+</a>
+
+---
+
+<div align="center">
+  <sub>Built with ❤️ by <a href="https://github.com/zosmaai">zosmaai</a></sub>
+</div>
+
 ## License
 
 MIT

package/docs/configuration.md

@@ -23,6 +23,12 @@ Wiki configuration lives in `.wiki/config.json`.
 | `auto_fix_lint`            | false   | Auto-fix lint issues               |
 | `batch_ingest_size`        | 3       | Sources processed per ingest batch |
 
+## Environment Variables
+
+| Variable                      | Default | Description                                     |
+| ----------------------------- | ------- | ----------------------------------------------- |
+| `WIKI_MARKITDOWN_TIMEOUT_MS` | 180000  | Timeout (ms) for MarkItDown PDF/text extraction |
+
 ## Page Frontmatter
 
 ```yaml

package/extensions/llm-wiki/lib/source-packet.ts

@@ -1,5 +1,5 @@
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
+import { extname, join } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { appendEvent } from "./metadata.js";
 import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
@@ -52,6 +52,39 @@ function pdfExtractionFailureMessage(source: string): string {
   return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
 }
 
+const URL_ORIGINAL_EXTENSIONS = new Set([".html", ".htm", ".md", ".pdf", ".txt", ".xml", ".json"]);
+
+function originalFileNameForUrl(url: string): string {
+  try {
+    const parsed = new URL(url);
+    const ext = extname(parsed.pathname).toLowerCase();
+    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
+  } catch {
+    const path = url.split(/[?#]/, 1)[0] ?? "";
+    const ext = extname(path).toLowerCase();
+    if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
+  }
+
+  return "source.html";
+}
+
+async function preserveUrlOriginal(
+  pi: ExtensionAPI,
+  packetPath: string,
+  url: string,
+  signal?: AbortSignal,
+): Promise<void> {
+  const originalPath = join(packetPath, "original", originalFileNameForUrl(url));
+  try {
+    await exec(pi, "curl", ["-sL", "--max-time", "30", "-o", originalPath, url], {
+      signal,
+      timeout: 35_000,
+    });
+  } catch {
+    // Preserve best-effort extraction behavior even when the original artifact cannot be saved.
+  }
+}
+
 /** Capture a URL into a source packet. */
 export async function captureUrl(
   pi: ExtensionAPI,
@@ -65,6 +98,8 @@ export async function captureUrl(
   mkdirSync(join(packetPath, "original"), { recursive: true });
   mkdirSync(join(packetPath, "attachments"), { recursive: true });
 
+  await preserveUrlOriginal(pi, packetPath, url, signal);
+
   // Try to fetch and extract content
   let extracted = "";
   let title = url;
@@ -171,12 +206,20 @@ export async function captureFile(
   mkdirSync(join(packetPath, "original"), { recursive: true });
   mkdirSync(join(packetPath, "attachments"), { recursive: true });
 
-  const isPdf = filePath.toLowerCase().endsWith(".pdf");
+  const lowerPath = filePath.toLowerCase();
+  const isPdf = lowerPath.endsWith(".pdf");
+  const isXml = lowerPath.endsWith(".xml");
   const content = isPdf ? "" : readText(filePath);
   const fileName = filePath.split("/").pop() || "unknown";
 
-  // Try MarkItDown for PDFs.
   let extracted = content;
+
+  // Convert XML to markdown
+  if (isXml && content) {
+    extracted = xmlToMarkdown(content);
+  }
+
+  // Try MarkItDown for PDFs.
   if (isPdf) {
     const markitdown = await exec(
       pi,
@@ -274,7 +317,7 @@ export function captureText(paths: VaultPaths, text: string, title?: string): Ca
 function buildSourcePageSkeleton(manifest: Record<string, unknown>, extracted: string): string {
   const id = String(manifest.id);
   const title = String(manifest.title || id);
-  const url = manifest.url ? `\n> _Original: ${manifest.url}_` : "";
+  const url = manifest.url ? `\n> _Original: [${manifest.url}](${manifest.url})_` : "";
   const format = String(manifest.format || "unknown");
   const captured = String(manifest.captured || fmtDate());
 
@@ -328,12 +371,58 @@ status: skeleton
 `;
 }
 
+/** Basic XML to markdown conversion: strip tags while preserving text structure. */
+function xmlToMarkdown(xml: string): string {
+  // Extract title from first <title> or root element
+  let title = "";
+  const titleMatch = xml.match(/<title[^>]*>([^<]*)<\/title>/i);
+  if (titleMatch) title = titleMatch[1].trim();
+
+  // Strip XML declaration and doctype
+  let text = xml.replace(/<\?xml[^>]*\?>\s*/gi, "");
+  text = text.replace(/<!DOCTYPE[^>]*>\s*/gi, "");
+
+  // Replace block-level tags with newlines
+  text = text.replace(/<\/(p|div|section|article|li|h\d|tr|blockquote|pre)>/gi, "\n");
+  text = text.replace(/<br\s*\/?>/gi, "\n");
+
+  // Strip remaining tags — match < followed by tag name characters to >
+  // Using a loop to handle malformed/broken tags that lack a closing >
+  let prev = "";
+  while (prev !== text) {
+    prev = text;
+    text = text.replace(/<[a-zA-Z\/!?][^>]*>/g, "");
+  }
+  // Remove any stray < that didn't form a complete tag
+  text = text.replace(/</g, "");
+
+  // Decode XML entities in a single pass to avoid double-unescaping
+  text = text.replace(/&(?:amp|lt|gt|quot|#\d+);/gi, (entity) => {
+    const map: Record<string, string> = { "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"' };
+    const lower = entity.toLowerCase();
+    if (map[lower]) return map[lower];
+    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
+    return entity;
+  });
+
+  // Clean up excessive blank lines
+  text = text.replace(/\n{3,}/g, "\n\n").trim();
+
+  if (!text) return xml; // fallback: return raw if stripping produced nothing
+
+  const lines = [];
+  if (title) lines.push(`# ${title}\n`);
+  lines.push(text);
+  return lines.join("\n\n");
+}
+
 function guessFormat(filePath: string): string {
   const lower = filePath.toLowerCase();
   if (lower.endsWith(".pdf")) return "pdf";
   if (lower.endsWith(".md")) return "markdown";
   if (lower.endsWith(".txt")) return "text";
   if (lower.endsWith(".html") || lower.endsWith(".htm")) return "html";
+  if (lower.endsWith(".xml")) return "xml";
   if (lower.endsWith(".docx")) return "docx";
   return "file";
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "LLM Wiki for Pi — self-maintaining knowledge base following Karpathy's pattern. Obsidian-friendly, auto-updating, personal & company wiki.",
   "keywords": [
     "pi-package",

package/test/llm-wiki.test.ts

@@ -3,6 +3,8 @@ import { tmpdir } from "node:os";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { captureFile, captureText, captureUrl } from "../extensions/llm-wiki/lib/source-packet.js";
+import { ensureVaultStructure, getVaultPaths } from "../extensions/llm-wiki/lib/utils.js";
 
 // ─── Helpers ────────────────────────────────────────────
 
@@ -232,6 +234,212 @@ describe("skill frontmatter validation", () => {
 
 // ─── Wiki Directory Structure Tests ─────────────────────
 
+describe("source packet capture", () => {
+  const html = "<html><head><title>Example Page</title></head><body>Hello</body></html>";
+  const pdfBytes = "%PDF-1.7\n1 0 obj\n<</Type/Catalog>>\nendobj\n";
+
+  beforeEach(() => {
+    tempDir = join(tmpdir(), `pi-llm-wiki-capture-${Date.now()}`);
+    mkdirSync(tempDir, { recursive: true });
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+  });
+
+  /** Build a mock pi.exec that optionally handles curl -o file writes. */
+  function mockPi(stdout?: string, writeOriginal = true) {
+    return {
+      exec: async (command: string, args: string[]) => {
+        if (command === "sh") return { stdout: "no\n", stderr: "", code: 0 };
+        if (command === "curl" && args.includes("-o")) {
+          if (writeOriginal) {
+            const outputPath = args[args.indexOf("-o") + 1];
+            writeFileSync(outputPath, stdout ?? html, "utf-8");
+          }
+          return { stdout: "", stderr: "", code: 0 };
+        }
+        if (command === "curl") return { stdout: stdout ?? html, stderr: "", code: 0 };
+        throw new Error(`Unexpected command: ${command}`);
+      },
+    };
+  }
+
+  function makePaths() {
+    const p = getVaultPaths(join(tempDir, `wiki-${Math.random().toString(36).slice(2)}`));
+    ensureVaultStructure(p);
+    return p;
+  }
+
+  it("should preserve the original artifact for URL captures and render clickable links", async () => {
+    const paths = makePaths();
+    const pi = mockPi();
+
+    const url = "https://example.com/article";
+    const result = await captureUrl(pi as never, paths, url);
+
+    // Original artifact preserved
+    expect(existsSync(join(result.packetPath, "original", "source.html"))).toBe(true);
+    expect(readFile(join(result.packetPath, "original", "source.html"))).toBe(html);
+    expect(readFile(join(result.packetPath, "extracted.md"))).toBe(html);
+
+    // Clickable URL in source page
+    const sourcePage = readFile(result.sourcePagePath);
+    expect(sourcePage).toContain(`> _Original: [${url}](${url})_`);
+  });
+
+  it("should write PDF extraction failure message for .pdf URLs when MarkItDown is unavailable", async () => {
+    const paths = makePaths();
+    const pi = mockPi();
+
+    const url = "https://example.com/report.pdf";
+    const result = await captureUrl(pi as never, paths, url);
+
+    // Should NOT have written raw original for .pdf (no uvx)
+    expect(existsSync(join(result.packetPath, "original", "source.pdf"))).toBe(true);
+
+    // extracted.md should contain a failure message, not raw bytes
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).not.toContain("%PDF-");
+    expect(extracted).toContain("PDF content could not be converted");
+    expect(extracted).toContain(url);
+  });
+
+  it("should sniff %PDF- bytes from non-.pdf URLs and write a failure message instead", async () => {
+    const paths = makePaths();
+    // curl returns PDF bytes from a non-.pdf URL
+    const pi = mockPi(pdfBytes);
+
+    const url = "https://example.com/download?format=pdf";
+    const result = await captureUrl(pi as never, paths, url);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).not.toContain("%PDF-");
+    expect(extracted).toContain("PDF content could not be converted");
+    expect(extracted).toContain(url);
+  });
+
+  it("should name original artifacts based on URL extension", async () => {
+    const cases: Array<{ url: string; expected: string }> = [
+      { url: "https://example.com/article.html", expected: "source.html" },
+      { url: "https://example.com/doc.pdf", expected: "source.pdf" },
+      { url: "https://example.com/notes.md", expected: "source.md" },
+      { url: "https://example.com/data.xml", expected: "source.xml" },
+      { url: "https://example.com/readme.txt", expected: "source.txt" },
+      { url: "https://example.com/page", expected: "source.html" },
+      { url: "https://example.com/page?format=pdf", expected: "source.html" },
+    ];
+
+    for (const { url, expected } of cases) {
+      const paths = makePaths();
+      const pi = mockPi();
+      const result = await captureUrl(pi as never, paths, url);
+      const originalFile = join(result.packetPath, "original", expected);
+      expect(existsSync(originalFile)).toBe(true);
+    }
+  });
+
+  it("should render source page without an Original: line for text captures", async () => {
+    const paths = makePaths();
+    const result = captureText(paths, "Some text content", "My Note");
+
+    const sourcePage = readFile(result.sourcePagePath);
+    expect(sourcePage).toContain("# My Note");
+    expect(sourcePage).not.toContain("Original:");
+    expect(sourcePage).toContain("_Auto-preview: Some text content_");
+  });
+
+  it("should truncate auto-preview to 500 characters", async () => {
+    const paths = makePaths();
+    const longText = "A".repeat(1000);
+    const result = captureText(paths, longText, "Long Note");
+
+    const sourcePage = readFile(result.sourcePagePath);
+    // Preview should be 500 chars + "..."
+    expect(sourcePage).toContain(`_Auto-preview: ${"A".repeat(500)}..._`);
+  });
+
+  it("should handle local PDF file capture failure message when MarkItDown is unavailable", async () => {
+    const paths = makePaths();
+    // Create a temporary PDF file
+    const pdfPath = join(tempDir, "test.pdf");
+    writeFileSync(pdfPath, pdfBytes, "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, pdfPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).not.toContain("%PDF-");
+    expect(extracted).toContain("PDF content could not be converted");
+  });
+
+  it("should copy local non-PDF file content to extracted.md", async () => {
+    const paths = makePaths();
+    const mdPath = join(tempDir, "notes.md");
+    writeFileSync(mdPath, "# My Notes\n\nHello world.", "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, mdPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    expect(extracted).toContain("# My Notes");
+    expect(extracted).toContain("Hello world.");
+
+    // Original file should be preserved
+    expect(existsSync(join(result.packetPath, "original", "notes.md"))).toBe(true);
+  });
+
+  it("should convert XML files to readable markdown in extracted.md", async () => {
+    const paths = makePaths();
+    const xmlContent = `<?xml version="1.0" encoding="UTF-8"?>
+<document>
+  <title>Project Report</title>
+  <section>
+    <heading>Findings</heading>
+    <p>The analysis revealed several key insights.</p>
+    <list>
+      <item>First finding</item>
+      <item>Second finding</item>
+    </list>
+  </section>
+</document>`;
+    const xmlPath = join(tempDir, "report.xml");
+    writeFileSync(xmlPath, xmlContent, "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, xmlPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    // Should have extracted title
+    expect(extracted).toContain("Project Report");
+    // Should have extracted text content
+    expect(extracted).toContain("The analysis revealed several key insights.");
+    expect(extracted).toContain("First finding");
+    expect(extracted).toContain("Second finding");
+    // Should NOT contain raw XML tags
+    expect(extracted).not.toContain("<?xml");
+    expect(extracted).not.toContain("<document>");
+    expect(extracted).not.toContain("</document>");
+
+    // Original file should be preserved
+    expect(existsSync(join(result.packetPath, "original", "report.xml"))).toBe(true);
+  });
+
+  it("should fall back to raw XML content when tag stripping produces nothing", async () => {
+    const paths = makePaths();
+    const xmlContent = `<?xml version="1.0"?><data><![CDATA[Hello]]></data>`;
+    const xmlPath = join(tempDir, "minimal.xml");
+    writeFileSync(xmlPath, xmlContent, "utf-8");
+
+    const pi = mockPi();
+    const result = await captureFile(pi as never, paths, xmlPath);
+
+    const extracted = readFile(join(result.packetPath, "extracted.md"));
+    // Should have the text content at minimum
+    expect(extracted).toContain("Hello");
+  });
+});
+
 describe("wiki directory structure", () => {
   let wikiDir: string;