@zosmaai/pi-llm-wiki 0.8.1 → 0.8.2

package/README.md

@@ -399,9 +399,39 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup, test patterns, and
 
 ## Contributors
 
-<a href="https://github.com/zosmaai/pi-llm-wiki/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=zosmaai/pi-llm-wiki" alt="Contributors" />
-</a>
+Thanks to everyone who has contributed! This list is regenerated automatically by [`.github/workflows/contributors.yml`](.github/workflows/contributors.yml) — see [#60](https://github.com/zosmaai/pi-llm-wiki/issues/60) for the rationale.
+
+<!-- readme: contributors -start -->
+<table>
+	<tbody>
+		<tr>
+            <td align="center">
+                <a href="https://github.com/arjun-zosma">
+                    <img src="https://avatars.githubusercontent.com/u/25246034?v=4" width="64;" alt="arjun-zosma"/>
+                    <br />
+                    <sub><b>Arjun Nayak</b></sub>
+                </a>
+            </td>
+            <td align="center">
+                <a href="https://github.com/jfraser">
+                    <img src="https://avatars.githubusercontent.com/u/165964?v=4" width="64;" alt="jfraser"/>
+                    <br />
+                    <sub><b>James Fraser</b></sub>
+                </a>
+            </td>
+            <td align="center">
+                <a href="https://github.com/Shanvit7">
+                    <img src="https://avatars.githubusercontent.com/u/64424817?v=4" width="64;" alt="Shanvit7"/>
+                    <br />
+                    <sub><b>Shanvit S Shetty</b></sub>
+                </a>
+            </td>
+		</tr>
+	<tbody>
+</table>
+<!-- readme: contributors -end -->
+
+<sub>Full history: [contributors graph](https://github.com/zosmaai/pi-llm-wiki/graphs/contributors).</sub>
 
 ---
 

package/extensions/llm-wiki/lib/source-extractors.ts

@@ -1,4 +1,6 @@
+import { open } from "node:fs/promises";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { NodeHtmlMarkdown } from "node-html-markdown";
 import { exec } from "./utils.js";
 
 export type ExtractionStatus = "success" | "failed" | "unsupported";
@@ -38,6 +40,67 @@ interface UrlExtractArgs {
   signal?: AbortSignal;
 }
 
+// ---------------------------------------------------------------------------
+// Binary magic byte detection
+// ---------------------------------------------------------------------------
+
+const BINARY_SIGNATURES: Array<{ bytes: number[]; format: string }> = [
+  // Archives & documents
+  { bytes: [0x50, 0x4b, 0x03, 0x04], format: "zip" }, // ZIP / DOCX / XLSX / PPTX / JAR
+  { bytes: [0x25, 0x50, 0x44, 0x46], format: "pdf" }, // %PDF
+  { bytes: [0x37, 0x7a, 0xbc, 0xaf], format: "7z" }, // 7-Zip
+  { bytes: [0x1f, 0x8b], format: "gzip" }, // gzip / .tar.gz
+  // Images
+  { bytes: [0x89, 0x50, 0x4e, 0x47], format: "png" }, // PNG
+  { bytes: [0xff, 0xd8, 0xff], format: "jpeg" }, // JPEG
+  { bytes: [0x47, 0x49, 0x46, 0x38], format: "gif" }, // GIF8
+  { bytes: [0x42, 0x4d], format: "bmp" }, // BMP
+  { bytes: [0x49, 0x49, 0x2a, 0x00], format: "tiff" }, // TIFF (little-endian)
+  { bytes: [0x4d, 0x4d, 0x00, 0x2a], format: "tiff" }, // TIFF (big-endian)
+  { bytes: [0x52, 0x49, 0x46, 0x46], format: "riff" }, // RIFF (WAV / AVI / WebP)
+  // Executables & binaries
+  { bytes: [0x4d, 0x5a], format: "exe" }, // Windows PE (EXE / DLL)
+  { bytes: [0xcf, 0xfa, 0xed, 0xfe], format: "macho" }, // Mach-O 64-bit LE
+  { bytes: [0xce, 0xfa, 0xed, 0xfe], format: "macho" }, // Mach-O 32-bit LE
+  { bytes: [0xfe, 0xed, 0xfa, 0xcf], format: "macho" }, // Mach-O 64-bit BE
+  { bytes: [0xfe, 0xed, 0xfa, 0xce], format: "macho" }, // Mach-O 32-bit BE
+  { bytes: [0xca, 0xfe, 0xba, 0xbe], format: "class" }, // Java .class / Mach-O FAT
+  { bytes: [0x7f, 0x45, 0x4c, 0x46], format: "elf" }, // ELF binary
+  { bytes: [0x00, 0x61, 0x73, 0x6d], format: "wasm" }, // WebAssembly
+  // Data & media
+  { bytes: [0x53, 0x51, 0x4c, 0x69], format: "sqlite" }, // SQLite
+  { bytes: [0x49, 0x44, 0x33], format: "mp3" }, // MP3 (ID3 tag)
+];
+
+/**
+ * Reads the first 8 bytes of `filePath` and checks them against known binary
+ * magic byte signatures. Returns the detected format name or `null` for text.
+ */
+export async function detectBinaryMagicBytes(filePath: string): Promise<string | null> {
+  let handle: import("node:fs/promises").FileHandle | undefined;
+  try {
+    handle = await open(filePath, "r");
+    const buf = Buffer.alloc(8);
+    const { bytesRead } = await handle.read(buf, 0, 8, 0);
+    const header = buf.subarray(0, bytesRead);
+
+    for (const { bytes, format } of BINARY_SIGNATURES) {
+      if (bytes.every((b, i) => header[i] === b)) return format;
+    }
+    return null;
+  } catch {
+    return null; // Unreadable file — let the extractor deal with it
+  } finally {
+    await handle?.close();
+  }
+}
+
+export function binaryExtractionFailureMessage(format: string): string {
+  return `_Binary file could not be converted to markdown (detected format: ${format}).\nCapture a text-based version or a URL pointing to readable content instead._\n`;
+}
+
+// ---------------------------------------------------------------------------
+
 const DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000;
 const DEFAULT_CURL_TIMEOUT_SECONDS = 30;
 
@@ -193,10 +256,11 @@ async function extractTextUrl(
       content_type: "application/pdf",
     };
   }
+  const normalized = htmlToMarkdown(curlExtracted);
   return {
-    extracted: curlExtracted,
-    title: titleFromHtml(curlExtracted),
-    extractor: "curl",
+    extracted: normalized,
+    title: titleFromMarkdown(normalized) ?? titleFromHtml(curlExtracted),
+    extractor: "htmlToMarkdown",
     extraction_status: "success",
   };
 }
@@ -279,6 +343,23 @@ function titleFromHtml(html: string): string | undefined {
   return html.match(/<title>([^<]*)<\/title>/i)?.[1]?.trim();
 }
 
+/** Decode common HTML/XML entities. Shared by xmlToMarkdown and htmlToMarkdown. */
+function decodeHtmlEntities(text: string): string {
+  return text.replace(/&(?:amp|lt|gt|quot|apos|#\d+);/gi, (entity) => {
+    const map: Record<string, string> = {
+      "&amp;": "&",
+      "&lt;": "<",
+      "&gt;": ">",
+      "&quot;": '"',
+      "&apos;": "'",
+    };
+    const lower = entity.toLowerCase();
+    if (map[lower]) return map[lower];
+    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
+    return entity;
+  });
+}
+
 /** Basic XML to markdown conversion: strip tags while preserving text structure. */
 function xmlToMarkdown(xml: string): string {
   let title = "";
@@ -297,14 +378,7 @@ function xmlToMarkdown(xml: string): string {
   }
   text = text.replace(/</g, "");
 
-  text = text.replace(/&(?:amp|lt|gt|quot|#\d+);/gi, (entity) => {
-    const map: Record<string, string> = { "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"' };
-    const lower = entity.toLowerCase();
-    if (map[lower]) return map[lower];
-    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
-    return entity;
-  });
-
+  text = decodeHtmlEntities(text);
   text = text.replace(/\n{3,}/g, "\n\n").trim();
   if (!text) return xml;
 
@@ -314,6 +388,41 @@ function xmlToMarkdown(xml: string): string {
   return lines.join("\n\n");
 }
 
+/**
+ * Lightweight HTML-to-markdown normalizer for the curl fallback path.
+ *
+ * Pre-strips page chrome (nav, header, footer, script, style) that
+ * node-html-markdown does not remove, then delegates full conversion —
+ * bold, italic, code blocks, tables, ordered lists, image alt text — to
+ * node-html-markdown. Prepends the <title> as a # heading when the body
+ * has no <h1> of its own.
+ *
+ * Falls back to the original HTML if conversion yields an empty string.
+ */
+export function htmlToMarkdown(input: string): string {
+  // 1. Extract <title> from original before stripping head
+  const title = input.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.trim() ?? "";
+
+  // 2. Strip <head> and noise blocks that node-html-markdown won't remove
+  let html = input.replace(/<head[\s\S]*?<\/head>/gi, "");
+  let previousHtml = "";
+  while (previousHtml !== html) {
+    previousHtml = html;
+    html = html.replace(/<(script|style|nav|header|footer|noscript)[\s\S]*?<\/\1>/gi, "");
+  }
+
+  // 3. Delegate to node-html-markdown for full semantic conversion
+  const converted = NodeHtmlMarkdown.translate(html).trim();
+  if (!converted) return input;
+
+  // 4. Prepend <title> as # heading only if body has no <h1> of its own
+  const hasBodyH1 = /<h1[^>]*>[\s\S]*?<\/h1>/i.test(html);
+  const lines: string[] = [];
+  if (title && !hasBodyH1) lines.push(`# ${title}\n`);
+  lines.push(converted);
+  return lines.join("\n");
+}
+
 function jsonToMarkdown(json: string): string {
   let value: unknown;
   try {

package/extensions/llm-wiki/lib/source-packet.ts

@@ -2,7 +2,13 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { extname, join } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { appendEvent } from "./metadata.js";
-import { type ExtractedContent, extractUrlContent, fileExtractorFor } from "./source-extractors.js";
+import {
+  type ExtractedContent,
+  binaryExtractionFailureMessage,
+  detectBinaryMagicBytes,
+  extractUrlContent,
+  fileExtractorFor,
+} from "./source-extractors.js";
 import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
 
 /**
@@ -107,6 +113,18 @@ function fileCaptureSource(
     preserveOriginal: (packetPath) =>
       preserveFileOriginal(pi, packetPath, filePath, fileName, content, signal),
     extract: async () => {
+      // Guard: if we hit the generic catch-all extractor, check for binary magic bytes first
+      if (extractor.format === "file") {
+        const binaryFormat = await detectBinaryMagicBytes(filePath);
+        if (binaryFormat) {
+          return {
+            extracted: binaryExtractionFailureMessage(binaryFormat),
+            extractor: "magicBytes",
+            extraction_status: "unsupported" as const,
+          };
+        }
+      }
+
       const extractedStr = await extractor.extract({ pi, filePath, content, signal });
       const failed = extractedStr.includes("could not be converted");
       return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zosmaai/pi-llm-wiki",
-  "version": "0.8.1",
+  "version": "0.8.2",
   "description": "Self-maintaining LLM Wiki for Pi — Karpathy-pattern knowledge base with immutable source capture, automated ingestion, search, linting, and Obsidian-compatible vault. auto-updating personal & company wiki.",
   "keywords": [
     "pi",
@@ -76,7 +76,8 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@modelcontextprotocol/server": "^2.0.0-alpha.2"
+    "@modelcontextprotocol/server": "^2.0.0-alpha.2",
+    "node-html-markdown": "^2.0.0"
   },
   "devDependencies": {
     "@biomejs/biome": "^1.9.4",