@zosmaai/pi-llm-wiki 0.8.1 → 0.9.0

package/extensions/llm-wiki/lib/runtime.ts

@@ -0,0 +1,216 @@
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { TASK_DEFAULTS, type TaskConfig, loadTaskConfig } from "./task-config.js";
+
+/**
+ * Background-task runtime for the LLM Wiki (issue #64, part of #63).
+ *
+ * Provides two primitives, ported from pi-observational-memory's proven
+ * pattern, that let the extension perform LLM work WITHOUT blocking the main
+ * agent turn:
+ *
+ *   - launchTask(): fire-and-forget a detached promise that may outlive the
+ *     current turn. The in-flight promise is stored so callers can await it at
+ *     compaction / session exit (so background work is never silently lost),
+ *     but the agent loop itself never blocks on it. Single-flight per label
+ *     to avoid pile-ups.
+ *
+ *   - resolveModel(): pick the model for background work — configured
+ *     `taskModel` → session model fallback → API-key resolution. Returns a
+ *     discriminated result so callers degrade gracefully (keep the existing
+ *     synchronous main-agent flow) when no model / API key is available.
+ *
+ * This module introduces NO user-facing behavior on its own; it is the
+ * infrastructure that issues #65 (background ingest), #66 (background
+ * embeddings) and #69 (model selection) build upon.
+ */
+
+export type ResolveResult =
+  | { ok: true; model: unknown; apiKey: string; headers?: Record<string, string> }
+  | { ok: false; reason: string };
+
+type NotifyLevel = "info" | "warning" | "error";
+type Notify = (message: string, type?: NotifyLevel) => void;
+
+export interface ResolveCtx {
+  /** Current session model (may be undefined when the session has no model). */
+  model: unknown;
+  modelRegistry: {
+    find(provider: string, id: string): unknown;
+    getApiKeyAndHeaders(
+      model: unknown,
+    ): Promise<{ ok: boolean; apiKey?: string; headers?: Record<string, string> }>;
+  };
+  hasUI: boolean;
+  ui?: { notify: Notify };
+}
+
+export interface LaunchCtx {
+  hasUI: boolean;
+  ui?: { notify: Notify };
+}
+
+export class Runtime {
+  config: TaskConfig = { ...TASK_DEFAULTS };
+  configLoaded = false;
+
+  /** Labels of tasks currently in flight (single-flight guard per label). */
+  private inFlightLabels = new Set<string>();
+  /** All in-flight task promises, keyed for await-at-exit and dedupe. */
+  private inFlight = new Map<string, Promise<void>>();
+  /** Whether we've already surfaced a model-resolution failure (avoid spam). */
+  resolveFailureNotified = false;
+
+  ensureConfig(cwd: string): void {
+    if (this.configLoaded) return;
+    this.config = loadTaskConfig(cwd);
+    this.configLoaded = true;
+  }
+
+  /** True if a task with this label is currently running. */
+  isInFlight(label: string): boolean {
+    return this.inFlightLabels.has(label);
+  }
+
+  /** Number of background tasks currently running. */
+  get pendingCount(): number {
+    return this.inFlight.size;
+  }
+
+  /**
+   * Resolve the model + auth for background work.
+   *
+   * Precedence (issue #69): per-call `override` → configured `taskModel` →
+   * session model. Each configured layer is applied only when the model is
+   * found in the registry; a missing layer warns (when UI is available) and
+   * falls through to the next. Returns { ok: false } when nothing resolves or
+   * no API key exists, so callers can fall back to the synchronous
+   * main-agent path.
+   */
+  async resolveModel(
+    ctx: ResolveCtx,
+    override?: { provider: string; id: string },
+  ): Promise<ResolveResult> {
+    let model = ctx.model;
+
+    // Configured taskModel layer (beats the session model).
+    const configured = this.config.taskModel;
+    if (configured) {
+      const found = ctx.modelRegistry.find(configured.provider, configured.id);
+      if (found) {
+        model = found;
+      } else if (ctx.hasUI && ctx.ui) {
+        ctx.ui.notify(
+          `LLM Wiki: configured task model ${configured.provider}/${configured.id} not found, using session model`,
+          "warning",
+        );
+      }
+    }
+
+    // Per-call override layer (beats both config and session).
+    if (override) {
+      const found = ctx.modelRegistry.find(override.provider, override.id);
+      if (found) {
+        model = found;
+      } else if (ctx.hasUI && ctx.ui) {
+        ctx.ui.notify(
+          `LLM Wiki: model override ${override.provider}/${override.id} not found, using ${
+            configured ? "configured/session" : "session"
+          } model`,
+          "warning",
+        );
+      }
+    }
+
+    if (!model) {
+      return {
+        ok: false,
+        reason: "no model available (session has no model and no taskModel configured)",
+      };
+    }
+
+    const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
+    if (!auth.ok || !auth.apiKey) {
+      const provider = (model as { provider?: string }).provider ?? "unknown";
+      return { ok: false, reason: `no API key for provider "${provider}"` };
+    }
+    return { ok: true, model, apiKey: auth.apiKey, headers: auth.headers };
+  }
+
+  /**
+   * Fire-and-forget a background task.
+   *
+   * The work runs in a detached promise so the caller (an agent hook/tool)
+   * is never blocked. Errors are caught and surfaced via the UI (when
+   * available) instead of crashing the agent. Single-flight per label: if a
+   * task with the same label is already running, the new request is dropped
+   * and the existing promise is returned.
+   *
+   * The returned promise resolves when the work completes; hold onto it (or
+   * call awaitAll) to drain background work before compaction/exit.
+   */
+  launchTask(ctx: LaunchCtx, label: string, work: () => Promise<void>): Promise<void> {
+    const existing = this.inFlight.get(label);
+    if (existing) return existing;
+
+    // Capture ctx properties synchronously — after `await work()` the extension
+    // ctx may be stale (e.g. after newSession/fork/switchSession/reload), and
+    // accessing ctx.hasUI or ctx.ui on a stale proxy throws.
+    const hasUI = ctx.hasUI;
+    const ui = ctx.ui;
+
+    this.inFlightLabels.add(label);
+    // biome-ignore lint/style/useConst: referenced inside its own initializer (finally block)
+    let promise!: Promise<void>;
+    promise = (async () => {
+      try {
+        await work();
+      } catch (error) {
+        const msg = error instanceof Error ? error.message : String(error);
+        if (hasUI && ui) ui.notify(`LLM Wiki: ${label} failed: ${msg}`, "warning");
+      } finally {
+        this.inFlightLabels.delete(label);
+        if (this.inFlight.get(label) === promise) this.inFlight.delete(label);
+      }
+    })();
+    this.inFlight.set(label, promise);
+    return promise;
+  }
+
+  /**
+   * Await all in-flight background tasks. Call at compaction / session exit so
+   * background work is not lost. Never rejects — task errors are already
+   * isolated inside launchTask.
+   */
+  async awaitAll(): Promise<void> {
+    while (this.inFlight.size > 0) {
+      await Promise.allSettled([...this.inFlight.values()]);
+    }
+  }
+}
+
+/**
+ * Register the shared background runtime and wire it into the extension
+ * lifecycle: config is loaded lazily per turn, and in-flight tasks are drained
+ * before compaction and on shutdown so background work is never lost.
+ *
+ * Returns the Runtime instance so concrete background workers (issues #65,
+ * #66) can launch tasks on it.
+ */
+export function registerBackgroundRuntime(pi: ExtensionAPI): Runtime {
+  const runtime = new Runtime();
+
+  pi.on("turn_start", (_event, ctx) => {
+    runtime.ensureConfig(ctx.cwd);
+  });
+
+  // Drain in-flight background work before the session is compacted or shut
+  // down, so nothing is lost mid-flight.
+  pi.on("session_before_compact", async () => {
+    await runtime.awaitAll();
+  });
+  pi.on("session_shutdown", async () => {
+    await runtime.awaitAll();
+  });
+
+  return runtime;
+}

package/extensions/llm-wiki/lib/source-extractors.ts

@@ -1,4 +1,6 @@
+import { open } from "node:fs/promises";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
+import { NodeHtmlMarkdown } from "node-html-markdown";
 import { exec } from "./utils.js";
 
 export type ExtractionStatus = "success" | "failed" | "unsupported";
@@ -38,6 +40,67 @@ interface UrlExtractArgs {
   signal?: AbortSignal;
 }
 
+// ---------------------------------------------------------------------------
+// Binary magic byte detection
+// ---------------------------------------------------------------------------
+
+const BINARY_SIGNATURES: Array<{ bytes: number[]; format: string }> = [
+  // Archives & documents
+  { bytes: [0x50, 0x4b, 0x03, 0x04], format: "zip" }, // ZIP / DOCX / XLSX / PPTX / JAR
+  { bytes: [0x25, 0x50, 0x44, 0x46], format: "pdf" }, // %PDF
+  { bytes: [0x37, 0x7a, 0xbc, 0xaf], format: "7z" }, // 7-Zip
+  { bytes: [0x1f, 0x8b], format: "gzip" }, // gzip / .tar.gz
+  // Images
+  { bytes: [0x89, 0x50, 0x4e, 0x47], format: "png" }, // PNG
+  { bytes: [0xff, 0xd8, 0xff], format: "jpeg" }, // JPEG
+  { bytes: [0x47, 0x49, 0x46, 0x38], format: "gif" }, // GIF8
+  { bytes: [0x42, 0x4d], format: "bmp" }, // BMP
+  { bytes: [0x49, 0x49, 0x2a, 0x00], format: "tiff" }, // TIFF (little-endian)
+  { bytes: [0x4d, 0x4d, 0x00, 0x2a], format: "tiff" }, // TIFF (big-endian)
+  { bytes: [0x52, 0x49, 0x46, 0x46], format: "riff" }, // RIFF (WAV / AVI / WebP)
+  // Executables & binaries
+  { bytes: [0x4d, 0x5a], format: "exe" }, // Windows PE (EXE / DLL)
+  { bytes: [0xcf, 0xfa, 0xed, 0xfe], format: "macho" }, // Mach-O 64-bit LE
+  { bytes: [0xce, 0xfa, 0xed, 0xfe], format: "macho" }, // Mach-O 32-bit LE
+  { bytes: [0xfe, 0xed, 0xfa, 0xcf], format: "macho" }, // Mach-O 64-bit BE
+  { bytes: [0xfe, 0xed, 0xfa, 0xce], format: "macho" }, // Mach-O 32-bit BE
+  { bytes: [0xca, 0xfe, 0xba, 0xbe], format: "class" }, // Java .class / Mach-O FAT
+  { bytes: [0x7f, 0x45, 0x4c, 0x46], format: "elf" }, // ELF binary
+  { bytes: [0x00, 0x61, 0x73, 0x6d], format: "wasm" }, // WebAssembly
+  // Data & media
+  { bytes: [0x53, 0x51, 0x4c, 0x69], format: "sqlite" }, // SQLite
+  { bytes: [0x49, 0x44, 0x33], format: "mp3" }, // MP3 (ID3 tag)
+];
+
+/**
+ * Reads the first 8 bytes of `filePath` and checks them against known binary
+ * magic byte signatures. Returns the detected format name or `null` for text.
+ */
+export async function detectBinaryMagicBytes(filePath: string): Promise<string | null> {
+  let handle: import("node:fs/promises").FileHandle | undefined;
+  try {
+    handle = await open(filePath, "r");
+    const buf = Buffer.alloc(8);
+    const { bytesRead } = await handle.read(buf, 0, 8, 0);
+    const header = buf.subarray(0, bytesRead);
+
+    for (const { bytes, format } of BINARY_SIGNATURES) {
+      if (bytes.every((b, i) => header[i] === b)) return format;
+    }
+    return null;
+  } catch {
+    return null; // Unreadable file — let the extractor deal with it
+  } finally {
+    await handle?.close();
+  }
+}
+
+export function binaryExtractionFailureMessage(format: string): string {
+  return `_Binary file could not be converted to markdown (detected format: ${format}).\nCapture a text-based version or a URL pointing to readable content instead._\n`;
+}
+
+// ---------------------------------------------------------------------------
+
 const DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000;
 const DEFAULT_CURL_TIMEOUT_SECONDS = 30;
 
@@ -193,10 +256,11 @@ async function extractTextUrl(
       content_type: "application/pdf",
     };
   }
+  const normalized = htmlToMarkdown(curlExtracted);
   return {
-    extracted: curlExtracted,
-    title: titleFromHtml(curlExtracted),
-    extractor: "curl",
+    extracted: normalized,
+    title: titleFromMarkdown(normalized) ?? titleFromHtml(curlExtracted),
+    extractor: "htmlToMarkdown",
     extraction_status: "success",
   };
 }
@@ -279,6 +343,23 @@ function titleFromHtml(html: string): string | undefined {
   return html.match(/<title>([^<]*)<\/title>/i)?.[1]?.trim();
 }
 
+/** Decode common HTML/XML entities. Shared by xmlToMarkdown and htmlToMarkdown. */
+function decodeHtmlEntities(text: string): string {
+  return text.replace(/&(?:amp|lt|gt|quot|apos|#\d+);/gi, (entity) => {
+    const map: Record<string, string> = {
+      "&amp;": "&",
+      "&lt;": "<",
+      "&gt;": ">",
+      "&quot;": '"',
+      "&apos;": "'",
+    };
+    const lower = entity.toLowerCase();
+    if (map[lower]) return map[lower];
+    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
+    return entity;
+  });
+}
+
 /** Basic XML to markdown conversion: strip tags while preserving text structure. */
 function xmlToMarkdown(xml: string): string {
   let title = "";
@@ -297,14 +378,7 @@ function xmlToMarkdown(xml: string): string {
   }
   text = text.replace(/</g, "");
 
-  text = text.replace(/&(?:amp|lt|gt|quot|#\d+);/gi, (entity) => {
-    const map: Record<string, string> = { "&amp;": "&", "&lt;": "<", "&gt;": ">", "&quot;": '"' };
-    const lower = entity.toLowerCase();
-    if (map[lower]) return map[lower];
-    if (lower.startsWith("&#")) return String.fromCodePoint(Number.parseInt(entity.slice(2, -1)));
-    return entity;
-  });
-
+  text = decodeHtmlEntities(text);
   text = text.replace(/\n{3,}/g, "\n\n").trim();
   if (!text) return xml;
 
@@ -314,6 +388,41 @@ function xmlToMarkdown(xml: string): string {
   return lines.join("\n\n");
 }
 
+/**
+ * Lightweight HTML-to-markdown normalizer for the curl fallback path.
+ *
+ * Pre-strips page chrome (nav, header, footer, script, style) that
+ * node-html-markdown does not remove, then delegates full conversion —
+ * bold, italic, code blocks, tables, ordered lists, image alt text — to
+ * node-html-markdown. Prepends the <title> as a # heading when the body
+ * has no <h1> of its own.
+ *
+ * Falls back to the original HTML if conversion yields an empty string.
+ */
+export function htmlToMarkdown(input: string): string {
+  // 1. Extract <title> from original before stripping head
+  const title = input.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.trim() ?? "";
+
+  // 2. Strip <head> and noise blocks that node-html-markdown won't remove
+  let html = input.replace(/<head[\s\S]*?<\/head>/gi, "");
+  let previousHtml = "";
+  while (previousHtml !== html) {
+    previousHtml = html;
+    html = html.replace(/<(script|style|nav|header|footer|noscript)[\s\S]*?<\/\1>/gi, "");
+  }
+
+  // 3. Delegate to node-html-markdown for full semantic conversion
+  const converted = NodeHtmlMarkdown.translate(html).trim();
+  if (!converted) return input;
+
+  // 4. Prepend <title> as # heading only if body has no <h1> of its own
+  const hasBodyH1 = /<h1[^>]*>[\s\S]*?<\/h1>/i.test(html);
+  const lines: string[] = [];
+  if (title && !hasBodyH1) lines.push(`# ${title}\n`);
+  lines.push(converted);
+  return lines.join("\n");
+}
+
 function jsonToMarkdown(json: string): string {
   let value: unknown;
   try {

package/extensions/llm-wiki/lib/source-packet.ts

@@ -2,7 +2,13 @@ import { mkdirSync, writeFileSync } from "node:fs";
 import { extname, join } from "node:path";
 import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
 import { appendEvent } from "./metadata.js";
-import { type ExtractedContent, extractUrlContent, fileExtractorFor } from "./source-extractors.js";
+import {
+  type ExtractedContent,
+  binaryExtractionFailureMessage,
+  detectBinaryMagicBytes,
+  extractUrlContent,
+  fileExtractorFor,
+} from "./source-extractors.js";
 import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
 
 /**
@@ -107,6 +113,18 @@ function fileCaptureSource(
     preserveOriginal: (packetPath) =>
       preserveFileOriginal(pi, packetPath, filePath, fileName, content, signal),
     extract: async () => {
+      // Guard: if we hit the generic catch-all extractor, check for binary magic bytes first
+      if (extractor.format === "file") {
+        const binaryFormat = await detectBinaryMagicBytes(filePath);
+        if (binaryFormat) {
+          return {
+            extracted: binaryExtractionFailureMessage(binaryFormat),
+            extractor: "magicBytes",
+            extraction_status: "unsupported" as const,
+          };
+        }
+      }
+
       const extractedStr = await extractor.extract({ pi, filePath, content, signal });
       const failed = extractedStr.includes("could not be converted");
       return {

package/extensions/llm-wiki/lib/subagent.ts

@@ -0,0 +1,82 @@
+import {
+  type AgentContext,
+  type AgentLoopConfig,
+  type AgentTool,
+  agentLoop,
+} from "@mariozechner/pi-agent-core";
+import type { Api, Message, Model } from "@mariozechner/pi-ai";
+
+/**
+ * Thin sub-agent runner for the LLM Wiki background lane (issue #64, part of #63).
+ *
+ * Wraps `agentLoop` so background tasks (ingest synthesis, topic inference,
+ * etc.) can run a focused, single-purpose agent on a resolved model with its
+ * own system prompt and tools — mirroring pi-observational-memory's
+ * `runObserver`. The caller drives behavior entirely through `tools`
+ * (tool-side effects accumulate results); this wrapper just drives the loop to
+ * completion and drains its event stream.
+ *
+ * This is infrastructure: it makes no wiki-specific decisions. Concrete
+ * background workers (issues #65, #66) supply the prompts and tools.
+ */
+export interface RunSubAgentArgs<TApi extends Api = Api> {
+  model: Model<TApi>;
+  apiKey: string;
+  headers?: Record<string, string>;
+  /** System prompt that defines the sub-agent's role. */
+  systemPrompt: string;
+  /** The user-turn instruction/payload to process. */
+  userPrompt: string;
+  /** Tools the sub-agent may call (side effects accumulate caller-side). */
+  tools: AgentTool[];
+  /** Max output tokens per model call. Default 4096. */
+  maxTokens?: number;
+  signal?: AbortSignal;
+}
+
+/**
+ * Run a sub-agent loop to completion.
+ *
+ * Returns nothing useful directly — by design, results are collected by the
+ * `tools` the caller passes (their `execute` accumulates into caller-owned
+ * state). This keeps the runner generic across every background task type.
+ */
+export async function runSubAgent<TApi extends Api = Api>(
+  args: RunSubAgentArgs<TApi>,
+): Promise<void> {
+  const { model, apiKey, headers, systemPrompt, userPrompt, tools, maxTokens, signal } = args;
+
+  const text = userPrompt.trim();
+  if (!text) return;
+
+  const prompts: Message[] = [
+    {
+      role: "user",
+      content: [{ type: "text", text }],
+      timestamp: Date.now(),
+    },
+  ];
+
+  const context: AgentContext = {
+    systemPrompt,
+    messages: [],
+    tools,
+  };
+
+  const reasoning = (model as unknown as { reasoning?: unknown }).reasoning;
+  const config: AgentLoopConfig = {
+    model,
+    apiKey,
+    headers,
+    maxTokens: maxTokens ?? 4096,
+    convertToLlm: (msgs) => msgs as Message[],
+    toolExecution: "sequential",
+    ...(reasoning ? { reasoning: "high" as const } : {}),
+  };
+
+  const stream = agentLoop(prompts, context, config, signal);
+  for await (const _event of stream) {
+    // Drain events; tool `execute` callbacks collect results caller-side.
+  }
+  await stream.result();
+}