@bismawy/pi-vision-watcher 1.0.7

package/src/error-log.ts

@@ -0,0 +1,131 @@
+/**
+ * Best-effort structured error logging for pi-vision-watcher.
+ *
+ * Every describer failure (auth error, network error, abort, empty response,
+ * stopReason error/aborted, timeout) and every user-facing "image description
+ * failed" warning is appended as one JSONL line to
+ *   ~/.pi/agent/logs/pi-vision-watcher/errors.log
+ * (resolvable via $PI_CODING_AGENT_DIR, like every other pi path).
+ *
+ * This is the troubleshooting surface for the "image description failed —
+ * unknown error" warning. That warning fires with reason "unknown error" when
+ * the engine's shared last-error string was already null — typically because a
+ * concurrent batch reset it between the failure and the warn — so the detailed
+ * reason lives only here, captured at the describer's failure source where the
+ * real exception/stopReason is still in hand. A `warn` entry with reason
+ * "unknown error" carries the failing image hashes; correlate them (and the
+ * timestamp) with the matching `batch`/`single` entry to recover the real
+ * cause.
+ *
+ * Best-effort: logging never throws and never breaks a describer turn. The
+ * directory is created lazily; a single size-based rotation (errors.log →
+ * errors.log.1) bounds growth to ~2× the cap.
+ */
+
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import { appendFileSync, existsSync, mkdirSync, renameSync, statSync } from "node:fs";
+import { join } from "node:path";
+
+/** Subdirectory under the pi agent dir holding this extension's logs. */
+const LOG_SUBDIR = "logs/pi-vision-watcher";
+/** Active log file name. */
+const LOG_FILENAME = "errors.log";
+
+/** A single error log is capped at this many bytes before it rotates to the
+ *  single `.1` backup, bounding total on-disk log to ~2× this. Generous enough
+ *  to retain a long troubleshooting history; small enough that a runaway
+ *  broken vision model (failures aren't cached, so they re-fire per turn)
+ *  can't fill the disk. */
+export const MAX_LOG_BYTES = 10 * 1024 * 1024;
+
+/** One structured error-log record (one JSONL line). */
+export interface VisionErrorLogEntry {
+  /** ISO 8601 timestamp (filled by {@link appendVisionError}). */
+  timestamp: string;
+  /** Where the entry originated: "batch"/"single" = describer failure source
+   *  (rich detail), "warn" = the user-facing warning (may carry reason
+   *  "unknown error" when the engine's shared error was already reset). */
+  phase: "batch" | "single" | "warn";
+  /** Human-readable failure reason (what `setLastError` received, or the warn
+   *  reason). "unknown error" in a `warn` entry means no describer error was
+   *  captured — correlate via {@link imageHashes} + {@link timestamp} with the
+   *  matching batch/single entry. */
+  reason: string;
+  /** Configured vision model ref ("provider/id"), or null if unset. */
+  visionModel: string | null;
+  /** Image hashes the failure covered (batch/single) or the warning named. */
+  imageHashes: string[];
+  /** Number of images involved. */
+  imageCount: number;
+  /** Describer stopReason, when the failure came from a completed response. */
+  stopReason?: string;
+  /** True when the describer's timeout fired. */
+  timedOut?: boolean;
+  /** The timeout budget in ms, when timedOut applies. */
+  timeoutMs?: number;
+  /** Provider error message from the response (response.errorMessage). */
+  errorMessage?: string;
+  /** Stack trace of a thrown error, when available. */
+  errorStack?: string;
+  /** Small config snapshot relevant to troubleshooting. */
+  config?: { maxTokens?: number; thinking: boolean; thinkingLevel: string };
+  /** The model being handed off to ("provider/id"), at the warn point. */
+  activeModel?: string;
+}
+
+/** Directory holding this extension's logs:
+ *  ~/.pi/agent/logs/pi-vision-watcher (overridable via $PI_CODING_AGENT_DIR). */
+export function getErrorLogDir(): string {
+  return join(getAgentDir(), LOG_SUBDIR);
+}
+
+/** Path to the active error log file. */
+export function getErrorLogPath(): string {
+  return join(getErrorLogDir(), LOG_FILENAME);
+}
+
+/** Serialize an entry to one JSONL line (JSON + terminating newline). Pure and
+ *  unit-testable independent of the disk. */
+export function formatErrorLogLine(entry: VisionErrorLogEntry): string {
+  return JSON.stringify(entry) + "\n";
+}
+
+/** Rotate the log when it has grown to at least `maxSize`: the current file
+ *  becomes the `.1` backup (overwriting any prior backup) and the active path
+ *  is cleared for a fresh log. Bounds total on-disk log to ~2× `maxSize`.
+ *  No-op when the file doesn't exist or is under the cap. Never throws. */
+export function rotateLogIfNeeded(path: string = getErrorLogPath(), maxSize: number = MAX_LOG_BYTES): void {
+  let size: number;
+  try {
+    size = statSync(path).size;
+  } catch {
+    return; // missing file — nothing to rotate
+  }
+  if (size < maxSize) return;
+  try {
+    renameSync(path, path + ".1");
+  } catch {
+    // rename failed (permissions, etc.) — leave the file; it'll keep growing
+    // until the condition clears. Logging must not throw.
+  }
+}
+
+/**
+ * Append one structured error entry to the log. Fills `timestamp`, ensures the
+ * log directory exists, rotates on size, and writes a JSONL line. Best-effort:
+ * swallows every error so a logging failure can NEVER break a describer turn
+ * (the describer calls this from its failure paths, where throwing would mask
+ * the real error and abort the batch).
+ */
+export function appendVisionError(input: Omit<VisionErrorLogEntry, "timestamp">): void {
+  try {
+    const dir = getErrorLogDir();
+    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+    const path = getErrorLogPath();
+    rotateLogIfNeeded(path);
+    const entry: VisionErrorLogEntry = { timestamp: new Date().toISOString(), ...input };
+    appendFileSync(path, formatErrorLogLine(entry), "utf8");
+  } catch {
+    // never break the describer on a logging failure
+  }
+}

package/src/image.ts

@@ -0,0 +1,331 @@
+/**
+ * Pure image IO utilities: MIME sniffing, dimension parsing, and resolving a
+ * pasted clipboard image file to the SAME {@link ExtractedImage} pi's `read`
+ * tool will emit — so a pre-warm's cache key matches the later `tool_result`'s
+ * key (no wasted vision call).
+ *
+ * Why this matters: pi's `read` tool runs `resizeImage` (on by default). For a
+ * small image (≤2000×2000 AND <4.5MB base64) the resize is a no-op that returns
+ * the raw input bytes unchanged — so our raw read matches. For an oversized
+ * image pi RE-ENCODES (Photon resize + possible JPEG@80), producing different
+ * bytes; pre-warming the raw file would then cache-miss at `tool_result` time
+ * and waste a vision call. {@link willBeResized} mirrors pi's threshold so we
+ * run the same {@link resolvePrewarmImage} pipeline and match the key in both
+ * cases.
+ *
+ * MIME sniffing is aligned with pi's `detectSupportedImageMimeType` (incl.
+ * rejecting animated PNG, which pi treats as a non-image → text) so our sniff
+ * agrees with pi on whether a file is an image at all.
+ */
+
+import { readFileSync, statSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { isAbsolute, join, sep } from "node:path";
+import crypto from "node:crypto";
+import type { ExtractedImage } from "./index.js";
+
+// pi's resize defaults (see @earendil-works/pi-coding-agent utils/image-resize).
+// The `read` tool returns raw input bytes unchanged when the image fits BOTH
+// the dimension limit AND the base64-payload size limit; otherwise it resizes.
+const RESIZE_MAX_WIDTH = 2000;
+const RESIZE_MAX_HEIGHT = 2000;
+const RESIZE_MAX_BYTES = 4.5 * 1024 * 1024; // base64 payload size
+
+const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
+
+/** Stable hash of an image's MIME + base64 data, used as the dataloader key. */
+export function imageHash(mimeType: string, data: string): string {
+  return crypto.createHash("sha256").update(`${mimeType}\x00${data}`).digest("hex").slice(0, 32);
+}
+
+function readUint32BE(buf: Buffer, offset: number): number {
+  return (
+    (buf[offset] ?? 0) * 0x1000000 +
+    ((buf[offset + 1] ?? 0) << 16) +
+    ((buf[offset + 2] ?? 0) << 8) +
+    (buf[offset + 3] ?? 0)
+  );
+}
+
+function readUint16BE(buf: Buffer, offset: number): number {
+  return ((buf[offset] ?? 0) << 8) + (buf[offset + 1] ?? 0);
+}
+
+function readUint16LE(buf: Buffer, offset: number): number {
+  return (buf[offset] ?? 0) + ((buf[offset + 1] ?? 0) << 8);
+}
+
+function startsWithAscii(buf: Buffer, offset: number, text: string): boolean {
+  for (let i = 0; i < text.length; i++) {
+    if (buf[offset + i] !== text.charCodeAt(i)) return false;
+  }
+  return true;
+}
+
+function isPng(buf: Buffer): boolean {
+  // IHDR chunk: length == 13 at offset 8, "IHDR" at offset 12.
+  return buf.length >= 16 && readUint32BE(buf, PNG_SIGNATURE.length) === 13 && startsWithAscii(buf, 12, "IHDR");
+}
+
+function isAnimatedPng(buf: Buffer): boolean {
+  // Scan chunks for "acTL" (animation control) before "IDAT". Matches pi's
+  // detectSupportedImageMimeType so an animated PNG is treated as a non-image
+  // (pi reads it as text) and we skip pre-warming it.
+  let offset = PNG_SIGNATURE.length;
+  while (offset + 8 <= buf.length) {
+    const chunkLength = readUint32BE(buf, offset);
+    const typeOffset = offset + 4;
+    if (startsWithAscii(buf, typeOffset, "acTL")) return true;
+    if (startsWithAscii(buf, typeOffset, "IDAT")) return false;
+    const next = offset + 8 + chunkLength + 4;
+    if (next <= offset || next > buf.length) return false;
+    offset = next;
+  }
+  return false;
+}
+
+/** Sniff an image MIME type from magic bytes, aligned with pi's
+ *  detectSupportedImageMimeType. Returns null for unsupported/animated PNG. */
+export function sniffImageMime(buf: Buffer): string | null {
+  if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {
+    return buf[3] === 0xf7 ? null : "image/jpeg";
+  }
+  if (buf.length >= 8 && PNG_SIGNATURE.every((b, i) => buf[i] === b)) {
+    return isPng(buf) && !isAnimatedPng(buf) ? "image/png" : null;
+  }
+  if (buf.length >= 3 && startsWithAscii(buf, 0, "GIF")) return "image/gif";
+  if (buf.length >= 12 && startsWithAscii(buf, 0, "RIFF") && startsWithAscii(buf, 8, "WEBP")) {
+    return "image/webp";
+  }
+  return null;
+}
+
+/** Parse an image's pixel dimensions from its headers (no decode). Returns null
+ *  for an unsupported/unparseable format. The dimension check is orientation-
+ *  invariant (`max(w,h) > 2000` is unchanged by EXIF rotation swaps), so it
+ *  agrees with pi's resize decision even for EXIF-oriented images. */
+export function imageDimensions(buf: Buffer, mimeType: string): { width: number; height: number } | null {
+  switch (mimeType) {
+    case "image/png": {
+      // IHDR: width (4 BE) at offset 16, height (4 BE) at offset 20.
+      if (buf.length < 24) return null;
+      return { width: readUint32BE(buf, 16), height: readUint32BE(buf, 20) };
+    }
+    case "image/gif": {
+      // Logical screen descriptor: width (2 LE) at 6, height (2 LE) at 8.
+      if (buf.length < 10) return null;
+      return { width: readUint16LE(buf, 6), height: readUint16LE(buf, 8) };
+    }
+    case "image/jpeg": {
+      return jpegDimensions(buf);
+    }
+    case "image/webp": {
+      return webpDimensions(buf);
+    }
+    default:
+      return null;
+  }
+}
+
+/** Scan JPEG segments for a SOF marker and read width/height. */
+function jpegDimensions(buf: Buffer): { width: number; height: number } | null {
+  if (buf.length < 4 || buf[0] !== 0xff || buf[1] !== 0xd8) return null;
+  let offset = 2;
+  while (offset + 9 <= buf.length) {
+    if (buf[offset] !== 0xff) return null;
+    // Skip fill bytes.
+    let marker = buf[offset + 1];
+    while (marker === 0xff && offset + 2 < buf.length) {
+      offset++;
+      marker = buf[offset + 1];
+    }
+    // Standalone markers (RSTn, SOI, EOI) have no length payload.
+    if (marker === 0xd8 || marker === 0xd9 || (marker >= 0xd0 && marker <= 0xd7)) {
+      offset += 2;
+      continue;
+    }
+    if (offset + 4 > buf.length) return null;
+    const segLen = readUint16BE(buf, offset + 2);
+    // SOF0–SOF15 (excluding RST/sof-defined non-SOF): C0–CF except C4 (DHT),
+    // C8 (JPG), CC (DAC).
+    if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+      if (offset + 9 > buf.length) return null;
+      const height = readUint16BE(buf, offset + 5);
+      const width = readUint16BE(buf, offset + 7);
+      return { width, height };
+    }
+    // SOS: image data follows, no more SOF.
+    if (marker === 0xda) return null;
+    offset += 2 + segLen;
+  }
+  return null;
+}
+
+/** Parse WEBP dimensions across VP8 (lossy), VP8L (lossless), VP8X (extended). */
+function webpDimensions(buf: Buffer): { width: number; height: number } | null {
+  if (buf.length < 30 || !startsWithAscii(buf, 0, "RIFF") || !startsWithAscii(buf, 8, "WEBP")) return null;
+  const fourcc = buf.subarray(12, 16).toString("ascii");
+  if (fourcc === "VP8 ") {
+    // Lossy: 3-byte frame tag at 20, 3-byte start code at 23, width (2 LE &
+    // 0x3FFF) at 26, height (2 LE & 0x3FFF) at 28.
+    return { width: readUint16LE(buf, 26) & 0x3fff, height: readUint16LE(buf, 28) & 0x3fff };
+  }
+  if (fourcc === "VP8L") {
+    // Lossless: 0x2F signature at 20, then 14-bit (width-1) + 14-bit (height-1)
+    // packed LSB-first across bytes 21–24.
+    if (buf[20] !== 0x2f) return null;
+    const val = (buf[21] ?? 0) | ((buf[22] ?? 0) << 8) | ((buf[23] ?? 0) << 16) | ((buf[24] ?? 0) << 24);
+    return { width: 1 + (val & 0x3fff), height: 1 + ((val >> 14) & 0x3fff) };
+  }
+  if (fourcc === "VP8X") {
+    // Extended: canvas width-1 (24-bit LE) at 24, height-1 (24-bit LE) at 27.
+    const w = 1 + ((buf[24] ?? 0) | ((buf[25] ?? 0) << 8) | ((buf[26] ?? 0) << 16));
+    const h = 1 + ((buf[27] ?? 0) | ((buf[28] ?? 0) << 8) | ((buf[29] ?? 0) << 16));
+    return { width: w, height: h };
+  }
+  return null;
+}
+
+/** Base64-encoded size of `byteLength` raw bytes (4/3 ratio, ceil). */
+function base64Size(byteLength: number): number {
+  return Math.ceil(byteLength / 3) * 4;
+}
+
+/** Whether pi's `read` tool would RESIZE (re-encode) this image. When false,
+ *  pi returns the raw input bytes unchanged — so a raw pre-warm matches the
+ *  `tool_result` key. When true, pi re-encodes — the caller must run the same
+ *  resize pipeline to match. Mirrors pi's resize threshold exactly. Unknown
+ *  dimensions default to `true` (force the resize path, which always matches). */
+export function willBeResized(buf: Buffer, mimeType: string): boolean {
+  const dims = imageDimensions(buf, mimeType);
+  if (!dims) return true;
+  return (
+    dims.width > RESIZE_MAX_WIDTH ||
+    dims.height > RESIZE_MAX_HEIGHT ||
+    base64Size(buf.length) >= RESIZE_MAX_BYTES
+  );
+}
+
+/** A resize function shaped like pi's `resizeImage` (injected so this module
+ *  stays free of the pi-coding-agent dependency and fully unit-testable). */
+export type ResizeFn = (
+  inputBytes: Uint8Array,
+  mimeType: string,
+) => Promise<{ data: string; mimeType: string } | null>;
+
+/** Resolve a clipboard image buffer to the SAME {@link ExtractedImage} pi's
+ *  `read` tool will emit, so the pre-warm's cache key matches the later
+ *  `tool_result`'s key (no wasted vision call):
+ *  - no-resize case (≤2000² & <4.5MB base64): return the raw bytes — pi's
+ *    no-resize path returns `inputBytes` unchanged.
+ *  - resize case: run the same `resize` pipeline (pi's `resizeImage`) and
+ *    return the re-encoded data — matches pi's resize path.
+ *  Returns null if resize failed (pi then emits no image block, so there is
+ *  nothing to pre-warm) or if the bytes aren't a supported image. */
+export async function resolvePrewarmImage(
+  buf: Buffer,
+  mimeType: string,
+  resize: ResizeFn,
+): Promise<ExtractedImage | null> {
+  if (!willBeResized(buf, mimeType)) {
+    return { data: buf.toString("base64"), mimeType };
+  }
+  const resized = await resize(buf, mimeType);
+  if (!resized) return null;
+  return { data: resized.data, mimeType: resized.mimeType };
+}
+
+/** Read an image file into its raw buffer + sniffed MIME. Returns null if the
+ *  file can't be read or isn't a supported image (aligned with pi's sniff). */
+export function readImageBuffer(filePath: string): { buf: Buffer; mimeType: string } | null {
+  try {
+    const buf = readFileSync(filePath);
+    const mimeType = sniffImageMime(buf);
+    if (!mimeType) return null;
+    return { buf, mimeType };
+  } catch {
+    return null;
+  }
+}
+
+/** Max raw file size the omitted-image recovery will re-read. pi's `read`
+ *  already read the file; this bounds the handoff's re-read so a pathologically
+ *  huge file doesn't double the memory. 20MB covers the most generous vision
+ *  model inline-image limit; a larger file would be rejected by the vision
+ *  model anyway. */
+export const MAX_RECOVER_IMAGE_BYTES = 20 * 1024 * 1024;
+
+/** pi core's `read` tool emits this text note (with NO image block) when it
+ *  detected an image but `processImage` failed — Photon/WASM unavailable,
+ *  decode failure, convert-to-PNG failure, or couldn't resize below the inline
+ *  size limit. The handoff's image-block path never sees these (no image
+ *  block), so the image goes undescribed and the model is told the image was
+ *  "omitted". This detects that note so the tool_result handler can re-read the
+ *  raw file and describe its bytes directly (the vision model decodes them —
+ *  no Photon needed). */
+export function isOmittedImageNote(text: string): boolean {
+  return text.includes("Read image file [") && text.includes("[Image omitted:");
+}
+
+/** Read an image file for the omitted-image recovery, bounded by
+ *  {@link MAX_RECOVER_IMAGE_BYTES}. Returns null if the file can't be read, is
+ *  too large, or isn't a supported image (aligned with pi's sniff — APNG and
+ *  unsupported formats are rejected, since the vision model can't decode them
+ *  either). */
+export function readImageBufferBounded(filePath: string): { buf: Buffer; mimeType: string } | null {
+  try {
+    const stat = statSync(filePath);
+    if (stat.size > MAX_RECOVER_IMAGE_BYTES) return null;
+  } catch {
+    return null;
+  }
+  return readImageBuffer(filePath);
+}
+
+/** Regex matching any pasted temp-image file path in the prompt text,
+ *  regardless of which paste mechanism wrote it (pi-clipboard,
+ *  localterm-paste, and others). We match any non-whitespace token ending in
+ *  a supported image extension, then confine to the OS temp directory below.
+ *  The confinement, not the filename pattern, is the security boundary. */
+const PASTED_IMAGE_PATH_RE = /(\S+\.(?:png|jpe?g|gif|webp))/gi;
+const URL_RE = new RegExp('^[a-z][a-z0-9+.-]*://', 'i');
+const LEADING_WRAP_RE = new RegExp('^[^A-Za-z0-9_/@.~-]+');
+
+/** Extract pasted image file paths from a prompt. Confined to the OS temp
+ *  directory so an attacker-crafted prompt can't trick the extension into
+ *  reading arbitrary files — only temp-dir image files qualify (this covers
+ *  every paste mechanism: pi-clipboard, localterm-paste, and others). URLs
+ *  (http(s)://, file://, ...) are skipped so they aren't joined into the temp
+ *  dir and mistaken for local files; leading wrapping chars (parentheses,
+ *  quotes) are stripped so quoted/parenthesized paths still resolve. */
+export function findPastedImagePaths(prompt: string): string[] {
+  const tmp = tmpdir();
+  const paths = new Set<string>();
+  for (const m of prompt.matchAll(PASTED_IMAGE_PATH_RE)) {
+    const token = m[1];
+    if (!token) continue;
+    if (URL_RE.test(token)) continue;
+    const p = token.replace(LEADING_WRAP_RE, "");
+    const abs = isAbsolute(p) ? p : join(tmp, p);
+    // Ensure the resolved candidate stays inside the temp directory.
+    if (abs.startsWith(tmp + sep) || abs === tmp) paths.add(abs);
+  }
+  return [...paths];
+}
+
+/** Diff the pasted image paths in `text` against `known`, returning those
+ *  that are newly appeared (not yet seen). Used by the paste-time prewarm
+ *  editor to prewarm only newly-pasted paths on each text change, without
+ *  re-reading already-seen ones. `findPastedImagePaths` already dedups
+ *  within a single text, so each returned path is unique and seen at most
+ *  once. Returns [] when `text` holds no pasted paths (e.g. ordinary
+ *  typing) — so the editor's per-keystroke cost when the opt-in is on is one
+ *  regex scan that almost always yields nothing. */
+export function diffPrewarmPaths(text: string, known: Set<string>): string[] {
+  const paths = findPastedImagePaths(text);
+  const newPaths: string[] = [];
+  for (const p of paths) {
+    if (!known.has(p)) newPaths.push(p);
+  }
+  return newPaths;
+}