@zocomputer/agent-sdk 0.5.0

package/src/walk.ts

@@ -0,0 +1,112 @@
+import { readFileSync, readdirSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+import ignore, { type Ignore } from "ignore";
+
+// The *fallback* candidate source — `listGitFiles` (git's own ignore
+// semantics) is preferred — so it honors `.gitignore` files itself (root,
+// nested, and ancestors between `base` and `root` for scoped walks) rather
+// than duplicating the repo's ignore list here. The hardcoded set is only a
+// safety net for trees with no `.gitignore` at all: VCS stores plus the one
+// universally-huge dependency dir. Exported so the sandbox search backend
+// (../sandbox-io.ts) applies the same unconditional skips — the two backends
+// must not drift on what a search can wander into.
+export const ALWAYS_IGNORED = new Set([".git", ".jj", ".hg", ".svn", "node_modules"]);
+
+// One .gitignore's matcher, scoped to the directory that holds it. `prefix` is
+// that directory relative to the walk base (forward-slash, "" for the base
+// itself); patterns only apply to paths under it. Matching is delegated to the
+// `ignore` npm package (the battle-tested matcher ESLint/Prettier use), so we
+// get the full gitignore spec — escapes and all — instead of maintaining one.
+type ScopedIgnore = { prefix: string; matcher: Ignore };
+
+// Read `<absDir>/.gitignore` into a scope. Null when absent or unreadable —
+// callers just skip pushing a scope.
+function loadGitignore(absDir: string, prefix: string): ScopedIgnore | null {
+  let text: string;
+  try {
+    text = readFileSync(join(absDir, ".gitignore"), "utf8");
+  } catch {
+    return null;
+  }
+  return { prefix, matcher: ignore().add(text) };
+}
+
+// Git-style decision for a base-relative, forward-slash path: evaluate every
+// scope whose directory contains the path (callers pass them outer→inner) and
+// let the innermost explicit verdict — ignore or `!` re-include — win. A
+// trailing slash tells the matcher "directory", so dir-only patterns (`out/`)
+// apply. Files under an ignored directory never get here — the walker prunes
+// the directory, matching git's "can't re-include under an excluded dir" rule.
+function isIgnored(scopes: readonly ScopedIgnore[], relPath: string, isDir: boolean): boolean {
+  let ignored = false;
+  for (const scope of scopes) {
+    let sub: string;
+    if (scope.prefix === "") sub = relPath;
+    else if (relPath.startsWith(`${scope.prefix}/`)) sub = relPath.slice(scope.prefix.length + 1);
+    else continue;
+    const verdict = scope.matcher.test(isDir ? `${sub}/` : sub);
+    if (verdict.ignored) ignored = true;
+    else if (verdict.unignored) ignored = false;
+  }
+  return ignored;
+}
+
+type Frame = {
+  dir: string; // absolute
+  rel: string; // base-relative, forward-slash ("" for base itself)
+  scopes: ScopedIgnore[]; // .gitignore scopes from ancestors (not `dir` itself)
+};
+
+// Depth-first walk yielding `base`-relative, forward-slash file paths. glob
+// and grep run in-process (no ripgrep dependency), so this is deliberately
+// modest — it exists to serve interactive searches, not to index the world.
+// `base` defaults to `root` for whole-workspace walks; a scoped walk passes
+// the subtree as `root` and the workspace as `base` so paths stay
+// base-relative and ancestor .gitignores still apply.
+export function* walkFiles(root: string, base: string = root): Generator<string> {
+  // Ancestor .gitignore chain: when walking a subtree, patterns from `base`
+  // down to `root`'s parent still apply to it (git semantics). `root`'s own
+  // .gitignore loads when its frame is processed, like every other dir.
+  const scopes: ScopedIgnore[] = [];
+  const relRoot = relative(base, root).split(sep).join("/");
+  if (relRoot !== "" && !relRoot.startsWith("..")) {
+    let absDir = base;
+    let prefix = "";
+    const own = loadGitignore(base, "");
+    if (own !== null) scopes.push(own);
+    const segments = relRoot.split("/");
+    for (const segment of segments.slice(0, -1)) {
+      absDir = join(absDir, segment);
+      prefix = prefix === "" ? segment : `${prefix}/${segment}`;
+      const scope = loadGitignore(absDir, prefix);
+      if (scope !== null) scopes.push(scope);
+    }
+  }
+
+  const stack: Frame[] = [{ dir: root, rel: relRoot === "." ? "" : relRoot, scopes }];
+  while (stack.length > 0) {
+    const frame = stack.pop();
+    if (frame === undefined) break;
+    let entries;
+    try {
+      entries = readdirSync(frame.dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    const own = loadGitignore(frame.dir, frame.rel);
+    const active = own === null ? frame.scopes : [...frame.scopes, own];
+    for (const entry of entries) {
+      const rel = frame.rel === "" ? entry.name : `${frame.rel}/${entry.name}`;
+      if (entry.isDirectory()) {
+        if (ALWAYS_IGNORED.has(entry.name)) continue;
+        if (isIgnored(active, rel, true)) continue;
+        stack.push({ dir: join(frame.dir, entry.name), rel, scopes: active });
+      } else if (entry.isFile()) {
+        // In a linked worktree `.git` is a file (a gitdir pointer), not a dir.
+        if (entry.name === ".git") continue;
+        if (isIgnored(active, rel, false)) continue;
+        yield rel;
+      }
+    }
+  }
+}

package/src/watch-output.ts

@@ -0,0 +1,104 @@
+// Pure output watcher for background commands: feed it output chunks, get back
+// the complete lines that match a regex — debounced and capped so a chatty
+// pattern can't flood the session with notification turns. The effect side
+// (posting a park notification per match batch) lives with the tools; this
+// module is the testable decision core.
+
+export interface OutputWatcherOptions {
+  /** Regex source matched against complete output lines. */
+  pattern: string;
+  /** Minimum ms between match batches; matches inside the window drop. */
+  debounceMs?: number | undefined;
+  /** Max match batches over the watcher's lifetime; later matches drop. */
+  maxNotifications?: number;
+  /** Clock, injectable for tests. */
+  now?: () => number;
+}
+
+export interface OutputWatcher {
+  /**
+   * Consume an output chunk (any framing — watcher handles line buffering).
+   * Returns the matching complete lines that clear debounce/cap, or null.
+   */
+  feed(chunk: string): string[] | null;
+  /** Flush the unterminated tail as a final line (call when the command ends). */
+  flush(): string[] | null;
+}
+
+export const DEFAULT_WATCH_DEBOUNCE_MS = 5_000;
+export const DEFAULT_MAX_WATCH_NOTIFICATIONS = 5;
+
+/**
+ * Build a watcher. An invalid regex throws here — at tool-input time, where
+ * the error surfaces as a normal tool failure the model can correct.
+ */
+export function createOutputWatcher(options: OutputWatcherOptions): OutputWatcher {
+  const regex = new RegExp(options.pattern);
+  const debounceMs = options.debounceMs ?? DEFAULT_WATCH_DEBOUNCE_MS;
+  const maxNotifications = options.maxNotifications ?? DEFAULT_MAX_WATCH_NOTIFICATIONS;
+  const now = options.now ?? Date.now;
+
+  let buffer = "";
+  let notifications = 0;
+  let lastNotifiedAt: number | null = null;
+
+  function emit(lines: string[]): string[] | null {
+    const matches = lines.filter((line) => regex.test(line));
+    if (matches.length === 0) return null;
+    if (notifications >= maxNotifications) return null;
+    const at = now();
+    if (lastNotifiedAt !== null && at - lastNotifiedAt < debounceMs) return null;
+    notifications += 1;
+    lastNotifiedAt = at;
+    return matches;
+  }
+
+  return {
+    feed(chunk) {
+      buffer += chunk;
+      const parts = buffer.split("\n");
+      // The last part is an unterminated tail; keep buffering it.
+      buffer = parts.pop() ?? "";
+      return emit(parts);
+    },
+    flush() {
+      if (buffer.length === 0) return null;
+      const tail = buffer;
+      buffer = "";
+      // Bypass debounce for the final output: a command ending is the signal.
+      const matches = [tail].filter((line) => regex.test(line));
+      if (matches.length === 0) return null;
+      if (notifications >= maxNotifications) return null;
+      notifications += 1;
+      lastNotifiedAt = now();
+      return matches;
+    },
+  };
+}
+
+/** The message text for a watcher match, complete and self-describing. */
+export function formatWatchNotification(opts: {
+  taskId: string;
+  label: string;
+  reason: string;
+  lines: readonly string[];
+}): string {
+  return (
+    `Background task ${opts.taskId} (${opts.label}) — ${opts.reason}. ` +
+    `Output matched your watch pattern:\n${opts.lines.join("\n")}`
+  );
+}
+
+/** The message text for a completion notice (run_async notify_on_complete). */
+export function formatCompletionNotification(opts: {
+  taskId: string;
+  label: string;
+  status: "done" | "error";
+  error?: string;
+}): string {
+  const outcome =
+    opts.status === "done"
+      ? "finished"
+      : `failed${opts.error ? `: ${opts.error}` : ""}`;
+  return `Background task ${opts.taskId} (${opts.label}) ${outcome}. Call await_task to collect its result.`;
+}

package/src/web-fetch.ts

@@ -0,0 +1,324 @@
+import { Parser } from "htmlparser2";
+import { parseHTML } from "linkedom";
+import TurndownService from "turndown";
+import {
+  buildContentCollapseNote,
+  buildMetadataHeader,
+  extractMainContent,
+  looksLikeRawHtmlOutput,
+  visibleTextLength,
+} from "./web-page";
+
+// The fetch + render core for the `webfetch` tool. Conventions follow the
+// converged Claude Code / opencode / eve standard: browser UA with per-format
+// Accept q-values, markdown as the default rendering for HTML, a one-shot
+// honest-UA retry when Cloudflare's bot check rejects the browser UA (TLS
+// fingerprint mismatch), and a hard response-size cap. HTML reduces to its
+// main content with a metadata header (./web-page.ts) before conversion. Kept
+// framework-free (no eve imports) with `fetchImpl` injectable so tests never
+// touch the network; the tool wrapper owns bounding/spill and the attachment
+// contract.
+
+/** Hard cap on the fetched body; larger responses error rather than truncate. */
+export const WEB_FETCH_MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
+export const WEB_FETCH_DEFAULT_TIMEOUT_SECONDS = 30;
+/** PDFs are routinely tens of MB behind slow servers; give them a longer default. */
+export const WEB_FETCH_PDF_DEFAULT_TIMEOUT_SECONDS = 60;
+export const WEB_FETCH_MAX_TIMEOUT_SECONDS = 120;
+
+export type WebFetchFormat = "markdown" | "text" | "html";
+
+// A real browser UA gets past naive bot filters; sites that inspect the TLS
+// fingerprint (Cloudflare) see through it, hence the honest-UA retry below.
+export const BROWSER_USER_AGENT =
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36";
+export const FALLBACK_USER_AGENT = "agent-sdk";
+
+function acceptHeader(format: WebFetchFormat): string {
+  switch (format) {
+    case "markdown":
+      return "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1";
+    case "text":
+      return "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1";
+    case "html":
+      return "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1";
+  }
+}
+
+export function buildWebFetchHeaders(
+  format: WebFetchFormat,
+  userAgent: string = BROWSER_USER_AGENT,
+): Record<string, string> {
+  return {
+    Accept: acceptHeader(format),
+    "Accept-Language": "en-US,en;q=0.9",
+    "User-Agent": userAgent,
+  };
+}
+
+export function convertHtmlToMarkdown(html: string): string {
+  const turndown = new TurndownService({
+    headingStyle: "atx",
+    hr: "---",
+    bulletListMarker: "-",
+    codeBlockStyle: "fenced",
+    emDelimiter: "*",
+  });
+  turndown.remove(["script", "style", "meta", "link"]);
+  // Parse with linkedom explicitly instead of handing turndown the string:
+  // given a string, turndown sniffs a global DOMParser at import time and
+  // prefers it, and DOM-polyfilled hosts (e.g. a happy-dom test preload)
+  // produce a document it can't traverse — output silently collapses to "".
+  // The wrapper element mirrors turndown's own string path (elements arranged
+  // in one root instead of head/body).
+  const { document } = parseHTML(`<x-turndown id="turndown-root">${html}</x-turndown>`);
+  const root = document.getElementById("turndown-root");
+  if (root === null) throw new Error("unreachable: the turndown-root wrapper always parses");
+  // linkedom's nodes implement the DOM surface turndown needs but aren't
+  // lib.dom types (we compile without the DOM lib) — the one sanctioned cast.
+  return turndown.turndown(root as unknown as Parameters<TurndownService["turndown"]>[0]);
+}
+
+const SKIPPED_HTML_TAGS = ["script", "style", "noscript", "iframe", "object", "embed"];
+
+export function extractTextFromHtml(html: string): string {
+  let text = "";
+  const skipStack: string[] = [];
+  const parser = new Parser({
+    onopentag(name: string) {
+      if (skipStack.length > 0 || SKIPPED_HTML_TAGS.includes(name)) {
+        skipStack.push(name);
+      }
+    },
+    ontext(input: string) {
+      if (skipStack.length === 0) text += input;
+    },
+    onclosetag(name: string) {
+      if (skipStack.length > 0 && skipStack[skipStack.length - 1] === name) {
+        skipStack.pop();
+      }
+    },
+  });
+  parser.write(html);
+  parser.end();
+  // Collapse the whitespace runs left behind by stripped markup.
+  return text
+    .split("\n")
+    .map((line) => line.replace(/[ \t]+/g, " ").trim())
+    .join("\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+export function isHtmlContentType(contentType: string): boolean {
+  const mime = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
+  return mime === "text/html" || mime === "application/xhtml+xml";
+}
+
+/**
+ * Sniff content to detect HTML even when the Content-Type header is wrong or
+ * missing. Checks for common HTML markers at the start of the content.
+ */
+export function looksLikeHtml(content: string): boolean {
+  const trimmed = content.trimStart().slice(0, 512).toLowerCase();
+  return (
+    trimmed.startsWith("<!doctype html") ||
+    trimmed.startsWith("<html") ||
+    /^<(!--|head|body|div|p|h[1-6]|span|a|ul|ol|li|table|script|style)\b/.test(trimmed)
+  );
+}
+
+export interface RenderedWebText {
+  readonly text: string;
+  /** Honest-failure signal (content collapse, leftover raw HTML); absent when the render looks healthy. */
+  readonly note?: string;
+}
+
+/**
+ * Render fetched text per the requested format; non-HTML (and format "html")
+ * passes through untouched. For markdown/text, the page first reduces to its
+ * main content with a metadata header (falling back to the whole document
+ * when extraction can't find one), and a render that comes back suspiciously
+ * empty or tag-heavy carries a note saying so.
+ */
+export function renderWebText(
+  content: string,
+  contentType: string,
+  format: WebFetchFormat,
+  url: string,
+): RenderedWebText {
+  // Route by sniffed content in addition to Content-Type, since servers lie.
+  const isHtml = isHtmlContentType(contentType) || looksLikeHtml(content);
+  if (!isHtml || format === "html") return { text: content };
+  const page = extractMainContent(content, url);
+  const bodyHtml = page === null ? content : page.contentHtml;
+  const converted =
+    format === "markdown" ? convertHtmlToMarkdown(bodyHtml) : extractTextFromHtml(bodyHtml);
+  const header = page === null ? null : buildMetadataHeader(page);
+  const text = header === null ? converted : `${header}\n\n${converted}`;
+  const notes: string[] = [];
+  const collapse = buildContentCollapseNote({
+    url,
+    renderedChars: converted.trim().length,
+    htmlChars: content.length,
+    documentTextChars: visibleTextLength(content),
+  });
+  if (collapse !== null) notes.push(collapse);
+  if (format === "markdown" && looksLikeRawHtmlOutput(converted)) {
+    notes.push(
+      'The markdown conversion left substantial raw HTML in the output — treat it as partially converted, or re-fetch with format "html" for the true page.',
+    );
+  }
+  return notes.length === 0 ? { text } : { text, note: notes.join(" ") };
+}
+
+/**
+ * The slice of `fetch` this module needs. Narrower than `typeof fetch` (Bun
+ * adds a `preconnect` property to the global) so test stubs stay plain
+ * functions.
+ */
+export type FetchLike = (url: string, init?: RequestInit) => Promise<Response>;
+
+export interface FetchedWebResource {
+  readonly body: Buffer;
+  /** Raw `content-type` header value; empty string when absent. */
+  readonly contentType: string;
+  /** URL after redirects when the fetch implementation exposes it; else the request URL. */
+  readonly finalUrl: string;
+}
+
+/** Throws with a clear message for anything that isn't a plain http(s) URL. */
+export function assertHttpUrl(url: string): void {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`Not a valid URL: ${url}`);
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error("URL must start with http:// or https://");
+  }
+}
+
+/**
+ * GET a URL with the standard headers, the Cloudflare-challenge retry, and the
+ * response-size cap. Throws on non-2xx, oversized, or timed-out responses.
+ */
+export async function fetchWebResource(opts: {
+  url: string;
+  format: WebFetchFormat;
+  timeoutMs: number;
+  /**
+   * Deadline (from request start) to extend to when the response headers
+   * reveal a PDF the request URL didn't — a redirect to `.pdf` or an
+   * extensionless URL served as `application/pdf`. Omit when the caller set
+   * an explicit timeout, which always wins.
+   */
+  pdfTimeoutMs?: number;
+  fetchImpl?: FetchLike;
+}): Promise<FetchedWebResource> {
+  const { url, format, timeoutMs, pdfTimeoutMs } = opts;
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  assertHttpUrl(url);
+  // One deadline shared by both attempts — the retry doesn't reset the clock.
+  // An AbortController (not AbortSignal.timeout) so the deadline can extend
+  // once the headers show the resource is a PDF.
+  const controller = new AbortController();
+  const startedAt = Date.now();
+  let deadlineMs = timeoutMs;
+  let timer = setTimeout(() => controller.abort(), deadlineMs);
+  const timedOut = () => new Error(`Request timed out after ${deadlineMs / 1000}s: ${url}`);
+  try {
+    const headers = buildWebFetchHeaders(format);
+    let response: Response;
+    try {
+      response = await fetchImpl(url, { headers, signal: controller.signal });
+      if (
+        response.status === 403 &&
+        response.headers.get("cf-mitigated") === "challenge"
+      ) {
+        response = await fetchImpl(url, {
+          headers: { ...headers, "User-Agent": FALLBACK_USER_AGENT },
+          signal: controller.signal,
+        });
+      }
+    } catch (error) {
+      if (controller.signal.aborted) throw timedOut();
+      throw error;
+    }
+    if (!response.ok) {
+      throw new Error(`Request failed with status ${response.status}: ${url}`);
+    }
+    const contentType = response.headers.get("content-type") ?? "";
+    // Some fetch stubs/implementations leave Response.url empty; fall back to
+    // the request URL so callers can always compare for cross-host redirects.
+    const finalUrl = response.url === "" ? url : response.url;
+    // The request URL can hide that the resource is a PDF; once the headers
+    // say so, the body download gets the PDF deadline instead of the page one.
+    if (
+      pdfTimeoutMs !== undefined &&
+      pdfTimeoutMs > deadlineMs &&
+      responseLooksLikePdf(contentType, finalUrl)
+    ) {
+      clearTimeout(timer);
+      deadlineMs = pdfTimeoutMs;
+      const remaining = Math.max(deadlineMs - (Date.now() - startedAt), 0);
+      timer = setTimeout(() => controller.abort(), remaining);
+    }
+    const contentLength = response.headers.get("content-length");
+    if (contentLength !== null && Number.parseInt(contentLength, 10) > WEB_FETCH_MAX_RESPONSE_BYTES) {
+      throw new Error(
+        `Response too large (content-length exceeds the ${WEB_FETCH_MAX_RESPONSE_BYTES}-byte limit): ${url}`,
+      );
+    }
+    let arrayBuffer: ArrayBuffer;
+    try {
+      arrayBuffer = await response.arrayBuffer();
+    } catch (error) {
+      if (controller.signal.aborted) throw timedOut();
+      throw error;
+    }
+    if (arrayBuffer.byteLength > WEB_FETCH_MAX_RESPONSE_BYTES) {
+      throw new Error(
+        `Response too large (exceeds the ${WEB_FETCH_MAX_RESPONSE_BYTES}-byte limit): ${url}`,
+      );
+    }
+    return {
+      body: Buffer.from(arrayBuffer),
+      contentType,
+      finalUrl,
+    };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+function responseLooksLikePdf(contentType: string, finalUrl: string): boolean {
+  const mime = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? "";
+  return mime === "application/pdf" || urlLooksLikePdf(finalUrl);
+}
+
+/**
+ * Clamp the model-supplied timeout (seconds) into the allowed range, in ms.
+ * An explicit timeout always wins; without one, `.pdf` URLs default higher
+ * than pages (models rarely think to raise the timeout for a slow PDF).
+ */
+export function resolveWebFetchTimeoutMs(
+  timeoutSeconds: number | undefined,
+  url?: string,
+): number {
+  const fallback = urlLooksLikePdf(url)
+    ? WEB_FETCH_PDF_DEFAULT_TIMEOUT_SECONDS
+    : WEB_FETCH_DEFAULT_TIMEOUT_SECONDS;
+  const seconds = timeoutSeconds ?? fallback;
+  return Math.min(Math.max(seconds, 1), WEB_FETCH_MAX_TIMEOUT_SECONDS) * 1000;
+}
+
+function urlLooksLikePdf(url: string | undefined): boolean {
+  if (url === undefined) return false;
+  try {
+    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return false;
+  }
+}