@zocomputer/agent-sdk 0.5.0

package/src/web-page.ts

@@ -0,0 +1,179 @@
+import Defuddle from "defuddle";
+import { parseHTML } from "linkedom";
+
+// Main-content extraction and honest-failure signals for fetched web pages.
+// Defuddle (Obsidian Web Clipper's reader-mode engine) prunes nav/footer/
+// cookie-banner boilerplate and reads title/author/date metadata in the same
+// pass; linkedom supplies the server-side DOM it runs on. Pure and
+// framework-free: `renderWebText` (../web-fetch.ts) orchestrates, this module
+// owns the extraction, the metadata header, and the collapse/domain notes.
+
+export interface ExtractedWebPage {
+  /** Cleaned main-content HTML, ready for markdown/text conversion. */
+  readonly contentHtml: string;
+  readonly title: string | null;
+  readonly author: string | null;
+  readonly published: string | null;
+  readonly site: string | null;
+}
+
+const asField = (value: unknown): string | null => {
+  if (typeof value !== "string") return null;
+  const trimmed = value.trim();
+  return trimmed === "" ? null : trimmed;
+};
+
+/** Visible-ish text length: script/style/noscript bodies and tags stripped. */
+export function visibleTextLength(html: string): number {
+  return html
+    .replace(/<(script|style|noscript)\b[\s\S]*?<\/\1\s*>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/\s+/g, " ")
+    .trim().length;
+}
+
+/**
+ * Extract the page's main content and metadata. Returns null — meaning
+ * "convert the whole document instead" — when extraction throws, comes back
+ * empty, or prunes so aggressively it lost most of a modest page (better to
+ * show boilerplate than to silently drop real content).
+ */
+export function extractMainContent(html: string, url: string): ExtractedWebPage | null {
+  let parsed: ReturnType<Defuddle["parse"]>;
+  try {
+    const { document } = parseHTML(html);
+    // linkedom's document isn't lib.dom's Document (we compile without the DOM
+    // lib); defuddle only needs the query/traversal surface linkedom provides,
+    // so this is the one sanctioned cast at the seam.
+    parsed = new Defuddle(document as unknown as ConstructorParameters<typeof Defuddle>[0], {
+      url,
+    }).parse();
+  } catch {
+    return null;
+  }
+  const contentHtml = typeof parsed.content === "string" ? parsed.content : "";
+  const extractedLength = visibleTextLength(contentHtml);
+  if (extractedLength === 0) return null;
+  // Over-pruning guard: a small extraction from a page with several times more
+  // visible text means the cleaner guessed wrong about what's boilerplate.
+  if (extractedLength < 500 && extractedLength * 4 < visibleTextLength(html)) return null;
+  return {
+    contentHtml,
+    title: asField(parsed.title),
+    author: asField(parsed.author),
+    published: asField(parsed.published),
+    site: asField(parsed.site),
+  };
+}
+
+/**
+ * A markdown header for the extracted page: the title as `#`, then one
+ * blockquote line with whichever of author/site/published exist. Null when
+ * there's nothing to say.
+ */
+export function buildMetadataHeader(
+  page: Pick<ExtractedWebPage, "title" | "author" | "published" | "site">,
+): string | null {
+  const byline = [
+    page.author === null ? null : `By ${page.author}`,
+    page.site,
+    page.published === null ? null : `Published ${page.published}`,
+  ]
+    .filter((part): part is string => part !== null)
+    .join(" · ");
+  const lines: string[] = [];
+  if (page.title !== null) lines.push(`# ${page.title}`);
+  if (byline !== "") lines.push(`> ${byline}`);
+  return lines.length === 0 ? null : lines.join("\n\n");
+}
+
+// Domains whose pages are client-rendered or login-walled: a fetch sees the
+// empty shell, so the collapse note names the reason and a way forward.
+const JS_RENDERED_DOMAIN_HINTS: readonly {
+  readonly suffixes: readonly string[];
+  readonly hint: string;
+}[] = [
+  {
+    suffixes: ["x.com", "twitter.com"],
+    hint: "X (Twitter) renders posts with client-side JavaScript and blocks anonymous scraping, so a plain fetch can't see the post.",
+  },
+  {
+    suffixes: ["reddit.com"],
+    hint: "Reddit renders client-side — fetch the JSON view (append .json to the post URL) or the same path on old.reddit.com instead.",
+  },
+  {
+    suffixes: ["linkedin.com"],
+    hint: "LinkedIn requires login; anonymous fetches get a wall instead of the content.",
+  },
+  {
+    suffixes: ["instagram.com", "threads.net", "tiktok.com", "facebook.com"],
+    hint: "This site renders client-side and gates content behind login, so a plain fetch can't see it.",
+  },
+  {
+    suffixes: ["youtube.com", "youtu.be"],
+    hint: "YouTube pages are a client-rendered player; the video itself isn't fetchable as text.",
+  },
+];
+
+/** The domain-specific reason a fetch of this URL comes back empty, if known. */
+export function jsRenderedDomainHint(url: string): string | null {
+  let hostname: string;
+  try {
+    hostname = new URL(url).hostname.toLowerCase();
+  } catch {
+    return null;
+  }
+  for (const { suffixes, hint } of JS_RENDERED_DOMAIN_HINTS) {
+    for (const suffix of suffixes) {
+      if (hostname === suffix || hostname.endsWith(`.${suffix}`)) return hint;
+    }
+  }
+  return null;
+}
+
+/** Rendered text this small from HTML this big means the content never made it. */
+export const COLLAPSE_RENDERED_MAX_CHARS = 200;
+export const COLLAPSE_HTML_MIN_CHARS = 2_000;
+
+/**
+ * When a substantial HTML page renders to almost nothing, say so — the model
+ * would otherwise read the near-empty result as "the page has no content"
+ * instead of "the fetch couldn't see the content".
+ *
+ * `documentTextChars` — the whole document's visible text, before
+ * main-content extraction — is what separates the two ways a render comes
+ * back short: a JS shell or login wall has almost no visible text anywhere
+ * in the document, while a legitimately short article sits inside a template
+ * whose nav/footer boilerplate still carries plenty. Only the former earns
+ * the note.
+ */
+export function buildContentCollapseNote(opts: {
+  url: string;
+  renderedChars: number;
+  htmlChars: number;
+  documentTextChars: number;
+}): string | null {
+  if (opts.renderedChars >= COLLAPSE_RENDERED_MAX_CHARS) return null;
+  if (opts.documentTextChars >= COLLAPSE_RENDERED_MAX_CHARS) return null;
+  if (opts.htmlChars < COLLAPSE_HTML_MIN_CHARS) return null;
+  const base = `The page produced almost no readable text (${opts.renderedChars} chars from ${opts.htmlChars} chars of HTML) — its content likely renders with client-side JavaScript or sits behind a login or bot wall.`;
+  const hint = jsRenderedDomainHint(opts.url);
+  return hint === null ? base : `${base} ${hint}`;
+}
+
+/**
+ * Detect a markdown conversion that left the output substantially raw HTML
+ * (e.g. markup turndown couldn't traverse), so the result can say so instead
+ * of passing tag soup off as a clean read.
+ */
+export function looksLikeRawHtmlOutput(rendered: string): boolean {
+  if (rendered.length === 0) return false;
+  // `[^<>]` (not `[^>]`) bounds the attribute scan to a single tag: an
+  // unterminated `<tag …` fails at the next `<` instead of scanning to the
+  // end of the string, which kept this linear on hostile input (a fetched
+  // page of `"<a ".repeat(n)` was polynomial — CodeQL js/polynomial-redos).
+  const tags = rendered.match(/<\/?[a-z][a-z0-9-]*(?:\s[^<>]*)?>/gi);
+  if (tags === null || tags.length < 10) return false;
+  const tagChars = tags.reduce((sum, tag) => sum + tag.length, 0);
+  return tagChars / rendered.length > 0.1;
+}

package/src/workspace-io.ts

@@ -0,0 +1,225 @@
+import { mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { globToRegExp } from "./glob-match";
+import { listGitFiles } from "./list-files";
+import { readTextForSearch } from "./read-text";
+import { walkFiles } from "./walk";
+import { relativizeWithin } from "./workspace";
+
+// The I/O seam under every stdlib file tool. Tools do path math through
+// `Workspace` (pure) and effects through a `WorkspaceIO` — so the same
+// read/edit/write/glob/grep factories run against the harness process's own
+// disk (rib, the coder example) or a remote eve sandbox (hosted agents, where
+// the eve process and the workspace are different machines — see
+// ./sandbox-io.ts). The local backend lives here; it is exactly the node:fs +
+// `git ls-files` + in-process-scan behavior the tools shipped with.
+//
+// A sandbox arrives per tool call (eve's `ctx.getSandbox()`), not per factory
+// build, so tools hold a `WorkspaceIoProvider` — `(ctx) => WorkspaceIO` — and
+// resolve it at the top of each execute. The local provider ignores `ctx`
+// entirely; the tools' no-session usability (direct factory calls in tests)
+// is preserved.
+
+/** Stat result for one path. */
+export interface IoStat {
+  readonly isFile: boolean;
+  readonly size: number;
+  readonly mtimeMs: number;
+}
+
+/** One matching line from a content search. `file` is workspace-root-relative. */
+export interface IoSearchMatch {
+  readonly file: string;
+  readonly line: number;
+  readonly text: string;
+}
+
+export interface IoSearchOptions {
+  /** JavaScript regex source. The tool validates it before calling. */
+  readonly pattern: string;
+  readonly ignoreCase: boolean;
+  /**
+   * Absolute file or directory to search. The whole workspace when omitted.
+   * Callers stat the scope first, so it exists.
+   */
+  readonly scope?: string | undefined;
+  /** Filename glob filter over root-relative paths (e.g. `**​/*.ts`). */
+  readonly glob?: string | undefined;
+  /** Stop scanning once this many matching lines have been collected. */
+  readonly maxMatches: number;
+}
+
+export interface IoSearchResult {
+  readonly matches: readonly IoSearchMatch[];
+  /**
+   * Why the scan ended early, or `false` when it covered everything:
+   * `"max-matches"` = the `maxMatches` bound; `"output-cap"` = a remote
+   * backend's byte cap cut the stream mid-scan (fewer than `maxMatches`
+   * lines were parsed, and more may exist).
+   */
+  readonly stopped: false | "max-matches" | "output-cap";
+  /**
+   * Files skipped for being over the search size cap, or `null` when the
+   * backend can't know (a remote searcher enforces the cap but doesn't
+   * report a count). Consumers must omit the figure rather than show 0.
+   */
+  readonly skippedLargeFiles: number | null;
+}
+
+/**
+ * Byte-oriented workspace effects, async so a remote backend fits. All paths
+ * are absolute (already resolved through `Workspace`); returned file lists
+ * and match paths are workspace-root-relative with forward slashes.
+ */
+export interface WorkspaceIO {
+  /** Stat one path; `null` when it doesn't exist. */
+  stat(abs: string): Promise<IoStat | null>;
+  /** Read one file's bytes; `null` when it doesn't exist. */
+  readFile(abs: string): Promise<Buffer | null>;
+  /** Write one file, creating parent directories and overwriting. */
+  writeFile(abs: string, content: string | Uint8Array): Promise<void>;
+  /**
+   * Candidate file paths for glob/grep — root-relative, gitignore-aware.
+   * `scope` (absolute directory) narrows the listing.
+   */
+  listFiles(scope?: string): Promise<Iterable<string>>;
+  /** Regex content search, backend-native (in-process locally, rg/grep remotely). */
+  search(options: IoSearchOptions): Promise<IoSearchResult>;
+}
+
+/**
+ * The slice of eve's `SandboxSession` the sandbox backend needs, declared
+ * structurally so lib modules stay framework-free (eve's type satisfies it).
+ * Shapes mirror the AI SDK sandbox surface: reads resolve `null` for a
+ * missing file, writes create parent directories.
+ */
+export interface SandboxSessionLike {
+  readonly readBinaryFile: (options: {
+    path: string;
+  }) => PromiseLike<Uint8Array | null>;
+  readonly writeBinaryFile: (options: {
+    path: string;
+    content: Uint8Array;
+  }) => PromiseLike<void>;
+  readonly run: (options: {
+    command: string;
+    workingDirectory?: string;
+  }) => PromiseLike<{ exitCode: number; stdout: string; stderr: string }>;
+}
+
+/**
+ * The slice of eve's `ToolContext` an IO provider may use. Structural, so
+ * tools can hand their eve context straight through without the lib importing
+ * `eve/*`. The session id is exposed for `resolveSession` hooks that run
+ * per-session setup (the Builder's workspace bootstrap keys on it).
+ */
+export interface IoToolContext {
+  readonly session?: { readonly id: string };
+  getSandbox(): PromiseLike<SandboxSessionLike>;
+}
+
+/**
+ * Resolves the IO for one tool call. The local provider ignores `ctx`; the
+ * sandbox provider (see ./sandbox-io.ts) resolves `ctx.getSandbox()` lazily,
+ * so constructing the IO never touches the session.
+ */
+export type WorkspaceIoProvider = (ctx: IoToolContext | undefined) => WorkspaceIO;
+
+/** The local backend: node:fs against the harness process's own disk. */
+export function createLocalIo(root: string): WorkspaceIO {
+  return {
+    async stat(abs) {
+      try {
+        const st = statSync(abs);
+        return { isFile: st.isFile(), size: st.size, mtimeMs: st.mtimeMs };
+      } catch {
+        return null;
+      }
+    },
+    async readFile(abs) {
+      try {
+        return readFileSync(abs);
+      } catch (err) {
+        if (isMissingFileError(err)) return null;
+        throw err;
+      }
+    },
+    async writeFile(abs, content) {
+      mkdirSync(dirname(abs), { recursive: true });
+      writeFileSync(abs, content);
+    },
+    async listFiles(scope) {
+      if (scope === undefined) {
+        return listGitFiles(root) ?? walkFiles(root);
+      }
+      const rel = relativizeWithin(root, scope);
+      return listGitFiles(root, rel) ?? walkFiles(scope, root);
+    },
+    async search(options) {
+      return searchLocal(root, options);
+    },
+  };
+}
+
+/** A provider over one shared local IO — every call gets the same instance. */
+export function localIoProvider(root: string): WorkspaceIoProvider {
+  const io = createLocalIo(root);
+  return () => io;
+}
+
+function isMissingFileError(err: unknown): boolean {
+  return (
+    typeof err === "object" &&
+    err !== null &&
+    "code" in err &&
+    (err as { code?: unknown }).code === "ENOENT"
+  );
+}
+
+/** The in-process scan the local grep has always run; exported for tests. */
+export async function searchLocal(
+  root: string,
+  options: IoSearchOptions,
+): Promise<IoSearchResult> {
+  const re = new RegExp(options.pattern, options.ignoreCase ? "i" : "");
+  const globRe = options.glob ? globToRegExp(options.glob) : null;
+
+  let candidates: Iterable<string>;
+  if (options.scope !== undefined) {
+    const rel = relativizeWithin(root, options.scope);
+    let isFile = false;
+    try {
+      isFile = statSync(options.scope).isFile();
+    } catch {
+      isFile = false;
+    }
+    candidates = isFile
+      ? [rel]
+      : (listGitFiles(root, rel) ?? walkFiles(options.scope, root));
+  } else {
+    candidates = listGitFiles(root) ?? walkFiles(root);
+  }
+
+  const matches: IoSearchMatch[] = [];
+  let stopped: false | "max-matches" = false;
+  let skippedLargeFiles = 0;
+  scan: for (const file of candidates) {
+    if (globRe && !globRe.test(file)) continue;
+    const read = readTextForSearch(join(root, file));
+    if (read.kind === "too-large") {
+      skippedLargeFiles += 1;
+      continue;
+    }
+    if (read.kind !== "text") continue;
+    const lines = read.content.split("\n");
+    for (const [index, line] of lines.entries()) {
+      if (!re.test(line)) continue;
+      matches.push({ file, line: index + 1, text: line });
+      if (matches.length >= options.maxMatches) {
+        stopped = "max-matches";
+        break scan;
+      }
+    }
+  }
+  return { matches, stopped, skippedLargeFiles };
+}

package/src/workspace.ts

@@ -0,0 +1,41 @@
+import { isAbsolute, relative, resolve, sep } from "node:path";
+
+// Every file tool resolves paths against one root and refuses anything that
+// escapes it — an agent is scoped to the workspace it was launched in. `bash`
+// is deliberately not confined this way (it's a real shell); the file tools are.
+
+/**
+ * Resolve `path` against `root` and refuse anything that escapes it. Relative
+ * paths resolve from `root`; an absolute path must already sit inside it.
+ */
+export function resolveWithin(root: string, path: string): string {
+  const abs = isAbsolute(path) ? resolve(path) : resolve(root, path);
+  // `root + sep` (not bare `root`) so a sibling like `/repo-evil` isn't read as
+  // inside `/repo`.
+  if (abs !== root && !abs.startsWith(root + sep)) {
+    throw new Error(`Path escapes the workspace root (${root}): ${path}`);
+  }
+  return abs;
+}
+
+/** A root-relative, forward-slash path for display. Pure counterpart to resolveWithin. */
+export function relativizeWithin(root: string, abs: string): string {
+  const rel = relative(root, abs);
+  return rel === "" ? "." : rel.split(sep).join("/");
+}
+
+/** One workspace root plus the two path operations every file tool needs. */
+export interface Workspace {
+  readonly root: string;
+  resolve(path: string): string;
+  relativize(abs: string): string;
+}
+
+export function createWorkspace(root: string): Workspace {
+  const abs = resolve(root);
+  return {
+    root: abs,
+    resolve: (path) => resolveWithin(abs, path),
+    relativize: (path) => relativizeWithin(abs, path),
+  };
+}