@nxisai/agent-harness 0.4.1 → 0.5.0

package/src/harness/tools/retry.ts

@@ -0,0 +1,117 @@
+/**
+ * Waiting out a server that is rate limiting us.
+ *
+ * The site's MCP server answers a read over its limit with the limit, and how
+ * long to wait: "You have used all 3 read requests allowed per 60s on the public
+ * plan. Retry after 52s." That is an instruction, not a failure — a turn that
+ * gives up on it tells the visitor the site has no data, which is false, and a
+ * turn that retries immediately makes it worse.
+ *
+ * The wait is the server's number, never a guess of ours, and it is bounded: a
+ * visitor is owed an answer sooner than a minute, so a long wait is refused and
+ * reported rather than sat through.
+ *
+ * @module
+ */
+
+/** How many times a rate-limited call is retried, by default. */
+const ATTEMPTS = 2;
+/** The longest we will wait on one attempt. Past this, the visitor is told. */
+const MAX_WAIT_MS = 60_000;
+
+/** "Retry after 52s", "retry-after: 52", "try again in 52 seconds". */
+const AFTER = /retry[-\s]?after:?\s*(\d+)\s*(s|sec|secs|seconds)?\b|try again in\s*(\d+)\s*(s|sec|secs|seconds)\b/i;
+/** What a server says when it is us, not the request, that it is refusing. */
+const LIMITED = /rate[-\s]?limit|resource[-_\s]?exhausted|too many requests|requests allowed per|429/i;
+
+const textOf = (err: unknown): string => {
+  if (typeof err === 'string') return err;
+  const e = err as { message?: string; content?: { text?: string }[] };
+  const message = typeof e?.message === 'string' ? e.message : '';
+  const content = Array.isArray(e?.content) ? e.content.map((c) => c?.text ?? '').join(' ') : '';
+  return `${message} ${content}`.trim();
+};
+
+/**
+ * Whether a failure is the server asking us to wait.
+ *
+ * Read from what it said. The transport carries a numeric code too, but each
+ * hop maps it differently, and the sentence survives every hop.
+ */
+export function isRateLimited(err: unknown): boolean {
+  const text = textOf(err);
+  return LIMITED.test(text) || AFTER.test(text);
+}
+
+/**
+ * How long the server asked us to wait, in milliseconds.
+ *
+ * @returns The advised wait, or null when it named none.
+ */
+export function retryAfterMs(err: unknown): number | null {
+  const match = AFTER.exec(textOf(err));
+  if (!match) return null;
+  const seconds = Number(match[1] ?? match[3]);
+  return Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : null;
+}
+
+/** Overrides, so the waiting can be exercised without spending it. */
+export interface RetryOptions {
+  /** Retries after the first try. Defaults to two. */
+  attempts?: number;
+  /** The longest single wait to accept. Defaults to a minute. */
+  maxWaitMs?: number;
+  /** Abort while waiting. */
+  signal?: AbortSignal;
+  /** How to wait. Replaced in tests. */
+  sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+  /** Told before each wait, for a surface that wants to say "one moment". */
+  onWait?: (ms: number, err: unknown) => void;
+}
+
+const defaultSleep = (ms: number, signal?: AbortSignal): Promise<void> =>
+  new Promise((resolve, reject) => {
+    if (signal?.aborted) return reject(signal.reason ?? new Error('aborted'));
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      'abort',
+      () => {
+        clearTimeout(timer);
+        reject(signal.reason ?? new Error('aborted'));
+      },
+      { once: true },
+    );
+  });
+
+/**
+ * Run a call, waiting out a rate limit as the server asked.
+ *
+ * Anything else — a network failure, a bad request, a tool that refused — is
+ * raised as it happened: this waits for permission, it does not paper over
+ * errors by trying again.
+ *
+ * @param run - The call, run afresh on each attempt.
+ * @param opts - Attempts, ceiling, the clock and the abort signal.
+ */
+export async function withRetry<T>(run: () => Promise<T>, opts: RetryOptions = {}): Promise<T> {
+  const attempts = opts.attempts ?? ATTEMPTS;
+  const ceiling = opts.maxWaitMs ?? MAX_WAIT_MS;
+  const sleep = opts.sleep ?? defaultSleep;
+
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return await run();
+    } catch (err) {
+      const advised = retryAfterMs(err);
+      const waitable = isRateLimited(err) && attempt < attempts;
+      // A wait longer than the ceiling is the server saying "not for a while":
+      // sitting on it would hold the turn open with nothing happening.
+      if (!waitable || (advised ?? 0) > ceiling) throw err;
+      // A limit with no number attached: back off, so a second client does not
+      // retry in lockstep with this one.
+      const wait = advised ?? Math.min(ceiling, 1000 * 2 ** attempt + Math.floor(Math.random() * 400));
+      opts.onWait?.(wait, err);
+      await sleep(wait, opts.signal);
+    }
+  }
+}

package/src/harness/tools/schema.ts

@@ -0,0 +1,245 @@
+/**
+ * Reading a tool's JSON Schema, and reconciling model output against it.
+ *
+ * Model-agnostic: these helpers work the same whichever provider produced the
+ * arguments, so they sit with the harness rather than behind
+ * {@link ModelProvider}.
+ *
+ * @module
+ */
+import type {
+  JsonSchema,
+  ModelContextToolInfo,
+  ModelContextToolResponse,
+} from '../../types.js';
+
+/**
+ * Read a tool's input schema, whichever key and form it arrives in.
+ *
+ * Accepts every alias on {@link ModelContextToolInfo}, parses a schema supplied
+ * as a JSON string, merges in any required fields the page's own markup
+ * declares, and gives every node a `type` so a chat template that formats types
+ * cannot fail on an untyped one.
+ *
+ * @param tool - The tool as reported by discovery.
+ * @returns The normalised schema, or an empty object when there is none.
+ */
+export function toolSchema(tool: ModelContextToolInfo): JsonSchema {
+  let raw: JsonSchema | string | undefined =
+    tool && (tool.inputSchema ?? tool.input_schema ?? tool.parameters ?? tool.schema);
+  if (typeof raw === 'string') {
+    try {
+      raw = JSON.parse(raw);
+    } catch {
+      raw = undefined;
+    }
+  }
+  const schema: JsonSchema = raw && typeof raw === 'object' ? raw : {};
+  // The page's own markup is authoritative about what is required: a backend
+  // schema may omit fields the rendered form still demands.
+  const props = schema.properties ? Object.keys(schema.properties) : [];
+  if (props.length) {
+    const domReq = _domRequired(props);
+    if (domReq.length) schema.required = [...new Set([...(schema.required || []), ...domReq])];
+  }
+  // Some schemas declare properties with no `type`. Gemma's tool chat template
+  // applies `| upper` to each parameter type and throws on an undefined one, so
+  // every node is given a type before prompt building.
+  return _normalizeSchema(schema, true);
+}
+
+/**
+ * Which of a tool's required fields have no usable value.
+ *
+ * @param tool - The tool whose schema declares the requirements.
+ * @param args - The arguments so far.
+ * @returns Required field names that are absent, null, or blank.
+ */
+/**
+ * A read's answer when it is prose rather than records.
+ *
+ * Some reads answer in words — where the visitor is, that they are already on
+ * the page they asked for — and a surface that can only draw record cards shows
+ * an empty one reading "No results found." directly under the answer it was
+ * given. This is how a surface tells the two apart.
+ *
+ * Empty when the answer parses as JSON, or says nothing: that is records a card
+ * knows how to draw, or nothing worth putting in front of anyone. The envelope
+ * a registry wraps a response in is never consulted — a native
+ * `document.modelContext` attaches fields of its own to every response, and
+ * judging by those is what put "No results found." under an answer in words.
+ *
+ * @param res - The tool response, or the text it was flattened to.
+ * @returns The sentence to show, or an empty string to draw records instead.
+ */
+export function proseAnswer(res: ModelContextToolResponse | string | null): string {
+  // Always through extractResult, strings included. Chrome's native registry returns the whole
+  // response serialized — '{"content":[{"type":"text","text":"You are on …"}]}' — and read as-is that
+  // parses as JSON and was taken for records, so the answer only ever reached the debug trace.
+  const text = extractResult(res);
+  const trimmed = (text ?? '').trim();
+  if (!trimmed) return '';
+  // Decided from the answer itself, never from the envelope around it. A registry attaches its own
+  // fields — `structuredContent`, a wrapper, whatever the implementation adds — and judging by
+  // those is how an answer in plain words ended up drawn as an empty result card.
+  try {
+    const parsed: unknown = JSON.parse(trimmed);
+    if (parsed && typeof parsed === 'object') return '';
+  } catch {
+    // Not records. Words.
+  }
+  return trimmed;
+}
+
+export function missingRequired(tool: ModelContextToolInfo, args: Record<string, unknown> | undefined): string[] {
+  const required = toolSchema(tool).required || [];
+  return required.filter((k) => {
+    const v = args ? args[k] : undefined;
+    return v === undefined || v === null || (typeof v === 'string' && v.trim() === '');
+  });
+}
+
+/**
+ * Map a model's argument keys onto the schema's real property names.
+ *
+ * Models paraphrase field names. Keys are matched exactly first, then
+ * case- and punctuation-insensitively, then by prefix, then by shared word.
+ * Values matching an enum are snapped to the enum's own casing. A key with no
+ * plausible match is passed through unchanged.
+ *
+ * @param args - The arguments the model supplied.
+ * @param schema - The tool's normalised schema.
+ * @returns The arguments rekeyed to schema property names.
+ */
+export function reconcileArgs(
+  args: Record<string, unknown> | undefined,
+  schema: JsonSchema,
+): Record<string, unknown> {
+  const props = schema && schema.properties ? Object.keys(schema.properties) : [];
+  if (!props.length || !args) return args || {};
+  const norm = (s: string) => String(s).toLowerCase().replace(/[^a-z0-9]/g, '');
+  const toks = (s: string) =>
+    String(s)
+      .toLowerCase()
+      .split(/[^a-z0-9]+/)
+      .filter(Boolean);
+  const out: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(args)) {
+    if (props.includes(key)) {
+      out[key] = _snapEnum(value, schema.properties![key]);
+      continue;
+    }
+    const nk = norm(key);
+    let best: string | undefined = props.find((p) => norm(p) === nk);
+    if (!best) best = props.find((p) => nk.startsWith(norm(p)) || norm(p).startsWith(nk));
+    if (!best) {
+      const ktoks = new Set(toks(key));
+      let bestScore = 0;
+      for (const p of props) {
+        const score = toks(p).filter((t) => ktoks.has(t)).length;
+        if (score > bestScore) {
+          bestScore = score;
+          best = p;
+        }
+      }
+      if (bestScore === 0) best = undefined;
+    }
+    const targetKey = best || key;
+    out[targetKey] = _snapEnum(value, schema.properties![targetKey]);
+  }
+  return out;
+}
+
+/**
+ * Flatten a tool response to text.
+ *
+ * @param res - The response, a plain string, or nothing.
+ * @returns The response's text blocks joined, or its JSON form when it carries
+ * no text. Empty for a null or undefined response.
+ */
+export function extractResult(res: ModelContextToolResponse | string | null | undefined): string {
+  if (res == null) return '';
+  if (typeof res === 'string') return unwrapResponse(res);
+  if (Array.isArray(res.content)) {
+    return res.content
+      .map((c) => unwrapResponse((c && c.text) || ''))
+      .filter(Boolean)
+      .join('\n');
+  }
+  try {
+    return JSON.stringify(res);
+  } catch {
+    return String(res);
+  }
+}
+
+// ── internals ───────────────────────────────────────────────────────────────
+
+/**
+ * A response the registry stringified rather than passed through.
+ *
+ * A tool may answer with a response object, and a registry is free to hand
+ * that back as JSON in a single text block — which puts `{"content":[{"type":
+ * "text"…` in front of the user where a sentence belongs. Only a shape that is
+ * plainly a tool response is unwrapped; a tool whose own result happens to be
+ * JSON keeps it.
+ */
+function unwrapResponse(text: string): string {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith('{') || !trimmed.includes('"content"')) return text;
+  try {
+    const parsed = JSON.parse(trimmed) as { content?: { text?: string }[] };
+    if (!Array.isArray(parsed.content)) return text;
+    const inner = parsed.content
+      .map((c) => (c && c.text) || '')
+      .filter(Boolean)
+      .join('\n');
+    return inner || text;
+  } catch {
+    return text;
+  }
+}
+
+/**
+ * Give every schema node a `type`, which the chat template requires.
+ *
+ * Nodes with properties become `object`, nodes with items become `array`, and
+ * remaining leaves become `string`. Mutates and returns the schema.
+ */
+function _normalizeSchema(schema: JsonSchema, isRoot = false): JsonSchema {
+  if (!schema || typeof schema !== 'object') return schema;
+  if (schema.properties && typeof schema.properties === 'object') {
+    if (!schema.type) schema.type = 'object';
+    for (const key of Object.keys(schema.properties)) _normalizeSchema(schema.properties[key]);
+  }
+  if (schema.items && typeof schema.items === 'object') {
+    if (!schema.type) schema.type = 'array';
+    _normalizeSchema(schema.items);
+  }
+  if (!schema.type) schema.type = isRoot ? 'object' : 'string';
+  if (isRoot && schema.type === 'object' && !schema.properties) schema.properties = {};
+  return schema;
+}
+
+/** Which of these property names the page's own markup marks required. */
+function _domRequired(propNames: string[]): string[] {
+  const out: string[] = [];
+  for (const name of propNames) {
+    let el: Element | null = null;
+    try {
+      el = document.querySelector(`[name="${CSS.escape(name)}"]`);
+    } catch {
+      // Not a usable selector.
+    }
+    if (el && ((el as HTMLInputElement).required || el.hasAttribute('data-required'))) out.push(name);
+  }
+  return out;
+}
+
+/** Correct a value's casing to match its schema's enum, when one matches. */
+function _snapEnum(value: unknown, propSchema: JsonSchema | undefined): unknown {
+  const enumVals = propSchema && (propSchema.enum || (propSchema.items && propSchema.items.enum));
+  if (!enumVals || typeof value !== 'string' || enumVals.includes(value)) return value;
+  const ci = enumVals.find((e) => String(e).toLowerCase() === value.toLowerCase());
+  return ci !== undefined ? ci : value;
+}