flint-agent 1.14.0

package/src/agent/vision.js

@@ -0,0 +1,94 @@
+// Whether the main model can look at images, and what it is told when it
+// cannot.
+//
+// WHY: a tool that returns an image (a desktop screenshot, a browser observe,
+// view_image) had its picture put into the conversation as image_url. A model
+// with no image input then got the whole payload refused by the provider
+// (OpenRouter: 404 "No endpoints found that support image input"), three
+// retries of the same payload, and the turn died. 18 bench runs on 2026-09-26/27
+// hit it; W-desk-013 never got past its first screenshot.
+//
+// The answer is known, not guessed. It comes from the provider's model list
+// when the provider publishes input modalities, or from the provider refusing
+// an image, which is the same fact learned the hard way. Until one of those
+// arrives it is null: unknown, and images are sent as before.
+//
+// When the model cannot see, it is told so as a fact about the situation:
+// where the image is saved and what text came with it. What to do about it,
+// use the file, read the screen as text, look for another way to see, or tell
+// the user, is the model's call. Naming a tool here would measure the hint.
+
+import { createLogger } from "../logging/logger.js";
+
+const log = createLogger("vision");
+
+let seesImages = null;
+let learnedFrom = null;
+
+/** true, false, or null when nothing has told us yet. */
+export function modelSeesImages() {
+  return seesImages;
+}
+
+/** Record what is known. `from` says how: "provider-metadata" or "provider-refusal". */
+export function setModelSeesImages(value, from) {
+  if (value !== true && value !== false) return;
+  if (seesImages === value) return;
+  // A refusal is the model's own answer; a list saying otherwise is the
+  // stale one. The metadata fetch is async and can land after it.
+  if (learnedFrom === "provider-refusal" && from !== "provider-refusal") {
+    log.warn("model-sees-images-conflict", { kept: seesImages, ignored: value, from });
+    return;
+  }
+  seesImages = value;
+  learnedFrom = from;
+  log.info("model-sees-images", { value, from });
+}
+
+/** For tests and for a model switch: forget what was learned. */
+export function resetVision() {
+  seesImages = null;
+  learnedFrom = null;
+}
+
+export function visionLearnedFrom() {
+  return learnedFrom;
+}
+
+// A client error whose text is about image input. Providers word it
+// differently, so this matches the subject (images as input), not one
+// provider's sentence; a 5xx or a rate limit is never this.
+const IMAGE_REFUSAL = /image input|support(?:s|ed)? (?:for )?images?|image_url is only supported|does not support images?|images? (?:are|is) not supported/i;
+
+export function isImageRefusal(err) {
+  const status = err?.statusCode;
+  if (!(status >= 400 && status < 500) || status === 429) return false;
+  return IMAGE_REFUSAL.test(String(err?.message || ""));
+}
+
+/** What the model reads in place of an image it cannot see. */
+export function imageUnseenText(toolName, imagePath, caption) {
+  const where = imagePath ? `It is saved at ${imagePath}.` : "It was not saved to a file.";
+  const text = caption && caption.trim() ? ` Text that came with it: ${caption.trim()}` : "";
+  return `[The model you are running on cannot view images. ${toolName} returned an image. ${where}${text}]`;
+}
+
+/**
+ * Replace every image part in `messages` with the text the model can read.
+ * Used once the provider has refused an image, so the same payload is not sent
+ * again. Returns how many messages changed.
+ */
+export function stripImages(messages) {
+  let changed = 0;
+  for (const m of messages) {
+    if (!Array.isArray(m.content) || !m.content.some((p) => p?.type === "image_url")) continue;
+    // A tool's image already has its caption and OCR in the tool result next
+    // to it; the text part here is only "[Tool result image ...]". An image
+    // from anywhere else keeps whatever text travelled with it.
+    const caption = m._isImage ? "" : m.content.filter((p) => p?.type === "text").map((p) => p.text).join("\n");
+    m.content = imageUnseenText(m._imageTool || "A message", m._imagePath || null, caption);
+    m._compressed = true;
+    changed++;
+  }
+  return changed;
+}

package/src/agent/watchdog.js

@@ -0,0 +1,139 @@
+// The first answer, and the clock on it.
+//
+// A call sent at 20:40:49 on 2026-09-29 never came back. The 10-minute timeout
+// fired at 20:50:49, the call was retried, and for the whole ten minutes the
+// console said `thinking #15 468s` — a number that is the age of the TASK, not
+// of the call, so it looks equally alarming after ten seconds of normal work.
+//
+// Two separate faults, and the second is why this module exists:
+//
+//   1. The status line counted from the wrong clock (fixed in the UI layer).
+//   2. Nothing gave up on the call. A provider that accepts the request and
+//      never answers is the one failure where the only useful move is to drop
+//      it and send it again, and Flint sat on it for the full 600s hard
+//      timeout, twice, before it did.
+//
+// So: the watchdog is armed on the wait for the FIRST answer, not the whole
+// call. 90 seconds of silence on a request that has been sent is a hung
+// connection, not a slow model. The attempt is dropped (the request is aborted,
+// not merely abandoned — the socket stays open otherwise), the operator is told
+// which attempt it is, and after the last attempt the turn stops with the
+// reason and how to continue.
+
+/** Silence before a hung connection is dropped. Env override for tests. */
+export function firstTokenTimeoutMs() {
+  const v = parseInt(process.env.AGENT_FIRST_TOKEN_TIMEOUT_MS || "", 10);
+  return Number.isFinite(v) && v > 0 ? v : 90_000;
+}
+
+/** How many hung calls in a row before the turn stops. */
+export function maxStallAttempts() {
+  const v = parseInt(process.env.AGENT_STALL_MAX_ATTEMPTS || "", 10);
+  return Number.isFinite(v) && v > 0 ? v : 3;
+}
+
+/**
+ * What the operator sees when one attempt is dropped.
+ *
+ * The attempt number is the whole point: "retrying" alone, once, looks like the
+ * loop hung.
+ *
+ * @param {number} seconds — silence before the drop
+ * @param {number} attempt  — 1-based
+ * @param {number} max      — total attempts allowed
+ * @returns {string}
+ */
+export function stallNote(seconds, attempt, max) {
+  return `no answer for ${Math.round(seconds)}s, retrying, ${attempt} of ${max}`;
+}
+
+/**
+ * The line the turn ends on after the last hung call.
+ *
+ * @param {number} attempts
+ * @param {number} timeoutMs
+ * @returns {string}
+ */
+export function stallStopNote(attempts, timeoutMs) {
+  const secs = Math.round((timeoutMs || 0) / 1000);
+  return `Stopped: the provider sent no answer for ${secs}s on ${attempts} attempt${attempts === 1 ? "" : "s"} in a row, ` +
+    `and Flint dropped each one. Nothing ran, and the work may be unfinished. ` +
+    `To continue, type: continue. If it keeps happening, switch the model with /model or the provider with /provider.`;
+}
+
+/**
+ * Run one model call under a watchdog on the FIRST answer.
+ *
+ * `run` receives `(onToken, signal)`. The signal aborts the in-flight request
+ * when the watchdog fires, because an abandoned promise still holds an open
+ * socket: on 2026-09-29 two of those were still open when the retry started.
+ *
+ * @param {(onToken: Function, signal: AbortSignal) => Promise<any>} run
+ * @param {object} opts - { timeoutMs, signal, onToken, onFirstToken, onStall }
+ * @returns {Promise<any>} the call's result
+ * @throws {Error} with `.isStall` set when the watchdog fired
+ */
+export async function callWithStallWatchdog(run, { timeoutMs, signal, onToken, onFirstToken, onStall } = {}) {
+  const controller = new AbortController();
+  let timer = null;
+  let watched = false;
+
+  const onExternalAbort = () => controller.abort(signal?.reason);
+  if (signal) {
+    if (signal.aborted) controller.abort(signal.reason);
+    else signal.addEventListener("abort", onExternalAbort, { once: true });
+  }
+
+  // Armed until the first token or the answer itself. Once the model starts
+  // writing, there is no silence left to measure.
+  const disarm = () => {
+    watched = false;
+    if (timer) { clearTimeout(timer); timer = null; }
+  };
+
+  const forwardToken = (token) => {
+    if (watched) {
+      // The model has started writing: there is no silence left to measure.
+      // Told here rather than on the way out, because a call that streams for
+      // a minute and then fails should still have reported that it started.
+      disarm();
+      onFirstToken?.();
+    }
+    onToken?.(token);
+  };
+
+  let rejectStall = null;
+  const stalled = new Promise((_, reject) => { rejectStall = reject; });
+
+  const call = (async () => run(forwardToken, controller.signal))();
+  // The race decides; if the watchdog wins, the call's own later rejection is
+  // nobody's problem and must not become an unhandled rejection.
+  call.catch(() => {});
+
+  if (timeoutMs > 0) {
+    watched = true;
+    timer = setTimeout(() => {
+      const err = new Error(`no answer for ${timeoutMs}ms`);
+      err.isStall = true;
+      onStall?.(err);
+      controller.abort(err);
+      rejectStall(err);
+    }, timeoutMs);
+    // Never hold the process open for a watchdog on a call that already ended.
+    timer.unref?.();
+  }
+
+  try {
+    const out = await Promise.race([call, stalled]);
+    // A non-streaming call (stream: false) never fires onToken, so the
+    // watchdog is still armed here. Disarm it and report the first answer.
+    if (watched) {
+      disarm();
+      onFirstToken?.();
+    }
+    return out;
+  } finally {
+    disarm();
+    if (signal) signal.removeEventListener?.("abort", onExternalAbort);
+  }
+}

package/src/agent/workspace-changes.js

@@ -0,0 +1,177 @@
+// What a turn changed, read off the folders it works in rather than off the
+// names of the tools it called.
+//
+// The count used to come from a list of file tools (write_file, edit_file, ...).
+// A file made any other way did not exist for it: ffmpeg run through
+// run_command, a Python script, an MCP server writing for the agent. 20 of 28
+// answers in one benchmark run ended "[Nothing was changed on disk this turn.]"
+// over a file the task had just produced, and each of those turns also paid for
+// a model call asking why nothing had been done.
+//
+// A snapshot is path -> mtime and size for every file under the roots. Two
+// snapshots differ exactly where something was written, whoever wrote it.
+
+import { readdirSync, statSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+
+// Past this many files a snapshot gives up and says so (null), and the caller
+// makes no claim about the disk. Walking a big tree on every tool call would
+// cost more than the answer is worth, and "unknown" is honest where a partial
+// count would not be.
+export const MAX_FILES = 20000;
+
+// Stores that churn on their own or hold other people's code. A change there is
+// not the work the operator asked for, and node_modules alone can exceed
+// MAX_FILES in an ordinary project.
+const SKIP_DIRS = new Set([".git", "node_modules"]);
+
+/**
+ * @param {string[]} roots - folders to read; duplicates and nesting are fine
+ * @param {{maxFiles?: number, skip?: string[]}} [opts] - skip: folders not to
+ *   descend into, for Flint's own state. Flint writes its session log on every
+ *   step, so when it runs from its own folder that log would count as the
+ *   turn's work. A root inside a skipped folder (the session workspace sits
+ *   under sessions/) is still read, because it was named.
+ * @returns {Map<string, string> | null} absolute path -> "mtimeMs:size", or null past maxFiles
+ */
+export function snapshotWorkspace(roots, { maxFiles = MAX_FILES, skip = [] } = {}) {
+  const files = new Map();
+  const seenDirs = new Set(skip.filter(Boolean).map((s) => resolve(s)));
+  const stack = [...new Set(roots.filter(Boolean).map((r) => resolve(r)))];
+  for (const r of stack) seenDirs.delete(r);
+  while (stack.length) {
+    const dir = stack.pop();
+    if (seenDirs.has(dir)) continue;
+    seenDirs.add(dir);
+    let entries;
+    try {
+      entries = readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue; // a root that does not exist yet, or a folder we may not read
+    }
+    for (const e of entries) {
+      const p = join(dir, e.name);
+      if (e.isDirectory()) {
+        if (!SKIP_DIRS.has(e.name)) stack.push(p);
+        continue;
+      }
+      if (!e.isFile()) continue;
+      let st;
+      try {
+        st = statSync(p);
+      } catch {
+        continue; // removed between readdir and stat
+      }
+      files.set(p, `${st.mtimeMs}:${st.size}`);
+      if (files.size > maxFiles) return null;
+    }
+  }
+  return files;
+}
+
+/**
+ * Paths that were added, modified or removed between two snapshots.
+ * @returns {string[] | null} null when either side is unknown
+ */
+export function changedFiles(before, after) {
+  if (!before || !after) return null;
+  const out = [];
+  for (const [p, sig] of after) if (before.get(p) !== sig) out.push(p);
+  for (const p of before.keys()) if (!after.has(p)) out.push(p);
+  return out;
+}
+
+// Absolute paths in free text: a POSIX path not preceded by a URL's "//" or a
+// word character, or a Windows drive path. Pseudo filesystems are left out:
+// nothing the operator asked for lives there, and walking them is slow.
+const POSIX_PATH = /(?<![\w:/.~-])\/(?:[\w.@%+~-]+\/?)+/g;
+const WINDOWS_PATH = /\b[A-Za-z]:[\\/][^\s"'`;|&<>()*?]*/g;
+const PSEUDO_FS = /^\/(proc|sys|dev)(\/|$)/;
+
+/**
+ * The folders that absolute paths in `value` point into: the path itself when
+ * it is a folder, otherwise its nearest existing parent (a file the call is
+ * about to create does not exist yet). Strings are read anywhere inside
+ * `value`, so this does not know or care which tool the arguments belong to.
+ */
+export function pathRootsIn(value) {
+  const strings = [];
+  (function collect(v) {
+    if (typeof v === "string") strings.push(v);
+    else if (Array.isArray(v)) v.forEach(collect);
+    else if (v && typeof v === "object") Object.values(v).forEach(collect);
+  })(value);
+  const roots = new Set();
+  for (const s of strings) {
+    for (const m of [...(s.match(POSIX_PATH) || []), ...(s.match(WINDOWS_PATH) || [])]) {
+      let p = resolve(m.replace(/[\\/.,:]+$/, "") || m);
+      if (PSEUDO_FS.test(p.replace(/\\/g, "/"))) continue;
+      for (;;) {
+        let st = null;
+        try { st = statSync(p); } catch {}
+        if (st?.isDirectory()) break;
+        const up = dirname(p);
+        if (up === p) break;
+        p = up;
+      }
+      // The filesystem root is not a place anyone works in; reading all of it
+      // would only hit MAX_FILES.
+      if (dirname(p) !== p) roots.add(p);
+    }
+  }
+  return [...roots];
+}
+
+/**
+ * Follow what a turn changes. The roots named at the start (the working folder
+ * and the session workspace) must be readable, or the whole answer is unknown.
+ * Folders met later in arguments are read once, before the call that names
+ * them runs; one too big to read is simply not watched, and not named.
+ *
+ * Two full readings per turn, at the start and when an answer is due, not one
+ * per tool call: on an 18k-file repository a reading takes about a second on
+ * Windows. The order of events (was anything run after the last change) comes
+ * from the files' own times instead.
+ */
+export function createChangeTracker({ roots, skip = [], maxFiles = MAX_FILES }) {
+  const baseline = new Map();
+  const watched = new Set();
+  let unknown = false;
+
+  function watch(dirs, required) {
+    for (const r of dirs.filter(Boolean).map((d) => resolve(d))) {
+      if (watched.has(r)) continue;
+      const snap = snapshotWorkspace([r], { skip, maxFiles });
+      if (!snap) {
+        if (required) unknown = true;
+        continue;
+      }
+      watched.add(r);
+      for (const [p, sig] of snap) if (!baseline.has(p)) baseline.set(p, sig);
+    }
+  }
+  watch(roots, true);
+
+  return {
+    /** Start watching the folders any absolute path in `value` points into. */
+    watchPathsIn(value) {
+      watch(pathRootsIn(value), false);
+    },
+    /** @returns {{files: string[], newestMs: number} | null} null when unknown */
+    changes() {
+      if (unknown) return null;
+      const now = snapshotWorkspace([...watched], { skip, maxFiles });
+      const files = changedFiles(baseline, now);
+      if (!files) return null;
+      let newestMs = 0;
+      for (const f of files) {
+        const sig = now.get(f);
+        if (sig) newestMs = Math.max(newestMs, parseFloat(sig));
+      }
+      return { files, newestMs };
+    },
+    roots() {
+      return [...watched];
+    },
+  };
+}

package/src/api/address.js

@@ -0,0 +1,14 @@
+// Where Flint's HTTP API listens, and the URL every caller uses to reach it.
+//
+// One address for both sides (2026-10-02). The server binds 127.0.0.1 only,
+// while the callers (a parent asking its child agents, the agent registry,
+// the orphan check) fetched http://localhost. On Node 22, which Flint needs,
+// `localhost` resolves to ::1 first and nothing listens there, so every one of
+// those calls failed with ECONNREFUSED; GitHub's Windows runners showed it.
+
+export const API_HOST = "127.0.0.1";
+
+/** http://127.0.0.1:<port><path> */
+export function apiUrl(port, path = "") {
+  return `http://${API_HOST}:${port}${path}`;
+}