@nodetool-ai/agents 0.7.0-rc.23 → 0.8.1

package/dist/capabilities/local-browser.js

@@ -0,0 +1,105 @@
+/**
+ * Headless Chrome screenshots, taken on this machine.
+ *
+ * `take_screenshot` used to need a `BROWSER_URL` service, so on a normal
+ * install — desktop app, CLI, dev checkout — it could not take a screenshot at
+ * all, even with Chrome installed. This drives a local Chrome over CDP
+ * instead, with the same lazy import the `lib.browser.*` nodes use: nothing is
+ * loaded, and no browser is required, until a screenshot is actually taken.
+ */
+const CHROME_FLAGS = [
+    "--headless=new",
+    "--disable-gpu",
+    "--no-sandbox",
+    "--disable-setuid-sandbox",
+    "--disable-dev-shm-usage",
+    "--hide-scrollbars",
+    "--mute-audio"
+];
+/**
+ * Navigate a freshly launched Chrome to `url` and return the PNG bytes.
+ *
+ * A navigation that never fires `load` is captured anyway once `timeoutMs`
+ * elapses — a screenshot of a half-loaded page answers more than an error.
+ */
+export async function captureScreenshot(url, opts = {}) {
+    const width = opts.width ?? 1280;
+    const height = opts.height ?? 900;
+    const timeoutMs = opts.timeoutMs ?? 30_000;
+    const { launch } = await import("chrome-launcher");
+    const chrome = await launch({
+        chromeFlags: [...CHROME_FLAGS, `--window-size=${width},${height}`],
+        // The sandbox container points CHROME_PATH at /usr/bin/chromium; locally,
+        // chrome-launcher finds the installed browser itself.
+        chromePath: process.env.CHROME_PATH || undefined
+    });
+    let client = null;
+    try {
+        const CDP = (await import("chrome-remote-interface")).default;
+        client = await CDP({ port: chrome.port });
+        const { Page, Emulation } = client;
+        await Page.enable();
+        await Emulation.setDeviceMetricsOverride({
+            width,
+            height,
+            deviceScaleFactor: 1,
+            mobile: false
+        });
+        const loaded = Page.loadEventFired();
+        const nav = await Page.navigate({ url });
+        // Without this a DNS failure or a refused connection is captured as
+        // Chrome's own error page and handed back as a successful screenshot.
+        if (nav.errorText) {
+            throw new Error(`Could not load ${url}: ${nav.errorText}`);
+        }
+        await raceTimeout(loaded, timeoutMs);
+        const params = { format: "png" };
+        if (opts.fullPage) {
+            const metrics = await Page.getLayoutMetrics();
+            const content = metrics.cssContentSize ?? metrics.contentSize;
+            params["captureBeyondViewport"] = true;
+            params["clip"] = {
+                x: 0,
+                y: 0,
+                width: Math.ceil(content.width),
+                height: Math.ceil(content.height),
+                scale: 1
+            };
+        }
+        const { data } = await Page.captureScreenshot(params);
+        return Uint8Array.from(Buffer.from(data, "base64"));
+    }
+    finally {
+        if (client) {
+            try {
+                await client.close();
+            }
+            catch {
+                // The client is being discarded with the browser below.
+            }
+        }
+        try {
+            await chrome.kill();
+        }
+        catch {
+            // Chrome already exited.
+        }
+    }
+}
+/** Resolve when `promise` settles or `ms` elapses, leaving no pending timer. */
+async function raceTimeout(promise, ms) {
+    let timer;
+    try {
+        await Promise.race([
+            promise,
+            new Promise((resolve) => {
+                timer = setTimeout(resolve, ms);
+            })
+        ]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+//# sourceMappingURL=local-browser.js.map

package/dist/capabilities/local-browser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-browser.js","sourceRoot":"","sources":["../../src/capabilities/local-browser.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAKH,MAAM,YAAY,GAAG;IACnB,gBAAgB;IAChB,eAAe;IACf,cAAc;IACd,0BAA0B;IAC1B,yBAAyB;IACzB,mBAAmB;IACnB,cAAc;CACf,CAAC;AAUF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,IAAI,GAAsB,EAAE;IAE5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC;IAClC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC;IAE3C,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;QAC1B,WAAW,EAAE,CAAC,GAAG,YAAY,EAAE,iBAAiB,KAAK,IAAI,MAAM,EAAE,CAAC;QAClE,0EAA0E;QAC1E,sDAAsD;QACtD,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,SAAS;KACjD,CAAC,CAAC;IAEH,IAAI,MAAM,GAAqB,IAAI,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC,CAAC,OAAO,CAAC;QAC9D,MAAM,GAAG,MAAM,GAAG,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1C,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;QACnC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,SAAS,CAAC,wBAAwB,CAAC;YACvC,KAAK;YACL,MAAM;YACN,iBAAiB,EAAE,CAAC;YACpB,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QACzC,oEAAoE;QACpE,sEAAsE;QACtE,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,KAAK,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAErC,MAAM,MAAM,GAA4B,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QAC1D,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9C,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,IAAI,OAAO,CAAC,WAAW,CAAC;YAC9D,MAAM,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC;YACvC,MAAM,CAAC,MAAM,CAAC,GAAG;gBACf,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;gBACJ,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;gBAC/B,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;gBACjC,KAAK,EAAE,CAAC;aACT,CAAC;QACJ,CAAC;QACD,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QACtD,OAAO,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAc,EAAE,QAAQ,CAAC,CAAC,CAAC;IAChE,CAAC;YAAS,CAAC;QACT,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;YACvB,CAAC;YAAC,MAAM,CAAC;gBACP,wDAAwD;YAC1D,CAAC;QACH,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,WAAW,CAAC,OAAyB,EAAE,EAAU;IAC9D,IAAI,KAAiC,CAAC;IACtC,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,OAAO;YACP,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBACtB,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAClC,CAAC,CAAC;SACH,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC;AACH,CAAC"}

package/dist/capabilities/media.d.ts

@@ -0,0 +1,121 @@
+/**
+ * The `media` capability module — provider-backed generation, plus the vision
+ * judges that grade what it produced.
+ *
+ * These capabilities replace the generation tools in `../tools/media-tools.ts`
+ * and the judging tools in `../tools/creative-critique-tools.ts`. Wire names,
+ * descriptions and schemas are unchanged; `getAllMcpTools` builds them through
+ * `toolFromCapability`.
+ *
+ * Every generation capability is a thin wrapper around
+ * `ProcessingContext.runProviderPrediction` (or `streamProviderPrediction`
+ * where the capability requires it), so it dispatches to whichever
+ * `BaseProvider` the id names and needs nothing on the run beyond its context.
+ *
+ * Outputs are saved as assets when the context exposes a `createAsset` model
+ * interface, and fall back to the caller-supplied `output_file` under the
+ * workspace when it does not.
+ *
+ * Design: docs/tool-class-retirement-design.md § Migration.
+ */
+import type { CapabilityExport, CapabilityModule } from "./types.js";
+declare const generateText: CapabilityExport;
+declare const generateImage: CapabilityExport;
+declare const editImage: CapabilityExport;
+declare const segmentImage: CapabilityExport;
+/** The listed ratio closest to `width/height`, or null for a bad size. */
+export declare function nearestAspectRatio(width: number, height: number): string | null;
+declare const generateVideo: CapabilityExport;
+declare const animateImage: CapabilityExport;
+/**
+ * Whether a reference is an image or a video. The provider takes the two in
+ * separate lists — a video handed over as an image is rejected with an opaque
+ * provider error — so the kind has to be settled here.
+ *
+ * The extension decides when it says anything, because it is what the caller
+ * stated. Otherwise the bytes do: the image sniffer first (it owns the
+ * `ftyp`-based AVIF/HEIC cases), then the three video containers. A reference
+ * neither names nor looks like either is null, and the call refuses it rather
+ * than guessing.
+ */
+export declare function classifyReferenceMedia(file: string, bytes: Uint8Array): "image" | "video" | null;
+declare const videoFromReferences: CapabilityExport;
+declare const generateSpeech: CapabilityExport;
+/**
+ * Music from a prompt — the counterpart to {@link generateSpeech}, and the
+ * hole every other generation capability made conspicuous.
+ *
+ * Without it, scoring a cut meant running `nodetool.audio.TextToMusic` through
+ * `run_node`: a one-node graph, and therefore the graph validator, a typed
+ * `music_model` property and a model ref to satisfy before a single second of
+ * audio existed. A live session spent five rounds there and shipped silent.
+ */
+declare const generateMusic: CapabilityExport;
+declare const transcribeAudio: CapabilityExport;
+declare const embedText: CapabilityExport;
+/**
+ * Whether a model reply reads as cut off rather than merely wrong.
+ *
+ * A judge on a reasoning model spends its ceiling thinking and returns half a
+ * sentence; the caller then sees "did not return parseable JSON" and re-runs
+ * the same call at the same budget. Naming the cause in the error is the
+ * difference between one retry and three.
+ */
+export declare function looksTruncated(text: string): boolean;
+declare const critiqueImage: CapabilityExport;
+declare const compareImages: CapabilityExport;
+declare const scoreImageAdherence: CapabilityExport;
+/**
+ * Read a video with a multimodal chat model.
+ *
+ * The video rides as a `video` content part next to the instruction; the
+ * context's media resolver inlines an `asset://` URI and the provider decides
+ * whether the bytes go inline or through its files API.
+ *
+ * A provider with no video content part is not refused here any more: the
+ * runtime samples the clip into stills with ffmpeg and sends those. The reply
+ * carries `read_as` so the caller knows which of the two it got — a frame read
+ * has no audio and nothing between the samples, and an answer that depends on
+ * either should be treated as a guess.
+ */
+declare const understandVideo: CapabilityExport;
+/**
+ * The gated read behind a media reference.
+ *
+ * The sandbox's own `media.bytes` needs a `ProcessingContext`, which a chat
+ * action deliberately runs without (#4780) — so a chat user could generate an
+ * image and never read it back. This is the same resolution past the
+ * permission gate, with the context the run already carries.
+ *
+ * A filesystem path is refused rather than contained here: `read_file` is the
+ * gated way to a workspace file, and a second containment rule beside
+ * `resolveGuestPath` is the thing worth not having.
+ */
+declare const readMediaBytes: CapabilityExport;
+declare const ffmpeg: CapabilityExport;
+/**
+ * What ffprobe reports as numbers, as numbers.
+ *
+ * ffprobe's JSON is all strings — `"duration": "8.000000"` — so arithmetic on
+ * it silently produces `"8.000000".toFixed is not a function`, or worse, a
+ * concatenation. The raw payload stays exactly as ffprobe wrote it; this is a
+ * second, typed view of the four fields a caller actually plans a cut with.
+ */
+export declare function ffprobeSummary(parsed: Record<string, unknown>): Record<string, unknown>;
+/**
+ * ffprobe with an argv NodeTool writes, not the caller.
+ *
+ * Everything a caller can say is one workspace path, so the whole boundary is
+ * the confinement check — there is no flag surface to guard, and no way to ask
+ * ffprobe to open a socket. `inputs` stages assets under workspace names the
+ * same way `ffmpeg`'s does: without it the two halves of one loop disagreed,
+ * and a caller who had just staged a clip for ffmpeg could not ask ffprobe how
+ * long it was without staging it a second time through a no-op ffmpeg copy.
+ */
+declare const ffprobe: CapabilityExport;
+declare const ytDlp: CapabilityExport;
+/** Every media capability, in the order `getAllMcpTools` offered them. */
+export declare const MEDIA_CAPABILITIES: readonly CapabilityExport[];
+export declare const module: CapabilityModule;
+export { generateText, generateImage, editImage, segmentImage, generateVideo, animateImage, videoFromReferences, generateSpeech, generateMusic, transcribeAudio, embedText, readMediaBytes, critiqueImage, compareImages, scoreImageAdherence, understandVideo, ffmpeg, ffprobe, ytDlp };
+//# sourceMappingURL=media.d.ts.map

package/dist/capabilities/media.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"media.d.ts","sourceRoot":"","sources":["../../src/capabilities/media.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAmEH,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AAoCpB,QAAA,MAAM,YAAY,EAAE,gBAqDnB,CAAC;AAgVF,QAAA,MAAM,aAAa,EAAE,gBAqBpB,CAAC;AAEF,QAAA,MAAM,SAAS,EAAE,gBA2ChB,CAAC;AAyBF,QAAA,MAAM,YAAY,EAAE,gBA6DnB,CAAC;AAsDF,0EAA0E;AAC1E,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,MAAM,GAAG,IAAI,CAWf;AAED,QAAA,MAAM,aAAa,EAAE,gBAoBpB,CAAC;AAEF,QAAA,MAAM,YAAY,EAAE,gBAmCnB,CAAC;AA4BF;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,UAAU,GAChB,OAAO,GAAG,OAAO,GAAG,IAAI,CAgB1B;AAED,QAAA,MAAM,mBAAmB,EAAE,gBAmE1B,CAAC;AA4IF,QAAA,MAAM,cAAc,EAAE,gBA0LrB,CAAC;AAEF;;;;;;;;GAQG;AACH,QAAA,MAAM,aAAa,EAAE,gBA+FpB,CAAC;AAEF,QAAA,MAAM,eAAe,EAAE,gBAyCtB,CAAC;AAEF,QAAA,MAAM,SAAS,EAAE,gBAkChB,CAAC;AAgGF;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAKpD;AAwDD,QAAA,MAAM,aAAa,EAAE,gBAqDpB,CAAC;AAyBF,QAAA,MAAM,aAAa,EAAE,gBA8GpB,CAAC;AAYF,QAAA,MAAM,mBAAmB,EAAE,gBAgF1B,CAAC;AAMF;;;;;;;;;;;;GAYG;AACH,QAAA,MAAM,eAAe,EAAE,gBA6CtB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,QAAA,MAAM,cAAc,EAAE,gBA0CrB,CAAC;AA8OF,QAAA,MAAM,MAAM,EAAE,gBAqEb,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAC9B,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAyBzB;AA0BD;;;;;;;;;GASG;AACH,QAAA,MAAM,OAAO,EAAE,gBAiFd,CAAC;AAIF,QAAA,MAAM,KAAK,EAAE,gBAoHZ,CAAC;AAEF,0EAA0E;AAC1E,eAAO,MAAM,kBAAkB,EAAE,SAAS,gBAAgB,EAoBzD,CAAC;AAEF,eAAO,MAAM,MAAM,EAAE,gBAGpB,CAAC;AAEF,OAAO,EACL,YAAY,EACZ,aAAa,EACb,SAAS,EACT,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,eAAe,EACf,SAAS,EACT,cAAc,EACd,aAAa,EACb,aAAa,EACb,mBAAmB,EACnB,eAAe,EACf,MAAM,EACN,OAAO,EACP,KAAK,EACN,CAAC"}