@sema-agent/core 5.10.0 → 5.12.0

package/dist/tools/web.js

@@ -25,6 +25,25 @@ function resolveWebMaxBytes(value) {
     }
     return value;
 }
+export const WEBFETCH_GROUNDING_MIN_TEXT_CHARS = 200;
+const GROUNDING_MIN_TEXT_RATIO = 0.01;
+const GROUNDING_RATIO_MAX_TEXT_CHARS = 2_000;
+const GROUNDING_ECHO_MAX_CHARS = GROUNDING_RATIO_MAX_TEXT_CHARS + 100;
+const GROUNDING_SHELL_MIN_BYTES = 500;
+function assessGrounding(text, bytes) {
+    const textChars = text.replace(/\s+/g, " ").trim().length;
+    const textRatio = bytes > 0 ? (textChars / bytes) : 0;
+    const low = textChars < WEBFETCH_GROUNDING_MIN_TEXT_CHARS ||
+        (textChars < GROUNDING_RATIO_MAX_TEXT_CHARS && bytes > 0 && textRatio < GROUNDING_MIN_TEXT_RATIO);
+    return { level: low ? "low" : "ok", textChars, bytes, textRatio: Math.round(textRatio * 1000) / 1000 };
+}
+function recoveryToolRuledOut(ctx, toolName) {
+    if (ctx.excludeTools?.includes(toolName) ?? false)
+        return true;
+    const deferred = ctx.deferTools?.includes(toolName) ?? false;
+    const pinnedInline = ctx.alwaysLoadTools?.includes(toolName) ?? false;
+    return deferred && !pinnedInline;
+}
 const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
 const ERROR_BODY_EXCERPT_CHARS = 2048;
 const ERROR_BODY_CONVERT_MAX_CHARS = 64 * 1024;
@@ -471,10 +490,14 @@ export function webFetchToolSpec(config = {}) {
                         : bodyCut
                             ? `\n\n[WebFetch: ${cutPhrase} — the ${bodyBytes.length} bytes are a partial prefix received before the cutoff, NOT the object's full size]`
                             : "";
+                    const shellRecoveryRuledOut = recoveryToolRuledOut(ctx, "Bash") || ctx.handsReadOnly === true;
+                    const recoveryLine = shellRecoveryRuledOut
+                        ? "Retrieve it outside this tool — this run has no shell capability that can download it to a file."
+                        : `Download and inspect it with Bash instead, e.g.: curl -L -o /tmp/download "${parsed.toString()}"`;
                     return {
                         content: `Error (WebFetch): binary content detected (${kind}${mt && signature ? `, content-type: ${mt}` : ""}, ` +
                             `${bodyBytes.length} bytes) — the body was NOT added to the context (it does not decode as text). ` +
-                            `Download and inspect it with Bash instead, e.g.: curl -L -o /tmp/download "${parsed.toString()}"` +
+                            recoveryLine +
                             binaryStateNote,
                         details: {
                             type: "web-fetch",
@@ -503,8 +526,11 @@ export function webFetchToolSpec(config = {}) {
                 raw = raw.slice(0, maxBytes);
             }
             const text = /html/i.test(contentType) || /^\s*</.test(raw) ? htmlToText(raw) : raw;
+            const grounding = assessGrounding(text, bodyBytes ? bodyBytes.length : raw.length);
             let out = text;
             let summaryTruncated = false;
+            let summaryApplied = false;
+            let summaryInputNote;
             let note;
             if (prompt && bodyCut) {
                 note =
@@ -516,13 +542,24 @@ export function webFetchToolSpec(config = {}) {
                     const summarized = await config.summarize(text, prompt, ctx.signal);
                     if (typeof summarized === "string") {
                         out = summarized;
+                        summaryApplied = true;
                     }
                     else {
                         out = summarized.text;
+                        summaryApplied = true;
                         if (summarized.truncated) {
                             summaryTruncated = true;
                             note = `[note: the summary below is INCOMPLETE — the summarizer hit its output limit before finishing]`;
                         }
+                        if (summarized.inputTruncated) {
+                            const sizes = summarized.usedChars !== undefined && summarized.inputChars !== undefined
+                                ? ` (only the first ${summarized.usedChars} of ${summarized.inputChars} characters were read)`
+                                : "";
+                            summaryInputNote =
+                                `[note: the summary below covers only the BEGINNING of the page${sizes} — the page exceeded what the ` +
+                                    `summarizer could feed its model, so the rest was never read. Absence of something from the summary ` +
+                                    `does NOT mean it is absent from the page.]`;
+                        }
                     }
                 }
                 catch (e) {
@@ -537,12 +574,44 @@ export function webFetchToolSpec(config = {}) {
                     `[note: summarization unavailable in this deployment — raw page content follows; ` +
                         `the requested analysis ("${inlineUntrusted(prompt, 120)}") was NOT applied]`;
             }
+            const bodyIncomplete = truncationNote !== "" || bodyCut !== undefined;
+            let groundingNote;
+            let sourceEcho = "";
+            if (grounding.level === "low") {
+                const shellShape = /html/i.test(contentType) && grounding.bytes >= GROUNDING_SHELL_MIN_BYTES
+                    ? bodyIncomplete
+                        ? " The retrieved bytes are almost entirely markup/script — which is either a client-rendered shell or " +
+                            "simply the head of a document whose content sits past the retrieved prefix; a larger byte limit (or a " +
+                            "completed transfer) would distinguish the two."
+                        : " The retrieved bytes are almost entirely markup/script, which is the shape of a page whose content is " +
+                            "loaded by JavaScript after the document — this tool does not run JavaScript, so that content was never present."
+                    : "";
+                const nothingExtractable = grounding.textChars === 0;
+                const sourceLabel = bodyIncomplete ? "extracted text of the retrieved prefix" : "complete extracted page text";
+                const head = `[WebFetch grounding: LOW — the ${bodyIncomplete ? "retrieved prefix" : "page"} yielded ` +
+                    (nothingExtractable ? "NO extractable text at all" : `only ${grounding.textChars} characters of extractable text`) +
+                    ` out of ${grounding.bytes} retrieved bytes`;
+                groundingNote = summaryApplied
+                    ? `${head}. ` +
+                        (nothingExtractable
+                            ? `There is no source text at all, so every specific claim in the summary below is unsourced — do not use it as a factual source.`
+                            : `That may be all this URL serves, or it may be a page whose content is not present in the fetched bytes — this tool ` +
+                                `cannot tell the two apart, so the summary below is NOT usable as a source on its own: check every specific claim ` +
+                                `in it against the ${sourceLabel} reproduced after it.`) +
+                        shellShape +
+                        `]`
+                    : `${head}; the content below is ${bodyIncomplete ? "all the retrieved prefix yielded" : "all of it"}.${shellShape}]`;
+                if (summaryApplied && !nothingExtractable) {
+                    sourceEcho = `\n\n${delimitUntrusted(`WebFetch ${parsed.hostname} — ${sourceLabel}`, text.replace(/\s+/g, " ").trim(), GROUNDING_ECHO_MAX_CHARS)}`;
+                }
+            }
             const fenced = delimitUntrusted(`WebFetch ${parsed.hostname}`, out);
-            const withNote = note ? `${note}\n\n${fenced}` : fenced;
+            const headNotes = [groundingNote, note, summaryInputNote].filter((n) => n !== undefined);
+            const withNote = headNotes.length > 0 ? `${headNotes.join("\n\n")}\n\n${fenced}` : fenced;
             const partialNote = bodyCut
                 ? `\n\n[WebFetch: ${cutPhrase} — the content above is PARTIAL: ${bodyBytes?.length ?? 0} bytes were received before the cutoff and the tail is missing. Treat absent information as unfetched, not absent from the source.]`
                 : "";
-            const modelText = withNote + truncationNote + partialNote;
+            const modelText = withNote + truncationNote + partialNote + sourceEcho;
             const RESULT_PREVIEW_CHARS = 8_000;
             return {
                 content: modelText,
@@ -554,7 +623,9 @@ export function webFetchToolSpec(config = {}) {
                     bytes: bodyBytes ? bodyBytes.length : raw.length,
                     result: out.length > RESULT_PREVIEW_CHARS ? `${out.slice(0, RESULT_PREVIEW_CHARS)}\n…[${out.length - RESULT_PREVIEW_CHARS} chars truncated — full text in the tool output]` : out,
                     durationMs: Date.now() - startedAt,
-                    ...(truncationNote || summaryTruncated ? { truncated: true } : {}),
+                    ...(truncationNote || summaryTruncated || summaryInputNote ? { truncated: true } : {}),
+                    grounding,
+                    ...(summaryInputNote ? { summaryInputTruncated: true } : {}),
                     ...transferStateDetails,
                 },
             };
@@ -570,12 +641,50 @@ export const WEBFETCH_SUMMARY_GUIDELINES = `Provide a concise response based onl
     ` - Use quotation marks for exact language from articles; any language outside of the quotation should never be word-for-word the same.\n` +
     ` - You are not a lawyer and never comment on the legality of your own prompts and responses.\n` +
     ` - Never produce or reproduce exact song lyrics.\n`;
-export function createWebFetchSummarizer(brain, model) {
+export const WEBFETCH_SUMMARY_GROUNDING_CLAUSE = `Ground every statement in the content above:\n` +
+    ` - If the content does not contain what was asked for, say exactly that and describe what the content IS instead. Never fill the gap with general knowledge or plausible inference.\n` +
+    ` - If the content is a listing (file names, links, titles) rather than the data itself, a name is evidence only about naming — do not conclude from names alone that the underlying resource does or does not contain something; report what the listing shows and what would have to be opened to answer.\n`;
+export const WEBFETCH_SUMMARY_INPUT_HEADROOM = 0.8;
+export const WEBFETCH_SUMMARY_MIN_CONTENT = 4_000;
+const SUMMARY_PROMPT_OVERHEAD_TOKENS = 1_000;
+const SUMMARY_PROMPT_ALLOWANCE_CHARS = 2_000;
+const SUMMARY_MIN_CONTENT_PER_CALL = 1_024;
+function invalidSummaryBudget(message) {
+    const e = new Error(message);
+    e.code = "config.web_summary_max_content_invalid";
+    return e;
+}
+export function resolveSummaryInputChars(model, override) {
+    if (override !== undefined) {
+        if (!Number.isFinite(override) || override < 1) {
+            throw invalidSummaryBudget(`WebFetch summarizer maxContentChars must be a finite number of characters >= 1 (got ${String(override)})`);
+        }
+        return Math.floor(override);
+    }
+    const window = model.contextTokens ?? model.contextWindow;
+    const charsPerToken = model.charsPerToken ?? 4;
+    for (const [name, value] of [
+        ["contextTokens/contextWindow", window],
+        ["maxTokens", model.maxTokens],
+        ["charsPerToken", charsPerToken],
+    ]) {
+        if (!Number.isFinite(value) || value <= 0) {
+            throw invalidSummaryBudget(`WebFetch summarizer cannot size its input: model "${model.id}" declares a non-finite or non-positive ${name} (got ${String(value)})`);
+        }
+    }
+    const reservedOutputTokens = Math.min(model.maxTokens, Math.floor(window / 4));
+    const inputTokens = window - reservedOutputTokens - SUMMARY_PROMPT_OVERHEAD_TOKENS;
+    const derived = Math.floor(inputTokens * charsPerToken * WEBFETCH_SUMMARY_INPUT_HEADROOM);
+    return Math.min(WEBFETCH_SUMMARY_MAX_CONTENT, Math.max(WEBFETCH_SUMMARY_MIN_CONTENT, derived));
+}
+export function createWebFetchSummarizer(brain, model, options = {}) {
+    const maxContentChars = resolveSummaryInputChars(model, options.maxContentChars);
     return async (content, prompt, signal) => {
-        const truncated = content.length > WEBFETCH_SUMMARY_MAX_CONTENT
-            ? content.slice(0, WEBFETCH_SUMMARY_MAX_CONTENT) + "\n\n[Content truncated due to length...]"
-            : content;
-        const userPrompt = `\nWeb page content:\n---\n${truncated}\n---\n\n${prompt}\n\n` + WEBFETCH_SUMMARY_GUIDELINES;
+        const promptOverflow = Math.max(0, prompt.length - SUMMARY_PROMPT_ALLOWANCE_CHARS);
+        const budget = promptOverflow === 0 ? maxContentChars : Math.max(SUMMARY_MIN_CONTENT_PER_CALL, maxContentChars - promptOverflow);
+        const inputTruncated = content.length > budget;
+        const truncated = inputTruncated ? content.slice(0, budget) + "\n\n[Content truncated due to length...]" : content;
+        const userPrompt = `\nWeb page content:\n---\n${truncated}\n---\n\n${prompt}\n\n` + WEBFETCH_SUMMARY_GROUNDING_CLAUSE + "\n" + WEBFETCH_SUMMARY_GUIDELINES;
         const context = { messages: [{ role: "user", content: userPrompt, timestamp: Date.now() }] };
         const msg = brain.complete
             ? await brain.complete(model, context, { signal })
@@ -590,7 +699,14 @@ export function createWebFetchSummarizer(brain, model) {
             .trim();
         if (!text)
             throw new Error("summarizer returned no text");
-        return msg.stopReason === "length" || msg.partialFinalized === true ? { text, truncated: true } : text;
+        const outputTruncated = msg.stopReason === "length" || msg.partialFinalized === true;
+        if (!outputTruncated && !inputTruncated)
+            return text;
+        return {
+            text,
+            ...(outputTruncated ? { truncated: true } : {}),
+            ...(inputTruncated ? { inputTruncated: true, inputChars: content.length, usedChars: budget } : {}),
+        };
     };
 }
 const DEFAULT_SEARCH_TIMEOUT_MS = 30_000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sema-agent/core",
-  "version": "5.10.0",
+  "version": "5.12.0",
   "description": "Stateless, task-oriented AI agent core",
   "type": "module",
   "license": "BUSL-1.1",