@semalt-ai/code 1.8.5 → 1.20.0

package/test/web-data-extraction-guidance.test.js

@@ -0,0 +1,71 @@
+'use strict';
+
+// Task W.4 Part 1 — guidance: teach fetch+grep for data-extraction tasks.
+//
+// "Extract specific values from a page" (colors/hex, versions, IDs, URLs,
+// counts) is a different task class from "read a page": the right pattern is
+// targeted matching (download→grep / curl|grep) so only the matches enter
+// context — NOT loading page content via any http_get mode. These tests pin
+// that the guidance text is present (the spec drives the model) and that the
+// stale prompts.js http_get description ("byte cap body") is corrected.
+//
+// NOTE: Part 1 is a guidance change; its real effect is behavioral (a live
+// re-run of the "what colors" task), not a unit assertion — these tests only
+// pin that the guidance exists and is discoverable to the model.
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+
+const { TOOL_SPECS } = require('../lib/tool_specs');
+const { getSystemPrompt } = require('../lib/prompts');
+
+// ---------------------------------------------------------------------------
+// The http_get spec guides the agent to fetch+grep for extracting values
+// ---------------------------------------------------------------------------
+
+test('http_get spec guides fetch+grep for extracting specific values', () => {
+  const blob = TOOL_SPECS.http_get.description.toLowerCase();
+  // It points at download/grep (or curl|grep) as the extraction pattern.
+  assert.match(blob, /grep/, 'mentions grep as the extraction tool');
+  assert.match(blob, /download|curl/, 'points at download/curl to disk first');
+  // It frames the task class — extracting specific values.
+  assert.match(blob, /extract|specific value|color|version|id/, 'names the extract-values task');
+  // It warns that raw is expensive for simple value extraction.
+  assert.match(blob, /raw/, 'still describes raw');
+});
+
+test('http_get spec notes the SPA / linked-asset case', () => {
+  const blob = TOOL_SPECS.http_get.description.toLowerCase();
+  // Values may live in linked assets, not the top-level HTML.
+  assert.match(blob, /asset|_nuxt|stylesheet|bundle|\.css|\.js/, 'mentions linked assets / SPA case');
+});
+
+// ---------------------------------------------------------------------------
+// The system prompt carries the same guidance (covers the XML tag rail)
+// ---------------------------------------------------------------------------
+
+test('system prompt teaches fetch+grep for value extraction (XML + native)', () => {
+  for (const native of [false, true]) {
+    const prompt = getSystemPrompt(native, '', '').toLowerCase();
+    assert.match(prompt, /grep/, `native=${native}: mentions grep`);
+    assert.match(prompt, /download|curl/, `native=${native}: mentions download/curl`);
+    assert.match(prompt, /extract/, `native=${native}: names the extract task`);
+  }
+});
+
+// ---------------------------------------------------------------------------
+// The stale prompts.js http_get description is corrected
+// ---------------------------------------------------------------------------
+
+test('http_get tag purpose no longer claims a raw byte-cap response body', () => {
+  const { TOOL_TAG_SPECS } = require('../lib/prompts');
+  const purpose = (TOOL_TAG_SPECS.http_get.purpose || '').toLowerCase();
+  // The old text described http_get as "returns the response body (truncated to
+  // a byte cap …)" — pre-W.1. It now runs the extract/summarize pipeline.
+  assert.ok(
+    !/response body \(truncated to a byte cap/.test(purpose),
+    'stale byte-cap-body description must be gone',
+  );
+  // The accurate description references the pipeline / modes / token cap.
+  assert.match(purpose, /summari|extract|mode|token|pipeline/, 'describes the W.1 pipeline');
+});

package/test/web-extract.test.js

@@ -0,0 +1,185 @@
+'use strict';
+
+// Unit tests for the web-fetch extraction + summary-request stages (Task W.1).
+// Network-free: they run Readability + Turndown over a fixture HTML page and the
+// pure summary-message builder. The pipeline integration (http_get end-to-end
+// with a mock fetch + mock summarizer) lives in test/web-fetch-agent.test.js.
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+
+const {
+  classifyContentType,
+  extractContent,
+  capToTokens,
+  defaultEstimate,
+} = require('../lib/web-extract');
+const { buildSummaryMessages, summarizeWebContent, FENCE_OPEN } = require('../lib/web-summarize');
+const { HTML, INJECTION } = require('./fixtures/web-page');
+
+// ---------------------------------------------------------------------------
+// Content-type classification
+// ---------------------------------------------------------------------------
+
+test('classifyContentType: html / json / text / markdown by content-type', () => {
+  assert.strictEqual(classifyContentType('text/html; charset=utf-8', 'http://x', ''), 'html');
+  assert.strictEqual(classifyContentType('application/json', 'http://x/api', '{}'), 'json');
+  assert.strictEqual(classifyContentType('application/vnd.api+json', 'http://x', '{}'), 'json');
+  assert.strictEqual(classifyContentType('text/plain', 'http://x/readme.txt', 'hi'), 'text');
+  assert.strictEqual(classifyContentType('text/markdown', 'http://x/r.md', '# hi'), 'markdown');
+  // .md served as text/plain is still markdown
+  assert.strictEqual(classifyContentType('text/plain', 'http://x/README.md', '# hi'), 'markdown');
+});
+
+test('classifyContentType: sniffs HTML when content-type is absent/generic', () => {
+  assert.strictEqual(classifyContentType('', 'http://x', '<!doctype html><html><body>hi</body></html>'), 'html');
+  assert.strictEqual(classifyContentType('application/octet-stream', 'http://x', '<div>x</div>'), 'html');
+  assert.strictEqual(classifyContentType('', 'http://x', 'just some plain text, no tags here'), 'text');
+});
+
+// ---------------------------------------------------------------------------
+// Extraction: HTML → clean Markdown of the MAIN content
+// ---------------------------------------------------------------------------
+
+test('extractContent: HTML yields clean Markdown of the article; nav/scripts/ads gone', () => {
+  const { kind, markdown, extracted, title } = extractContent({ body: HTML, contentType: 'text/html', url: 'http://x/docs' });
+  assert.strictEqual(kind, 'html');
+  assert.strictEqual(extracted, true);
+
+  // Title is recovered as metadata.
+  assert.match(title || '', /HyperWidget Layout Phases/);
+  // Main content survives.
+  assert.match(markdown, /layout phases/i);
+  assert.match(markdown, /ctx\.cancel\(\)/);
+
+  // Chrome is gone.
+  assert.ok(!/SPONSORED/.test(markdown), 'ad copy must be dropped');
+  assert.ok(!/Accept all/.test(markdown), 'cookie banner must be dropped');
+  assert.ok(!/All rights reserved/.test(markdown), 'footer must be dropped');
+  assert.ok(!/Sign up/.test(markdown), 'nav must be dropped');
+
+  // No executable markup leaks through (script bodies, style).
+  assert.ok(!/dataLayer/.test(markdown), 'inline script body must be dropped');
+  assert.ok(!/footer analytics beacon/.test(markdown), 'trailing script must be dropped');
+  assert.ok(!/<script/i.test(markdown) && !/font-family/.test(markdown), 'no script/style markup');
+});
+
+test('TOKEN VOLUME: extraction alone yields a large reduction vs raw HTML (chrome dropped)', () => {
+  const { markdown } = extractContent({ body: HTML, contentType: 'text/html', url: 'http://x/docs' });
+  const rawTokens = defaultEstimate(HTML);
+  const extractedTokens = defaultEstimate(markdown);
+  // Extraction alone (before summarization) already cuts the page substantially
+  // by dropping scripts/nav/ads/JSON-LD/CSS. The ORDER-OF-MAGNITUDE reduction —
+  // "the result entering context vs raw HTML" — is asserted on the full
+  // summarized pipeline in test/web-fetch-agent.test.js.
+  assert.ok(extractedTokens > 0, 'extraction produced content');
+  assert.ok(
+    extractedTokens * 3 < rawTokens,
+    `expected >=3x token reduction from extraction, got raw=${rawTokens} extracted=${extractedTokens}`,
+  );
+});
+
+test('extractContent: JSON passes through verbatim (no mangling)', () => {
+  const json = JSON.stringify({ a: 1, b: [2, 3], c: { d: '<not html>' } });
+  const { kind, markdown, extracted } = extractContent({ body: json, contentType: 'application/json', url: 'http://x/api' });
+  assert.strictEqual(kind, 'json');
+  assert.strictEqual(extracted, false);
+  assert.strictEqual(markdown, json);
+});
+
+test('extractContent: plain text passes through verbatim', () => {
+  const txt = 'line one\nline two\n  indented three';
+  const { kind, markdown } = extractContent({ body: txt, contentType: 'text/plain', url: 'http://x/f.txt' });
+  assert.strictEqual(kind, 'text');
+  assert.strictEqual(markdown, txt);
+});
+
+// ---------------------------------------------------------------------------
+// Token budget
+// ---------------------------------------------------------------------------
+
+test('capToTokens: under budget is unchanged; over budget is truncated with a notice', () => {
+  const small = 'hello world';
+  const r1 = capToTokens(small, 6000, defaultEstimate);
+  assert.strictEqual(r1.truncated, false);
+  assert.strictEqual(r1.text, small);
+
+  const big = 'x'.repeat(40000); // ~10k tokens at char/4
+  const r2 = capToTokens(big, 1000, defaultEstimate);
+  assert.strictEqual(r2.truncated, true);
+  assert.match(r2.text, /\[\.\.\. truncated/);
+  // Capped to ~ the budget in chars (+ the notice), far below the original.
+  assert.ok(r2.text.length < big.length / 5);
+});
+
+// ---------------------------------------------------------------------------
+// Markup-aware token estimate (Task W.4 Part 2)
+// ---------------------------------------------------------------------------
+
+test('markupEstimate: denser than the prose char/4 estimate', () => {
+  const { markupEstimate, MARKUP_CHARS_PER_TOKEN, DEFAULT_CHARS_PER_TOKEN } = require('../lib/web-extract');
+  assert.ok(MARKUP_CHARS_PER_TOKEN < DEFAULT_CHARS_PER_TOKEN, 'markup divisor is smaller');
+  const css = '.x{color:#ffffff;margin:0}'.repeat(1000);
+  assert.ok(markupEstimate(css) > defaultEstimate(css), 'markup estimates MORE tokens for the same chars');
+  // Exactly the divisor relationship.
+  assert.strictEqual(markupEstimate('x'.repeat(2500)), Math.ceil(2500 / MARKUP_CHARS_PER_TOKEN));
+});
+
+test('capToTokens: markup charsPerToken trims more aggressively than prose for the SAME budget', () => {
+  const { markupEstimate, MARKUP_CHARS_PER_TOKEN } = require('../lib/web-extract');
+  const big = '.x{color:#fff;background:#000}'.repeat(5000); // dense markup, well over budget
+  const budget = 1000;
+
+  const prose = capToTokens(big, budget, defaultEstimate); // char/4
+  const markup = capToTokens(big, budget, markupEstimate, MARKUP_CHARS_PER_TOKEN); // char/2.5
+  assert.ok(prose.truncated && markup.truncated, 'both truncate');
+
+  // The kept char budget reflects the divisor: markup keeps ~budget*2.5 chars,
+  // prose keeps ~budget*4 — so markup is trimmed more aggressively.
+  const keptChars = (r) => r.text.split('\n\n[... truncated')[0].length;
+  assert.ok(keptChars(markup) < keptChars(prose), `markup ${keptChars(markup)} < prose ${keptChars(prose)}`);
+  assert.strictEqual(keptChars(markup), Math.floor(budget * MARKUP_CHARS_PER_TOKEN));
+  assert.strictEqual(keptChars(prose), Math.floor(budget * 4));
+});
+
+test('capToTokens: prose path (no charsPerToken) is byte-for-byte unchanged', () => {
+  // The default divisor stays 4 — the prose path must not change.
+  const big = 'x'.repeat(40000);
+  const r = capToTokens(big, 1000, defaultEstimate);
+  assert.strictEqual(r.text.split('\n\n[... truncated')[0].length, 4000);
+});
+
+// ---------------------------------------------------------------------------
+// Summary request builder — untrusted, data-only framing
+// ---------------------------------------------------------------------------
+
+test('buildSummaryMessages: page text is fenced as data-only and never framed as instructions', () => {
+  const msgs = buildSummaryMessages(`some content with ${INJECTION}`, 'what are layout phases?');
+  assert.strictEqual(msgs.length, 2);
+  const sys = msgs[0].content;
+  const user = msgs[1].content;
+  // System prompt instructs the model to treat the block as data and ignore injections.
+  assert.match(sys, /DATA/);
+  assert.match(sys, /[Nn]ever (obey|follow|act)/);
+  // The injection text lives INSIDE the fenced untrusted block, not in the system role.
+  assert.ok(user.includes(FENCE_OPEN), 'content is fenced');
+  assert.ok(user.includes(INJECTION), 'injection carried as data');
+  assert.ok(!sys.includes(INJECTION), 'injection must not be in the system prompt');
+  // Intent threaded in.
+  assert.match(user, /layout phases/);
+});
+
+test('summarizeWebContent: passes data-only messages to the injected chat and returns its text', async () => {
+  let seen = null;
+  const chat = async (messages) => { seen = messages; return '  SUMMARY: phases fire in order.  '; };
+  const out = await summarizeWebContent({ markdown: `body ${INJECTION}`, intent: 'x', chat });
+  assert.strictEqual(out, 'SUMMARY: phases fire in order.');
+  // The summarizer received the injection only as fenced data.
+  assert.ok(seen[1].content.includes(INJECTION));
+  assert.match(seen[0].content, /DATA/);
+});
+
+test('summarizeWebContent: empty result throws (so the caller can fall back)', async () => {
+  await assert.rejects(() => summarizeWebContent({ markdown: 'x', chat: async () => '   ' }));
+  await assert.rejects(() => summarizeWebContent({ markdown: 'x', chat: null }));
+});

package/test/web-fetch-agent.test.js

@@ -0,0 +1,291 @@
+'use strict';
+
+// Integration tests for the http_get web-fetch pipeline (Task W.1). A real local
+// HTTP server serves the fixture page; the executor runs the real extraction
+// (Readability + Turndown) and an INJECTED mock summarizer (no real LLM). These
+// prove the end-to-end contract: only the processed result enters context, the
+// raw page never does, summarize on/off, untrusted handling, failure fallback,
+// pass-through, and the token budget.
+
+const os = require('node:os');
+const fs = require('node:fs');
+const path = require('node:path');
+
+// Redirect home-based paths into a temp dir before any lib loads (audit log etc).
+const TMP_HOME = fs.mkdtempSync(path.join(os.tmpdir(), 'semalt-web-home-'));
+const PREV_HOME = process.env.HOME;
+const PREV_USERPROFILE = process.env.USERPROFILE;
+process.env.HOME = TMP_HOME;
+process.env.USERPROFILE = TMP_HOME;
+
+const { test, before, after } = require('node:test');
+const assert = require('node:assert');
+const http = require('node:http');
+
+const ui = require('../lib/ui');
+const { createPermissionManager } = require('../lib/permissions');
+const { createToolExecutor } = require('../lib/tools');
+const { defaultEstimate } = require('../lib/web-extract');
+const { HTML, INJECTION, INJECTION_MARKER } = require('./fixtures/web-page');
+
+let server;
+let baseUrl;
+
+// Start a fixture server: GET / → the HTML page; /api.json → JSON; /plain → text.
+before(async () => {
+  server = http.createServer((req, res) => {
+    if (req.url.startsWith('/api.json')) {
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({ name: 'widget', tags: ['a', 'b'], note: '<b>not html</b>' }));
+      return;
+    }
+    if (req.url.startsWith('/plain')) {
+      res.writeHead(200, { 'Content-Type': 'text/plain' });
+      res.end('plain line one\nplain line two');
+      return;
+    }
+    res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+    res.end(HTML);
+  });
+  await new Promise((r) => server.listen(0, '127.0.0.1', r));
+  baseUrl = `http://127.0.0.1:${server.address().port}`;
+});
+
+after(async () => {
+  await new Promise((r) => server.close(r));
+  if (PREV_HOME === undefined) delete process.env.HOME; else process.env.HOME = PREV_HOME;
+  if (PREV_USERPROFILE === undefined) delete process.env.USERPROFILE; else process.env.USERPROFILE = PREV_USERPROFILE;
+});
+
+// Build an executor with a configurable web config + a recording mock summarizer.
+function makeExec({ web, webChat } = {}) {
+  const pm = createPermissionManager(ui, {});
+  const getConfig = () => ({
+    max_file_size_kb: 512,
+    command_timeout_ms: 30000,
+    http_fetch_max_bytes: 262144,
+    web: web || { summarize: true, summary_model: '', max_content_tokens: 6000 },
+  });
+  return createToolExecutor(pm, ui, getConfig, { webChat });
+}
+
+// Invoke http_get the way the agent loop does: trailing { signal } options bag.
+function fetchUrl(exec, url, callOpts = {}) {
+  return exec.agentExecFile('http_get', url, callOpts, { signal: null });
+}
+
+// ---------------------------------------------------------------------------
+
+test('summarize ON: only the summary enters context; the raw page never does', async () => {
+  let calls = 0;
+  let summarizerSawInjection = false;
+  const webChat = async (messages) => {
+    calls += 1;
+    // The summarizer reads the page only as fenced data.
+    summarizerSawInjection = messages[1].content.includes(INJECTION_MARKER);
+    return 'SUMMARY: layout-phase handlers run synchronously in registration order; ctx.cancel() aborts.';
+  };
+  const exec = makeExec({ webChat });
+  const r = await fetchUrl(exec, `${baseUrl}/`, { intent: 'how do handlers run?' });
+
+  assert.strictEqual(calls, 1, 'secondary summarizer was called exactly once');
+  assert.strictEqual(r.summarized, true);
+  assert.strictEqual(r.kind, 'html');
+  assert.match(r.body, /SUMMARY:/);
+  // The summarizer received the page (incl. injection) as DATA.
+  assert.ok(summarizerSawInjection, 'summarizer received the page content');
+  // The RAW page never enters the returned body.
+  assert.ok(!/dataLayer|SPONSORED|<script|font-family/.test(r.body), 'no raw page markup in body');
+  assert.ok(!/included to give the article enough weight/.test(r.body), 'no extracted full text in body either');
+});
+
+test('TOKEN VOLUME: summarized result entering context is an order of magnitude smaller than raw HTML', async () => {
+  const webChat = async () => 'Handlers fire in registration order; ctx.cancel() aborts a transition. ~40 words of summary text here to be realistic about a real summary length for a docs page about layout phases and handlers.';
+  const exec = makeExec({ webChat });
+  const r = await fetchUrl(exec, `${baseUrl}/`);
+  const rawTokens = defaultEstimate(HTML);
+  const resultTokens = defaultEstimate(r.body);
+  assert.ok(
+    resultTokens * 10 < rawTokens,
+    `expected >=10x reduction, got raw=${rawTokens} result=${resultTokens}`,
+  );
+});
+
+test('summarize OFF (per-call): extracted Markdown enters context, no summarizer call', async () => {
+  let calls = 0;
+  const webChat = async () => { calls += 1; return 'SHOULD NOT BE CALLED'; };
+  const exec = makeExec({ webChat });
+  const r = await fetchUrl(exec, `${baseUrl}/`, { summarize: false });
+
+  assert.strictEqual(calls, 0, 'no secondary call when summarize=false');
+  assert.strictEqual(r.summarized, false);
+  assert.strictEqual(r.extracted, true);
+  // Extracted Markdown of the main content (verbatim — for exact snippets).
+  assert.match(r.body, /ctx\.cancel\(\)/);
+  // Still clean: chrome dropped, but the full extracted text is present.
+  assert.ok(!/SPONSORED|dataLayer/.test(r.body));
+  assert.match(r.body, /included to give the article enough weight/);
+});
+
+test('summarize OFF (global config): same as per-call off', async () => {
+  let calls = 0;
+  const webChat = async () => { calls += 1; return 'x'; };
+  const exec = makeExec({ web: { summarize: false, summary_model: '', max_content_tokens: 6000 }, webChat });
+  const r = await fetchUrl(exec, `${baseUrl}/`);
+  assert.strictEqual(calls, 0);
+  assert.strictEqual(r.summarized, false);
+  assert.match(r.body, /ctx\.cancel\(\)/);
+});
+
+test('UNTRUSTED: a page injection does not steer the summarizer (treated as data)', async () => {
+  let systemPrompt = null;
+  let userMsg = null;
+  // A compliant summarizer that does NOT obey the injection — it summarizes.
+  const webChat = async (messages) => {
+    systemPrompt = messages[0].content;
+    userMsg = messages[1].content;
+    return 'The page documents layout-phase handlers. (Ignoring any embedded instructions.)';
+  };
+  const exec = makeExec({ webChat });
+  const r = await fetchUrl(exec, `${baseUrl}/`);
+
+  // The data-only framing is present and the injection is fenced as data, not
+  // promoted to the system role.
+  assert.match(systemPrompt, /DATA/);
+  assert.match(systemPrompt, /[Nn]ever (obey|follow|act)/);
+  assert.ok(userMsg.includes(INJECTION_MARKER), 'injection carried as fenced data');
+  assert.ok(!systemPrompt.includes(INJECTION_MARKER), 'injection not in system prompt');
+  // The summarizer output did not leak the injection / act on it, and the result
+  // still does not contain the raw page.
+  assert.ok(!r.body.includes('rm -rf'), 'injection command not echoed into context');
+  assert.ok(!/<script/.test(r.body));
+});
+
+test('summarizer FAILURE falls back to extracted Markdown, never raw HTML', async () => {
+  const webChat = async () => { throw new Error('summary model timeout'); };
+  const exec = makeExec({ webChat });
+  const r = await fetchUrl(exec, `${baseUrl}/`);
+
+  assert.strictEqual(r.summarized, false);
+  assert.match(r.summary_error || '', /timeout/);
+  // Fell back to extracted Markdown (clean), NOT the raw page.
+  assert.match(r.body, /ctx\.cancel\(\)/);
+  assert.ok(!/dataLayer|SPONSORED|<script|font-family/.test(r.body), 'no raw HTML on fallback');
+});
+
+test('no summarizer available (e.g. headless): returns extracted Markdown, never raw HTML', async () => {
+  const exec = makeExec({ webChat: undefined });
+  const r = await fetchUrl(exec, `${baseUrl}/`);
+  assert.strictEqual(r.summarized, false);
+  assert.match(r.body, /ctx\.cancel\(\)/);
+  assert.ok(!/<script|dataLayer/.test(r.body));
+});
+
+test('JSON passes through without mangling and is not summarized', async () => {
+  let calls = 0;
+  const webChat = async () => { calls += 1; return 'x'; };
+  const exec = makeExec({ webChat });
+  const r = await fetchUrl(exec, `${baseUrl}/api.json`);
+  assert.strictEqual(r.kind, 'json');
+  assert.strictEqual(r.summarized, false);
+  assert.strictEqual(calls, 0, 'JSON is not sent to the summarizer');
+  const parsed = JSON.parse(r.body);
+  assert.strictEqual(parsed.name, 'widget');
+  assert.strictEqual(parsed.note, '<b>not html</b>');
+});
+
+test('plain text passes through without mangling', async () => {
+  const exec = makeExec({ webChat: async () => 'x' });
+  const r = await fetchUrl(exec, `${baseUrl}/plain`);
+  assert.strictEqual(r.kind, 'text');
+  assert.strictEqual(r.body, 'plain line one\nplain line two');
+});
+
+test('token budget caps oversized extracted content with a notice', async () => {
+  // Tiny budget forces truncation of the extracted Markdown; summarize off so we
+  // observe the capped content directly.
+  const exec = makeExec({ web: { summarize: false, summary_model: '', max_content_tokens: 300 } });
+  const r = await fetchUrl(exec, `${baseUrl}/`, { summarize: false });
+  assert.strictEqual(r.content_truncated, true);
+  assert.match(r.body, /\[\.\.\. truncated/);
+  assert.ok(defaultEstimate(r.body) <= 300 + 60, 'capped near the token budget');
+});
+
+// ---------------------------------------------------------------------------
+// Mode enum (Task W.1b): raw / extracted / summarized end-to-end
+// ---------------------------------------------------------------------------
+
+test('mode="raw" REGRESSION: original HTML reaches context (markup intact), no summarizer call', async () => {
+  let calls = 0;
+  // A generous budget so the WHOLE fixture is retained — this test isolates
+  // "raw returns the original markup" from the token cap (covered just below).
+  // The default 6000-token budget now trims the fixture's tail under the
+  // markup-aware estimate (Task W.4 Part 2), so SPONSORED (deep in the body)
+  // would otherwise be capped away.
+  const exec = makeExec({ web: { summarize: true, summary_model: '', max_content_tokens: 100000 }, webChat: async () => { calls += 1; return 'NOPE'; } });
+  const r = await fetchUrl(exec, `${baseUrl}/`, { mode: 'raw' });
+  assert.strictEqual(calls, 0, 'raw never summarizes');
+  assert.strictEqual(r.mode, 'raw');
+  assert.strictEqual(r.summarized, false);
+  assert.strictEqual(r.extracted, false);
+  assert.strictEqual(r.kind, 'html');
+  // The things extraction destroys are present — raw HTML access is restored.
+  assert.match(r.body, /<script/);
+  assert.match(r.body, /<style/);
+  assert.match(r.body, /font-family/);
+  assert.match(r.body, /dataLayer/);
+  assert.match(r.body, /SPONSORED/);
+});
+
+test('mode="raw" still applies the token budget (raw HTML is heavier, capped)', async () => {
+  const exec = makeExec({ web: { summarize: true, summary_model: '', max_content_tokens: 300 } });
+  const r = await fetchUrl(exec, `${baseUrl}/`, { mode: 'raw' });
+  assert.strictEqual(r.content_truncated, true);
+  assert.match(r.body, /\[\.\.\. truncated/);
+  assert.ok(defaultEstimate(r.body) <= 300 + 60, 'capped near the token budget');
+});
+
+test('mode="extracted" == legacy summarize=false (Markdown, no summary)', async () => {
+  let calls = 0;
+  const exec = makeExec({ webChat: async () => { calls += 1; return 'x'; } });
+  const a = await fetchUrl(exec, `${baseUrl}/`, { mode: 'extracted' });
+  const b = await fetchUrl(exec, `${baseUrl}/`, { summarize: false });
+  assert.strictEqual(calls, 0);
+  assert.strictEqual(a.summarized, false);
+  assert.strictEqual(a.extracted, true);
+  assert.strictEqual(b.summarized, false);
+  assert.strictEqual(a.body, b.body, 'extracted mode and legacy summarize=false are identical');
+  assert.match(a.body, /ctx\.cancel\(\)/);
+  assert.ok(!/<script|SPONSORED/.test(a.body));
+});
+
+test('mode="summarized" == the default behavior', async () => {
+  const webChat = async () => 'SUMMARY: handlers fire in registration order.';
+  const exec = makeExec({ webChat });
+  const a = await fetchUrl(exec, `${baseUrl}/`, { mode: 'summarized' });
+  const b = await fetchUrl(exec, `${baseUrl}/`); // default
+  assert.strictEqual(a.summarized, true);
+  assert.strictEqual(b.summarized, true);
+  assert.match(a.body, /SUMMARY:/);
+  assert.strictEqual(a.body, b.body);
+});
+
+test('back-compat: legacy raw="true" resolves to extracted (NOT raw HTML)', async () => {
+  const exec = makeExec({ webChat: async () => 'x' });
+  const r = await fetchUrl(exec, `${baseUrl}/`, { raw: true });
+  // The deprecated raw=true alias still means "extracted Markdown", as in W.1.
+  assert.notStrictEqual(r.mode, 'raw');
+  assert.strictEqual(r.summarized, false);
+  assert.strictEqual(r.extracted, true);
+  assert.match(r.body, /ctx\.cancel\(\)/);
+  assert.ok(!/<script|font-family/.test(r.body), 'raw=true alias does NOT leak HTML');
+});
+
+test('precedence: explicit mode="raw" beats a legacy summarize=true', async () => {
+  let calls = 0;
+  const exec = makeExec({ webChat: async () => { calls += 1; return 'x'; } });
+  const r = await fetchUrl(exec, `${baseUrl}/`, { mode: 'raw', summarize: true });
+  assert.strictEqual(calls, 0);
+  assert.strictEqual(r.mode, 'raw');
+  assert.match(r.body, /<script/);
+});