@semalt-ai/code 1.8.5 → 1.19.0

package/test/web-activity.test.js

@@ -0,0 +1,207 @@
+'use strict';
+
+// Web-activity process summary (Task W.3, Part 1). The default view collapses a
+// run of web ops (web_search → http_get) into ONE compact summary line; --debug
+// keeps the full per-operation lines. These tests pin the pure renderer (counts:
+// queries / sources read / failures), the debug-vs-default branch, that a failed
+// fetch (403/timeout) is reflected (not dropped), that non-web tools are out of
+// scope, and the stateful tracker's collapse-to-one-committed-line behaviour.
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+
+const { stripAnsi } = require('../lib/ui/utils');
+const { formatToolLine } = require('../lib/ui/format');
+const {
+  isWebTool,
+  opSucceeded,
+  aggregateWebOps,
+  webSummaryText,
+  formatWebSummaryLine,
+  renderWebActivity,
+  createWebActivityTracker,
+} = require('../lib/ui/web-activity');
+
+// ---------------------------------------------------------------------------
+// Scope: which tools are collapsed
+// ---------------------------------------------------------------------------
+
+test('isWebTool: only web_search and http_get are in scope', () => {
+  assert.strictEqual(isWebTool('web_search'), true);
+  assert.strictEqual(isWebTool('http_get'), true);
+  // download writes a file, not a page read — keeps its own line.
+  assert.strictEqual(isWebTool('download'), false);
+  assert.strictEqual(isWebTool('shell'), false);
+  assert.strictEqual(isWebTool('read_file'), false);
+  assert.strictEqual(isWebTool('write_file'), false);
+});
+
+// ---------------------------------------------------------------------------
+// Success classification (the 403/406 "blocked" rule)
+// ---------------------------------------------------------------------------
+
+test('opSucceeded: http_get >= 400 is a failure even with no transport error', () => {
+  assert.strictEqual(opSucceeded({ tag: 'http_get', status: 200 }), true);
+  assert.strictEqual(opSucceeded({ tag: 'http_get', status: 403 }), false);
+  assert.strictEqual(opSucceeded({ tag: 'http_get', status: 406 }), false);
+  // A transport error (timeout/DNS) is a failure regardless of status.
+  assert.strictEqual(opSucceeded({ tag: 'http_get', error: 'Request timeout' }), false);
+  // web_search is ok unless the backend errored.
+  assert.strictEqual(opSucceeded({ tag: 'web_search' }), true);
+  assert.strictEqual(opSucceeded({ tag: 'web_search', error: 'web search unavailable' }), false);
+});
+
+// ---------------------------------------------------------------------------
+// Pure summary text — reflects queries, sources read, failures
+// ---------------------------------------------------------------------------
+
+test('webSummaryText: reflects query count, sources read, and blocked count', () => {
+  const ops = [
+    { tag: 'web_search', query: 'коррупционные скандалы 2024' },
+    { tag: 'web_search', query: 'follow-up query' },
+    { tag: 'http_get', url: 'https://a.example/1', status: 200 },
+    { tag: 'http_get', url: 'https://b.example/2', status: 200 },
+    { tag: 'http_get', url: 'https://ru.wikipedia.org/x', status: 403 },
+  ];
+  const text = webSummaryText(aggregateWebOps(ops));
+  assert.match(text, /search "коррупционные/);   // leads with the query
+  assert.match(text, /2 queries/);               // query count visible
+  assert.match(text, /2 sources read/);          // successful reads
+  assert.match(text, /1 blocked/);               // the 403 is surfaced, not dropped
+});
+
+test('webSummaryText: a timeout counts as blocked, not silently dropped', () => {
+  const ops = [
+    { tag: 'http_get', url: 'https://slow.example', error: 'Request timeout' },
+    { tag: 'http_get', url: 'https://ok.example', status: 200 },
+  ];
+  const text = webSummaryText(aggregateWebOps(ops));
+  assert.match(text, /1 source read/);
+  assert.match(text, /1 blocked/);
+});
+
+test('webSummaryText: a failed web_search is surfaced', () => {
+  const ops = [{ tag: 'web_search', query: 'q', error: 'web search unavailable: backend down' }];
+  const text = webSummaryText(aggregateWebOps(ops));
+  assert.match(text, /search failed/);
+});
+
+test('webSummaryText: fetch-only flow (no search) still reads cleanly', () => {
+  const ops = [{ tag: 'http_get', url: 'https://x', status: 200 }];
+  assert.match(webSummaryText(aggregateWebOps(ops)), /1 source read/);
+});
+
+test('aggregateWebOps: counts are exact', () => {
+  const s = aggregateWebOps([
+    { tag: 'web_search', query: 'a' },
+    { tag: 'http_get', status: 200 },
+    { tag: 'http_get', status: 200 },
+    { tag: 'http_get', status: 500 },
+  ]);
+  assert.deepStrictEqual(
+    { searchCount: s.searchCount, fetchCount: s.fetchCount, fetchOk: s.fetchOk, fetchFailed: s.fetchFailed },
+    { searchCount: 1, fetchCount: 3, fetchOk: 2, fetchFailed: 1 },
+  );
+});
+
+// ---------------------------------------------------------------------------
+// renderWebActivity — debug branch keeps full per-op detail; default collapses
+// ---------------------------------------------------------------------------
+
+const SAMPLE_OPS = [
+  { tag: 'web_search', query: 'how do tariffs work', durationMs: 941 },
+  { tag: 'http_get', url: 'https://24tv.ua/article', status: 200, bytes: 406 * 1024, durationMs: 171 },
+  { tag: 'http_get', url: 'https://ru.wikipedia.org/page', status: 403, bytes: 126, durationMs: 25 },
+];
+
+test('renderWebActivity (default): a sequence of web ops → ONE compact summary line', () => {
+  const lines = renderWebActivity(SAMPLE_OPS, { debug: false, formatToolLine });
+  assert.strictEqual(lines.length, 1, 'collapsed to a single line');
+  const plain = stripAnsi(lines[0]);
+  assert.match(plain, /web/);
+  assert.match(plain, /search "how do tariffs work"/);
+  assert.match(plain, /1 source read/);
+  assert.match(plain, /1 blocked/);
+});
+
+test('renderWebActivity (--debug): full per-operation lines, nothing hidden', () => {
+  const lines = renderWebActivity(SAMPLE_OPS, { debug: true, formatToolLine });
+  assert.strictEqual(lines.length, SAMPLE_OPS.length, 'one line per op');
+  const all = lines.map(stripAnsi);
+  // The query and both URLs survive in the detailed view.
+  assert.ok(all.some((l) => /how do tariffs work/.test(l)));
+  assert.ok(all.some((l) => /24tv\.ua/.test(l)));
+  assert.ok(all.some((l) => /ru\.wikipedia\.org/.test(l)));
+  // The HTTP status codes (200 / 403) are present in the per-op meta.
+  assert.ok(all.some((l) => /\b200\b/.test(l)));
+  assert.ok(all.some((l) => /\b403\b/.test(l)));
+});
+
+// ---------------------------------------------------------------------------
+// Styled line: glyph + failures coloured, plain text correct
+// ---------------------------------------------------------------------------
+
+test('formatWebSummaryLine: pending shows ●, committed shows ✓', () => {
+  const state = aggregateWebOps(SAMPLE_OPS);
+  assert.match(formatWebSummaryLine(state, { pending: true, durationMs: 500 }), /●/);
+  assert.match(formatWebSummaryLine(state, { pending: false }), /✓/);
+});
+
+// ---------------------------------------------------------------------------
+// Stateful tracker — collapse a multi-op group into one committed line
+// ---------------------------------------------------------------------------
+
+function fakeWriter() {
+  const calls = { start: [], update: 0, end: [] };
+  return {
+    calls,
+    startActivity(id) { calls.start.push(id); },
+    updateActivity() { calls.update += 1; },
+    endActivity(id, line) { calls.end.push({ id, line }); },
+  };
+}
+
+test('tracker: a run of web ops commits exactly ONE summary line on flush', () => {
+  const w = fakeWriter();
+  const t = createWebActivityTracker({ writerModule: w });
+
+  t.start('web_search', 'коррупционные скандалы');
+  t.end('web_search', { results: [] }, 900, { attrs: { query: 'коррупционные скандалы' } });
+  t.start('http_get', 'https://a.example');
+  t.end('http_get', {}, 170, { attrs: { url: 'https://a.example' }, meta: { status_code: 200, bytes: 1000 } });
+  t.start('http_get', 'https://ru.wikipedia.org/x');
+  t.end('http_get', {}, 25, { attrs: { url: 'https://ru.wikipedia.org/x' }, meta: { status_code: 403, bytes: 126 } });
+
+  assert.strictEqual(w.calls.start.length, 1, 'one activity opened for the whole group');
+  assert.strictEqual(t.isOpen(), true);
+
+  t.flush();
+  assert.strictEqual(t.isOpen(), false);
+  assert.strictEqual(w.calls.end.length, 1, 'one committed summary line');
+  const plain = stripAnsi(w.calls.end[0].line);
+  assert.match(plain, /search "коррупционные скандалы"/);
+  assert.match(plain, /1 source read/);
+  assert.match(plain, /1 blocked/);
+});
+
+test('tracker: flush with no open group is a no-op', () => {
+  const w = fakeWriter();
+  const t = createWebActivityTracker({ writerModule: w });
+  t.flush();
+  assert.strictEqual(w.calls.start.length, 0);
+  assert.strictEqual(w.calls.end.length, 0);
+});
+
+test('tracker: a second group after flush opens a fresh activity', () => {
+  const w = fakeWriter();
+  const t = createWebActivityTracker({ writerModule: w });
+  t.start('web_search', 'q1');
+  t.end('web_search', {}, 10, { attrs: { query: 'q1' } });
+  t.flush();
+  t.start('http_get', 'https://x');
+  t.end('http_get', {}, 10, { attrs: { url: 'https://x' }, meta: { status_code: 200 } });
+  t.flush();
+  assert.strictEqual(w.calls.start.length, 2, 'two distinct groups');
+  assert.strictEqual(w.calls.end.length, 2);
+  assert.notStrictEqual(w.calls.start[0], w.calls.start[1], 'distinct group ids');
+});

package/test/web-data-extraction-guidance.test.js

@@ -0,0 +1,71 @@
+'use strict';
+
+// Task W.4 Part 1 — guidance: teach fetch+grep for data-extraction tasks.
+//
+// "Extract specific values from a page" (colors/hex, versions, IDs, URLs,
+// counts) is a different task class from "read a page": the right pattern is
+// targeted matching (download→grep / curl|grep) so only the matches enter
+// context — NOT loading page content via any http_get mode. These tests pin
+// that the guidance text is present (the spec drives the model) and that the
+// stale prompts.js http_get description ("byte cap body") is corrected.
+//
+// NOTE: Part 1 is a guidance change; its real effect is behavioral (a live
+// re-run of the "what colors" task), not a unit assertion — these tests only
+// pin that the guidance exists and is discoverable to the model.
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+
+const { TOOL_SPECS } = require('../lib/tool_specs');
+const { getSystemPrompt } = require('../lib/prompts');
+
+// ---------------------------------------------------------------------------
+// The http_get spec guides the agent to fetch+grep for extracting values
+// ---------------------------------------------------------------------------
+
+test('http_get spec guides fetch+grep for extracting specific values', () => {
+  const blob = TOOL_SPECS.http_get.description.toLowerCase();
+  // It points at download/grep (or curl|grep) as the extraction pattern.
+  assert.match(blob, /grep/, 'mentions grep as the extraction tool');
+  assert.match(blob, /download|curl/, 'points at download/curl to disk first');
+  // It frames the task class — extracting specific values.
+  assert.match(blob, /extract|specific value|color|version|id/, 'names the extract-values task');
+  // It warns that raw is expensive for simple value extraction.
+  assert.match(blob, /raw/, 'still describes raw');
+});
+
+test('http_get spec notes the SPA / linked-asset case', () => {
+  const blob = TOOL_SPECS.http_get.description.toLowerCase();
+  // Values may live in linked assets, not the top-level HTML.
+  assert.match(blob, /asset|_nuxt|stylesheet|bundle|\.css|\.js/, 'mentions linked assets / SPA case');
+});
+
+// ---------------------------------------------------------------------------
+// The system prompt carries the same guidance (covers the XML tag rail)
+// ---------------------------------------------------------------------------
+
+test('system prompt teaches fetch+grep for value extraction (XML + native)', () => {
+  for (const native of [false, true]) {
+    const prompt = getSystemPrompt(native, '', '').toLowerCase();
+    assert.match(prompt, /grep/, `native=${native}: mentions grep`);
+    assert.match(prompt, /download|curl/, `native=${native}: mentions download/curl`);
+    assert.match(prompt, /extract/, `native=${native}: names the extract task`);
+  }
+});
+
+// ---------------------------------------------------------------------------
+// The stale prompts.js http_get description is corrected
+// ---------------------------------------------------------------------------
+
+test('http_get tag purpose no longer claims a raw byte-cap response body', () => {
+  const { TOOL_TAG_SPECS } = require('../lib/prompts');
+  const purpose = (TOOL_TAG_SPECS.http_get.purpose || '').toLowerCase();
+  // The old text described http_get as "returns the response body (truncated to
+  // a byte cap …)" — pre-W.1. It now runs the extract/summarize pipeline.
+  assert.ok(
+    !/response body \(truncated to a byte cap/.test(purpose),
+    'stale byte-cap-body description must be gone',
+  );
+  // The accurate description references the pipeline / modes / token cap.
+  assert.match(purpose, /summari|extract|mode|token|pipeline/, 'describes the W.1 pipeline');
+});

package/test/web-extract.test.js

@@ -0,0 +1,185 @@
+'use strict';
+
+// Unit tests for the web-fetch extraction + summary-request stages (Task W.1).
+// Network-free: they run Readability + Turndown over a fixture HTML page and the
+// pure summary-message builder. The pipeline integration (http_get end-to-end
+// with a mock fetch + mock summarizer) lives in test/web-fetch-agent.test.js.
+
+const { test } = require('node:test');
+const assert = require('node:assert');
+
+const {
+  classifyContentType,
+  extractContent,
+  capToTokens,
+  defaultEstimate,
+} = require('../lib/web-extract');
+const { buildSummaryMessages, summarizeWebContent, FENCE_OPEN } = require('../lib/web-summarize');
+const { HTML, INJECTION } = require('./fixtures/web-page');
+
+// ---------------------------------------------------------------------------
+// Content-type classification
+// ---------------------------------------------------------------------------
+
+test('classifyContentType: html / json / text / markdown by content-type', () => {
+  assert.strictEqual(classifyContentType('text/html; charset=utf-8', 'http://x', ''), 'html');
+  assert.strictEqual(classifyContentType('application/json', 'http://x/api', '{}'), 'json');
+  assert.strictEqual(classifyContentType('application/vnd.api+json', 'http://x', '{}'), 'json');
+  assert.strictEqual(classifyContentType('text/plain', 'http://x/readme.txt', 'hi'), 'text');
+  assert.strictEqual(classifyContentType('text/markdown', 'http://x/r.md', '# hi'), 'markdown');
+  // .md served as text/plain is still markdown
+  assert.strictEqual(classifyContentType('text/plain', 'http://x/README.md', '# hi'), 'markdown');
+});
+
+test('classifyContentType: sniffs HTML when content-type is absent/generic', () => {
+  assert.strictEqual(classifyContentType('', 'http://x', '<!doctype html><html><body>hi</body></html>'), 'html');
+  assert.strictEqual(classifyContentType('application/octet-stream', 'http://x', '<div>x</div>'), 'html');
+  assert.strictEqual(classifyContentType('', 'http://x', 'just some plain text, no tags here'), 'text');
+});
+
+// ---------------------------------------------------------------------------
+// Extraction: HTML → clean Markdown of the MAIN content
+// ---------------------------------------------------------------------------
+
+test('extractContent: HTML yields clean Markdown of the article; nav/scripts/ads gone', () => {
+  const { kind, markdown, extracted, title } = extractContent({ body: HTML, contentType: 'text/html', url: 'http://x/docs' });
+  assert.strictEqual(kind, 'html');
+  assert.strictEqual(extracted, true);
+
+  // Title is recovered as metadata.
+  assert.match(title || '', /HyperWidget Layout Phases/);
+  // Main content survives.
+  assert.match(markdown, /layout phases/i);
+  assert.match(markdown, /ctx\.cancel\(\)/);
+
+  // Chrome is gone.
+  assert.ok(!/SPONSORED/.test(markdown), 'ad copy must be dropped');
+  assert.ok(!/Accept all/.test(markdown), 'cookie banner must be dropped');
+  assert.ok(!/All rights reserved/.test(markdown), 'footer must be dropped');
+  assert.ok(!/Sign up/.test(markdown), 'nav must be dropped');
+
+  // No executable markup leaks through (script bodies, style).
+  assert.ok(!/dataLayer/.test(markdown), 'inline script body must be dropped');
+  assert.ok(!/footer analytics beacon/.test(markdown), 'trailing script must be dropped');
+  assert.ok(!/<script/i.test(markdown) && !/font-family/.test(markdown), 'no script/style markup');
+});
+
+test('TOKEN VOLUME: extraction alone yields a large reduction vs raw HTML (chrome dropped)', () => {
+  const { markdown } = extractContent({ body: HTML, contentType: 'text/html', url: 'http://x/docs' });
+  const rawTokens = defaultEstimate(HTML);
+  const extractedTokens = defaultEstimate(markdown);
+  // Extraction alone (before summarization) already cuts the page substantially
+  // by dropping scripts/nav/ads/JSON-LD/CSS. The ORDER-OF-MAGNITUDE reduction —
+  // "the result entering context vs raw HTML" — is asserted on the full
+  // summarized pipeline in test/web-fetch-agent.test.js.
+  assert.ok(extractedTokens > 0, 'extraction produced content');
+  assert.ok(
+    extractedTokens * 3 < rawTokens,
+    `expected >=3x token reduction from extraction, got raw=${rawTokens} extracted=${extractedTokens}`,
+  );
+});
+
+test('extractContent: JSON passes through verbatim (no mangling)', () => {
+  const json = JSON.stringify({ a: 1, b: [2, 3], c: { d: '<not html>' } });
+  const { kind, markdown, extracted } = extractContent({ body: json, contentType: 'application/json', url: 'http://x/api' });
+  assert.strictEqual(kind, 'json');
+  assert.strictEqual(extracted, false);
+  assert.strictEqual(markdown, json);
+});
+
+test('extractContent: plain text passes through verbatim', () => {
+  const txt = 'line one\nline two\n  indented three';
+  const { kind, markdown } = extractContent({ body: txt, contentType: 'text/plain', url: 'http://x/f.txt' });
+  assert.strictEqual(kind, 'text');
+  assert.strictEqual(markdown, txt);
+});
+
+// ---------------------------------------------------------------------------
+// Token budget
+// ---------------------------------------------------------------------------
+
+test('capToTokens: under budget is unchanged; over budget is truncated with a notice', () => {
+  const small = 'hello world';
+  const r1 = capToTokens(small, 6000, defaultEstimate);
+  assert.strictEqual(r1.truncated, false);
+  assert.strictEqual(r1.text, small);
+
+  const big = 'x'.repeat(40000); // ~10k tokens at char/4
+  const r2 = capToTokens(big, 1000, defaultEstimate);
+  assert.strictEqual(r2.truncated, true);
+  assert.match(r2.text, /\[\.\.\. truncated/);
+  // Capped to ~ the budget in chars (+ the notice), far below the original.
+  assert.ok(r2.text.length < big.length / 5);
+});
+
+// ---------------------------------------------------------------------------
+// Markup-aware token estimate (Task W.4 Part 2)
+// ---------------------------------------------------------------------------
+
+test('markupEstimate: denser than the prose char/4 estimate', () => {
+  const { markupEstimate, MARKUP_CHARS_PER_TOKEN, DEFAULT_CHARS_PER_TOKEN } = require('../lib/web-extract');
+  assert.ok(MARKUP_CHARS_PER_TOKEN < DEFAULT_CHARS_PER_TOKEN, 'markup divisor is smaller');
+  const css = '.x{color:#ffffff;margin:0}'.repeat(1000);
+  assert.ok(markupEstimate(css) > defaultEstimate(css), 'markup estimates MORE tokens for the same chars');
+  // Exactly the divisor relationship.
+  assert.strictEqual(markupEstimate('x'.repeat(2500)), Math.ceil(2500 / MARKUP_CHARS_PER_TOKEN));
+});
+
+test('capToTokens: markup charsPerToken trims more aggressively than prose for the SAME budget', () => {
+  const { markupEstimate, MARKUP_CHARS_PER_TOKEN } = require('../lib/web-extract');
+  const big = '.x{color:#fff;background:#000}'.repeat(5000); // dense markup, well over budget
+  const budget = 1000;
+
+  const prose = capToTokens(big, budget, defaultEstimate); // char/4
+  const markup = capToTokens(big, budget, markupEstimate, MARKUP_CHARS_PER_TOKEN); // char/2.5
+  assert.ok(prose.truncated && markup.truncated, 'both truncate');
+
+  // The kept char budget reflects the divisor: markup keeps ~budget*2.5 chars,
+  // prose keeps ~budget*4 — so markup is trimmed more aggressively.
+  const keptChars = (r) => r.text.split('\n\n[... truncated')[0].length;
+  assert.ok(keptChars(markup) < keptChars(prose), `markup ${keptChars(markup)} < prose ${keptChars(prose)}`);
+  assert.strictEqual(keptChars(markup), Math.floor(budget * MARKUP_CHARS_PER_TOKEN));
+  assert.strictEqual(keptChars(prose), Math.floor(budget * 4));
+});
+
+test('capToTokens: prose path (no charsPerToken) is byte-for-byte unchanged', () => {
+  // The default divisor stays 4 — the prose path must not change.
+  const big = 'x'.repeat(40000);
+  const r = capToTokens(big, 1000, defaultEstimate);
+  assert.strictEqual(r.text.split('\n\n[... truncated')[0].length, 4000);
+});
+
+// ---------------------------------------------------------------------------
+// Summary request builder — untrusted, data-only framing
+// ---------------------------------------------------------------------------
+
+test('buildSummaryMessages: page text is fenced as data-only and never framed as instructions', () => {
+  const msgs = buildSummaryMessages(`some content with ${INJECTION}`, 'what are layout phases?');
+  assert.strictEqual(msgs.length, 2);
+  const sys = msgs[0].content;
+  const user = msgs[1].content;
+  // System prompt instructs the model to treat the block as data and ignore injections.
+  assert.match(sys, /DATA/);
+  assert.match(sys, /[Nn]ever (obey|follow|act)/);
+  // The injection text lives INSIDE the fenced untrusted block, not in the system role.
+  assert.ok(user.includes(FENCE_OPEN), 'content is fenced');
+  assert.ok(user.includes(INJECTION), 'injection carried as data');
+  assert.ok(!sys.includes(INJECTION), 'injection must not be in the system prompt');
+  // Intent threaded in.
+  assert.match(user, /layout phases/);
+});
+
+test('summarizeWebContent: passes data-only messages to the injected chat and returns its text', async () => {
+  let seen = null;
+  const chat = async (messages) => { seen = messages; return '  SUMMARY: phases fire in order.  '; };
+  const out = await summarizeWebContent({ markdown: `body ${INJECTION}`, intent: 'x', chat });
+  assert.strictEqual(out, 'SUMMARY: phases fire in order.');
+  // The summarizer received the injection only as fenced data.
+  assert.ok(seen[1].content.includes(INJECTION));
+  assert.match(seen[0].content, /DATA/);
+});
+
+test('summarizeWebContent: empty result throws (so the caller can fall back)', async () => {
+  await assert.rejects(() => summarizeWebContent({ markdown: 'x', chat: async () => '   ' }));
+  await assert.rejects(() => summarizeWebContent({ markdown: 'x', chat: null }));
+});