@myagentroam/node 0.11.61 → 0.11.63

package/dist/service/web-snapshots.js

@@ -0,0 +1,188 @@
+import { randomUUID } from 'node:crypto';
+import { AgentError, countModelTokens } from '@myagentroam/agent';
+const DEFAULT_BUDGET = 2_000;
+const SNAPSHOT_BUDGET = 32 * 1024 * 1024;
+/** Process-local documents; callers supply the trusted storage/session scope. */
+export class WebSnapshots {
+    maxBytes;
+    #entries = new Map();
+    #bytes = 0;
+    constructor(maxBytes = SNAPSHOT_BUDGET) {
+        this.maxBytes = maxBytes;
+    }
+    add(scope, document) {
+        const ref = randomUUID();
+        const size = Buffer.byteLength(JSON.stringify({ ...document, scope, ref })) + 128;
+        if (size > this.maxBytes)
+            throw unavailable();
+        while (this.#bytes + size > this.maxBytes) {
+            const oldest = this.#entries.keys().next().value;
+            if (oldest === undefined)
+                break;
+            this.#delete(oldest);
+        }
+        this.#entries.set(ref, { ...document, scope, size });
+        this.#bytes += size;
+        return ref;
+    }
+    clearScope(scope) {
+        for (const [ref, entry] of this.#entries)
+            if (entry.scope === scope)
+                this.#delete(ref);
+    }
+    clearHostedStorage(storage) {
+        for (const [ref, entry] of this.#entries) {
+            try {
+                const scope = JSON.parse(entry.scope);
+                if (Array.isArray(scope) && scope[0] === 'hosted' && scope[1] === storage)
+                    this.#delete(ref);
+            }
+            catch {
+                /* Ignore unrelated scopes. */
+            }
+        }
+    }
+    clear() {
+        this.#entries.clear();
+        this.#bytes = 0;
+    }
+    open(scope, ref, options) {
+        const entry = this.#get(scope, ref);
+        const lines = entry.content.split('\n');
+        const budget = options.maxTokens ?? DEFAULT_BUDGET;
+        let line = options.line ?? 0;
+        let column = 0;
+        if (options.cursor !== undefined) {
+            const match = /^(\d+):(\d+)$/u.exec(options.cursor);
+            if (!match)
+                throw invalidCursor();
+            line = Number(match[1]);
+            column = Number(match[2]);
+        }
+        if (!Number.isSafeInteger(line) ||
+            !Number.isSafeInteger(column) ||
+            line < 0 ||
+            line >= lines.length ||
+            column < 0 ||
+            column > lines[line].length ||
+            (column > 0 && column < lines[line].length && /[\uDC00-\uDFFF]/u.test(lines[line][column])))
+            throw invalidCursor();
+        const startLine = line;
+        const header = `Ref: ${ref}\nTitle: ${short(entry.title ?? '', 160)}\nURL: ${short(entry.url, 240)}\n`;
+        let body = '';
+        let lastLine = line;
+        for (let seen = 0; line < lines.length && seen < 200; seen++) {
+            const remaining = lines[line].slice(column);
+            const points = Array.from(remaining);
+            const prefix = `${line}: `;
+            if (countModelTokens(header + body + prefix + '\n' + 'x'.repeat(180)) > budget)
+                break;
+            let low = 0;
+            let high = points.length;
+            while (low < high) {
+                const mid = Math.ceil((low + high) / 2);
+                if (countModelTokens(header + body + prefix + points.slice(0, mid).join('') + '\n' + 'x'.repeat(180)) <= budget)
+                    low = mid;
+                else
+                    high = mid - 1;
+            }
+            if (low === 0 && points.length > 0)
+                break;
+            const part = points.slice(0, low).join('');
+            body += prefix + part + '\n';
+            lastLine = line;
+            column += part.length;
+            if (column < lines[line].length)
+                break;
+            line++;
+            column = 0;
+        }
+        const hasMore = line < lines.length;
+        const nextCursor = hasMore ? `${line}:${column}` : undefined;
+        const footer = `Lines: ${startLine}-${lastLine} / ${lines.length}\nsourceTruncated: ${entry.sourceTruncated}\nhasMore: ${hasMore}${nextCursor ? `\nnextCursor: ${nextCursor}` : ''}`;
+        return {
+            content: header + body + footer,
+            data: {
+                ref,
+                url: entry.url,
+                title: entry.title,
+                status: entry.status,
+                contentType: entry.contentType,
+                format: /^text\/html\b/iu.test(entry.contentType) ? 'markdown' : 'text',
+                totalLines: lines.length,
+                startLine,
+                endLine: lastLine,
+                hasMore,
+                ...(nextCursor ? { nextCursor } : {}),
+                sourceTruncated: entry.sourceTruncated
+            }
+        };
+    }
+    find(scope, ref, pattern, maxTokens = DEFAULT_BUDGET) {
+        const entry = this.#get(scope, ref);
+        const needle = pattern.toLowerCase();
+        if (!needle)
+            throw new AgentError('MAR_AGENT_TOOL_INPUT_INVALID', 'Find pattern must not be empty.');
+        const matchLines = [];
+        const lines = entry.content.split('\n');
+        for (let i = 0; i < lines.length; i++)
+            if (lines[i].toLowerCase().includes(needle))
+                matchLines.push(i);
+        if (!matchLines.length)
+            return {
+                content: `Ref: ${ref}\nNo matches in the retained snapshot.\nsourceTruncated: ${entry.sourceTruncated}`,
+                data: { ref, matchLines: [], sourceTruncated: entry.sourceTruncated, partial: false }
+            };
+        const first = matchLines[0];
+        const matchOffset = lines[first].toLowerCase().indexOf(needle);
+        let originalOffset = 0;
+        let foldedOffset = 0;
+        for (const point of lines[first]) {
+            const next = foldedOffset + point.toLowerCase().length;
+            if (next > matchOffset)
+                break;
+            foldedOffset = next;
+            originalOffset += point.length;
+        }
+        let column = Math.max(0, originalOffset - 80);
+        if (column > 0 && /[\uDC00-\uDFFF]/u.test(lines[first][column]))
+            column--;
+        const shown = matchLines.slice(0, 40);
+        const partial = shown.length < matchLines.length;
+        const prefix = `Matches${partial ? ' (partial)' : ''}: ${shown.join(', ')}\n`;
+        const window = this.open(scope, ref, {
+            cursor: `${first}:${column}`,
+            maxTokens: maxTokens - countModelTokens(prefix) - 1
+        });
+        return {
+            content: prefix + window.content,
+            data: { ...window.data, matchLines: shown, partial }
+        };
+    }
+    #get(scope, ref) {
+        const entry = this.#entries.get(ref);
+        if (!entry || entry.scope !== scope)
+            throw unavailable();
+        this.#entries.delete(ref);
+        this.#entries.set(ref, entry);
+        return entry;
+    }
+    #delete(ref) {
+        const entry = this.#entries.get(ref);
+        if (entry)
+            this.#bytes -= entry.size;
+        this.#entries.delete(ref);
+    }
+}
+function short(value, maxBytes) {
+    if (Buffer.byteLength(value) <= maxBytes)
+        return value;
+    const bytes = Buffer.from(value);
+    return `${new TextDecoder().decode(bytes.subarray(0, maxBytes - 3), { stream: true })}…`;
+}
+function unavailable() {
+    return new AgentError('MAR_AGENT_WEB_SNAPSHOT_UNAVAILABLE', 'Page reference is unavailable. Open the URL again.');
+}
+function invalidCursor() {
+    return new AgentError('MAR_AGENT_TOOL_INPUT_INVALID', 'Page position is invalid.');
+}

package/dist/service/web-tool-service.js

@@ -1,12 +1,17 @@
-import { AgentError } from '@myagentroam/agent';
+import { AgentError, countModelTokens } from '@myagentroam/agent';
+import { z } from 'zod';
 import { readWebSearchError } from './web-search-error.js';
 import { WebFetchService } from './web-fetch-service.js';
-import { webFetchResultSchema } from './web-fetch-contracts.js';
+import { browserFetchResultSchema } from './web-fetch-contracts.js';
+import { WebSnapshots } from './web-snapshots.js';
 /** Owns Node web reads and the authenticated Server search adapter. */
 export class WebToolService {
     #config;
     #request;
     #fetcher;
+    #snapshots = new WebSnapshots();
+    #pending = new Map();
+    #closed = false;
     constructor(config, request = fetch) {
         this.#config = config;
         this.#request = request;
@@ -15,19 +20,80 @@ export class WebToolService {
     webSearch(input, signal) {
         return this.#execute(input, signal);
     }
-    webFetch = async (input, signal, allowUntrustedTls) => {
-        const result = await this.#fetcher.fetch(input, signal, {
-            ...(allowUntrustedTls === undefined ? {} : { allowUntrustedTls })
-        });
-        const parsed = webFetchResultSchema.safeParse(result);
+    webFetch = async (input, signal, scope, allowUntrustedTls) => {
+        signal.throwIfAborted();
+        if (this.#closed)
+            throw new AgentError('MAR_AGENT_WEB_FETCH_CANCELLED', 'Web service is closed.');
+        if (input.action === 'find')
+            return this.#snapshots.find(scope, input.ref, input.pattern, input.maxTokens);
+        if ('ref' in input)
+            return this.#snapshots.open(scope, input.ref, {
+                maxTokens: input.maxTokens,
+                ...(input.line === undefined ? {} : { line: input.line }),
+                ...(input.cursor === undefined ? {} : { cursor: input.cursor })
+            });
+        const controller = new AbortController();
+        const pending = this.#pending.get(scope) ?? new Set();
+        this.#pending.set(scope, pending);
+        pending.add(controller);
+        const combined = AbortSignal.any([signal, controller.signal]);
+        let result;
+        try {
+            result = await this.#fetcher.fetch({ url: input.url, mode: 'browser' }, combined, {
+                ...(allowUntrustedTls === undefined ? {} : { allowUntrustedTls })
+            });
+            combined.throwIfAborted();
+        }
+        finally {
+            pending.delete(controller);
+            if (pending.size === 0)
+                this.#pending.delete(scope);
+        }
+        const parsed = browserFetchResultSchema.safeParse(result);
         if (!parsed.success)
             throw new AgentError('MAR_AGENT_WEB_FETCH_FAILED', 'Web fetch returned an invalid result.');
-        return parsed.data;
+        signal.throwIfAborted();
+        if (this.#closed)
+            throw new AgentError('MAR_AGENT_WEB_FETCH_CANCELLED', 'Web service is closed.');
+        const { content, data, truncated } = parsed.data;
+        const ref = this.#snapshots.add(scope, {
+            content,
+            url: data.url,
+            status: data.status,
+            contentType: data.contentType,
+            ...(data.title === undefined ? {} : { title: data.title }),
+            sourceTruncated: truncated
+        });
+        return this.#snapshots.open(scope, ref, { maxTokens: input.maxTokens });
     };
-    close() {
-        return this.#fetcher.close();
+    clearSession(scope) {
+        for (const controller of this.#pending.get(scope) ?? [])
+            controller.abort();
+        this.#snapshots.clearScope(scope);
+    }
+    clearHostedStorage(storage) {
+        for (const [scope, controllers] of this.#pending) {
+            try {
+                const parts = JSON.parse(scope);
+                if (Array.isArray(parts) && parts[0] === 'hosted' && parts[1] === storage)
+                    for (const controller of controllers)
+                        controller.abort();
+            }
+            catch {
+                /* Ignore unrelated scopes. */
+            }
+        }
+        this.#snapshots.clearHostedStorage(storage);
+    }
+    async close() {
+        this.#closed = true;
+        await this.#fetcher.close();
+        this.#snapshots.clear();
     }
     async #execute(input, signal) {
+        signal.throwIfAborted();
+        if (this.#closed)
+            throw new AgentError('MAR_AGENT_WEB_SEARCH_FAILED', 'Web service is closed.');
         const config = this.#config();
         if (config?.nodeId === undefined || config.credential === undefined)
             throw new Error('NODE_UNAUTHENTICATED');
@@ -39,7 +105,7 @@ export class WebToolService {
                     'x-mar-node-id': config.nodeId,
                     authorization: `Bearer ${config.credential}`
                 },
-                body: JSON.stringify(input),
+                body: JSON.stringify({ query: input.query, maxResults: input.maxResults }),
                 signal
             });
             if (!response.ok)
@@ -51,12 +117,55 @@ export class WebToolService {
             catch {
                 throw invalidResult();
             }
-            if (result === null || typeof result !== 'object')
+            const parsed = z
+                .object({
+                query: z.string().max(500),
+                results: z
+                    .array(z.object({ title: z.string(), url: z.string().max(8192), snippet: z.string() }))
+                    .max(10)
+            })
+                .safeParse(result);
+            if (!parsed.success)
                 throw invalidResult();
-            const content = JSON.stringify(result);
-            if (content === undefined)
+            const budget = input.maxTokens ?? 2000;
+            let snippetLimit = 2000;
+            let metadataLimit = 8192;
+            let results = parsed.data.results;
+            let content = '';
+            let truncated = false;
+            do {
+                results = parsed.data.results.map((item) => ({
+                    title: clip(item.title, metadataLimit),
+                    url: clip(item.url, metadataLimit),
+                    snippet: clip(item.snippet, snippetLimit)
+                }));
+                truncated = results.some((item, i) => item.title !== parsed.data.results[i].title ||
+                    item.url !== parsed.data.results[i].url ||
+                    item.snippet !== parsed.data.results[i].snippet);
+                content =
+                    `Search results${truncated ? ' (truncated)' : ''}:\n` +
+                        results
+                            .map((item, i) => `${i + 1}. ${item.title}\nURL: ${item.url}\nSearch snippet: ${item.snippet}`)
+                            .join('\n');
+                if (snippetLimit > 0)
+                    snippetLimit = snippetLimit <= 32 ? 0 : Math.floor(snippetLimit / 2);
+                else
+                    metadataLimit = Math.floor(metadataLimit / 2);
+            } while (countModelTokens(content) > budget && metadataLimit >= 32);
+            if (countModelTokens(content) > budget)
                 throw invalidResult();
-            return { content, data: result };
+            return {
+                content,
+                data: {
+                    query: parsed.data.query,
+                    results: results.map((item, index) => ({
+                        ...item,
+                        url: parsed.data.results[index].url
+                    })),
+                    truncated
+                },
+                truncated
+            };
         }
         catch (error) {
             if (error instanceof AgentError || signal.aborted)
@@ -67,6 +176,14 @@ export class WebToolService {
         }
     }
 }
+function clip(value, maxBytes) {
+    if (maxBytes <= 0)
+        return value ? '…' : '';
+    const bytes = Buffer.from(value);
+    return bytes.length <= maxBytes
+        ? value
+        : `${new TextDecoder().decode(bytes.subarray(0, maxBytes - 3), { stream: true })}…`;
+}
 function invalidResult() {
     return new AgentError('MAR_AGENT_WEB_SEARCH_FAILED', 'Web search returned an invalid result.');
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@myagentroam/node",
-  "version": "0.11.61",
+  "version": "0.11.63",
   "description": "MyAgentRoam Node runtime CLI.",
   "license": "Apache-2.0 AND GPL-2.0-or-later",
   "type": "module",
@@ -36,14 +36,17 @@
     "npm": "11.12.1",
     "playwright": "1.62.1",
     "sharp": "0.35.5",
+    "turndown": "^7.2.4",
+    "turndown-plugin-gfm": "^1.0.2",
     "ws": "^8.21.3",
     "zod": "4.4.3",
-    "@myagentroam/agent": "0.11.61",
+    "@myagentroam/agent": "0.11.63",
     "@myagentroam/code-navigation": "0.11.1",
-    "@myagentroam/protocol": "0.11.61",
-    "@myagentroam/marctl": "0.11.11"
+    "@myagentroam/marctl": "0.11.11",
+    "@myagentroam/protocol": "0.11.63"
   },
   "devDependencies": {
+    "@types/turndown": "^5.0.6",
     "@types/ws": "^8.18.1"
   },
   "exports": {