@qvac/harness 0.0.0 → 0.1.1

package/dist-lib/lib/tools/url-guard.js

@@ -0,0 +1,40 @@
+import { AbortedError, raceAbort } from "../helpers/race-abort.js";
+import { isPrivateOrBlocked, stripIPv6Brackets } from "./private-ip.js";
+const LITERAL_IPV4 = /^\d{1,3}(\.\d{1,3}){3}$/;
+export function checkOutboundHost(url) {
+    const host = stripIPv6Brackets(url.hostname);
+    return isPrivateOrBlocked(host)
+        ? `blocked host: ${host} (private, loopback, link-local or metadata address)`
+        : null;
+}
+export async function resolveOutboundUrl(url, resolveHost, signal) {
+    const blocked = checkOutboundHost(url);
+    if (blocked) {
+        return { blocked };
+    }
+    const host = stripIPv6Brackets(url.hostname);
+    if (host.includes(':') || LITERAL_IPV4.test(host)) {
+        return { blocked: null };
+    }
+    if (!resolveHost) {
+        return { blocked: null };
+    }
+    let addresses;
+    try {
+        addresses = await raceAbort(resolveHost(host), signal);
+    }
+    catch (err) {
+        if (err instanceof AbortedError)
+            throw err;
+        return { blocked: `could not resolve host: ${host}` };
+    }
+    if (addresses.length === 0) {
+        return { blocked: `could not resolve host: ${host}` };
+    }
+    for (const address of addresses) {
+        if (isPrivateOrBlocked(address)) {
+            return { blocked: `blocked host: ${host} resolves to ${address}` };
+        }
+    }
+    return { blocked: null, addresses };
+}

package/dist-lib/lib/tools/web-fetch.d.ts

@@ -0,0 +1,8 @@
+import type { ToolDefinition } from '../capabilities/completion.ts';
+import type { FetchFn } from './http-request.ts';
+import { type HostResolver } from './url-guard.ts';
+export interface WebFetchToolOptions {
+    fetch?: FetchFn;
+    resolveHost?: HostResolver;
+}
+export declare function createWebFetchTool({ fetch: fetchImpl, resolveHost }?: WebFetchToolOptions): ToolDefinition;

package/dist-lib/lib/tools/web-fetch.js

@@ -0,0 +1,302 @@
+import defaultFetch from '#no-redirect-fetch';
+import { AbortController } from 'bare-abort-controller';
+import { lookupHost } from "../dns-lookup.js";
+import { AbortedError } from "../helpers/race-abort.js";
+import { extractEmbeddedState } from "../web-fetch/embedded-state.js";
+import { extractPageText, looksLikeHtml } from "../web-fetch/extract.js";
+import { imageFileName, pageImageUrls } from "../web-fetch/images.js";
+import { pageVideoRef } from "../web-fetch/video.js";
+import { MAX_HTML_CHARS } from "../web-search/fetch-html.js";
+import { BROWSER_UA } from "../web-search/user-agents.js";
+import { isRedirectStatus } from "./http-request/utils.js";
+import { resolveOutboundUrl } from "./url-guard.js";
+const SMALL_CONTEXT_TOKENS = 12_288;
+const SMALL_CONTEXT_CHARS = 6_000;
+const LARGE_CONTEXT_CHARS = 16_000;
+const PAGE_TIMEOUT_MS = 15_000;
+const SOURCE_EXCERPT_CHARS = 2_000;
+const MIN_MARKUP_CHARS = 200;
+const MAX_REDIRECTS = 5;
+const MAX_IMAGE_BYTES = 3_000_000;
+const MAX_PAGE_BYTES = 8_000_000;
+function charBudget(ctxSize) {
+    return ctxSize && ctxSize <= SMALL_CONTEXT_TOKENS ? SMALL_CONTEXT_CHARS : LARGE_CONTEXT_CHARS;
+}
+export function createWebFetchTool({ fetch: fetchImpl, resolveHost = lookupHost } = {}) {
+    return {
+        name: 'web_fetch',
+        toolset: 'web-search',
+        description: 'Fetch a public web page and return its readable text. ' +
+            'Use this to read the full content behind a web_search result URL, or a page the user ' +
+            'shares, when a search snippet is not enough to answer.',
+        parameters: {
+            type: 'object',
+            properties: {
+                url: { type: 'string', description: 'Full http:// or https:// URL of the page to read' }
+            },
+            required: ['url']
+        },
+        effect: () => 'read',
+        resource: (args) => {
+            try {
+                return { kind: 'host', label: new URL(String(args.url)).hostname };
+            }
+            catch {
+                return null;
+            }
+        },
+        async execute(args, ctx) {
+            const raw = args.url;
+            if (typeof raw !== 'string' || raw.trim().length === 0) {
+                return { error: 'url must be a non-empty string' };
+            }
+            let url;
+            try {
+                url = new URL(raw.trim());
+            }
+            catch {
+                return { error: `"${raw}" is not a valid URL` };
+            }
+            if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+                return { error: `Scheme "${url.protocol}" not allowed. Use http: or https:.` };
+            }
+            if (ctx?.signal?.aborted)
+                return { error: 'web_fetch cancelled' };
+            const fetch = fetchImpl ?? defaultFetch;
+            const controller = new AbortController();
+            const timeout = setTimeout(() => controller.abort(undefined), PAGE_TIMEOUT_MS);
+            const onCallerAbort = () => controller.abort(undefined);
+            ctx?.signal?.addEventListener('abort', onCallerAbort);
+            try {
+                const got = await guardedGet(fetch, url, {
+                    resolveHost,
+                    signal: controller.signal,
+                    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+                    maxBytes: MAX_PAGE_BYTES,
+                    truncate: true
+                });
+                if (!got.ok)
+                    return { error: got.error };
+                const { response, finalUrl: currentUrl } = got;
+                if (response.status < 200 || response.status >= 300) {
+                    return { error: `HTTP ${response.status} fetching ${currentUrl.toString()}` };
+                }
+                const rawBody = await response.text();
+                const body = rawBody.length > MAX_HTML_CHARS ? rawBody.slice(0, MAX_HTML_CHARS) : rawBody;
+                let mimeType = response.headers?.get('content-type')?.split(';')[0]?.trim().toLowerCase() ?? '';
+                if (body.startsWith('%PDF-'))
+                    mimeType = 'application/pdf';
+                if (isBinaryMime(mimeType) || body.includes('\u0000')) {
+                    const attachMime = isBinaryMime(mimeType) ? mimeType : 'application/octet-stream';
+                    return await attachDocument(response, currentUrl, attachMime, ctx?.attach);
+                }
+                const isHtml = looksLikeHtml(body);
+                let page = isHtml ? extractPageText(body) : { title: null, text: body.trim() };
+                if (isHtml && page.text.length < MIN_MARKUP_CHARS) {
+                    const embedded = extractEmbeddedState(body);
+                    if (embedded.length > page.text.length)
+                        page = { title: page.title, text: embedded };
+                }
+                const budget = charBudget(ctx?.ctxSize);
+                const text = page.text.length > budget ? `${page.text.slice(0, budget)}…[truncated]` : page.text;
+                if (text.length === 0) {
+                    return {
+                        content: isHtml
+                            ? `No readable text found at ${url.toString()} — the page likely requires JavaScript to render.`
+                            : `The response at ${url.toString()} is empty.`
+                    };
+                }
+                const attached = isHtml
+                    ? await attachPageImages(body, currentUrl, {
+                        fetch,
+                        resolveHost,
+                        attach: ctx?.attach,
+                        signal: controller.signal
+                    })
+                    : [];
+                const imagesLine = attached.length
+                    ? `\n\n[Attached ${attached.length === 1 ? 'image' : 'images'}: ${attached.join(', ')} — shown to the user automatically]`
+                    : '';
+                const video = isHtml ? pageVideoRef(body, currentUrl) : null;
+                const videoLine = video
+                    ? `\n\n[This page's video: ${video.url} — share this link when the user wants to watch it]`
+                    : '';
+                const header = page.title ? `${page.title}\nSource: ${url}` : `Source: ${url}`;
+                return {
+                    content: `${header}\n\n${text}${imagesLine}${videoLine}`,
+                    data: {
+                        results: [
+                            {
+                                title: page.title ?? url.toString(),
+                                url: url.toString(),
+                                content: text.slice(0, SOURCE_EXCERPT_CHARS)
+                            }
+                        ]
+                    }
+                };
+            }
+            catch (err) {
+                if (ctx?.signal?.aborted || err instanceof AbortedError) {
+                    return { error: 'web_fetch cancelled' };
+                }
+                if (controller.signal.aborted) {
+                    return { error: `${url.hostname} timed out after ${PAGE_TIMEOUT_MS}ms` };
+                }
+                return { error: err instanceof Error ? err.message : String(err) };
+            }
+            finally {
+                clearTimeout(timeout);
+                ctx?.signal?.removeEventListener('abort', onCallerAbort);
+            }
+        }
+    };
+}
+const BINARY_MIME_PREFIXES = ['image/', 'audio/', 'video/', 'font/', 'model/'];
+function isBinaryMime(mimeType) {
+    if (mimeType in MIME_EXTENSIONS)
+        return true;
+    return BINARY_MIME_PREFIXES.some((prefix) => mimeType.startsWith(prefix));
+}
+async function attachDocument(response, url, mimeType, attach) {
+    const bytes = await response.bytes?.();
+    if (!attach || !bytes || bytes.byteLength === 0) {
+        return { error: 'This URL serves a file, not a readable page, and it could not be attached.' };
+    }
+    if (bytes.byteLength >= MAX_PAGE_BYTES) {
+        return {
+            error: `This file is larger than ${MAX_PAGE_BYTES / 1_000_000}MB and was not attached.`
+        };
+    }
+    const mime = mimeType || 'application/octet-stream';
+    const record = await attach({
+        fileName: attachmentFileName(url, mime),
+        mimeType: mime,
+        bytes,
+        raw: !mime.startsWith('image/')
+    });
+    return {
+        content: `Attached the file "${record.fileName}" (${Math.round(record.byteLength / 1024)}KB, ${mime}) from ${url.toString()} — ` +
+            'the user can open or save it. Its contents were NOT read: do not claim to know what is inside.'
+    };
+}
+const MIME_EXTENSIONS = {
+    'application/pdf': 'pdf',
+    'application/msword': 'doc',
+    'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',
+    'application/vnd.ms-excel': 'xls',
+    'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
+    'application/vnd.ms-powerpoint': 'ppt',
+    'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx',
+    'application/vnd.oasis.opendocument.text': 'odt',
+    'application/vnd.oasis.opendocument.spreadsheet': 'ods',
+    'application/epub+zip': 'epub',
+    'application/zip': 'zip',
+    'application/gzip': 'gz',
+    'image/svg+xml': 'svg',
+    'image/jpeg': 'jpg',
+    'audio/mpeg': 'mp3',
+    'application/octet-stream': 'bin'
+};
+function attachmentFileName(url, mimeType) {
+    const last = url.pathname.split('/').filter(Boolean).pop() ?? '';
+    let decoded = last;
+    try {
+        decoded = decodeURIComponent(last);
+    }
+    catch { }
+    const base = decoded.split(/[/\\]/).filter(Boolean).pop() ?? '';
+    if (!base)
+        return `${url.hostname}.${extensionFor(mimeType)}`;
+    const extension = /\.([\w-]{1,8})$/.exec(base)?.[1]?.toLowerCase() ?? '';
+    if (extension && acceptableExtensions(mimeType).has(extension))
+        return base;
+    if (mimeType === 'application/octet-stream' && /^[a-z][\w-]*$/.test(extension))
+        return base;
+    return `${base}.${extensionFor(mimeType)}`;
+}
+function acceptableExtensions(mimeType) {
+    const accepted = new Set([extensionFor(mimeType)]);
+    const subtype = mimeType.split('/')[1] ?? '';
+    if (/^[a-z0-9]{1,8}$/.test(subtype))
+        accepted.add(subtype);
+    return accepted;
+}
+function extensionFor(mimeType) {
+    const known = MIME_EXTENSIONS[mimeType];
+    if (known)
+        return known;
+    const subtype = mimeType.split('/')[1] ?? '';
+    return /^[a-z0-9]{1,5}$/.test(subtype) ? subtype : 'bin';
+}
+async function guardedGet(fetch, url, { resolveHost, signal, accept, maxBytes, truncate }) {
+    const initialCheck = await resolveOutboundUrl(url, resolveHost, signal);
+    if (initialCheck.blocked)
+        return { ok: false, error: initialCheck.blocked };
+    let currentUrl = url;
+    let validatedAddresses = initialCheck.addresses;
+    for (let redirects = 0;; redirects++) {
+        const response = await fetch(currentUrl, {
+            method: 'GET',
+            headers: { 'User-Agent': BROWSER_UA, Accept: accept },
+            signal,
+            redirect: 'manual',
+            validatedAddresses,
+            maxBytes,
+            truncate
+        });
+        if (!isRedirectStatus(response.status))
+            return { ok: true, response, finalUrl: currentUrl };
+        const location = response.headers?.get('location');
+        if (!location)
+            return { ok: true, response, finalUrl: currentUrl };
+        let nextUrl;
+        try {
+            nextUrl = new URL(location, currentUrl);
+        }
+        catch {
+            return { ok: true, response, finalUrl: currentUrl };
+        }
+        if (nextUrl.protocol !== 'http:' && nextUrl.protocol !== 'https:') {
+            return { ok: true, response, finalUrl: currentUrl };
+        }
+        if (redirects >= MAX_REDIRECTS)
+            return { ok: false, error: 'too many redirects' };
+        const hopCheck = await resolveOutboundUrl(nextUrl, resolveHost, signal);
+        if (hopCheck.blocked)
+            return { ok: false, error: hopCheck.blocked };
+        currentUrl = nextUrl;
+        validatedAddresses = hopCheck.addresses;
+    }
+}
+async function attachPageImages(html, pageUrl, { fetch, resolveHost, attach, signal }) {
+    if (!attach)
+        return [];
+    const attached = [];
+    for (const href of pageImageUrls(html, pageUrl)) {
+        try {
+            const got = await guardedGet(fetch, new URL(href), {
+                resolveHost,
+                signal,
+                accept: 'image/*',
+                maxBytes: MAX_IMAGE_BYTES
+            });
+            if (!got.ok)
+                continue;
+            const { response, finalUrl } = got;
+            if (response.status < 200 || response.status >= 300)
+                continue;
+            const mimeType = response.headers?.get('content-type')?.split(';')[0]?.trim() ?? '';
+            if (!mimeType.startsWith('image/'))
+                continue;
+            const bytes = await response.bytes?.();
+            if (!bytes || bytes.byteLength === 0 || bytes.byteLength > MAX_IMAGE_BYTES)
+                continue;
+            const record = await attach({ fileName: imageFileName(finalUrl, mimeType), mimeType, bytes });
+            attached.push(record.fileName);
+        }
+        catch {
+            continue;
+        }
+    }
+    return attached;
+}

package/dist-lib/lib/tools/web-search.d.ts

@@ -0,0 +1,6 @@
+import type { ToolDefinition } from '../capabilities/completion.ts';
+import type { Fetcher } from '../web-search/types.ts';
+export interface WebSearchToolOptions {
+    fetch?: Fetcher;
+}
+export declare function createWebSearchTool({ fetch }?: WebSearchToolOptions): ToolDefinition;

package/dist-lib/lib/tools/web-search.js

@@ -0,0 +1,55 @@
+import defaultFetch from '#fetch';
+import { formatResultsForPrompt } from "../web-search/format-results.js";
+import { searchWeb } from "../web-search/orchestrator.js";
+const nodeFetch = async (url, init) => {
+    const res = await defaultFetch(url, init);
+    return { ok: res.ok, status: res.status, text: () => res.text() };
+};
+const SMALL_CONTEXT_TOKENS = 12_288;
+const SMALL_CONTEXT_RESULTS = 8;
+const LARGE_CONTEXT_RESULTS = 12;
+function resultBudget(ctxSize) {
+    return ctxSize && ctxSize <= SMALL_CONTEXT_TOKENS ? SMALL_CONTEXT_RESULTS : LARGE_CONTEXT_RESULTS;
+}
+export function createWebSearchTool({ fetch } = {}) {
+    const fetcher = fetch ?? nodeFetch;
+    return {
+        name: 'web_search',
+        toolset: 'web-search',
+        description: 'Search the web for current information using multiple search engines. ' +
+            'Returns a concise aggregated and ranked result set. Use this when the user asks about ' +
+            'recent events, current information, or anything that may require ' +
+            'up-to-date knowledge beyond your training data. When a snippet is not enough to answer, ' +
+            'read the full page behind a result with web_fetch.',
+        parameters: {
+            type: 'object',
+            properties: {
+                query: { type: 'string', description: 'The search query' }
+            },
+            required: ['query']
+        },
+        effect: () => 'read',
+        async execute(args, ctx) {
+            const query = args.query;
+            if (typeof query !== 'string' || query.trim().length === 0) {
+                return { error: 'query must be a non-empty string' };
+            }
+            try {
+                const { results, engineSummary } = await searchWeb(fetcher, query, resultBudget(ctx?.ctxSize), ctx?.signal);
+                if (ctx?.signal?.aborted)
+                    return { error: 'web_search cancelled' };
+                if (results.length > 0) {
+                    return { content: formatResultsForPrompt(results), data: { results } };
+                }
+                const failed = engineSummary.filter((e) => e.status !== 'ok').length;
+                if (engineSummary.length > 0 && failed === engineSummary.length) {
+                    return { error: `web_search unavailable: all ${failed} engines failed (timeout/error)` };
+                }
+                return { content: `No web results found for "${query}".` };
+            }
+            catch (err) {
+                return { error: err instanceof Error ? err.message : String(err) };
+            }
+        }
+    };
+}

package/dist-lib/lib/transport.d.ts

@@ -0,0 +1,12 @@
+import { Duplex } from 'streamx';
+export { settleOnClose, type Stream, type Transport } from './rpc-session.ts';
+export interface Pipe {
+    write(data: Buffer): unknown;
+    on(event: 'data', listener: (chunk: unknown) => void): unknown;
+}
+export interface ClientSessions {
+    readonly current: Duplex;
+    reset(): Duplex;
+}
+export declare function clientSessions(pipe: Pipe): ClientSessions;
+export declare function duplexPair(): [Duplex, Duplex];

package/dist-lib/lib/transport.js

@@ -0,0 +1,59 @@
+import c from 'compact-encoding';
+import { Duplex } from 'streamx';
+export { settleOnClose } from "./rpc-session.js";
+export function clientSessions(pipe) {
+    let current = session(pipe);
+    pipe.on('data', (chunk) => current.push(chunk));
+    return {
+        get current() {
+            return current;
+        },
+        reset() {
+            const previous = current;
+            current = session(pipe);
+            previous.destroy();
+            return current;
+        }
+    };
+}
+function session(pipe) {
+    return new Duplex({
+        write(data, cb) {
+            pipe.write(data);
+            cb(null);
+        }
+    });
+}
+export function duplexPair() {
+    let a;
+    let b;
+    a = new Duplex({
+        write(data, cb) {
+            b.push(data);
+            cb(null);
+        },
+        final(cb) {
+            b.push(null);
+            cb(null);
+        },
+        destroy(cb) {
+            b.destroy();
+            cb(null);
+        }
+    });
+    b = new Duplex({
+        write(data, cb) {
+            a.push(data);
+            cb(null);
+        },
+        final(cb) {
+            a.push(null);
+            cb(null);
+        },
+        destroy(cb) {
+            a.destroy();
+            cb(null);
+        }
+    });
+    return [a, b];
+}

package/dist-lib/lib/user-agent.d.ts

@@ -0,0 +1 @@
+export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";

package/dist-lib/lib/user-agent.js

@@ -0,0 +1 @@
+export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';

package/dist-lib/lib/validating-lookup.d.ts

@@ -0,0 +1,17 @@
+export type IPFamily = 4 | 6;
+export type LookupAddress = {
+    readonly address: string;
+    readonly family: IPFamily;
+};
+export type LookupAllCb = (err: Error | null, addresses: ReadonlyArray<LookupAddress> | null) => void;
+export type BareLookupAllOpts = {
+    readonly all: true;
+    readonly family?: IPFamily | 0;
+    readonly hints?: number;
+};
+export type BareLookupAll = (hostname: string, opts: BareLookupAllOpts, cb: LookupAllCb) => void;
+export declare class BlockedAddressError extends Error {
+    readonly code = "EBLOCKEDADDR";
+    constructor(hostname: string, address: string);
+}
+export declare function makeValidatingLookup(rawLookup: BareLookupAll): BareLookupAll;

package/dist-lib/lib/validating-lookup.js

@@ -0,0 +1,25 @@
+import { isPrivateOrBlocked } from "./tools/private-ip.js";
+export class BlockedAddressError extends Error {
+    code = 'EBLOCKEDADDR';
+    constructor(hostname, address) {
+        super(`Requests to "${hostname}" resolve to blocked address "${address}".`);
+        this.name = 'BlockedAddressError';
+    }
+}
+export function makeValidatingLookup(rawLookup) {
+    return (hostname, opts, cb) => {
+        rawLookup(hostname, opts, (err, addresses) => {
+            if (err || addresses === null) {
+                cb(err, addresses);
+                return;
+            }
+            for (const record of addresses) {
+                if (isPrivateOrBlocked(record.address)) {
+                    cb(new BlockedAddressError(hostname, record.address), null);
+                    return;
+                }
+            }
+            cb(null, addresses);
+        });
+    };
+}

package/dist-lib/lib/wav-encode.d.ts

@@ -0,0 +1,5 @@
+export declare const TARGET_SAMPLE_RATE = 16000;
+export declare const WAV_HEADER_SIZE = 44;
+export declare const BITS_PER_SAMPLE = 16;
+export declare const CHANNELS = 1;
+export declare function buildWavBuffer(pcm: Buffer): Buffer;

package/dist-lib/lib/wav-encode.js

@@ -0,0 +1,24 @@
+export const TARGET_SAMPLE_RATE = 16000;
+export const WAV_HEADER_SIZE = 44;
+export const BITS_PER_SAMPLE = 16;
+export const CHANNELS = 1;
+export function buildWavBuffer(pcm) {
+    const dataSize = pcm.length;
+    const buf = Buffer.alloc(WAV_HEADER_SIZE + dataSize);
+    const blockAlign = CHANNELS * (BITS_PER_SAMPLE / 8);
+    buf.write('RIFF', 0, 'ascii');
+    buf.writeUInt32LE(WAV_HEADER_SIZE + dataSize - 8, 4);
+    buf.write('WAVE', 8, 'ascii');
+    buf.write('fmt ', 12, 'ascii');
+    buf.writeUInt32LE(16, 16);
+    buf.writeUInt16LE(1, 20);
+    buf.writeUInt16LE(CHANNELS, 22);
+    buf.writeUInt32LE(TARGET_SAMPLE_RATE, 24);
+    buf.writeUInt32LE(TARGET_SAMPLE_RATE * blockAlign, 28);
+    buf.writeUInt16LE(blockAlign, 32);
+    buf.writeUInt16LE(BITS_PER_SAMPLE, 34);
+    buf.write('data', 36, 'ascii');
+    buf.writeUInt32LE(dataSize, 40);
+    buf.set(pcm, WAV_HEADER_SIZE);
+    return buf;
+}

package/dist-lib/lib/web-fetch/embedded-state.d.ts

@@ -0,0 +1 @@
+export declare function extractEmbeddedState(html: string): string;