@qvac/harness 0.1.0 → 0.1.2

package/dist-lib/lib/tools/fs/local.js

@@ -0,0 +1,29 @@
+import path from 'path';
+import Localdrive from 'localdrive';
+import { defaultRealpath } from "./scope.js";
+function withSkipList(localdrive) {
+    return {
+        entry: localdrive.entry.bind(localdrive),
+        get: localdrive.get.bind(localdrive),
+        list: (folder, opts) => localdrive.list(folder, {
+            ignore: (key) => key.split('/').some((segment) => opts?.skip?.has(segment))
+        }),
+        createReadStream: localdrive.createReadStream.bind(localdrive),
+        createWriteStream: localdrive.createWriteStream.bind(localdrive)
+    };
+}
+export async function createLocalRoots(dirs, realpath = defaultRealpath) {
+    const roots = [];
+    for (const dir of dirs) {
+        let canonical;
+        try {
+            canonical = await realpath(path.resolve(dir));
+        }
+        catch {
+            continue;
+        }
+        const localdrive = new Localdrive(canonical, { atomic: true });
+        roots.push({ kind: 'local', root: canonical, drive: withSkipList(localdrive) });
+    }
+    return roots;
+}

package/dist-lib/lib/tools/fs/read.d.ts

@@ -0,0 +1,2 @@
+import type { Drive } from './drive.ts';
+export declare function readLines(drive: Drive, key: string): AsyncGenerator<string>;

package/dist-lib/lib/tools/fs/read.js

@@ -0,0 +1,17 @@
+import b4a from 'b4a';
+const NEWLINE = 0x0a;
+export async function* readLines(drive, key) {
+    let carry = null;
+    for await (const chunk of drive.createReadStream(key)) {
+        const buffer = carry === null ? chunk : b4a.concat([carry, chunk]);
+        let start = 0;
+        for (let index = 0; index < buffer.length; index++) {
+            if (buffer[index] !== NEWLINE)
+                continue;
+            yield b4a.toString(buffer.subarray(start, index), 'utf8');
+            start = index + 1;
+        }
+        carry = start === buffer.length ? null : buffer.subarray(start);
+    }
+    yield carry === null ? '' : b4a.toString(carry, 'utf8');
+}

package/dist-lib/lib/tools/fs/remote-protocol.d.ts

@@ -0,0 +1,40 @@
+import type { DriveEntry } from './drive.ts';
+export type DriveEntryRequest = {
+    op: 'entry';
+    driveKey: string;
+    key: string;
+};
+export type DriveGetRequest = {
+    op: 'get';
+    driveKey: string;
+    key: string;
+};
+export type DriveReadRequest = {
+    op: 'read';
+    driveKey: string;
+    key: string;
+};
+export type DriveListRequest = {
+    op: 'list';
+    driveKey: string;
+    folder: string;
+    skip: string[];
+    limit: number;
+    cursor: string | null;
+};
+export type DriveWriteRequest = {
+    op: 'write';
+    driveKey: string;
+    key: string;
+    path: string;
+    executable?: boolean;
+};
+export type DriveRequest = DriveEntryRequest | DriveGetRequest | DriveReadRequest | DriveListRequest | DriveWriteRequest;
+export interface DriveResponse {
+    error?: string;
+    entry?: DriveEntry | null;
+    path?: string | null;
+    truncated?: boolean;
+    entries?: DriveEntry[];
+    cursor?: string | null;
+}

package/dist-lib/lib/tools/fs/remote-protocol.js

@@ -0,0 +1 @@
+export {};

package/dist-lib/lib/tools/fs/remote.d.ts

@@ -0,0 +1,17 @@
+import type { DriveRequest, DriveResponse } from './remote-protocol.ts';
+import type { HyperRoot } from './scope.ts';
+export interface GrantedDrive {
+    readonly key: string;
+    readonly writable: boolean;
+}
+export type DriveOpFn = (request: DriveRequest) => Promise<DriveResponse>;
+export declare const DRIVE_CALL_TIMEOUT_MS = 20000;
+export declare const MAX_DRIVE_BYTES: number;
+export declare const DRIVE_LIST_PAGE = 256;
+export declare class DriveReadTruncatedError extends Error {
+    constructor(key: string);
+}
+export declare function isDriveReadTruncated(err: unknown): err is DriveReadTruncatedError;
+export declare function createRemoteRoots(drives: readonly GrantedDrive[], driveOp: DriveOpFn, opts?: {
+    readonly timeoutMs?: number;
+}): HyperRoot[];

package/dist-lib/lib/tools/fs/remote.js

@@ -0,0 +1,131 @@
+import fs from 'fs';
+import path from 'path';
+import { Writable } from 'streamx';
+import { STAGING_DIR } from "../../staging.js";
+export const DRIVE_CALL_TIMEOUT_MS = 20_000;
+export const MAX_DRIVE_BYTES = 8 * 1024 * 1024;
+export const DRIVE_LIST_PAGE = 256;
+const noop = () => { };
+export class DriveReadTruncatedError extends Error {
+    constructor(key) {
+        super(`read stopped at ${MAX_DRIVE_BYTES} bytes: ${key}`);
+        this.name = 'DriveReadTruncatedError';
+    }
+}
+export function isDriveReadTruncated(err) {
+    return err instanceof DriveReadTruncatedError;
+}
+function withTimeout(promise, ms, op) {
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(new Error(`drive ${op} timed out`)), ms);
+        promise.then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, (err) => {
+            clearTimeout(timer);
+            reject(err);
+        });
+    });
+}
+async function call(driveOp, request, timeoutMs) {
+    const response = await withTimeout(driveOp(request), timeoutMs, request.op);
+    if (response.error)
+        throw new Error(response.error);
+    return response;
+}
+async function readStaged(stagedPath) {
+    try {
+        return await fs.promises.readFile(stagedPath);
+    }
+    finally {
+        await fs.promises.unlink(stagedPath).catch(noop);
+    }
+}
+function stagedWritePath() {
+    fs.mkdirSync(STAGING_DIR, { recursive: true, mode: 0o700 });
+    const unique = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+    return path.join(STAGING_DIR, `drive-write-${unique}`);
+}
+function remoteDrive(driveKey, driveOp, timeoutMs) {
+    return {
+        async entry(entryKey) {
+            const response = await call(driveOp, { op: 'entry', driveKey, key: entryKey }, timeoutMs);
+            return response.entry ?? null;
+        },
+        async get(getKey) {
+            const response = await call(driveOp, { op: 'get', driveKey, key: getKey }, timeoutMs);
+            if (!response.path)
+                return null;
+            return readStaged(response.path);
+        },
+        async *list(folder = '/', opts) {
+            const skip = [...(opts?.skip ?? [])];
+            let cursor = null;
+            while (true) {
+                const response = await call(driveOp, { op: 'list', driveKey, folder, skip, limit: DRIVE_LIST_PAGE, cursor }, timeoutMs);
+                const entries = response.entries ?? [];
+                for (const entry of entries)
+                    yield entry;
+                if (!response.cursor || entries.length === 0)
+                    return;
+                cursor = response.cursor;
+            }
+        },
+        async *createReadStream(readKey) {
+            const response = await call(driveOp, { op: 'read', driveKey, key: readKey }, timeoutMs);
+            if (!response.path)
+                return;
+            try {
+                yield* fs.createReadStream(response.path);
+            }
+            finally {
+                await fs.promises.unlink(response.path).catch(noop);
+            }
+            if (response.truncated)
+                throw new DriveReadTruncatedError(readKey);
+        },
+        createWriteStream(writeKey, writeOpts) {
+            const stagedPath = stagedWritePath();
+            const sink = fs.createWriteStream(stagedPath, { mode: 0o600 });
+            let sinkError = null;
+            sink.on('error', (err) => {
+                sinkError = err;
+            });
+            return new Writable({
+                write(data, cb) {
+                    sink.write(data, (err) => cb(err ?? null));
+                },
+                final(cb) {
+                    sink.end();
+                    sink.on('close', () => {
+                        if (sinkError) {
+                            fs.promises.unlink(stagedPath).catch(noop);
+                            cb(sinkError);
+                            return;
+                        }
+                        call(driveOp, {
+                            op: 'write',
+                            driveKey,
+                            key: writeKey,
+                            path: stagedPath,
+                            ...(writeOpts?.executable ? { executable: true } : {})
+                        }, timeoutMs).then(() => cb(null), (err) => {
+                            fs.promises.unlink(stagedPath).catch(noop);
+                            cb(err);
+                        });
+                    });
+                }
+            });
+        }
+    };
+}
+export function createRemoteRoots(drives, driveOp, opts = {}) {
+    const timeoutMs = opts.timeoutMs ?? DRIVE_CALL_TIMEOUT_MS;
+    return drives.map((drive) => ({
+        kind: 'hyper',
+        key: drive.key,
+        writable: drive.writable,
+        prefix: '/',
+        drive: remoteDrive(drive.key, driveOp, timeoutMs)
+    }));
+}

package/dist-lib/lib/tools/fs/scope.d.ts

@@ -0,0 +1,37 @@
+import type { Drive } from './drive.ts';
+export type Realpath = (p: string) => Promise<string>;
+export declare const defaultRealpath: Realpath;
+export interface LocalRoot {
+    readonly kind: 'local';
+    readonly root: string;
+    readonly drive: Drive;
+}
+export interface HyperRoot {
+    readonly kind: 'hyper';
+    readonly key: string;
+    readonly writable: boolean;
+    readonly prefix: string;
+    readonly drive: Drive;
+}
+export type DriveRoot = LocalRoot | HyperRoot;
+export interface FsToolOptions {
+    readonly realpath?: Realpath;
+    readonly deniedRoots?: readonly string[];
+}
+export declare const NO_DIRS_GRANTED = "no directories are granted";
+export declare const NO_STORAGE_ROOT = "cannot fence the app storage root: the SDK home is unknown";
+export declare function deniedRootsFor(sdkHome: string | undefined): readonly string[];
+export declare const PROTECTED_PATH_ERROR = "path is inside a protected app directory";
+export declare function canonicalDenyRoots(deniedRoots: readonly string[] | undefined, realpath: Realpath): Promise<readonly string[]>;
+export declare function isDeniedPath(hostPath: string, canonicalDenyRoots: readonly string[]): boolean;
+export declare function hostPath(root: LocalRoot, key: string): string;
+export declare function displayPath(root: DriveRoot, key: string): string;
+export declare function normalizeDriveKey(prefix: string, input: string): string | null;
+export interface ResolvedKey {
+    root: DriveRoot;
+    key: string;
+    exists: boolean;
+}
+export declare function resolveKey(p: string, roots: readonly DriveRoot[], realpath: Realpath, deny?: readonly string[]): Promise<ResolvedKey | {
+    error: string;
+}>;

package/dist-lib/lib/tools/fs/scope.js

@@ -0,0 +1,130 @@
+import fs from 'fs';
+import path from 'path';
+import process from 'process';
+export const defaultRealpath = (p) => fs.promises.realpath(p);
+export const NO_DIRS_GRANTED = 'no directories are granted';
+const HYPER_PREFIX = 'hyper://';
+function rootLabel(root) {
+    return root.kind === 'local' ? root.root : `${HYPER_PREFIX}${root.key}`;
+}
+function outside(roots) {
+    return { error: `path must be inside one of: ${roots.map(rootLabel).join(', ')}` };
+}
+export const NO_STORAGE_ROOT = 'cannot fence the app storage root: the SDK home is unknown';
+export function deniedRootsFor(sdkHome) {
+    if (!sdkHome)
+        throw new Error(NO_STORAGE_ROOT);
+    return [path.dirname(sdkHome)];
+}
+export const PROTECTED_PATH_ERROR = 'path is inside a protected app directory';
+function protectedPath() {
+    return { error: PROTECTED_PATH_ERROR };
+}
+export async function canonicalDenyRoots(deniedRoots, realpath) {
+    if (!deniedRoots?.length)
+        return [];
+    return Promise.all(deniedRoots.map(async (raw) => {
+        try {
+            return await realpath(raw);
+        }
+        catch {
+            return path.resolve(raw);
+        }
+    }));
+}
+export function isDeniedPath(hostPath, canonicalDenyRoots) {
+    return canonicalDenyRoots.some((root) => contained(hostPath, root, true));
+}
+const FOLD_CASE = process.platform === 'darwin' || process.platform === 'win32';
+function within(candidate, root) {
+    return contained(candidate, root, FOLD_CASE);
+}
+function contained(candidate, root, ignoreCase) {
+    const a = ignoreCase ? candidate.toLowerCase() : candidate;
+    const b = ignoreCase ? root.toLowerCase() : root;
+    return a === b || a.startsWith(b + path.sep);
+}
+function withinPrefix(candidate, prefix) {
+    if (prefix === '/')
+        return true;
+    return candidate === prefix || candidate.startsWith(prefix + '/');
+}
+function toKey(root, canonical) {
+    const rel = path.relative(root.root, canonical);
+    return rel ? `/${rel.split(path.sep).join('/')}` : '/';
+}
+export function hostPath(root, key) {
+    return path.join(root.root, ...key.split('/').filter(Boolean));
+}
+export function displayPath(root, key) {
+    return root.kind === 'local' ? hostPath(root, key) : `${HYPER_PREFIX}${root.key}${key}`;
+}
+const CONTROL_OR_NUL = /[\x00-\x1f]/;
+const WINDOWS_DRIVE_LETTER = /^[A-Za-z]:/;
+export function normalizeDriveKey(prefix, input) {
+    if (input.includes('\\'))
+        return null;
+    if (WINDOWS_DRIVE_LETTER.test(input))
+        return null;
+    if (CONTROL_OR_NUL.test(input))
+        return null;
+    if (input.includes('://'))
+        return null;
+    const segments = [];
+    for (const segment of input.split('/')) {
+        if (segment === '' || segment === '.')
+            continue;
+        if (segment === '..') {
+            if (segments.length === 0)
+                return null;
+            segments.pop();
+            continue;
+        }
+        segments.push(segment);
+    }
+    const normalized = segments.length ? `/${segments.join('/')}` : '/';
+    return withinPrefix(normalized, prefix) ? normalized : null;
+}
+function matchHyperRoot(rest, roots) {
+    return roots.find((r) => rest === r.key || rest.startsWith(r.key + '/'));
+}
+export async function resolveKey(p, roots, realpath, deny = []) {
+    if (roots.length === 0)
+        return { error: NO_DIRS_GRANTED };
+    if (p.startsWith(HYPER_PREFIX)) {
+        const rest = p.slice(HYPER_PREFIX.length);
+        const hyperRoots = roots.filter((r) => r.kind === 'hyper');
+        const root = matchHyperRoot(rest, hyperRoots);
+        if (!root)
+            return outside(roots);
+        const tail = rest.slice(root.key.length);
+        const key = normalizeDriveKey(root.prefix, tail || '/');
+        if (key === null)
+            return outside(roots);
+        return { root, key, exists: true };
+    }
+    const localRoots = roots.filter((r) => r.kind === 'local');
+    let ancestor = path.resolve(p);
+    const tail = [];
+    while (true) {
+        try {
+            const canonical = await realpath(ancestor);
+            const target = tail.length ? path.join(canonical, ...tail) : canonical;
+            const root = localRoots.find((r) => within(target, r.root));
+            if (!root)
+                return outside(roots);
+            if (isDeniedPath(target, deny)) {
+                return protectedPath();
+            }
+            return { root, key: toKey(root, target), exists: tail.length === 0 };
+        }
+        catch {
+            const parent = path.dirname(ancestor);
+            if (parent === ancestor)
+                break;
+            tail.unshift(path.basename(ancestor));
+            ancestor = parent;
+        }
+    }
+    return outside(roots);
+}

package/dist-lib/lib/tools/fs/walk.d.ts

@@ -0,0 +1,15 @@
+import { type DriveRoot } from './scope.ts';
+export declare const WALK_SKIP_DIRS: Set<string>;
+export declare const MAX_WALK_ENTRIES = 10000;
+export declare const MAX_WALK_MS = 30000;
+export interface WalkedFile {
+    path: string;
+    key: string;
+    byteLength: number;
+}
+export interface WalkResult {
+    files: WalkedFile[];
+    truncated: boolean;
+    skippedLinks: number;
+}
+export declare function walkFiles(root: DriveRoot, folder?: string): Promise<WalkResult>;

package/dist-lib/lib/tools/fs/walk.js

@@ -0,0 +1,38 @@
+import { hlog } from "../../log.js";
+import { displayPath } from "./scope.js";
+export const WALK_SKIP_DIRS = new Set(['.git', 'node_modules']);
+export const MAX_WALK_ENTRIES = 10_000;
+export const MAX_WALK_MS = 30_000;
+export async function walkFiles(root, folder = '/') {
+    const files = [];
+    let visited = 0;
+    let skippedLinks = 0;
+    const deadline = Date.now() + MAX_WALK_MS;
+    try {
+        for await (const entry of root.drive.list(folder, { skip: WALK_SKIP_DIRS })) {
+            if (++visited > MAX_WALK_ENTRIES)
+                return { files, truncated: true, skippedLinks };
+            if (Date.now() > deadline)
+                return { files, truncated: true, skippedLinks };
+            if (entry.value.linkname !== null) {
+                skippedLinks++;
+                continue;
+            }
+            files.push({
+                path: displayPath(root, entry.key),
+                key: entry.key,
+                byteLength: entry.value.blob?.byteLength ?? 0
+            });
+        }
+    }
+    catch (err) {
+        hlog('warn: walk aborted mid-listing', {
+            root: displayPath(root, '/'),
+            folder,
+            visited,
+            err: err.message
+        });
+        return { files, truncated: true, skippedLinks };
+    }
+    return { files, truncated: false, skippedLinks };
+}

package/dist-lib/lib/tools/fs/write.d.ts

@@ -0,0 +1,4 @@
+import type { Drive } from './drive.ts';
+export declare function writeText(drive: Drive, key: string, text: string, opts?: {
+    executable?: boolean;
+}): Promise<void>;

package/dist-lib/lib/tools/fs/write.js

@@ -0,0 +1,5 @@
+import b4a from 'b4a';
+import { pipelinePromise, Readable } from 'streamx';
+export async function writeText(drive, key, text, opts) {
+    await pipelinePromise(Readable.from([b4a.from(text, 'utf8')]), drive.createWriteStream(key, opts));
+}

package/dist-lib/lib/tools/generate-image.d.ts

@@ -0,0 +1,111 @@
+import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
+import type { ToolDefinition } from '../capabilities/completion.ts';
+import type { ModelSrc } from '../models/context.ts';
+import { type ImageClamps } from '../models/model-catalog.ts';
+import { type LoadTracker, type TrackedLoad } from '../models/model-pool.ts';
+import { type ModelRegistrySdk } from '../models/model-source.ts';
+export declare const MAX_DIM = 1024;
+export declare const DIM_STEP = 64;
+export declare const MAX_STEPS = 100;
+export interface DiffusionTick {
+    step: number;
+    totalSteps: number;
+    elapsedMs: number;
+}
+interface DiffusionRun {
+    progressStream: AsyncGenerator<DiffusionTick>;
+    outputs: Promise<Uint8Array[]>;
+    stats: Promise<unknown>;
+}
+export interface DiffusionParams {
+    prompt: string;
+    vae_tiling?: boolean;
+    width?: number;
+    height?: number;
+    steps?: number;
+    seed?: number;
+    negative_prompt?: string;
+    init_image?: Uint8Array;
+    init_images?: Uint8Array[];
+    auto_resize_ref_image?: boolean;
+    strength?: number;
+    guidance?: number;
+    cfg_scale?: number;
+}
+export interface DiffusionModelConfig {
+    prediction?: 'flux2_flow';
+    llmModelSrc?: ModelSrc;
+    vaeModelSrc?: ModelSrc;
+}
+export type DiffusionSdk = ModelRegistrySdk & {
+    loadModel(params: {
+        modelSrc: ModelSrc;
+        modelType: 'diffusion';
+        modelConfig?: DiffusionModelConfig;
+        onProgress?: (progress: Record<string, unknown>) => void;
+    }): TrackedLoad;
+    unloadModel(params: {
+        modelId: string;
+    }): Promise<unknown>;
+    diffusion(params: DiffusionParams & {
+        modelId: string;
+    }): DiffusionRun;
+    cancel?(params: {
+        operation: 'inference';
+        modelId: string;
+    } | {
+        requestId: string;
+    }): Promise<void>;
+};
+export interface ImageModelSet {
+    model: string;
+    llmModel?: string;
+    vaeModel?: string;
+    prediction?: 'flux2_flow';
+}
+export declare const IMAGE_LOAD_STALL_MS: number;
+export interface DiffusionRuntimeOptions extends ImageModelSet {
+    loadSdk: () => Promise<DiffusionSdk>;
+    onProgress?: (model: string, progress: Record<string, unknown>) => void;
+    onBeforeLoad?: (signal?: BareAbortSignal) => Promise<void>;
+    loads?: LoadTracker;
+    stallMs?: number;
+}
+export interface LoadedDiffusionModel {
+    modelId: string;
+    prediction?: 'flux2_flow';
+}
+export interface DiffusionRuntime {
+    loadSdk: () => Promise<DiffusionSdk>;
+    ensureLoaded: (sdk: DiffusionSdk, signal?: BareAbortSignal, modelChoice?: string) => Promise<LoadedDiffusionModel>;
+    predictionFor(modelChoice?: string): 'flux2_flow' | undefined;
+    unload(): Promise<void>;
+    unloadIdle(): Promise<void>;
+    acquire(): boolean;
+    release(): void;
+}
+export declare function generationPercentTick(onPercent: (percentage: number) => void, encodePhases?: number): (tick: DiffusionTick) => void;
+export declare const IMAGE_LOAD_STALL_ERROR = "image model download stalled";
+export declare function createDiffusionRuntime({ loadSdk, onProgress, onBeforeLoad, loads, stallMs, ...base }: DiffusionRuntimeOptions): DiffusionRuntime;
+export declare const BUSY_ERROR = "an image generation is already running; try again when it finishes";
+export declare function runDiffusion(sdk: DiffusionSdk, modelId: string, params: DiffusionParams, signal?: BareAbortSignal, onTick?: (tick: DiffusionTick) => void): Promise<Uint8Array | {
+    error: string;
+}>;
+export interface CommonImageArgs {
+    prompt: string;
+    steps?: number;
+    seed?: number;
+    negativePrompt?: string;
+}
+export declare function parseCommonArgs(args: Record<string, unknown>, maxSteps?: number): CommonImageArgs | {
+    error: string;
+};
+export declare function normalizeClamps(clamps?: ImageClamps): {
+    maxDim: number;
+    maxSteps: number;
+};
+export declare function settingsDimension(value: number | undefined, maxDim?: number): number | undefined;
+export declare function settingsSteps(value: number | undefined, maxSteps?: number): number | undefined;
+export declare function effectiveSteps(parsedSteps: number | undefined, settingsValue: number | undefined, stepsCap: number | undefined): number | undefined;
+export declare function createGenerateImageTool(runtime: DiffusionRuntime, clamps?: ImageClamps): ToolDefinition;
+export {};