@qvac/harness 0.1.0 → 0.1.2

package/dist-lib/lib/tools/edit-image.js

@@ -0,0 +1,271 @@
+import fs from 'fs';
+import path from 'path';
+import { AbortedError } from "../helpers/race-abort.js";
+import { BUSY_ERROR, DIM_STEP, effectiveSteps, generationPercentTick, IMAGE_LOAD_STALL_ERROR, normalizeClamps, parseCommonArgs, runDiffusion } from "./generate-image.js";
+const DEFAULT_STRENGTH = 0.7;
+const FLUX2_GUIDANCE = 3.5;
+const FLUX2_CFG_SCALE = 1;
+const MULTI_SOURCE_UNSUPPORTED = 'this image model edits one image at a time — fusing multiple sources needs an in-context editing model (FLUX.2 today)';
+const IMAGE_REF_TOKEN_PATTERN = /@image\d+/i;
+async function turnSources(turnImages) {
+    const sources = [];
+    for (const image of turnImages) {
+        const bytes = await fs.promises.readFile(image.path).catch(toNull);
+        if (!bytes)
+            return { error: `could not read attached image ${path.basename(image.path)}` };
+        sources.push({ bytes, fileName: path.basename(image.path) });
+    }
+    return sources;
+}
+async function resolveSources(fetchAttachment, ids, turnImages, signal) {
+    if (!ids.length) {
+        if (turnImages.length) {
+            const sources = await turnSources(turnImages);
+            return 'error' in sources ? sources : { sources };
+        }
+        const latest = await fetchAttachment({ signal });
+        if (!latest || !('bytes' in latest) || !latest.bytes) {
+            return { error: 'no image in this chat to edit' };
+        }
+        return { sources: [latest] };
+    }
+    const sources = [];
+    const missing = [];
+    for (const id of ids) {
+        const found = await fetchAttachment({ attachmentId: id, signal });
+        if (found && 'bytes' in found && found.bytes)
+            sources.push(found);
+        else
+            missing.push(id);
+    }
+    if (!missing.length)
+        return { sources };
+    if (sources.length) {
+        return {
+            sources,
+            note: `ignored unknown attachment id(s) ${missing.join(', ')} — edited the found source(s) only`
+        };
+    }
+    if (turnImages.length) {
+        const fallback = await turnSources(turnImages);
+        if ('error' in fallback)
+            return fallback;
+        return {
+            sources: fallback,
+            note: `ignored unknown attachment id(s) ${missing.join(', ')} — edited the image(s) from the user's latest message`
+        };
+    }
+    return {
+        error: `attachment ${missing.join(', ')} not found in this chat — ids come only from earlier ` +
+            "tool results. Call again WITHOUT ids to edit the image(s) from the user's latest message"
+    };
+}
+async function defaultLoadMedia() {
+    try {
+        return (await import('bare-media'));
+    }
+    catch {
+        return null;
+    }
+}
+async function defaultResample(rgba, width, height) {
+    const { resize } = await import('bare-image-resample');
+    return resize(rgba, width, height);
+}
+export function createEditImageTool(runtime, { loadMedia = defaultLoadMedia, resample = defaultResample } = {}, clamps) {
+    const cap = normalizeClamps(clamps);
+    const stepsCap = clamps?.maxSteps ? cap.maxSteps : undefined;
+    const snap = (n) => Math.min(cap.maxDim, Math.max(DIM_STEP, Math.round(n / DIM_STEP) * DIM_STEP));
+    return {
+        name: 'edit_image',
+        toolset: null,
+        skillGated: true,
+        description: 'Edit or restyle an existing image from this chat using the local diffusion model ' +
+            '(image-to-image). To edit what the user just attached, call with the prompt ONLY — the ' +
+            "image(s) from the user's latest message (or the chat's most recent image) are the " +
+            'sources automatically. Never invent attachment ids; `attachment_ids` accepts only ids ' +
+            'returned by earlier tool results. Multiple sources fuse into one output (e.g. an object ' +
+            'from one image placed into another) — only image models with in-context editing support ' +
+            'this (FLUX.2 today); other models edit exactly one image. When fusing, reference the sources in the prompt as ' +
+            '@image1, @image2, ... in source order (e.g. "put @image1\'s hat on the man in @image2") ' +
+            '— without @imageN tokens the model collapses toward single-reference behavior. ' +
+            '`strength` 0..1 controls how far to move from the source: low keeps it, ' +
+            'high follows the prompt (default 0.7); in-context editing models ignore it. ' +
+            `Output keeps the source aspect ratio, fitted to the 64px grid (max ${cap.maxDim} per side on this device). ` +
+            'The edited image is saved as a chat attachment and shown to the user automatically; ' +
+            'the attachment id is returned to you.',
+        parameters: {
+            type: 'object',
+            properties: {
+                prompt: { type: 'string', description: 'How the image should change' },
+                attachment_id: {
+                    type: 'string',
+                    description: 'Chat attachment to edit; omitted = the latest image in the chat'
+                },
+                attachment_ids: {
+                    type: 'array',
+                    items: { type: 'string' },
+                    description: 'Source attachments to fuse (in-context editing models only); the first sets the output ' +
+                        'size. Only ids ' +
+                        "from earlier tool results — OMIT entirely for the user's attached images"
+                },
+                strength: {
+                    type: 'number',
+                    description: '0..1, how far to move from the source (default 0.7); in-context models ignore it'
+                },
+                steps: {
+                    type: 'number',
+                    description: `Diffusion steps (model default when omitted, max ${cap.maxSteps})`
+                },
+                seed: { type: 'number', description: 'Reproducibility seed; -1 or omitted = random' },
+                negative_prompt: { type: 'string', description: 'What the result should NOT contain' }
+            },
+            required: ['prompt']
+        },
+        effect: () => 'write',
+        resource: (args) => ({
+            kind: 'image-edit',
+            label: String(args.prompt ?? '').slice(0, 80)
+        }),
+        execute: async (args, ctx) => {
+            const common = parseCommonArgs(args, cap.maxSteps);
+            if ('error' in common)
+                return common;
+            let strength = DEFAULT_STRENGTH;
+            if (args.strength !== undefined && args.strength !== null) {
+                strength = Number(args.strength);
+                if (!Number.isFinite(strength) || strength < 0 || strength > 1) {
+                    return { error: 'strength must be a number between 0 and 1' };
+                }
+            }
+            const attachmentId = typeof args.attachment_id === 'string' && args.attachment_id
+                ? args.attachment_id
+                : undefined;
+            const attachmentIds = [];
+            if (Array.isArray(args.attachment_ids)) {
+                for (const id of args.attachment_ids) {
+                    if (typeof id === 'string' && id)
+                        attachmentIds.push(id);
+                }
+            }
+            if (attachmentId && attachmentIds.length) {
+                return { error: 'pass attachment_id or attachment_ids, not both' };
+            }
+            if (attachmentId)
+                attachmentIds.push(attachmentId);
+            if (!ctx?.attach || !ctx.fetchAttachment) {
+                return { error: 'this device cannot edit chat images' };
+            }
+            const inContext = runtime.predictionFor(ctx.image?.modelChoice) === 'flux2_flow';
+            const turnImages = ctx.turnImages ?? [];
+            if (!attachmentIds.length && turnImages.length > 1 && !inContext) {
+                return { error: MULTI_SOURCE_UNSUPPORTED };
+            }
+            const resolved = await resolveSources(ctx.fetchAttachment, attachmentIds, turnImages, ctx.signal);
+            if (ctx.signal?.aborted)
+                return { error: 'image generation canceled' };
+            if ('error' in resolved)
+                return resolved;
+            const { sources, note } = resolved;
+            if (sources.length > 1 && !inContext)
+                return { error: MULTI_SOURCE_UNSUPPORTED };
+            if (!runtime.acquire())
+                return { error: BUSY_ERROR };
+            const report = ctx.progress;
+            const onTick = report && generationPercentTick((percentage) => report({ percentage }), sources.length);
+            try {
+                const media = await loadMedia();
+                if (!media) {
+                    return {
+                        error: 'image editing needs the native media codecs, unavailable on this device'
+                    };
+                }
+                const inits = [];
+                let width = 0;
+                let height = 0;
+                for (const source of sources) {
+                    let decoded;
+                    try {
+                        decoded = await media.image.decode(Buffer.from(source.bytes), { maxFrames: 1 });
+                    }
+                    catch {
+                        return { error: `could not decode ${source.fileName} as an image` };
+                    }
+                    const frame = decoded.frames?.length ? decoded.frames[0] : decoded;
+                    const scale = Math.min(1, cap.maxDim / Math.max(frame.width, frame.height));
+                    const fitWidth = snap(frame.width * scale);
+                    const fitHeight = snap(frame.height * scale);
+                    const fitted = frame.width === fitWidth && frame.height === fitHeight
+                        ? frame
+                        : await resample(frame, fitWidth, fitHeight);
+                    inits.push(Buffer.from(await media.image.encode(fitted, { mimetype: 'image/png' })));
+                    if (!width) {
+                        width = fitWidth;
+                        height = fitHeight;
+                    }
+                }
+                const sdk = await runtime.loadSdk();
+                const { modelId } = await runtime.ensureLoaded(sdk, ctx.signal, ctx.image?.modelChoice);
+                const fusion = inits.length > 1;
+                const image = await runDiffusion(sdk, modelId, {
+                    prompt: common.prompt,
+                    width,
+                    height,
+                    steps: effectiveSteps(common.steps, ctx.image?.steps, stepsCap),
+                    seed: common.seed,
+                    negative_prompt: common.negativePrompt,
+                    ...(inContext
+                        ? {
+                            init_images: inits,
+                            auto_resize_ref_image: true,
+                            guidance: FLUX2_GUIDANCE,
+                            cfg_scale: FLUX2_CFG_SCALE
+                        }
+                        : { init_image: inits[0], strength })
+                }, ctx.signal, onTick);
+                if (!(image instanceof Uint8Array))
+                    return image;
+                const record = await ctx.attach({
+                    fileName: `edited-${Date.now()}.png`,
+                    mimeType: 'image/png',
+                    bytes: image,
+                    raw: true
+                });
+                const notes = note ? [note] : [];
+                if (fusion) {
+                    notes.push(IMAGE_REF_TOKEN_PATTERN.test(common.prompt)
+                        ? `multi-reference fusion of ${sources.length} images`
+                        : `multi-reference fusion of ${sources.length} images; the prompt did not ` +
+                            'reference @image1/@image2 — the model may have favored the first reference. ' +
+                            'Next time write the prompt with explicit @imageN tokens in source order');
+                }
+                return {
+                    attachmentId: record.id,
+                    ...(sources.length === 1 && sources[0].id
+                        ? { sourceAttachmentId: sources[0].id }
+                        : { sourceFileNames: sources.map((source) => source.fileName) }),
+                    ...(notes.length ? { note: notes.join('; ') } : {}),
+                    fileName: record.fileName,
+                    byteLength: record.byteLength,
+                    width,
+                    height
+                };
+            }
+            catch (err) {
+                if (err instanceof AbortedError)
+                    return { error: 'image generation canceled' };
+                if (err instanceof Error && err.message === IMAGE_LOAD_STALL_ERROR) {
+                    return { error: IMAGE_LOAD_STALL_ERROR };
+                }
+                throw err;
+            }
+            finally {
+                runtime.release();
+                report?.(null);
+            }
+        }
+    };
+}
+function toNull() {
+    return null;
+}

package/dist-lib/lib/tools/exec/adapters/adapter.d.ts

@@ -0,0 +1,27 @@
+import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
+import type { AttachFn, FetchAttachmentFn, RunBudget } from '../../../capabilities/completion.ts';
+import type { ExecResult } from '../runner.ts';
+export type ExecutionOptions = {
+    readonly cwd?: string;
+    readonly env?: Readonly<Record<string, string>>;
+    readonly timeoutMs?: number;
+    readonly maxOutputChars?: number;
+    readonly signal?: BareAbortSignal;
+};
+export type ExecutionContext = {
+    readonly attach?: AttachFn;
+    readonly fetchAttachment?: FetchAttachmentFn;
+    readonly turnImages?: readonly {
+        path: string;
+    }[];
+    readonly runBudget?: RunBudget;
+};
+export type ExecutionPlan = {
+    run(options: ExecutionOptions): Promise<ExecResult> | ExecResult;
+};
+export type ExecutionAdapter = {
+    resolve(args: Record<string, unknown>, ctx?: ExecutionContext): ExecutionPlan | null;
+};
+export type FallbackExecutionAdapter = {
+    resolve(args: Record<string, unknown>, ctx?: ExecutionContext): ExecutionPlan;
+};

package/dist-lib/lib/tools/exec/adapters/adapter.js

@@ -0,0 +1 @@
+export {};

package/dist-lib/lib/tools/exec/adapters/python/attachments.d.ts

@@ -0,0 +1,26 @@
+import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
+import type { AttachFn, FetchAttachmentFn, RunBudget } from '../../../../capabilities/completion.ts';
+import type { ExecAttachment, RejectedOutput } from '../../runner.ts';
+export type ExecInput = {
+    readonly attachmentId?: string;
+    readonly path: string;
+};
+export type StagedInput = {
+    readonly path: string;
+    readonly bytes: Uint8Array;
+};
+export declare function stageInputs(fetchAttachment: FetchAttachmentFn, inputs: readonly ExecInput[], opts?: {
+    turnImages?: readonly {
+        path: string;
+    }[];
+    signal?: BareAbortSignal;
+}): Promise<{
+    files: readonly StagedInput[];
+} | {
+    error: string;
+}>;
+export declare function attachOutputs(attach: AttachFn, files: ReadonlyMap<string, Uint8Array>, declared: readonly string[], runBudget?: RunBudget): Promise<{
+    attachments: readonly ExecAttachment[];
+    missingOutputs: readonly string[];
+    rejectedOutputs: readonly RejectedOutput[];
+}>;

package/dist-lib/lib/tools/exec/adapters/python/attachments.js

@@ -0,0 +1,160 @@
+import fs from 'fs';
+import path from 'path';
+import { hlog } from "../../../../log.js";
+import { checkOutputBytes, MAX_INPUT_TOTAL_BYTES, RUN_OUTPUT_BUDGET_BYTES } from "./output-policy.js";
+const STAGE_TIMEOUT_MS = 60_000;
+const STAGE_TIMEOUT_MESSAGE = `staging attachments timed out after ${STAGE_TIMEOUT_MS}ms`;
+const OVER_BUDGET_ERROR = 'inputs exceed maximum total size';
+const NO_UPLOAD_ERROR = 'no uploaded image in this chat — attach an image or pass an attachmentId';
+const DOCUMENT_INPUT = /\.(pdf|docx?|xlsx?|xlsm|pptx?|csv|tsv|txt|md|markdown|rtf|odt|ods|odp|epub|jsonl?|xhtml|html?|xml|ya?ml|log|zip)$/i;
+const NO_DOCUMENT_ERROR = 'an id-less input stages an uploaded image, and this path names a document — pass its ' +
+    'attachmentId, which the [Attached file "…"] line on the user\'s message carries';
+function bounded(work, ms, message) {
+    let timer;
+    const bound = new Promise((_, reject) => {
+        timer = setTimeout(() => reject(new Error(message)), ms);
+    });
+    return Promise.race([work, bound]).finally(() => {
+        if (timer)
+            clearTimeout(timer);
+    });
+}
+function hasBytes(record) {
+    return 'bytes' in record && record.bytes !== null;
+}
+function byteLengthOf(record) {
+    return hasBytes(record) ? record.bytes.byteLength : record.byteLength;
+}
+async function uploadedBytes(fetchAttachment, turnImages, index, signal) {
+    const image = turnImages[index];
+    if (image) {
+        const bytes = await fs.promises.readFile(image.path).catch(toNull);
+        if (!bytes)
+            return { error: `could not read attached image ${path.basename(image.path)}` };
+        return bytes;
+    }
+    if (turnImages.length) {
+        return {
+            error: `the user's latest message has only ${turnImages.length} image(s) — one id-less input per uploaded image`
+        };
+    }
+    if (index > 0)
+        return { error: NO_UPLOAD_ERROR };
+    const latest = await fetchAttachment({ signal });
+    if (!latest || !hasBytes(latest))
+        return { error: NO_UPLOAD_ERROR };
+    return latest.bytes;
+}
+export async function stageInputs(fetchAttachment, inputs, opts = {}) {
+    const { turnImages = [], signal } = opts;
+    const files = inputs.map(() => null);
+    const pending = [];
+    const deadline = Date.now() + STAGE_TIMEOUT_MS;
+    let total = 0;
+    let uploads = 0;
+    for (let i = 0; i < inputs.length; i++) {
+        const input = inputs[i];
+        const id = input.attachmentId;
+        try {
+            if (id === undefined) {
+                if (DOCUMENT_INPUT.test(input.path))
+                    return { error: NO_DOCUMENT_ERROR };
+                const bytes = await bounded(uploadedBytes(fetchAttachment, turnImages, uploads++, signal), deadline - Date.now(), STAGE_TIMEOUT_MESSAGE);
+                if ('error' in bytes)
+                    return bytes;
+                total += bytes.byteLength;
+                if (total > MAX_INPUT_TOTAL_BYTES) {
+                    hlog('python: input attachments over budget; skipping blob download', {
+                        total,
+                        cap: MAX_INPUT_TOTAL_BYTES,
+                        count: inputs.length
+                    });
+                    return { error: OVER_BUDGET_ERROR };
+                }
+                files[i] = { path: input.path, bytes };
+                continue;
+            }
+            const head = await bounded(fetchAttachment({ attachmentId: id, meta: true, signal }), deadline - Date.now(), STAGE_TIMEOUT_MESSAGE);
+            if (!head)
+                return { error: `attachment ${id} not found in this chat` };
+            total += byteLengthOf(head);
+            if (total > MAX_INPUT_TOTAL_BYTES) {
+                hlog('python: input attachments over budget; skipping blob download', {
+                    total,
+                    cap: MAX_INPUT_TOTAL_BYTES,
+                    count: inputs.length
+                });
+                return { error: OVER_BUDGET_ERROR };
+            }
+            if (hasBytes(head))
+                files[i] = { path: input.path, bytes: head.bytes };
+            else
+                pending.push(i);
+        }
+        catch (error) {
+            return { error: error instanceof Error ? error.message : String(error) };
+        }
+    }
+    for (const i of pending) {
+        const input = inputs[i];
+        const id = input.attachmentId;
+        try {
+            const fetched = await bounded(fetchAttachment({ attachmentId: id, signal }), deadline - Date.now(), STAGE_TIMEOUT_MESSAGE);
+            if (!fetched || !hasBytes(fetched)) {
+                return { error: `attachment ${id} not found in this chat` };
+            }
+            files[i] = { path: input.path, bytes: fetched.bytes };
+        }
+        catch (error) {
+            return { error: error instanceof Error ? error.message : String(error) };
+        }
+    }
+    return { files: files };
+}
+export async function attachOutputs(attach, files, declared, runBudget) {
+    const attachments = [];
+    const missingOutputs = [];
+    const rejectedOutputs = [];
+    let total = 0;
+    for (const name of declared) {
+        const bytes = files.get(name);
+        if (!bytes) {
+            missingOutputs.push(name);
+            continue;
+        }
+        const verdict = checkOutputBytes(name, bytes, total);
+        if ('error' in verdict) {
+            rejectedOutputs.push({ name, reason: verdict.error });
+            continue;
+        }
+        if (runBudget && runBudget.outputBytesUsed + bytes.length > RUN_OUTPUT_BUDGET_BYTES) {
+            rejectedOutputs.push({ name, reason: 'run output budget exceeded' });
+            continue;
+        }
+        let record;
+        try {
+            record = await attach({
+                fileName: name,
+                mimeType: verdict.mimeType,
+                bytes,
+                raw: true
+            });
+        }
+        catch {
+            rejectedOutputs.push({ name, reason: 'the host declined to persist this file' });
+            continue;
+        }
+        total += bytes.byteLength;
+        if (runBudget)
+            runBudget.outputBytesUsed += bytes.byteLength;
+        attachments.push({
+            attachmentId: record.id,
+            fileName: record.fileName,
+            byteLength: record.byteLength
+        });
+    }
+    return { attachments, missingOutputs, rejectedOutputs };
+}
+function toNull() {
+    return null;
+}

package/dist-lib/lib/tools/exec/adapters/python/create-effect.d.ts

@@ -0,0 +1,7 @@
+type SourceHalves = {
+    code: string;
+    text: string;
+};
+export declare function splitPythonSource(source: string): SourceHalves | null;
+export declare function pythonCreateEffect(args: Record<string, unknown>): 'read' | 'write';
+export {};