@qvac/harness 0.1.0 → 0.1.1

package/dist-lib/lib/serve.js

@@ -0,0 +1,612 @@
+import { AbortController } from 'bare-abort-controller';
+import fs from 'fs';
+import { settleOnClose } from "./rpc-session.js";
+import { pipeline, Readable } from 'streamx';
+import { bindApi } from '../spec/bind.js';
+import HRPC from '../spec/hrpc/index.js';
+import { unsupportedKvCacheDeletion } from "./capabilities/kv-cache.js";
+import { errorMessage } from "./error-fields.js";
+import { hlog } from "./log.js";
+import { pushQueue } from "./queue.js";
+import { TextCoalescer } from "./helpers/text-coalescer.js";
+import { prepareOpenclCache } from "./models/opencl-cache.js";
+import { stageFile, sweepStaging } from "./staging.js";
+import { readSetup } from "./tools/skills/manifest-parse.js";
+import { packVectors, toDownloadWire, toWire } from "./wire.js";
+const MODEL_STATUSES = new Set([
+    'loading',
+    'downloading',
+    'ready',
+    'error',
+    'unloading',
+    'unloaded'
+]);
+const DOWNLOAD_PROGRESS_INTERVAL_MS = 500;
+const WIRE_HIGH_WATER_MARK = 64;
+const TEXT_COALESCE_CAP = 8192;
+const TEXT_COALESCE_MS = 60;
+function noop() { }
+function tick() {
+    return new Promise((resolve) => setTimeout(resolve, 0));
+}
+export function serveHarness(stream, harness) {
+    sweepStaging();
+    prepareOpenclCache();
+    const rpc = settleOnClose(new HRPC(stream));
+    bindApi(rpc, {
+        setCredentials: async ({ entries }) => {
+            await harness.setCredentials?.(entries ?? []);
+            return { ok: true };
+        },
+        setSkillSettings: async ({ entries }) => {
+            await harness.setSkillSettings?.(entries ?? []);
+            return { ok: true };
+        },
+        listManagedSkills: async () => ({
+            skills: (await harness.listManagedSkills?.()) ?? []
+        }),
+        readSkillFile: async ({ name }) => {
+            const file = await harness.readSkillFile?.(name);
+            return file
+                ? { found: true, content: file.content, origin: file.origin }
+                : { found: false, content: '', origin: '' };
+        },
+        writeSkillFile: async ({ name, content }) => {
+            if (!harness.writeSkillFile)
+                return { ok: false };
+            return harness.writeSkillFile(name, content);
+        },
+        copySkillForEdit: async ({ name }) => ({
+            copied: (await harness.copyBaseForEdit?.(name))?.copied ?? false
+        }),
+        deleteOverlaySkill: async ({ name }) => ({
+            removed: (await harness.deleteOverlaySkill?.(name))?.removed ?? false
+        }),
+        setCustomModels: async ({ models }) => {
+            await harness.setCustomModels?.((models ?? []).map(({ name, src, projectionSrc }) => ({
+                name,
+                src,
+                projectionSrc: projectionSrc ?? undefined
+            })));
+            return { ok: true };
+        },
+        describe: async () => (await harness.describe?.()) ?? {
+            toolNames: [],
+            skills: [],
+            supportsImages: false,
+            supportsKvCacheDeletion: false
+        },
+        diagnostics: async () => harness.diagnostics?.() ?? {
+            models: [],
+            suspended: false,
+            sdkLoaded: false,
+            toolNames: [],
+            skills: []
+        },
+        embed: async ({ model, texts }) => {
+            if (!harness.embed)
+                throw new Error('harness does not support embed');
+            return packVectors(await harness.embed({ model, texts: texts ?? [] }));
+        },
+        transcribe: async ({ model, audio }) => {
+            if (!harness.transcribe)
+                throw new Error('harness does not support transcribe');
+            return {
+                text: await harness.transcribe({ model: model ?? '', audio: audio ?? Buffer.alloc(0) })
+            };
+        },
+        speak: async ({ model, text }) => {
+            if (!harness.speak)
+                throw new Error('harness does not support speak');
+            return harness.speak({ model: model ?? '', text: text ?? '' });
+        },
+        speakStream: speakStream(harness),
+        getEmbeddingDevice: async () => {
+            if (!harness.getEmbeddingDevice)
+                throw unsupportedEmbeddingDeviceControl();
+            return { device: await harness.getEmbeddingDevice() };
+        },
+        setEmbeddingDevice: async ({ device }) => {
+            if (!harness.setEmbeddingDevice)
+                throw unsupportedEmbeddingDeviceControl();
+            await harness.setEmbeddingDevice(device);
+            return { ok: true };
+        },
+        skills: async (req) => ({
+            skills: ((await harness.skills?.(req?.agentId, req?.modelName, {
+                refresh: req?.refresh === true,
+                skipCredentialProbe: req?.skipCredentialProbe === true
+            })) ?? []).map(toWireSkill)
+        }),
+        tools: async () => ({
+            tools: (await harness.tools?.()) ?? []
+        }),
+        listModels: async (req) => ({
+            models: (await harness.list?.({
+                type: (req?.type || undefined),
+                cachedOnly: req?.cachedOnly ?? undefined,
+                fullRegistry: req?.fullRegistry ?? undefined
+            })) ?? []
+        }),
+        deleteModel: async (req) => {
+            const name = req?.name ?? '';
+            return (await harness.deleteModel?.(name)) ?? { name, removed: 0, freedBytes: 0 };
+        },
+        clearKvCache: async () => (await harness.clearKvCache?.()) ?? unsupportedKvCacheDeletion(),
+        infraModels: async ({ profile }) => ({
+            models: (await harness.infraModels?.(profile ?? '')) ?? []
+        }),
+        capability: async () => {
+            if (!harness.capability)
+                throw new Error('harness does not support capability');
+            return harness.capability();
+        },
+        warmModel: async ({ model, agent }) => {
+            if (!harness.warmModel)
+                throw new Error('harness does not support warmModel');
+            await harness.warmModel({ model, agent: agent ?? {} });
+            return { ok: true };
+        },
+        modelEvents: () => modelEvents(harness),
+        completion: completion(harness)
+    });
+    rpc.onDownload((out) => {
+        const src = downloadStream(harness, out.data);
+        pipeline(src, out, noop);
+        out.writeStream?.on('close', () => src.destroy());
+    });
+}
+function toWireSkill(skill) {
+    const setupV2 = readSetup(skill.setup);
+    const binaries = (skill.requires?.bins ?? []).map((bin) => ({
+        bin,
+        minVersion: skill.requires?.binMinVersions?.[bin]?.min,
+        command: skill.requires?.binMinVersions?.[bin]?.command
+    }));
+    return {
+        name: skill.name,
+        description: skill.description,
+        tools: skill.tools,
+        credentials: skill.credentials,
+        allowList: skill.allowList,
+        platforms: skill.platform,
+        status: skill.status,
+        ...(skill.requires
+            ? {
+                requirements: {
+                    binaries,
+                    credentials: skill.requires.credentials ?? []
+                }
+            }
+            : {}),
+        ...(skill.install ? { install: skill.install } : {}),
+        ...(skill.setup ? { setup: skill.setup, ...(setupV2 ? { setupV2 } : {}) } : {}),
+        ...(skill.stub !== undefined ? { stub: skill.stub } : {}),
+        ...(skill.minCapabilityProfile ? { minCapabilityProfile: skill.minCapabilityProfile } : {}),
+        ...(skill.emoji ? { emoji: skill.emoji } : {})
+    };
+}
+function unsupportedEmbeddingDeviceControl() {
+    return new Error('harness does not support embedding device control');
+}
+function modelEvents(harness) {
+    let last = '';
+    let pendingProgress = null;
+    let lastProgressAt = 0;
+    const flushProgress = (at) => {
+        if (pendingProgress === null)
+            return;
+        stream.push(pendingProgress);
+        pendingProgress = null;
+        lastProgressAt = at;
+    };
+    const onModel = (state) => {
+        if (!MODEL_STATUSES.has(state.status))
+            return;
+        const percentage = Math.floor(state.percentage ?? 0);
+        const downloaded = Math.floor(state.downloaded ?? 0);
+        const total = Math.floor(state.total ?? 0);
+        const loaded = state.loaded ?? [];
+        const catalogChanged = state.catalogChanged === true;
+        const key = [
+            state.status,
+            state.model ?? '',
+            percentage,
+            downloaded,
+            total,
+            state.message ?? '',
+            catalogChanged,
+            ...loaded.map((entry) => `${entry.model}:${entry.refs}`)
+        ].join('\0');
+        if (key === last)
+            return;
+        last = key;
+        const frame = {
+            status: state.status,
+            model: state.model,
+            percentage,
+            downloaded,
+            total,
+            message: state.message,
+            catalogChanged,
+            loaded
+        };
+        const now = Date.now();
+        if (state.status === 'downloading') {
+            pendingProgress = frame;
+            if (lastProgressAt === 0 || now - lastProgressAt >= DOWNLOAD_PROGRESS_INTERVAL_MS) {
+                flushProgress(now);
+            }
+            return;
+        }
+        flushProgress(now);
+        lastProgressAt = 0;
+        stream.push(frame);
+    };
+    const stream = new Readable({
+        open(cb) {
+            harness.on?.('model', onModel);
+            cb(null);
+        },
+        destroy(cb) {
+            harness.off?.('model', onModel);
+            cb(null);
+        }
+    });
+    return stream;
+}
+function speakStream(harness) {
+    return async function* (duplex) {
+        if (!harness.speakStream)
+            throw new Error('harness does not support speakStream');
+        const text = pushQueue();
+        const end = () => text.end();
+        duplex.on('close', end);
+        duplex.on('end', end);
+        duplex.readStream?.on('close', end);
+        duplex.on('data', (frame) => {
+            if (frame.text)
+                text.push(frame.text);
+        });
+        for await (const chunk of harness.speakStream({ model: '', text })) {
+            yield { audio: chunk.audio, sampleRate: chunk.sampleRate };
+        }
+    };
+}
+function completion(harness) {
+    return async function* (duplex) {
+        const events = pushQueue({ highWaterMark: WIRE_HIGH_WATER_MARK });
+        const pending = new Map();
+        const pendingAttach = new Map();
+        const pendingFetch = new Map();
+        const pendingRag = new Map();
+        const pendingQuestion = new Map();
+        const pendingDrive = new Map();
+        const controller = new AbortController();
+        let calls = 0;
+        let started = false;
+        const gone = () => {
+            controller.abort('client gone');
+            for (const resolve of pending.values())
+                resolve(false);
+            pending.clear();
+            for (const { reject, staged } of pendingAttach.values()) {
+                void fs.promises.unlink(staged).catch(noop);
+                reject(new Error('client gone'));
+            }
+            pendingAttach.clear();
+            for (const { reject } of pendingFetch.values())
+                reject(new Error('client gone'));
+            pendingFetch.clear();
+            for (const { reject } of pendingRag.values())
+                reject(new Error('client gone'));
+            pendingRag.clear();
+            for (const { resolve } of pendingQuestion.values())
+                resolve(null);
+            pendingQuestion.clear();
+            for (const { reject, path } of pendingDrive.values()) {
+                if (path)
+                    void fs.promises.unlink(path).catch(noop);
+                reject(new Error('client gone'));
+            }
+            pendingDrive.clear();
+            events.end();
+        };
+        duplex.on('close', gone);
+        duplex.on('end', gone);
+        duplex.readStream?.on('close', gone);
+        duplex.on('data', (frame) => {
+            if (frame.type === 'decision') {
+                if (typeof frame.callId !== 'string')
+                    return;
+                pending.get(frame.callId)?.(frame.approved === true);
+                pending.delete(frame.callId);
+                return;
+            }
+            if (frame.type === 'attach-result') {
+                if (typeof frame.callId !== 'string')
+                    return;
+                const waiter = pendingAttach.get(frame.callId);
+                pendingAttach.delete(frame.callId);
+                if (!waiter)
+                    return;
+                void fs.promises.unlink(waiter.staged).catch(noop);
+                if (frame.attachment)
+                    waiter.resolve(frame.attachment);
+                else
+                    waiter.reject(new Error(frame.message || 'the host failed to store the attachment'));
+                return;
+            }
+            if (frame.type === 'fetch-result') {
+                if (typeof frame.callId !== 'string')
+                    return;
+                const waiter = pendingFetch.get(frame.callId);
+                pendingFetch.delete(frame.callId);
+                if (!waiter)
+                    return;
+                if (frame.message)
+                    return waiter.reject(new Error(frame.message));
+                const staged = frame.path;
+                const { fileName, mimeType, attachmentId, size } = frame;
+                if (frame.meta === true) {
+                    return waiter.resolve({
+                        id: attachmentId ?? '',
+                        fileName: fileName ?? '',
+                        mimeType: mimeType ?? '',
+                        byteLength: typeof size === 'number' ? size : 0
+                    });
+                }
+                if (!staged)
+                    return waiter.resolve(null);
+                fs.promises.readFile(staged).then((bytes) => waiter.resolve({
+                    id: attachmentId ?? '',
+                    fileName: fileName ?? '',
+                    mimeType: mimeType ?? '',
+                    byteLength: typeof size === 'number' ? size : bytes.byteLength,
+                    bytes
+                }), (err) => waiter.reject(err));
+                return;
+            }
+            if (frame.type === 'rag-result') {
+                if (typeof frame.callId !== 'string')
+                    return;
+                const waiter = pendingRag.get(frame.callId);
+                pendingRag.delete(frame.callId);
+                if (!waiter)
+                    return;
+                if (frame.message)
+                    return waiter.reject(new Error(frame.message));
+                waiter.resolve(frame.result ?? { enabled: false, indexEmpty: true, hasKnowledge: false, hits: [] });
+            }
+            if (frame.type === 'question-result') {
+                if (typeof frame.callId !== 'string')
+                    return;
+                const waiter = pendingQuestion.get(frame.callId);
+                pendingQuestion.delete(frame.callId);
+                if (!waiter)
+                    return;
+                if (frame.message)
+                    hlog('question failed', { callId: frame.callId, message: frame.message });
+                waiter.resolve(frame.answer || null);
+                return;
+            }
+            if (frame.type === 'drive-result') {
+                if (typeof frame.callId !== 'string')
+                    return;
+                const waiter = pendingDrive.get(frame.callId);
+                pendingDrive.delete(frame.callId);
+                if (!waiter)
+                    return;
+                if (frame.message)
+                    return waiter.reject(new Error(frame.message));
+                waiter.resolve((frame.payload ?? {}));
+                return;
+            }
+            if (frame.type !== 'start' || started)
+                return;
+            started = true;
+            const context = {
+                model: frame.model ?? '',
+                agent: frame.agent ?? {},
+                cacheKey: frame.cacheKey ?? undefined,
+                thinkingDefault: frame.thinkingDefault ?? undefined,
+                generationParams: frame.generationParams ?? undefined,
+                responseFormat: frame.responseFormat ?? undefined,
+                messages: (frame.messages ?? []).map(({ role, content, attachments }) => ({
+                    role,
+                    content,
+                    ...(attachments?.length
+                        ? { attachments: attachments.map(({ path: file }) => ({ path: file })) }
+                        : {})
+                })),
+                toolConfigs: frame.toolConfigs ?? undefined,
+                grantedDirs: frame.grantedDirs ?? undefined,
+                grantedDrives: frame.grantedDrives?.length
+                    ? frame.grantedDrives.map((drive) => ({
+                        key: drive.key,
+                        writable: drive.writable === true
+                    }))
+                    : undefined,
+                driveOp: frame.grantedDrives?.length
+                    ? (request) => controller.signal.aborted
+                        ? Promise.reject(new Error('client gone'))
+                        : new Promise((resolve, reject) => {
+                            const callId = String(calls++);
+                            pendingDrive.set(callId, {
+                                resolve,
+                                reject,
+                                ...(request.op === 'write' ? { path: request.path } : {})
+                            });
+                            events.push({ type: 'drive-ask', callId, op: request.op, payload: request });
+                        })
+                    : undefined,
+                grantedWriteDirs: frame.grantedWriteDirs ?? undefined,
+                signal: controller.signal,
+                approve: (call) => new Promise((resolve) => {
+                    const callId = String(calls++);
+                    pending.set(callId, resolve);
+                    events.push({
+                        type: 'approval-ask',
+                        callId,
+                        name: call.name,
+                        arguments: call.arguments,
+                        resource: call.resource ?? undefined,
+                        effect: call.effect,
+                        intent: call.intent ?? undefined,
+                        display: call.display ?? undefined
+                    });
+                }),
+                attach: async (input) => {
+                    const staged = await stageFile('attach', input.bytes);
+                    if (controller.signal.aborted) {
+                        void fs.promises.unlink(staged).catch(noop);
+                        throw new Error('client gone');
+                    }
+                    return new Promise((resolve, reject) => {
+                        const callId = String(calls++);
+                        pendingAttach.set(callId, { resolve, reject, staged });
+                        events.push({
+                            type: 'attach-ask',
+                            callId,
+                            path: staged,
+                            fileName: input.fileName,
+                            mimeType: input.mimeType,
+                            raw: input.raw ?? false
+                        });
+                    });
+                },
+                fetchAttachment: ({ attachmentId, meta } = {}) => controller.signal.aborted
+                    ? Promise.reject(new Error('client gone'))
+                    : new Promise((resolve, reject) => {
+                        const callId = String(calls++);
+                        pendingFetch.set(callId, { resolve, reject });
+                        events.push({
+                            type: 'fetch-ask',
+                            callId,
+                            attachmentId,
+                            ...(meta === true ? { meta: true } : {})
+                        });
+                    }),
+                ragSearch: ({ query, limit }) => controller.signal.aborted
+                    ? Promise.reject(new Error('client gone'))
+                    : new Promise((resolve, reject) => {
+                        const callId = String(calls++);
+                        pendingRag.set(callId, { resolve, reject });
+                        events.push({ type: 'rag-ask', callId, query, limit });
+                    }),
+                progress: (update) => {
+                    events.push(update
+                        ? { type: 'progress', percentage: update.percentage }
+                        : { type: 'progress', cleared: true });
+                },
+                askUser: (question) => controller.signal.aborted
+                    ? Promise.resolve(null)
+                    : new Promise((resolve, reject) => {
+                        const callId = String(calls++);
+                        pendingQuestion.set(callId, { resolve, reject });
+                        events.push({
+                            type: 'question-ask',
+                            callId,
+                            question: question.question,
+                            options: question.options
+                        });
+                    })
+            };
+            void (async () => {
+                let produced = 0;
+                let frames = 0;
+                const text = new TextCoalescer((type, merged) => {
+                    frames++;
+                    return events.push({ type, text: merged });
+                }, { cap: TEXT_COALESCE_CAP, holdMs: TEXT_COALESCE_MS });
+                try {
+                    for await (const event of harness.completion(context)) {
+                        produced++;
+                        const wire = toWire(event);
+                        if ((wire.type === 'content' || wire.type === 'thinking') && wire.text) {
+                            await text.add(wire.type, wire.text);
+                            continue;
+                        }
+                        await text.flush();
+                        frames++;
+                        await events.push(wire);
+                    }
+                    await text.flush();
+                }
+                catch (err) {
+                    await text.flush();
+                    events.push({ type: 'error', message: errorMessage(err) });
+                }
+                events.push({ type: 'done' });
+                hlog('wire end', { produced, frames });
+                events.end();
+            })();
+        });
+        for await (const event of events) {
+            yield event;
+            await tick();
+        }
+        hlog('wire handed off');
+    };
+}
+function downloadStream(harness, req) {
+    const controller = new AbortController();
+    const stream = new Readable({
+        open(cb) {
+            void run();
+            cb(null);
+        },
+        destroy(cb) {
+            controller.abort('client gone');
+            cb(null);
+        }
+    });
+    const run = async () => {
+        if (!harness.download) {
+            stream.push({
+                type: 'error',
+                message: 'harness does not support download'
+            });
+            stream.push(null);
+            return;
+        }
+        let pendingProgress = null;
+        let lastProgressAt = 0;
+        const flushProgress = (at = Date.now()) => {
+            if (pendingProgress === null)
+                return;
+            stream.push(pendingProgress);
+            pendingProgress = null;
+            lastProgressAt = at;
+        };
+        try {
+            for await (const event of harness.download({
+                model: req.model,
+                addon: req.addon,
+                seed: req.seed,
+                signal: controller.signal
+            })) {
+                const wire = toDownloadWire(event);
+                if (wire.type === 'progress') {
+                    const now = Date.now();
+                    pendingProgress = wire;
+                    if (lastProgressAt === 0 || now - lastProgressAt >= DOWNLOAD_PROGRESS_INTERVAL_MS) {
+                        flushProgress(now);
+                    }
+                    continue;
+                }
+                flushProgress();
+                stream.push(wire);
+            }
+            flushProgress();
+        }
+        catch (err) {
+            if (!controller.signal.aborted) {
+                flushProgress();
+                stream.push({ type: 'error', message: errorMessage(err) });
+            }
+        }
+        stream.push(null);
+    };
+    return stream;
+}

package/dist-lib/lib/spawn.d.ts

@@ -0,0 +1,62 @@
+import type { SdkStorage } from './storage.ts';
+import { type StowIpc, type StowRunner } from 'bare-supervisor/stow';
+import type { ChildSpec } from 'bare-supervisor';
+import { type RemoteHarness } from './connect.ts';
+import { type EmbedDevice } from './models/embed-config.ts';
+import type { ImageClamps } from './models/model-catalog.ts';
+import type { ImageModelSet } from './tools/generate-image.ts';
+import type { MusicClamps, MusicModelStages } from './tools/generate-music.ts';
+export type MusicModelArg = string | (MusicModelStages & {
+    inferenceSteps?: number;
+    useGpu?: boolean;
+});
+export interface Logger {
+    info(...args: unknown[]): void;
+    warn(...args: unknown[]): void;
+}
+export interface HarnessLlmConfig {
+    llmDevice?: 'cpu' | 'gpu';
+    llmGpuLayers?: number;
+    contextSize?: number;
+    kvCacheK?: string;
+    kvCacheV?: string;
+}
+export interface HarnessChildOptions {
+    runner: StowRunner;
+    skillsDir?: string;
+    wheelsDir?: string;
+    sdkStorage?: SdkStorage;
+    embedDevice?: EmbedDevice;
+    embedGpuLayers?: number;
+    embedBatchSize?: number;
+    imageModel?: string | ImageModelSet;
+    imageClamps?: ImageClamps;
+    musicModel?: MusicModelArg;
+    musicClamps?: MusicClamps;
+    llm?: HarnessLlmConfig;
+    onIpc?: (ipc: StowIpc) => void;
+    logger?: Logger | null;
+    deps?: string[];
+    restart?: 'always' | 'never';
+    maxRestarts?: number;
+    window?: number;
+    backoff?: number;
+    maxBackoff?: number;
+}
+export declare function harnessChild(entry: string, opts: HarnessChildOptions): ChildSpec<RemoteHarness>;
+export interface SpawnHarnessOptions {
+    entry: string;
+    skillsDir?: string;
+    wheelsDir?: string;
+    sdkStorage: SdkStorage;
+    embedDevice?: EmbedDevice;
+    embedGpuLayers?: number;
+    embedBatchSize?: number;
+    imageModel?: string | ImageModelSet;
+    imageClamps?: ImageClamps;
+    musicModel?: MusicModelArg;
+    musicClamps?: MusicClamps;
+    llm?: HarnessLlmConfig;
+    logger?: Logger | null;
+}
+export declare function spawnHarness(opts: SpawnHarnessOptions): RemoteHarness;