@qvac/harness 0.1.0 → 0.1.1

package/dist-lib/lib/capabilities/usage-breakdown.js

@@ -0,0 +1,44 @@
+import { detectToolDialect } from "../models/tool-dialect.js";
+const QWEN35 = { prose: 4.38, schema: 4.08, toolBlockTokens: 189 };
+const DENSITY_BY_DIALECT = new Map([['qwen35', QWEN35]]);
+export function densityFor(modelName) {
+    const dialect = modelName ? detectToolDialect(modelName, modelName) : undefined;
+    return (dialect && DENSITY_BY_DIALECT.get(dialect)) ?? QWEN35;
+}
+export function promptUsageParts({ system, toolSchemas }) {
+    return {
+        systemPrompt: system.text.length - system.bioChars - system.skillsChars,
+        agentInstructions: system.bioChars,
+        activeSkills: system.skillsChars,
+        toolSchemas: (toolSchemas ?? []).reduce((total, schema) => total + JSON.stringify(schema).length, 0)
+    };
+}
+export function usageBreakdown(chars, totalTokens, modelName) {
+    if (!totalTokens || totalTokens <= 0)
+        return null;
+    const density = densityFor(modelName);
+    const prose = (value) => Math.round(value / density.prose);
+    const rows = {
+        systemPrompt: prose(chars.systemPrompt),
+        agentInstructions: prose(chars.agentInstructions),
+        activeSkills: prose(chars.activeSkills),
+        tools: chars.toolSchemas > 0
+            ? Math.round(chars.toolSchemas / density.schema) + density.toolBlockTokens
+            : 0
+    };
+    const prompt = rows.systemPrompt + rows.agentInstructions + rows.activeSkills + rows.tools;
+    if (prompt > totalTokens)
+        return scaledToWindow(rows, totalTokens);
+    return { ...rows, conversation: totalTokens - prompt, totalTokens };
+}
+function scaledToWindow(rows, totalTokens) {
+    const scale = totalTokens / (rows.systemPrompt + rows.agentInstructions + rows.activeSkills + rows.tools);
+    const scaled = {
+        systemPrompt: Math.floor(rows.systemPrompt * scale),
+        agentInstructions: Math.floor(rows.agentInstructions * scale),
+        activeSkills: Math.floor(rows.activeSkills * scale),
+        tools: Math.floor(rows.tools * scale)
+    };
+    const prompt = scaled.systemPrompt + scaled.agentInstructions + scaled.activeSkills + scaled.tools;
+    return { ...scaled, conversation: totalTokens - prompt, totalTokens };
+}

package/dist-lib/lib/capabilities/vision-history.d.ts

@@ -0,0 +1,3 @@
+import type { Message } from './completion.ts';
+export declare function projectVisionHistory(history: ReadonlyArray<Message>): Message[];
+export declare function visionHistoryCacheKey(cacheKey: string | undefined, history: ReadonlyArray<Message>): string | undefined;

package/dist-lib/lib/capabilities/vision-history.js

@@ -0,0 +1,21 @@
+const CACHE_VERSION = 'vision-history-v1';
+export function projectVisionHistory(history) {
+    return history.flatMap((message) => {
+        const attachments = message.attachments;
+        if (message.role !== 'user' || !attachments || attachments.length < 2)
+            return [message];
+        return attachments.map((attachment, index) => ({
+            ...message,
+            content: index === attachments.length - 1 ? message.content : '\n',
+            attachments: [attachment]
+        }));
+    });
+}
+export function visionHistoryCacheKey(cacheKey, history) {
+    if (!cacheKey || !history.some(hasMultipleImages))
+        return cacheKey;
+    return `${cacheKey}-${CACHE_VERSION}`;
+}
+function hasMultipleImages(message) {
+    return message.role === 'user' && (message.attachments?.length ?? 0) > 1;
+}

package/dist-lib/lib/capabilities/voice.d.ts

@@ -0,0 +1,78 @@
+import type { TextToSpeechStreamClientParams, TextToSpeechStreamSession } from '@qvac/sdk';
+import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
+export declare const SUPERTONIC_SAMPLE_RATE = 44100;
+export type TtsStreamSession = TextToSpeechStreamSession;
+export interface TranscribeStreamSegment {
+    text: string;
+    append: boolean;
+    startMs?: number;
+    endMs?: number;
+    id?: number;
+}
+export interface TranscribeStreamEvent {
+    type: 'text' | 'segment' | 'vad' | 'endOfTurn';
+    text?: string;
+    segment?: TranscribeStreamSegment;
+}
+export interface TranscribeStreamSession {
+    write(audioChunk: Uint8Array): void;
+    end(): void;
+    destroy(): void;
+    [Symbol.asyncIterator](): AsyncIterator<TranscribeStreamEvent>;
+}
+export interface VoiceSdk {
+    transcribe(params: {
+        modelId: string;
+        audioChunk: Buffer;
+    }): Promise<string>;
+    transcribeStream(params: {
+        modelId: string;
+        metadata?: boolean;
+        parakeetStreamingConfig?: {
+            chunkMs?: number;
+            emitPartials?: boolean;
+        };
+    }): Promise<TranscribeStreamSession>;
+    textToSpeech(params: {
+        modelId: string;
+        text: string;
+        inputType: 'text';
+        stream: boolean;
+    }): {
+        bufferStream: AsyncGenerator<number>;
+        buffer: Promise<number[]>;
+        done: Promise<boolean>;
+    };
+    textToSpeechStream(params: TextToSpeechStreamClientParams): Promise<TtsStreamSession>;
+    cancel(params: {
+        modelId: string;
+        operation?: 'inference';
+    }): Promise<unknown>;
+}
+export interface TranscribeContext {
+    model: string;
+    audio: Buffer;
+    signal?: BareAbortSignal;
+}
+export interface SpeakContext {
+    model: string;
+    text: string;
+    signal?: BareAbortSignal;
+}
+export interface SpeakResult {
+    audio: Buffer;
+    sampleRate: number;
+}
+export interface SpeakStreamContext {
+    model: string;
+    text: AsyncIterable<string>;
+    signal?: BareAbortSignal;
+}
+export type VoiceLane = <T>(job: () => Promise<T>) => Promise<T>;
+export declare function createVoiceLane(): VoiceLane;
+export declare function transcribeOnce(sdk: VoiceSdk, modelId: string, audio: Buffer): Promise<string>;
+export declare function transcribeStreaming(sdk: VoiceSdk, modelId: string, pcm: Buffer, signal?: BareAbortSignal): Promise<string>;
+export declare function speakStreamOnce(sdk: VoiceSdk, modelId: string, text: AsyncIterable<string>): AsyncGenerator<SpeakResult>;
+export declare function speakable(text: string): string;
+export declare function speakOnce(sdk: VoiceSdk, modelId: string, text: string): Promise<number[]>;
+export declare function packSamples(samples: number[]): Buffer;

package/dist-lib/lib/capabilities/voice.js

@@ -0,0 +1,189 @@
+import b4a from 'b4a';
+import { AbortedError } from "../helpers/race-abort.js";
+import { hlog } from "../log.js";
+import { normalizeForSpeech } from "./speech-text.js";
+export const SUPERTONIC_SAMPLE_RATE = 44100;
+export function createVoiceLane() {
+    let tail = Promise.resolve();
+    return (job) => {
+        const run = tail.then(job);
+        tail = run.catch(() => { });
+        return run;
+    };
+}
+const TRANSCRIBE_SAMPLE_RATE = 16000;
+const TRANSCRIBE_CHUNK_BYTES = TRANSCRIBE_SAMPLE_RATE * 2;
+const PARAKEET_STREAMING = { emitPartials: false, chunkMs: 5000 };
+const TRANSCRIBE_WITH_METADATA = false;
+const TRANSCRIBE_PACE_MS = 40;
+function pace(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+const TRANSCRIBE_DRAIN_IDLE_MS = 3000;
+const TRANSCRIBE_DRAIN_MAX_MS = 60_000;
+async function writePaced(session, data, signal) {
+    for (let offset = 0; offset < data.length; offset += TRANSCRIBE_CHUNK_BYTES) {
+        if (signal?.aborted)
+            return;
+        session.write(data.subarray(offset, offset + TRANSCRIBE_CHUNK_BYTES));
+        await pace(TRANSCRIBE_PACE_MS);
+    }
+}
+async function drainUntilIdle(lastPieceAt, streamClosed, signal) {
+    const deadline = Date.now() + TRANSCRIBE_DRAIN_MAX_MS;
+    for (;;) {
+        if (signal?.aborted || streamClosed() || Date.now() >= deadline)
+            return;
+        const last = lastPieceAt();
+        if (last > 0 && Date.now() - last >= TRANSCRIBE_DRAIN_IDLE_MS)
+            return;
+        await pace(TRANSCRIBE_PACE_MS);
+    }
+}
+export function transcribeOnce(sdk, modelId, audio) {
+    return sdk.transcribe({ modelId, audioChunk: audio });
+}
+function transcribePiece(piece) {
+    if (typeof piece === 'string') {
+        const text = piece.trim();
+        return text ? { text, append: false } : null;
+    }
+    if (piece.type === 'segment' && piece.segment) {
+        const text = piece.segment.text.trim();
+        return text ? { text, append: piece.segment.append } : null;
+    }
+    if (piece.type === 'text' && piece.text) {
+        return { text: piece.text, append: true };
+    }
+    return null;
+}
+export async function transcribeStreaming(sdk, modelId, pcm, signal) {
+    const session = await sdk.transcribeStream({
+        modelId,
+        metadata: TRANSCRIBE_WITH_METADATA,
+        parakeetStreamingConfig: PARAKEET_STREAMING
+    });
+    const onAbort = () => session.destroy();
+    signal?.addEventListener('abort', onAbort, { once: true });
+    let lastPieceAt = 0;
+    let streamClosed = false;
+    const collect = (async () => {
+        let out = '';
+        for await (const piece of session) {
+            lastPieceAt = Date.now();
+            const seg = transcribePiece(piece);
+            if (!seg)
+                continue;
+            out += !out || seg.append ? seg.text : ` ${seg.text}`;
+        }
+        streamClosed = true;
+        return out.replace(/\s+/g, ' ').trim();
+    })();
+    collect.catch(() => {
+        streamClosed = true;
+    });
+    try {
+        await writePaced(session, pcm, signal);
+        await drainUntilIdle(() => lastPieceAt, () => streamClosed, signal);
+        if (!signal?.aborted)
+            session.end();
+        const text = await collect;
+        if (signal?.aborted)
+            throw new AbortedError();
+        return text;
+    }
+    catch (err) {
+        session.destroy();
+        await collect.catch(() => { });
+        throw err;
+    }
+    finally {
+        signal?.removeEventListener('abort', onAbort);
+    }
+}
+export async function* speakStreamOnce(sdk, modelId, text) {
+    const session = await sdk.textToSpeechStream({
+        modelId,
+        inputType: 'text',
+        accumulateSentences: false
+    });
+    let wrotePronounceable = false;
+    let inputError = null;
+    const pump = (async () => {
+        try {
+            for await (const fragment of text) {
+                wrotePronounceable = writeSpeakable(session, fragment) || wrotePronounceable;
+            }
+            session.end();
+        }
+        catch (err) {
+            hlog('tts stream input failed', { err: err instanceof Error ? err.message : String(err) });
+            inputError = err;
+            destroyQuietly(session);
+        }
+    })();
+    let yieldedAudio = false;
+    try {
+        for await (const frame of session) {
+            if (frame.buffer.length) {
+                yieldedAudio = true;
+                yield { audio: packSamples(frame.buffer), sampleRate: SUPERTONIC_SAMPLE_RATE };
+            }
+        }
+        await pump;
+        if (inputError) {
+            throw inputError;
+        }
+        if (wrotePronounceable && !yieldedAudio) {
+            throw new Error('Text-to-speech returned no audio.');
+        }
+    }
+    catch (err) {
+        throw inputError ?? err;
+    }
+    finally {
+        destroyQuietly(session);
+        await pump;
+    }
+}
+function destroyQuietly(session) {
+    try {
+        session.destroy();
+    }
+    catch { }
+}
+function writeSpeakable(session, fragment) {
+    const spoken = speakable(fragment);
+    if (!/[A-Za-z0-9]/.test(spoken))
+        return false;
+    session.write(b4a.from(/^\s/.test(fragment) ? ` ${spoken}` : spoken, 'utf8'));
+    return true;
+}
+export function speakable(text) {
+    const { text: spoken, dropped } = normalizeForSpeech(text);
+    if (dropped.length) {
+        hlog('tts dropped unspeakable characters', { chars: [...new Set(dropped)].join('') });
+    }
+    return spoken;
+}
+export async function speakOnce(sdk, modelId, text) {
+    const spoken = speakable(text);
+    if (!spoken)
+        return [];
+    const result = sdk.textToSpeech({ modelId, text: spoken, inputType: 'text', stream: true });
+    const samples = [];
+    for await (const sample of result.bufferStream)
+        samples.push(sample);
+    if (!(await result.done))
+        throw new Error('tts synthesis did not complete');
+    return samples;
+}
+export function packSamples(samples) {
+    const bytes = b4a.alloc(samples.length * 4);
+    let offset = 0;
+    for (const sample of samples) {
+        b4a.writeFloatLE(bytes, sample / 32768, offset);
+        offset += 4;
+    }
+    return bytes;
+}

package/dist-lib/lib/connect.d.ts

@@ -0,0 +1,76 @@
+import EventEmitter from 'events';
+import { type Transport } from './rpc-session.ts';
+import { type KvCacheDeletionResult } from './capabilities/kv-cache.ts';
+import type { CapabilityInfo, CompletionContext, DownloadContext, EmbedContext, HarnessContract, HarnessDiagnostics, HarnessEvent, HarnessInfo, ListModelsOptions, ModelInfo, ResolvedSkillCatalogEntry, SpeakContext, SpeakResult, SpeakStreamContext, ToolCatalogEntry, TranscribeContext, WarmModelContext } from './contract-types.ts';
+import { type EmbedDevice } from './models/embed-config.ts';
+import type { DeleteModelSummary } from './models/model-delete.ts';
+import type { ManagedSkill, SkillFileContent } from './tools/skills/overlay-crud.ts';
+export declare const HARNESS_DIED = "HARNESS_DIED";
+export declare function connectHarness(transport: Transport): RemoteHarness;
+export declare function readEmbeddingDeviceResponse(response: unknown): EmbedDevice;
+export declare class RemoteHarness extends EventEmitter implements HarnessContract {
+    on: (event: 'model' | 'status', listener: (...args: never[]) => void) => this;
+    off: (event: 'model' | 'status', listener: (...args: never[]) => void) => this;
+    private readonly _transport;
+    private _session;
+    private _opening;
+    private _closed;
+    private _customModels;
+    constructor(transport: Transport);
+    private _ensure;
+    private _open;
+    completion(context: CompletionContext): AsyncIterable<HarnessEvent>;
+    download(context: DownloadContext): AsyncIterable<HarnessEvent>;
+    embed(context: EmbedContext): Promise<number[][]>;
+    transcribe(context: TranscribeContext): Promise<string>;
+    speak(context: SpeakContext): Promise<SpeakResult>;
+    speakStream(context: SpeakStreamContext): AsyncIterable<SpeakResult>;
+    getEmbeddingDevice(): Promise<EmbedDevice>;
+    setEmbeddingDevice(device: EmbedDevice): Promise<void>;
+    setCredentials(entries: ReadonlyArray<{
+        key: string;
+        token?: string | null;
+        expiresAt?: number | null;
+        agentId?: string | null;
+        skill?: string | null;
+    }>): Promise<void>;
+    setSkillSettings(entries: ReadonlyArray<{
+        skill: string;
+        key: string;
+        value: string;
+        agentId?: string | null;
+    }>): Promise<void>;
+    listManagedSkills(): Promise<ManagedSkill[]>;
+    readSkillFile(name: string): Promise<SkillFileContent | null>;
+    writeSkillFile(name: string, content: string): Promise<{
+        ok: boolean;
+    }>;
+    copyBaseForEdit(name: string): Promise<{
+        copied: boolean;
+    }>;
+    deleteOverlaySkill(name: string): Promise<{
+        removed: boolean;
+    }>;
+    setCustomModels(entries: ReadonlyArray<{
+        name: string;
+        src: string;
+        projectionSrc?: string;
+    }>): Promise<void>;
+    skills(agentId?: string | null, modelName?: string | null, options?: {
+        readonly refresh?: boolean;
+        readonly skipCredentialProbe?: boolean;
+    }): Promise<ResolvedSkillCatalogEntry[]>;
+    tools(): Promise<ToolCatalogEntry[]>;
+    warmModel(context: WarmModelContext): Promise<void>;
+    list(options?: ListModelsOptions): Promise<ModelInfo[]>;
+    deleteModel(name: string): Promise<DeleteModelSummary>;
+    clearKvCache(): Promise<KvCacheDeletionResult>;
+    infraModels(profile: string): Promise<Array<{
+        name: string;
+        addon: string;
+    }>>;
+    capability(): Promise<CapabilityInfo>;
+    close(): void;
+    describe(): Promise<HarnessInfo | null>;
+    fetchDiagnostics(): Promise<HarnessDiagnostics | null>;
+}