@qvac/core 0.0.0 → 0.1.0

package/dist-lib/lib/api/rag.d.ts

@@ -0,0 +1,7 @@
+import '../logger.ts';
+import type { CapabilityHandlers } from '../../spec/capabilities.d.ts';
+import type { QvacAgent } from '../../spec/mesh/hyperschema/types.d.ts';
+import { type IndexingEmbedder, type VectorIndex } from '../core/files/index.ts';
+import { type CurrentMesh } from './current-mesh.ts';
+export declare const MAX_RAG_SEARCH_LIMIT = 20;
+export declare function createRagApi(current: CurrentMesh, index: () => VectorIndex | null, embedder: () => IndexingEmbedder | null, effectiveCtxSize?: ((agent: QvacAgent | null) => Promise<number>) | null): Pick<CapabilityHandlers, 'ragSearch'>;

package/dist-lib/lib/api/rag.js

@@ -0,0 +1,216 @@
+import "../logger.js";
+import { toolsetEnabled } from '@qvac/harness/lib/tools/toolsets.ts';
+import { getAgent } from "../core/agent.js";
+import { getChat } from "../core/chat.js";
+import { isKnowledgeRoutedAttachment, fullTextBudgetChars, fullTextHits, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, referencedDocuments } from "../core/files/index.js";
+import { requireMesh } from "./current-mesh.js";
+export const MAX_RAG_SEARCH_LIMIT = 20;
+const DEFAULT_RAG_SEARCH_LIMIT = 5;
+function clampLimit(limit) {
+    if (typeof limit !== 'number' || limit <= 0)
+        return DEFAULT_RAG_SEARCH_LIMIT;
+    return Math.min(Math.max(1, Math.floor(limit)), MAX_RAG_SEARCH_LIMIT);
+}
+export function createRagApi(current, index, embedder, effectiveCtxSize = null) {
+    return {
+        ragSearch: async ({ chatId, query, mode, limit, attachmentIds }) => {
+            let pendingFiles = [];
+            const respond = (overrides) => response({ pendingFiles, ...overrides });
+            const mesh = await requireMesh(current);
+            const chat = await getChat(mesh, chatId);
+            const toolMode = mode === 'tool';
+            if (!chat)
+                return respond({ enabled: false });
+            const agent = await getAgent(mesh, chat.agentId);
+            let rows = await mesh.files.listKnowledge(chatId);
+            const initialFiles = (await mesh.files.status(chatId)).files;
+            pendingFiles = pendingFileNames(initialFiles);
+            const scopeIds = new Set(toolMode ? [] : (attachmentIds ?? []));
+            let scopeRows = rows.filter((row) => scopeIds.has(row.id));
+            if (scopeIds.size > 0 && scopeRows.length === scopeIds.size) {
+                const ctxSize = effectiveCtxSize ? await effectiveCtxSize(agent) : 0;
+                const fullText = await fullTextHits(mesh, scopeRows, fullTextBudgetChars(ctxSize), {
+                    head: !agentAllowsRagSearch(agent)
+                });
+                if (fullText) {
+                    return respond({
+                        pendingFiles: pendingFileNames(initialFiles, new Set(scopeRows.map((row) => row.id))),
+                        searchedCount: fullText.length,
+                        indexEmpty: false,
+                        hasKnowledge: true,
+                        availableFiles: [...new Set(scopeRows.map((row) => row.fileName))],
+                        hits: fullText
+                    });
+                }
+            }
+            if (!toolMode && !agentAllowsRagSearch(agent))
+                return respond({ enabled: false });
+            if (rows.length === 0)
+                return respond({});
+            const namedRows = referencedDocuments(query, rows.map((row) => ({ id: row.id, title: row.fileName })));
+            const requested = scopeIds.size ? scopeRows : namedRows.length ? namedRows : rows;
+            const waiting = requested.filter((row) => !mesh.files.hydratedArtifact(row.id));
+            if (waiting.length) {
+                try {
+                    await mesh.files.settle(waiting.map((row) => row.id));
+                }
+                catch (err) {
+                    logger.warn('rag', 'indexing settle failed', { err });
+                }
+                rows = await mesh.files.listKnowledge(chatId);
+                scopeRows = rows.filter((row) => scopeIds.has(row.id));
+            }
+            const hasKnowledge = rows.length > 0;
+            const vectorIndex = index();
+            const embed = embedder();
+            if (!vectorIndex || !embed) {
+                return respond({ hasKnowledge, errorMessage: 'knowledge index unavailable on this device' });
+            }
+            const artifactByRow = new Map(rows.map((row) => [row, mesh.files.hydratedArtifact(row.id)]));
+            const hydratedDocuments = [];
+            for (const [row, artifactId] of artifactByRow) {
+                if (artifactId)
+                    hydratedDocuments.push({ documentId: artifactId, title: row.fileName });
+            }
+            const indexEmpty = hydratedDocuments.length === 0;
+            const availableFiles = [...new Set(hydratedDocuments.map((document) => document.title))];
+            const failedRows = new Set();
+            const fileStates = (await mesh.files.status(chatId)).files;
+            for (const row of rows) {
+                if (artifactByRow.get(row))
+                    continue;
+                if (fileStates.find((file) => file.fileId === row.id)?.state === 'failed') {
+                    failedRows.add(row);
+                }
+            }
+            pendingFiles = pendingFileNames(fileStates, new Set(rows.filter((row) => artifactByRow.get(row)).map((row) => row.id)));
+            const failedFiles = scopeRows.filter((row) => failedRows.has(row)).map((row) => row.fileName);
+            const scopeDocumentIds = [];
+            let scopePending = false;
+            for (const row of scopeRows) {
+                const artifactId = artifactByRow.get(row);
+                if (artifactId)
+                    scopeDocumentIds.push(artifactId);
+                else if (!failedRows.has(row))
+                    scopePending = true;
+            }
+            if (scopePending && scopeDocumentIds.length === 0) {
+                return respond({
+                    pendingReferenced: true,
+                    indexEmpty,
+                    hasKnowledge,
+                    availableFiles,
+                    failedFiles
+                });
+            }
+            const pendingReferenced = scopeDocumentIds.length === 0 &&
+                referencesPendingFile(query, rows.map((row) => ({
+                    title: row.fileName,
+                    pending: !artifactByRow.get(row) && !failedRows.has(row)
+                })));
+            if (pendingReferenced) {
+                return respond({ pendingReferenced, indexEmpty, hasKnowledge, availableFiles, failedFiles });
+            }
+            const failedReferenced = scopeRows.length === 0 &&
+                failedRows.size > 0 &&
+                referencesPendingFile(query, rows.map((row) => ({ title: row.fileName, pending: failedRows.has(row) })));
+            if (failedReferenced) {
+                return respond({
+                    indexEmpty,
+                    hasKnowledge,
+                    availableFiles,
+                    failedFiles: rows.filter((row) => failedRows.has(row)).map((row) => row.fileName)
+                });
+            }
+            const allFailed = indexEmpty && rows.length > 0 && rows.every((row) => failedRows.has(row));
+            if (allFailed) {
+                return respond({
+                    indexEmpty,
+                    hasKnowledge,
+                    availableFiles,
+                    failedFiles: rows.map((row) => row.fileName)
+                });
+            }
+            if (toolMode) {
+                try {
+                    const hits = await searchRagChunks({
+                        query,
+                        chatId,
+                        limit: clampLimit(limit),
+                        liveSourceIds: rows.map((row) => row.id)
+                    }, { index: vectorIndex, embed: (texts) => embed.embed({ texts }) });
+                    return respond({
+                        searchedCount: hits.length,
+                        indexEmpty,
+                        hasKnowledge,
+                        availableFiles,
+                        hits: hits.map(toWireHit)
+                    });
+                }
+                catch (err) {
+                    return respond({
+                        indexEmpty,
+                        hasKnowledge,
+                        availableFiles,
+                        errorMessage: err instanceof Error ? err.message : String(err)
+                    });
+                }
+            }
+            try {
+                const result = await retrieveRagKnowledge({
+                    query,
+                    chatId,
+                    documents: hydratedDocuments,
+                    liveSourceIds: rows.map((row) => row.id),
+                    scopeDocumentIds: scopeDocumentIds.length > 0 ? scopeDocumentIds : undefined
+                }, { index: vectorIndex, embed: (texts) => embed.embed({ texts }) });
+                return respond({
+                    searchedCount: result.searchedCount,
+                    indexEmpty,
+                    hasKnowledge,
+                    targetDocumentId: result.targetDocumentId,
+                    availableFiles,
+                    failedFiles,
+                    hits: result.hits.map(toWireHit)
+                });
+            }
+            catch (err) {
+                return respond({
+                    indexEmpty,
+                    hasKnowledge,
+                    availableFiles,
+                    failedFiles,
+                    errorMessage: err instanceof Error ? err.message : String(err)
+                });
+            }
+        }
+    };
+}
+function toWireHit(hit) {
+    return { id: hit.id, content: hit.content, metadataJson: hit.metadataJson, score: hit.score };
+}
+function pendingFileNames(files, available = new Set()) {
+    return files
+        .filter((file) => !available.has(file.fileId) &&
+        ['pending', 'running', 'uploading'].includes(file.state) &&
+        (file.state === 'uploading'
+            ? isKnowledgeRoutedAttachment(file)
+            : file.processors.some((processor) => processor.processorId === 'rag' && processor.state !== 'not-applicable')))
+        .map((file) => file.fileName);
+}
+function response(overrides) {
+    return {
+        enabled: true,
+        searchedCount: 0,
+        indexEmpty: true,
+        hasKnowledge: false,
+        availableFiles: [],
+        hits: [],
+        ...overrides
+    };
+}
+function agentAllowsRagSearch(agent) {
+    if (!agent)
+        return false;
+    return toolsetEnabled('knowledge', agent.disabledToolsets);
+}

package/dist-lib/lib/api/sync.d.ts

@@ -0,0 +1,3 @@
+import type { CapabilityHandlers } from '../../spec/capabilities.d.ts';
+import type { CurrentMesh } from './current-mesh.ts';
+export declare function createSyncApi(current: CurrentMesh): Pick<CapabilityHandlers, 'syncStatusWatch'>;

package/dist-lib/lib/api/sync.js

@@ -0,0 +1,6 @@
+import { watchable } from "./watchable.js";
+export function createSyncApi(current) {
+    return {
+        syncStatusWatch: watchable('syncStatusWatch', async () => (await current())?.syncActivity ?? null, (_request, activity) => activity.snapshot())
+    };
+}

package/dist-lib/lib/api/tool-activity.d.ts

@@ -0,0 +1,5 @@
+import type { CapabilityHandlers } from '../../spec/capabilities.d.ts';
+import { type CurrentMesh } from './current-mesh.ts';
+type ToolActivityCapabilities = Pick<CapabilityHandlers, 'listToolActivity' | 'toolActivityWatch' | 'recordToolActivity'>;
+export declare function createToolActivityApi(current: CurrentMesh, deviceId: Buffer | null): ToolActivityCapabilities;
+export {};

package/dist-lib/lib/api/tool-activity.js

@@ -0,0 +1,53 @@
+import { listToolActivity, recordToolActivity } from "../core/tool-activity.js";
+import { requireMesh } from "./current-mesh.js";
+import { watchable } from "./watchable.js";
+const DEFAULT_LIMIT = 200;
+const MAX_LIMIT = 1000;
+const DECISIONS = new Set(['auto', 'allowed', 'denied', 'skip-all', 'blocked']);
+function normalizeDecision(decision) {
+    return DECISIONS.has(decision) ? decision : 'blocked';
+}
+function toEntry(row) {
+    return {
+        id: row.id,
+        agentId: row.agentId,
+        deviceId: row.originDeviceId.toString('hex'),
+        chatId: row.chatId ?? undefined,
+        runId: row.runId ?? undefined,
+        toolName: row.toolName,
+        arguments: row.arguments ?? undefined,
+        resource: row.resource ?? undefined,
+        effect: row.effect ?? undefined,
+        decision: row.decision,
+        at: row.at,
+        seq: row.seq ?? 0
+    };
+}
+export function createToolActivityApi(current, deviceId) {
+    const query = async ({ agentId, limit, beforeSeq }, mesh) => ({
+        activities: (await listToolActivity(mesh, agentId, {
+            limit: Math.min(limit || DEFAULT_LIMIT, MAX_LIMIT),
+            beforeSeq: beforeSeq || undefined
+        })).map(toEntry)
+    });
+    return {
+        listToolActivity: async (request) => query(request, await requireMesh(current)),
+        toolActivityWatch: watchable('toolActivityWatch', () => requireMesh(current), query),
+        recordToolActivity: async (request) => {
+            if (!deviceId)
+                throw new Error('cannot record tool activity without a device id');
+            return recordToolActivity(await requireMesh(current), {
+                agentId: request.agentId,
+                originDeviceId: deviceId,
+                toolName: request.toolName,
+                decision: normalizeDecision(request.decision),
+                effect: request.effect ?? undefined,
+                arguments: request.arguments ?? undefined,
+                resource: request.resource ?? undefined,
+                chatId: request.chatId ?? undefined,
+                runId: request.runId ?? undefined,
+                at: request.at
+            });
+        }
+    };
+}

package/dist-lib/lib/api/tool-configs.d.ts

@@ -0,0 +1,6 @@
+import type { CapabilityHandlers } from '../../spec/capabilities.d.ts';
+import { getToolConfigs } from '../core/tool-configs.ts';
+import { type CurrentMesh } from './current-mesh.ts';
+type ToolConfigsCapabilities = Pick<CapabilityHandlers, 'setToolConfigs' | 'deleteToolConfigs' | 'listToolConfigs' | 'toolConfigsWatch'>;
+export declare function createToolConfigsApi(current: CurrentMesh): ToolConfigsCapabilities;
+export { getToolConfigs };

package/dist-lib/lib/api/tool-configs.js

@@ -0,0 +1,27 @@
+import { deleteToolConfigs, getToolConfigs, listToolConfigs, setToolConfigs } from "../core/tool-configs.js";
+import { requireMesh } from "./current-mesh.js";
+import { watchable } from "./watchable.js";
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function ensureSettingsPayload(value) {
+    if (!isRecord(value))
+        throw new Error('tool configs must be a plain object');
+    return value;
+}
+export function createToolConfigsApi(current) {
+    const query = async ({ agentId }, mesh) => ({
+        configs: await listToolConfigs(mesh, agentId)
+    });
+    return {
+        setToolConfigs: async ({ agentId, toolName, settings }) => setToolConfigs(await requireMesh(current), {
+            agentId,
+            toolName,
+            settings: ensureSettingsPayload(settings)
+        }),
+        deleteToolConfigs: async ({ agentId, toolName }) => deleteToolConfigs(await requireMesh(current), agentId, toolName),
+        listToolConfigs: async (request) => query(request, await requireMesh(current)),
+        toolConfigsWatch: watchable('toolConfigsWatch', () => requireMesh(current), query)
+    };
+}
+export { getToolConfigs };

package/dist-lib/lib/api/watchable.d.ts

@@ -0,0 +1,12 @@
+import { Readable } from 'streamx';
+import '../logger.ts';
+interface WatchableStore {
+    watch(listener: () => void): void;
+    unwatch(listener: () => void): void;
+    once?(event: 'close', listener: () => void): unknown;
+    off?(event: 'close', listener: () => void): unknown;
+    closing?: unknown;
+    closed?: boolean;
+}
+export declare function watchable<TRequest, TResult, TStore extends WatchableStore = WatchableStore>(name: string, storeOf: () => TStore | null | Promise<TStore | null>, query: (request: TRequest, store: TStore) => Promise<TResult>): (request: TRequest) => Readable & AsyncIterable<TResult>;
+export {};

package/dist-lib/lib/api/watchable.js

@@ -0,0 +1,121 @@
+import b4a from 'b4a';
+import { Readable } from 'streamx';
+import "../logger.js";
+const SCOPE = ['api', 'watch'];
+const CHURN_WINDOW_MS = 1000;
+function snapshotEquals(a, b) {
+    if (a === b)
+        return true;
+    if (b4a.isBuffer(a) || b4a.isBuffer(b)) {
+        return b4a.isBuffer(a) && b4a.isBuffer(b) && b4a.equals(a, b);
+    }
+    if (Array.isArray(a) || Array.isArray(b)) {
+        if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length)
+            return false;
+        return a.every((item, i) => snapshotEquals(item, b[i]));
+    }
+    if (!isRecord(a) || !isRecord(b))
+        return false;
+    const keys = Object.keys(a);
+    if (keys.length !== Object.keys(b).length)
+        return false;
+    return keys.every((key) => snapshotEquals(a[key], b[key]));
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+export function watchable(name, storeOf, query) {
+    return (request) => {
+        const resolved = storeOf();
+        const pending = resolved instanceof Promise ? resolved.catch(noop) : null;
+        let store = resolved instanceof Promise ? null : resolved;
+        let dirty = true;
+        let ended = false;
+        let last = null;
+        let wake = null;
+        let churn = 0;
+        let reportedAt = Date.now();
+        const reportChurn = (force) => {
+            if (churn === 0)
+                return;
+            const now = Date.now();
+            const elapsed = now - reportedAt;
+            if (!force && elapsed < CHURN_WINDOW_MS)
+                return;
+            logger.debug(SCOPE, `${name} woke with no change`, { woke: churn, ms: elapsed });
+            churn = 0;
+            reportedAt = now;
+        };
+        const onChange = () => {
+            dirty = true;
+            wake?.resolve();
+            wake = null;
+        };
+        const onStoreClose = () => {
+            ended = true;
+            wake?.resolve();
+            wake = null;
+        };
+        const storeGone = () => store === null || ended || Boolean(store.closed) || Boolean(store.closing);
+        const stream = new Readable({
+            async open(cb) {
+                if (pending)
+                    store = (await pending) ?? null;
+                store?.watch(onChange);
+                store?.once?.('close', onStoreClose);
+                cb(null);
+            },
+            async read(cb) {
+                try {
+                    while (!stream.destroying) {
+                        if (store === null || storeGone()) {
+                            stream.push(null);
+                            break;
+                        }
+                        if (!dirty) {
+                            wake = Promise.withResolvers();
+                            await wake.promise;
+                            continue;
+                        }
+                        dirty = false;
+                        const next = await query(request, store);
+                        if (storeGone()) {
+                            stream.push(null);
+                            break;
+                        }
+                        if (snapshotEquals(next, last)) {
+                            churn++;
+                            reportChurn(false);
+                            continue;
+                        }
+                        reportChurn(true);
+                        last = next;
+                        stream.push(next);
+                        break;
+                    }
+                    cb(null);
+                }
+                catch (err) {
+                    if (storeGone()) {
+                        stream.push(null);
+                        cb(null);
+                    }
+                    else {
+                        cb(err);
+                    }
+                }
+            },
+            predestroy() {
+                onChange();
+            },
+            destroy(cb) {
+                reportChurn(true);
+                store?.unwatch(onChange);
+                store?.off?.('close', onStoreClose);
+                cb(null);
+            }
+        });
+        return stream;
+    };
+}
+function noop() { }

package/dist-lib/lib/assistant-client.d.ts

@@ -0,0 +1,21 @@
+import type { Core } from './core.ts';
+import type { DeviceMetadata } from './core/mesh.ts';
+import { Client, type ClientOptions } from './client.ts';
+export interface AssistantClientOptions extends ClientOptions {
+    sidecar?: boolean;
+    storagePath: string;
+    device?: DeviceMetadata;
+    meshSeed?: Buffer;
+    meshKey?: Buffer;
+    bootstrapNodes?: ReadonlyArray<{
+        host: string;
+        port: number;
+    }>;
+    entry?: string;
+}
+export declare class AssistantClient extends Client {
+    private readonly _inline;
+    constructor({ sidecar: spawnSidecar, storagePath, device, meshSeed, meshKey, bootstrapNodes, entry, ...engineOptions }: AssistantClientOptions);
+    get core(): Core | null;
+    _close(): Promise<void>;
+}

package/dist-lib/lib/assistant-client.js

@@ -0,0 +1,52 @@
+import { Client } from "./client.js";
+import { duplexPair } from '@qvac/harness/transport';
+export class AssistantClient extends Client {
+    _inline;
+    constructor({ sidecar: spawnSidecar = true, storagePath, device, meshSeed, meshKey, bootstrapNodes, entry, ...engineOptions }) {
+        const inline = { core: null };
+        if (spawnSidecar && !entry)
+            throw new Error('sidecar mode needs a stowed bundle entry');
+        const args = [
+            storagePath,
+            device?.name ?? '',
+            device?.capabilityProfile ?? '',
+            device?.recommendedModelName ?? '',
+            meshSeed?.toString('hex') ?? '',
+            meshKey?.toString('hex') ?? '',
+            (bootstrapNodes ?? []).map((node) => `${node.host}:${node.port}`).join(',')
+        ];
+        const transport = spawnSidecar
+            ? async () => {
+                const { default: sidecarRunner } = await import('bare-supervisor/runner/sidecar');
+                const { ipc, exit } = sidecarRunner(entry, args, engineOptions.logger ?? null);
+                const died = exit.then((code) => `sidecar exited (${code}) before ready`);
+                const failure = await Promise.race([ipc.ready.then(() => null), died]);
+                if (failure)
+                    throw new Error(failure);
+                return ipc;
+            }
+            : async () => {
+                const [serverStream, clientStream] = duplexPair();
+                const { Core } = await import("./core.js");
+                inline.core = new Core(() => serverStream, {
+                    logger: engineOptions.logger ?? null,
+                    storagePath,
+                    device,
+                    meshSeed,
+                    meshKey,
+                    bootstrapNodes
+                });
+                await inline.core.ready();
+                return clientStream;
+            };
+        super(transport, engineOptions);
+        this._inline = inline;
+    }
+    get core() {
+        return this._inline.core;
+    }
+    async _close() {
+        await super._close();
+        await this._inline.core?.close();
+    }
+}

package/dist-lib/lib/assistant-tree.d.ts

@@ -0,0 +1,50 @@
+import Supervisor, { type ChildSpec } from 'bare-supervisor';
+import { type Harness } from './assistant.ts';
+import { Client } from './client.ts';
+import type { Core } from './core.ts';
+import type { IndexingEmbedder, RagExtractors } from './core/files/index.ts';
+import type { Logger } from './log.ts';
+export interface HarnessContext {
+    engine: Client;
+    onDeath: (error?: Error) => void;
+}
+export type HarnessFactory = (ctx: HarnessContext) => Harness | Promise<Harness>;
+export type EmbedderFactory = (harness: Harness) => IndexingEmbedder | null | undefined;
+export type OnHarnessDisposer = (() => void | Promise<void>) & {
+    sync?: () => Promise<void>;
+};
+export type OnHarness = (harness: Harness, ctx: {
+    engine: Client;
+}) => void | OnHarnessDisposer;
+export interface AssistantTreeOptions {
+    core: Core;
+    harness: HarnessFactory | ChildSpec<Harness>;
+    logger?: Logger | null;
+    imageCacheDir?: string;
+    imageCacheMaxBytes?: number;
+    imageCacheMaxAgeMs?: number;
+    embedder?: EmbedderFactory;
+    extractors?: RagExtractors;
+    onHarness?: OnHarness;
+    platform?: string;
+    preloadCurrentAgentModel?: boolean;
+}
+export declare function assistantTree(opts: AssistantTreeOptions): Supervisor;
+export interface AssistantChildOptions {
+    harness: HarnessFactory | ChildSpec<Harness>;
+    core?: string;
+    logger?: Logger | null;
+    imageCacheDir?: string;
+    imageCacheMaxBytes?: number;
+    imageCacheMaxAgeMs?: number;
+    embedder?: EmbedderFactory;
+    extractors?: RagExtractors;
+    onHarness?: OnHarness;
+    preloadCurrentAgentModel?: boolean;
+    restart?: 'always' | 'never';
+    maxRestarts?: number;
+    window?: number;
+    backoff?: number;
+    maxBackoff?: number;
+}
+export declare function assistantChild(opts: AssistantChildOptions): ChildSpec<Supervisor>;