@qvac/core 0.0.0 → 0.1.1

package/dist-lib/lib/core/departures.js

@@ -0,0 +1,145 @@
+import ScopeLock from 'scope-lock';
+import { Mesh } from "./mesh.js";
+const SCOPE = ['p2p', 'departure'];
+export class PendingDepartures {
+    _store;
+    _local;
+    _swarm;
+    _log;
+    _blindPeerKeys;
+    _blindPeeringOpts;
+    _wakeup;
+    _active = new Map();
+    _parked = new Set();
+    _lock = new ScopeLock();
+    _closing = false;
+    constructor(store, local, opts) {
+        this._store = store;
+        this._local = local;
+        this._swarm = opts.swarm;
+        this._log = opts.logger ?? null;
+        this._blindPeerKeys = opts.blindPeerKeys ?? [];
+        this._blindPeeringOpts = opts.blindPeeringOpts;
+        this._wakeup = opts.wakeup;
+    }
+    flush(activeKey) {
+        void this._flush(activeKey ?? null).catch((err) => this._log?.warn(SCOPE, 'departure flush failed', err));
+    }
+    async _flush(activeKey) {
+        await this._lock.lock();
+        try {
+            if (this._closing)
+                return;
+            for (const row of await this._local.departures()) {
+                if (this._closing)
+                    return;
+                const hex = row.key.toString('hex');
+                if (activeKey?.equals(row.key) || this._parked.has(hex) || this._active.has(hex)) {
+                    continue;
+                }
+                const mesh = new Mesh(this._store, {
+                    seed: row.seed,
+                    key: row.creator ? null : row.key,
+                    writerSeed: row.writerSeed ?? undefined,
+                    logger: this._log,
+                    blindPeerKeys: this._blindPeerKeys,
+                    blindPeeringOpts: this._blindPeeringOpts,
+                    wakeup: this._wakeup
+                });
+                const done = this._deliver(row, mesh)
+                    .catch((err) => this._log?.warn(SCOPE, 'departure delivery failed', err))
+                    .finally(() => this._active.delete(hex));
+                this._active.set(hex, { mesh, done });
+            }
+        }
+        finally {
+            this._lock.unlock();
+        }
+    }
+    async _deliver(row, mesh) {
+        const scope = [...SCOPE, row.key.toString('hex').slice(0, 8)];
+        try {
+            await mesh.ready();
+            if (this._closing || mesh.closing)
+                return;
+            this._swarm.join(mesh.discoveryKey, { server: true, client: true });
+            mesh.startBlindPeering(this._swarm.dht);
+            this._log?.info(scope, 'reopened a departed mesh to deliver its removal');
+            if (!(await waitForDelivery(mesh)))
+                return;
+            await this._local.deleteDeparture(row.key);
+            this._log?.info(scope, 'departure delivered');
+        }
+        finally {
+            await this._swarm.leave(mesh.discoveryKey).catch(noop);
+            await mesh.close().catch(noop);
+        }
+    }
+    async park(key) {
+        const hex = key.toString('hex');
+        await this._lock.lock();
+        try {
+            this._parked.add(hex);
+            const entry = this._active.get(hex);
+            if (!entry)
+                return;
+            await entry.mesh.close().catch(noop);
+            await entry.done;
+        }
+        finally {
+            this._lock.unlock();
+        }
+    }
+    unpark(key) {
+        this._parked.delete(key.toString('hex'));
+    }
+    drop(key) {
+        return this._local.deleteDeparture(key);
+    }
+    async suspend() {
+        for (const { mesh } of this._active.values())
+            await mesh.suspend();
+    }
+    async resume() {
+        for (const { mesh } of this._active.values())
+            await mesh.resume();
+    }
+    async close() {
+        this._closing = true;
+        await Promise.all([...this._active.values()].map(async ({ mesh, done }) => {
+            await mesh.close().catch(noop);
+            await done;
+        }));
+    }
+}
+function waitForDelivery(mesh) {
+    const local = mesh.autobee.local;
+    let uploaded = 0;
+    const delivered = () => local.length > 0 &&
+        (uploaded >= local.length || local.peers.some((peer) => peer.remoteLength >= local.length));
+    if (delivered())
+        return Promise.resolve(true);
+    return new Promise((resolve) => {
+        const settle = (ok) => {
+            local.off('upload', onUpload);
+            local.off('peer-add', check);
+            mesh.syncActivity.unwatch(check);
+            mesh.off('close', onClose);
+            resolve(ok);
+        };
+        const check = () => {
+            if (delivered())
+                settle(true);
+        };
+        const onUpload = (index) => {
+            uploaded = Math.max(uploaded, index + 1);
+            check();
+        };
+        const onClose = () => settle(false);
+        local.on('upload', onUpload);
+        local.on('peer-add', check);
+        mesh.syncActivity.watch(check);
+        mesh.on('close', onClose);
+    });
+}
+function noop() { }

package/dist-lib/lib/core/device-attestation.d.ts

@@ -0,0 +1,6 @@
+export interface DeviceAttestation {
+    readonly deviceId: Buffer;
+    readonly signature: Buffer;
+}
+export declare function signDeviceAttestation(secretKey: Buffer, deviceId: Buffer): DeviceAttestation;
+export declare function verifyDeviceAttestation(publicKey: Buffer, attestation: DeviceAttestation): boolean;

package/dist-lib/lib/core/device-attestation.js

@@ -0,0 +1,12 @@
+import crypto from 'hypercore-crypto';
+export function signDeviceAttestation(secretKey, deviceId) {
+    return { deviceId, signature: crypto.sign(deviceId, secretKey) };
+}
+export function verifyDeviceAttestation(publicKey, attestation) {
+    try {
+        return crypto.verify(attestation.deviceId, attestation.signature, publicKey);
+    }
+    catch {
+        return false;
+    }
+}

package/dist-lib/lib/core/devices.d.ts

@@ -0,0 +1,14 @@
+import type { QvacToolReportEntry } from '../../spec/mesh/hyperschema/types.d.ts';
+import type { AppBuildMetadata } from './app-build.ts';
+import type { Mesh } from './mesh.ts';
+export interface SaveDeviceCapabilitiesInput {
+    deviceId: Buffer;
+    installedToolNames: string[];
+    installedSkillNames: string[];
+    installedModelNames: string[];
+    cachedModelNames: string[];
+    readySkillNames?: string[] | null;
+    toolCatalog?: QvacToolReportEntry[] | null;
+}
+export declare function saveDeviceCapabilities(store: Mesh, input: SaveDeviceCapabilitiesInput): Promise<void>;
+export declare function advertiseDeviceVersion(store: Mesh, deviceId: Buffer, app: AppBuildMetadata): Promise<void>;

package/dist-lib/lib/core/devices.js

@@ -0,0 +1,11 @@
+export async function saveDeviceCapabilities(store, input) {
+    await store.dispatch('report-device-capabilities', input);
+}
+export async function advertiseDeviceVersion(store, deviceId, app) {
+    await store.dispatch('report-device-version', {
+        deviceId,
+        appVersion: app.version,
+        appBuild: app.build,
+        appChannel: app.channel
+    });
+}

package/dist-lib/lib/core/document-attachment.d.ts

@@ -0,0 +1,22 @@
+export interface DocumentAttachment {
+    id: string;
+    fileName: string;
+    mimeType: string;
+}
+export declare function isDocumentAttachment(ref: {
+    fileName: string;
+    mimeType: string;
+}): boolean;
+export declare function documentAttachmentLine(doc: DocumentAttachment): string;
+export declare const DOCUMENT_ATTACHMENT_ONLY_PROMPT = "The user sent one or more documents without any accompanying text, and no excerpts of them are available. Say what each file is and ask what they would like done with it.";
+export type DocumentUnreadReason = 'indexing' | 'knowledge-off' | 'unreadable';
+export declare function documentUnreadReason({ knowledgeOn, failedFiles, attachedCount }: {
+    knowledgeOn: boolean;
+    failedFiles: ReadonlyArray<string> | null | undefined;
+    attachedCount: number;
+}): DocumentUnreadReason;
+export declare function documentUnreadNotice(reason: DocumentUnreadReason, plural: boolean): string;
+export declare function annotateDocumentAttachments(messages: {
+    content: string;
+    documents?: DocumentAttachment[];
+}[]): void;

package/dist-lib/lib/core/document-attachment.js

@@ -0,0 +1,35 @@
+import { isAudioAttachment } from "./audio-attachment.js";
+import { IMAGE_MIMES } from "./image/normalize.js";
+export function isDocumentAttachment(ref) {
+    if (IMAGE_MIMES.has(ref.mimeType.toLowerCase()))
+        return false;
+    return !isAudioAttachment(ref.fileName, ref.mimeType);
+}
+export function documentAttachmentLine(doc) {
+    return `[Attached file "${doc.fileName}" (${doc.mimeType}) — attachmentId: ${doc.id}]`;
+}
+export const DOCUMENT_ATTACHMENT_ONLY_PROMPT = 'The user sent one or more documents without any accompanying text, and no excerpts of them are available. Say what each file is and ask what they would like done with it.';
+export function documentUnreadReason({ knowledgeOn, failedFiles, attachedCount }) {
+    if (!knowledgeOn)
+        return 'knowledge-off';
+    if ((failedFiles?.length ?? 0) >= attachedCount)
+        return 'unreadable';
+    return 'indexing';
+}
+export function documentUnreadNotice(reason, plural) {
+    const they = plural ? 'they are' : 'it is';
+    const why = reason === 'indexing'
+        ? `${they} still being indexed`
+        : reason === 'knowledge-off'
+            ? `${they} not something that could be included whole, and Knowledge is off for this agent`
+            : `${they} not readable on this device`;
+    return `[The attached document${plural ? 's' : ''} could not be read for this reply: ${why}.]`;
+}
+export function annotateDocumentAttachments(messages) {
+    for (const turn of messages.filter((message) => message.documents?.length)) {
+        const block = (turn.documents ?? []).map(documentAttachmentLine).join('\n');
+        const head = turn.content.trim() || DOCUMENT_ATTACHMENT_ONLY_PROMPT;
+        turn.content = `${head}\n${block}`;
+        delete turn.documents;
+    }
+}

package/dist-lib/lib/core/files/attachments.d.ts

@@ -0,0 +1,8 @@
+import type { Mesh } from '../mesh.ts';
+import type { CapabilityHandlers } from '../../../spec/capabilities.d.ts';
+export declare const MAX_ATTACHMENT_BYTES: number;
+export declare function attachmentFiles(mesh: Mesh, { maxBytes, deviceId, inactivityMs }?: {
+    maxBytes?: number;
+    deviceId?: Buffer | null;
+    inactivityMs?: number;
+}): Pick<CapabilityHandlers, 'uploadAttachment' | 'downloadAttachment' | 'deleteAttachment'>;

package/dist-lib/lib/core/files/attachments.js

@@ -0,0 +1,50 @@
+import { linkAttachment } from "../attachment-link.js";
+import { getChunk, updateChunk } from "../chunk.js";
+import { uploadFile } from "./upload.js";
+export const MAX_ATTACHMENT_BYTES = 100 * 1024 * 1024;
+export function attachmentFiles(mesh, { maxBytes = MAX_ATTACHMENT_BYTES, deviceId = null, inactivityMs } = {}) {
+    return {
+        async *uploadAttachment(frames) {
+            for await (const frame of uploadFile(mesh, frames, {
+                kind: 'attachment',
+                maxBytes,
+                inactivityMs
+            })) {
+                yield {
+                    bytesWritten: frame.bytesWritten,
+                    attachment: frame.attachment
+                        ? linkAttachment(mesh.attachmentUrl, frame.attachment)
+                        : undefined
+                };
+            }
+        },
+        downloadAttachment: async function* ({ blobId }) {
+            yield* mesh.files.read(blobId);
+        },
+        async deleteAttachment({ chunkId, at }) {
+            if (!at || at < 0) {
+                throw new Error(`delete-attachment needs a positive timestamp, got ${at}`);
+            }
+            const chunk = await getChunk(mesh, chunkId);
+            if (!chunk)
+                throw new Error(`chunk not found: ${chunkId}`);
+            if (chunk.type !== 'attachment' || !chunk.attachment) {
+                throw new Error(`not an attachment chunk: ${chunkId}`);
+            }
+            if (deviceId && !chunk.originDeviceId.equals(deviceId)) {
+                throw new Error('only the origin device may delete an attachment');
+            }
+            const attachment = chunk.attachment;
+            if (attachment.deletedAt)
+                return chunk;
+            const updated = await updateChunk(mesh, {
+                id: chunkId,
+                attachment: { ...attachment, deletedAt: at }
+            });
+            if (!updated.attachment?.deletedAt) {
+                throw new Error('delete-attachment dropped — only the origin device may delete an attachment');
+            }
+            return updated;
+        }
+    };
+}

package/dist-lib/lib/core/files/index.d.ts

@@ -0,0 +1,133 @@
+import type { AbortSignal } from 'bare-abort-controller';
+import { Buffer } from 'buffer';
+import EventEmitter from 'bare-events';
+import type { AssistantFileStatusList, AssistantFileReadRequest, AssistantFileReadFrame, AssistantFilePreviewResponse, AssistantIndexingStatusList } from '../../../spec/hyperschema/types.d.ts';
+import type { QvacChunk, QvacKnowledge, QvacKnowledgeArtifact } from '../../../spec/mesh/hyperschema/types.d.ts';
+import { attachmentFiles } from './attachments.ts';
+import { addKnowledgeArtifact, type FileRecord } from './records/index.ts';
+import { knowledgeFiles } from './knowledge.ts';
+import { FileState, type LocalWork, type IndexingProgressEntry, type FileProcessor } from './processing/index.ts';
+import { BlobStore } from './store/index.ts';
+import { type IndexingProgress, type IndexingEmbedder, type RagExtractors, type VectorIndex } from './rag/index.ts';
+import type { Mesh } from '../mesh.ts';
+export { fullTextBudgetChars, fullTextHits, retrieveRagKnowledge, searchRagChunks, referencesPendingFile, type VectorHit, type RagDocumentRef } from './rag/index.ts';
+export { isKnowledgeRoutedAttachment } from './records/index.ts';
+export { assertRemovableAt } from './records/index.ts';
+export { assertArtifactOutput } from './records/index.ts';
+export { claimSource } from './records/index.ts';
+export declare class Files extends EventEmitter {
+    private readonly _mesh;
+    private readonly _uploads;
+    readonly state: FileState;
+    private readonly _states;
+    private readonly _processors;
+    private readonly _store;
+    private readonly _deadline;
+    private readonly _updated;
+    private _model;
+    private _paused;
+    closed: boolean;
+    constructor(mesh: Mesh);
+    read(reference: string, options?: Parameters<BlobStore['read']>[1]): AsyncGenerator<{
+        data: Buffer;
+        bytesRead: number;
+        byteLength: number;
+    }, any, any>;
+    readBytes(reference: string, options?: Parameters<BlobStore['readBytes']>[1]): Promise<Buffer<ArrayBufferLike>>;
+    download(reference: string, options: Parameters<BlobStore['download']>[1]): Promise<void>;
+    write(chunks: AsyncIterable<Uint8Array>, options?: Parameters<BlobStore['write']>[1]): AsyncGenerator<import("./store/blob.ts").WriteProgress, any, any>;
+    put(bytes: Buffer): Promise<{
+        blobRef: import("hyperblobs").BlobId;
+        reference: string;
+    }>;
+    inline(bytes: Buffer, mimeType: string): string;
+    reference(id: Parameters<BlobStore['reference']>[0]): string;
+    clear(reference: string, id: string): Promise<void>;
+    attachments(options?: Parameters<typeof attachmentFiles>[1]): Pick<import("../../../spec/capabilities").CapabilityHandlers, "uploadAttachment" | "downloadAttachment" | "deleteAttachment">;
+    knowledge(options?: Parameters<typeof knowledgeFiles>[1]): Pick<import("../../../spec/capabilities").CapabilityHandlers, "uploadKnowledge" | "listKnowledge" | "removeKnowledge" | "downloadKnowledge" | "listIndexing">;
+    listKnowledge(chatId: string): Promise<QvacKnowledge[]>;
+    record(id: string): Promise<QvacKnowledge | null>;
+    resolve(request: {
+        chatId: string;
+        fileId: string;
+    }): Promise<FileRecord | null>;
+    fileRead(request: AssistantFileReadRequest, options?: {
+        signal?: AbortSignal;
+    }): AsyncGenerator<AssistantFileReadFrame>;
+    preview(request: {
+        chatId: string;
+        fileId: string;
+    }): Promise<AssistantFilePreviewResponse>;
+    retry(request: {
+        chatId: string;
+        fileId: string;
+        processorId: string;
+    }): Promise<void>;
+    artifact(id: string): Promise<QvacKnowledgeArtifact | null>;
+    artifacts(id: string): Promise<QvacKnowledgeArtifact[]>;
+    publish(input: Parameters<typeof addKnowledgeArtifact>[1]): Promise<QvacKnowledgeArtifact>;
+    hasLiveSource(chatId: string, reference: string): Promise<boolean>;
+    syncAttachment(chunk: QvacChunk): Promise<void>;
+    syncKnowledge(row: QvacKnowledge): Promise<void>;
+    syncArtifact(row: QvacKnowledgeArtifact): Promise<void>;
+    sweepTombstones(deviceId?: Buffer | null): Promise<void>;
+    sweepOrphans(coreLength: number): Promise<void>;
+    processorState(id: string): FileState;
+    start(options: {
+        deviceId: Buffer | null;
+    }): void;
+    startProcessor(processor: FileProcessor, options: {
+        deviceId: Buffer | null;
+    }): void;
+    stopProcessor(id: string): Promise<boolean>;
+    private get _processing();
+    startProcessing(options: {
+        embedder: IndexingEmbedder;
+        index: VectorIndex;
+        deviceId: Buffer | null;
+        extractors?: RagExtractors;
+    }): void;
+    stopProcessing(): Promise<boolean>;
+    suspend(): Promise<void>;
+    resume(): void;
+    listIndexing(chatId: string): Promise<AssistantIndexingStatusList>;
+    watch(listener: () => void): void;
+    unwatch(listener: () => void): void;
+    setModel(model: string | null): void;
+    work(id: string, state: LocalWork, processorId?: string): void;
+    load(id: string, state: LocalWork, options?: {
+        artifactId?: string;
+        processorId?: string;
+    }): void;
+    clearLoaded(processorId?: string): void;
+    hydratedArtifact(id: string): string | null;
+    settle(ids: string[], options?: {
+        timeoutMs?: number;
+        signal?: AbortSignal;
+    }): Promise<void>;
+    reconnect(): void;
+    report: (update: Parameters<IndexingProgress>[0], processorId?: string) => void;
+    progress(chatId: string): IndexingProgressEntry[];
+    beginUpload(input: {
+        fileId?: string | null;
+        chatId?: string | null;
+        fileName?: string | null;
+        mimeType?: string | null;
+        totalBytes?: number | null;
+        data?: Buffer | null;
+    }): Promise<{
+        id: string;
+    }>;
+    uploadProgress(id: string, bytes: number): void;
+    uploadComplete(id: string): void;
+    uploadFailed(id: string, error: unknown): void;
+    uploadClosed(id: string): void;
+    private _pruneUploads;
+    status(chatId: string): Promise<AssistantFileStatusList>;
+    private _processorStatus;
+    close(): Promise<void>;
+}
+export { MemoryVectorIndex, RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, WEB_TOOLS, chooseRagNotice, citedByPhraseOverlap, createRagSourceTagFilter, formatFullDocumentContext, formatRagContext, isFullTextHit, isWholeDocumentQuery, referencedDocuments, isTruncatedFullTextHit, sourcesFromHits, webSourcesFromToolResult, withoutDataChannel } from './rag/index.ts';
+export type { IndexingEmbedder, RagExtractors, RagSource, VectorIndex } from './rag/index.ts';
+export { attachTurnDocumentIds, groupAttachmentsByMessage } from './records/index.ts';
+export { CoreBlobServer } from './store/index.ts';