@qvac/core 0.0.0 → 0.1.1

package/dist-lib/lib/core/files/preview/index.d.ts

@@ -0,0 +1,33 @@
+import { Buffer } from 'buffer';
+import type { AbortSignal } from 'bare-abort-controller';
+import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
+import type { FileDocument, FileProcessor } from '../processing/index.ts';
+import { type FileRecord } from '../records/index.ts';
+import { type PreviewEncoder } from './image.ts';
+import type { Mesh } from '../../mesh.ts';
+export { previewEncoder } from './image.ts';
+export type { PreviewEncoder } from './image.ts';
+export declare const PREVIEW_VERSION = "processor:preview:1";
+export declare class PreviewProcessor implements FileProcessor {
+    readonly id = "preview";
+    readonly version = "processor:preview:1";
+    readonly input = "attachment";
+    readonly output = "inline";
+    readonly mediaKind = "image";
+    private readonly _deviceId;
+    private readonly _retryMs;
+    private readonly _encoder;
+    constructor(options: {
+        deviceId: Buffer | null;
+        retryMs?: number;
+        encoder?: () => Promise<PreviewEncoder | null>;
+    });
+    applies(file: FileRecord): boolean;
+    canProduce(file: FileRecord, host: {
+        mesh: Mesh;
+    }): Promise<boolean>;
+    accepts(output: QvacKnowledgeArtifact, file: FileRecord): boolean;
+    produce(document: FileDocument, signal: AbortSignal): Promise<void>;
+    close(): Promise<boolean>;
+}
+export declare function previewId(file: Pick<FileRecord, 'id' | 'blobId'>): string;

package/dist-lib/lib/core/files/preview/index.js

@@ -0,0 +1,161 @@
+import b4a from 'b4a';
+import { Buffer } from 'buffer';
+import crypto from 'hypercore-crypto';
+import { processorArtifactId } from "../records/index.js";
+import { parseBlobReference } from "../store/index.js";
+import { IMAGE_MIMES } from "../../image/normalize.js";
+import { previewEncoder } from "./image.js";
+import { AbortedError } from '@qvac/harness/race-abort';
+import { getChat } from "../../chat.js";
+export { previewEncoder } from "./image.js";
+export const PREVIEW_VERSION = 'processor:preview:1';
+const RETRY_MS = 30_000;
+export class PreviewProcessor {
+    id = 'preview';
+    version = PREVIEW_VERSION;
+    input = 'attachment';
+    output = 'inline';
+    mediaKind = 'image';
+    _deviceId;
+    _retryMs;
+    _encoder;
+    constructor(options) {
+        this._deviceId = options.deviceId;
+        this._retryMs = options.retryMs ?? RETRY_MS;
+        this._encoder = options.encoder ?? previewEncoder;
+    }
+    applies(file) {
+        return (file.kind === 'attachment' &&
+            file.preview === true &&
+            IMAGE_MIMES.has(file.mimeType.toLowerCase()));
+    }
+    async canProduce(file, host) {
+        return !file.deletedAt && !(await getChat(host.mesh, file.chatId))?.deletedAt;
+    }
+    accepts(output, file) {
+        if (output.embedderVersion !== this.version)
+            return false;
+        try {
+            return output.id === previewId(file);
+        }
+        catch {
+            return false;
+        }
+    }
+    async produce(document, signal) {
+        const file = document.record;
+        const id = previewId(file);
+        document.state.artifactIdMemo = id;
+        const existing = (await document.artifacts()).find((row) => this.accepts(row, file));
+        if (existing) {
+            document.work({ state: 'ready' });
+            await document.claims.settle(id);
+            return;
+        }
+        if (document.claims.isFailed(id))
+            return;
+        const encode = await this._encoder();
+        if (!encode) {
+            document.work({ state: 'pending', reason: 'image processor unavailable on this device' });
+            return;
+        }
+        if (!this._deviceId ||
+            !file.originDeviceId ||
+            !b4a.equals(this._deviceId, file.originDeviceId)) {
+            document.state.preferredUntil ??= Math.min(file.createdAt, Date.now()) + this._retryMs;
+            if (document.state.preferredUntil > Date.now()) {
+                document.work({
+                    state: 'pending',
+                    reason: 'waiting for uploader',
+                    retryAt: document.state.preferredUntil
+                });
+                document.schedule(document.state.preferredUntil);
+                return;
+            }
+        }
+        const retryAt = document.source.blockedUntil();
+        if (retryAt) {
+            document.schedule(retryAt);
+            return;
+        }
+        if (await document.claims.blockedByStatus(id))
+            return;
+        const source = await document.source.read(signal);
+        if (source.kind !== 'bytes') {
+            const retryAt = source.kind === 'missing' ? source.retryAt : undefined;
+            document.work({
+                state: source.kind === 'missing' ? 'pending' : 'failed',
+                reason: 'source unavailable',
+                retryAt
+            });
+            if (retryAt)
+                document.schedule(retryAt);
+            return;
+        }
+        if (signal.aborted || document.paused() || document.closed())
+            return;
+        const claim = await document.claims.claim(id);
+        if (!claim)
+            return;
+        document.work({ state: 'running' });
+        try {
+            const bytes = await encode(source.bytes, signal);
+            const current = await document.current();
+            if (signal.aborted)
+                throw new AbortedError();
+            if (!current || current.deletedAt || previewId(current) !== id) {
+                throw new Error('source no longer available');
+            }
+            await document.publish({
+                id,
+                sourceHash: previewSource(file),
+                embedderVersion: this.version,
+                dimensions: 0,
+                artifactBlobId: document.inline(bytes, 'image/jpeg')
+            });
+            if (claim.claimId) {
+                await document.resolve({ artifactId: id, claimId: claim.claimId, status: 'ready' });
+            }
+            document.source.clear();
+            document.work({ state: 'ready' });
+        }
+        catch (error) {
+            const interrupted = signal.aborted || document.paused() || document.closed();
+            const reason = interrupted
+                ? 'interrupted: suspend'
+                : error instanceof Error
+                    ? error.message
+                    : String(error);
+            document.work({ state: interrupted ? 'pending' : 'failed', reason });
+            if (!interrupted)
+                document.claims.markFailed(id);
+            if (claim.claimId && !document.closed()) {
+                await document
+                    .resolve({ artifactId: id, claimId: claim.claimId, status: 'failed', error: reason })
+                    .catch(() => { });
+            }
+        }
+    }
+    close() {
+        return Promise.resolve(true);
+    }
+}
+export function previewId(file) {
+    return processorArtifactId({
+        processorId: 'preview',
+        hash: hash([file.id, previewSource(file), PREVIEW_VERSION])
+    });
+}
+function previewSource(file) {
+    const { blobId, coreKey } = parseBlobReference(file.blobId);
+    return hash([
+        coreKey ?? '',
+        blobId.blockOffset,
+        blobId.blockLength,
+        blobId.byteOffset,
+        blobId.byteLength
+    ]);
+}
+function hash(values) {
+    return b4a.toString(crypto.hash(Buffer.from(JSON.stringify(values))), 'hex');
+}

package/dist-lib/lib/core/files/processing/claims.d.ts

@@ -0,0 +1,26 @@
+import type { Mesh } from '../../mesh.ts';
+import type { FileState } from './state.ts';
+export type IndexingClaimsOptions = {
+    mesh: Mesh;
+    schedule: (deadline: number) => void;
+    paused: () => boolean;
+    closed: () => boolean;
+    state?: FileState;
+    advisory?: boolean;
+};
+export declare class IndexingClaims {
+    private readonly _mesh;
+    private readonly _schedule;
+    private readonly _paused;
+    private readonly _closed;
+    private readonly _state;
+    private readonly _advisory;
+    constructor(opts: IndexingClaimsOptions);
+    markFailed(artifactId: string): void;
+    isFailed(artifactId: string): boolean;
+    claim(artifactId: string, knowledgeId: string): Promise<{
+        claimId: string;
+    } | null>;
+    blockedByStatus(artifactId: string): Promise<boolean>;
+    settle(artifactId: string, knowledgeId: string): Promise<void>;
+}

package/dist-lib/lib/core/files/processing/claims.js

@@ -0,0 +1,89 @@
+import { claimIndexing, getIndexingStatus, isClaimLive, resolveIndexing } from "./protocol.js";
+export class IndexingClaims {
+    _mesh;
+    _schedule;
+    _paused;
+    _closed;
+    _state;
+    _advisory;
+    constructor(opts) {
+        this._mesh = opts.mesh;
+        this._schedule = opts.schedule;
+        this._paused = opts.paused;
+        this._closed = opts.closed;
+        this._state = opts.state ?? opts.mesh.files.state;
+        this._advisory = opts.advisory ?? false;
+    }
+    markFailed(artifactId) {
+        this._state.artifact(artifactId).terminal = true;
+    }
+    isFailed(artifactId) {
+        return this._state.artifact(artifactId).terminal === true;
+    }
+    async claim(artifactId, knowledgeId) {
+        try {
+            const { row, won } = await claimIndexing(this._mesh, {
+                artifactId,
+                knowledgeId,
+                at: Date.now()
+            });
+            if (won)
+                return { claimId: row.claimId };
+            if (isClaimLive(row, Date.now()))
+                this._schedule(row.expiresAt);
+            return null;
+        }
+        catch (err) {
+            if (this._advisory &&
+                !this._closed() &&
+                !this._paused() &&
+                !(await getIndexingStatus(this._mesh, artifactId))) {
+                return { claimId: '' };
+            }
+            logger.debug('rag', 'claim not placed', { id: artifactId, err });
+            return null;
+        }
+    }
+    async blockedByStatus(artifactId) {
+        if (this.isFailed(artifactId))
+            return true;
+        const row = await getIndexingStatus(this._mesh, artifactId);
+        if (!row)
+            return false;
+        if (row.status === 'ready')
+            return true;
+        if (isClaimLive(row, Date.now())) {
+            this._schedule(row.expiresAt);
+            return true;
+        }
+        return false;
+    }
+    async settle(artifactId, knowledgeId) {
+        if (this._paused() || this._closed())
+            return;
+        const row = await getIndexingStatus(this._mesh, artifactId);
+        if (row?.status === 'ready')
+            return;
+        if (row && isClaimLive(row, Date.now())) {
+            this._schedule(row.expiresAt);
+            return;
+        }
+        if (this._paused() || this._closed())
+            return;
+        const claim = await this.claim(artifactId, knowledgeId);
+        if (!claim?.claimId)
+            return;
+        try {
+            const { applied } = await resolveIndexing(this._mesh, {
+                artifactId,
+                claimId: claim.claimId,
+                status: 'ready'
+            });
+            if (applied)
+                logger.info('rag', 'repaired to ready', { id: artifactId });
+        }
+        catch (err) {
+            logger.debug('rag', 'repair resolve dropped', { id: artifactId, err });
+        }
+    }
+}

package/dist-lib/lib/core/files/processing/document.d.ts

@@ -0,0 +1,100 @@
+import type { AbortSignal } from 'bare-abort-controller';
+import type { BlobId } from 'hyperblobs';
+import { type IndexingProgress } from '../rag/index.ts';
+import { IndexingClaims } from './claims.ts';
+import { ArtifactOwnership } from './ownership.ts';
+import { resolveIndexing } from './protocol.ts';
+import type { IndexingSource } from './source.ts';
+import { type FileRecord } from '../records/index.ts';
+import type { Mesh } from '../../mesh.ts';
+import type { LocalWork } from './state.ts';
+import type { FileProcessor } from './index.ts';
+export declare class FileDocument {
+    private readonly _mesh;
+    readonly record: FileRecord;
+    readonly state: ReturnType<Mesh['files']['state']['document']>;
+    readonly source: ReturnType<typeof fileSource>;
+    readonly claims: {
+        markFailed: IndexingClaims['markFailed'];
+        isFailed: IndexingClaims['isFailed'];
+        blockedByStatus: IndexingClaims['blockedByStatus'];
+        claim: (id: string) => ReturnType<IndexingClaims['claim']>;
+        settle: (id: string) => Promise<void>;
+    };
+    readonly ownership: ArtifactOwnership;
+    readonly schedule: (deadline: number) => void;
+    readonly closed: () => boolean;
+    readonly paused: () => boolean;
+    private readonly _output;
+    private readonly _progress?;
+    private readonly _processor;
+    constructor(mesh: Mesh, options: {
+        record: FileRecord;
+        processor: FileProcessor;
+        source: IndexingSource;
+        output: IndexingSource;
+        schedule: (deadline: number) => void;
+        closed: () => boolean;
+        paused: () => boolean;
+        progress?: IndexingProgress;
+    });
+    get localCoreKey(): string;
+    artifact(id: string): {
+        knowledgeId?: string;
+        terminal?: boolean;
+        failedReference?: string;
+        unavailable?: {
+            blobRef: string;
+            repeated: boolean;
+        };
+        rebuildFailures?: number;
+        published?: {
+            artifactBlobId: string;
+        };
+    };
+    readArtifact(id: string): Readonly<{
+        knowledgeId?: string;
+        terminal?: boolean;
+        failedReference?: string;
+        unavailable?: {
+            blobRef: string;
+            repeated: boolean;
+        };
+        rebuildFailures?: number;
+        published?: {
+            artifactBlobId: string;
+        };
+    }> | undefined;
+    output(id: string, reference: string): {
+        read: (signal?: AbortSignal) => Promise<import("./source.ts").SourceRead>;
+        blockedUntil: () => number | null;
+        miss: () => number;
+        clear: () => void;
+    };
+    artifacts(): Promise<import("../../../../spec/index").QvacKnowledgeArtifact[]>;
+    statuses(): Promise<import("../../../../spec/index").QvacIndexingStatus[]>;
+    current(): Promise<FileRecord | null>;
+    publish(input: Omit<Parameters<Mesh['files']['publish']>[0], 'knowledgeId'>): Promise<import("../../../../spec/index").QvacKnowledgeArtifact>;
+    resolve(input: Parameters<typeof resolveIndexing>[1]): Promise<{
+        row: import("../../../../spec/index").QvacIndexingStatus;
+        applied: boolean;
+    }>;
+    write(bytes: Buffer): Promise<{
+        blobRef: BlobId;
+        artifactBlobId: string;
+    }>;
+    inline(bytes: Buffer, mimeType: string): string;
+    work(state: LocalWork): void;
+    load(state: LocalWork, artifactId?: string): void;
+    report(update: Omit<Parameters<IndexingProgress>[0], 'knowledgeId' | 'chatId'>): void;
+}
+declare function fileSource(source: IndexingSource, { id, reference }: {
+    id: string;
+    reference: string;
+}): {
+    read: (signal?: AbortSignal) => Promise<import("./source.ts").SourceRead>;
+    blockedUntil: () => number | null;
+    miss: () => number;
+    clear: () => void;
+};
+export {};

package/dist-lib/lib/core/files/processing/document.js

@@ -0,0 +1,101 @@
+import {} from "../rag/index.js";
+import { IndexingClaims } from "./claims.js";
+import { ArtifactOwnership } from "./ownership.js";
+import { listIndexingByKnowledge, resolveIndexing } from "./protocol.js";
+import { fileRecord } from "../records/index.js";
+export class FileDocument {
+    _mesh;
+    record;
+    state;
+    source;
+    claims;
+    ownership;
+    schedule;
+    closed;
+    paused;
+    _output;
+    _progress;
+    _processor;
+    constructor(mesh, options) {
+        const { record } = options;
+        this._mesh = mesh;
+        this.record = record;
+        this._processor = options.processor;
+        const state = mesh.files.processorState(options.processor.id);
+        this.state = state.document(record.id);
+        this.source = fileSource(options.source, { id: record.id, reference: record.blobId });
+        this._output = options.output;
+        this.schedule = options.schedule;
+        this.closed = () => options.closed() || !!mesh.closing;
+        this.paused = options.paused;
+        this._progress = options.progress;
+        const claims = new IndexingClaims({
+            mesh,
+            ...options,
+            state,
+            advisory: record.kind === 'attachment'
+        });
+        this.claims = {
+            markFailed: (id) => claims.markFailed(id),
+            isFailed: (id) => claims.isFailed(id),
+            blockedByStatus: (id) => claims.blockedByStatus(id),
+            claim: (id) => claims.claim(id, record.id),
+            settle: (id) => claims.settle(id, record.id)
+        };
+        this.ownership = new ArtifactOwnership(mesh, state);
+    }
+    get localCoreKey() {
+        return this._mesh.blobs.core.key.toString('hex');
+    }
+    artifact(id) {
+        return this._mesh.files.processorState(this._processor.id).artifact(id);
+    }
+    readArtifact(id) {
+        return this._mesh.files.state.readArtifact(id);
+    }
+    output(id, reference) {
+        this.artifact(id).knowledgeId = this.record.id;
+        return fileSource(this._output, { id, reference });
+    }
+    artifacts() {
+        return this._mesh.files.artifacts(this.record.id);
+    }
+    statuses() {
+        return listIndexingByKnowledge(this._mesh, { knowledgeId: this.record.id });
+    }
+    current() {
+        return fileRecord(this._mesh.view, this.record.id, this._processor.input);
+    }
+    publish(input) {
+        return this._mesh.files.publish({ ...input, knowledgeId: this.record.id });
+    }
+    resolve(input) {
+        return resolveIndexing(this._mesh, input);
+    }
+    async write(bytes) {
+        const { blobRef, reference } = await this._mesh.files.put(bytes);
+        return { blobRef, artifactBlobId: reference };
+    }
+    inline(bytes, mimeType) {
+        return this._mesh.files.inline(bytes, mimeType);
+    }
+    work(state) {
+        this._mesh.files.work(this.record.id, state, this._processor.id);
+    }
+    load(state, artifactId) {
+        this._mesh.files.load(this.record.id, state, { artifactId, processorId: this._processor.id });
+    }
+    report(update) {
+        const progress = { ...update, knowledgeId: this.record.id, chatId: this.record.chatId };
+        this._mesh.files.report(progress, this._processor.id);
+        this._progress?.(progress);
+    }
+}
+function fileSource(source, { id, reference }) {
+    return {
+        read: (signal) => source.read(id, reference, signal),
+        blockedUntil: () => source.retryBlockedUntil(id, Date.now()),
+        miss: () => source.retryMiss(id, Date.now()),
+        clear: () => source.retryClear(id)
+    };
+}

package/dist-lib/lib/core/files/processing/index.d.ts

@@ -0,0 +1,75 @@
+import { type AbortSignal } from 'bare-abort-controller';
+import { type IndexingProgress } from '../rag/index.ts';
+import { type FileInput, type FileRecord } from '../records/index.ts';
+import type { Mesh } from '../../mesh.ts';
+import { FileDocument } from './document.ts';
+import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
+export interface FileProcessor {
+    readonly id: string;
+    readonly version: string;
+    readonly input: FileInput;
+    readonly output: 'blob' | 'inline';
+    readonly mediaKind?: string;
+    accepts?(output: QvacKnowledgeArtifact, document: FileRecord): boolean;
+    applies(document: FileRecord): boolean;
+    canProduce(document: FileRecord, host: {
+        mesh: Mesh;
+        deviceId: Buffer | null;
+    }): boolean | Promise<boolean>;
+    produce(document: FileDocument, signal: AbortSignal): Promise<void>;
+    load?(document: FileDocument, signal: AbortSignal): Promise<void>;
+    close(): Promise<boolean>;
+}
+export declare class FileProcessing {
+    private readonly _mesh;
+    private readonly _processor;
+    private readonly _deviceId;
+    private readonly _source;
+    private readonly _output;
+    private readonly _progress?;
+    private readonly _lanes;
+    private readonly _deadlines;
+    private readonly _scheduler;
+    private _scanning;
+    private _dirty;
+    private _paused;
+    private _closed;
+    private _close;
+    private _indexCleared;
+    private readonly _onDirty;
+    private readonly _onAll;
+    constructor(mesh: Mesh, options: {
+        processor: FileProcessor;
+        deviceId: Buffer | null;
+        retryMs?: number;
+        progress?: IndexingProgress;
+    });
+    start(): void;
+    suspend(): Promise<void>;
+    resume(): void;
+    reconnect(): void;
+    close(): Promise<void>;
+    get indexCleared(): boolean;
+    settle(): Promise<void>;
+    enqueue(ids: Iterable<string>, options?: {
+        wait?: boolean;
+    }): Promise<void>;
+    private _enqueue;
+    get active(): boolean;
+    pending(id: string): boolean;
+    prioritize(ids: string[]): void;
+    private _shutdown;
+    private _stop;
+    private _kick;
+    private _scan;
+    private _documents;
+    private _pump;
+    private _drain;
+    private _schedule;
+    private _retry;
+}
+export type { FileDocument } from './document.ts';
+export { isClaimLive, listIndexingByChat } from './protocol.ts';
+export { FileState } from './state.ts';
+export type { IndexingProgressEntry, LocalWork } from './state.ts';
+export { settleFiles } from './settle.ts';