@qvac/core 0.0.0 → 0.1.1

package/dist-lib/lib/core/files/processing/index.js

@@ -0,0 +1,295 @@
+import { AbortController } from 'bare-abort-controller';
+import {} from "../rag/index.js";
+import { DeadlineScheduler } from "../../../deadline.js";
+import { IndexingSource } from "./source.js";
+import { fileIds, fileRecord } from "../records/index.js";
+import { FileDocument } from "./document.js";
+export class FileProcessing {
+    _mesh;
+    _processor;
+    _deviceId;
+    _source;
+    _output;
+    _progress;
+    _lanes = [];
+    _deadlines = new Map();
+    _scheduler = new DeadlineScheduler(() => this._retry());
+    _scanning = null;
+    _dirty = false;
+    _paused = false;
+    _closed = false;
+    _close = null;
+    _indexCleared = true;
+    _onDirty = (ids) => {
+        if (ids)
+            void this.enqueue(ids);
+        else
+            this._kick();
+    };
+    _onAll = () => this._kick();
+    constructor(mesh, options) {
+        this._mesh = mesh;
+        this._processor = options.processor;
+        this._deviceId = options.deviceId;
+        this._progress = options.progress;
+        const readBytes = mesh.files.readBytes.bind(mesh.files);
+        const fetch = mesh.files.download.bind(mesh.files);
+        this._source = new IndexingSource(readBytes, {
+            retryMs: options.retryMs,
+            retries: mesh.files.processorState(options.processor.id).sourceRetries,
+            download: { fetch, ready: (id) => this._schedule(id, Date.now()) }
+        });
+        this._output = new IndexingSource(readBytes, {
+            retryMs: options.retryMs,
+            retries: mesh.files.processorState(options.processor.id).outputRetries,
+            download: {
+                fetch,
+                ready: (id) => this._schedule(mesh.files.processorState(options.processor.id).artifact(id).knowledgeId ?? id, Date.now())
+            }
+        });
+        this._lanes.push({ stage: 'produce', queue: new Map(), running: null, abort: null });
+        if (options.processor.load) {
+            this._lanes.push({ stage: 'load', queue: new Map(), running: null, abort: null });
+        }
+    }
+    start() {
+        this._mesh.on('indexing-dirty', this._onDirty);
+        this._mesh.autobee.on('move-to', this._onAll);
+        this._mesh.autobee.on('writable', this._onAll);
+        this._kick();
+    }
+    async suspend() {
+        this._paused = true;
+        await this._stop();
+    }
+    resume() {
+        if (this._closed)
+            return;
+        this._paused = false;
+        this._kick();
+    }
+    reconnect() {
+        this._source.reconnect((id, deadline) => this._schedule(id, deadline));
+        this._output.reconnect((id, deadline) => this._schedule(this._mesh.files.processorState(this._processor.id).artifact(id).knowledgeId ?? id, deadline));
+    }
+    close() {
+        this._close ??= this._shutdown();
+        return this._close;
+    }
+    get indexCleared() {
+        return this._indexCleared;
+    }
+    async settle() {
+        if (this._closed || this._paused)
+            return;
+        const ids = await this._documents();
+        for (const lane of this._lanes)
+            await this._enqueue(ids, [lane], { wait: true });
+    }
+    enqueue(ids, options = {}) {
+        return this._enqueue(ids, this._lanes, options);
+    }
+    async _enqueue(ids, lanes, { wait = false } = {}) {
+        if (this._closed || this._paused)
+            return;
+        const pending = [];
+        for (const id of ids) {
+            for (const lane of lanes) {
+                let wakes = lane.queue.get(id);
+                if (!wakes)
+                    lane.queue.set(id, (wakes = []));
+                if (wait)
+                    pending.push(new Promise((resolve) => wakes.push(resolve)));
+            }
+        }
+        for (const lane of lanes)
+            this._pump(lane);
+        await Promise.all(pending);
+    }
+    get active() {
+        return !this._paused && !this._closed && !this._mesh.closing;
+    }
+    pending(id) {
+        return (!this._paused &&
+            !this._closed &&
+            (this._lanes.some((lane) => lane.current === id || lane.queue.has(id)) ||
+                this._source.pending((key) => key === id) ||
+                this._output.pending((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id)));
+    }
+    prioritize(ids) {
+        for (const lane of this._lanes) {
+            const queued = new Map(lane.queue);
+            lane.queue.clear();
+            for (const id of ids)
+                lane.queue.set(id, queued.get(id) ?? []);
+            for (const [id, wakes] of queued)
+                if (!lane.queue.has(id))
+                    lane.queue.set(id, wakes);
+            this._pump(lane);
+        }
+    }
+    async _shutdown() {
+        this._closed = true;
+        this._mesh.off('indexing-dirty', this._onDirty);
+        this._mesh.autobee.off('move-to', this._onAll);
+        this._mesh.autobee.off('writable', this._onAll);
+        await this._stop();
+        if (this._processor.load)
+            this._mesh.files.clearLoaded(this._processor.id);
+        this._indexCleared = await this._processor.close();
+    }
+    async _stop() {
+        this._scheduler.clear();
+        this._deadlines.clear();
+        this._mesh.files.emit('update');
+        for (const lane of this._lanes) {
+            lane.abort?.abort({
+                type: this._closed ? 'file-processing-closed' : 'file-processing-suspended'
+            });
+        }
+        await Promise.all([this._scanning, ...this._lanes.map((lane) => lane.running)]);
+        await Promise.all([this._source.cancel(), this._output.cancel()]);
+        for (const lane of this._lanes) {
+            for (const wakes of lane.queue.values())
+                for (const wake of wakes)
+                    wake();
+            lane.queue.clear();
+        }
+    }
+    _kick() {
+        if (this._closed || this._paused)
+            return;
+        this._dirty = true;
+        if (this._scanning)
+            return;
+        this._scanning = this._scan().finally(() => {
+            this._scanning = null;
+            if (this._dirty)
+                this._kick();
+        });
+    }
+    async _scan() {
+        while (this._dirty && !this._closed && !this._paused) {
+            this._dirty = false;
+            try {
+                await this.enqueue(await this._documents());
+            }
+            catch (err) {
+                logger.warn('rag', 'file processing scan failed', { err });
+            }
+        }
+    }
+    async _documents() {
+        if (this._mesh.closing)
+            return [];
+        const snap = this._mesh.view.snapshot();
+        try {
+            return await fileIds(snap, this._processor.input, this._processor.mediaKind);
+        }
+        finally {
+            await snap.close();
+        }
+    }
+    _pump(lane) {
+        if (lane.running || !lane.queue.size || this._closed || this._paused || this._mesh.closing) {
+            return;
+        }
+        lane.running = this._drain(lane).finally(() => {
+            lane.running = null;
+            this._pump(lane);
+        });
+    }
+    async _drain(lane) {
+        while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
+            const [id, wakes] = lane.queue.entries().next().value;
+            lane.queue.delete(id);
+            lane.current = id;
+            lane.abort = new AbortController();
+            let produced = false;
+            try {
+                const record = await fileRecord(this._mesh.view, id, this._processor.input);
+                if (!record || record.deletedAt) {
+                    await this._source.cancel((key) => key === id);
+                    await this._output.cancel((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id);
+                }
+                if (!record || !this._processor.applies(record) || lane.abort.signal.aborted)
+                    continue;
+                if (lane.stage === 'produce' &&
+                    (!this._mesh.writable ||
+                        !(await this._processor.canProduce(record, {
+                            mesh: this._mesh,
+                            deviceId: this._deviceId
+                        })))) {
+                    continue;
+                }
+                const document = new FileDocument(this._mesh, {
+                    record,
+                    processor: this._processor,
+                    source: this._source,
+                    output: this._output,
+                    schedule: (deadline) => this._schedule(id, deadline),
+                    closed: () => this._closed,
+                    paused: () => this._paused,
+                    progress: this._progress
+                });
+                if (lane.abort.signal.aborted)
+                    continue;
+                await this._processor[lane.stage]?.(document, lane.abort.signal);
+                produced = lane.stage === 'produce';
+            }
+            catch (err) {
+                if (!lane.abort.signal.aborted) {
+                    const reason = err instanceof Error ? err.message : String(err);
+                    if (lane.stage === 'load') {
+                        this._mesh.files.load(id, { state: 'failed', reason }, {
+                            processorId: this._processor.id,
+                            artifactId: this._mesh.files.hydratedArtifact(id) ?? undefined
+                        });
+                    }
+                    else {
+                        this._mesh.files.work(id, { state: 'failed', reason }, this._processor.id);
+                    }
+                    logger.warn('rag', 'file processing failed', { id, stage: lane.stage, err });
+                }
+            }
+            finally {
+                lane.abort = null;
+                lane.current = undefined;
+                if (produced && this.active) {
+                    const loader = this._lanes.find((entry) => entry.stage === 'load');
+                    if (loader) {
+                        if (!loader.queue.has(id))
+                            loader.queue.set(id, []);
+                        this._pump(loader);
+                    }
+                }
+                for (const wake of wakes)
+                    wake();
+                this._mesh.files.emit('update');
+            }
+        }
+    }
+    _schedule(id, deadline) {
+        if (this._closed || this._paused)
+            return;
+        const previous = this._deadlines.get(id);
+        if (previous === undefined || deadline < previous)
+            this._deadlines.set(id, deadline);
+        this._scheduler.schedule(deadline);
+    }
+    _retry() {
+        const ids = [];
+        for (const [id, deadline] of this._deadlines) {
+            if (deadline <= Date.now()) {
+                this._deadlines.delete(id);
+                ids.push(id);
+            }
+            else
+                this._scheduler.schedule(deadline);
+        }
+        void this.enqueue(ids);
+    }
+}
+export { isClaimLive, listIndexingByChat } from "./protocol.js";
+export { FileState } from "./state.js";
+export { settleFiles } from "./settle.js";

package/dist-lib/lib/core/files/processing/ownership.d.ts

@@ -0,0 +1,11 @@
+import type { BlobId } from 'hyperblobs';
+import type { Mesh } from '../../mesh.ts';
+import type { FileState } from './state.ts';
+export declare class ArtifactOwnership {
+    private readonly _mesh;
+    private readonly _state;
+    constructor(mesh: Mesh, state?: FileState);
+    onPublished(artifactId: string, blobRef: BlobId, artifactBlobId: string, landedBlobId: string): Promise<void>;
+    reclaimSuperseded(artifactId: string, winnerBlobId: string): Promise<void>;
+    clearUnreferenced(artifactId: string, blobRef: BlobId): Promise<void>;
+}

package/dist-lib/lib/core/files/processing/ownership.js

@@ -0,0 +1,41 @@
+import { compareBlobReferences } from "../store/index.js";
+export class ArtifactOwnership {
+    _mesh;
+    _state;
+    constructor(mesh, state = mesh.files.state) {
+        this._mesh = mesh;
+        this._state = state;
+    }
+    async onPublished(artifactId, blobRef, artifactBlobId, landedBlobId) {
+        if (compareBlobReferences(landedBlobId, artifactBlobId, this._mesh.blobs.core.key.toString('hex')) === false) {
+            await this._mesh.files.clear(this._mesh.files.reference(blobRef), artifactId).catch(noop);
+            return;
+        }
+        this._state.artifact(artifactId).published = { artifactBlobId };
+    }
+    async reclaimSuperseded(artifactId, winnerBlobId) {
+        const local = this._state.artifact(artifactId);
+        const mine = local.published;
+        if (!mine ||
+            compareBlobReferences(mine.artifactBlobId, winnerBlobId, this._mesh.blobs.core.key.toString('hex')) !== false) {
+            return;
+        }
+        local.published = undefined;
+        await this._mesh.files.clear(mine.artifactBlobId, artifactId).catch(noop);
+    }
+    async clearUnreferenced(artifactId, blobRef) {
+        try {
+            const row = await this._mesh.files.artifact(artifactId);
+            const mine = this._mesh.files.reference(blobRef);
+            if (row &&
+                compareBlobReferences(row.artifactBlobId, mine, this._mesh.blobs.core.key.toString('hex')) !== false) {
+                this._state.artifact(artifactId).published = { artifactBlobId: mine };
+                return;
+            }
+            await this._mesh.files.clear(this._mesh.files.reference(blobRef), artifactId).catch(noop);
+        }
+        catch {
+        }
+    }
+}
+const noop = () => { };

package/dist-lib/lib/core/files/processing/protocol.d.ts

@@ -0,0 +1,29 @@
+import type { QvacIndexingResolution, QvacIndexingStatus } from '../../../../spec/mesh/hyperschema/types.d.ts';
+import type { Mesh } from '../../mesh.ts';
+export { INDEXING_CLAIM_TTL_MS, INDEXING_MAX_CLOCK_SKEW_MS } from '../../contract/indexing.ts';
+export declare function isClaimLive(row: QvacIndexingStatus, at: number): boolean;
+export interface ClaimIndexingInput {
+    artifactId: string;
+    knowledgeId: string;
+    at: number;
+}
+export declare function getIndexingStatus(store: Mesh, artifactId: string): Promise<QvacIndexingStatus | null>;
+export declare function listIndexingByKnowledge(store: Mesh, { knowledgeId }: {
+    knowledgeId: string;
+}): Promise<QvacIndexingStatus[]>;
+export declare function listIndexingByChat(store: Mesh, { chatId }: {
+    chatId: string;
+}): Promise<QvacIndexingStatus[]>;
+export declare function claimIndexing(store: Mesh, { artifactId, knowledgeId, at }: ClaimIndexingInput): Promise<{
+    row: QvacIndexingStatus;
+    won: boolean;
+}>;
+export declare function resolveIndexing(store: Mesh, { artifactId, claimId, status, error }: {
+    artifactId: string;
+    claimId: string;
+    status: QvacIndexingResolution;
+    error?: string;
+}): Promise<{
+    row: QvacIndexingStatus;
+    applied: boolean;
+}>;

package/dist-lib/lib/core/files/processing/protocol.js

@@ -0,0 +1,70 @@
+import { INDEXING_MAX_CLOCK_SKEW_MS } from "../../contract/indexing.js";
+import { randomId } from "../../id.js";
+export { INDEXING_CLAIM_TTL_MS, INDEXING_MAX_CLOCK_SKEW_MS } from "../../contract/indexing.js";
+export function isClaimLive(row, at) {
+    return (row.status === 'indexing' && row.expiresAt > at && row.at <= at + INDEXING_MAX_CLOCK_SKEW_MS);
+}
+const INDEXING_STATUS = '@qvac/indexing-status';
+const INDEXING_STATUS_BY_KNOWLEDGE = '@qvac/indexing-status-by-knowledge';
+const INDEXING_STATUS_BY_CHAT = '@qvac/indexing-status-by-chat';
+export async function getIndexingStatus(store, artifactId) {
+    const snap = store.view.snapshot();
+    try {
+        const status = await snap.get(INDEXING_STATUS, { id: artifactId });
+        return status;
+    }
+    finally {
+        await snap.close();
+    }
+}
+async function mustGetIndexingStatus(store, artifactId) {
+    const row = await getIndexingStatus(store, artifactId);
+    if (!row)
+        throw new Error(`indexing status not found: ${artifactId}`);
+    return row;
+}
+export async function listIndexingByKnowledge(store, { knowledgeId }) {
+    const snap = store.view.snapshot();
+    try {
+        const rows = await snap
+            .find(INDEXING_STATUS_BY_KNOWLEDGE, {
+            gte: { knowledgeId },
+            lte: { knowledgeId }
+        })
+            .toArray();
+        return rows;
+    }
+    finally {
+        await snap.close();
+    }
+}
+export async function listIndexingByChat(store, { chatId }) {
+    const snap = store.view.snapshot();
+    try {
+        const rows = await snap
+            .find(INDEXING_STATUS_BY_CHAT, {
+            gte: { chatId },
+            lte: { chatId }
+        })
+            .toArray();
+        return rows;
+    }
+    finally {
+        await snap.close();
+    }
+}
+export async function claimIndexing(store, { artifactId, knowledgeId, at }) {
+    if (!at || at < 0)
+        throw new Error(`claim-indexing needs a positive timestamp, got ${at}`);
+    const claimId = randomId();
+    await store.dispatch('claim-indexing', { id: artifactId, knowledgeId, claimId, at });
+    const row = await mustGetIndexingStatus(store, artifactId);
+    const won = row.status === 'indexing' && row.claimId === claimId;
+    return { row, won };
+}
+export async function resolveIndexing(store, { artifactId, claimId, status, error }) {
+    await store.dispatch('resolve-indexing', { id: artifactId, claimId, status, error });
+    const row = await mustGetIndexingStatus(store, artifactId);
+    const applied = row.status === status && row.claimId === claimId;
+    return { row, applied };
+}

package/dist-lib/lib/core/files/processing/settle.d.ts

@@ -0,0 +1,10 @@
+import type { AbortSignal } from 'bare-abort-controller';
+import type { Files } from '../index.ts';
+import type { FileProcessing } from './index.ts';
+export declare function settleFiles(files: Files, options: {
+    ids: string[];
+    processing: FileProcessing;
+    chatIds: () => Promise<string[]>;
+    timeoutMs?: number;
+    signal?: AbortSignal;
+}): Promise<void>;

package/dist-lib/lib/core/files/processing/settle.js

@@ -0,0 +1,75 @@
+import { AbortedError } from '@qvac/harness/race-abort';
+export function settleFiles(files, options) {
+    const { ids, processing, signal, timeoutMs = 5_000 } = options;
+    if (signal?.aborted)
+        return Promise.reject(new AbortedError());
+    if (!ids.length || timeoutMs <= 0 || files.closed)
+        return Promise.resolve();
+    const deadline = Date.now() + timeoutMs;
+    const wanted = new Set(ids);
+    return new Promise((resolve, reject) => {
+        let done = false;
+        let checking = false;
+        let dirty = false;
+        const timer = setTimeout(() => finish(), timeoutMs);
+        const finish = (error) => {
+            if (done)
+                return;
+            done = true;
+            clearTimeout(timer);
+            files.unwatch(check);
+            files.off('close', onClose);
+            signal?.removeEventListener('abort', onAbort);
+            if (error)
+                reject(error);
+            else
+                resolve();
+        };
+        const onClose = () => finish();
+        const onAbort = () => finish(new AbortedError());
+        const chats = options.chatIds();
+        const check = () => {
+            dirty = true;
+            if (checking || done)
+                return;
+            checking = true;
+            void (async () => {
+                while (dirty && !done) {
+                    dirty = false;
+                    const snapshots = await Promise.all((await chats).map((chatId) => files.status(chatId)));
+                    if (done)
+                        return;
+                    if (!processing.active)
+                        return finish();
+                    const waiting = snapshots
+                        .flatMap((snapshot) => snapshot.files)
+                        .some((file) => {
+                        if (!wanted.has(file.fileId) || file.readyForChat)
+                            return false;
+                        const rag = file.processors.find((entry) => entry.processorId === 'rag');
+                        if (!rag || rag.state === 'not-applicable')
+                            return false;
+                        if (rag.state === 'failed' && (!rag.retryAt || rag.retryAt > deadline))
+                            return false;
+                        return (processing.pending(file.fileId) ||
+                            rag.state === 'running' ||
+                            (rag.retryAt !== undefined && rag.retryAt <= deadline));
+                    });
+                    if (!waiting)
+                        finish();
+                }
+            })()
+                .catch(finish)
+                .finally(() => {
+                checking = false;
+                if (dirty && !done)
+                    check();
+            });
+        };
+        files.watch(check);
+        files.on('close', onClose);
+        signal?.addEventListener('abort', onAbort, { once: true });
+        processing.prioritize(ids);
+        check();
+    });
+}

package/dist-lib/lib/core/files/processing/source.d.ts

@@ -0,0 +1,36 @@
+import { type BlobStore } from '../store/index.ts';
+import { type AbortSignal } from 'bare-abort-controller';
+import type { FileRetries } from './state.ts';
+export type SourceRead = {
+    kind: 'bytes';
+    bytes: Buffer;
+} | {
+    kind: 'missing';
+    retryAt: number;
+} | {
+    kind: 'malformed';
+};
+export interface IndexingSourceOptions {
+    readonly retryMs?: number;
+    readonly retries: FileRetries;
+    readonly download?: {
+        fetch: BlobStore['download'];
+        ready: (id: string) => void;
+    };
+}
+export declare class IndexingSource {
+    private readonly _readBytes;
+    private readonly _retryMs;
+    private readonly _download;
+    private readonly _downloads;
+    readonly retry: FileRetries;
+    constructor(readBytes: BlobStore['readBytes'], opts: IndexingSourceOptions);
+    retryBlockedUntil(id: string, now: number): number | null;
+    retryMiss(id: string, now: number): number;
+    retryClear(id: string): void;
+    reconnect(onRetry?: (id: string, deadline: number) => void): number | null;
+    read(id: string, blobRef: string, signal?: AbortSignal): Promise<SourceRead>;
+    cancel(matches?: (id: string) => boolean): Promise<void>;
+    pending(matches: (id: string) => boolean): boolean;
+    private _fetch;
+}

package/dist-lib/lib/core/files/processing/source.js

@@ -0,0 +1,96 @@
+import { isInvalidBlobReferenceError, compareBlobReferences } from "../store/index.js";
+import { AbortController } from 'bare-abort-controller';
+import { AbortedError } from '@qvac/harness/race-abort';
+const BLOB_RETRY_MS = 30_000;
+export class IndexingSource {
+    _readBytes;
+    _retryMs;
+    _download;
+    _downloads = new Map();
+    retry;
+    constructor(readBytes, opts) {
+        this._readBytes = readBytes;
+        this._retryMs = opts.retryMs ?? BLOB_RETRY_MS;
+        this.retry = opts.retries;
+        this._download = opts.download;
+    }
+    retryBlockedUntil(id, now) {
+        return this.retry.blockedUntil(id, now);
+    }
+    retryMiss(id, now) {
+        return this.retry.miss(id, now, this._retryMs);
+    }
+    retryClear(id) {
+        this.retry.clear(id);
+    }
+    reconnect(onRetry) {
+        return this.retry.reconnect(onRetry);
+    }
+    async read(id, blobRef, signal) {
+        try {
+            const bytes = await this._readBytes(blobRef, { signal, wait: !this._download });
+            this.retry.missing(id, false);
+            return { kind: 'bytes', bytes };
+        }
+        catch (err) {
+            if (signal?.aborted || err instanceof AbortedError)
+                throw new AbortedError();
+            if (isInvalidBlobReferenceError(err)) {
+                logger.warn('rag', 'blob ref is malformed, never readable', { id });
+                return { kind: 'malformed' };
+            }
+            logger.debug('rag', 'blob bytes unavailable', { id, err });
+            this.retry.missing(id, true);
+            const retryAt = this.retryMiss(id, Date.now());
+            this._fetch(id, blobRef);
+            return { kind: 'missing', retryAt };
+        }
+    }
+    async cancel(matches = () => true) {
+        const pending = [];
+        for (const [id, transfer] of this._downloads) {
+            if (!matches(id))
+                continue;
+            transfer.abort.abort(new AbortedError());
+            this.retryClear(id);
+            pending.push(transfer.done);
+        }
+        await Promise.all(pending);
+    }
+    pending(matches) {
+        for (const id of this._downloads.keys())
+            if (matches(id))
+                return true;
+        return false;
+    }
+    _fetch(id, reference) {
+        const download = this._download;
+        if (!download)
+            return;
+        const current = this._downloads.get(id);
+        if (current && compareBlobReferences(current.reference, reference))
+            return;
+        current?.abort.abort(new AbortedError());
+        const abort = new AbortController();
+        const transfer = {
+            reference,
+            abort,
+            done: download
+                .fetch(reference, { signal: abort.signal })
+                .then(() => {
+                if (abort.signal.aborted)
+                    return;
+                this.retryClear(id);
+                download.ready(id);
+            }, (err) => {
+                if (!abort.signal.aborted)
+                    logger.debug('rag', 'blob download interrupted', { id, err });
+            })
+                .finally(() => {
+                if (this._downloads.get(id) === transfer)
+                    this._downloads.delete(id);
+            })
+        };
+        this._downloads.set(id, transfer);
+    }
+}