@qvac/core 0.0.0 → 0.1.0

package/dist-lib/lib/core/files/processing/state.d.ts

@@ -0,0 +1,64 @@
+import type { AssistantFileProcessorStatus } from '../../../../spec/hyperschema/types.d.ts';
+import type { IndexingProgress } from '../rag/index.ts';
+export type LocalWork = Pick<AssistantFileProcessorStatus, 'state' | 'reason' | 'retryAt'>;
+export type IndexingProgressEntry = Omit<Parameters<IndexingProgress>[0], 'error'> & {
+    error: string | null;
+};
+export type LoadedArtifact = {
+    artifactId: string;
+    blobRef?: string;
+    evict?: string;
+};
+type DocumentState = {
+    work?: LocalWork;
+    load?: LocalWork;
+    progress?: IndexingProgressEntry;
+    loaded?: LoadedArtifact;
+    sourceMisses?: number;
+    artifactIdMemo?: string;
+    embedFailures?: number;
+    preferredUntil?: number;
+};
+type ArtifactState = {
+    knowledgeId?: string;
+    terminal?: boolean;
+    failedReference?: string;
+    unavailable?: {
+        blobRef: string;
+        repeated: boolean;
+    };
+    rebuildFailures?: number;
+    published?: {
+        artifactBlobId: string;
+    };
+};
+export declare class FileRetries {
+    private readonly _entries;
+    private readonly _missing;
+    blockedUntil(id: string, now: number): number | null;
+    miss(id: string, now: number, retryMs: number): number;
+    missing(id: string, missing: boolean): void;
+    clear(id?: string): void;
+    reconnect(onRetry?: (id: string, deadline: number) => void): number | null;
+    get(id: string): Readonly<{
+        at: number;
+        delay: number;
+    }> | undefined;
+    rewind(id: string): void;
+}
+export declare class FileState {
+    private readonly _documents;
+    private readonly _artifacts;
+    readonly sourceRetries: FileRetries;
+    readonly outputRetries: FileRetries;
+    document(id: string): DocumentState;
+    artifact(id: string): ArtifactState;
+    get(id: string): Readonly<DocumentState> | undefined;
+    readArtifact(id: string): Readonly<ArtifactState> | undefined;
+    progress(chatId: string): IndexingProgressEntry[];
+    clearLoaded(): void;
+    retry(id: string): void;
+    reset(): void;
+    close(): void;
+}
+export {};

package/dist-lib/lib/core/files/processing/state.js

@@ -0,0 +1,118 @@
+export class FileRetries {
+    _entries = new Map();
+    _missing = new Set();
+    blockedUntil(id, now) {
+        const entry = this._entries.get(id);
+        return entry && now < entry.at ? entry.at : null;
+    }
+    miss(id, now, retryMs) {
+        const previous = this._entries.get(id);
+        const delay = previous ? Math.min(previous.delay * 2, 600_000) : retryMs;
+        const at = now + delay;
+        this._entries.set(id, { at, delay });
+        return at;
+    }
+    missing(id, missing) {
+        if (missing)
+            this._missing.add(id);
+        else
+            this._missing.delete(id);
+    }
+    clear(id) {
+        if (id !== undefined) {
+            this._entries.delete(id);
+            this._missing.delete(id);
+        }
+        else {
+            this._entries.clear();
+            this._missing.clear();
+        }
+    }
+    reconnect(onRetry) {
+        let deadline = null;
+        for (const id of this._missing) {
+            const entry = this._entries.get(id);
+            if (!entry)
+                continue;
+            deadline ??= Date.now() + 250;
+            entry.at = Math.min(entry.at, deadline);
+            onRetry?.(id, entry.at);
+        }
+        return deadline;
+    }
+    get(id) {
+        return this._entries.get(id);
+    }
+    rewind(id) {
+        const entry = this._entries.get(id);
+        if (entry)
+            entry.at = 0;
+    }
+}
+export class FileState {
+    _documents = new Map();
+    _artifacts = new Map();
+    sourceRetries = new FileRetries();
+    outputRetries = new FileRetries();
+    document(id) {
+        let entry = this._documents.get(id);
+        if (!entry)
+            this._documents.set(id, (entry = {}));
+        return entry;
+    }
+    artifact(id) {
+        let entry = this._artifacts.get(id);
+        if (!entry)
+            this._artifacts.set(id, (entry = {}));
+        return entry;
+    }
+    get(id) {
+        return this._documents.get(id);
+    }
+    readArtifact(id) {
+        return this._artifacts.get(id);
+    }
+    progress(chatId) {
+        return [...this._documents.values()].flatMap((entry) => entry.progress?.chatId === chatId ? [entry.progress] : []);
+    }
+    clearLoaded() {
+        for (const entry of this._documents.values()) {
+            entry.loaded = undefined;
+            entry.load = undefined;
+        }
+    }
+    retry(id) {
+        const entry = this.document(id);
+        entry.work = undefined;
+        entry.load = undefined;
+        entry.preferredUntil = 0;
+        entry.embedFailures = undefined;
+        entry.sourceMisses = undefined;
+        this.sourceRetries.clear(id);
+        this.outputRetries.clear(id);
+        for (const [artifactId, output] of this._artifacts) {
+            if (output.knowledgeId !== id && artifactId !== entry.artifactIdMemo)
+                continue;
+            output.terminal = false;
+            output.failedReference = undefined;
+            output.unavailable = undefined;
+            output.rebuildFailures = undefined;
+            this.outputRetries.clear(artifactId);
+        }
+    }
+    reset() {
+        this._documents.clear();
+        for (const [id, entry] of this._artifacts) {
+            if (entry.published)
+                this._artifacts.set(id, { published: entry.published });
+            else
+                this._artifacts.delete(id);
+        }
+        this.sourceRetries.clear();
+        this.outputRetries.clear();
+    }
+    close() {
+        this.reset();
+        this._artifacts.clear();
+    }
+}

package/dist-lib/lib/core/files/rag/artifact/codec.d.ts

@@ -0,0 +1,3 @@
+import { type RagArtifactPayload } from './types.ts';
+export declare function encodeRagArtifact(payload: RagArtifactPayload): Buffer;
+export declare function decodeRagArtifact(bytes: Buffer): RagArtifactPayload;

package/dist-lib/lib/core/files/rag/artifact/codec.js

@@ -0,0 +1,172 @@
+import b4a from 'b4a';
+import { ARTIFACT_FORMAT_VERSION, assertRagArtifactPayload } from "./types.js";
+const MAGIC = b4a.from(ARTIFACT_FORMAT_VERSION, 'ascii');
+const HEADER_LENGTH_BYTES = 4;
+export function encodeRagArtifact(payload) {
+    assertRagArtifactPayload(payload);
+    assertPayloadDimensions(payload);
+    const sections = buildSections(payload);
+    const header = b4a.from(JSON.stringify(sections.header), 'utf8');
+    const headerLength = b4a.alloc(HEADER_LENGTH_BYTES);
+    b4a.writeUInt32LE(headerLength, header.byteLength, 0);
+    return b4a.concat([
+        MAGIC,
+        headerLength,
+        header,
+        sections.text,
+        sections.metadata,
+        sections.embeddings
+    ]);
+}
+export function decodeRagArtifact(bytes) {
+    assertMagic(bytes);
+    const headerLengthOffset = MAGIC.byteLength;
+    const headerLength = b4a.readUInt32LE(bytes, headerLengthOffset);
+    const headerStart = headerLengthOffset + HEADER_LENGTH_BYTES;
+    const headerEnd = headerStart + headerLength;
+    let parsed;
+    try {
+        parsed = JSON.parse(b4a.toString(bytes.subarray(headerStart, headerEnd), 'utf8'));
+    }
+    catch {
+        throw new Error('RAG artifact header is not valid JSON');
+    }
+    const header = parseArtifactHeader(parsed);
+    const textStart = headerEnd;
+    const metadataStart = textStart + header.textByteLength;
+    const embeddingStart = metadataStart + header.metadataByteLength;
+    assertExpectedLength(bytes, header, embeddingStart);
+    return {
+        modelName: header.modelName,
+        dimensions: header.dimensions,
+        chunks: header.chunks.map((chunk, index) => decodeChunk(bytes, header, chunk, index, textStart, metadataStart, embeddingStart))
+    };
+}
+function buildSections(payload) {
+    const textBuffers = [];
+    const metadataBuffers = [];
+    const chunks = [];
+    let contentOffset = 0;
+    let metadataOffset = 0;
+    for (const chunk of payload.chunks) {
+        const content = b4a.from(chunk.content, 'utf8');
+        const metadata = b4a.from(chunk.metadataJson, 'utf8');
+        chunks.push({
+            id: chunk.id,
+            contentOffset,
+            contentLength: content.byteLength,
+            metadataOffset,
+            metadataLength: metadata.byteLength
+        });
+        textBuffers.push(content);
+        metadataBuffers.push(metadata);
+        contentOffset += content.byteLength;
+        metadataOffset += metadata.byteLength;
+    }
+    return {
+        header: {
+            modelName: payload.modelName,
+            dimensions: payload.dimensions,
+            chunkCount: payload.chunks.length,
+            textByteLength: contentOffset,
+            metadataByteLength: metadataOffset,
+            chunks
+        },
+        text: b4a.concat(textBuffers),
+        metadata: b4a.concat(metadataBuffers),
+        embeddings: encodeEmbeddings(payload)
+    };
+}
+function encodeEmbeddings(payload) {
+    const embeddings = b4a.alloc(payload.chunks.length * payload.dimensions * 4);
+    let offset = 0;
+    for (const chunk of payload.chunks) {
+        for (const value of chunk.embedding) {
+            b4a.writeFloatLE(embeddings, finite(value), offset);
+            offset += 4;
+        }
+    }
+    return embeddings;
+}
+function decodeChunk(bytes, header, chunk, index, textStart, metadataStart, embeddingStart) {
+    return {
+        id: chunk.id,
+        content: b4a.toString(bytes.subarray(textStart + chunk.contentOffset, textStart + chunk.contentOffset + chunk.contentLength), 'utf8'),
+        metadataJson: b4a.toString(bytes.subarray(metadataStart + chunk.metadataOffset, metadataStart + chunk.metadataOffset + chunk.metadataLength), 'utf8'),
+        embedding: decodeEmbedding(bytes, header.dimensions, embeddingStart, index)
+    };
+}
+function decodeEmbedding(bytes, dimensions, embeddingStart, chunkIndex) {
+    const values = [];
+    let offset = embeddingStart + chunkIndex * dimensions * 4;
+    for (let index = 0; index < dimensions; index++) {
+        const value = b4a.readFloatLE(bytes, offset);
+        if (!Number.isFinite(value)) {
+            throw new Error('RAG artifact has a non-finite embedding value');
+        }
+        values.push(value);
+        offset += 4;
+    }
+    return values;
+}
+function parseArtifactHeader(value) {
+    const header = value;
+    if (!header ||
+        typeof header.modelName !== 'string' ||
+        !isNonNegativeInteger(header.dimensions) ||
+        header.dimensions === 0 ||
+        !isNonNegativeInteger(header.chunkCount) ||
+        !isNonNegativeInteger(header.textByteLength) ||
+        !isNonNegativeInteger(header.metadataByteLength) ||
+        !Array.isArray(header.chunks) ||
+        !header.chunks.every(isChunkHeader)) {
+        throw new Error('RAG artifact header is malformed');
+    }
+    const parsed = header;
+    for (const chunk of parsed.chunks) {
+        if (chunk.contentOffset + chunk.contentLength > parsed.textByteLength ||
+            chunk.metadataOffset + chunk.metadataLength > parsed.metadataByteLength) {
+            throw new Error('RAG artifact chunk range is out of bounds');
+        }
+    }
+    return parsed;
+}
+function isChunkHeader(value) {
+    const chunk = value;
+    return Boolean(chunk &&
+        typeof chunk.id === 'string' &&
+        isNonNegativeInteger(chunk.contentOffset) &&
+        isNonNegativeInteger(chunk.contentLength) &&
+        isNonNegativeInteger(chunk.metadataOffset) &&
+        isNonNegativeInteger(chunk.metadataLength));
+}
+function isNonNegativeInteger(value) {
+    return typeof value === 'number' && Number.isInteger(value) && value >= 0;
+}
+function assertPayloadDimensions(payload) {
+    for (const chunk of payload.chunks) {
+        if (chunk.embedding.length !== payload.dimensions) {
+            throw new Error(`Chunk ${chunk.id} has ${chunk.embedding.length} dimensions`);
+        }
+    }
+}
+function assertMagic(bytes) {
+    if (bytes.byteLength < MAGIC.byteLength + HEADER_LENGTH_BYTES) {
+        throw new Error('RAG artifact is truncated');
+    }
+    if (!b4a.equals(bytes.subarray(0, MAGIC.byteLength), MAGIC)) {
+        throw new Error('RAG artifact has an unsupported format');
+    }
+}
+function assertExpectedLength(bytes, header, embeddingStart) {
+    const expected = embeddingStart + header.chunkCount * header.dimensions * 4;
+    if (bytes.byteLength !== expected || header.chunkCount !== header.chunks.length) {
+        throw new Error('RAG artifact length does not match its header');
+    }
+}
+function finite(value) {
+    if (!Number.isFinite(value)) {
+        throw new Error('Embedding value must be finite');
+    }
+    return value;
+}

package/dist-lib/lib/core/files/rag/artifact/id.d.ts

@@ -0,0 +1,10 @@
+export type ArtifactIdInput = {
+    readonly attachmentId: string;
+    readonly sourceBlobId: string;
+    readonly sourceHash: string;
+    readonly embeddingModel: string;
+};
+export declare function contentHash(bytes: Buffer): string;
+export declare function documentIndexArtifactId(input: ArtifactIdInput): string;
+export declare function pipelineVersionFor(model: string): string;
+export declare function stableHashString(value: string): string;

package/dist-lib/lib/core/files/rag/artifact/id.js

@@ -0,0 +1,39 @@
+import b4a from 'b4a';
+import { CHUNKER_VERSION, EMBEDDER_CODE_VERSION, EXTRACTOR_VERSION } from "./types.js";
+export function contentHash(bytes) {
+    return stableHashBytes(bytes);
+}
+export function documentIndexArtifactId(input) {
+    return [
+        'docidx',
+        stableHashString([
+            input.attachmentId,
+            input.sourceBlobId,
+            input.sourceHash,
+            input.embeddingModel,
+            EMBEDDER_CODE_VERSION,
+            EXTRACTOR_VERSION,
+            CHUNKER_VERSION
+        ].join('\u0000'))
+    ].join('-');
+}
+export function pipelineVersionFor(model) {
+    return `${model}|${EMBEDDER_CODE_VERSION}|${EXTRACTOR_VERSION}|${CHUNKER_VERSION}`;
+}
+export function stableHashString(value) {
+    return stableHashBytes(b4a.from(value, 'utf8'));
+}
+function stableHashBytes(bytes) {
+    let high = 0x811c9dc5;
+    let low = 0x01000193;
+    for (const byte of bytes) {
+        high ^= byte;
+        high = Math.imul(high, 0x01000193);
+        low ^= byte + 0x9e3779b9;
+        low = Math.imul(low, 0x85ebca6b);
+    }
+    return `${hex32(high)}${hex32(low)}`;
+}
+function hex32(value) {
+    return (value >>> 0).toString(16).padStart(8, '0');
+}

package/dist-lib/lib/core/files/rag/artifact/types.d.ts

@@ -0,0 +1,24 @@
+export declare const DOCUMENT_INDEX_ARTIFACT_STATUS: {
+    readonly pending: "pending";
+    readonly indexing: "indexing";
+    readonly ready: "ready";
+    readonly failed: "failed";
+};
+export type DocumentIndexArtifactStatus = (typeof DOCUMENT_INDEX_ARTIFACT_STATUS)[keyof typeof DOCUMENT_INDEX_ARTIFACT_STATUS];
+export declare const EXTRACTOR_VERSION = "document-extractor-v1";
+export declare const CHUNKER_VERSION = "rag-chunker-v1";
+export declare const ARTIFACT_FORMAT_VERSION = "QVRAGIDX1";
+export declare const EMBEDDER_CODE_VERSION = "embedder-v1";
+export type RagArtifactChunk = {
+    readonly id: string;
+    readonly content: string;
+    readonly metadataJson: string;
+    readonly embedding: ReadonlyArray<number>;
+};
+export type RagArtifactPayload = {
+    readonly modelName: string;
+    readonly dimensions: number;
+    readonly chunks: ReadonlyArray<RagArtifactChunk>;
+};
+export declare function isDocumentIndexArtifactStatus(value: unknown): value is DocumentIndexArtifactStatus;
+export declare function assertRagArtifactPayload(payload: RagArtifactPayload): void;

package/dist-lib/lib/core/files/rag/artifact/types.js

@@ -0,0 +1,41 @@
+export const DOCUMENT_INDEX_ARTIFACT_STATUS = {
+    pending: 'pending',
+    indexing: 'indexing',
+    ready: 'ready',
+    failed: 'failed'
+};
+export const EXTRACTOR_VERSION = 'document-extractor-v1';
+export const CHUNKER_VERSION = 'rag-chunker-v1';
+export const ARTIFACT_FORMAT_VERSION = 'QVRAGIDX1';
+export const EMBEDDER_CODE_VERSION = 'embedder-v1';
+export function isDocumentIndexArtifactStatus(value) {
+    return typeof value === 'string' && value in DOCUMENT_INDEX_ARTIFACT_STATUS;
+}
+export function assertRagArtifactPayload(payload) {
+    if (typeof payload.modelName !== 'string' || payload.modelName.length === 0) {
+        throw new Error('RAG artifact payload needs a model name');
+    }
+    if (!Number.isInteger(payload.dimensions) || payload.dimensions <= 0) {
+        throw new Error('RAG artifact payload needs positive integer dimensions');
+    }
+    if (!Array.isArray(payload.chunks)) {
+        throw new Error('RAG artifact payload needs a chunk array');
+    }
+    for (const chunk of payload.chunks) {
+        if (chunk === null || typeof chunk !== 'object') {
+            throw new Error('RAG artifact chunk must be an object');
+        }
+        if (typeof chunk.id !== 'string' || chunk.id.length === 0) {
+            throw new Error('RAG artifact chunk needs an id');
+        }
+        if (typeof chunk.content !== 'string' || typeof chunk.metadataJson !== 'string') {
+            throw new Error(`RAG artifact chunk ${chunk.id} needs string content and metadata`);
+        }
+        if (!Array.isArray(chunk.embedding) || !chunk.embedding.every(isNumber)) {
+            throw new Error(`RAG artifact chunk ${chunk.id} needs a numeric embedding`);
+        }
+    }
+}
+function isNumber(value) {
+    return typeof value === 'number';
+}

package/dist-lib/lib/core/files/rag/build.d.ts

@@ -0,0 +1,25 @@
+import type { AbortSignal } from 'bare-abort-controller';
+import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
+import { type RagExtractors } from './extract/extract.ts';
+import type { RagArtifactPayload } from './artifact/types.ts';
+export type BuildEmbedder = {
+    model: string;
+    embed(input: {
+        texts: string[];
+        signal?: AbortSignal;
+    }): Promise<number[][]>;
+};
+export type BuildProgress = (processedChunks: number, totalChunks: number) => void;
+export declare class EmbedderFailure extends Error {
+    constructor(cause: unknown);
+}
+export interface ArtifactBuildInput {
+    embedder: BuildEmbedder;
+    knowledge: QvacKnowledge;
+    artifactId: string;
+    bytes: Buffer;
+    signal: AbortSignal;
+    onProgress?: BuildProgress;
+    extractors?: RagExtractors;
+}
+export declare function buildArtifactPayload(input: ArtifactBuildInput): Promise<RagArtifactPayload>;

package/dist-lib/lib/core/files/rag/build.js

@@ -0,0 +1,60 @@
+import { chunkRagDocuments } from "./chunk/chunking.js";
+import { extractRagDocumentFromBytes } from "./extract/extract.js";
+import { AbortedError } from '@qvac/harness/race-abort';
+const EMBED_BATCH = 8;
+export class EmbedderFailure extends Error {
+    constructor(cause) {
+        super(cause instanceof Error ? cause.message : String(cause), { cause });
+        this.name = 'EmbedderFailure';
+    }
+}
+export async function buildArtifactPayload(input) {
+    const { embedder, knowledge, artifactId, bytes, signal, onProgress, extractors } = input;
+    const document = await extractRagDocumentFromBytes({
+        id: artifactId,
+        artifactId,
+        attachmentId: knowledge.id,
+        chatId: knowledge.chatId,
+        blobId: knowledge.blobId,
+        fileName: knowledge.fileName,
+        mimeType: knowledge.mimeType,
+        bytes
+    }, { extractors, signal });
+    const chunks = chunkRagDocuments([document], {});
+    if (!document.content.trim() || chunks.length === 0) {
+        throw new Error(`no extractable text: ${knowledge.fileName}`);
+    }
+    onProgress?.(0, chunks.length);
+    const vectors = [];
+    for (let start = 0; start < chunks.length; start += EMBED_BATCH) {
+        if (signal.aborted)
+            throw new AbortedError();
+        const batch = chunks.slice(start, start + EMBED_BATCH);
+        let rows;
+        try {
+            rows = await embedder.embed({ texts: batch.map((c) => c.content), signal });
+        }
+        catch (err) {
+            throw new EmbedderFailure(err);
+        }
+        vectors.push(...rows);
+        onProgress?.(Math.min(start + batch.length, chunks.length), chunks.length);
+    }
+    if (vectors.length !== chunks.length) {
+        throw new Error(`embedder returned ${vectors.length} vectors for ${chunks.length} chunks`);
+    }
+    const dimensions = vectors[0]?.length ?? 0;
+    if (dimensions <= 0 || vectors.some((v) => v.length !== dimensions)) {
+        throw new Error('embedder returned inconsistent dimensions');
+    }
+    return {
+        modelName: embedder.model,
+        dimensions,
+        chunks: chunks.map((chunk, i) => ({
+            id: chunk.id,
+            content: chunk.content,
+            metadataJson: chunk.metadataJson,
+            embedding: vectors[i]
+        }))
+    };
+}

package/dist-lib/lib/core/files/rag/chunk/chunking.d.ts

@@ -0,0 +1,12 @@
+import type { RagDocumentInput } from '../extract/types.ts';
+export type RagPreparedChunk = {
+    readonly id: string;
+    readonly content: string;
+    readonly tokenCount: number;
+    readonly metadataJson: string;
+};
+export type ChunkOptions = {
+    readonly chunkSize?: number;
+    readonly chunkOverlap?: number;
+};
+export declare function chunkRagDocuments(documents: ReadonlyArray<RagDocumentInput>, options: ChunkOptions): ReadonlyArray<RagPreparedChunk>;