@qvac/core 0.0.0 → 0.1.0

package/dist-lib/lib/core/files/rag/extract/text-decoding.js

@@ -0,0 +1,91 @@
+import { decodeLatin1, decodeUtf16 } from "./utf.js";
+const UTF16_LE_BOM = [0xff, 0xfe];
+const UTF16_BE_BOM = [0xfe, 0xff];
+const UTF8_BOM = [0xef, 0xbb, 0xbf];
+const LATIN1_LABEL = /^(?:iso-8859-1|iso8859-1|iso88591|latin-1|latin1)$/;
+const HTML_CHARSET = /<meta\s+[^>]*charset\s*=\s*["']?\s*([A-Za-z0-9._-]+)/i;
+const CONTENT_TYPE_CHARSET = /<meta\s+[^>]*http-equiv\s*=\s*["']?content-type["']?[^>]*content\s*=\s*["'][^"']*charset=([A-Za-z0-9._-]+)/i;
+export function decodeTextBytes(bytes) {
+    return decodeWithEncoding(bytes, detectEncoding(bytes) ?? 'utf-8');
+}
+function detectEncoding(bytes) {
+    return detectBom(bytes) ?? detectUtf16ByNuls(bytes) ?? detectHtmlCharset(bytes);
+}
+function detectBom(bytes) {
+    if (startsWith(bytes, UTF16_LE_BOM))
+        return 'utf-16le';
+    if (startsWith(bytes, UTF16_BE_BOM))
+        return 'utf-16be';
+    return startsWith(bytes, UTF8_BOM) ? 'utf-8' : null;
+}
+function startsWith(bytes, prefix) {
+    return prefix.every((value, index) => bytes[index] === value);
+}
+function detectUtf16ByNuls(bytes) {
+    const sampleLength = Math.min(bytes.length, 512);
+    let evenNuls = 0;
+    let oddNuls = 0;
+    for (let index = 0; index < sampleLength; index++) {
+        if (bytes[index] !== 0)
+            continue;
+        if (index % 2 === 0) {
+            evenNuls++;
+        }
+        else {
+            oddNuls++;
+        }
+    }
+    const threshold = Math.max(8, Math.floor(sampleLength / 8));
+    if (oddNuls >= threshold && oddNuls > evenNuls * 2)
+        return 'utf-16le';
+    return evenNuls >= threshold && evenNuls > oddNuls * 2 ? 'utf-16be' : null;
+}
+function detectHtmlCharset(bytes) {
+    const sample = decodeUtf8(bytes.slice(0, 4096));
+    const match = HTML_CHARSET.exec(sample) ?? CONTENT_TYPE_CHARSET.exec(sample);
+    return normalizeEncoding(match?.[1] ?? null);
+}
+function normalizeEncoding(value) {
+    if (!value)
+        return null;
+    const normalized = value.trim().toLowerCase().replace(/_/g, '-');
+    if (normalized === 'utf8')
+        return 'utf-8';
+    if (LATIN1_LABEL.test(normalized))
+        return 'latin1';
+    if (normalized === 'utf16' || normalized === 'utf-16')
+        return 'utf-16le';
+    if (normalized === 'utf16le')
+        return 'utf-16le';
+    if (normalized === 'utf16be')
+        return 'utf-16be';
+    return normalized;
+}
+function decodeWithEncoding(bytes, encoding) {
+    const normalized = normalizeEncoding(encoding) ?? 'utf-8';
+    if (normalized === 'utf-16le')
+        return decodeUtf16(bytes, true).replace(/^\uFEFF/, '');
+    if (normalized === 'utf-16be')
+        return decodeUtf16(bytes, false).replace(/^\uFEFF/, '');
+    if (normalized === 'latin1')
+        return decodeLatin1(bytes);
+    return decodeWithTextDecoder(bytes, normalized);
+}
+function decodeUtf8(bytes) {
+    return decodeWithTextDecoder(bytes, 'utf-8');
+}
+const TextDecoderCtor = globalThis
+    .TextDecoder;
+function decodeWithTextDecoder(bytes, encoding) {
+    if (TextDecoderCtor) {
+        try {
+            return new TextDecoderCtor(encoding).decode(bytes).replace(/^\uFEFF/, '');
+        }
+        catch {
+            return new TextDecoderCtor('utf-8').decode(bytes).replace(/^\uFEFF/, '');
+        }
+    }
+    return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
+        .toString('utf8')
+        .replace(/^\uFEFF/, '');
+}

package/dist-lib/lib/core/files/rag/extract/types.d.ts

@@ -0,0 +1,36 @@
+export type RagDocumentInput = {
+    readonly id?: string;
+    readonly blobId?: string;
+    readonly title: string;
+    readonly content: string;
+    readonly sourcePath?: string;
+    readonly sourceUrl?: string;
+    readonly mimeType?: string;
+    readonly artifactId?: string;
+    readonly attachmentId?: string;
+    readonly messageId?: string;
+    readonly chatId?: string;
+};
+export type RagFileInput = {
+    readonly id?: string;
+    readonly blobId?: string;
+    readonly title?: string;
+    readonly fileName?: string;
+    readonly mimeType?: string;
+    readonly sourcePath?: string;
+    readonly sourceUrl?: string;
+    readonly bytes?: Uint8Array;
+    readonly artifactId?: string;
+    readonly attachmentId?: string;
+    readonly messageId?: string;
+    readonly chatId?: string;
+};
+export declare const RAG_EXTRACT_ERROR_CODE: {
+    readonly unsupportedFile: "RAG_UNSUPPORTED_FILE";
+    readonly extractionFailed: "RAG_EXTRACTION_FAILED";
+};
+export type RagExtractErrorCode = (typeof RAG_EXTRACT_ERROR_CODE)[keyof typeof RAG_EXTRACT_ERROR_CODE];
+export declare class RagExtractError extends Error {
+    readonly code: RagExtractErrorCode;
+    constructor(code: RagExtractErrorCode, message: string);
+}

package/dist-lib/lib/core/files/rag/extract/types.js

@@ -0,0 +1,12 @@
+export const RAG_EXTRACT_ERROR_CODE = {
+    unsupportedFile: 'RAG_UNSUPPORTED_FILE',
+    extractionFailed: 'RAG_EXTRACTION_FAILED'
+};
+export class RagExtractError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.name = 'RagExtractError';
+        this.code = code;
+    }
+}

package/dist-lib/lib/core/files/rag/extract/utf.d.ts

@@ -0,0 +1,2 @@
+export declare function decodeUtf16(bytes: Uint8Array, littleEndian: boolean): string;
+export declare function decodeLatin1(bytes: Uint8Array): string;

package/dist-lib/lib/core/files/rag/extract/utf.js

@@ -0,0 +1,18 @@
+import b4a from 'b4a';
+export function decodeUtf16(bytes, littleEndian) {
+    let out = '';
+    const chunk = [];
+    for (let i = 0; i + 1 < bytes.length; i += 2) {
+        const a = bytes[i];
+        const b = bytes[i + 1];
+        chunk.push(littleEndian ? a | (b << 8) : (a << 8) | b);
+        if (chunk.length >= 8192) {
+            out += String.fromCharCode(...chunk);
+            chunk.length = 0;
+        }
+    }
+    return out + String.fromCharCode(...chunk);
+}
+export function decodeLatin1(bytes) {
+    return b4a.toString(b4a.from(bytes), 'latin1');
+}

package/dist-lib/lib/core/files/rag/index.d.ts

@@ -0,0 +1,54 @@
+import type { AbortSignal } from 'bare-abort-controller';
+import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
+import type { RagExtractors } from './extract/extract.ts';
+import type { VectorIndex } from './vector/types.ts';
+import { type IndexingEmbedder } from './produce.ts';
+import type { Mesh } from '../../mesh.ts';
+import type { FileDocument, FileProcessor } from '../processing/index.ts';
+export declare class RagProcessor implements FileProcessor {
+    readonly id = "rag";
+    readonly input = "knowledge";
+    readonly output = "blob";
+    readonly version: string;
+    private readonly _producer;
+    readonly load?: (document: FileDocument, signal: AbortSignal) => Promise<void>;
+    private readonly _index;
+    constructor(options: {
+        model: string;
+        embedder?: IndexingEmbedder;
+        index?: VectorIndex | null;
+        extractors?: RagExtractors;
+    });
+    applies(_document: QvacKnowledge): boolean;
+    canProduce(document: QvacKnowledge, host: {
+        mesh: Mesh;
+        deviceId: Buffer | null;
+    }): Promise<boolean>;
+    produce(document: FileDocument, signal: AbortSignal): Promise<void>;
+    close(): Promise<boolean>;
+}
+export { encodeRagArtifact } from './artifact/codec.ts';
+export type { RagArtifactPayload } from './artifact/types.ts';
+export { isSupportedRagDocument } from './extract/document-types.ts';
+export type { RagExtractors } from './extract/extract.ts';
+export { MAX_KNOWLEDGE_BYTES } from './limits.ts';
+export { canIndexChat } from './policy.ts';
+export type { IndexingEmbedder, IndexingProgress } from './produce.ts';
+export { citedByPhraseOverlap } from './retrieval/citation-match.ts';
+export { formatRagContext } from './retrieval/format.ts';
+export { formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit } from './retrieval/full-text.ts';
+export { RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, chooseRagNotice } from './retrieval/notices.ts';
+export { createRagSourceTagFilter } from './retrieval/source-tag-filter.ts';
+export { sourcesFromHits } from './retrieval/sources.ts';
+export type { RagSource } from './retrieval/sources.ts';
+export { WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from './retrieval/web-sources.ts';
+export { MemoryVectorIndex } from './vector/memory.ts';
+export type { VectorIndex } from './vector/types.ts';
+export { fullTextBudgetChars, fullTextHits } from './retrieval/full-text.ts';
+export { retrieveRagKnowledge } from './retrieval/knowledge.ts';
+export { searchRagChunks } from './retrieval/search.ts';
+export { referencesPendingFile } from './retrieval/target.ts';
+export type { RagDocumentRef } from './retrieval/target.ts';
+export type { VectorHit } from './vector/types.ts';
+export { isWholeDocumentQuery } from './retrieval/whole-document-query.ts';
+export { referencedDocuments } from './retrieval/target.ts';

package/dist-lib/lib/core/files/rag/index.js

@@ -0,0 +1,79 @@
+import ScopeLock from 'scope-lock';
+import { pipelineVersionFor } from "./artifact/id.js";
+import { AbortedError } from '@qvac/harness/race-abort';
+import { IndexingPass } from "./produce.js";
+import { HydratePass } from "./load.js";
+import { canIndexChat } from "./policy.js";
+export class RagProcessor {
+    id = 'rag';
+    input = 'knowledge';
+    output = 'blob';
+    version;
+    _producer;
+    load;
+    _index;
+    constructor(options) {
+        this.version = pipelineVersionFor(options.model);
+        this._index = options.index ?? null;
+        const lock = new ScopeLock();
+        const source = options.embedder;
+        const embedder = source && {
+            model: source.model,
+            embed: async (input) => {
+                await lock.lock();
+                try {
+                    if (input.signal?.aborted)
+                        throw new AbortedError();
+                    return await source.embed(input);
+                }
+                finally {
+                    lock.unlock();
+                }
+            }
+        };
+        this._producer = embedder ? new IndexingPass(embedder, options.extractors) : null;
+        if (this._index) {
+            const loader = new HydratePass({ ...options, index: this._index, embedder });
+            this.load = (document, signal) => loader.run(document, signal);
+        }
+    }
+    applies(_document) {
+        return true;
+    }
+    async canProduce(document, host) {
+        return (!!this._producer &&
+            !document.deletedAt &&
+            (await canIndexChat(host.mesh, host.deviceId, document.chatId)));
+    }
+    async produce(document, signal) {
+        await this._producer?.run(document, signal);
+    }
+    async close() {
+        try {
+            await this._index?.clear();
+            return true;
+        }
+        catch (err) {
+            logger.warn('rag', 'vector index clear failed on close; it must not be reused', { err });
+            return false;
+        }
+    }
+}
+export { encodeRagArtifact } from "./artifact/codec.js";
+export { isSupportedRagDocument } from "./extract/document-types.js";
+export { MAX_KNOWLEDGE_BYTES } from "./limits.js";
+export { canIndexChat } from "./policy.js";
+export { citedByPhraseOverlap } from "./retrieval/citation-match.js";
+export { formatRagContext } from "./retrieval/format.js";
+export { formatFullDocumentContext, isFullTextHit, isTruncatedFullTextHit } from "./retrieval/full-text.js";
+export { RAG_KNOWLEDGE_OFF_NOTICE, RAG_TRUNCATED_NOTICE, RAG_UNAVAILABLE_NOTICE, chooseRagNotice } from "./retrieval/notices.js";
+export { createRagSourceTagFilter } from "./retrieval/source-tag-filter.js";
+export { sourcesFromHits } from "./retrieval/sources.js";
+export { WEB_TOOLS, webSourcesFromToolResult, withoutDataChannel } from "./retrieval/web-sources.js";
+export { MemoryVectorIndex } from "./vector/memory.js";
+export { fullTextBudgetChars, fullTextHits } from "./retrieval/full-text.js";
+export { retrieveRagKnowledge } from "./retrieval/knowledge.js";
+export { searchRagChunks } from "./retrieval/search.js";
+export { referencesPendingFile } from "./retrieval/target.js";
+export { isWholeDocumentQuery } from "./retrieval/whole-document-query.js";
+export { referencedDocuments } from "./retrieval/target.js";

package/dist-lib/lib/core/files/rag/limits.d.ts

@@ -0,0 +1,2 @@
+export declare const MAX_KNOWLEDGE_BYTES: number;
+export declare const MAX_OFFICE_BYTES: number;

package/dist-lib/lib/core/files/rag/limits.js

@@ -0,0 +1,2 @@
+export const MAX_KNOWLEDGE_BYTES = 100 * 1024 * 1024;
+export const MAX_OFFICE_BYTES = 20 * 1024 * 1024;

package/dist-lib/lib/core/files/rag/load.d.ts

@@ -0,0 +1,28 @@
+import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
+import type { VectorIndex } from './vector/types.ts';
+import type { FileDocument } from '../processing/index.ts';
+import type { AbortSignal } from 'bare-abort-controller';
+import { type BuildEmbedder } from './build.ts';
+import type { RagExtractors } from './extract/extract.ts';
+export type HydratePassOptions = {
+    model: string;
+    index: VectorIndex;
+    embedder?: BuildEmbedder;
+    extractors?: RagExtractors;
+};
+export declare function pickArtifact(rows: QvacKnowledgeArtifact[], opts: {
+    model: string;
+    pinnedDimensions: number | null;
+    failed: (id: string) => string | undefined;
+    localCoreKey?: string;
+    skip: ReadonlySet<string>;
+}): QvacKnowledgeArtifact | null;
+export declare class HydratePass {
+    private readonly _model;
+    private readonly _index;
+    private readonly _embedder;
+    private readonly _extractors;
+    constructor(opts: HydratePassOptions);
+    run(document: FileDocument, signal: AbortSignal): Promise<void>;
+    private _rebuild;
+}

package/dist-lib/lib/core/files/rag/load.js

@@ -0,0 +1,250 @@
+import { decodeRagArtifact } from "./artifact/codec.js";
+import { pipelineVersionFor, contentHash } from "./artifact/id.js";
+import { compareBlobReferences } from "../store/index.js";
+import { buildArtifactPayload, EmbedderFailure } from "./build.js";
+import { EMBED_RETRY_LIMIT } from "./produce.js";
+export function pickArtifact(rows, opts) {
+    const candidates = rows.filter((row) => !opts.skip.has(row.id) &&
+        !sameReference(opts.failed(row.id), row.artifactBlobId, opts.localCoreKey) &&
+        row.embedderVersion.startsWith(`${opts.model}|`) &&
+        (opts.pinnedDimensions === null || row.dimensions === opts.pinnedDimensions));
+    if (candidates.length === 0)
+        return null;
+    const exactVersion = pipelineVersionFor(opts.model);
+    const exact = candidates.filter((row) => row.embedderVersion === exactVersion);
+    const pool = exact.length > 0 ? exact : candidates;
+    return pool.reduce((newest, row) => (row.createdAt > newest.createdAt ? row : newest));
+}
+export class HydratePass {
+    _model;
+    _index;
+    _embedder;
+    _extractors;
+    constructor(opts) {
+        this._model = opts.model;
+        this._index = opts.index;
+        this._embedder = opts.embedder;
+        this._extractors = opts.extractors;
+    }
+    async run(document, signal) {
+        const knowledge = document.record;
+        const local = document.state;
+        let entry = local.loaded;
+        if (knowledge.deletedAt) {
+            if (!entry)
+                return;
+            if (entry.evict)
+                await this._index.remove(entry.evict);
+            await this._index.remove(entry.artifactId);
+            local.loaded = undefined;
+            document.load({ state: 'pending' });
+            document.output(entry.artifactId, entry.blobRef ?? '').clear();
+            return;
+        }
+        const rows = await document.artifacts();
+        const localCoreKey = document.localCoreKey;
+        const skip = new Set();
+        for (;;) {
+            const pick = pickArtifact(rows, {
+                model: this._model,
+                pinnedDimensions: this._index.info().dimensions,
+                failed: (id) => document.readArtifact(id)?.failedReference,
+                localCoreKey,
+                skip
+            });
+            if (!pick) {
+                if (entry)
+                    document.load({ state: 'ready' }, entry.artifactId);
+                const repair = rows.find((row) => {
+                    const artifact = document.readArtifact(row.id);
+                    return (row.embedderVersion === pipelineVersionFor(this._model) &&
+                        (sameReference(artifact?.failedReference, row.artifactBlobId, localCoreKey) ||
+                            (sameReference(artifact?.unavailable?.blobRef, row.artifactBlobId, localCoreKey) &&
+                                artifact?.unavailable?.repeated)));
+                });
+                if (!entry && repair && signal)
+                    await this._rebuild(document, repair, signal);
+                return;
+            }
+            if (entry &&
+                entry.artifactId === pick.id &&
+                sameReference(entry.blobRef, pick.artifactBlobId, localCoreKey)) {
+                if (entry.evict) {
+                    await this._index.remove(entry.evict);
+                    local.loaded = { artifactId: entry.artifactId, blobRef: entry.blobRef };
+                }
+                document.load({ state: 'ready' }, entry.artifactId);
+                return;
+            }
+            const retryAt = document.output(pick.id, pick.artifactBlobId).blockedUntil();
+            if (retryAt !== null) {
+                document.schedule(retryAt);
+                skip.add(pick.id);
+                continue;
+            }
+            if (document.paused() || document.closed())
+                return;
+            const read = await document.output(pick.id, pick.artifactBlobId).read(signal);
+            if (read.kind === 'malformed') {
+                document.load({ state: 'failed', reason: 'invalid output reference' }, entry?.artifactId);
+                document.artifact(pick.id).failedReference = pick.artifactBlobId;
+                continue;
+            }
+            if (read.kind === 'missing') {
+                const artifact = document.artifact(pick.id);
+                artifact.unavailable = {
+                    blobRef: pick.artifactBlobId,
+                    repeated: sameReference(artifact.unavailable?.blobRef, pick.artifactBlobId, localCoreKey)
+                };
+                const { retryAt } = read;
+                document.schedule(retryAt);
+                document.load({
+                    state: 'pending',
+                    reason: 'output unavailable on this device',
+                    retryAt
+                }, entry?.artifactId);
+                skip.add(pick.id);
+                continue;
+            }
+            document.output(pick.id, pick.artifactBlobId).clear();
+            document.artifact(pick.id).unavailable = undefined;
+            let chunks;
+            try {
+                const payload = decodeRagArtifact(read.bytes);
+                if (payload.dimensions !== pick.dimensions || payload.modelName !== this._model) {
+                    throw new Error(`artifact payload is ${payload.modelName} at ${payload.dimensions} dimensions, ` +
+                        `the row says ${this._model} at ${pick.dimensions}`);
+                }
+                chunks = payload.chunks;
+            }
+            catch (err) {
+                document.artifact(pick.id).failedReference = pick.artifactBlobId;
+                logger.warn('rag', 'artifact undecodable, pinned for the session', { id: pick.id, err });
+                continue;
+            }
+            if (document.paused() || document.closed())
+                return;
+            const live = await document.current();
+            if (!live || live.deletedAt || signal?.aborted)
+                return;
+            try {
+                await this._index.upsert(pick.id, chunks);
+            }
+            catch (err) {
+                document.artifact(pick.id).failedReference = pick.artifactBlobId;
+                logger.warn('rag', 'artifact rejected by the vector index', { id: pick.id, err });
+                continue;
+            }
+            const evict = entry && entry.artifactId !== pick.id ? entry.artifactId : undefined;
+            local.loaded = { artifactId: pick.id, blobRef: pick.artifactBlobId, evict };
+            if (signal.aborted || document.closed())
+                return;
+            document.load({ state: 'ready' }, pick.id);
+            entry = undefined;
+            if (evict) {
+                await this._index.remove(evict);
+                local.loaded = { artifactId: pick.id, blobRef: pick.artifactBlobId };
+            }
+            logger.info('rag', 'hydrated knowledge into the vector index', {
+                artifactId: pick.id,
+                chunks: chunks.length
+            });
+            return;
+        }
+    }
+    async _rebuild(document, artifact, signal) {
+        const knowledge = document.record;
+        const local = document.artifact(artifact.id);
+        if (!this._embedder ||
+            (local.rebuildFailures ?? 0) > EMBED_RETRY_LIMIT ||
+            document.paused() ||
+            document.closed() ||
+            signal.aborted) {
+            return;
+        }
+        const retryAt = document.output(knowledge.id, knowledge.blobId).blockedUntil();
+        if (retryAt !== null) {
+            document.schedule(retryAt);
+            return;
+        }
+        const read = await document.output(knowledge.id, knowledge.blobId).read(signal);
+        if (read.kind === 'malformed') {
+            local.rebuildFailures = Infinity;
+            document.load({ state: 'failed', reason: 'invalid source reference' });
+            return;
+        }
+        if (read.kind !== 'bytes') {
+            document.schedule(read.retryAt);
+            document.load({
+                state: 'pending',
+                reason: 'source and output unavailable',
+                retryAt: read.retryAt
+            });
+            return;
+        }
+        try {
+            if (contentHash(read.bytes) !== artifact.sourceHash) {
+                throw new Error('source does not match the published output');
+            }
+            document.load({ state: 'running', reason: 'rebuilding local index' });
+            const payload = await buildArtifactPayload({
+                embedder: this._embedder,
+                extractors: this._extractors,
+                knowledge,
+                artifactId: artifact.id,
+                bytes: read.bytes,
+                signal,
+                onProgress: (processedChunks, totalChunks) => document.report({
+                    stage: 'embedding',
+                    processedChunks,
+                    totalChunks
+                })
+            });
+            const live = await document.current();
+            if (!live || live.deletedAt || signal.aborted || document.paused() || document.closed()) {
+                return;
+            }
+            if (payload.dimensions !== artifact.dimensions) {
+                throw new Error('rebuilt output has incompatible dimensions');
+            }
+            await this._index.upsert(artifact.id, payload.chunks);
+            document.state.loaded = {
+                artifactId: artifact.id,
+                blobRef: artifact.artifactBlobId
+            };
+            if (signal.aborted || document.closed())
+                return;
+            document.output(knowledge.id, knowledge.blobId).clear();
+            local.rebuildFailures = undefined;
+            document.load({ state: 'ready' }, artifact.id);
+            document.report({
+                stage: 'ready',
+                processedChunks: payload.chunks.length,
+                totalChunks: payload.chunks.length
+            });
+        }
+        catch (error) {
+            if (document.closed())
+                return;
+            if (signal.aborted || document.paused()) {
+                document.load({ state: 'pending', reason: 'processing interrupted' });
+                return;
+            }
+            const failures = error instanceof EmbedderFailure ? (local.rebuildFailures ?? 0) + 1 : Infinity;
+            local.rebuildFailures = failures;
+            const retryAt = failures <= EMBED_RETRY_LIMIT
+                ? document.output(knowledge.id, knowledge.blobId).miss()
+                : undefined;
+            if (retryAt)
+                document.schedule(retryAt);
+            document.load({
+                state: 'failed',
+                reason: error instanceof Error ? error.message : String(error),
+                retryAt
+            });
+        }
+    }
+}
+function sameReference(left, right, localCoreKey) {
+    return left !== undefined && (compareBlobReferences(left, right, localCoreKey) ?? left === right);
+}

package/dist-lib/lib/core/files/rag/models.d.ts

@@ -0,0 +1 @@
+export declare const DEFAULT_RAG_MODEL = "EMBEDDINGGEMMA_300M_Q4_0";

package/dist-lib/lib/core/files/rag/models.js

@@ -0,0 +1 @@
+export const DEFAULT_RAG_MODEL = 'EMBEDDINGGEMMA_300M_Q4_0';

package/dist-lib/lib/core/files/rag/policy.d.ts

@@ -0,0 +1,2 @@
+import type { Mesh } from '../../mesh.ts';
+export declare function canIndexChat(mesh: Mesh, deviceId: Buffer | null, chatId: string): Promise<boolean>;

package/dist-lib/lib/core/files/rag/policy.js

@@ -0,0 +1,23 @@
+import { toolsetEnabled } from '@qvac/harness/lib/tools/toolsets.ts';
+import { getAgent, isOrphaned } from "../../agent.js";
+import { getChat } from "../../chat.js";
+export async function canIndexChat(mesh, deviceId, chatId) {
+    if (!deviceId) {
+        return false;
+    }
+    const chat = await getChat(mesh, chatId);
+    if (!chat) {
+        return false;
+    }
+    const agent = await getAgent(mesh, chat.agentId);
+    if (!agent) {
+        return false;
+    }
+    if (!toolsetEnabled('knowledge', agent.disabledToolsets)) {
+        return false;
+    }
+    if (await isOrphaned(mesh, agent)) {
+        return true;
+    }
+    return agent.providerDeviceId.equals(deviceId);
+}

package/dist-lib/lib/core/files/rag/produce.d.ts

@@ -0,0 +1,21 @@
+import type { AbortSignal } from 'bare-abort-controller';
+import type { RagExtractors } from './extract/extract.ts';
+import { type BuildEmbedder } from './build.ts';
+import { type FileDocument } from '../processing/index.ts';
+export interface IndexingEmbedder extends BuildEmbedder {
+}
+export type IndexingProgress = (update: {
+    knowledgeId: string;
+    chatId: string;
+    stage: 'embedding' | 'ready' | 'failed';
+    processedChunks: number;
+    totalChunks: number;
+    error?: string;
+}) => void;
+export declare const EMBED_RETRY_LIMIT = 5;
+export declare class IndexingPass {
+    private readonly _embedder;
+    private readonly _extractors;
+    constructor(embedder: IndexingEmbedder, extractors?: RagExtractors);
+    run(document: FileDocument, signal: AbortSignal): Promise<void>;
+}