@qvac/core 0.0.0 → 0.1.0

package/dist-lib/lib/core/files/rag/retrieval/whole-document-query.js

@@ -0,0 +1,96 @@
+const SUMMARIZE_MARKERS = [
+    /\bsummar(?:ize|ise|y|ies)\b/,
+    /\btldr\b/,
+    /\boverview\b/,
+    /\bkey (?:points|takeaways)\b/,
+    /\bmain (?:points|ideas)\b/,
+    /\bgist\b/,
+    /\brecap\b/,
+    /\bwhat(?:'s| is| are)? (?:this|these|it) about\b/,
+    /\bwhat(?:'s| is| are) (?:in|on)\b/,
+    /\bwhat (?:do|does)\b.*\bcontain\b/
+];
+const FILLER = new Set([
+    'a',
+    'about',
+    'all',
+    'an',
+    'and',
+    'attached',
+    'brief',
+    'briefly',
+    'can',
+    'concise',
+    'content',
+    'contents',
+    'contain',
+    'do',
+    'does',
+    'doc',
+    'docs',
+    'document',
+    'documents',
+    'file',
+    'files',
+    'for',
+    'give',
+    'in',
+    'is',
+    'it',
+    'its',
+    'me',
+    'my',
+    'of',
+    'on',
+    'pdf',
+    'please',
+    'provide',
+    'short',
+    'some',
+    'the',
+    'their',
+    'them',
+    'these',
+    'this',
+    'those',
+    'to',
+    'up',
+    'us',
+    'what',
+    'whole',
+    'you',
+    'your'
+]);
+const MARKER_WORDS = new Set([
+    'are',
+    'dr',
+    'gist',
+    'ideas',
+    'key',
+    'main',
+    'overview',
+    'points',
+    'recap',
+    'summaries',
+    'summarise',
+    'summarize',
+    'summary',
+    'takeaways',
+    'tl',
+    'tldr'
+]);
+const CLAUSE_BOUNDARY = /[.!?;\n]+|\band\b|\balso\b|\bthen\b/;
+export function isWholeDocumentQuery(query) {
+    const normalized = query
+        .toLowerCase()
+        .replace(/\bwhat's\b/g, 'what is')
+        .replace(/\btl[;:\s-]?dr\b/g, 'tldr')
+        .replace(/[_-]+/g, ' ');
+    return normalized.split(CLAUSE_BOUNDARY).some(isBareSummarizeClause);
+}
+function isBareSummarizeClause(clause) {
+    if (!SUMMARIZE_MARKERS.some((marker) => marker.test(clause)))
+        return false;
+    const words = clause.match(/[a-z0-9]+/g) ?? [];
+    return words.every((word) => FILLER.has(word) || MARKER_WORDS.has(word));
+}

package/dist-lib/lib/core/files/rag/vector/memory.d.ts

@@ -0,0 +1,15 @@
+import type { VectorCandidate, VectorChunk, VectorHit, VectorIndex, VectorIndexInfo } from './types.ts';
+export declare class MemoryVectorIndex implements VectorIndex {
+    private readonly _artifacts;
+    private _chunks;
+    private _dimensions;
+    info(): VectorIndexInfo;
+    upsert(artifactId: string, chunks: ReadonlyArray<VectorChunk>): Promise<void>;
+    remove(artifactId: string): Promise<void>;
+    search(embedding: ReadonlyArray<number>, opts: {
+        readonly limit: number;
+        readonly filter?: (candidate: VectorCandidate) => boolean;
+    }): Promise<ReadonlyArray<VectorHit>>;
+    clear(): Promise<void>;
+    private _pack;
+}

package/dist-lib/lib/core/files/rag/vector/memory.js

@@ -0,0 +1,135 @@
+export class MemoryVectorIndex {
+    _artifacts = new Map();
+    _chunks = 0;
+    _dimensions = null;
+    info() {
+        return { artifacts: this._artifacts.size, chunks: this._chunks, dimensions: this._dimensions };
+    }
+    upsert(artifactId, chunks) {
+        const packed = this._pack(chunks);
+        const prev = this._artifacts.get(artifactId);
+        if (prev)
+            this._chunks -= prev.ids.length;
+        this._artifacts.set(artifactId, packed.artifact);
+        this._chunks += packed.artifact.ids.length;
+        if (packed.dimensions !== null && this._dimensions === null) {
+            this._dimensions = packed.dimensions;
+        }
+        if (this._chunks === 0)
+            this._dimensions = null;
+        return Promise.resolve();
+    }
+    remove(artifactId) {
+        const prev = this._artifacts.get(artifactId);
+        if (!prev)
+            return Promise.resolve();
+        this._artifacts.delete(artifactId);
+        this._chunks -= prev.ids.length;
+        if (this._chunks === 0)
+            this._dimensions = null;
+        return Promise.resolve();
+    }
+    search(embedding, opts) {
+        if (this._chunks === 0 || opts.limit <= 0)
+            return Promise.resolve([]);
+        if (this._dimensions !== null && embedding.length !== this._dimensions) {
+            throw new Error(`vector index: query has ${embedding.length} dimensions, the index holds ${this._dimensions}`);
+        }
+        const dimensions = embedding.length;
+        const query = new Float64Array(dimensions);
+        let sum = 0;
+        for (let d = 0; d < dimensions; d++) {
+            const value = embedding[d];
+            if (!Number.isFinite(value))
+                throw new Error('vector index: query embedding is not finite');
+            query[d] = value;
+            sum += value * value;
+        }
+        const queryNorm = Math.sqrt(sum);
+        const scored = [];
+        for (const artifact of this._artifacts.values()) {
+            for (let i = 0; i < artifact.ids.length; i++) {
+                if (opts.filter) {
+                    const candidate = {
+                        id: artifact.ids[i],
+                        content: artifact.contents[i],
+                        metadataJson: artifact.metadataJsons[i]
+                    };
+                    if (!opts.filter(candidate))
+                        continue;
+                }
+                let dot = 0;
+                const base = i * dimensions;
+                for (let d = 0; d < dimensions; d++)
+                    dot += artifact.vectors[base + d] * query[d];
+                const denominator = artifact.norms[i] * queryNorm;
+                scored.push({
+                    id: artifact.ids[i],
+                    content: artifact.contents[i],
+                    metadataJson: artifact.metadataJsons[i],
+                    cosine: denominator > 0 ? dot / denominator : 0
+                });
+            }
+        }
+        scored.sort((a, b) => b.cosine - a.cosine);
+        const results = scored.slice(0, opts.limit).map((entry) => ({
+            id: entry.id,
+            content: entry.content,
+            metadataJson: entry.metadataJson,
+            score: Math.max(0, entry.cosine)
+        }));
+        return Promise.resolve(results);
+    }
+    clear() {
+        this._artifacts.clear();
+        this._chunks = 0;
+        this._dimensions = null;
+        return Promise.resolve();
+    }
+    _pack(chunks) {
+        if (chunks.length === 0) {
+            return {
+                artifact: {
+                    ids: [],
+                    contents: [],
+                    metadataJsons: [],
+                    vectors: new Float32Array(0),
+                    norms: new Float64Array(0)
+                },
+                dimensions: null
+            };
+        }
+        const dimensions = chunks[0].embedding.length;
+        if (dimensions === 0)
+            throw new Error('vector index: chunk embedding is empty');
+        if (this._dimensions !== null && dimensions !== this._dimensions) {
+            throw new Error(`vector index: artifact has ${dimensions} dimensions, the index holds ${this._dimensions}`);
+        }
+        const ids = [];
+        const contents = [];
+        const metadataJsons = [];
+        const vectors = new Float32Array(chunks.length * dimensions);
+        const norms = new Float64Array(chunks.length);
+        for (let i = 0; i < chunks.length; i++) {
+            const chunk = chunks[i];
+            if (chunk.embedding.length !== dimensions) {
+                throw new Error(`vector index: chunk ${chunk.id} disagrees on dimensions`);
+            }
+            let sum = 0;
+            for (let d = 0; d < dimensions; d++) {
+                const value = chunk.embedding[d];
+                if (!Number.isFinite(value)) {
+                    throw new Error(`vector index: chunk ${chunk.id} embedding is not finite`);
+                }
+                const stored = Math.fround(value);
+                vectors[i * dimensions + d] = stored;
+                sum += stored * stored;
+            }
+            norms[i] = Math.sqrt(sum);
+            ids.push(chunk.id);
+            contents.push(chunk.content);
+            metadataJsons.push(chunk.metadataJson);
+        }
+        return { artifact: { ids, contents, metadataJsons, vectors, norms }, dimensions };
+    }
+}

package/dist-lib/lib/core/files/rag/vector/types.d.ts

@@ -0,0 +1,28 @@
+export type VectorChunk = {
+    readonly id: string;
+    readonly content: string;
+    readonly metadataJson: string;
+    readonly embedding: ReadonlyArray<number>;
+};
+export type VectorHit = {
+    readonly id: string;
+    readonly content: string;
+    readonly metadataJson: string;
+    readonly score: number;
+};
+export type VectorCandidate = Omit<VectorHit, 'score'>;
+export type VectorIndexInfo = {
+    readonly artifacts: number;
+    readonly chunks: number;
+    readonly dimensions: number | null;
+};
+export interface VectorIndex {
+    info(): VectorIndexInfo;
+    upsert(artifactId: string, chunks: ReadonlyArray<VectorChunk>): Promise<void>;
+    remove(artifactId: string): Promise<void>;
+    search(embedding: ReadonlyArray<number>, opts: {
+        readonly limit: number;
+        readonly filter?: (candidate: VectorCandidate) => boolean;
+    }): Promise<ReadonlyArray<VectorHit>>;
+    clear(): Promise<void>;
+}

package/dist-lib/lib/core/files/rag/vector/types.js

@@ -0,0 +1 @@
+export {};

package/dist-lib/lib/core/files/records/chunks.d.ts

@@ -0,0 +1,3 @@
+import type { QvacChunk } from '../../../../spec/mesh/hyperschema/types';
+import type { Mesh } from '../../mesh';
+export declare function syncAttachmentChunk(mesh: Mesh, chunk: QvacChunk): Promise<void>;

package/dist-lib/lib/core/files/records/chunks.js

@@ -0,0 +1,12 @@
+import { hasLiveSource } from "./routing.js";
+export async function syncAttachmentChunk(mesh, chunk) {
+    if (chunk.type !== 'attachment')
+        return;
+    const attachment = chunk.attachment;
+    if (!attachment?.deletedAt || typeof attachment.blobId !== 'string')
+        return;
+    if (await hasLiveSource(mesh, chunk.chatId, attachment.blobId))
+        return;
+    logger.debug('sync', 'attachment tombstone observed — reclaiming blob bytes', { id: chunk.id });
+    await mesh.files.clear(attachment.blobId, chunk.id);
+}

package/dist-lib/lib/core/files/records/index.d.ts

@@ -0,0 +1,10 @@
+export { addKnowledge, addKnowledgeArtifact, assertRemovableAt, findKnowledgeArtifactsByKnowledge, getKnowledge, getKnowledgeArtifact, listKnowledge, listKnowledgeArtifactsByKnowledge, removeKnowledge } from './knowledge.ts';
+export type { AddKnowledgeInput, AddKnowledgeArtifactInput } from './knowledge.ts';
+export { attachTurnDocumentIds, hasLiveSource, isKnowledgeRoutedAttachment, routeAttachmentToKnowledge } from './routing.ts';
+export { groupAttachmentsByMessage } from './linkage.ts';
+export { syncAttachmentChunk } from './chunks.ts';
+export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from './reclaim.ts';
+export { referencedBlobs } from './sweep.ts';
+export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from './output.ts';
+export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from './source.ts';
+export type { FileInput, FileRecord } from './source.ts';

package/dist-lib/lib/core/files/records/index.js

@@ -0,0 +1,8 @@
+export { addKnowledge, addKnowledgeArtifact, assertRemovableAt, findKnowledgeArtifactsByKnowledge, getKnowledge, getKnowledgeArtifact, listKnowledge, listKnowledgeArtifactsByKnowledge, removeKnowledge } from "./knowledge.js";
+export { attachTurnDocumentIds, hasLiveSource, isKnowledgeRoutedAttachment, routeAttachmentToKnowledge } from "./routing.js";
+export { groupAttachmentsByMessage } from "./linkage.js";
+export { syncAttachmentChunk } from "./chunks.js";
+export { sweepTombstonedBlobs, syncArtifactArrival, syncKnowledgeTombstone } from "./reclaim.js";
+export { referencedBlobs } from "./sweep.js";
+export { assertArtifactOutput, isProcessorOutput, processorArtifactId, isProcessorArtifactId } from "./output.js";
+export { attachmentRecord, fileRecord, fileIds, scopedFile, claimSource } from "./source.js";

package/dist-lib/lib/core/files/records/knowledge.d.ts

@@ -0,0 +1,22 @@
+import type { QvacKnowledge, QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
+import type { Mesh } from '../../mesh.ts';
+export type AddKnowledgeInput = Pick<QvacKnowledge, 'chatId' | 'fileName' | 'mimeType' | 'byteLength' | 'blobId'> & Partial<Pick<QvacKnowledge, 'id'>>;
+export declare function getKnowledge(store: Mesh, id: string): Promise<QvacKnowledge | null>;
+export declare function listKnowledge(store: Mesh, { chatId }: {
+    chatId: string;
+}): Promise<QvacKnowledge[]>;
+export declare function addKnowledge(store: Mesh, input: AddKnowledgeInput): Promise<QvacKnowledge>;
+export declare function assertRemovableAt(at: number): void;
+export declare function removeKnowledge(store: Mesh, { knowledgeId, at }: {
+    knowledgeId: string;
+    at: number;
+}): Promise<QvacKnowledge>;
+export type AddKnowledgeArtifactInput = Omit<QvacKnowledgeArtifact, 'createdAt'>;
+export declare function getKnowledgeArtifact(store: Mesh, id: string): Promise<QvacKnowledgeArtifact | null>;
+type KnowledgeSnapshot = ReturnType<Mesh['view']['snapshot']>;
+export declare function listKnowledgeArtifactsByKnowledge(store: Mesh, { knowledgeId }: {
+    knowledgeId: string;
+}): Promise<QvacKnowledgeArtifact[]>;
+export declare function findKnowledgeArtifactsByKnowledge(snap: KnowledgeSnapshot, knowledgeId: string): Promise<QvacKnowledgeArtifact[]>;
+export declare function addKnowledgeArtifact(store: Mesh, input: AddKnowledgeArtifactInput): Promise<QvacKnowledgeArtifact>;
+export {};

package/dist-lib/lib/core/files/records/knowledge.js

@@ -0,0 +1,105 @@
+import { randomId } from "../../id.js";
+import { assertArtifactOutput } from "./output.js";
+const KNOWLEDGE = '@qvac/knowledge';
+const ACTIVE_KNOWLEDGE_BY_CHAT = '@qvac/active-knowledge-by-chat';
+const KNOWLEDGE_ARTIFACTS = '@qvac/knowledge-artifacts';
+const KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE = '@qvac/knowledge-artifacts-by-knowledge';
+export async function getKnowledge(store, id) {
+    const snap = store.view.snapshot();
+    try {
+        const row = await snap.get(KNOWLEDGE, { id });
+        return row;
+    }
+    finally {
+        await snap.close();
+    }
+}
+async function mustGetKnowledge(store, id) {
+    const row = await getKnowledge(store, id);
+    if (!row)
+        throw new Error(`knowledge not found: ${id}`);
+    return row;
+}
+export async function listKnowledge(store, { chatId }) {
+    const snap = store.view.snapshot();
+    try {
+        const rows = await snap
+            .find(ACTIVE_KNOWLEDGE_BY_CHAT, { gte: { chatId }, lte: { chatId } })
+            .toArray();
+        return rows;
+    }
+    finally {
+        await snap.close();
+    }
+}
+export async function addKnowledge(store, input) {
+    const now = Date.now();
+    const knowledge = {
+        ...input,
+        id: input.id ?? randomId(),
+        createdAt: now,
+        updatedAt: now
+    };
+    await store.dispatch('add-knowledge', { knowledge });
+    return await mustGetKnowledge(store, knowledge.id);
+}
+export function assertRemovableAt(at) {
+    if (!at || at < 0) {
+        throw new Error(`remove-knowledge needs a positive timestamp, got ${at}`);
+    }
+}
+export async function removeKnowledge(store, { knowledgeId, at }) {
+    assertRemovableAt(at);
+    const existing = await mustGetKnowledge(store, knowledgeId);
+    if (existing.deletedAt)
+        return existing;
+    await store.dispatch('remove-knowledge', { knowledgeId, at });
+    const removed = await mustGetKnowledge(store, knowledgeId);
+    return removed;
+}
+const ARTIFACT_STRING_FIELDS = [
+    'id',
+    'knowledgeId',
+    'sourceHash',
+    'embedderVersion',
+    'artifactBlobId'
+];
+export async function getKnowledgeArtifact(store, id) {
+    const snap = store.view.snapshot();
+    try {
+        const artifact = await snap.get(KNOWLEDGE_ARTIFACTS, { id });
+        return artifact;
+    }
+    finally {
+        await snap.close();
+    }
+}
+export async function listKnowledgeArtifactsByKnowledge(store, { knowledgeId }) {
+    const snap = store.view.snapshot();
+    try {
+        return await findKnowledgeArtifactsByKnowledge(snap, knowledgeId);
+    }
+    finally {
+        await snap.close();
+    }
+}
+export async function findKnowledgeArtifactsByKnowledge(snap, knowledgeId) {
+    return await snap
+        .find(KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, {
+        gte: { knowledgeId },
+        lte: { knowledgeId }
+    })
+        .toArray();
+}
+export async function addKnowledgeArtifact(store, input) {
+    for (const field of ARTIFACT_STRING_FIELDS) {
+        if (!input[field])
+            throw new Error(`knowledge-artifact needs a non-empty ${field}`);
+    }
+    assertArtifactOutput(input);
+    await store.dispatch('add-knowledge-artifact', { artifact: { ...input, createdAt: Date.now() } });
+    const row = await getKnowledgeArtifact(store, input.id);
+    if (!row)
+        throw new Error(`knowledge-artifact not found after add: ${input.id}`);
+    return row;
+}

package/dist-lib/lib/core/files/records/linkage.d.ts

@@ -0,0 +1,11 @@
+export declare const REPLY_MESSAGE_PREFIX = "reply-";
+export declare function replyMessageId(runId: string): string;
+interface LinkableChunk {
+    readonly id: string;
+    readonly runId?: string | null;
+    readonly seq: number;
+    readonly attachment?: unknown;
+    readonly userMessage?: unknown;
+}
+export declare function groupAttachmentsByMessage<T extends LinkableChunk>(attachments: ReadonlyArray<T>, userMessages: ReadonlyArray<T>): Map<string, T[]>;
+export {};

package/dist-lib/lib/core/files/records/linkage.js

@@ -0,0 +1,27 @@
+export const REPLY_MESSAGE_PREFIX = 'reply-';
+export function replyMessageId(runId) {
+    return `${REPLY_MESSAGE_PREFIX}${runId}`;
+}
+export function groupAttachmentsByMessage(attachments, userMessages) {
+    const groups = new Map();
+    const push = (messageId, chunk) => {
+        const list = groups.get(messageId);
+        if (list)
+            list.push(chunk);
+        else
+            groups.set(messageId, [chunk]);
+    };
+    let boundary = 0;
+    for (const attachment of attachments) {
+        if (attachment.runId) {
+            push(replyMessageId(attachment.runId), attachment);
+            continue;
+        }
+        while (boundary < userMessages.length && userMessages[boundary].seq <= attachment.seq) {
+            boundary++;
+        }
+        if (boundary < userMessages.length)
+            push(userMessages[boundary].id, attachment);
+    }
+    return groups;
+}

package/dist-lib/lib/core/files/records/output.d.ts

@@ -0,0 +1,8 @@
+import type { QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types.d.ts';
+export declare function processorArtifactId({ processorId, hash }: {
+    processorId: string;
+    hash: string;
+}): string;
+export declare function isProcessorArtifactId(id: string): boolean;
+export declare function isProcessorOutput(version: string): boolean;
+export declare function assertArtifactOutput(output: Pick<QvacKnowledgeArtifact, 'embedderVersion' | 'dimensions' | 'artifactBlobId'>): void;

package/dist-lib/lib/core/files/records/output.js

@@ -0,0 +1,38 @@
+import { MAX_REFERENCE_CHARS, parseBlobReference } from "../store/index.js";
+const PROCESSOR_ID_SEPARATOR = ':';
+export function processorArtifactId({ processorId, hash }) {
+    return `${processorId}${PROCESSOR_ID_SEPARATOR}${hash}`;
+}
+export function isProcessorArtifactId(id) {
+    return id.includes(PROCESSOR_ID_SEPARATOR);
+}
+export function isProcessorOutput(version) {
+    return version.startsWith('processor:') && !version.includes('|');
+}
+export function assertArtifactOutput(output) {
+    const processor = isProcessorOutput(output.embedderVersion);
+    if (processor
+        ? output.dimensions !== 0
+        : !Number.isInteger(output.dimensions) || output.dimensions <= 0) {
+        throw new Error(processor
+            ? 'processor output needs zero dimensions'
+            : `knowledge-artifact needs positive integer dimensions, got ${output.dimensions}`);
+    }
+    if (output.artifactBlobId.length > MAX_REFERENCE_CHARS) {
+        throw new Error('inline output exceeds 512 KiB');
+    }
+    let value;
+    try {
+        value = JSON.parse(output.artifactBlobId);
+    }
+    catch {
+        if (!processor)
+            return;
+    }
+    const inline = value !== null && typeof value === 'object' && 'inline' in value;
+    if (!processor && !inline)
+        return;
+    const parsed = parseBlobReference(output.artifactBlobId);
+    if (parsed.inline && !processor)
+        throw new Error('inline outputs need a processor identity');
+}

package/dist-lib/lib/core/files/records/reclaim.d.ts

@@ -0,0 +1,5 @@
+import type { QvacKnowledge, QvacKnowledgeArtifact } from '../../../../spec/mesh/hyperschema/types';
+import type { Mesh } from '../../mesh.ts';
+export declare function syncKnowledgeTombstone(mesh: Mesh, row: QvacKnowledge): Promise<void>;
+export declare function syncArtifactArrival(mesh: Mesh, row: QvacKnowledgeArtifact): Promise<void>;
+export declare function sweepTombstonedBlobs(mesh: Mesh, deviceId?: Buffer | null): Promise<void>;

package/dist-lib/lib/core/files/records/reclaim.js

@@ -0,0 +1,73 @@
+import { hasLiveSource } from "./routing.js";
+import { removeChatAttachments, removeChatKnowledge } from "../../chat.js";
+import { listChunks } from "../../chunk.js";
+import { getKnowledge, listKnowledgeArtifactsByKnowledge } from "./knowledge.js";
+import { syncAttachmentChunk } from "./chunks.js";
+export async function syncKnowledgeTombstone(mesh, row) {
+    if (!row.deletedAt)
+        return;
+    logger.debug('sync', 'knowledge tombstone observed - reclaiming blob bytes', { id: row.id });
+    if (typeof row.blobId === 'string' && !(await hasLiveSource(mesh, row.chatId, row.blobId))) {
+        await mesh.files.clear(row.blobId, row.id);
+    }
+    for (const artifact of await listKnowledgeArtifactsByKnowledge(mesh, { knowledgeId: row.id })) {
+        if (mesh.closing)
+            return;
+        await mesh.files.clear(artifact.artifactBlobId, artifact.id);
+    }
+}
+export async function syncArtifactArrival(mesh, row) {
+    const knowledge = await getKnowledge(mesh, row.knowledgeId);
+    if (!knowledge?.deletedAt)
+        return;
+    await mesh.files.clear(row.artifactBlobId, row.id);
+}
+export async function sweepTombstonedBlobs(mesh, deviceId = null) {
+    if (deviceId && mesh.writable) {
+        const chats = mesh.view.snapshot();
+        try {
+            for await (const chat of chats.find('@qvac/chats')) {
+                if (mesh.closing)
+                    return;
+                if (!chat.deletedAt)
+                    continue;
+                await removeChatAttachments(mesh, deviceId, chat.id, chat.deletedAt);
+                await removeChatKnowledge(mesh, chat.id, chat.deletedAt);
+            }
+        }
+        catch (error) {
+            logger.warn('sync', 'chat-tombstone sweep pass failed - continuing with local reclaims', {
+                error
+            });
+        }
+        finally {
+            await chats.close();
+        }
+    }
+    const knowledge = mesh.view.snapshot();
+    try {
+        for await (const row of knowledge.find('@qvac/knowledge')) {
+            if (mesh.closing)
+                return;
+            await syncKnowledgeTombstone(mesh, row);
+        }
+    }
+    finally {
+        await knowledge.close();
+    }
+    const counters = mesh.view.snapshot();
+    try {
+        for await (const counter of counters.find('@qvac/chunk-counters')) {
+            if (mesh.closing)
+                return;
+            for (const chunk of await listChunks(mesh, { chatId: counter.chatId, type: 'attachment' })) {
+                if (mesh.closing)
+                    return;
+                await syncAttachmentChunk(mesh, chunk);
+            }
+        }
+    }
+    finally {
+        await counters.close();
+    }
+}

package/dist-lib/lib/core/files/records/routing.d.ts

@@ -0,0 +1,23 @@
+import type { QvacChunkContentAttachment } from '../../../../spec/mesh/hyperschema/types.d.ts';
+import type { Mesh } from '../../mesh.ts';
+export declare function isKnowledgeRoutedAttachment(input: {
+    readonly fileName: string;
+    readonly mimeType: string;
+}): boolean;
+export declare function routeAttachmentToKnowledge(mesh: Mesh, chatId: string, attachment: QvacChunkContentAttachment): Promise<import("../../../../spec/index").QvacKnowledge>;
+export declare function hasLiveSource(mesh: Mesh, chatId: string, reference: string): Promise<boolean>;
+interface AttachTurnChunk {
+    readonly id: string;
+    readonly seq: number;
+    readonly runId?: string | null;
+    readonly type?: string | null;
+    readonly attachment?: {
+        readonly id?: string | null;
+        readonly fileName?: string | null;
+        readonly mimeType?: string | null;
+        readonly deletedAt?: number | null;
+    } | null;
+    readonly userMessage?: unknown;
+}
+export declare function attachTurnDocumentIds(chunks: ReadonlyArray<AttachTurnChunk>): string[];
+export {};