@ontos-ai/knowhere-claw 0.2.3 → 0.2.5

package/dist/parser.js

@@ -3,11 +3,9 @@ import fs from "node:fs/promises";
 import path from "node:path";
 import { createHash } from "node:crypto";
 import { strFromU8, unzipSync } from "fflate";
+//#region src/parser.ts
 const CHUNKS_FILE_NAME = "chunks.json";
-const FULL_MARKDOWN_FILE_NAME = "full.md";
-const HIERARCHY_FILE_NAME = "hierarchy.json";
-const HIERARCHY_VIEW_FILE_NAME = "hierarchy_view.html";
-const KB_CSV_FILE_NAME = "kb.csv";
+const LEGACY_RESULT_DIRECTORY_NAME = "result";
 const MANIFEST_FILE_NAME = "manifest.json";
 function readZipText(entries, fileName) {
 	const entry = entries[fileName];
@@ -16,6 +14,15 @@ function readZipText(entries, fileName) {
 async function ensureDir(targetPath) {
 	await fs.mkdir(targetPath, { recursive: true });
 }
+async function pathExists(targetPath) {
+	try {
+		await fs.access(targetPath);
+		return true;
+	} catch (error) {
+		if (isNodeError(error) && error.code === "ENOENT") return false;
+		throw error;
+	}
+}
 async function readTextFile(targetPath) {
 	try {
 		return await fs.readFile(targetPath, "utf-8");
@@ -39,36 +46,6 @@ function resolveResultEntryPath(rootDir, entryPath) {
 function normalizeRelativePath(value) {
 	return value.replace(/\\/g, "/").replace(/^\/+/, "");
 }
-function normalizeStoredPath(value) {
-	if (typeof value !== "string") return null;
-	return value.trim() || null;
-}
-function readChunkNumber(rawChunk, metadata, key) {
-	const metadataValue = metadata[key];
-	if (typeof metadataValue === "number" && Number.isFinite(metadataValue)) return metadataValue;
-	const rawValue = rawChunk[key];
-	if (typeof rawValue === "number" && Number.isFinite(rawValue)) return rawValue;
-	return null;
-}
-function readChunkStringArray(rawChunk, metadata, key) {
-	const metadataValue = metadata[key];
-	if (Array.isArray(metadataValue)) return metadataValue.filter((entry) => typeof entry === "string");
-	const rawValue = rawChunk[key];
-	if (Array.isArray(rawValue)) return rawValue.filter((entry) => typeof entry === "string");
-	return [];
-}
-function readChunkArray(rawChunk, metadata, key) {
-	const metadataValue = metadata[key];
-	if (Array.isArray(metadataValue)) return metadataValue;
-	const rawValue = rawChunk[key];
-	if (Array.isArray(rawValue)) return rawValue;
-	return [];
-}
-function extractAssetFilePath(rawChunk, metadata) {
-	const candidates = [rawChunk.file_path, metadata.file_path];
-	for (const candidate of candidates) if (typeof candidate === "string" && candidate.trim()) return normalizeRelativePath(candidate.trim());
-	return null;
-}
 function parseRawChunks(value) {
 	if (Array.isArray(value)) return value.filter((entry) => isRecord(entry));
 	if (isRecord(value) && Array.isArray(value.chunks)) return value.chunks.filter((entry) => isRecord(entry));
@@ -77,24 +54,6 @@ function parseRawChunks(value) {
 function parseManifest(value) {
 	return isRecord(value) ? value : {};
 }
-function buildChunk(rawChunk) {
-	const metadata = isRecord(rawChunk.metadata) ? rawChunk.metadata : {};
-	const type = rawChunk.type === "image" || rawChunk.type === "table" || rawChunk.type === "text" ? rawChunk.type : "text";
-	return {
-		chunkId: typeof rawChunk.chunk_id === "string" ? rawChunk.chunk_id : "",
-		type,
-		path: normalizeStoredPath(rawChunk.path),
-		summary: typeof metadata.summary === "string" ? metadata.summary : typeof rawChunk.summary === "string" ? rawChunk.summary : "",
-		content: typeof rawChunk.content === "string" ? rawChunk.content : "",
-		tokens: readChunkNumber(rawChunk, metadata, "tokens"),
-		keywords: readChunkStringArray(rawChunk, metadata, "keywords"),
-		relationships: readChunkArray(rawChunk, metadata, "relationships"),
-		metadata,
-		assetFilePath: extractAssetFilePath(rawChunk, metadata),
-		originalName: typeof metadata.original_name === "string" ? metadata.original_name : typeof rawChunk.original_name === "string" ? rawChunk.original_name : null,
-		tableType: typeof metadata.table_type === "string" ? metadata.table_type : typeof rawChunk.table_type === "string" ? rawChunk.table_type : null
-	};
-}
 function normalizeStatistics(manifest, rawChunks) {
 	if (manifest.statistics) return manifest.statistics;
 	return {
@@ -110,185 +69,6 @@ function validateKnowhereResultChecksum(zipBuffer, manifest) {
 	if (typeof checksum !== "string" || !checksum) return;
 	if (createHash("sha256").update(zipBuffer).digest("hex") !== checksum) throw new Error("Knowhere result ZIP checksum mismatch.");
 }
-function tokenizeStoredPath(storedPath) {
-	const slashSegments = storedPath.split("/").map((segment) => segment.trim()).filter(Boolean);
-	const tokens = [];
-	for (const slashSegment of slashSegments) {
-		const arrowSegments = slashSegment.split("-->").map((segment) => segment.trim()).filter(Boolean);
-		if (arrowSegments.length === 0) continue;
-		tokens.push({
-			delimiter: tokens.length === 0 ? null : "/",
-			segment: arrowSegments[0] || ""
-		});
-		for (const arrowSegment of arrowSegments.slice(1)) tokens.push({
-			delimiter: "-->",
-			segment: arrowSegment
-		});
-	}
-	return tokens;
-}
-function buildStoredPathPrefixes(storedPath) {
-	const tokens = tokenizeStoredPath(storedPath);
-	const prefixes = [];
-	let currentPath = "";
-	for (const token of tokens) {
-		currentPath = token.delimiter ? `${currentPath}${token.delimiter}${token.segment}` : token.segment;
-		prefixes.push(currentPath);
-	}
-	return prefixes;
-}
-function ensurePathAccumulator(accumulators, pathValue, parentPath, depth) {
-	const existing = accumulators.get(pathValue);
-	if (existing) {
-		if (parentPath && !existing.parentPath) existing.parentPath = parentPath;
-		return existing;
-	}
-	const next = {
-		childPaths: /* @__PURE__ */ new Set(),
-		chunkCount: 0,
-		chunkIds: [],
-		depth,
-		directChunkCount: 0,
-		imageChunkCount: 0,
-		parentPath,
-		path: pathValue,
-		tableChunkCount: 0,
-		textChunkCount: 0
-	};
-	accumulators.set(pathValue, next);
-	return next;
-}
-function incrementPathCounters(accumulator, chunkType) {
-	accumulator.chunkCount += 1;
-	if (chunkType === "image") {
-		accumulator.imageChunkCount += 1;
-		return;
-	}
-	if (chunkType === "table") {
-		accumulator.tableChunkCount += 1;
-		return;
-	}
-	accumulator.textChunkCount += 1;
-}
-function buildPathRecords(chunks) {
-	const accumulators = /* @__PURE__ */ new Map();
-	for (const chunk of chunks) {
-		if (!chunk.path) continue;
-		const prefixes = buildStoredPathPrefixes(chunk.path);
-		for (const [index, prefix] of prefixes.entries()) {
-			const parentPath = index > 0 ? prefixes[index - 1] || null : null;
-			const accumulator = ensurePathAccumulator(accumulators, prefix, parentPath, index + 1);
-			incrementPathCounters(accumulator, chunk.type);
-			if (parentPath) ensurePathAccumulator(accumulators, parentPath, index > 1 ? prefixes[index - 2] || null : null, index).childPaths.add(prefix);
-			if (index === prefixes.length - 1) {
-				accumulator.directChunkCount += 1;
-				if (chunk.chunkId) accumulator.chunkIds.push(chunk.chunkId);
-			}
-		}
-	}
-	return [...accumulators.values()].sort((left, right) => left.depth - right.depth || left.path.localeCompare(right.path)).map((entry) => ({
-		path: entry.path,
-		parentPath: entry.parentPath,
-		depth: entry.depth,
-		childPaths: [...entry.childPaths].sort((left, right) => left.localeCompare(right)),
-		chunkIds: [...entry.chunkIds],
-		directChunkCount: entry.directChunkCount,
-		chunkCount: entry.chunkCount,
-		textChunkCount: entry.textChunkCount,
-		imageChunkCount: entry.imageChunkCount,
-		tableChunkCount: entry.tableChunkCount
-	}));
-}
-function readManifestAssetEntries(manifest, key) {
-	const rawEntries = (isRecord(manifest.files) ? manifest.files : {})[key];
-	if (!Array.isArray(rawEntries)) return [];
-	return rawEntries.filter((entry) => isRecord(entry));
-}
-function buildResultFileChunkLookup(manifest, chunks) {
-	const entries = /* @__PURE__ */ new Map();
-	for (const key of ["images", "tables"]) {
-		const assetEntries = readManifestAssetEntries(manifest, key);
-		for (const entry of assetEntries) {
-			const filePath = typeof entry.file_path === "string" && entry.file_path.trim() ? normalizeRelativePath(entry.file_path.trim()) : null;
-			if (!filePath) continue;
-			entries.set(filePath, {
-				chunkId: typeof entry.id === "string" ? entry.id : null,
-				format: typeof entry.format === "string" ? entry.format : null
-			});
-		}
-	}
-	for (const chunk of chunks) {
-		if (!chunk.assetFilePath || entries.has(chunk.assetFilePath)) continue;
-		entries.set(chunk.assetFilePath, {
-			chunkId: chunk.chunkId || null,
-			format: null
-		});
-	}
-	return entries;
-}
-function inferResultFileKind(relativePath) {
-	if (relativePath === MANIFEST_FILE_NAME) return "manifest";
-	if (relativePath === CHUNKS_FILE_NAME) return "chunks";
-	if (relativePath === FULL_MARKDOWN_FILE_NAME) return "fullMarkdown";
-	if (relativePath === KB_CSV_FILE_NAME) return "kbCsv";
-	if (relativePath === HIERARCHY_FILE_NAME) return "hierarchy";
-	if (relativePath === HIERARCHY_VIEW_FILE_NAME) return "hierarchyView";
-	if (relativePath.startsWith("images/")) return "image";
-	if (relativePath.startsWith("tables/")) return "table";
-	return "other";
-}
-function inferResultFileFormat(relativePath) {
-	return path.posix.extname(relativePath).replace(/^\./, "").trim() || null;
-}
-function isStringArray(value) {
-	return Array.isArray(value) && value.every((entry) => typeof entry === "string");
-}
-function isStoredBrowseIndex(value) {
-	if (!isRecord(value)) return false;
-	if (value.version !== 2) return false;
-	if (!isStringArray(value.chunkOrder)) return false;
-	if (!Array.isArray(value.paths) || !Array.isArray(value.resultFiles)) return false;
-	if (!value.paths.every((entry) => isRecord(entry) && typeof entry.path === "string" && (entry.parentPath === null || typeof entry.parentPath === "string") && typeof entry.depth === "number" && Number.isFinite(entry.depth) && isStringArray(entry.childPaths) && isStringArray(entry.chunkIds) && typeof entry.directChunkCount === "number" && typeof entry.chunkCount === "number" && typeof entry.textChunkCount === "number" && typeof entry.imageChunkCount === "number" && typeof entry.tableChunkCount === "number")) return false;
-	return value.resultFiles.every((entry) => isRecord(entry) && typeof entry.relativePath === "string" && typeof entry.kind === "string" && (entry.chunkId === null || typeof entry.chunkId === "string") && (entry.format === null || typeof entry.format === "string") && (entry.sizeBytes === null || typeof entry.sizeBytes === "number" && Number.isFinite(entry.sizeBytes)));
-}
-async function listResultFiles(rootDir, currentDir = rootDir) {
-	const entries = await fs.readdir(currentDir, { withFileTypes: true });
-	const files = [];
-	for (const entry of entries) {
-		const absolutePath = path.join(currentDir, entry.name);
-		if (entry.isDirectory()) {
-			files.push(...await listResultFiles(rootDir, absolutePath));
-			continue;
-		}
-		if (!entry.isFile()) continue;
-		files.push(normalizeRelativePath(path.relative(rootDir, absolutePath)));
-	}
-	return files.sort((left, right) => left.localeCompare(right));
-}
-async function buildResultFileRecords(resultDir, manifest, chunks) {
-	const lookup = buildResultFileChunkLookup(manifest, chunks);
-	const relativePaths = await listResultFiles(resultDir);
-	return Promise.all(relativePaths.map(async (relativePath) => {
-		const absolutePath = resolveResultEntryPath(resultDir, relativePath);
-		const stats = await fs.stat(absolutePath);
-		const manifestEntry = lookup.get(relativePath);
-		return {
-			relativePath,
-			kind: inferResultFileKind(relativePath),
-			chunkId: manifestEntry?.chunkId ?? null,
-			format: manifestEntry?.format ?? inferResultFileFormat(relativePath),
-			sizeBytes: stats.isFile() ? stats.size : null
-		};
-	}));
-}
-async function buildStoredBrowseIndex(resultDir, manifest, chunks) {
-	return {
-		version: 2,
-		paths: buildPathRecords(chunks),
-		chunkOrder: chunks.map((chunk) => chunk.chunkId).filter((chunkId) => chunkId.length > 0),
-		resultFiles: await buildResultFileRecords(resultDir, manifest, chunks)
-	};
-}
 async function extractKnowhereResultArchive(downloadedResult, targetDir) {
 	const zipBuffer = Buffer.isBuffer(downloadedResult.zipBytes) ? downloadedResult.zipBytes : Buffer.from(downloadedResult.zipBytes);
 	const entries = unzipSync(new Uint8Array(zipBuffer));
@@ -302,22 +82,24 @@ async function extractKnowhereResultArchive(downloadedResult, targetDir) {
 		await fs.writeFile(outputPath, entryBytes);
 	}
 }
-async function readStoredKnowhereResultSummary(resultDir) {
-	const manifest = parseManifest(await readJsonFile(path.join(resultDir, MANIFEST_FILE_NAME)));
-	const rawChunks = parseRawChunks(await readJsonFile(path.join(resultDir, CHUNKS_FILE_NAME)));
+async function resolveStoredKnowhereResultRoot(documentDir) {
+	if (await pathExists(path.join(documentDir, MANIFEST_FILE_NAME))) return documentDir;
+	const legacyResultDir = path.join(documentDir, LEGACY_RESULT_DIRECTORY_NAME);
+	if (await pathExists(path.join(legacyResultDir, MANIFEST_FILE_NAME))) return legacyResultDir;
+	return documentDir;
+}
+async function resolveStoredKnowhereArtifactPath(documentDir, entryPath) {
+	return resolveResultEntryPath(await resolveStoredKnowhereResultRoot(documentDir), entryPath);
+}
+async function readStoredKnowhereResultSummary(documentDir) {
+	const resultRoot = await resolveStoredKnowhereResultRoot(documentDir);
+	const manifest = parseManifest(await readJsonFile(path.join(resultRoot, MANIFEST_FILE_NAME)));
+	const rawChunks = parseRawChunks(await readJsonFile(path.join(resultRoot, CHUNKS_FILE_NAME)));
 	return {
 		manifest,
 		chunkCount: rawChunks.length,
 		statistics: normalizeStatistics(manifest, rawChunks)
 	};
 }
-async function readStoredKnowhereResultContent(resultDir) {
-	return {
-		manifest: parseManifest(await readJsonFile(path.join(resultDir, MANIFEST_FILE_NAME))),
-		chunks: parseRawChunks(await readJsonFile(path.join(resultDir, CHUNKS_FILE_NAME))).map((rawChunk) => buildChunk(rawChunk)),
-		fullMarkdown: await readTextFile(path.join(resultDir, FULL_MARKDOWN_FILE_NAME)) || "",
-		hierarchy: await readJsonFile(path.join(resultDir, HIERARCHY_FILE_NAME))
-	};
-}
 //#endregion
-export { buildStoredBrowseIndex, extractKnowhereResultArchive, isStoredBrowseIndex, readStoredKnowhereResultContent, readStoredKnowhereResultSummary, resolveResultEntryPath };
+export { extractKnowhereResultArchive, readStoredKnowhereResultSummary, resolveStoredKnowhereArtifactPath, resolveStoredKnowhereResultRoot };

package/dist/store.d.ts

@@ -1,10 +1,10 @@
-import type { ChannelRouteRecord, KnowhereScope, PluginLogger, SaveStoredDocumentPayload, ScopeMode, StoredDocumentPayload, StoredDocumentRecord } from "./types";
+import type { ChannelRouteRecord, KnowhereScope, PluginLogger, SaveStoredDocumentPayload, ScopeMode, StoredDocumentMetadata, StoredDocumentRecord } from "./types";
+type StoredDocumentScopePaths = Pick<KnowhereScope, "documentsDir" | "metadataDir">;
 export declare class KnowhereStore {
     private readonly rootDir;
     private readonly scopeMode;
     private readonly logger;
     private readonly indexCache;
-    private readonly documentPayloadCache;
     private readonly scopeAccessChains;
     private readonly scopeKeyAliases;
     private readonly sessionScopeKeysBySessionId;
@@ -48,14 +48,8 @@ export declare class KnowhereStore {
         sessionKey?: string;
         sessionId?: string;
     }): KnowhereScope;
+    readDocumentMetadata(scope: StoredDocumentScopePaths, docId: string): Promise<StoredDocumentMetadata | null>;
     listDocuments(scope: KnowhereScope): Promise<StoredDocumentRecord[]>;
-    loadDocumentPayload(scope: KnowhereScope, docId: string): Promise<StoredDocumentPayload | null>;
-    getResultFileAbsolutePath(scope: KnowhereScope, docId: string, relativePath: string): string;
-    readResultFile(scope: KnowhereScope, docId: string, relativePath: string): Promise<{
-        document: StoredDocumentRecord;
-        relativePath: string;
-        text: string | null;
-    } | null>;
     saveDownloadedDocument(scope: KnowhereScope, payload: SaveStoredDocumentPayload, options?: {
         overwrite?: boolean;
     }): Promise<StoredDocumentRecord>;
@@ -65,11 +59,6 @@ export declare class KnowhereStore {
     private persistIndex;
     private runWithScopeAccessLock;
     private removeDocumentArtifacts;
-    private buildDocumentPayloadCacheKey;
-    private touchDocumentPayloadCache;
-    private deleteDocumentPayloadCache;
-    private deleteScopeDocumentPayloadCaches;
-    private loadOrBuildBrowseIndex;
     private buildRouteKey;
     private ensureRoutesLoaded;
     private persistRoutes;
@@ -77,3 +66,4 @@ export declare class KnowhereStore {
     private resolveKnownScopeKey;
     private rebuildIndex;
 }
+export {};

package/dist/store.js

@@ -1,5 +1,5 @@
 import { isNodeError } from "./types.js";
-import { buildStoredBrowseIndex, extractKnowhereResultArchive, isStoredBrowseIndex, readStoredKnowhereResultContent, readStoredKnowhereResultSummary, resolveResultEntryPath } from "./parser.js";
+import { extractKnowhereResultArchive, readStoredKnowhereResultSummary } from "./parser.js";
 import { hashString, normalizeWhitespace, sanitizeStringArray, slugify } from "./text.js";
 import { deriveMessageContextScopeKey, findConversationSegmentValue, parseConversationSessionKey } from "./session.js";
 import fs from "node:fs/promises";
@@ -7,10 +7,8 @@ import path from "node:path";
 import { randomUUID } from "node:crypto";
 //#region src/store.ts
 const INDEX_VERSION = 1;
-const BROWSE_INDEX_FILE_NAME = "browse-index.json";
-const DOCUMENT_PAYLOAD_CACHE_LIMIT = 16;
+const METADATA_DIRECTORY_NAME = "metadata";
 const METADATA_FILE_NAME = "metadata.json";
-const RESULT_DIRECTORY_NAME = "result";
 const ROUTES_FILE_NAME = "routes.json";
 async function pathExists(targetPath) {
 	try {
@@ -119,10 +117,9 @@ function createEmptyIndex(scope) {
 function buildStoredDocumentPaths(scope, docId) {
 	const documentDir = path.join(scope.documentsDir, docId);
 	return {
-		browseIndexPath: path.join(documentDir, BROWSE_INDEX_FILE_NAME),
 		documentDir,
-		metadataPath: path.join(documentDir, METADATA_FILE_NAME),
-		resultDir: path.join(documentDir, RESULT_DIRECTORY_NAME)
+		metadataPath: path.join(scope.metadataDir, `${docId}.json`),
+		legacyMetadataPath: path.join(documentDir, METADATA_FILE_NAME)
 	};
 }
 async function readStoredDocumentMetadata(metadataPath) {
@@ -142,7 +139,6 @@ var KnowhereStore = class {
 	scopeMode;
 	logger;
 	indexCache;
-	documentPayloadCache;
 	scopeAccessChains;
 	scopeKeyAliases;
 	sessionScopeKeysBySessionId;
@@ -154,7 +150,6 @@ var KnowhereStore = class {
 		this.scopeMode = params.scopeMode;
 		this.logger = params.logger;
 		this.indexCache = /* @__PURE__ */ new Map();
-		this.documentPayloadCache = /* @__PURE__ */ new Map();
 		this.scopeAccessChains = /* @__PURE__ */ new Map();
 		this.scopeKeyAliases = /* @__PURE__ */ new Map();
 		this.sessionScopeKeysBySessionId = /* @__PURE__ */ new Map();
@@ -277,80 +272,32 @@ var KnowhereStore = class {
 			key: rawKey,
 			label: rawKey === "global" ? "global" : `${mode}:${rawKey}`,
 			rootDir: scopeRoot,
+			metadataDir: path.join(scopeRoot, METADATA_DIRECTORY_NAME),
 			documentsDir: path.join(scopeRoot, "documents"),
 			indexPath: path.join(scopeRoot, "index.json")
 		};
 	}
+	async readDocumentMetadata(scope, docId) {
+		const paths = buildStoredDocumentPaths(scope, docId);
+		const preferredMetadata = await readStoredDocumentMetadata(paths.metadataPath);
+		if (preferredMetadata) return preferredMetadata;
+		return readStoredDocumentMetadata(paths.legacyMetadataPath);
+	}
 	async listDocuments(scope) {
 		return this.runWithScopeAccessLock(scope, async () => {
 			return [...(await this.getIndex(scope, true)).documents].sort((left, right) => String(right.updatedAt || right.ingestedAt || "").localeCompare(String(left.updatedAt || left.ingestedAt || "")));
 		});
 	}
-	async loadDocumentPayload(scope, docId) {
-		return this.runWithScopeAccessLock(scope, async () => {
-			const cacheKey = this.buildDocumentPayloadCacheKey(scope, docId);
-			const cachedPayload = this.documentPayloadCache.get(cacheKey);
-			if (cachedPayload) {
-				this.touchDocumentPayloadCache(cacheKey, cachedPayload);
-				return cachedPayload;
-			}
-			const paths = buildStoredDocumentPaths(scope, docId);
-			const metadata = await readStoredDocumentMetadata(paths.metadataPath);
-			if (!metadata) return null;
-			const resultContent = await readStoredKnowhereResultContent(paths.resultDir);
-			const browseIndex = await this.loadOrBuildBrowseIndex(paths, resultContent.manifest, resultContent.chunks);
-			const payload = {
-				version: metadata.version,
-				document: metadata.document,
-				manifest: resultContent.manifest,
-				jobResult: metadata.jobResult,
-				fullMarkdown: resultContent.fullMarkdown,
-				hierarchy: resultContent.hierarchy,
-				browseIndex,
-				rawZipSha1: metadata.rawZipSha1,
-				chunks: resultContent.chunks
-			};
-			this.touchDocumentPayloadCache(cacheKey, payload);
-			return payload;
-		});
-	}
-	getResultFileAbsolutePath(scope, docId, relativePath) {
-		return resolveResultEntryPath(buildStoredDocumentPaths(scope, docId).resultDir, relativePath);
-	}
-	async readResultFile(scope, docId, relativePath) {
-		return this.runWithScopeAccessLock(scope, async () => {
-			const paths = buildStoredDocumentPaths(scope, docId);
-			const metadata = await readStoredDocumentMetadata(paths.metadataPath);
-			if (!metadata) return null;
-			const filePath = resolveResultEntryPath(paths.resultDir, relativePath);
-			try {
-				const text = await fs.readFile(filePath, "utf-8");
-				return {
-					document: metadata.document,
-					relativePath: path.posix.normalize(relativePath.replace(/\\/g, "/")),
-					text
-				};
-			} catch (error) {
-				if (isNodeError(error) && error.code === "ENOENT") return {
-					document: metadata.document,
-					relativePath: path.posix.normalize(relativePath.replace(/\\/g, "/")),
-					text: null
-				};
-				throw error;
-			}
-		});
-	}
 	async saveDownloadedDocument(scope, payload, options = {}) {
 		return this.runWithScopeAccessLock(scope, async () => {
 			const index = await this.getIndex(scope, true);
 			const existingIds = new Set(index.documents.map((document) => document.id));
 			const now = (/* @__PURE__ */ new Date()).toISOString();
 			const tempPaths = buildStoredDocumentPaths(scope, `.tmp-${randomUUID()}`);
+			let finalPaths = null;
 			try {
-				await extractKnowhereResultArchive(payload.downloadedResult, tempPaths.resultDir);
-				const resultSummary = await readStoredKnowhereResultSummary(tempPaths.resultDir);
-				const resultContent = await readStoredKnowhereResultContent(tempPaths.resultDir);
-				const browseIndex = await buildStoredBrowseIndex(tempPaths.resultDir, resultContent.manifest, resultContent.chunks);
+				await extractKnowhereResultArchive(payload.downloadedResult, tempPaths.documentDir);
+				const resultSummary = await readStoredKnowhereResultSummary(tempPaths.documentDir);
 				const documentIdSeed = [
 					payload.sourceType,
 					payload.source,
@@ -364,6 +311,7 @@ var KnowhereStore = class {
 				if (payload.docId && existingDocument && options.overwrite !== true) throw new Error(`Document ${documentIdCandidate} already exists in scope ${scope.label}.`);
 				const docId = existingDocument && options.overwrite === true ? existingDocument.id : buildUniqueDocumentId(documentIdCandidate, documentIdSeed, existingIds);
 				const paths = buildStoredDocumentPaths(scope, docId);
+				finalPaths = paths;
 				const originalFileName = deriveOriginalFileName(payload, resultSummary.manifest);
 				const documentRecord = {
 					id: docId,
@@ -392,16 +340,17 @@ var KnowhereStore = class {
 					rawZipSha1: payload.downloadedResult.rawZipSha1
 				};
 				await writeJsonAtomic(tempPaths.metadataPath, metadata);
-				await writeJsonAtomic(tempPaths.browseIndexPath, browseIndex);
 				await this.removeDocumentArtifacts(paths);
 				await ensureDir(scope.documentsDir);
+				await ensureDir(scope.metadataDir);
 				await fs.rename(tempPaths.documentDir, paths.documentDir);
+				await fs.rename(tempPaths.metadataPath, paths.metadataPath);
 				index.documents = index.documents.filter((document) => document.id !== docId).concat(documentRecord);
-				this.deleteDocumentPayloadCache(scope, docId);
 				await this.persistIndex(scope, index);
 				return documentRecord;
 			} catch (error) {
 				await this.removeDocumentArtifacts(tempPaths).catch(() => void 0);
+				if (finalPaths) await this.removeDocumentArtifacts(finalPaths).catch(() => void 0);
 				throw error;
 			}
 		});
@@ -413,7 +362,6 @@ var KnowhereStore = class {
 			if (!existingDocument) return null;
 			await this.removeDocumentArtifacts(buildStoredDocumentPaths(scope, docId));
 			index.documents = index.documents.filter((document) => document.id !== docId);
-			this.deleteDocumentPayloadCache(scope, docId);
 			await this.persistIndex(scope, index);
 			return existingDocument;
 		});
@@ -426,7 +374,6 @@ var KnowhereStore = class {
 				force: true
 			});
 			this.indexCache.delete(scope.rootDir);
-			this.deleteScopeDocumentPayloadCaches(scope);
 			this.logger.info(`knowhere: cleared scope ${scope.label} (${removedDocuments.length} document${removedDocuments.length === 1 ? "" : "s"})`);
 			return removedDocuments;
 		});
@@ -473,39 +420,8 @@ var KnowhereStore = class {
 			recursive: true,
 			force: true
 		});
-	}
-	buildDocumentPayloadCacheKey(scope, docId) {
-		return `${scope.rootDir}:${docId}`;
-	}
-	touchDocumentPayloadCache(cacheKey, payload) {
-		this.documentPayloadCache.delete(cacheKey);
-		this.documentPayloadCache.set(cacheKey, payload);
-		while (this.documentPayloadCache.size > DOCUMENT_PAYLOAD_CACHE_LIMIT) {
-			const oldestKey = this.documentPayloadCache.keys().next().value;
-			if (!oldestKey) break;
-			this.documentPayloadCache.delete(oldestKey);
-		}
-	}
-	deleteDocumentPayloadCache(scope, docId) {
-		this.documentPayloadCache.delete(this.buildDocumentPayloadCacheKey(scope, docId));
-	}
-	deleteScopeDocumentPayloadCaches(scope) {
-		const cacheKeyPrefix = `${scope.rootDir}:`;
-		for (const cacheKey of this.documentPayloadCache.keys()) if (cacheKey.startsWith(cacheKeyPrefix)) this.documentPayloadCache.delete(cacheKey);
-	}
-	async loadOrBuildBrowseIndex(paths, manifest, chunks) {
-		if (await pathExists(paths.browseIndexPath)) {
-			try {
-				const existingBrowseIndex = await readJson(paths.browseIndexPath, null);
-				if (isStoredBrowseIndex(existingBrowseIndex)) return existingBrowseIndex;
-			} catch (error) {
-				this.logger.warn(`knowhere: failed to read browse index ${paths.browseIndexPath}; rebuilding. ${error instanceof Error ? error.message : String(error)}`);
-			}
-			this.logger.info(`knowhere: rebuilding browse index for ${paths.documentDir} (expected version 2)`);
-		}
-		const browseIndex = await buildStoredBrowseIndex(paths.resultDir, manifest, chunks);
-		await writeJsonAtomic(paths.browseIndexPath, browseIndex);
-		return browseIndex;
+		if (await pathExists(paths.metadataPath)) await fs.rm(paths.metadataPath, { force: true });
+		if (await pathExists(paths.legacyMetadataPath)) await fs.rm(paths.legacyMetadataPath, { force: true });
 	}
 	buildRouteKey(channelId, conversationId) {
 		const normalizedChannel = normalizeWhitespace(channelId)?.toLowerCase();
@@ -557,8 +473,7 @@ var KnowhereStore = class {
 		const documentEntries = await fs.readdir(scope.documentsDir, { withFileTypes: true });
 		for (const documentEntry of documentEntries) {
 			if (!documentEntry.isDirectory()) continue;
-			const documentPath = path.join(scope.documentsDir, documentEntry.name);
-			const metadata = await readStoredDocumentMetadata(path.join(documentPath, METADATA_FILE_NAME));
+			const metadata = await this.readDocumentMetadata(scope, documentEntry.name);
 			if (!metadata?.document) continue;
 			rebuiltIndex.documents.push(metadata.document);
 		}

package/dist/text.js

@@ -18,17 +18,5 @@ function sanitizeStringArray(value) {
 	}
 	return [];
 }
-function stripHtmlTags(text) {
-	return text.replace(/<[^>]*>/g, "");
-}
-function stripLatex(text) {
-	return text.replace(/\$([^$]*)\$/g, "$1").replace(/\\text\{([^}]*)}/g, "$1").replace(/\\(?:text(?:bf|it|tt|sf|sc|rm)|math(?:rm|bf|it|sf|tt|cal|bb|frak))\{([^}]*)}/g, "$1").replace(/\\(?:emph|underline|overline)\{([^}]*)}/g, "$1").replace(/\\([%$&#_])/g, "$1").replace(/\\(?:right|Right)arrow/g, "→").replace(/\\(?:left|Left)arrow/g, "←").replace(/\\leftrightarrow/g, "↔").replace(/\\times/g, "×").replace(/\\cdot/g, "·").replace(/\\pm/g, "±").replace(/\\leq/g, "≤").replace(/\\geq/g, "≥").replace(/\\neq/g, "≠").replace(/\\approx/g, "≈").replace(/\\(sup|inf|max|min|log|ln|sin|cos|tan|exp|lim)\b/g, "$1").replace(/\\([{}])/g, "$1").replace(/\\\\/g, " ").replace(/\\[a-zA-Z]+/g, "");
-}
-function normalizeUnicode(text) {
-	return text.replace(/[\u2018\u2019\u201A]/g, "'").replace(/[\u201C\u201D\u201E]/g, "\"").replace(/[\u2013\u2014]/g, "-").replace(/[\u00A0\u2009\u200A\u200B\u2007\u202F]/g, " ").replace(/\u2026/g, "...").replace(/\u2022/g, "-");
-}
-function normalizeForGrep(text) {
-	return stripHtmlTags(stripLatex(normalizeUnicode(text))).replace(/\s+/g, " ").trim();
-}
 //#endregion
-export { hashString, normalizeForGrep, normalizeWhitespace, sanitizeStringArray, slugify };
+export { hashString, normalizeWhitespace, sanitizeStringArray, slugify };