@ontos-ai/knowhere-claw 0.2.3 → 0.2.4

package/dist/store.js

@@ -1,5 +1,5 @@
 import { isNodeError } from "./types.js";
-import { buildStoredBrowseIndex, extractKnowhereResultArchive, isStoredBrowseIndex, readStoredKnowhereResultContent, readStoredKnowhereResultSummary, resolveResultEntryPath } from "./parser.js";
+import { extractKnowhereResultArchive, readStoredKnowhereResultSummary } from "./parser.js";
 import { hashString, normalizeWhitespace, sanitizeStringArray, slugify } from "./text.js";
 import { deriveMessageContextScopeKey, findConversationSegmentValue, parseConversationSessionKey } from "./session.js";
 import fs from "node:fs/promises";
@@ -7,10 +7,8 @@ import path from "node:path";
 import { randomUUID } from "node:crypto";
 //#region src/store.ts
 const INDEX_VERSION = 1;
-const BROWSE_INDEX_FILE_NAME = "browse-index.json";
-const DOCUMENT_PAYLOAD_CACHE_LIMIT = 16;
+const METADATA_DIRECTORY_NAME = "metadata";
 const METADATA_FILE_NAME = "metadata.json";
-const RESULT_DIRECTORY_NAME = "result";
 const ROUTES_FILE_NAME = "routes.json";
 async function pathExists(targetPath) {
 	try {
@@ -119,10 +117,9 @@ function createEmptyIndex(scope) {
 function buildStoredDocumentPaths(scope, docId) {
 	const documentDir = path.join(scope.documentsDir, docId);
 	return {
-		browseIndexPath: path.join(documentDir, BROWSE_INDEX_FILE_NAME),
 		documentDir,
-		metadataPath: path.join(documentDir, METADATA_FILE_NAME),
-		resultDir: path.join(documentDir, RESULT_DIRECTORY_NAME)
+		metadataPath: path.join(scope.metadataDir, `${docId}.json`),
+		legacyMetadataPath: path.join(documentDir, METADATA_FILE_NAME)
 	};
 }
 async function readStoredDocumentMetadata(metadataPath) {
@@ -142,7 +139,6 @@ var KnowhereStore = class {
 	scopeMode;
 	logger;
 	indexCache;
-	documentPayloadCache;
 	scopeAccessChains;
 	scopeKeyAliases;
 	sessionScopeKeysBySessionId;
@@ -154,7 +150,6 @@ var KnowhereStore = class {
 		this.scopeMode = params.scopeMode;
 		this.logger = params.logger;
 		this.indexCache = /* @__PURE__ */ new Map();
-		this.documentPayloadCache = /* @__PURE__ */ new Map();
 		this.scopeAccessChains = /* @__PURE__ */ new Map();
 		this.scopeKeyAliases = /* @__PURE__ */ new Map();
 		this.sessionScopeKeysBySessionId = /* @__PURE__ */ new Map();
@@ -277,80 +272,32 @@ var KnowhereStore = class {
 			key: rawKey,
 			label: rawKey === "global" ? "global" : `${mode}:${rawKey}`,
 			rootDir: scopeRoot,
+			metadataDir: path.join(scopeRoot, METADATA_DIRECTORY_NAME),
 			documentsDir: path.join(scopeRoot, "documents"),
 			indexPath: path.join(scopeRoot, "index.json")
 		};
 	}
+	async readDocumentMetadata(scope, docId) {
+		const paths = buildStoredDocumentPaths(scope, docId);
+		const preferredMetadata = await readStoredDocumentMetadata(paths.metadataPath);
+		if (preferredMetadata) return preferredMetadata;
+		return readStoredDocumentMetadata(paths.legacyMetadataPath);
+	}
 	async listDocuments(scope) {
 		return this.runWithScopeAccessLock(scope, async () => {
 			return [...(await this.getIndex(scope, true)).documents].sort((left, right) => String(right.updatedAt || right.ingestedAt || "").localeCompare(String(left.updatedAt || left.ingestedAt || "")));
 		});
 	}
-	async loadDocumentPayload(scope, docId) {
-		return this.runWithScopeAccessLock(scope, async () => {
-			const cacheKey = this.buildDocumentPayloadCacheKey(scope, docId);
-			const cachedPayload = this.documentPayloadCache.get(cacheKey);
-			if (cachedPayload) {
-				this.touchDocumentPayloadCache(cacheKey, cachedPayload);
-				return cachedPayload;
-			}
-			const paths = buildStoredDocumentPaths(scope, docId);
-			const metadata = await readStoredDocumentMetadata(paths.metadataPath);
-			if (!metadata) return null;
-			const resultContent = await readStoredKnowhereResultContent(paths.resultDir);
-			const browseIndex = await this.loadOrBuildBrowseIndex(paths, resultContent.manifest, resultContent.chunks);
-			const payload = {
-				version: metadata.version,
-				document: metadata.document,
-				manifest: resultContent.manifest,
-				jobResult: metadata.jobResult,
-				fullMarkdown: resultContent.fullMarkdown,
-				hierarchy: resultContent.hierarchy,
-				browseIndex,
-				rawZipSha1: metadata.rawZipSha1,
-				chunks: resultContent.chunks
-			};
-			this.touchDocumentPayloadCache(cacheKey, payload);
-			return payload;
-		});
-	}
-	getResultFileAbsolutePath(scope, docId, relativePath) {
-		return resolveResultEntryPath(buildStoredDocumentPaths(scope, docId).resultDir, relativePath);
-	}
-	async readResultFile(scope, docId, relativePath) {
-		return this.runWithScopeAccessLock(scope, async () => {
-			const paths = buildStoredDocumentPaths(scope, docId);
-			const metadata = await readStoredDocumentMetadata(paths.metadataPath);
-			if (!metadata) return null;
-			const filePath = resolveResultEntryPath(paths.resultDir, relativePath);
-			try {
-				const text = await fs.readFile(filePath, "utf-8");
-				return {
-					document: metadata.document,
-					relativePath: path.posix.normalize(relativePath.replace(/\\/g, "/")),
-					text
-				};
-			} catch (error) {
-				if (isNodeError(error) && error.code === "ENOENT") return {
-					document: metadata.document,
-					relativePath: path.posix.normalize(relativePath.replace(/\\/g, "/")),
-					text: null
-				};
-				throw error;
-			}
-		});
-	}
 	async saveDownloadedDocument(scope, payload, options = {}) {
 		return this.runWithScopeAccessLock(scope, async () => {
 			const index = await this.getIndex(scope, true);
 			const existingIds = new Set(index.documents.map((document) => document.id));
 			const now = (/* @__PURE__ */ new Date()).toISOString();
 			const tempPaths = buildStoredDocumentPaths(scope, `.tmp-${randomUUID()}`);
+			let finalPaths = null;
 			try {
-				await extractKnowhereResultArchive(payload.downloadedResult, tempPaths.resultDir);
-				const resultSummary = await readStoredKnowhereResultSummary(tempPaths.resultDir);
-				const resultContent = await readStoredKnowhereResultContent(tempPaths.resultDir);
-				const browseIndex = await buildStoredBrowseIndex(tempPaths.resultDir, resultContent.manifest, resultContent.chunks);
+				await extractKnowhereResultArchive(payload.downloadedResult, tempPaths.documentDir);
+				const resultSummary = await readStoredKnowhereResultSummary(tempPaths.documentDir);
 				const documentIdSeed = [
 					payload.sourceType,
 					payload.source,
@@ -364,6 +311,7 @@ var KnowhereStore = class {
 				if (payload.docId && existingDocument && options.overwrite !== true) throw new Error(`Document ${documentIdCandidate} already exists in scope ${scope.label}.`);
 				const docId = existingDocument && options.overwrite === true ? existingDocument.id : buildUniqueDocumentId(documentIdCandidate, documentIdSeed, existingIds);
 				const paths = buildStoredDocumentPaths(scope, docId);
+				finalPaths = paths;
 				const originalFileName = deriveOriginalFileName(payload, resultSummary.manifest);
 				const documentRecord = {
 					id: docId,
@@ -392,16 +340,17 @@ var KnowhereStore = class {
 					rawZipSha1: payload.downloadedResult.rawZipSha1
 				};
 				await writeJsonAtomic(tempPaths.metadataPath, metadata);
-				await writeJsonAtomic(tempPaths.browseIndexPath, browseIndex);
 				await this.removeDocumentArtifacts(paths);
 				await ensureDir(scope.documentsDir);
+				await ensureDir(scope.metadataDir);
 				await fs.rename(tempPaths.documentDir, paths.documentDir);
+				await fs.rename(tempPaths.metadataPath, paths.metadataPath);
 				index.documents = index.documents.filter((document) => document.id !== docId).concat(documentRecord);
-				this.deleteDocumentPayloadCache(scope, docId);
 				await this.persistIndex(scope, index);
 				return documentRecord;
 			} catch (error) {
 				await this.removeDocumentArtifacts(tempPaths).catch(() => void 0);
+				if (finalPaths) await this.removeDocumentArtifacts(finalPaths).catch(() => void 0);
 				throw error;
 			}
 		});
@@ -413,7 +362,6 @@ var KnowhereStore = class {
 			if (!existingDocument) return null;
 			await this.removeDocumentArtifacts(buildStoredDocumentPaths(scope, docId));
 			index.documents = index.documents.filter((document) => document.id !== docId);
-			this.deleteDocumentPayloadCache(scope, docId);
 			await this.persistIndex(scope, index);
 			return existingDocument;
 		});
@@ -426,7 +374,6 @@ var KnowhereStore = class {
 				force: true
 			});
 			this.indexCache.delete(scope.rootDir);
-			this.deleteScopeDocumentPayloadCaches(scope);
 			this.logger.info(`knowhere: cleared scope ${scope.label} (${removedDocuments.length} document${removedDocuments.length === 1 ? "" : "s"})`);
 			return removedDocuments;
 		});
@@ -473,39 +420,8 @@ var KnowhereStore = class {
 			recursive: true,
 			force: true
 		});
-	}
-	buildDocumentPayloadCacheKey(scope, docId) {
-		return `${scope.rootDir}:${docId}`;
-	}
-	touchDocumentPayloadCache(cacheKey, payload) {
-		this.documentPayloadCache.delete(cacheKey);
-		this.documentPayloadCache.set(cacheKey, payload);
-		while (this.documentPayloadCache.size > DOCUMENT_PAYLOAD_CACHE_LIMIT) {
-			const oldestKey = this.documentPayloadCache.keys().next().value;
-			if (!oldestKey) break;
-			this.documentPayloadCache.delete(oldestKey);
-		}
-	}
-	deleteDocumentPayloadCache(scope, docId) {
-		this.documentPayloadCache.delete(this.buildDocumentPayloadCacheKey(scope, docId));
-	}
-	deleteScopeDocumentPayloadCaches(scope) {
-		const cacheKeyPrefix = `${scope.rootDir}:`;
-		for (const cacheKey of this.documentPayloadCache.keys()) if (cacheKey.startsWith(cacheKeyPrefix)) this.documentPayloadCache.delete(cacheKey);
-	}
-	async loadOrBuildBrowseIndex(paths, manifest, chunks) {
-		if (await pathExists(paths.browseIndexPath)) {
-			try {
-				const existingBrowseIndex = await readJson(paths.browseIndexPath, null);
-				if (isStoredBrowseIndex(existingBrowseIndex)) return existingBrowseIndex;
-			} catch (error) {
-				this.logger.warn(`knowhere: failed to read browse index ${paths.browseIndexPath}; rebuilding. ${error instanceof Error ? error.message : String(error)}`);
-			}
-			this.logger.info(`knowhere: rebuilding browse index for ${paths.documentDir} (expected version 2)`);
-		}
-		const browseIndex = await buildStoredBrowseIndex(paths.resultDir, manifest, chunks);
-		await writeJsonAtomic(paths.browseIndexPath, browseIndex);
-		return browseIndex;
+		if (await pathExists(paths.metadataPath)) await fs.rm(paths.metadataPath, { force: true });
+		if (await pathExists(paths.legacyMetadataPath)) await fs.rm(paths.legacyMetadataPath, { force: true });
 	}
 	buildRouteKey(channelId, conversationId) {
 		const normalizedChannel = normalizeWhitespace(channelId)?.toLowerCase();
@@ -557,8 +473,7 @@ var KnowhereStore = class {
 		const documentEntries = await fs.readdir(scope.documentsDir, { withFileTypes: true });
 		for (const documentEntry of documentEntries) {
 			if (!documentEntry.isDirectory()) continue;
-			const documentPath = path.join(scope.documentsDir, documentEntry.name);
-			const metadata = await readStoredDocumentMetadata(path.join(documentPath, METADATA_FILE_NAME));
+			const metadata = await this.readDocumentMetadata(scope, documentEntry.name);
 			if (!metadata?.document) continue;
 			rebuiltIndex.documents.push(metadata.document);
 		}

package/dist/text.js

@@ -18,17 +18,5 @@ function sanitizeStringArray(value) {
 	}
 	return [];
 }
-function stripHtmlTags(text) {
-	return text.replace(/<[^>]*>/g, "");
-}
-function stripLatex(text) {
-	return text.replace(/\$([^$]*)\$/g, "$1").replace(/\\text\{([^}]*)}/g, "$1").replace(/\\(?:text(?:bf|it|tt|sf|sc|rm)|math(?:rm|bf|it|sf|tt|cal|bb|frak))\{([^}]*)}/g, "$1").replace(/\\(?:emph|underline|overline)\{([^}]*)}/g, "$1").replace(/\\([%$&#_])/g, "$1").replace(/\\(?:right|Right)arrow/g, "→").replace(/\\(?:left|Left)arrow/g, "←").replace(/\\leftrightarrow/g, "↔").replace(/\\times/g, "×").replace(/\\cdot/g, "·").replace(/\\pm/g, "±").replace(/\\leq/g, "≤").replace(/\\geq/g, "≥").replace(/\\neq/g, "≠").replace(/\\approx/g, "≈").replace(/\\(sup|inf|max|min|log|ln|sin|cos|tan|exp|lim)\b/g, "$1").replace(/\\([{}])/g, "$1").replace(/\\\\/g, " ").replace(/\\[a-zA-Z]+/g, "");
-}
-function normalizeUnicode(text) {
-	return text.replace(/[\u2018\u2019\u201A]/g, "'").replace(/[\u201C\u201D\u201E]/g, "\"").replace(/[\u2013\u2014]/g, "-").replace(/[\u00A0\u2009\u200A\u200B\u2007\u202F]/g, " ").replace(/\u2026/g, "...").replace(/\u2022/g, "-");
-}
-function normalizeForGrep(text) {
-	return stripHtmlTags(stripLatex(normalizeUnicode(text))).replace(/\s+/g, " ").trim();
-}
 //#endregion
-export { hashString, normalizeForGrep, normalizeWhitespace, sanitizeStringArray, slugify };
+export { hashString, normalizeWhitespace, sanitizeStringArray, slugify };