@ontos-ai/knowhere-claw 0.1.0-beta.0

package/dist/parser.d.ts

@@ -0,0 +1,16 @@
+import type { KnowhereDownloadedResult, KnowhereManifest, KnowhereParseResult, KnowhereStatistics, StoredBrowseIndex, StoredChunk } from "./types";
+type KnowhereStoredResultSummary = {
+    manifest: KnowhereManifest;
+    chunkCount: number;
+    statistics: KnowhereStatistics;
+};
+export declare const STORED_BROWSE_INDEX_VERSION = 2;
+export declare function resolveResultEntryPath(rootDir: string, entryPath: string): string;
+export declare function buildStoredPathPrefixes(storedPath: string): string[];
+export declare function isStoredBrowseIndex(value: unknown): value is StoredBrowseIndex;
+export declare function buildStoredBrowseIndex(resultDir: string, manifest: KnowhereManifest, chunks: StoredChunk[]): Promise<StoredBrowseIndex>;
+export declare function extractKnowhereResultArchive(downloadedResult: KnowhereDownloadedResult, targetDir: string): Promise<void>;
+export declare function readStoredKnowhereResultSummary(resultDir: string): Promise<KnowhereStoredResultSummary>;
+export declare function readStoredKnowhereResultContent(resultDir: string): Promise<Omit<KnowhereParseResult, "browseIndex">>;
+export declare function readStoredKnowhereParseResult(resultDir: string): Promise<KnowhereParseResult>;
+export {};

package/dist/parser.js

@@ -0,0 +1,323 @@
+import { isNodeError, isRecord } from "./types.js";
+import { strFromU8, unzipSync } from "./node_modules/.pnpm/fflate@0.8.2/node_modules/fflate/esm/browser.js";
+import path from "node:path";
+import fs from "node:fs/promises";
+import { createHash } from "node:crypto";
+const CHUNKS_FILE_NAME = "chunks.json";
+const FULL_MARKDOWN_FILE_NAME = "full.md";
+const HIERARCHY_FILE_NAME = "hierarchy.json";
+const HIERARCHY_VIEW_FILE_NAME = "hierarchy_view.html";
+const KB_CSV_FILE_NAME = "kb.csv";
+const MANIFEST_FILE_NAME = "manifest.json";
+function readZipText(entries, fileName) {
+	const entry = entries[fileName];
+	return entry ? strFromU8(entry) : null;
+}
+async function ensureDir(targetPath) {
+	await fs.mkdir(targetPath, { recursive: true });
+}
+async function readTextFile(targetPath) {
+	try {
+		return await fs.readFile(targetPath, "utf-8");
+	} catch (error) {
+		if (isNodeError(error) && error.code === "ENOENT") return null;
+		throw error;
+	}
+}
+async function readJsonFile(targetPath) {
+	const text = await readTextFile(targetPath);
+	if (text === null) return null;
+	return JSON.parse(text);
+}
+function resolveResultEntryPath(rootDir, entryPath) {
+	const normalizedEntryPath = normalizeRelativePath(entryPath);
+	const resolvedPath = path.resolve(rootDir, normalizedEntryPath);
+	const relativePath = path.relative(rootDir, resolvedPath);
+	if (relativePath.startsWith("..") || path.isAbsolute(relativePath)) throw new Error(`Knowhere result entry escapes storage root: ${entryPath}`);
+	return resolvedPath;
+}
+function normalizeRelativePath(value) {
+	return value.replace(/\\/g, "/").replace(/^\/+/, "");
+}
+function normalizeStoredPath(value) {
+	if (typeof value !== "string") return null;
+	return value.trim() || null;
+}
+function readChunkNumber(rawChunk, metadata, key) {
+	const metadataValue = metadata[key];
+	if (typeof metadataValue === "number" && Number.isFinite(metadataValue)) return metadataValue;
+	const rawValue = rawChunk[key];
+	if (typeof rawValue === "number" && Number.isFinite(rawValue)) return rawValue;
+	return null;
+}
+function readChunkStringArray(rawChunk, metadata, key) {
+	const metadataValue = metadata[key];
+	if (Array.isArray(metadataValue)) return metadataValue.filter((entry) => typeof entry === "string");
+	const rawValue = rawChunk[key];
+	if (Array.isArray(rawValue)) return rawValue.filter((entry) => typeof entry === "string");
+	return [];
+}
+function readChunkArray(rawChunk, metadata, key) {
+	const metadataValue = metadata[key];
+	if (Array.isArray(metadataValue)) return metadataValue;
+	const rawValue = rawChunk[key];
+	if (Array.isArray(rawValue)) return rawValue;
+	return [];
+}
+function extractAssetFilePath(rawChunk, metadata) {
+	const candidates = [rawChunk.file_path, metadata.file_path];
+	for (const candidate of candidates) if (typeof candidate === "string" && candidate.trim()) return normalizeRelativePath(candidate.trim());
+	return null;
+}
+function parseRawChunks(value) {
+	if (Array.isArray(value)) return value.filter((entry) => isRecord(entry));
+	if (isRecord(value) && Array.isArray(value.chunks)) return value.chunks.filter((entry) => isRecord(entry));
+	return [];
+}
+function parseManifest(value) {
+	return isRecord(value) ? value : {};
+}
+function buildChunk(rawChunk) {
+	const metadata = isRecord(rawChunk.metadata) ? rawChunk.metadata : {};
+	const type = rawChunk.type === "image" || rawChunk.type === "table" || rawChunk.type === "text" ? rawChunk.type : "text";
+	return {
+		chunkId: typeof rawChunk.chunk_id === "string" ? rawChunk.chunk_id : "",
+		type,
+		path: normalizeStoredPath(rawChunk.path),
+		summary: typeof metadata.summary === "string" ? metadata.summary : typeof rawChunk.summary === "string" ? rawChunk.summary : "",
+		content: typeof rawChunk.content === "string" ? rawChunk.content : "",
+		tokens: readChunkNumber(rawChunk, metadata, "tokens"),
+		keywords: readChunkStringArray(rawChunk, metadata, "keywords"),
+		relationships: readChunkArray(rawChunk, metadata, "relationships"),
+		metadata,
+		assetFilePath: extractAssetFilePath(rawChunk, metadata),
+		originalName: typeof metadata.original_name === "string" ? metadata.original_name : typeof rawChunk.original_name === "string" ? rawChunk.original_name : null,
+		tableType: typeof metadata.table_type === "string" ? metadata.table_type : typeof rawChunk.table_type === "string" ? rawChunk.table_type : null
+	};
+}
+function normalizeStatistics(manifest, rawChunks) {
+	if (manifest.statistics) return manifest.statistics;
+	return {
+		total_chunks: rawChunks.length,
+		text_chunks: rawChunks.filter((chunk) => chunk.type !== "image" && chunk.type !== "table").length,
+		image_chunks: rawChunks.filter((chunk) => chunk.type === "image").length,
+		table_chunks: rawChunks.filter((chunk) => chunk.type === "table").length,
+		total_pages: 0
+	};
+}
+function validateKnowhereResultChecksum(zipBuffer, manifest) {
+	const checksum = manifest.checksum?.value;
+	if (typeof checksum !== "string" || !checksum) return;
+	if (createHash("sha256").update(zipBuffer).digest("hex") !== checksum) throw new Error("Knowhere result ZIP checksum mismatch.");
+}
+function tokenizeStoredPath(storedPath) {
+	const slashSegments = storedPath.split("/").map((segment) => segment.trim()).filter(Boolean);
+	const tokens = [];
+	for (const slashSegment of slashSegments) {
+		const arrowSegments = slashSegment.split("-->").map((segment) => segment.trim()).filter(Boolean);
+		if (arrowSegments.length === 0) continue;
+		tokens.push({
+			delimiter: tokens.length === 0 ? null : "/",
+			segment: arrowSegments[0] || ""
+		});
+		for (const arrowSegment of arrowSegments.slice(1)) tokens.push({
+			delimiter: "-->",
+			segment: arrowSegment
+		});
+	}
+	return tokens;
+}
+function buildStoredPathPrefixes(storedPath) {
+	const tokens = tokenizeStoredPath(storedPath);
+	const prefixes = [];
+	let currentPath = "";
+	for (const token of tokens) {
+		currentPath = token.delimiter ? `${currentPath}${token.delimiter}${token.segment}` : token.segment;
+		prefixes.push(currentPath);
+	}
+	return prefixes;
+}
+function ensurePathAccumulator(accumulators, pathValue, parentPath, depth) {
+	const existing = accumulators.get(pathValue);
+	if (existing) {
+		if (parentPath && !existing.parentPath) existing.parentPath = parentPath;
+		return existing;
+	}
+	const next = {
+		childPaths: /* @__PURE__ */ new Set(),
+		chunkCount: 0,
+		chunkIds: [],
+		depth,
+		directChunkCount: 0,
+		imageChunkCount: 0,
+		parentPath,
+		path: pathValue,
+		tableChunkCount: 0,
+		textChunkCount: 0
+	};
+	accumulators.set(pathValue, next);
+	return next;
+}
+function incrementPathCounters(accumulator, chunkType) {
+	accumulator.chunkCount += 1;
+	if (chunkType === "image") {
+		accumulator.imageChunkCount += 1;
+		return;
+	}
+	if (chunkType === "table") {
+		accumulator.tableChunkCount += 1;
+		return;
+	}
+	accumulator.textChunkCount += 1;
+}
+function buildPathRecords(chunks) {
+	const accumulators = /* @__PURE__ */ new Map();
+	for (const chunk of chunks) {
+		if (!chunk.path) continue;
+		const prefixes = buildStoredPathPrefixes(chunk.path);
+		for (const [index, prefix] of prefixes.entries()) {
+			const parentPath = index > 0 ? prefixes[index - 1] || null : null;
+			const accumulator = ensurePathAccumulator(accumulators, prefix, parentPath, index + 1);
+			incrementPathCounters(accumulator, chunk.type);
+			if (parentPath) ensurePathAccumulator(accumulators, parentPath, index > 1 ? prefixes[index - 2] || null : null, index).childPaths.add(prefix);
+			if (index === prefixes.length - 1) {
+				accumulator.directChunkCount += 1;
+				if (chunk.chunkId) accumulator.chunkIds.push(chunk.chunkId);
+			}
+		}
+	}
+	return [...accumulators.values()].sort((left, right) => left.depth - right.depth || left.path.localeCompare(right.path)).map((entry) => ({
+		path: entry.path,
+		parentPath: entry.parentPath,
+		depth: entry.depth,
+		childPaths: [...entry.childPaths].sort((left, right) => left.localeCompare(right)),
+		chunkIds: [...entry.chunkIds],
+		directChunkCount: entry.directChunkCount,
+		chunkCount: entry.chunkCount,
+		textChunkCount: entry.textChunkCount,
+		imageChunkCount: entry.imageChunkCount,
+		tableChunkCount: entry.tableChunkCount
+	}));
+}
+function readManifestAssetEntries(manifest, key) {
+	const rawEntries = (isRecord(manifest.files) ? manifest.files : {})[key];
+	if (!Array.isArray(rawEntries)) return [];
+	return rawEntries.filter((entry) => isRecord(entry));
+}
+function buildResultFileChunkLookup(manifest, chunks) {
+	const entries = /* @__PURE__ */ new Map();
+	for (const key of ["images", "tables"]) {
+		const assetEntries = readManifestAssetEntries(manifest, key);
+		for (const entry of assetEntries) {
+			const filePath = typeof entry.file_path === "string" && entry.file_path.trim() ? normalizeRelativePath(entry.file_path.trim()) : null;
+			if (!filePath) continue;
+			entries.set(filePath, {
+				chunkId: typeof entry.id === "string" ? entry.id : null,
+				format: typeof entry.format === "string" ? entry.format : null
+			});
+		}
+	}
+	for (const chunk of chunks) {
+		if (!chunk.assetFilePath || entries.has(chunk.assetFilePath)) continue;
+		entries.set(chunk.assetFilePath, {
+			chunkId: chunk.chunkId || null,
+			format: null
+		});
+	}
+	return entries;
+}
+function inferResultFileKind(relativePath) {
+	if (relativePath === MANIFEST_FILE_NAME) return "manifest";
+	if (relativePath === CHUNKS_FILE_NAME) return "chunks";
+	if (relativePath === FULL_MARKDOWN_FILE_NAME) return "fullMarkdown";
+	if (relativePath === KB_CSV_FILE_NAME) return "kbCsv";
+	if (relativePath === HIERARCHY_FILE_NAME) return "hierarchy";
+	if (relativePath === HIERARCHY_VIEW_FILE_NAME) return "hierarchyView";
+	if (relativePath.startsWith("images/")) return "image";
+	if (relativePath.startsWith("tables/")) return "table";
+	return "other";
+}
+function inferResultFileFormat(relativePath) {
+	return path.posix.extname(relativePath).replace(/^\./, "").trim() || null;
+}
+function isStringArray(value) {
+	return Array.isArray(value) && value.every((entry) => typeof entry === "string");
+}
+function isStoredBrowseIndex(value) {
+	if (!isRecord(value)) return false;
+	if (value.version !== 2) return false;
+	if (!isStringArray(value.chunkOrder)) return false;
+	if (!Array.isArray(value.paths) || !Array.isArray(value.resultFiles)) return false;
+	if (!value.paths.every((entry) => isRecord(entry) && typeof entry.path === "string" && (entry.parentPath === null || typeof entry.parentPath === "string") && typeof entry.depth === "number" && Number.isFinite(entry.depth) && isStringArray(entry.childPaths) && isStringArray(entry.chunkIds) && typeof entry.directChunkCount === "number" && typeof entry.chunkCount === "number" && typeof entry.textChunkCount === "number" && typeof entry.imageChunkCount === "number" && typeof entry.tableChunkCount === "number")) return false;
+	return value.resultFiles.every((entry) => isRecord(entry) && typeof entry.relativePath === "string" && typeof entry.kind === "string" && (entry.chunkId === null || typeof entry.chunkId === "string") && (entry.format === null || typeof entry.format === "string") && (entry.sizeBytes === null || typeof entry.sizeBytes === "number" && Number.isFinite(entry.sizeBytes)));
+}
+async function listResultFiles(rootDir, currentDir = rootDir) {
+	const entries = await fs.readdir(currentDir, { withFileTypes: true });
+	const files = [];
+	for (const entry of entries) {
+		const absolutePath = path.join(currentDir, entry.name);
+		if (entry.isDirectory()) {
+			files.push(...await listResultFiles(rootDir, absolutePath));
+			continue;
+		}
+		if (!entry.isFile()) continue;
+		files.push(normalizeRelativePath(path.relative(rootDir, absolutePath)));
+	}
+	return files.sort((left, right) => left.localeCompare(right));
+}
+async function buildResultFileRecords(resultDir, manifest, chunks) {
+	const lookup = buildResultFileChunkLookup(manifest, chunks);
+	const relativePaths = await listResultFiles(resultDir);
+	return Promise.all(relativePaths.map(async (relativePath) => {
+		const absolutePath = resolveResultEntryPath(resultDir, relativePath);
+		const stats = await fs.stat(absolutePath);
+		const manifestEntry = lookup.get(relativePath);
+		return {
+			relativePath,
+			kind: inferResultFileKind(relativePath),
+			chunkId: manifestEntry?.chunkId ?? null,
+			format: manifestEntry?.format ?? inferResultFileFormat(relativePath),
+			sizeBytes: stats.isFile() ? stats.size : null
+		};
+	}));
+}
+async function buildStoredBrowseIndex(resultDir, manifest, chunks) {
+	return {
+		version: 2,
+		paths: buildPathRecords(chunks),
+		chunkOrder: chunks.map((chunk) => chunk.chunkId).filter((chunkId) => chunkId.length > 0),
+		resultFiles: await buildResultFileRecords(resultDir, manifest, chunks)
+	};
+}
+async function extractKnowhereResultArchive(downloadedResult, targetDir) {
+	const zipBuffer = Buffer.isBuffer(downloadedResult.zipBytes) ? downloadedResult.zipBytes : Buffer.from(downloadedResult.zipBytes);
+	const entries = unzipSync(new Uint8Array(zipBuffer));
+	const manifestText = readZipText(entries, MANIFEST_FILE_NAME);
+	if (!manifestText) throw new Error("Knowhere result ZIP does not contain manifest.json.");
+	validateKnowhereResultChecksum(zipBuffer, parseManifest(JSON.parse(manifestText)));
+	await ensureDir(targetDir);
+	for (const [entryPath, entryBytes] of Object.entries(entries)) {
+		const outputPath = resolveResultEntryPath(targetDir, entryPath);
+		await ensureDir(path.dirname(outputPath));
+		await fs.writeFile(outputPath, entryBytes);
+	}
+}
+async function readStoredKnowhereResultSummary(resultDir) {
+	const manifest = parseManifest(await readJsonFile(path.join(resultDir, MANIFEST_FILE_NAME)));
+	const rawChunks = parseRawChunks(await readJsonFile(path.join(resultDir, CHUNKS_FILE_NAME)));
+	return {
+		manifest,
+		chunkCount: rawChunks.length,
+		statistics: normalizeStatistics(manifest, rawChunks)
+	};
+}
+async function readStoredKnowhereResultContent(resultDir) {
+	return {
+		manifest: parseManifest(await readJsonFile(path.join(resultDir, MANIFEST_FILE_NAME))),
+		chunks: parseRawChunks(await readJsonFile(path.join(resultDir, CHUNKS_FILE_NAME))).map((rawChunk) => buildChunk(rawChunk)),
+		fullMarkdown: await readTextFile(path.join(resultDir, FULL_MARKDOWN_FILE_NAME)) || "",
+		hierarchy: await readJsonFile(path.join(resultDir, HIERARCHY_FILE_NAME))
+	};
+}
+//#endregion
+export { buildStoredBrowseIndex, extractKnowhereResultArchive, isStoredBrowseIndex, readStoredKnowhereResultContent, readStoredKnowhereResultSummary, resolveResultEntryPath };

package/dist/session.d.ts

@@ -0,0 +1,11 @@
+export type ParsedConversationSessionKey = {
+    agentId: string;
+    surface: string;
+    remainder: string;
+    segments: string[];
+};
+export declare function parseConversationSessionKey(sessionKey: string | undefined): ParsedConversationSessionKey | undefined;
+export declare function normalizeConversationScopeKey(sessionKey: string | undefined): string | undefined;
+export declare function resolveSharedConversationScopeKey(sessionKey: string | undefined): string | undefined;
+export declare function findConversationSegmentValue(session: ParsedConversationSessionKey | undefined, ...keys: string[]): string | undefined;
+export declare function deriveMessageContextScopeKey(context: unknown): string | undefined;

package/dist/session.js

@@ -0,0 +1,78 @@
+import { normalizeWhitespace } from "./text.js";
+//#region src/session.ts
+const RESERVED_AGENT_SESSION_ROOTS = new Set([
+	"main",
+	"subagent",
+	"direct",
+	"global"
+]);
+const CONVERSATION_KIND_SEGMENTS = new Set([
+	"channel",
+	"chat",
+	"conversation",
+	"direct",
+	"dm",
+	"forum",
+	"group",
+	"guild",
+	"member",
+	"room",
+	"space",
+	"thread",
+	"user"
+]);
+function isRecord(value) {
+	return typeof value === "object" && value !== null;
+}
+function readNormalizedString(value) {
+	return normalizeWhitespace(value) || void 0;
+}
+function looksLikeConversationSegments(segments) {
+	return segments.some((segment) => CONVERSATION_KIND_SEGMENTS.has(normalizeWhitespace(segment).toLowerCase()));
+}
+function parseConversationSessionKey(sessionKey) {
+	const normalized = normalizeWhitespace(sessionKey);
+	if (!normalized) return;
+	const match = normalized.match(/^agent:([^:]+):(.+)$/);
+	if (!match?.[1] || !match[2]) return;
+	const remainder = match[2];
+	const segments = remainder.split(":").filter(Boolean);
+	if (segments.length < 2) return;
+	const surface = normalizeWhitespace(segments[0]).toLowerCase();
+	if (!surface || RESERVED_AGENT_SESSION_ROOTS.has(surface)) return;
+	if (!looksLikeConversationSegments(segments.slice(1))) return;
+	return {
+		agentId: normalizeWhitespace(match[1]),
+		surface,
+		remainder,
+		segments
+	};
+}
+function findConversationSegmentValue(session, ...keys) {
+	if (!session || keys.length === 0) return;
+	const expected = new Set(keys.map((key) => normalizeWhitespace(key).toLowerCase()));
+	for (let index = 0; index < session.segments.length - 1; index += 1) {
+		const segment = normalizeWhitespace(session.segments[index]).toLowerCase();
+		if (!expected.has(segment)) continue;
+		const value = normalizeWhitespace(session.segments[index + 1]);
+		if (value) return value;
+	}
+}
+function deriveMessageContextScopeKey(context) {
+	if (!isRecord(context)) return;
+	const surface = readNormalizedString(context.channelId ?? context.provider ?? context.surface)?.toLowerCase();
+	if (!surface) return;
+	const conversationId = readNormalizedString(context.conversationId);
+	const groupId = readNormalizedString(context.groupId);
+	const threadId = readNormalizedString(context.threadId) ?? readNormalizedString(context.messageThreadId);
+	if (!conversationId && !groupId && !threadId) return;
+	const parts = ["shared", surface];
+	const accountId = readNormalizedString(context.accountId);
+	if (accountId) parts.push("account", accountId);
+	if (groupId) parts.push("group", groupId);
+	if (conversationId) parts.push("conversation", conversationId);
+	if (threadId) parts.push("thread", threadId);
+	return parts.join(":");
+}
+//#endregion
+export { deriveMessageContextScopeKey, findConversationSegmentValue, parseConversationSessionKey };

package/dist/store.d.ts

@@ -0,0 +1,62 @@
+import type { KnowhereScope, PluginLogger, SaveStoredDocumentPayload, ScopeMode, StoredDocumentPayload, StoredDocumentRecord } from "./types";
+export declare class KnowhereStore {
+    private readonly rootDir;
+    private readonly scopeMode;
+    private readonly logger;
+    private readonly indexCache;
+    private readonly documentPayloadCache;
+    private readonly scopeAccessChains;
+    private readonly scopeKeyAliases;
+    private readonly sessionScopeKeysBySessionId;
+    private readonly sessionScopeKeysBySessionKey;
+    constructor(params: {
+        rootDir: string;
+        scopeMode: ScopeMode;
+        logger: PluginLogger;
+    });
+    registerScopeAlias(context: {
+        scopeKey: string;
+        sessionKey?: string;
+        sessionId?: string;
+    }): string;
+    registerMessageScope(context: {
+        sessionKey?: string;
+        sessionId?: string;
+        messageContext: unknown;
+    }): string | undefined;
+    inheritScopeAlias(context: {
+        childSessionKey?: string;
+        childSessionId?: string;
+        requesterSessionKey?: string;
+        requesterSessionId?: string;
+    }): string | undefined;
+    resolveScope(context?: {
+        agentId?: string;
+        sessionKey?: string;
+        sessionId?: string;
+    }): KnowhereScope;
+    listDocuments(scope: KnowhereScope): Promise<StoredDocumentRecord[]>;
+    loadDocumentPayload(scope: KnowhereScope, docId: string): Promise<StoredDocumentPayload | null>;
+    readResultFile(scope: KnowhereScope, docId: string, relativePath: string): Promise<{
+        document: StoredDocumentRecord;
+        relativePath: string;
+        text: string | null;
+    } | null>;
+    saveDownloadedDocument(scope: KnowhereScope, payload: SaveStoredDocumentPayload, options?: {
+        overwrite?: boolean;
+    }): Promise<StoredDocumentRecord>;
+    removeDocument(scope: KnowhereScope, docId: string): Promise<StoredDocumentRecord | null>;
+    clearScope(scope: KnowhereScope): Promise<StoredDocumentRecord[]>;
+    private getIndex;
+    private persistIndex;
+    private runWithScopeAccessLock;
+    private removeDocumentArtifacts;
+    private buildDocumentPayloadCacheKey;
+    private touchDocumentPayloadCache;
+    private deleteDocumentPayloadCache;
+    private deleteScopeDocumentPayloadCaches;
+    private loadOrBuildBrowseIndex;
+    private resolveCanonicalScopeKey;
+    private resolveKnownScopeKey;
+    private rebuildIndex;
+}