@ontos-ai/knowhere-claw 0.1.4 → 0.1.5

package/dist/kg-service.js

@@ -0,0 +1,249 @@
+import { __toESM } from "./_virtual/_rolldown/runtime.js";
+import { require_lib } from "./node_modules/.pnpm/fs-extra@11.3.4/node_modules/fs-extra/lib/index.js";
+import { require_nodejieba } from "./node_modules/.pnpm/nodejieba@2.6.0/node_modules/nodejieba/index.js";
+import { buildConnections, init_connect_builder } from "./connect-builder.js";
+import { buildKnowledgeGraph } from "./graph-builder.js";
+import path from "node:path";
+import { spawn } from "node:child_process";
+import os from "node:os";
+//#region src/kg-service.ts
+var import_lib = /* @__PURE__ */ __toESM(require_lib(), 1);
+var import_nodejieba = /* @__PURE__ */ __toESM(require_nodejieba(), 1);
+init_connect_builder();
+const DEFAULT_CONNECT_CONFIG = {
+	minKeywordOverlap: 3,
+	keywordScoreWeight: 1,
+	maxConnectionsPerChunk: 10,
+	minScoreThreshold: .8,
+	crossFileOnly: true,
+	maxContentOverlap: .8
+};
+const DEFAULT_KG_CONFIG = {
+	enableKnowledgeGraph: true,
+	kbIdSource: "disabled",
+	concurrentBuildStrategy: "queue",
+	buildTimeout: 3e5,
+	notifyOnGraphFailure: false
+};
+var KnowledgeGraphService = class {
+	config;
+	logger;
+	degradationMode = "full";
+	buildQueues = /* @__PURE__ */ new Map();
+	jiebaInitialized = false;
+	constructor(params) {
+		this.config = {
+			...DEFAULT_KG_CONFIG,
+			...params.config,
+			kbId: params.config.kbId || void 0,
+			connectConfig: {
+				...DEFAULT_CONNECT_CONFIG,
+				...params.config.connectConfig
+			}
+		};
+		this.logger = params.logger;
+	}
+	async initialize() {
+		if (!this.config.enableKnowledgeGraph) {
+			this.degradationMode = "disabled";
+			this.logger.info("Knowledge graph disabled by configuration");
+			return;
+		}
+		try {
+			await this.checkPythonEnvironment();
+			await this.initializeJieba();
+			this.degradationMode = "full";
+			this.logger.info("Knowledge graph service initialized in full mode");
+		} catch (error) {
+			this.logger.warn(`Knowledge graph initialization failed: ${error instanceof Error ? error.message : String(error)}`);
+			if (error.code === "PYTHON_MISSING") {
+				this.degradationMode = "disabled";
+				this.logger.warn("Python not found, knowledge graph disabled");
+			} else if (error.code === "NODEJIEBA_MISSING") {
+				this.degradationMode = "basic";
+				this.logger.warn("Nodejieba missing, using basic tokenization");
+			} else this.degradationMode = "disabled";
+		}
+	}
+	async checkPythonEnvironment() {
+		return new Promise((resolve, reject) => {
+			const python = spawn("python3", ["--version"]);
+			python.on("error", (err) => {
+				const error = /* @__PURE__ */ new Error("Python3 not found in PATH");
+				error.code = "PYTHON_MISSING";
+				reject(error);
+			});
+			python.on("close", (code) => {
+				if (code === 0) {
+					this.logger.info("Python3 environment detected");
+					resolve();
+				} else {
+					const error = /* @__PURE__ */ new Error(`Python3 check failed with code ${code}`);
+					error.code = "PYTHON_MISSING";
+					reject(error);
+				}
+			});
+		});
+	}
+	async initializeJieba() {
+		try {
+			import_nodejieba.load();
+			this.jiebaInitialized = true;
+			this.logger.info("Nodejieba initialized successfully");
+		} catch (error) {
+			const err = /* @__PURE__ */ new Error("Failed to initialize nodejieba");
+			err.code = "NODEJIEBA_MISSING";
+			throw err;
+		}
+	}
+	async extractKeywords(text, topK = 20) {
+		if (this.degradationMode === "disabled") return [];
+		if (this.degradationMode === "full" && this.jiebaInitialized) try {
+			const rawKeywords = import_nodejieba.extract(text, topK);
+			return (Array.isArray(rawKeywords) ? rawKeywords : []).map((item) => typeof item === "string" ? item : String(item.word || item)).filter((kw) => {
+				if (kw.length <= 1) return false;
+				if (/^\d+$/.test(kw)) return false;
+				return true;
+			}).slice(0, topK);
+		} catch (error) {
+			this.logger.warn(`Jieba extraction failed, falling back to basic: ${error}`);
+			this.degradationMode = "basic";
+		}
+		return text.split(/\s+/).filter((w) => w.length > 1).slice(0, topK);
+	}
+	resolveKbId(context) {
+		if (!this.config.enableKnowledgeGraph) return null;
+		if (this.config.kbId?.trim()) return this.config.kbId.trim();
+		switch (this.config.kbIdSource || "disabled") {
+			case "disabled": return null;
+			case "agent": return context.agentId ? `agent-${context.agentId}` : null;
+			case "agent-session":
+				if (context.agentId && context.sessionId) return `agent-${context.agentId}-session-${context.sessionId}`;
+				if (context.agentId && context.sessionKey) return `agent-${context.agentId}-session-${context.sessionKey}`;
+				return context.agentId ? `agent-${context.agentId}` : null;
+			case "global": return "global";
+			default: return null;
+		}
+	}
+	getKbPath(kbId) {
+		return path.join(os.homedir(), ".knowhere", kbId);
+	}
+	async ensureKbDirectory(kbId) {
+		const kbPath = this.getKbPath(kbId);
+		await import_lib.default.ensureDir(kbPath);
+		return kbPath;
+	}
+	async saveDocumentToKb(params) {
+		const kbPath = await this.ensureKbDirectory(params.kbId);
+		const docDir = path.join(kbPath, params.docId);
+		await import_lib.default.ensureDir(docDir);
+		await import_lib.default.copy(params.sourcePath, docDir, { overwrite: true });
+		const keywordsPath = path.join(docDir, "keywords.json");
+		await import_lib.default.writeJSON(keywordsPath, params.keywords, { spaces: 2 });
+		const metadataPath = path.join(docDir, "metadata.json");
+		await import_lib.default.writeJSON(metadataPath, params.metadata, { spaces: 2 });
+		this.logger.info(`Document saved to knowledge base: kb=${params.kbId} doc=${params.docId}`);
+	}
+	async scheduleBuild(kbId, task) {
+		if ((this.config.concurrentBuildStrategy || "queue") === "skip") {
+			if (this.buildQueues.has(kbId)) {
+				this.logger.info(`Skipping KG build for ${kbId} (already building)`);
+				return;
+			}
+		}
+		const next = (this.buildQueues.get(kbId) || Promise.resolve()).then(() => task()).catch((err) => {
+			this.logger.error(`KG build failed for ${kbId}: ${err}`);
+			throw err;
+		}).finally(() => {
+			if (this.buildQueues.get(kbId) === next) this.buildQueues.delete(kbId);
+		});
+		this.buildQueues.set(kbId, next);
+		await next;
+	}
+	async buildKnowledgeGraph(kbId) {
+		const kbPath = this.getKbPath(kbId);
+		const docs = await import_lib.default.readdir(kbPath);
+		const docDirs = [];
+		for (const doc of docs) {
+			const docPath = path.join(kbPath, doc);
+			if ((await import_lib.default.stat(docPath)).isDirectory() && doc !== "knowledge_graph.json" && doc !== "chunk_stats.json" && doc !== "kb_metadata.json") docDirs.push(doc);
+		}
+		if (docDirs.length < 2) {
+			this.logger.info(`Not enough documents for graph building (need >=2, have ${docDirs.length}), skipping`);
+			return;
+		}
+		this.logger.info(`Building knowledge graph for ${docDirs.length} documents in kb=${kbId}`);
+		try {
+			const allChunks = [];
+			for (const docDir of docDirs) {
+				const chunksPath = path.join(kbPath, docDir, "chunks.json");
+				if (await import_lib.default.pathExists(chunksPath)) {
+					const chunksData = await import_lib.default.readJSON(chunksPath);
+					if (chunksData.chunks && Array.isArray(chunksData.chunks)) allChunks.push(...chunksData.chunks);
+				}
+			}
+			if (allChunks.length === 0) {
+				this.logger.warn(`No chunks found in kb=${kbId}, skipping graph build`);
+				return;
+			}
+			this.logger.info(`Loaded ${allChunks.length} chunks from ${docDirs.length} documents`);
+			const connections = buildConnections(allChunks, this.config.connectConfig, this.logger);
+			this.logger.info(`Built ${connections.length} connections`);
+			const chunkStatsPath = path.join(kbPath, "chunk_stats.json");
+			let chunkStats = {};
+			if (await import_lib.default.pathExists(chunkStatsPath)) chunkStats = await import_lib.default.readJSON(chunkStatsPath);
+			const knowledgeGraph = buildKnowledgeGraph(allChunks, connections, chunkStats, this.jiebaInitialized, this.logger);
+			const graphFile = path.join(kbPath, "knowledge_graph.json");
+			await import_lib.default.writeJSON(graphFile, knowledgeGraph, { spaces: 2 });
+			this.logger.info(`Knowledge graph saved to ${graphFile}`);
+			await this.updateKbMetadata(kbPath, {
+				lastUpdated: (/* @__PURE__ */ new Date()).toISOString(),
+				documentCount: docDirs.length
+			});
+			this.logger.info(`Knowledge graph built successfully for kb=${kbId}`);
+		} catch (error) {
+			this.logger.error(`Failed to build knowledge graph: ${error}`);
+			throw error;
+		}
+	}
+	async updateKbMetadata(kbPath, updates) {
+		const metadataPath = path.join(kbPath, "kb_metadata.json");
+		let metadata = {};
+		if (await import_lib.default.pathExists(metadataPath)) metadata = await import_lib.default.readJSON(metadataPath);
+		const updated = {
+			...metadata,
+			...updates
+		};
+		await import_lib.default.writeJSON(metadataPath, updated, { spaces: 2 });
+	}
+	async queryGraph(kbId, fileKey) {
+		const graphPath = path.join(this.getKbPath(kbId), "knowledge_graph.json");
+		if (!await import_lib.default.pathExists(graphPath)) return [];
+		const graph = await import_lib.default.readJSON(graphPath);
+		if (!fileKey) return graph.edges;
+		return graph.edges.filter((edge) => edge.source === fileKey || edge.target === fileKey);
+	}
+	async getKnowledgeGraph(kbId) {
+		const graphPath = path.join(this.getKbPath(kbId), "knowledge_graph.json");
+		if (!await import_lib.default.pathExists(graphPath)) return null;
+		return await import_lib.default.readJSON(graphPath);
+	}
+	async listKnowledgeBases() {
+		const knowhereRoot = path.join(os.homedir(), ".knowhere");
+		if (!await import_lib.default.pathExists(knowhereRoot)) return [];
+		return (await import_lib.default.readdir(knowhereRoot, { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => e.name);
+	}
+	async getKbMetadata(kbId) {
+		const metadataPath = path.join(this.getKbPath(kbId), "kb_metadata.json");
+		if (!await import_lib.default.pathExists(metadataPath)) return null;
+		return await import_lib.default.readJSON(metadataPath);
+	}
+	isEnabled() {
+		return this.degradationMode !== "disabled";
+	}
+	getDegradationMode() {
+		return this.degradationMode;
+	}
+};
+//#endregion
+export { KnowledgeGraphService };

package/dist/node_modules/.pnpm/@mapbox_node-pre-gyp@1.0.11/node_modules/@mapbox/node-pre-gyp/lib/node-pre-gyp.js

@@ -0,0 +1,212 @@
+import { __commonJSMin, __require } from "../../../../../../../_virtual/_rolldown/runtime.js";
+import { require_s3_setup } from "./util/s3_setup.js";
+import { require_nopt } from "../../../../../nopt@5.0.0/node_modules/nopt/lib/nopt.js";
+import { require_log } from "../../../../../npmlog@5.0.1/node_modules/npmlog/log.js";
+import { require_napi } from "./util/napi.js";
+import { require_pre_binding } from "./pre-binding.js";
+import { require_package } from "../package.js";
+//#region node_modules/.pnpm/@mapbox+node-pre-gyp@1.0.11/node_modules/@mapbox/node-pre-gyp/lib/node-pre-gyp.js
+var require_node_pre_gyp = /* @__PURE__ */ __commonJSMin(((exports, module) => {
+	/**
+	* Module exports.
+	*/
+	module.exports = exports;
+	/**
+	* Module dependencies.
+	*/
+	exports.mockS3Http = require_s3_setup().get_mockS3Http();
+	exports.mockS3Http("on");
+	const mocking = exports.mockS3Http("get");
+	const fs = __require("fs");
+	const path = __require("path");
+	const nopt = require_nopt();
+	const log = require_log();
+	log.disableProgress();
+	const napi = require_napi();
+	const EE = __require("events").EventEmitter;
+	const inherits = __require("util").inherits;
+	const cli_commands = [
+		"clean",
+		"install",
+		"reinstall",
+		"build",
+		"rebuild",
+		"package",
+		"testpackage",
+		"publish",
+		"unpublish",
+		"info",
+		"testbinary",
+		"reveal",
+		"configure"
+	];
+	const aliases = {};
+	log.heading = "node-pre-gyp";
+	if (mocking) log.warn(`mocking s3 to ${process.env.node_pre_gyp_mock_s3}`);
+	Object.defineProperty(exports, "find", {
+		get: function() {
+			return require_pre_binding().find;
+		},
+		enumerable: true
+	});
+	function Run({ package_json_path = "./package.json", argv }) {
+		this.package_json_path = package_json_path;
+		this.commands = {};
+		const self = this;
+		cli_commands.forEach((command) => {
+			self.commands[command] = function(argvx, callback) {
+				log.verbose("command", command, argvx);
+				return __require("./" + command)(self, argvx, callback);
+			};
+		});
+		this.parseArgv(argv);
+		this.binaryHostSet = false;
+	}
+	inherits(Run, EE);
+	exports.Run = Run;
+	const proto = Run.prototype;
+	/**
+	* Export the contents of the package.json.
+	*/
+	proto.package = require_package();
+	/**
+	* nopt configuration definitions
+	*/
+	proto.configDefs = {
+		help: Boolean,
+		arch: String,
+		debug: Boolean,
+		directory: String,
+		proxy: String,
+		loglevel: String
+	};
+	/**
+	* nopt shorthands
+	*/
+	proto.shorthands = {
+		release: "--no-debug",
+		C: "--directory",
+		debug: "--debug",
+		j: "--jobs",
+		silent: "--loglevel=silent",
+		silly: "--loglevel=silly",
+		verbose: "--loglevel=verbose"
+	};
+	/**
+	* expose the command aliases for the bin file to use.
+	*/
+	proto.aliases = aliases;
+	/**
+	* Parses the given argv array and sets the 'opts', 'argv',
+	* 'command', and 'package_json' properties.
+	*/
+	proto.parseArgv = function parseOpts(argv) {
+		this.opts = nopt(this.configDefs, this.shorthands, argv);
+		this.argv = this.opts.argv.remain.slice();
+		const commands = this.todo = [];
+		argv = this.argv.map((arg) => {
+			if (arg in this.aliases) arg = this.aliases[arg];
+			return arg;
+		});
+		argv.slice().forEach((arg) => {
+			if (arg in this.commands) {
+				const args = argv.splice(0, argv.indexOf(arg));
+				argv.shift();
+				if (commands.length > 0) commands[commands.length - 1].args = args;
+				commands.push({
+					name: arg,
+					args: []
+				});
+			}
+		});
+		if (commands.length > 0) commands[commands.length - 1].args = argv.splice(0);
+		let package_json_path = this.package_json_path;
+		if (this.opts.directory) package_json_path = path.join(this.opts.directory, package_json_path);
+		this.package_json = JSON.parse(fs.readFileSync(package_json_path));
+		this.todo = napi.expand_commands(this.package_json, this.opts, commands);
+		const npm_config_prefix = "npm_config_";
+		Object.keys(process.env).forEach((name) => {
+			if (name.indexOf(npm_config_prefix) !== 0) return;
+			const val = process.env[name];
+			if (name === npm_config_prefix + "loglevel") log.level = val;
+			else {
+				name = name.substring(11);
+				if (name === "argv") if (this.opts.argv && this.opts.argv.remain && this.opts.argv.remain.length) {} else this.opts[name] = val;
+				else this.opts[name] = val;
+			}
+		});
+		if (this.opts.loglevel) log.level = this.opts.loglevel;
+		log.resume();
+	};
+	/**
+	* allow the binary.host property to be set at execution time.
+	*
+	* for this to take effect requires all the following to be true.
+	* - binary is a property in package.json
+	* - binary.host is falsey
+	* - binary.staging_host is not empty
+	* - binary.production_host is not empty
+	*
+	* if any of the previous checks fail then the function returns an empty string
+	* and makes no changes to package.json's binary property.
+	*
+	*
+	* if command is "publish" then the default is set to "binary.staging_host"
+	* if command is not "publish" the the default is set to "binary.production_host"
+	*
+	* if the command-line option '--s3_host' is set to "staging" or "production" then
+	* "binary.host" is set to the specified "staging_host" or "production_host". if
+	* '--s3_host' is any other value an exception is thrown.
+	*
+	* if '--s3_host' is not present then "binary.host" is set to the default as above.
+	*
+	* this strategy was chosen so that any command other than "publish" or "unpublish" uses "production"
+	* as the default without requiring any command-line options but that "publish" and "unpublish" require
+	* '--s3_host production_host' to be specified in order to *really* publish (or unpublish). publishing
+	* to staging can be done freely without worrying about disturbing any production releases.
+	*/
+	proto.setBinaryHostProperty = function(command) {
+		if (this.binaryHostSet) return this.package_json.binary.host;
+		const p = this.package_json;
+		if (!p || !p.binary || p.binary.host) return "";
+		if (!p.binary.staging_host || !p.binary.production_host) return "";
+		let target = "production_host";
+		if (command === "publish" || command === "unpublish") target = "staging_host";
+		const npg_s3_host = process.env.node_pre_gyp_s3_host;
+		if (npg_s3_host === "staging" || npg_s3_host === "production") target = `${npg_s3_host}_host`;
+		else if (this.opts["s3_host"] === "staging" || this.opts["s3_host"] === "production") target = `${this.opts["s3_host"]}_host`;
+		else if (this.opts["s3_host"] || npg_s3_host) throw new Error(`invalid s3_host ${this.opts["s3_host"] || npg_s3_host}`);
+		p.binary.host = p.binary[target];
+		this.binaryHostSet = true;
+		return p.binary.host;
+	};
+	/**
+	* Returns the usage instructions for node-pre-gyp.
+	*/
+	proto.usage = function usage() {
+		return [
+			"",
+			"  Usage: node-pre-gyp <command> [options]",
+			"",
+			"  where <command> is one of:",
+			cli_commands.map((c) => {
+				return "    - " + c + " - " + __require("./" + c).usage;
+			}).join("\n"),
+			"",
+			"node-pre-gyp@" + this.version + "  " + path.resolve(__dirname, ".."),
+			"node@" + process.versions.node
+		].join("\n");
+	};
+	/**
+	* Version number getter.
+	*/
+	Object.defineProperty(proto, "version", {
+		get: function() {
+			return this.package.version;
+		},
+		enumerable: true
+	});
+}));
+//#endregion
+export default require_node_pre_gyp();
+export { require_node_pre_gyp };

package/dist/node_modules/.pnpm/@mapbox_node-pre-gyp@1.0.11/node_modules/@mapbox/node-pre-gyp/lib/pre-binding.js

@@ -0,0 +1,35 @@
+import { __commonJSMin, __require } from "../../../../../../../_virtual/_rolldown/runtime.js";
+import { require_napi } from "./util/napi.js";
+import { require_versioning } from "./util/versioning.js";
+import { require_node_pre_gyp } from "./node-pre-gyp.js";
+//#region node_modules/.pnpm/@mapbox+node-pre-gyp@1.0.11/node_modules/@mapbox/node-pre-gyp/lib/pre-binding.js
+var require_pre_binding = /* @__PURE__ */ __commonJSMin(((exports, module) => {
+	const npg = require_node_pre_gyp();
+	const versioning = require_versioning();
+	const napi = require_napi();
+	const existsSync = __require("fs").existsSync || __require("path").existsSync;
+	const path = __require("path");
+	module.exports = exports;
+	exports.usage = "Finds the require path for the node-pre-gyp installed module";
+	exports.validate = function(package_json, opts) {
+		versioning.validate_config(package_json, opts);
+	};
+	exports.find = function(package_json_path, opts) {
+		if (!existsSync(package_json_path)) throw new Error(package_json_path + "does not exist");
+		const prog = new npg.Run({
+			package_json_path,
+			argv: process.argv
+		});
+		prog.setBinaryHostProperty();
+		const package_json = prog.package_json;
+		versioning.validate_config(package_json, opts);
+		let napi_build_version;
+		if (napi.get_napi_build_versions(package_json, opts)) napi_build_version = napi.get_best_napi_build_version(package_json, opts);
+		opts = opts || {};
+		if (!opts.module_root) opts.module_root = path.dirname(package_json_path);
+		return versioning.evaluate(package_json, opts, napi_build_version).module;
+	};
+}));
+//#endregion
+export default require_pre_binding();
+export { require_pre_binding };