@yugoo-design/dsh-design-knowledge-ladybug 0.1.0-beta.3 → 0.1.0

package/lib/semantic-index.js

@@ -1,367 +1,2 @@
-import { existsSync } from "node:fs";
-import { isAbsolute, resolve } from "node:path";
-import { SEMANTIC_TEXT_VERSION, normalizeVector, semanticDigest, semanticEdges } from "@yugoo-design/dsh-design-core";
-import { fork } from "node:child_process";
-import { homedir } from "node:os";
-//#region src/semantic-index.ts
-const localEmbeddingFailures = {
-	runtime: "本地 embedding 运行依赖不可用；请按当前操作系统安装依赖（包括原生模块），不要复用其他平台的 node_modules。",
-	model: "本地 embedding 模型加载失败；请检查固定版本模型缓存，离线模式需要先准备完整模型文件。",
-	inference: "本地 embedding 推理失败；请检查输入限制和本地内存、CPU 运行资源。"
-};
-/** Disposable, project-local cache. Never stores Knowledge/GRA facts. */
-var MemorySemanticCache = class {
-	limit;
-	byteLimit;
-	values = /* @__PURE__ */ new Map();
-	bytes = 0;
-	constructor(limit = 300, byteLimit = 4e6) {
-		this.limit = limit;
-		this.byteLimit = byteLimit;
-		if (!Number.isSafeInteger(limit) || limit < 1 || !Number.isSafeInteger(byteLimit) || byteLimit < 1) throw new Error("Invalid volatile cache bounds");
-	}
-	async readSemanticCache(key) {
-		return this.values.get(key);
-	}
-	async writeSemanticCache(key, value) {
-		const size = Buffer.byteLength(key) + Buffer.byteLength(value);
-		if (size > this.byteLimit) return;
-		const old = this.values.get(key);
-		if (old !== void 0) {
-			this.bytes -= Buffer.byteLength(key) + Buffer.byteLength(old);
-			this.values.delete(key);
-		}
-		while (this.values.size && (this.values.size >= this.limit || this.bytes + size > this.byteLimit)) {
-			const [id, text] = this.values.entries().next().value;
-			this.values.delete(id);
-			this.bytes -= Buffer.byteLength(id) + Buffer.byteLength(text);
-		}
-		this.values.set(key, value);
-		this.bytes += size;
-	}
-};
-async function stopEmbeddingChild(worker) {
-	if (worker.exitCode !== null || worker.signalCode !== null) return;
-	await new Promise((resolveExit) => {
-		const timer = setTimeout(() => worker.kill("SIGKILL"), 2e3);
-		worker.once("exit", () => {
-			clearTimeout(timer);
-			resolveExit();
-		});
-		worker.kill("SIGTERM");
-	});
-}
-var DesignSemanticIndex = class {
-	idleMs;
-	config;
-	identity;
-	get publicationIdentity() {
-		return semanticDigest(this.identity + ":similarity-v1:" + this.config.threshold);
-	}
-	jobs = /* @__PURE__ */ new Map();
-	tail = Promise.resolve();
-	closed = false;
-	constructor(config = {}, idleMs = 3e5) {
-		this.idleMs = idleMs;
-		if (!Number.isSafeInteger(idleMs) || idleMs < 1) throw new Error("Invalid semantic idle timeout");
-		this.config = {
-			provider: "local",
-			model: "Xenova/multilingual-e5-small",
-			revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
-			cacheDir: resolve(homedir(), ".cache/design-system/embeddings"),
-			localOnly: false,
-			prefix: "query: ",
-			threshold: .8,
-			...config
-		};
-		if (![
-			"local",
-			"remote",
-			"disabled"
-		].includes(this.config.provider)) throw new Error("Unknown embedding provider");
-		if (!this.config.model || !this.config.revision || !isAbsolute(this.config.cacheDir)) throw new Error("Invalid embedding configuration");
-		if (!(this.config.threshold >= 0 && this.config.threshold <= 1)) throw new Error("Invalid semantic threshold");
-		if (this.config.provider === "remote") {
-			if (!this.config.endpoint || !config.model || !config.revision) throw new Error("Remote embedding requires explicit endpoint, model and revision");
-			const url = new URL(this.config.endpoint);
-			if (!["https:", "http:"].includes(url.protocol) || url.username || url.password) throw new Error("Invalid embedding endpoint");
-			if (url.protocol === "http:" && ![
-				"localhost",
-				"127.0.0.1",
-				"[::1]"
-			].includes(url.hostname)) throw new Error("Remote embedding requires HTTPS except loopback");
-		}
-		this.identity = semanticDigest(JSON.stringify({
-			provider: this.config.provider,
-			model: this.config.model,
-			revision: this.config.revision,
-			endpoint: this.config.endpoint,
-			prefix: this.config.prefix,
-			text: SEMANTIC_TEXT_VERSION,
-			pooling: "mean-token-windows-v1",
-			dtype: "q8",
-			engine: "transformers-3.8.1"
-		}));
-	}
-	read(project, graph, cache) {
-		const units = (graph.semanticUnits ?? []).map((unit) => ({
-			...unit,
-			fieldPointers: unit.fieldPointers?.slice()
-		}));
-		const snapshot = {
-			checkpoint: graph.checkpoint,
-			semanticUnits: units
-		};
-		const key = semanticDigest(graph.checkpoint + (graph.generation ?? "") + this.publicationIdentity + (cache ? "persistent" : "memory") + JSON.stringify(units.map((u) => [
-			u.id,
-			u.nodeId,
-			u.file,
-			u.pointer,
-			u.fieldPointers,
-			u.digest
-		])));
-		if (this.closed || this.config.provider === "disabled") return {
-			edges: [],
-			status: {
-				state: "disabled",
-				completed: 0,
-				total: units.length,
-				model: this.config.model
-			}
-		};
-		let job = this.jobs.get(project);
-		if (job?.key !== key) {
-			if (this.jobs.size >= 8 && !job) return {
-				edges: [],
-				status: {
-					state: "error",
-					completed: 0,
-					total: units.length,
-					model: this.config.model,
-					message: "语义索引同时最多处理8个项目。"
-				}
-			};
-			job?.abort.abort();
-			clearTimeout(job?.idle);
-			job = {
-				key,
-				status: {
-					state: "indexing",
-					completed: 0,
-					total: units.length,
-					model: this.config.model,
-					cache: cache ? "persistent" : "memory",
-					message: this.config.provider === "local" ? "本地索引准备中；首次可能下载模型，文本不上传" : "使用已配置远程 embedding 服务处理设计文本"
-				},
-				edges: [],
-				abort: new AbortController(),
-				memory: job?.memory ?? new MemorySemanticCache()
-			};
-			this.jobs.set(project, job);
-			const current = job;
-			this.tail = this.tail.catch(() => {}).then(async () => {
-				if (current.abort.signal.aborted) return;
-				try {
-					await this.build(snapshot, cache, current);
-				} catch (error) {
-					const phase = error?.embeddingFailure;
-					const detail = phase && Object.hasOwn(localEmbeddingFailures, phase) ? localEmbeddingFailures[phase] : "语义索引失败；请检查本地运行资源或已配置的 embedding 服务。";
-					if (!current.abort.signal.aborted) current.status = {
-						...current.status,
-						state: "error",
-						message: detail + " 保留结构图，不自动上传或重试。"
-					};
-				}
-			});
-			current.settled = this.tail;
-		}
-		clearTimeout(job.idle);
-		if (!job.waiters) {
-			job.idle = setTimeout(() => this.cancelProject(project), this.idleMs);
-			job.idle.unref();
-		}
-		return {
-			edges: job.edges,
-			status: { ...job.status }
-		};
-	}
-	/** Await the existing inference job for background persistence; no polling and
-	* no project write lock held during model execution. Partial edges never escape. */
-	async compute(project, graph, cache, signal) {
-		signal?.throwIfAborted();
-		const status = this.read(project, graph, cache);
-		const job = this.jobs.get(project);
-		if (!job || status.status.state === "disabled" || status.status.state === "error") throw new Error(status.status.message ?? "Semantic indexing unavailable");
-		clearTimeout(job.idle);
-		job.waiters = (job.waiters ?? 0) + 1;
-		let rejectAbort;
-		const cancelled = new Promise((_resolve, reject) => {
-			rejectAbort = reject;
-		});
-		const abort = () => rejectAbort(signal?.reason ?? /* @__PURE__ */ new Error("Semantic consumer cancelled"));
-		signal?.addEventListener("abort", abort, { once: true });
-		try {
-			signal?.throwIfAborted();
-			await Promise.race([job.settled, cancelled]);
-			signal?.throwIfAborted();
-			job.abort.signal.throwIfAborted();
-			if (this.jobs.get(project) !== job || job.status.state !== "ready") throw new Error(job.status.message ?? "Semantic indexing did not complete");
-			return {
-				edges: structuredClone(job.edges),
-				status: { ...job.status }
-			};
-		} finally {
-			signal?.removeEventListener("abort", abort);
-			job.waiters--;
-			if (!job.waiters && signal?.aborted && job.status.state === "indexing") job.abort.abort(signal.reason);
-			if (this.jobs.get(project) === job && !job.waiters) {
-				job.idle = setTimeout(() => this.cancelProject(project), this.idleMs);
-				job.idle.unref();
-			}
-		}
-	}
-	cancelProject(project) {
-		const job = this.jobs.get(project);
-		clearTimeout(job?.idle);
-		job?.abort.abort();
-		this.jobs.delete(project);
-	}
-	async close() {
-		this.closed = true;
-		for (const project of this.jobs.keys()) this.cancelProject(project);
-		await this.tail;
-	}
-	async build(graph, cache, job) {
-		const units = graph.semanticUnits ?? [], vectors = [], missing = [];
-		const signal = job.abort.signal;
-		for (let i = 0; i < units.length; i++) {
-			signal.throwIfAborted();
-			const key = semanticDigest(this.identity + units[i].digest);
-			let value;
-			if (cache && job.status.cache === "persistent") try {
-				value = await cache.readSemanticCache(key);
-			} catch {
-				job.status.cache = "memory";
-			}
-			value ??= await job.memory.readSemanticCache(key);
-			if (value) try {
-				vectors[i] = normalizeVector(JSON.parse(value));
-				job.status.completed++;
-				continue;
-			} catch {}
-			missing.push(i);
-		}
-		let worker;
-		const kill = () => {
-			worker?.kill("SIGTERM");
-		};
-		signal.addEventListener("abort", kill, { once: true });
-		try {
-			if (missing.length && this.config.provider === "local") {
-				const built = new URL("./embedding-worker.js", import.meta.url);
-				worker = fork(existsSync(built) ? built : new URL("./embedding-worker.mjs", import.meta.url), [], {
-					windowsHide: true,
-					execArgv: ["--max-old-space-size=512"],
-					stdio: [
-						"ignore",
-						"ignore",
-						"ignore",
-						"ipc"
-					]
-				});
-			}
-			for (let start = 0; start < missing.length; start += 4) {
-				signal.throwIfAborted();
-				const indices = missing.slice(start, start + 4), texts = indices.map((i) => units[i].text);
-				const values = worker ? await this.localBatch(worker, texts, job) : await this.remoteBatch(texts, signal);
-				if (!Array.isArray(values) || values.length !== texts.length) throw new Error("Embedding batch count mismatch");
-				for (let offset = 0; offset < indices.length; offset++) {
-					signal.throwIfAborted();
-					const i = indices[offset], vector = normalizeVector(values[offset]);
-					vectors[i] = vector;
-					const key = semanticDigest(this.identity + units[i].digest), value = JSON.stringify(vector);
-					await job.memory.writeSemanticCache(key, value);
-					if (cache && job.status.cache === "persistent") try {
-						await cache.writeSemanticCache(key, value);
-					} catch {
-						job.status.cache = "memory";
-					}
-					job.status.completed++;
-				}
-			}
-			signal.throwIfAborted();
-			job.edges = semanticEdges(units, vectors, `${this.config.model}@${this.config.revision}/${this.identity.slice(0, 12)}`, graph.checkpoint, this.config.threshold);
-			job.status = {
-				...job.status,
-				state: "ready",
-				message: `${job.edges.length} 条跨文件语义候选；相似度不等于确认关系`
-			};
-		} finally {
-			signal.removeEventListener("abort", kill);
-			if (worker) await stopEmbeddingChild(worker);
-		}
-	}
-	localBatch(worker, texts, job) {
-		return new Promise((resolveBatch, reject) => {
-			const cleanup = () => {
-				clearTimeout(timer);
-				worker.off("message", message);
-				worker.off("error", failure);
-				worker.off("exit", failure);
-				job.abort.signal.removeEventListener("abort", failure);
-			};
-			const failure = () => {
-				cleanup();
-				reject(/* @__PURE__ */ new Error("Embedding worker stopped"));
-			};
-			const message = (data) => {
-				if (data.progress) {
-					job.status.message = data.progress;
-					return;
-				}
-				cleanup();
-				if (data.error || !data.vectors) reject(Object.assign(/* @__PURE__ */ new Error("Embedding worker failed"), { embeddingFailure: data.error }));
-				else resolveBatch(data.vectors);
-			};
-			const timer = setTimeout(() => {
-				failure();
-				worker.kill("SIGTERM");
-			}, 18e4);
-			worker.on("message", message);
-			worker.once("error", failure);
-			worker.once("exit", failure);
-			job.abort.signal.addEventListener("abort", failure, { once: true });
-			if (job.abort.signal.aborted) {
-				failure();
-				return;
-			}
-			worker.send({
-				texts,
-				config: this.config
-			}, (error) => {
-				if (error) failure();
-			});
-		});
-	}
-	async remoteBatch(texts, signal) {
-		const response = await fetch(this.config.endpoint, {
-			method: "POST",
-			redirect: "error",
-			headers: {
-				"Content-Type": "application/json",
-				...this.config.apiKey ? { Authorization: `Bearer ${this.config.apiKey}` } : {}
-			},
-			body: JSON.stringify({
-				model: this.config.model,
-				input: texts.map((t) => this.config.prefix + t)
-			}),
-			signal: AbortSignal.any([signal, AbortSignal.timeout(6e4)])
-		});
-		if (!response.ok) throw new Error("Remote embedding request failed");
-		const data = await response.json();
-		if (!Array.isArray(data.data) || data.data.length !== texts.length || new Set(data.data.map((d) => d.index)).size !== texts.length || data.data.some((d) => !Number.isInteger(d.index) || d.index < 0 || d.index >= texts.length)) throw new Error("Invalid remote embedding response");
-		return data.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
-	}
-};
-//#endregion
+import { n as MemorySemanticCache, r as stopEmbeddingChild, t as DesignSemanticIndex } from "./semantic-index-CigpMVTO.js";
 export { DesignSemanticIndex, MemorySemanticCache, stopEmbeddingChild };