@yugoo-design/dsh-design-knowledge-ladybug 0.1.0-beta.2 → 0.1.0-beta.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -0
- package/lib/{dsh-plugin-C0TG9tJI.js → dsh-plugin-BWO-rNYF.js} +67 -4
- package/lib/dsh-plugin.js +1 -2
- package/lib/embedding-diagnostic-CvIl3TWJ.js +104 -0
- package/lib/embedding-worker.js +12 -2
- package/lib/index.d.ts +7 -0
- package/lib/index.js +2 -2
- package/lib/semantic-index-CigpMVTO.js +599 -0
- package/lib/semantic-index.d.ts +1 -0
- package/lib/semantic-index.js +1 -366
- package/licenses/dsh-distribution.json +16 -8
- package/package.json +3 -3
|
@@ -0,0 +1,599 @@
|
|
|
1
|
+
import { n as embeddingDiagnosticSuffix, r as readEmbeddingDiagnostic } from "./embedding-diagnostic-CvIl3TWJ.js";
|
|
2
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
3
|
+
import { copyFile, lstat, mkdir, open, rename } from "node:fs/promises";
|
|
4
|
+
import { constants, createReadStream, existsSync } from "node:fs";
|
|
5
|
+
import { dirname, isAbsolute, join, resolve } from "node:path";
|
|
6
|
+
import { SEMANTIC_TEXT_VERSION, normalizeVector, semanticDigest, semanticEdges } from "@yugoo-design/dsh-design-core";
|
|
7
|
+
import { lock } from "proper-lockfile";
|
|
8
|
+
import { execFile, fork } from "node:child_process";
|
|
9
|
+
import { homedir } from "node:os";
|
|
10
|
+
import { promisify } from "node:util";
|
|
11
|
+
import { EnvHttpProxyAgent, fetch as fetch$1 } from "undici";
|
|
12
|
+
//#region src/embedding-model-assets.ts
|
|
13
|
+
const run = promisify(execFile);
|
|
14
|
+
const DEFAULT_EMBEDDING_ASSETS = {
|
|
15
|
+
model: "Xenova/multilingual-e5-small",
|
|
16
|
+
revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
|
|
17
|
+
files: [
|
|
18
|
+
{
|
|
19
|
+
path: "config.json",
|
|
20
|
+
bytes: 658,
|
|
21
|
+
hash: "4104f38273cc595fd9500fd243124e9f6cf383dc",
|
|
22
|
+
kind: "git-blob-sha1"
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
path: "tokenizer_config.json",
|
|
26
|
+
bytes: 443,
|
|
27
|
+
hash: "059214673d9d6d2ee319411e2ffec8c024b816d5",
|
|
28
|
+
kind: "git-blob-sha1"
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
path: "tokenizer.json",
|
|
32
|
+
bytes: 17082730,
|
|
33
|
+
hash: "0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39",
|
|
34
|
+
kind: "sha256"
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
path: "onnx/model_quantized.onnx",
|
|
38
|
+
bytes: 118308185,
|
|
39
|
+
hash: "f80102d3f2a1229f387d3c81909990d8945513e347b0eab049f7de3c6f98c193",
|
|
40
|
+
kind: "sha256"
|
|
41
|
+
}
|
|
42
|
+
]
|
|
43
|
+
};
|
|
44
|
+
var EmbeddingAssetError = class extends Error {
|
|
45
|
+
code;
|
|
46
|
+
constructor(code) {
|
|
47
|
+
super(`Embedding resource preparation: ${code}`);
|
|
48
|
+
this.code = code;
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
const hasProcessProxy = (env) => (env.https_proxy ?? env.HTTPS_PROXY) !== void 0 || (env.http_proxy ?? env.HTTP_PROXY) !== void 0;
|
|
52
|
+
const hasProcessBypass = (env) => (env.no_proxy ?? env.NO_PROXY) !== void 0;
|
|
53
|
+
/** Pure projection; explicit process proxy configuration always wins. No env mutation. */
|
|
54
|
+
function modelProxyOptions(platform, env, system = "") {
|
|
55
|
+
if (hasProcessProxy(env)) return void 0;
|
|
56
|
+
if (platform !== "darwin") return void 0;
|
|
57
|
+
const value = (key) => system.match(new RegExp(`^\\s*${key}\\s*:\\s*([^\\r\\n]+)`, "m"))?.[1]?.trim();
|
|
58
|
+
if (value("HTTPSEnable") !== "1") return void 0;
|
|
59
|
+
const host = value("HTTPSProxy"), port = Number(value("HTTPSPort"));
|
|
60
|
+
if (!host || !/^(?:[a-zA-Z0-9.-]+|[a-fA-F0-9:]+)$/.test(host) || !Number.isInteger(port) || port < 1 || port > 65535) throw new EmbeddingAssetError("download");
|
|
61
|
+
const noProxy = (system.match(/ExceptionsList\s*:\s*<array>\s*\{([^}]+)\}/)?.[1] ?? "").split("\n").map((line) => line.match(/^\s*\d+\s*:\s*(\S+)\s*$/)?.[1]).filter(Boolean).join(",");
|
|
62
|
+
return {
|
|
63
|
+
httpsProxy: `http://${host.includes(":") ? `[${host}]` : host}:${port}`,
|
|
64
|
+
...hasProcessBypass(env) ? {} : { noProxy }
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
async function systemProxyOptions(options = {}) {
|
|
68
|
+
const platform = options.platform ?? process.platform, env = options.env ?? process.env;
|
|
69
|
+
if (platform !== "darwin" || hasProcessProxy(env)) return void 0;
|
|
70
|
+
let system;
|
|
71
|
+
try {
|
|
72
|
+
system = options.readSystem ? await options.readSystem() : (await run("/usr/sbin/scutil", ["--proxy"], {
|
|
73
|
+
timeout: 3e3,
|
|
74
|
+
maxBuffer: 131072
|
|
75
|
+
})).stdout;
|
|
76
|
+
} catch {
|
|
77
|
+
throw new EmbeddingAssetError("download");
|
|
78
|
+
}
|
|
79
|
+
return modelProxyOptions(platform, env, system);
|
|
80
|
+
}
|
|
81
|
+
/** Private per-child environment; never persisted or returned through IPC/status. */
|
|
82
|
+
async function embeddingChildEnvironment(options = {}) {
|
|
83
|
+
const env = options.env ?? process.env, proxy = await systemProxyOptions(options);
|
|
84
|
+
if (!proxy) return void 0;
|
|
85
|
+
return {
|
|
86
|
+
...env,
|
|
87
|
+
HTTPS_PROXY: proxy.httpsProxy,
|
|
88
|
+
...proxy.noProxy !== void 0 ? { NO_PROXY: proxy.noProxy } : {}
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
async function assetNetwork() {
|
|
92
|
+
const dispatcher = new EnvHttpProxyAgent(await systemProxyOptions());
|
|
93
|
+
return {
|
|
94
|
+
request: (async (url, init) => {
|
|
95
|
+
const response = await fetch$1(url, {
|
|
96
|
+
...init,
|
|
97
|
+
dispatcher,
|
|
98
|
+
redirect: "follow"
|
|
99
|
+
});
|
|
100
|
+
return {
|
|
101
|
+
status: response.status,
|
|
102
|
+
headers: response.headers,
|
|
103
|
+
body: response.body
|
|
104
|
+
};
|
|
105
|
+
}),
|
|
106
|
+
close: () => dispatcher.close()
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
async function size(path) {
|
|
110
|
+
try {
|
|
111
|
+
const value = await lstat(path);
|
|
112
|
+
if (!value.isFile() || value.isSymbolicLink()) throw new EmbeddingAssetError("integrity");
|
|
113
|
+
return value.size;
|
|
114
|
+
} catch (error) {
|
|
115
|
+
if (error.code === "ENOENT") return void 0;
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
async function valid(path, asset) {
|
|
120
|
+
if (await size(path) !== asset.bytes) return false;
|
|
121
|
+
const hash = createHash(asset.kind === "git-blob-sha1" ? "sha1" : "sha256");
|
|
122
|
+
if (asset.kind === "git-blob-sha1") hash.update(`blob ${asset.bytes}\0`);
|
|
123
|
+
for await (const bytes of createReadStream(path)) hash.update(bytes);
|
|
124
|
+
return hash.digest("hex") === asset.hash;
|
|
125
|
+
}
|
|
126
|
+
/** Private resource preparation. No model/native session or Design text enters this path.
|
|
127
|
+
* Download has its own bounded lifetime; the existing 180s inference budget is unchanged.
|
|
128
|
+
* A failed pass keeps resumable staging bytes and never publishes a partial canonical file.
|
|
129
|
+
*/
|
|
130
|
+
async function prepareEmbeddingAssetSet(root, files, localOnly, options) {
|
|
131
|
+
if (files.some((file) => !/^[a-zA-Z0-9_.-]+(?:\/[a-zA-Z0-9_.-]+)*$/.test(file.path) || file.path.split("/").some((p) => p === "..") || !Number.isSafeInteger(file.bytes) || file.bytes < 1 || !["sha256", "git-blob-sha1"].includes(file.kind))) throw new EmbeddingAssetError("integrity");
|
|
132
|
+
options.signal.throwIfAborted();
|
|
133
|
+
await mkdir(root, { recursive: true });
|
|
134
|
+
const compromised = new AbortController();
|
|
135
|
+
let release;
|
|
136
|
+
let network;
|
|
137
|
+
try {
|
|
138
|
+
try {
|
|
139
|
+
release = await lock(root, {
|
|
140
|
+
retries: 0,
|
|
141
|
+
stale: 6e5,
|
|
142
|
+
update: 1e4,
|
|
143
|
+
onCompromised: () => compromised.abort()
|
|
144
|
+
});
|
|
145
|
+
} catch {
|
|
146
|
+
throw new EmbeddingAssetError("busy");
|
|
147
|
+
}
|
|
148
|
+
const signal = AbortSignal.any([
|
|
149
|
+
options.signal,
|
|
150
|
+
compromised.signal,
|
|
151
|
+
AbortSignal.timeout(6e5)
|
|
152
|
+
]);
|
|
153
|
+
for (const asset of files) {
|
|
154
|
+
signal.throwIfAborted();
|
|
155
|
+
const target = join(root, asset.path);
|
|
156
|
+
if (await valid(target, asset)) continue;
|
|
157
|
+
if (localOnly) throw new EmbeddingAssetError("missing");
|
|
158
|
+
await mkdir(dirname(target), { recursive: true });
|
|
159
|
+
const staged = target + ".yugoo-download.part";
|
|
160
|
+
let offset = await size(staged);
|
|
161
|
+
if (offset === void 0) {
|
|
162
|
+
const old = await size(target);
|
|
163
|
+
if (old !== void 0 && old > 0 && old < asset.bytes) {
|
|
164
|
+
await copyFile(target, staged, constants.COPYFILE_EXCL);
|
|
165
|
+
offset = old;
|
|
166
|
+
} else offset = 0;
|
|
167
|
+
}
|
|
168
|
+
if (offset >= asset.bytes) offset = 0;
|
|
169
|
+
if (!options.fetch) network ??= await assetNetwork();
|
|
170
|
+
const response = await (options.fetch ?? network.request)(options.baseUrl + "/" + asset.path, {
|
|
171
|
+
headers: offset ? { Range: `bytes=${offset}-` } : {},
|
|
172
|
+
signal
|
|
173
|
+
});
|
|
174
|
+
let received = offset;
|
|
175
|
+
let append = false;
|
|
176
|
+
if (offset && response.status === 206 && response.headers.get("content-range") === `bytes ${offset}-${asset.bytes - 1}/${asset.bytes}`) append = true;
|
|
177
|
+
else if (response.status === 200) received = 0;
|
|
178
|
+
else {
|
|
179
|
+
if (response.body) for await (const _ of response.body) break;
|
|
180
|
+
throw new EmbeddingAssetError("download");
|
|
181
|
+
}
|
|
182
|
+
if (!response.body) throw new EmbeddingAssetError("download");
|
|
183
|
+
const output = await open(staged, append ? "a" : "w");
|
|
184
|
+
try {
|
|
185
|
+
for await (const bytes of response.body) {
|
|
186
|
+
signal.throwIfAborted();
|
|
187
|
+
received += bytes.byteLength;
|
|
188
|
+
if (received > asset.bytes) throw new EmbeddingAssetError("integrity");
|
|
189
|
+
await output.writeFile(bytes);
|
|
190
|
+
options.progress?.(received, asset.bytes);
|
|
191
|
+
}
|
|
192
|
+
await output.sync();
|
|
193
|
+
} finally {
|
|
194
|
+
await output.close();
|
|
195
|
+
}
|
|
196
|
+
if (!await valid(staged, asset)) throw new EmbeddingAssetError("integrity");
|
|
197
|
+
signal.throwIfAborted();
|
|
198
|
+
if (await size(target) !== void 0) await copyFile(target, target + ".rejected-" + randomUUID(), constants.COPYFILE_EXCL);
|
|
199
|
+
await rename(staged, target);
|
|
200
|
+
}
|
|
201
|
+
} catch (error) {
|
|
202
|
+
if (options.signal.aborted) throw options.signal.reason;
|
|
203
|
+
if (error instanceof EmbeddingAssetError) throw error;
|
|
204
|
+
throw new EmbeddingAssetError("download");
|
|
205
|
+
} finally {
|
|
206
|
+
await network?.close();
|
|
207
|
+
await release?.();
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async function prepareDefaultEmbeddingAssets(config, signal, progress) {
|
|
211
|
+
if (config.model !== DEFAULT_EMBEDDING_ASSETS.model || config.revision !== DEFAULT_EMBEDDING_ASSETS.revision) return;
|
|
212
|
+
await prepareEmbeddingAssetSet(join(config.cacheDir, config.model, config.revision), DEFAULT_EMBEDDING_ASSETS.files, config.localOnly, {
|
|
213
|
+
signal,
|
|
214
|
+
progress,
|
|
215
|
+
baseUrl: `https://huggingface.co/${config.model}/resolve/${config.revision}`
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
//#endregion
|
|
219
|
+
//#region src/semantic-index.ts
|
|
220
|
+
const localEmbeddingFailures = {
|
|
221
|
+
runtime: "本地 embedding 运行依赖不可用;请按当前操作系统安装依赖(包括原生模块),不要复用其他平台的 node_modules。",
|
|
222
|
+
model: "本地 embedding 模型加载失败;请检查固定版本模型缓存,离线模式需要先准备完整模型文件。",
|
|
223
|
+
inference: "本地 embedding 推理失败;请检查输入限制和本地内存、CPU 运行资源。"
|
|
224
|
+
};
|
|
225
|
+
const modelAssetFailures = {
|
|
226
|
+
missing: "本地 embedding 模型资源缺失或不完整;离线模式不能下载,请先准备固定版本资源。",
|
|
227
|
+
download: "本地 embedding 模型资源下载未完成;请检查网络及代理后显式重试,本次未开始模型加载。",
|
|
228
|
+
integrity: "本地 embedding 模型资源校验失败;完整性通过后才能加载,本次未发布语义索引。",
|
|
229
|
+
busy: "本地 embedding 模型资源正由另一进程准备;本次未开始模型加载,可在准备完成后重试。"
|
|
230
|
+
};
|
|
231
|
+
/** Disposable, project-local cache. Never stores Knowledge/GRA facts. */
|
|
232
|
+
var MemorySemanticCache = class {
|
|
233
|
+
limit;
|
|
234
|
+
byteLimit;
|
|
235
|
+
values = /* @__PURE__ */ new Map();
|
|
236
|
+
bytes = 0;
|
|
237
|
+
constructor(limit = 300, byteLimit = 4e6) {
|
|
238
|
+
this.limit = limit;
|
|
239
|
+
this.byteLimit = byteLimit;
|
|
240
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || !Number.isSafeInteger(byteLimit) || byteLimit < 1) throw new Error("Invalid volatile cache bounds");
|
|
241
|
+
}
|
|
242
|
+
async readSemanticCache(key) {
|
|
243
|
+
return this.values.get(key);
|
|
244
|
+
}
|
|
245
|
+
async writeSemanticCache(key, value) {
|
|
246
|
+
const size = Buffer.byteLength(key) + Buffer.byteLength(value);
|
|
247
|
+
if (size > this.byteLimit) return;
|
|
248
|
+
const old = this.values.get(key);
|
|
249
|
+
if (old !== void 0) {
|
|
250
|
+
this.bytes -= Buffer.byteLength(key) + Buffer.byteLength(old);
|
|
251
|
+
this.values.delete(key);
|
|
252
|
+
}
|
|
253
|
+
while (this.values.size && (this.values.size >= this.limit || this.bytes + size > this.byteLimit)) {
|
|
254
|
+
const [id, text] = this.values.entries().next().value;
|
|
255
|
+
this.values.delete(id);
|
|
256
|
+
this.bytes -= Buffer.byteLength(id) + Buffer.byteLength(text);
|
|
257
|
+
}
|
|
258
|
+
this.values.set(key, value);
|
|
259
|
+
this.bytes += size;
|
|
260
|
+
}
|
|
261
|
+
};
|
|
262
|
+
async function stopEmbeddingChild(worker) {
|
|
263
|
+
if (worker.exitCode !== null || worker.signalCode !== null) return;
|
|
264
|
+
await new Promise((resolveExit) => {
|
|
265
|
+
const timer = setTimeout(() => worker.kill("SIGKILL"), 2e3);
|
|
266
|
+
worker.once("exit", () => {
|
|
267
|
+
clearTimeout(timer);
|
|
268
|
+
resolveExit();
|
|
269
|
+
});
|
|
270
|
+
worker.kill("SIGTERM");
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
var DesignSemanticIndex = class {
|
|
274
|
+
idleMs;
|
|
275
|
+
config;
|
|
276
|
+
identity;
|
|
277
|
+
get publicationIdentity() {
|
|
278
|
+
return semanticDigest(this.identity + ":similarity-v1:" + this.config.threshold);
|
|
279
|
+
}
|
|
280
|
+
jobs = /* @__PURE__ */ new Map();
|
|
281
|
+
tail = Promise.resolve();
|
|
282
|
+
closed = false;
|
|
283
|
+
constructor(config = {}, idleMs = 3e5) {
|
|
284
|
+
this.idleMs = idleMs;
|
|
285
|
+
if (!Number.isSafeInteger(idleMs) || idleMs < 1) throw new Error("Invalid semantic idle timeout");
|
|
286
|
+
this.config = {
|
|
287
|
+
provider: "local",
|
|
288
|
+
model: "Xenova/multilingual-e5-small",
|
|
289
|
+
revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78",
|
|
290
|
+
cacheDir: resolve(homedir(), ".cache/design-system/embeddings"),
|
|
291
|
+
localOnly: false,
|
|
292
|
+
prefix: "query: ",
|
|
293
|
+
threshold: .8,
|
|
294
|
+
...config
|
|
295
|
+
};
|
|
296
|
+
if (![
|
|
297
|
+
"local",
|
|
298
|
+
"remote",
|
|
299
|
+
"disabled"
|
|
300
|
+
].includes(this.config.provider)) throw new Error("Unknown embedding provider");
|
|
301
|
+
if (!this.config.model || !this.config.revision || !isAbsolute(this.config.cacheDir)) throw new Error("Invalid embedding configuration");
|
|
302
|
+
if (!(this.config.threshold >= 0 && this.config.threshold <= 1)) throw new Error("Invalid semantic threshold");
|
|
303
|
+
if (this.config.provider === "remote") {
|
|
304
|
+
if (!this.config.endpoint || !config.model || !config.revision) throw new Error("Remote embedding requires explicit endpoint, model and revision");
|
|
305
|
+
const url = new URL(this.config.endpoint);
|
|
306
|
+
if (!["https:", "http:"].includes(url.protocol) || url.username || url.password) throw new Error("Invalid embedding endpoint");
|
|
307
|
+
if (url.protocol === "http:" && ![
|
|
308
|
+
"localhost",
|
|
309
|
+
"127.0.0.1",
|
|
310
|
+
"[::1]"
|
|
311
|
+
].includes(url.hostname)) throw new Error("Remote embedding requires HTTPS except loopback");
|
|
312
|
+
}
|
|
313
|
+
this.identity = semanticDigest(JSON.stringify({
|
|
314
|
+
provider: this.config.provider,
|
|
315
|
+
model: this.config.model,
|
|
316
|
+
revision: this.config.revision,
|
|
317
|
+
endpoint: this.config.endpoint,
|
|
318
|
+
prefix: this.config.prefix,
|
|
319
|
+
text: SEMANTIC_TEXT_VERSION,
|
|
320
|
+
pooling: "mean-token-windows-v1",
|
|
321
|
+
dtype: "q8",
|
|
322
|
+
engine: "transformers-3.8.1"
|
|
323
|
+
}));
|
|
324
|
+
}
|
|
325
|
+
read(project, graph, cache) {
|
|
326
|
+
const units = (graph.semanticUnits ?? []).map((unit) => ({
|
|
327
|
+
...unit,
|
|
328
|
+
fieldPointers: unit.fieldPointers?.slice()
|
|
329
|
+
}));
|
|
330
|
+
const snapshot = {
|
|
331
|
+
checkpoint: graph.checkpoint,
|
|
332
|
+
semanticUnits: units
|
|
333
|
+
};
|
|
334
|
+
const key = semanticDigest(graph.checkpoint + (graph.generation ?? "") + this.publicationIdentity + (cache ? "persistent" : "memory") + JSON.stringify(units.map((u) => [
|
|
335
|
+
u.id,
|
|
336
|
+
u.nodeId,
|
|
337
|
+
u.file,
|
|
338
|
+
u.pointer,
|
|
339
|
+
u.fieldPointers,
|
|
340
|
+
u.digest
|
|
341
|
+
])));
|
|
342
|
+
if (this.closed || this.config.provider === "disabled") return {
|
|
343
|
+
edges: [],
|
|
344
|
+
status: {
|
|
345
|
+
state: "disabled",
|
|
346
|
+
completed: 0,
|
|
347
|
+
total: units.length,
|
|
348
|
+
model: this.config.model
|
|
349
|
+
}
|
|
350
|
+
};
|
|
351
|
+
let job = this.jobs.get(project);
|
|
352
|
+
if (job?.key !== key) {
|
|
353
|
+
if (this.jobs.size >= 8 && !job) return {
|
|
354
|
+
edges: [],
|
|
355
|
+
status: {
|
|
356
|
+
state: "error",
|
|
357
|
+
completed: 0,
|
|
358
|
+
total: units.length,
|
|
359
|
+
model: this.config.model,
|
|
360
|
+
message: "语义索引同时最多处理8个项目。"
|
|
361
|
+
}
|
|
362
|
+
};
|
|
363
|
+
job?.abort.abort();
|
|
364
|
+
clearTimeout(job?.idle);
|
|
365
|
+
job = {
|
|
366
|
+
key,
|
|
367
|
+
status: {
|
|
368
|
+
state: "indexing",
|
|
369
|
+
completed: 0,
|
|
370
|
+
total: units.length,
|
|
371
|
+
model: this.config.model,
|
|
372
|
+
cache: cache ? "persistent" : "memory",
|
|
373
|
+
message: this.config.provider === "local" ? "本地索引准备中;首次可能下载模型,文本不上传" : "使用已配置远程 embedding 服务处理设计文本"
|
|
374
|
+
},
|
|
375
|
+
edges: [],
|
|
376
|
+
abort: new AbortController(),
|
|
377
|
+
memory: job?.memory ?? new MemorySemanticCache()
|
|
378
|
+
};
|
|
379
|
+
this.jobs.set(project, job);
|
|
380
|
+
const current = job;
|
|
381
|
+
this.tail = this.tail.catch(() => {}).then(async () => {
|
|
382
|
+
if (current.abort.signal.aborted) return;
|
|
383
|
+
try {
|
|
384
|
+
await this.build(snapshot, cache, current);
|
|
385
|
+
} catch (error) {
|
|
386
|
+
const phase = error?.embeddingFailure;
|
|
387
|
+
const detail = error instanceof EmbeddingAssetError ? modelAssetFailures[error.code] : phase && Object.hasOwn(localEmbeddingFailures, phase) ? localEmbeddingFailures[phase] : "语义索引失败;请检查本地运行资源或已配置的 embedding 服务。";
|
|
388
|
+
const diagnostic = embeddingDiagnosticSuffix(error?.embeddingDiagnostic);
|
|
389
|
+
if (!current.abort.signal.aborted) current.status = {
|
|
390
|
+
...current.status,
|
|
391
|
+
state: "error",
|
|
392
|
+
message: detail + " 保留结构图,不自动上传或重试。" + diagnostic
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
});
|
|
396
|
+
current.settled = this.tail;
|
|
397
|
+
}
|
|
398
|
+
clearTimeout(job.idle);
|
|
399
|
+
if (!job.waiters) {
|
|
400
|
+
job.idle = setTimeout(() => this.cancelProject(project), this.idleMs);
|
|
401
|
+
job.idle.unref();
|
|
402
|
+
}
|
|
403
|
+
return {
|
|
404
|
+
edges: job.edges,
|
|
405
|
+
status: { ...job.status }
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
/** Await the existing inference job for background persistence; no polling and
|
|
409
|
+
* no project write lock held during model execution. Partial edges never escape. */
|
|
410
|
+
async compute(project, graph, cache, signal) {
|
|
411
|
+
signal?.throwIfAborted();
|
|
412
|
+
const status = this.read(project, graph, cache);
|
|
413
|
+
const job = this.jobs.get(project);
|
|
414
|
+
if (!job || status.status.state === "disabled" || status.status.state === "error") throw new Error(status.status.message ?? "Semantic indexing unavailable");
|
|
415
|
+
clearTimeout(job.idle);
|
|
416
|
+
job.waiters = (job.waiters ?? 0) + 1;
|
|
417
|
+
let rejectAbort;
|
|
418
|
+
const cancelled = new Promise((_resolve, reject) => {
|
|
419
|
+
rejectAbort = reject;
|
|
420
|
+
});
|
|
421
|
+
const abort = () => rejectAbort(signal?.reason ?? /* @__PURE__ */ new Error("Semantic consumer cancelled"));
|
|
422
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
423
|
+
try {
|
|
424
|
+
signal?.throwIfAborted();
|
|
425
|
+
await Promise.race([job.settled, cancelled]);
|
|
426
|
+
signal?.throwIfAborted();
|
|
427
|
+
job.abort.signal.throwIfAborted();
|
|
428
|
+
if (this.jobs.get(project) !== job || job.status.state !== "ready") throw new Error(job.status.message ?? "Semantic indexing did not complete");
|
|
429
|
+
return {
|
|
430
|
+
edges: structuredClone(job.edges),
|
|
431
|
+
status: { ...job.status }
|
|
432
|
+
};
|
|
433
|
+
} finally {
|
|
434
|
+
signal?.removeEventListener("abort", abort);
|
|
435
|
+
job.waiters--;
|
|
436
|
+
if (!job.waiters && signal?.aborted && job.status.state === "indexing") job.abort.abort(signal.reason);
|
|
437
|
+
if (this.jobs.get(project) === job && !job.waiters) {
|
|
438
|
+
job.idle = setTimeout(() => this.cancelProject(project), this.idleMs);
|
|
439
|
+
job.idle.unref();
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
cancelProject(project) {
|
|
444
|
+
const job = this.jobs.get(project);
|
|
445
|
+
clearTimeout(job?.idle);
|
|
446
|
+
job?.abort.abort();
|
|
447
|
+
this.jobs.delete(project);
|
|
448
|
+
}
|
|
449
|
+
async close() {
|
|
450
|
+
this.closed = true;
|
|
451
|
+
for (const project of this.jobs.keys()) this.cancelProject(project);
|
|
452
|
+
await this.tail;
|
|
453
|
+
}
|
|
454
|
+
async prepareLocalAssets(job) {
|
|
455
|
+
await prepareDefaultEmbeddingAssets(this.config, job.abort.signal, (done, total) => {
|
|
456
|
+
job.status.message = `正在准备固定版本模型资源 ${done}/${total} 字节;设计文本不上传`;
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
async build(graph, cache, job) {
|
|
460
|
+
const units = graph.semanticUnits ?? [], vectors = [], missing = [];
|
|
461
|
+
const signal = job.abort.signal;
|
|
462
|
+
for (let i = 0; i < units.length; i++) {
|
|
463
|
+
signal.throwIfAborted();
|
|
464
|
+
const key = semanticDigest(this.identity + units[i].digest);
|
|
465
|
+
let value;
|
|
466
|
+
if (cache && job.status.cache === "persistent") try {
|
|
467
|
+
value = await cache.readSemanticCache(key);
|
|
468
|
+
} catch {
|
|
469
|
+
job.status.cache = "memory";
|
|
470
|
+
}
|
|
471
|
+
value ??= await job.memory.readSemanticCache(key);
|
|
472
|
+
if (value) try {
|
|
473
|
+
vectors[i] = normalizeVector(JSON.parse(value));
|
|
474
|
+
job.status.completed++;
|
|
475
|
+
continue;
|
|
476
|
+
} catch {}
|
|
477
|
+
missing.push(i);
|
|
478
|
+
}
|
|
479
|
+
let worker;
|
|
480
|
+
const kill = () => {
|
|
481
|
+
worker?.kill("SIGTERM");
|
|
482
|
+
};
|
|
483
|
+
signal.addEventListener("abort", kill, { once: true });
|
|
484
|
+
try {
|
|
485
|
+
if (missing.length && this.config.provider === "local") {
|
|
486
|
+
await this.prepareLocalAssets(job);
|
|
487
|
+
signal.throwIfAborted();
|
|
488
|
+
const built = new URL("./embedding-worker.js", import.meta.url);
|
|
489
|
+
const options = {
|
|
490
|
+
windowsHide: true,
|
|
491
|
+
execArgv: ["--max-old-space-size=512"],
|
|
492
|
+
stdio: [
|
|
493
|
+
"ignore",
|
|
494
|
+
"ignore",
|
|
495
|
+
"ignore",
|
|
496
|
+
"ipc"
|
|
497
|
+
],
|
|
498
|
+
env: await embeddingChildEnvironment()
|
|
499
|
+
};
|
|
500
|
+
worker = fork(existsSync(built) ? built : new URL("./embedding-worker.mjs", import.meta.url), [], options);
|
|
501
|
+
}
|
|
502
|
+
for (let start = 0; start < missing.length; start += 4) {
|
|
503
|
+
signal.throwIfAborted();
|
|
504
|
+
const indices = missing.slice(start, start + 4), texts = indices.map((i) => units[i].text);
|
|
505
|
+
const values = worker ? await this.localBatch(worker, texts, job) : await this.remoteBatch(texts, signal);
|
|
506
|
+
if (!Array.isArray(values) || values.length !== texts.length) throw new Error("Embedding batch count mismatch");
|
|
507
|
+
for (let offset = 0; offset < indices.length; offset++) {
|
|
508
|
+
signal.throwIfAborted();
|
|
509
|
+
const i = indices[offset], vector = normalizeVector(values[offset]);
|
|
510
|
+
vectors[i] = vector;
|
|
511
|
+
const key = semanticDigest(this.identity + units[i].digest), value = JSON.stringify(vector);
|
|
512
|
+
await job.memory.writeSemanticCache(key, value);
|
|
513
|
+
if (cache && job.status.cache === "persistent") try {
|
|
514
|
+
await cache.writeSemanticCache(key, value);
|
|
515
|
+
} catch {
|
|
516
|
+
job.status.cache = "memory";
|
|
517
|
+
}
|
|
518
|
+
job.status.completed++;
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
signal.throwIfAborted();
|
|
522
|
+
job.edges = semanticEdges(units, vectors, `${this.config.model}@${this.config.revision}/${this.identity.slice(0, 12)}`, graph.checkpoint, this.config.threshold);
|
|
523
|
+
job.status = {
|
|
524
|
+
...job.status,
|
|
525
|
+
state: "ready",
|
|
526
|
+
message: `${job.edges.length} 条跨文件语义候选;相似度不等于确认关系`
|
|
527
|
+
};
|
|
528
|
+
} finally {
|
|
529
|
+
signal.removeEventListener("abort", kill);
|
|
530
|
+
if (worker) await stopEmbeddingChild(worker);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
localBatch(worker, texts, job) {
|
|
534
|
+
return new Promise((resolveBatch, reject) => {
|
|
535
|
+
const cleanup = () => {
|
|
536
|
+
clearTimeout(timer);
|
|
537
|
+
worker.off("message", message);
|
|
538
|
+
worker.off("error", failure);
|
|
539
|
+
worker.off("exit", failure);
|
|
540
|
+
job.abort.signal.removeEventListener("abort", failure);
|
|
541
|
+
};
|
|
542
|
+
const failure = () => {
|
|
543
|
+
cleanup();
|
|
544
|
+
reject(/* @__PURE__ */ new Error("Embedding worker stopped"));
|
|
545
|
+
};
|
|
546
|
+
const message = (data) => {
|
|
547
|
+
if (data.progress) {
|
|
548
|
+
job.status.message = data.progress;
|
|
549
|
+
return;
|
|
550
|
+
}
|
|
551
|
+
cleanup();
|
|
552
|
+
if (data.error || !data.vectors) reject(Object.assign(/* @__PURE__ */ new Error("Embedding worker failed"), {
|
|
553
|
+
embeddingFailure: data.error,
|
|
554
|
+
embeddingDiagnostic: readEmbeddingDiagnostic(data.diagnostic)
|
|
555
|
+
}));
|
|
556
|
+
else resolveBatch(data.vectors);
|
|
557
|
+
};
|
|
558
|
+
const timer = setTimeout(() => {
|
|
559
|
+
failure();
|
|
560
|
+
worker.kill("SIGTERM");
|
|
561
|
+
}, 18e4);
|
|
562
|
+
worker.on("message", message);
|
|
563
|
+
worker.once("error", failure);
|
|
564
|
+
worker.once("exit", failure);
|
|
565
|
+
job.abort.signal.addEventListener("abort", failure, { once: true });
|
|
566
|
+
if (job.abort.signal.aborted) {
|
|
567
|
+
failure();
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
worker.send({
|
|
571
|
+
texts,
|
|
572
|
+
config: this.config
|
|
573
|
+
}, (error) => {
|
|
574
|
+
if (error) failure();
|
|
575
|
+
});
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
async remoteBatch(texts, signal) {
|
|
579
|
+
const response = await fetch(this.config.endpoint, {
|
|
580
|
+
method: "POST",
|
|
581
|
+
redirect: "error",
|
|
582
|
+
headers: {
|
|
583
|
+
"Content-Type": "application/json",
|
|
584
|
+
...this.config.apiKey ? { Authorization: `Bearer ${this.config.apiKey}` } : {}
|
|
585
|
+
},
|
|
586
|
+
body: JSON.stringify({
|
|
587
|
+
model: this.config.model,
|
|
588
|
+
input: texts.map((t) => this.config.prefix + t)
|
|
589
|
+
}),
|
|
590
|
+
signal: AbortSignal.any([signal, AbortSignal.timeout(6e4)])
|
|
591
|
+
});
|
|
592
|
+
if (!response.ok) throw new Error("Remote embedding request failed");
|
|
593
|
+
const data = await response.json();
|
|
594
|
+
if (!Array.isArray(data.data) || data.data.length !== texts.length || new Set(data.data.map((d) => d.index)).size !== texts.length || data.data.some((d) => !Number.isInteger(d.index) || d.index < 0 || d.index >= texts.length)) throw new Error("Invalid remote embedding response");
|
|
595
|
+
return data.data.sort((a, b) => a.index - b.index).map((d) => d.embedding);
|
|
596
|
+
}
|
|
597
|
+
};
|
|
598
|
+
//#endregion
|
|
599
|
+
export { MemorySemanticCache as n, stopEmbeddingChild as r, DesignSemanticIndex as t };
|