@modusensus/dsh-mneme 0.8.13 → 0.8.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -5
- package/bin/dsh-mneme-serve.mjs +151 -0
- package/lib/api-standalone.js +110 -9
- package/lib/config.js +7 -0
- package/lib/index.js +18 -186
- package/lib/semantic.js +229 -0
- package/lib/serve.js +215 -0
- package/lib/service.js +15 -6
- package/lib/settings.js +2 -0
- package/package.json +118 -114
- package/scripts/changelog-prep.mjs +36 -0
- package/src/api-standalone.js +110 -9
- package/src/config.js +7 -0
- package/src/index.js +18 -186
- package/src/semantic.js +229 -0
- package/src/serve.js +215 -0
- package/src/service.js +15 -6
- package/src/settings.js +2 -0
- package/test/api.test.js +3 -1
- package/test/changelog-prep.test.js +60 -0
- package/test/inject-heat.test.js +31 -15
- package/test/scope-strict.test.js +24 -0
- package/test/serve-bin.test.js +121 -0
- package/test/serve.test.js +196 -0
- package/test/standalone-api.test.js +212 -5
package/src/api-standalone.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Standalone HTTP API
|
|
1
|
+
// Standalone HTTP API: a plain node:http server for ecosystem
|
|
2
2
|
// integrations that live outside the DSH host and cannot reach the plugin's
|
|
3
3
|
// internal webServer routes (/api/dsh-mneme/*). Mirrors the JSON semantics of
|
|
4
4
|
// those routes but with mandatory Bearer-token auth on everything except
|
|
@@ -17,12 +17,10 @@ import { createServer } from "node:http";
|
|
|
17
17
|
import { randomBytes, timingSafeEqual } from "node:crypto";
|
|
18
18
|
import { TYPES } from "./store.js";
|
|
19
19
|
import { bootstrapFromDirectory } from "./bootstrap.js";
|
|
20
|
+
import { PACKAGE_VERSION } from "./version-check.js";
|
|
20
21
|
|
|
21
22
|
const DEFAULT_PORT = 8790;
|
|
22
23
|
const DEFAULT_HOST = "127.0.0.1";
|
|
23
|
-
// Hardcoded release version (package.json is bumped at publish time and may
|
|
24
|
-
// lag the code that ships in between).
|
|
25
|
-
const VERSION = "0.7.12";
|
|
26
24
|
const MAX_PORT_ATTEMPTS = 20;
|
|
27
25
|
|
|
28
26
|
/**
|
|
@@ -30,8 +28,13 @@ const MAX_PORT_ATTEMPTS = 20;
|
|
|
30
28
|
* the next MAX_PORT_ATTEMPTS-1 ports, and finally falls back to port 0 so the
|
|
31
29
|
* OS assigns a free port. Multiple DSH profiles/instances sharing the default
|
|
32
30
|
* port no longer leave the standalone API permanently unavailable.
|
|
31
|
+
*
|
|
32
|
+
* strictPort opts out of that recovery (dsh-mneme-serve daemon, #363): a
|
|
33
|
+
* long-lived third-party integration pins the URL, so silently hopping ports
|
|
34
|
+
* would make clients talk to nothing (or, worse, to a future different data
|
|
35
|
+
* plane). A busy configured port is a configuration error there — fail loud.
|
|
33
36
|
*/
|
|
34
|
-
function listenWithRetry(server, startPort, host, logger) {
|
|
37
|
+
function listenWithRetry(server, startPort, host, logger, strictPort = false) {
|
|
35
38
|
return new Promise((resolve, reject) => {
|
|
36
39
|
let attempt = 0;
|
|
37
40
|
const tryListen = (port) => {
|
|
@@ -42,6 +45,10 @@ function listenWithRetry(server, startPort, host, logger) {
|
|
|
42
45
|
};
|
|
43
46
|
const onError = (error) => {
|
|
44
47
|
server.off("listening", onListening);
|
|
48
|
+
if (error?.code === "EADDRINUSE" && strictPort) {
|
|
49
|
+
reject(error);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
45
52
|
if (error?.code === "EADDRINUSE" && attempt < MAX_PORT_ATTEMPTS - 1) {
|
|
46
53
|
attempt++;
|
|
47
54
|
const next = startPort + attempt;
|
|
@@ -186,18 +193,70 @@ async function handlePutBody(res, service, logger, id, text) {
|
|
|
186
193
|
sendJson(res, 200, { memory: service.toApiList([service.getById(id)])[0] });
|
|
187
194
|
}
|
|
188
195
|
|
|
196
|
+
/**
|
|
197
|
+
* /context 的查询嵌入:best-effort,embedder 接口宽容与 searchMemories 同款
|
|
198
|
+
* (embedSingle 优先,兼容 embed-only 的 OpenAI 兼容客户端,issue #10)。
|
|
199
|
+
* 调用方负责 catch——嵌入失败降级规则档,绝不因此 500。
|
|
200
|
+
*/
|
|
201
|
+
async function embedQueryVector(embedder, q) {
|
|
202
|
+
const embedSingle = typeof embedder.embedSingle === "function"
|
|
203
|
+
? embedder.embedSingle.bind(embedder)
|
|
204
|
+
: typeof embedder.embed === "function"
|
|
205
|
+
? embedder.embed.bind(embedder)
|
|
206
|
+
: null;
|
|
207
|
+
if (!embedSingle) return undefined;
|
|
208
|
+
const vector = await embedSingle(q);
|
|
209
|
+
return Array.isArray(vector) && vector.length ? vector : undefined;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* /context 查询嵌入的超时护栏(CodeRabbit on #371):embedder 挂起(不抛错,
|
|
214
|
+
* 典型如第三方 ollama/openai 端点无响应)时不能让 HTTP 连接被无限占住——超时
|
|
215
|
+
* 按嵌入缺失处理,降级规则档。输掉的 embed promise 无法取消、自行结束后被丢弃,
|
|
216
|
+
* 无害;挂起的主要成本是一个泄漏的挂起任务,可接受。嵌入先完成时清掉定时器
|
|
217
|
+
* (CodeRabbit 同轮 Minor:别让每个请求都留一枚要等满超时的计时器)。
|
|
218
|
+
*/
|
|
219
|
+
function withEmbedTimeout(promise, timeoutMs) {
|
|
220
|
+
let timer;
|
|
221
|
+
const timeout = new Promise((resolve) => {
|
|
222
|
+
timer = setTimeout(() => resolve(undefined), timeoutMs);
|
|
223
|
+
});
|
|
224
|
+
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* scope 查询参数(/context 与 /search 同口径,issue #370):两参全缺 = 返回
|
|
229
|
+
* undefined(调用方不传 scope 走各自默认,行为与既往逐字节一致);任一给出则
|
|
230
|
+
* 缺的一维按「解析不到」(null)走 isVisibleInScope 的 fail-closed——身份不明
|
|
231
|
+
* 的维度只见全局,不冒认,与宿主会话 scope 解析器同构。
|
|
232
|
+
*/
|
|
233
|
+
function scopeFromSearchParams(url) {
|
|
234
|
+
const agentScope = url.searchParams.get("agent_scope");
|
|
235
|
+
const workspaceScope = url.searchParams.get("workspace_scope");
|
|
236
|
+
if (agentScope === null && workspaceScope === null) return undefined;
|
|
237
|
+
return { agent_scope: agentScope, workspace_scope: workspaceScope };
|
|
238
|
+
}
|
|
239
|
+
|
|
189
240
|
/**
|
|
190
241
|
* Create (and start) the standalone API server.
|
|
191
|
-
* Accepts { service, store, config, logger, settings, port, host }:
|
|
242
|
+
* Accepts { service, store, config, logger, settings, port, host, embedder }:
|
|
192
243
|
* - token: persisted settings kv "external_api" wins; auto-generated
|
|
193
244
|
* (crypto.randomBytes(24).toString("base64url")) and persisted when empty.
|
|
194
245
|
* - port: explicit arg > persisted settings > config.externalApiPort > 8790.
|
|
195
246
|
* - host: explicit arg > config.externalApiHost > "127.0.0.1".
|
|
247
|
+
* - strictPort: EADDRINUSE rejects instead of hopping ports (daemon mode;
|
|
248
|
+
* default false keeps the in-host sidecar recovery described above).
|
|
249
|
+
* - embedder: query-embedding handle for GET /context (issue #370) — same
|
|
250
|
+
* instance the caller handed to service.setEmbedder; null degrades /context
|
|
251
|
+
* to the rule + BM25 tier. Best-effort: embed failures never fail the route.
|
|
252
|
+
* - embedTimeoutMs: /context query-embed guard (default 3000) — a hung
|
|
253
|
+
* embedder must not hold the HTTP connection; timeout degrades to the rule
|
|
254
|
+
* tier. Factory option (like strictPort), not a user config key.
|
|
196
255
|
* Returns { server, port, host, token, ready }: `port` is the effective bound
|
|
197
256
|
* port (updated to the OS-assigned one after `ready` resolves when asked to
|
|
198
257
|
* bind port 0), `ready` resolves once listening and rejects if the bind fails.
|
|
199
258
|
*/
|
|
200
|
-
export function createStandaloneApi({ service, store, config = {}, logger, settings, port, host, maintenance }) {
|
|
259
|
+
export function createStandaloneApi({ service, store, config = {}, logger, settings, port, host, maintenance, strictPort = false, embedder = null, embedTimeoutMs = 3000 }) {
|
|
201
260
|
const persisted = settings?.getExternalApi?.() ?? {};
|
|
202
261
|
|
|
203
262
|
let token = typeof persisted.token === "string" ? persisted.token : "";
|
|
@@ -249,7 +308,7 @@ export function createStandaloneApi({ service, store, config = {}, logger, setti
|
|
|
249
308
|
entities = store.db.prepare("SELECT count(*) AS c FROM entities").get().c;
|
|
250
309
|
} catch { /* entities storage unavailable → 0 */ }
|
|
251
310
|
sendJson(res, 200, {
|
|
252
|
-
version:
|
|
311
|
+
version: PACKAGE_VERSION,
|
|
253
312
|
memories: { total: service.count(), byType },
|
|
254
313
|
entities,
|
|
255
314
|
uptime_s: Math.floor(process.uptime())
|
|
@@ -517,6 +576,10 @@ export function createStandaloneApi({ service, store, config = {}, logger, setti
|
|
|
517
576
|
const rerank = url.searchParams.get("rerank") !== "false";
|
|
518
577
|
const occurredFrom = url.searchParams.get("occurred_from") ?? null;
|
|
519
578
|
const occurredTo = url.searchParams.get("occurred_to") ?? null;
|
|
579
|
+
// scope 透传(issue #370):searchMemories 本就收 scope,此前 8790 没有
|
|
580
|
+
// 入口——第三方(daemon 桥接侧)声明身份后,A2 软加权与 strictScope 硬
|
|
581
|
+
// 过滤才在检索面同样成立;与 /context 同口径(scopeFromSearchParams)。
|
|
582
|
+
const scope = scopeFromSearchParams(url);
|
|
520
583
|
const query = q.trim();
|
|
521
584
|
if (!query) {
|
|
522
585
|
sendJson(res, 200, { items: [], mode: "keyword" });
|
|
@@ -528,6 +591,7 @@ export function createStandaloneApi({ service, store, config = {}, logger, setti
|
|
|
528
591
|
mode,
|
|
529
592
|
topK: limit,
|
|
530
593
|
useRerank: rerank,
|
|
594
|
+
...(scope ? { scope } : {}),
|
|
531
595
|
...(occurredFrom !== null || occurredTo !== null ? { occurredFrom, occurredTo } : {})
|
|
532
596
|
})
|
|
533
597
|
).then((rows) => {
|
|
@@ -539,6 +603,43 @@ export function createStandaloneApi({ service, store, config = {}, logger, setti
|
|
|
539
603
|
return;
|
|
540
604
|
}
|
|
541
605
|
|
|
606
|
+
// --- GET /context: 注入候选一站式聚合(issue #370,#363 承诺)------------
|
|
607
|
+
// 给定话题返回「宿主此刻会注入什么」的数据等价物:injectCandidates 全语义
|
|
608
|
+
// (优先级分层 / 编码门控 / heat / scope 软加权 / hybrid 语义路 / pin 前置,
|
|
609
|
+
// service.js:1494)+ 画像 + 规则。结构化条目、不含渲染文本——语言 / 时间
|
|
610
|
+
// 前缀 / 热记忆是宿主会话概念,调用方(Mneme Bridge 等网页端桥接)自行渲染。
|
|
611
|
+
// 与宿主注入的两点差异都是 HTTP 面的有意为之:① 首次调用即走语义路(宿主
|
|
612
|
+
// 渲染必须同步,查询向量只能异步 prefetch 给下一轮,inject.js Bug4;这里
|
|
613
|
+
// 没有该约束);② 不做跨轮轮换(daemon 无会话状态,调用方自管)。
|
|
614
|
+
// recall_runs 记账随 injectCandidates 内建(#217:注入是曝光型访问事件,
|
|
615
|
+
// mode='inject' 与检索命中同表分账)——宿主注入同样记,/context 自动同口径。
|
|
616
|
+
if (req.method === "GET" && pathname === "/context") {
|
|
617
|
+
const q = (url.searchParams.get("q") ?? "").trim();
|
|
618
|
+
const maxItemsRaw = Number(url.searchParams.get("topK") ?? url.searchParams.get("limit") ?? 5);
|
|
619
|
+
const maxItems = Number.isInteger(maxItemsRaw) && maxItemsRaw > 0 ? maxItemsRaw : 5;
|
|
620
|
+
// threshold 是 importance 阈值(injectCandidates 语义),与 /search 的
|
|
621
|
+
// 相似度 threshold 同名不同义——文档已点名,勿混。
|
|
622
|
+
const thresholdRaw = Number(url.searchParams.get("threshold") ?? 3);
|
|
623
|
+
const threshold = Number.isFinite(thresholdRaw) ? thresholdRaw : 3;
|
|
624
|
+
const scope = scopeFromSearchParams(url);
|
|
625
|
+
void Promise.resolve(q && embedder ? withEmbedTimeout(embedQueryVector(embedder, q), embedTimeoutMs) : undefined)
|
|
626
|
+
.catch(() => undefined) // 查询嵌入失败降级规则档(与 searchMemories 同口径)
|
|
627
|
+
.then((queryVector) => {
|
|
628
|
+
const pinnedStats = {};
|
|
629
|
+
const items = service.injectCandidates({ query: q, maxItems, threshold, queryVector, scope, pinnedStats });
|
|
630
|
+
sendJson(res, 200, {
|
|
631
|
+
profile: (settings?.getProfile?.() ?? "").trim(),
|
|
632
|
+
rules: settings?.getRules?.() ?? [],
|
|
633
|
+
items: service.toApiList(items),
|
|
634
|
+
pinnedCount: pinnedStats.shown ?? 0
|
|
635
|
+
});
|
|
636
|
+
})
|
|
637
|
+
.catch(() => {
|
|
638
|
+
sendJson(res, 500, { error: "internal" });
|
|
639
|
+
});
|
|
640
|
+
return;
|
|
641
|
+
}
|
|
642
|
+
|
|
542
643
|
sendJson(res, 404, { error: "not-found" });
|
|
543
644
|
} catch {
|
|
544
645
|
sendJson(res, 500, { error: "internal" });
|
|
@@ -552,7 +653,7 @@ export function createStandaloneApi({ service, store, config = {}, logger, setti
|
|
|
552
653
|
});
|
|
553
654
|
const ready = new Promise((resolve, reject) => {
|
|
554
655
|
// listening handled by listenWithRetry
|
|
555
|
-
listenWithRetry(server, boundPort, boundHost, logger).then(resolve, reject);
|
|
656
|
+
listenWithRetry(server, boundPort, boundHost, logger, strictPort).then(resolve, reject);
|
|
556
657
|
});
|
|
557
658
|
// server.listen is called inside listenWithRetry
|
|
558
659
|
ready.then(() => {
|
package/src/config.js
CHANGED
|
@@ -708,6 +708,13 @@ export const Config = z.object({
|
|
|
708
708
|
// 恒 1,sleep 降级退回纯时间分层。
|
|
709
709
|
// 也走 feature_flags(FEATURE_FLAG_BOOLEANS 白名单),面板可启停=线上回滚开关。
|
|
710
710
|
heatEnabled: z.boolean().default(false),
|
|
711
|
+
// issue #218 / E5 效用考卷:heat 乘进注入排序在真实年龄混合下饿死老约束
|
|
712
|
+
// (现行量级 ≡ 拟合参数,注入集逐条相同;importance-only 遵从 +12.7pp),
|
|
713
|
+
// 故注入侧 heat 改为独立 opt-in:默认关 = heatEnabled 开启后热度字段/sleep
|
|
714
|
+
// 保护/前端投影照常,但注入排序乘数恒 1(与 heat 关闭逐字节一致);
|
|
715
|
+
// 显式开启才恢复「层内乘 heat」的旧序。召回侧时钟(touchLastAccess)与
|
|
716
|
+
// sleep 降级联判不受本键影响——它们不参与注入排序。
|
|
717
|
+
injectHeatEnabled: z.boolean().default(false),
|
|
711
718
|
// 广义指数形状参数 β(heat = exp(-λ·Δt^β),issue #218 拍板):β=1 纯指数。
|
|
712
719
|
// 快慢以 Δt>1 小时为准——β<1 衰减更慢(亚线性长尾)、β>1 更快(超线性);
|
|
713
720
|
// 0<Δt<1 的首小时内方向相反(Δt^β 随 β 增大而变小)。专家调优项,不进面板白名单。
|
package/src/index.js
CHANGED
|
@@ -19,10 +19,10 @@ import { createStandaloneApi } from "./api-standalone.js";
|
|
|
19
19
|
import { createMaintenance } from "./maintenance.js";
|
|
20
20
|
import { createSettings } from "./settings.js";
|
|
21
21
|
import { createCommandManager } from "./commands.js";
|
|
22
|
-
import { createEmbedder } from "./embedding.js";
|
|
23
|
-
import { createEmbedderByProvider } from "./local-embedder.js";
|
|
24
|
-
import { LocalReranker } from "./reranker.js";
|
|
25
22
|
import { createVectorIndex } from "./vector-index.js";
|
|
23
|
+
// semantic(embedder/reranker/boot 回填)纯搬移至 src/semantic.js(PR2):宿主与
|
|
24
|
+
// daemon(dsh-mneme-serve,#363)共用同一套装配,原文件保留 barrel 出口。
|
|
25
|
+
import { createSemantic } from "./semantic.js";
|
|
26
26
|
import { Config, applyLightModePreset, injectChildEnabled } from "./config.js";
|
|
27
27
|
import { langOf } from "./lang.js";
|
|
28
28
|
import { extractEntities } from "./entities/extractor.js";
|
|
@@ -149,43 +149,9 @@ export function createEntityStreamAdapter({ llm, agentDefaultModel, logger, serv
|
|
|
149
149
|
};
|
|
150
150
|
}
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
*
|
|
156
|
-
* Runs regardless of the model fingerprint: the old call-site gate returned
|
|
157
|
-
* early when vector_meta already held the embedder's hash, permanently
|
|
158
|
-
* orphaning rows whose embed failed at write time (embedder not ready /
|
|
159
|
-
* provider rate limit) — one successful embed was enough to never backfill
|
|
160
|
-
* again. markModel is idempotent when the fingerprint already matches, so
|
|
161
|
-
* re-running costs nothing beyond the actually-missing rows.
|
|
162
|
-
*/
|
|
163
|
-
export async function backfillMissingEmbeddings({
|
|
164
|
-
store, embedder, vectorIndex, logger,
|
|
165
|
-
maxTotal = 500, batchSize = 10, rateLimitMs = 200
|
|
166
|
-
}) {
|
|
167
|
-
let indexed = 0;
|
|
168
|
-
for (let done = 0; done < maxTotal;) {
|
|
169
|
-
const rows = store.needsEmbedding(batchSize);
|
|
170
|
-
if (!rows.length) break;
|
|
171
|
-
for (const row of rows) {
|
|
172
|
-
try {
|
|
173
|
-
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
174
|
-
const vector = await embedder.embedSingle(text);
|
|
175
|
-
if (vector?.length) {
|
|
176
|
-
store.setEmbedding(row.id, vector);
|
|
177
|
-
indexed++;
|
|
178
|
-
}
|
|
179
|
-
} catch { /* skip the bad row */ }
|
|
180
|
-
}
|
|
181
|
-
done += rows.length;
|
|
182
|
-
// Rate limit: space out batches so the provider is not hammered.
|
|
183
|
-
if (store.needsEmbedding(1).length) await new Promise((r) => setTimeout(r, rateLimitMs));
|
|
184
|
-
}
|
|
185
|
-
if (indexed > 0 && embedder.modelHash) vectorIndex.markModel?.(embedder.modelHash, embedder.dimension);
|
|
186
|
-
logger?.info?.(`[dsh-mneme] auto-reindex backfilled ${indexed} embeddings on boot`);
|
|
187
|
-
return indexed;
|
|
188
|
-
}
|
|
152
|
+
// backfillMissingEmbeddings 已随语义装配整体搬至 src/semantic.js(纯搬移);
|
|
153
|
+
// 保留 barrel 再出口 —— test/reindex-backfill.test.js 仍从本模块 import,调用方零改动。
|
|
154
|
+
export { backfillMissingEmbeddings } from "./semantic.js";
|
|
189
155
|
|
|
190
156
|
export const apply = (ctx, config) => {
|
|
191
157
|
const rawCfg = Config(config);
|
|
@@ -339,150 +305,14 @@ export const apply = (ctx, config) => {
|
|
|
339
305
|
}
|
|
340
306
|
};
|
|
341
307
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
//
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
// human mirror edits still merge on boot.
|
|
351
|
-
applyHumanEdits();
|
|
352
|
-
} else if (cfg.embedProvider === "openai") {
|
|
353
|
-
// vectorIndex is passed so the legacy OpenAI embedder records the producing
|
|
354
|
-
// model fingerprint after each successful embed (Bug3).
|
|
355
|
-
embedder = createEmbedder({ store, settings, logger: ctx.logger, vectorIndex });
|
|
356
|
-
service.setEmbedder(embedder);
|
|
357
|
-
// issue #135: 未配置时明确告警一次。此前 legacy OpenAI embedder 恒报
|
|
358
|
-
// ready=true,向量层「绿的但全哑」可以静默存在很久(本机持续了数周)。
|
|
359
|
-
// 只记日志、不阻断启动:轻量模式与「先跑起来再补配置」都是正当用法。
|
|
360
|
-
if (embedder.configured === false) {
|
|
361
|
-
ctx.logger?.warn?.(
|
|
362
|
-
"[dsh-mneme] 向量层未配置(vector-config 的 enabled/baseUrl/apiKey/model 有缺):"
|
|
363
|
-
+ "语义召回、语义去重、rerank、sleep 冲突检测将静默失效,"
|
|
364
|
-
+ "dream 的语义聚类会退化为全量窗口兜底。"
|
|
365
|
-
+ "请在设置面板补全 embedding 端点与模型,或把 embedProvider 改为 local/ollama。"
|
|
366
|
-
);
|
|
367
|
-
}
|
|
368
|
-
// legacy OpenAI embedder needs no async init → human edits apply right away
|
|
369
|
-
applyHumanEdits();
|
|
370
|
-
} else {
|
|
371
|
-
try {
|
|
372
|
-
embedder = createEmbedderByProvider(cfg.embedProvider, {
|
|
373
|
-
model: cfg.embedProvider === "ollama" ? cfg.ollamaModel : cfg.localEmbedModel,
|
|
374
|
-
dimension: cfg.localEmbedDimension,
|
|
375
|
-
device: cfg.localEmbedDevice,
|
|
376
|
-
batchSize: cfg.localEmbedBatchSize,
|
|
377
|
-
// 池化方式必须与模型的训练口径一致(BGE 系 = CLS)。它既进 embed() 的调用,
|
|
378
|
-
// 也进 modelHash —— 池化改了就是换向量空间,既有索引会被判失配并重建。
|
|
379
|
-
pooling: cfg.localEmbedPooling,
|
|
380
|
-
cacheDir: cfg.embedModelCacheDir,
|
|
381
|
-
runtimeDir: cfg.runtimeDir,
|
|
382
|
-
// #188:embedModelMirror 接成 transformers 的下载镜像(此前死配置)。
|
|
383
|
-
remoteHost: cfg.embedModelMirror,
|
|
384
|
-
resilientModelDownload: cfg.resilientModelDownload,
|
|
385
|
-
baseUrl: cfg.ollamaBaseUrl,
|
|
386
|
-
logger: ctx.logger
|
|
387
|
-
});
|
|
388
|
-
service.setEmbedder(embedder);
|
|
389
|
-
// issue #6: wait for extractor init before applying human edits, so
|
|
390
|
-
// scheduled embeddings see a ready embedder.
|
|
391
|
-
const bootEmbedder = () => embedder.init()
|
|
392
|
-
.then(() => { applyHumanEdits(); return true; })
|
|
393
|
-
.catch(() => false);
|
|
394
|
-
// #118: the old one-shot probe permanently degraded search to keyword
|
|
395
|
-
// when Ollama was briefly unreachable at boot (recoverable only by
|
|
396
|
-
// restart). Retry briefly (5 attempts total: 1 initial + 4 × 15s);
|
|
397
|
-
// search degrades to keyword meanwhile because per-query embed failures
|
|
398
|
-
// are swallowed.
|
|
399
|
-
bootEmbedder().then((ok) => {
|
|
400
|
-
if (ok) return;
|
|
401
|
-
let tries = 4;
|
|
402
|
-
const retry = () => {
|
|
403
|
-
if (tries-- <= 0) {
|
|
404
|
-
ctx.logger?.warn?.("[dsh-mneme] embedder init retries exhausted, search degrades to keyword");
|
|
405
|
-
service.setEmbedder(null);
|
|
406
|
-
applyHumanEdits();
|
|
407
|
-
return;
|
|
408
|
-
}
|
|
409
|
-
embedRetryTimer = setTimeout(async () => {
|
|
410
|
-
if (await bootEmbedder()) return;
|
|
411
|
-
retry();
|
|
412
|
-
}, 15_000);
|
|
413
|
-
};
|
|
414
|
-
ctx.logger?.warn?.("[dsh-mneme] embedder init failed, retrying");
|
|
415
|
-
retry();
|
|
416
|
-
});
|
|
417
|
-
} catch (error) {
|
|
418
|
-
ctx.logger?.warn?.(`[dsh-mneme] embedder unavailable, search degrades to keyword: ${String(error)}`);
|
|
419
|
-
applyHumanEdits();
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
// Cross-encoder rerank over recall candidates. Best-effort: a failed model
|
|
424
|
-
// load only disables reranking, never search itself. Explicit opt-in only
|
|
425
|
-
// (rerankEnabled defaults to false): constructing LocalReranker is what pulls
|
|
426
|
-
// in onnxruntime, so the default config never loads it (item ⑥).
|
|
427
|
-
if (cfg.rerankEnabled && cfg.rerankProvider === "local") {
|
|
428
|
-
try {
|
|
429
|
-
reranker = new LocalReranker({
|
|
430
|
-
model: cfg.rerankModel,
|
|
431
|
-
batchSize: cfg.rerankBatchSize,
|
|
432
|
-
maxCandidates: cfg.rerankMaxCandidates,
|
|
433
|
-
scoreThreshold: cfg.rerankScoreThreshold,
|
|
434
|
-
device: cfg.localEmbedDevice,
|
|
435
|
-
cacheDir: cfg.embedModelCacheDir,
|
|
436
|
-
runtimeDir: cfg.runtimeDir,
|
|
437
|
-
// #188:量化档默认 q8(此前不传 dtype 会去要 1GB 级 fp32 模型);
|
|
438
|
-
// embedModelMirror 此前是死配置,现接成 transformers 的下载镜像。
|
|
439
|
-
useDtype: cfg.rerankDtype,
|
|
440
|
-
remoteHost: cfg.embedModelMirror,
|
|
441
|
-
resilientModelDownload: cfg.resilientModelDownload,
|
|
442
|
-
logger: ctx.logger
|
|
443
|
-
});
|
|
444
|
-
service.setReranker(reranker);
|
|
445
|
-
reranker.init().catch((error) => {
|
|
446
|
-
ctx.logger?.warn?.(`[dsh-mneme] reranker init failed, rerank disabled: ${String(error)}`);
|
|
447
|
-
service.setReranker(null);
|
|
448
|
-
});
|
|
449
|
-
} catch (error) {
|
|
450
|
-
ctx.logger?.warn?.(`[dsh-mneme] reranker unavailable, rerank disabled: ${String(error)}`);
|
|
451
|
-
}
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
// Bug2: lazy auto-backfill of missing embeddings on boot. When the vector API
|
|
455
|
-
// is configured and rows still lack an embedding (e.g. written before vector
|
|
456
|
-
// search was enabled), the backfill runs in the background after a short
|
|
457
|
-
// delay. Gated on cfg.autoReindexOnBoot; rate-limited in small batches so a
|
|
458
|
-
// large backlog never floods the provider. Failures degrade silently —
|
|
459
|
-
// search stays keyword.
|
|
460
|
-
function scheduleAutoReindex() {
|
|
461
|
-
if (cfg.autoReindexOnBoot === false) return;
|
|
462
|
-
const attempt = (tries) => {
|
|
463
|
-
try {
|
|
464
|
-
if (!embedder || typeof embedder.embedSingle !== "function") return;
|
|
465
|
-
if ("ready" in embedder && embedder.ready !== true) {
|
|
466
|
-
// Local/ollama embedders init asynchronously; give them a moment
|
|
467
|
-
// before giving up on this boot (next boot retries).
|
|
468
|
-
if (tries > 0) setTimeout(() => attempt(tries - 1), 2000);
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
471
|
-
if (!store.needsEmbedding(1).length) return; // nothing to backfill
|
|
472
|
-
// Issue #128: no fingerprint gate here anymore — a matching fingerprint
|
|
473
|
-
// used to return early and permanently orphan rows whose embed failed
|
|
474
|
-
// at write time. See backfillMissingEmbeddings().
|
|
475
|
-
backfillMissingEmbeddings({ store, embedder, vectorIndex, logger: ctx.logger })
|
|
476
|
-
.catch((error) => {
|
|
477
|
-
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
478
|
-
});
|
|
479
|
-
} catch (error) {
|
|
480
|
-
ctx.logger?.warn?.(`[dsh-mneme] auto-reindex failed: ${String(error)}`);
|
|
481
|
-
}
|
|
482
|
-
};
|
|
483
|
-
setTimeout(() => attempt(5), 5000);
|
|
484
|
-
}
|
|
485
|
-
scheduleAutoReindex();
|
|
308
|
+
// embedder/reranker/boot 回填装配已整体搬至 src/semantic.js(纯搬移,宿主与 daemon
|
|
309
|
+
// 共用):调用时序(applyHumanEdits 在各分支的触发点、#118 重试、autoReindexOnBoot)
|
|
310
|
+
// 原样保留在 createSemantic 内部,这里只拿引用。init 全失败的 embedder 引用仍会进入
|
|
311
|
+
// dream/sleep 的 semantic 面 —— 与搬移前一致:检索侧 setEmbedder(null) 降级关键词。
|
|
312
|
+
const semantic = createSemantic({
|
|
313
|
+
store, service, settings, cfg, logger: ctx.logger, vectorIndex, applyHumanEdits, lightMode
|
|
314
|
+
});
|
|
315
|
+
const { embedder, reranker } = semantic;
|
|
486
316
|
|
|
487
317
|
// Custom commands: register persisted commands into the DSH command registry
|
|
488
318
|
// on boot; add/remove re-register live through the API.
|
|
@@ -601,7 +431,7 @@ export const apply = (ctx, config) => {
|
|
|
601
431
|
|
|
602
432
|
// #118: never let a pending embedder init retry fire after unload and touch
|
|
603
433
|
// a torn-down context.
|
|
604
|
-
disposers.push(() =>
|
|
434
|
+
disposers.push(() => semantic.dispose()); // #118 重试计时器 + boot 回填计时器(搬入 semantic.js 后由它自持)
|
|
605
435
|
|
|
606
436
|
ctx.inject(["systemPrompt"], (promptCtx) => {
|
|
607
437
|
if (cfg.autoInject) disposers.push(createInjector(promptCtx, service, settings, cfg));
|
|
@@ -649,7 +479,9 @@ export const apply = (ctx, config) => {
|
|
|
649
479
|
// kv and is auto-generated on first boot by createStandaloneApi. Binding a
|
|
650
480
|
// non-loopback host is the operator's documented responsibility.
|
|
651
481
|
if ((settings.getExternalApi?.()?.enabled ?? cfg.externalApiEnabled) === true) {
|
|
652
|
-
|
|
482
|
+
// embedder 交给 /context 的查询嵌入(issue #370):与 service.setEmbedder 同一
|
|
483
|
+
// 实例(semantic.js 装配产物),lightMode 下为 null,路由侧降级规则档。
|
|
484
|
+
const standalone = createStandaloneApi({ service, store, config: cfg, logger: ctx.logger, settings, maintenance, embedder });
|
|
653
485
|
disposers.push(() => standalone.server.close());
|
|
654
486
|
standalone.ready.catch((error) => {
|
|
655
487
|
ctx.logger?.warn?.(`[dsh-mneme] standalone API failed to start: ${String(error)}`);
|