@modusensus/dsh-mneme 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +727 -636
- package/bin/cli.mjs +0 -0
- package/bin/dsh-mneme-mcp.mjs +510 -0
- package/cordis.patch.yml +21 -21
- package/lib/api-standalone.js +506 -316
- package/lib/api.js +1403 -1316
- package/lib/bootstrap.js +211 -0
- package/lib/client.js +4657 -4366
- package/lib/commands.js +95 -95
- package/lib/config.js +615 -510
- package/lib/document.js +221 -0
- package/lib/dream/clustering.js +118 -118
- package/lib/dream/decisions.js +658 -658
- package/lib/dream/narratives.js +44 -0
- package/lib/dream/sleep.js +702 -700
- package/lib/dream.js +147 -16
- package/lib/embedding.js +177 -177
- package/lib/entities/extractor.js +249 -249
- package/lib/guide.js +44 -0
- package/lib/heat.js +144 -136
- package/lib/hot-memory.js +46 -46
- package/lib/index.js +515 -515
- package/lib/inject.js +375 -295
- package/lib/lang.js +208 -179
- package/lib/local-embedder.js +320 -320
- package/lib/mirror.js +198 -198
- package/lib/organize.js +450 -0
- package/lib/quality-filter.js +126 -126
- package/lib/recall-stats.js +125 -0
- package/lib/reranker.js +269 -269
- package/lib/runtime/adopt.js +176 -176
- package/lib/runtime/closure.js +172 -172
- package/lib/runtime/download.js +247 -247
- package/lib/runtime/layout.js +265 -235
- package/lib/runtime/loader.js +252 -249
- package/lib/runtime/model-fetch.js +361 -361
- package/lib/runtime/provision.js +237 -237
- package/lib/runtime/tarball.js +130 -130
- package/lib/runtime/verify.js +204 -191
- package/lib/scope.js +118 -118
- package/lib/search/adaptive.js +60 -22
- package/lib/search/bm25.js +96 -96
- package/lib/service.js +2324 -2087
- package/lib/settings.js +459 -420
- package/lib/store.js +2549 -2367
- package/lib/summarize.js +820 -449
- package/lib/tools.js +702 -574
- package/lib/vector-index.js +110 -110
- package/lib/version-check.js +122 -122
- package/package.json +109 -108
- package/runtime-manifest.json +526 -596
- package/scripts/benchmark-embed.js +201 -201
- package/scripts/benchmark-recall.js +199 -199
- package/scripts/benchmark-rerank.js +166 -166
- package/scripts/build-runtime-manifest.mjs +224 -217
- package/scripts/check-sync.js +42 -42
- package/scripts/e2e-dsh.js +220 -220
- package/scripts/mneme-runtime.mjs +277 -277
- package/scripts/stress-dsh.js +255 -255
- package/scripts/sync-lib.js +47 -47
- package/scripts/sync-test-badge.mjs +67 -61
- package/src/api-standalone.js +506 -316
- package/src/api.js +1403 -1316
- package/src/bootstrap.js +211 -0
- package/src/commands.js +95 -95
- package/src/config.js +615 -510
- package/src/document.js +221 -0
- package/src/dream/clustering.js +118 -118
- package/src/dream/decisions.js +658 -658
- package/src/dream/narratives.js +44 -0
- package/src/dream/sleep.js +702 -700
- package/src/dream.js +147 -16
- package/src/embedding.js +177 -177
- package/src/entities/extractor.js +249 -249
- package/src/guide.js +44 -0
- package/src/heat.js +144 -136
- package/src/hot-memory.js +46 -46
- package/src/index.js +515 -515
- package/src/inject.js +375 -295
- package/src/lang.js +208 -179
- package/src/local-embedder.js +320 -320
- package/src/mirror.js +198 -198
- package/src/organize.js +450 -0
- package/src/quality-filter.js +126 -126
- package/src/recall-stats.js +125 -0
- package/src/reranker.js +269 -269
- package/src/runtime/adopt.js +176 -176
- package/src/runtime/closure.js +172 -172
- package/src/runtime/download.js +247 -247
- package/src/runtime/layout.js +265 -235
- package/src/runtime/loader.js +252 -249
- package/src/runtime/model-fetch.js +361 -361
- package/src/runtime/provision.js +237 -237
- package/src/runtime/tarball.js +130 -130
- package/src/runtime/verify.js +204 -191
- package/src/scope.js +118 -118
- package/src/search/adaptive.js +60 -22
- package/src/search/bm25.js +96 -96
- package/src/service.js +2324 -2087
- package/src/settings.js +459 -420
- package/src/store.js +2549 -2367
- package/src/summarize.js +820 -449
- package/src/tools.js +702 -574
- package/src/vector-index.js +110 -110
- package/src/version-check.js +122 -122
- package/test/adaptive-inject-budget.test.js +85 -0
- package/test/api-routes-gaps.test.js +218 -218
- package/test/api.test.js +1640 -1482
- package/test/audit.test.js +448 -448
- package/test/benchmark.test.js +94 -94
- package/test/bootstrap.test.js +220 -0
- package/test/client.test.js +860 -736
- package/test/clustering.test.js +100 -100
- package/test/commands.test.js +106 -106
- package/test/config.test.js +66 -66
- package/test/conflict-freeze.test.js +300 -300
- package/test/document.test.js +523 -0
- package/test/dream-archive-guardrail.test.js +116 -116
- package/test/dream-candidate-hybrid.test.js +281 -281
- package/test/dream-coverage-degrade.test.js +86 -86
- package/test/dream.test.js +1308 -1254
- package/test/embedding-ready.test.js +196 -196
- package/test/entities.test.js +547 -547
- package/test/entity-recall.test.js +179 -0
- package/test/epistemic.test.js +298 -298
- package/test/fnew-0112.test.js +311 -311
- package/test/fnew-03.test.js +422 -422
- package/test/graph-api.test.js +175 -175
- package/test/heat.test.js +175 -148
- package/test/helpers/dream-mock.js +82 -82
- package/test/helpers/peer-worker.mjs +25 -25
- package/test/helpers/runtime-source.js +60 -60
- package/test/helpers/tar-builder.js +80 -80
- package/test/hot-memory.test.js +145 -145
- package/test/inject-content-cap.test.js +91 -0
- package/test/inject-first-round-bm25.test.js +89 -89
- package/test/inject-heat.test.js +61 -0
- package/test/inject-pools.test.js +170 -0
- package/test/inject-recall.test.js +105 -0
- package/test/inject-rotation.test.js +161 -161
- package/test/inject.test.js +283 -249
- package/test/lang.test.js +87 -87
- package/test/lib-smoke.test.js +96 -96
- package/test/llm-audit.test.js +462 -309
- package/test/llm-message-source.test.js +177 -177
- package/test/local-embedder.test.js +249 -249
- package/test/mcp-stdio.test.js +385 -0
- package/test/mirror-dirty.test.js +424 -424
- package/test/mirror-edit-digest.test.js +187 -187
- package/test/mirror-generation.test.js +499 -499
- package/test/mirror.test.js +249 -249
- package/test/narrative-bars.test.js +162 -0
- package/test/organize.test.js +418 -0
- package/test/peer-blockers.test.js +193 -193
- package/test/policy-epoch.test.js +259 -259
- package/test/quality-filter.test.js +186 -186
- package/test/reasoning-effort.test.js +746 -746
- package/test/recall-evals.test.js +235 -235
- package/test/recall-layer.test.js +326 -326
- package/test/recall-stats.test.js +123 -0
- package/test/receipt-chain.test.js +451 -451
- package/test/reflection.test.js +226 -226
- package/test/reindex-backfill.test.js +62 -62
- package/test/reranker-single-logit.test.js +73 -73
- package/test/reranker.test.js +197 -197
- package/test/resident-status.test.js +83 -0
- package/test/runtime-adopt.test.js +206 -206
- package/test/runtime-cli.test.js +89 -89
- package/test/runtime-closure.test.js +193 -193
- package/test/runtime-download.test.js +302 -302
- package/test/runtime-layout.test.js +163 -163
- package/test/runtime-loader.test.js +255 -237
- package/test/runtime-manifest.test.js +159 -158
- package/test/runtime-model-fetch.test.js +348 -348
- package/test/runtime-provision.test.js +101 -101
- package/test/runtime-tarball.test.js +81 -81
- package/test/runtime-verify.test.js +279 -214
- package/test/scope-candidates.test.js +239 -239
- package/test/scope-explicit.test.js +452 -452
- package/test/scope-retrieval.test.js +197 -197
- package/test/scope-storage.test.js +255 -255
- package/test/scope-strict.test.js +231 -231
- package/test/search-fusion.test.js +90 -90
- package/test/semantic.test.js +124 -124
- package/test/service-search.test.js +174 -174
- package/test/service.test.js +405 -405
- package/test/settings.test.js +261 -261
- package/test/skip-invalid-restore.test.js +262 -262
- package/test/sleep-action-set.test.js +422 -422
- package/test/sleep-heat.test.js +126 -125
- package/test/sleep-scheduler-rearm.test.js +89 -89
- package/test/sleep.test.js +410 -373
- package/test/standalone-api.test.js +544 -359
- package/test/store-vector-cache.test.js +98 -98
- package/test/store.test.js +396 -383
- package/test/stress.test.js +209 -209
- package/test/summarize.test.js +1288 -575
- package/test/tools-dto-schema-matrix.test.js +325 -0
- package/test/tools-memory-get-schema.test.js +65 -65
- package/test/tools.test.js +374 -341
- package/test/updated-at-semantics.test.js +113 -113
- package/test/vector-index.test.js +221 -221
- package/test/version-check.test.js +152 -152
- package/test/webServer-optional.test.js +82 -82
package/lib/dream.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { validateDecisions, applyDecisions } from "./dream/decisions.js";
|
|
2
2
|
import { clusterMemories, findPotentialConflicts, cosineSimilarity } from "./dream/clustering.js";
|
|
3
|
+
import { clusterByTag, intersectEvidence } from "./dream/narratives.js";
|
|
3
4
|
import { scopeKeyOf } from "./scope.js";
|
|
4
5
|
import { createHash, randomUUID } from "node:crypto";
|
|
5
6
|
import { STR, langOf } from "./lang.js";
|
|
6
7
|
export { validateDecisions, applyDecisions, withEffortFallback, describeStreamFailure, resolveDreamEffort, resolveRoute };
|
|
7
8
|
|
|
8
|
-
|
|
9
9
|
// Extract the first JSON array from LLM output, tolerating markdown fences,
|
|
10
10
|
// leading/trailing prose, and common wrapper noise. Returns an array or null.
|
|
11
11
|
function extractJsonArray(text) {
|
|
@@ -231,7 +231,12 @@ async function streamText(ctx, options, onUsage, onStreamError) {
|
|
|
231
231
|
let text = "";
|
|
232
232
|
for await (const chunk of ctx.llm.stream(options)) {
|
|
233
233
|
if (chunk.type === "text-delta" && typeof chunk.text === "string") text += chunk.text;
|
|
234
|
-
|
|
234
|
+
// DSH 的 StreamChunk 契约把用量嵌在 chunk.usage(TokenUsage:inputTokens /
|
|
235
|
+
// outputTokens / …),chunk 顶层没有 token 字段——把整个 chunk 当用量对象交给
|
|
236
|
+
// reporter,input/output 恒为 undefined,审计行落 0(v0.8.4 实测:7 天 49 次
|
|
237
|
+
// success 调用 token 全 0,面板「LLM 消耗」长期显示 0)。`?? chunk` 兜底把用量
|
|
238
|
+
// 平铺在顶层的替身。
|
|
239
|
+
if (chunk.type === "usage" && typeof onUsage === "function") onUsage(chunk.usage ?? chunk);
|
|
235
240
|
if (chunk.type === "finish" && (chunk.reason?.kind === "error" || chunk.reason?.kind === "aborted")) {
|
|
236
241
|
// dsh-llm rc.1 turns adapter-stage failures (unknown provider route,
|
|
237
242
|
// UNSUPPORTED_REASONING_EFFORT from resolveCallWithInfo, …) into a
|
|
@@ -362,6 +367,85 @@ async function withEffortFallback(ctx, effort, attempt, fallback, getStreamError
|
|
|
362
367
|
}
|
|
363
368
|
}
|
|
364
369
|
|
|
370
|
+
/** 向量索引同步(总览与叙述条共用):fail-safe,失败只 warn 不反噬主流程。 */
|
|
371
|
+
async function reEmbedMemory(semantic, memory, logger) {
|
|
372
|
+
if (!semantic?.embedder || !semantic?.vectorIndex || !memory) return;
|
|
373
|
+
try {
|
|
374
|
+
const v = await semantic.embedder.embedSingle([memory.title, memory.content].filter(Boolean).join("\n"));
|
|
375
|
+
if (v?.length) semantic.vectorIndex.saveEmbedding(memory.id, v);
|
|
376
|
+
if (semantic.embedder.modelHash) semantic.vectorIndex.markModel?.(semantic.embedder.modelHash, semantic.embedder.dimension);
|
|
377
|
+
} catch (error) {
|
|
378
|
+
logger?.warn?.(`dsh-mneme dream: re-embed failed: ${String(error)}`);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/**
|
|
383
|
+
* 叙述条阶段(#164 对齐,opt-in dreamNarrativeEnabled):共享 tag 主题聚类 →
|
|
384
|
+
* 单次 LLM 调用按簇合成叙述 → 证据求交(防捏造,dream/narratives.js 纯函数)
|
|
385
|
+
* → saveWithDedupe(source="narrative", _overwrite) 落库。按需检索不常驻注入。
|
|
386
|
+
* 任何失败降级为 0 条,绝不反噬 dream 主流程。
|
|
387
|
+
*/
|
|
388
|
+
async function generateNarratives({ ctx, service, config, route, language, effort, semantic, logger }) {
|
|
389
|
+
const inputs = service.all().filter((m) => !m.archived && !m.forgotten && m.type !== "summary" && m.type !== "document");
|
|
390
|
+
const clusters = clusterByTag(inputs, { minCluster: config.dreamNarrativeMinCluster ?? 3 });
|
|
391
|
+
if (!clusters.length) return 0;
|
|
392
|
+
|
|
393
|
+
const listing = clusters
|
|
394
|
+
.map(({ tag, members }) =>
|
|
395
|
+
[`# ${tag}`, ...members.map((m) => `- id=${m.id} | title=${m.title} | ${m.content}`)].join("\n"))
|
|
396
|
+
.join("\n\n");
|
|
397
|
+
let streamFailure = "";
|
|
398
|
+
const runNarratives = (withEffort) => {
|
|
399
|
+
streamFailure = "";
|
|
400
|
+
return runAuditedLlm(ctx, service, config, {
|
|
401
|
+
triggerSource: "autoDream",
|
|
402
|
+
operationType: "dream_narrative",
|
|
403
|
+
modelId: `${route.provider}:${route.model}`,
|
|
404
|
+
relatedMemoryIds: clusters.flatMap((c) => c.members.map((m) => m.id)),
|
|
405
|
+
streamError: () => streamFailure
|
|
406
|
+
}, (reportUsage) => streamText(ctx, {
|
|
407
|
+
provider: route.provider,
|
|
408
|
+
model: route.model,
|
|
409
|
+
purpose: "compaction",
|
|
410
|
+
maxTokens: config.dreamMaxTokens ?? 2048,
|
|
411
|
+
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
412
|
+
messages: [
|
|
413
|
+
{ role: "system", content: [{ type: "text", text: STR.prompts.narrative[language] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
414
|
+
{ role: "user", content: [{ type: "text", text: listing }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
415
|
+
]
|
|
416
|
+
}, reportUsage, (reason) => { streamFailure = describeStreamFailure(reason); }));
|
|
417
|
+
};
|
|
418
|
+
const text = await withEffortFallback(ctx, effort, () => runNarratives(true), () => runNarratives(false), () => streamFailure);
|
|
419
|
+
|
|
420
|
+
const entries = extractJsonArray(text);
|
|
421
|
+
if (!Array.isArray(entries)) return 0;
|
|
422
|
+
const clusterByOfferedTag = new Map(clusters.map((c) => [c.tag, c]));
|
|
423
|
+
const now = new Date().toISOString();
|
|
424
|
+
let stored = 0;
|
|
425
|
+
for (const entry of entries) {
|
|
426
|
+
if (!entry || typeof entry !== "object") continue;
|
|
427
|
+
const cluster = clusterByOfferedTag.get(typeof entry.tag === "string" ? entry.tag : "");
|
|
428
|
+
const content = String(entry.content ?? "").trim();
|
|
429
|
+
if (!cluster || !content) continue; // 未提供的 tag / 空叙述 → 跳过
|
|
430
|
+
const evidenceIds = intersectEvidence(entry.evidence, cluster.members);
|
|
431
|
+
const { memory } = service.saveWithDedupe({
|
|
432
|
+
type: "summary",
|
|
433
|
+
title: STR.narrativeTitle[language](cluster.tag),
|
|
434
|
+
content,
|
|
435
|
+
importance: 3,
|
|
436
|
+
tags: [cluster.tag],
|
|
437
|
+
source: "narrative",
|
|
438
|
+
evidence: evidenceIds.map((id) => ({ memory_id: id, op: "support", at: now })),
|
|
439
|
+
_overwrite: true
|
|
440
|
+
});
|
|
441
|
+
if (!memory) continue;
|
|
442
|
+
stored++;
|
|
443
|
+
// 叙述条按需可检索:向量索引同步(与总览共用 reEmbedMemory,fail-safe)。
|
|
444
|
+
await reEmbedMemory(semantic, memory, logger);
|
|
445
|
+
}
|
|
446
|
+
return stored;
|
|
447
|
+
}
|
|
448
|
+
|
|
365
449
|
/**
|
|
366
450
|
* Resolve the reasoning effort to actually send for a dream/sleep route.
|
|
367
451
|
*
|
|
@@ -671,7 +755,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
671
755
|
let lastRunAt = 0;
|
|
672
756
|
|
|
673
757
|
function shouldTrigger(service) {
|
|
674
|
-
const memories = service.all().filter((m) => !m.archived && m.type !== "summary");
|
|
758
|
+
const memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
|
|
675
759
|
const count = memories.length;
|
|
676
760
|
const chars = totalChars(memories);
|
|
677
761
|
const overBase = count >= baseline.count + thresholdCount || chars >= baseline.chars + thresholdChars;
|
|
@@ -734,7 +818,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
734
818
|
async function runDream(ctx, service, config) {
|
|
735
819
|
const language = langOf(config);
|
|
736
820
|
const logger = ctx.logger;
|
|
737
|
-
let memories = service.all().filter((m) => !m.archived && m.type !== "summary");
|
|
821
|
+
let memories = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
|
|
738
822
|
if (memories.length === 0) return { ok: true, applied: 0, skipped: true, summary: false };
|
|
739
823
|
// v0.4.4 滑动窗口:只 consolidation 最近 dreamMaxSnapshotSize 条记忆,
|
|
740
824
|
// 窗口外的旧记忆不进 snapshot(大记忆量下全量快照会撑爆 LLM 输入,配合
|
|
@@ -1126,24 +1210,59 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1126
1210
|
let summaryStreamFailure = "";
|
|
1127
1211
|
const runSummary = (withEffort) => {
|
|
1128
1212
|
summaryStreamFailure = "";
|
|
1213
|
+
// Issue #258:总览阶段可用 dreamSummaryProvider/dreamSummaryModel 走独立路由
|
|
1214
|
+
//(默认空 = 沿用 dream 路由);输入侧有 dreamSummaryMaxInputs 硬上限,
|
|
1215
|
+
// 按窗口同款 updated_at 倒序保留最新的(见下方 summaryInputs)。
|
|
1216
|
+
const summaryRoute = (config.dreamSummaryProvider && config.dreamSummaryModel)
|
|
1217
|
+
? { provider: config.dreamSummaryProvider, model: config.dreamSummaryModel }
|
|
1218
|
+
: route;
|
|
1129
1219
|
return runAuditedLlm(ctx, service, config, {
|
|
1130
1220
|
triggerSource: "autoDream",
|
|
1131
1221
|
operationType: "dream_summarize",
|
|
1132
|
-
modelId: `${
|
|
1222
|
+
modelId: `${summaryRoute.provider}:${summaryRoute.model}`,
|
|
1133
1223
|
relatedMemoryIds: [],
|
|
1134
1224
|
streamError: () => summaryStreamFailure
|
|
1135
1225
|
}, (reportUsage) => streamText(ctx, {
|
|
1136
|
-
provider:
|
|
1137
|
-
model:
|
|
1226
|
+
provider: summaryRoute.provider,
|
|
1227
|
+
model: summaryRoute.model,
|
|
1138
1228
|
purpose: "compaction",
|
|
1139
1229
|
maxTokens: config.dreamMaxTokens ?? 2048,
|
|
1140
1230
|
...(withEffort && effort ? { reasoningEffort: effort } : {}),
|
|
1141
1231
|
messages: [
|
|
1142
1232
|
{ role: "system", content: [{ type: "text", text: STR.prompts.dreamSummary[language] }], source: { kind: "plugin", plugin: "dsh-mneme" } },
|
|
1143
|
-
{ role: "user", content: [{ type: "text", text:
|
|
1233
|
+
{ role: "user", content: [{ type: "text", text: summaryInputs.map((m) => `- ${m.title}: ${m.content}`).join("\n") }], source: { kind: "plugin", plugin: "dsh-mneme" } }
|
|
1144
1234
|
]
|
|
1145
1235
|
}, reportUsage, (reason) => { summaryStreamFailure = describeStreamFailure(reason); }));
|
|
1146
1236
|
};
|
|
1237
|
+
// Resident status bar (#164 alignment): the overview is THE one resident
|
|
1238
|
+
// narrative — summary tier 0, refreshed every run via _overwrite (supersede
|
|
1239
|
+
// v1, content_history traceable). Its 口径 (what snapshot produced it) is
|
|
1240
|
+
// stamped into the content so the standing answer is always auditable; the
|
|
1241
|
+
// formal evidence column lands with the narrative-bars batch.
|
|
1242
|
+
// Issue #258:输入硬上限(条数)——原全库无界,把 dream 路由指向小 ctx
|
|
1243
|
+
// 模型时 120k token 当场 CONTEXT_WINDOW_EXCEEDED(#258 实测),且随库增长
|
|
1244
|
+
// 渐进恶化。倒序排序与 consolidate 窗口同款(updated_at desc, id tiebreak),
|
|
1245
|
+
// 保留最新的;0 = 关闭上限(回归全库行为,调用方自担 ctx)。
|
|
1246
|
+
// #230:document 指针行不进总览(与 dream/sleep 五个候选池的同款排除)。
|
|
1247
|
+
const summaryAll = service.all().filter((m) => !m.archived && m.type !== "summary" && m.type !== "document");
|
|
1248
|
+
const summaryMaxInputs = Number.isInteger(config.dreamSummaryMaxInputs) ? config.dreamSummaryMaxInputs : 0;
|
|
1249
|
+
const summaryInputs = summaryMaxInputs > 0 && summaryAll.length > summaryMaxInputs
|
|
1250
|
+
? [...summaryAll]
|
|
1251
|
+
.sort((a, b) => {
|
|
1252
|
+
const ta = String(a.updated_at ?? "");
|
|
1253
|
+
const tb = String(b.updated_at ?? "");
|
|
1254
|
+
if (ta < tb) return 1;
|
|
1255
|
+
if (ta > tb) return -1;
|
|
1256
|
+
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
|
|
1257
|
+
})
|
|
1258
|
+
.slice(0, summaryMaxInputs)
|
|
1259
|
+
: summaryAll;
|
|
1260
|
+
|
|
1261
|
+
const summaryScope = STR.summaryScope[language](
|
|
1262
|
+
summaryInputs.length,
|
|
1263
|
+
runId.slice(0, 8),
|
|
1264
|
+
new Date().toISOString().slice(0, 10)
|
|
1265
|
+
);
|
|
1147
1266
|
try {
|
|
1148
1267
|
// Bug8: the summary call is audited too (operation dream_summarize).
|
|
1149
1268
|
summaryText = await withEffortFallback(ctx, effort, () => runSummary(true), () => runSummary(false), () => summaryStreamFailure);
|
|
@@ -1156,21 +1275,32 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1156
1275
|
// Bug5 carve-out: the library overview is regenerated every run, so it
|
|
1157
1276
|
// must REPLACE the previous overview (not append — that would grow the
|
|
1158
1277
|
// summary unboundedly). `_overwrite` still archives the old overview into
|
|
1159
|
-
// content_history before replacing it.
|
|
1160
|
-
|
|
1278
|
+
// content_history before replacing it. Content carries the snapshot-scope
|
|
1279
|
+
// footer computed above — the resident bar states its own 口径.
|
|
1280
|
+
service.saveWithDedupe({ type: "summary", title: STR.summaryTitle[language], content: `${summaryText.trim()}${summaryScope}`, importance: 5, source: "dream", _overwrite: true });
|
|
1161
1281
|
summaryStored = true;
|
|
1162
1282
|
// Re-embed the fresh summary so the index stays in sync with the store.
|
|
1163
1283
|
if (semantic?.embedder && semantic?.vectorIndex) {
|
|
1164
1284
|
try {
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
if (semantic.embedder.modelHash) semantic.vectorIndex.markModel?.(semantic.embedder.modelHash, semantic.embedder.dimension);
|
|
1170
|
-
}
|
|
1285
|
+
// source="dream" 限定:叙述条(source=narrative)同为 summary 类型,
|
|
1286
|
+
// 不加限定时 find 可能截胡,把总览的 re-embed 花在叙述条上。
|
|
1287
|
+
const summary = service.all().find((m) => m.type === "summary" && m.source === "dream");
|
|
1288
|
+
if (summary) await reEmbedMemory(semantic, summary, logger);
|
|
1171
1289
|
} catch { /* best-effort */ }
|
|
1172
1290
|
}
|
|
1173
1291
|
}
|
|
1292
|
+
// Narrative bars (#164 alignment, opt-in dreamNarrativeEnabled): per-topic
|
|
1293
|
+
// on-demand narratives after consolidation + summary. Additive phase —
|
|
1294
|
+
// failures degrade to zero narratives, never break the run.
|
|
1295
|
+
let narrativesStored = 0;
|
|
1296
|
+
if (config.dreamNarrativeEnabled === true) {
|
|
1297
|
+
try {
|
|
1298
|
+
narrativesStored = await generateNarratives({ ctx, service, config, route, language, effort, semantic, logger });
|
|
1299
|
+
} catch (error) {
|
|
1300
|
+
logger?.warn?.(`dsh-mneme dream: narrative phase failed: ${String(error)}`);
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1174
1304
|
// Honest status assignment (never a fake ok):
|
|
1175
1305
|
// reconcile — some decisions validated but did not commit (CAS/rollback).
|
|
1176
1306
|
// noop — nothing changed and no summary persisted: truly an empty
|
|
@@ -1206,6 +1336,7 @@ export function createDreamScheduler({ onRun, thresholdCount = 10, thresholdChar
|
|
|
1206
1336
|
failures,
|
|
1207
1337
|
frozen: frozenCount,
|
|
1208
1338
|
summary: summaryStored,
|
|
1339
|
+
narratives: narrativesStored,
|
|
1209
1340
|
// Issue #104:只有真发生跳过时才落库(degraded 也可能仅因 summary 为空),
|
|
1210
1341
|
// ok 轮留 NULL,避免「空数组」与「无此字段」两种假明细占据审计行。
|
|
1211
1342
|
skipped: skippedInvalid ? skipped : undefined
|
package/lib/embedding.js
CHANGED
|
@@ -1,177 +1,177 @@
|
|
|
1
|
-
// OpenAI-compatible embedding client for vector search. DSH's LLM service is
|
|
2
|
-
// chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
|
|
3
|
-
// Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
|
|
4
|
-
// proxy) and any provider exposing the standard embeddings API.
|
|
5
|
-
const DEFAULT_TIMEOUT_MS = 15000;
|
|
6
|
-
|
|
7
|
-
/** djb2 — stable, fast fingerprint for a provider/model string. Mirrors the
|
|
8
|
-
* hash used by the local embedders so all backends share one fingerprint
|
|
9
|
-
* format (model#hex) for vector_meta consistency checks. */
|
|
10
|
-
function hashString(s) {
|
|
11
|
-
let h = 5381;
|
|
12
|
-
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
13
|
-
return h.toString(16);
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/** Full provider+model fingerprint used for index-consistency checks. */
|
|
17
|
-
function modelHashOf(model) {
|
|
18
|
-
return `${model}#${hashString(model)}`;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
|
|
22
|
-
function embeddingsUrl(baseUrl) {
|
|
23
|
-
const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
|
|
24
|
-
if (!base) return "";
|
|
25
|
-
// Accept both "https://host/v1" and a full path ending in /embeddings.
|
|
26
|
-
if (/\/embeddings$/i.test(base)) return base;
|
|
27
|
-
return `${base}/embeddings`;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Call the embeddings API for one text. Resolves to a Float64 array, or null
|
|
32
|
-
* when the provider is not configured, the call fails, or the response is
|
|
33
|
-
* unusable. Never throws: failures degrade to keyword search.
|
|
34
|
-
*/
|
|
35
|
-
export async function embedText({ baseUrl, apiKey, model }, text) {
|
|
36
|
-
const url = embeddingsUrl(baseUrl);
|
|
37
|
-
if (!url || !apiKey || !model || !text) return null;
|
|
38
|
-
let res;
|
|
39
|
-
try {
|
|
40
|
-
res = await fetch(url, {
|
|
41
|
-
method: "POST",
|
|
42
|
-
headers: {
|
|
43
|
-
"Content-Type": "application/json",
|
|
44
|
-
"Authorization": `Bearer ${apiKey}`
|
|
45
|
-
},
|
|
46
|
-
body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
|
|
47
|
-
signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
|
|
48
|
-
});
|
|
49
|
-
} catch {
|
|
50
|
-
return null;
|
|
51
|
-
}
|
|
52
|
-
if (!res.ok) return null;
|
|
53
|
-
let body;
|
|
54
|
-
try {
|
|
55
|
-
body = await res.json();
|
|
56
|
-
} catch {
|
|
57
|
-
return null;
|
|
58
|
-
}
|
|
59
|
-
const vec = body?.data?.[0]?.embedding;
|
|
60
|
-
return Array.isArray(vec) && vec.length ? Array.from(vec) : null;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
/**
|
|
64
|
-
* Embedder bound to the current settings + store: on each write it re-embeds
|
|
65
|
-
* the row's title+content and stores the vector. Failures are swallowed so a
|
|
66
|
-
* flaky embedding endpoint never breaks memory writes.
|
|
67
|
-
*
|
|
68
|
-
* `vectorIndex` (optional) is the vector_meta fingerprint holder: after any
|
|
69
|
-
* successful embed the model that produced the vectors is recorded, so the
|
|
70
|
-
* index can detect drift and the auto-reindex backfill knows what to rebuild.
|
|
71
|
-
*/
|
|
72
|
-
export function createEmbedder({ store, settings, logger, vectorIndex }) {
|
|
73
|
-
// Dimension of the most recent successful embed, exposed for fingerprinting.
|
|
74
|
-
let _dimension = 0;
|
|
75
|
-
|
|
76
|
-
/** Record the producing model fingerprint in vector_meta (best-effort). */
|
|
77
|
-
function markModel(cfg, dimension) {
|
|
78
|
-
if (!vectorIndex || typeof vectorIndex.markModel !== "function") return;
|
|
79
|
-
try {
|
|
80
|
-
vectorIndex.markModel(modelHashOf(cfg.model), dimension);
|
|
81
|
-
} catch { /* metadata write is best-effort */ }
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// issue #135: 「配置是否齐备」的唯一判据 —— embed()/embedFor() 的守卫与下面
|
|
85
|
-
// ready/configured 两个 getter 共用,避免以后改一处漏一处。
|
|
86
|
-
const configComplete = (cfg) => !!(cfg?.enabled && cfg.baseUrl && cfg.apiKey && cfg.model);
|
|
87
|
-
|
|
88
|
-
async function embedFor(id, title, content) {
|
|
89
|
-
const cfg = settings.getVectorConfig();
|
|
90
|
-
if (!configComplete(cfg)) return;
|
|
91
|
-
const text = [title, content].filter(Boolean).join("\n");
|
|
92
|
-
const vector = await embedText(cfg, text);
|
|
93
|
-
if (vector) {
|
|
94
|
-
store.setEmbedding(id, vector);
|
|
95
|
-
_dimension = vector.length;
|
|
96
|
-
// Bug3: record which model produced the current vectors so the index can
|
|
97
|
-
// detect drift and skip a redundant backfill when nothing changed.
|
|
98
|
-
markModel(cfg, vector.length);
|
|
99
|
-
logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
return {
|
|
104
|
-
// Display name for /semantic: a literal's constructor.name is "Object",
|
|
105
|
-
// which the status card would render verbatim.
|
|
106
|
-
name: "OpenAI",
|
|
107
|
-
/**
|
|
108
|
-
* issue #135: 这个 embedder 没有异步初始化,`"ready" in embedder` 的通用
|
|
109
|
-
* 分支会把它恒判为可用 —— 而 vector-config 四项有缺时它一条都嵌不出来。
|
|
110
|
-
* 对外报真实可用性,让 /semantic 的 ready 不再是「绿色假阳性」。
|
|
111
|
-
* 用 getter 而非快照字段:设置面板改完配置无需重启即生效(与 embed() 同源读取)。
|
|
112
|
-
*/
|
|
113
|
-
get ready() {
|
|
114
|
-
return configComplete(settings.getVectorConfig());
|
|
115
|
-
},
|
|
116
|
-
/**
|
|
117
|
-
* 与 ready 同源,分开两个名字是给 /semantic 用:local/ollama 的
|
|
118
|
-
* ready=false 表示「还在初始化」,而这里的 false 表示「根本没配」。
|
|
119
|
-
*/
|
|
120
|
-
get configured() {
|
|
121
|
-
return configComplete(settings.getVectorConfig());
|
|
122
|
-
},
|
|
123
|
-
/** Fire-and-forget re-embed of a memory after any write. */
|
|
124
|
-
schedule(memory) {
|
|
125
|
-
if (!memory?.id) return;
|
|
126
|
-
embedFor(memory.id, memory.title, memory.content).catch(() => {});
|
|
127
|
-
},
|
|
128
|
-
|
|
129
|
-
/** Embed one text and return its vector (null on failure/disabled). */
|
|
130
|
-
async embed(query) {
|
|
131
|
-
const cfg = settings.getVectorConfig();
|
|
132
|
-
if (!configComplete(cfg)) return null;
|
|
133
|
-
const vector = await embedText(cfg, query);
|
|
134
|
-
if (vector) _dimension = vector.length;
|
|
135
|
-
return vector;
|
|
136
|
-
},
|
|
137
|
-
|
|
138
|
-
// Bug1: single-text adapter. Local/ollama embedders expose embedSingle
|
|
139
|
-
// natively; the legacy OpenAI-compatible client only has embed. This
|
|
140
|
-
// adapter unifies the interface so vector-index rebuildIndex (which guards
|
|
141
|
-
// on `typeof embedder.embedSingle === "function"`) accepts this embedder.
|
|
142
|
-
async embedSingle(text) {
|
|
143
|
-
if (typeof this.embed === "function") return this.embed(text);
|
|
144
|
-
return null;
|
|
145
|
-
},
|
|
146
|
-
|
|
147
|
-
/** Model fingerprint (model#hex), or undefined when not configured. */
|
|
148
|
-
get modelHash() {
|
|
149
|
-
const cfg = settings.getVectorConfig();
|
|
150
|
-
return cfg?.enabled && cfg.model ? modelHashOf(cfg.model) : undefined;
|
|
151
|
-
},
|
|
152
|
-
|
|
153
|
-
/** Dimension of the last successful embed (0 when never embedded). */
|
|
154
|
-
get dimension() {
|
|
155
|
-
return _dimension || undefined;
|
|
156
|
-
},
|
|
157
|
-
|
|
158
|
-
/** Batch re-index rows still missing an embedding. */
|
|
159
|
-
async reindexMissing(limit = 50) {
|
|
160
|
-
const cfg = settings.getVectorConfig();
|
|
161
|
-
if (!cfg?.enabled) return { indexed: 0, skipped: 0 };
|
|
162
|
-
const rows = store.needsEmbedding(limit);
|
|
163
|
-
let indexed = 0;
|
|
164
|
-
for (const row of rows) {
|
|
165
|
-
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
166
|
-
const vector = await embedText(cfg, text);
|
|
167
|
-
if (vector) {
|
|
168
|
-
store.setEmbedding(row.id, vector);
|
|
169
|
-
_dimension = vector.length;
|
|
170
|
-
indexed++;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
if (indexed > 0) markModel(cfg, _dimension || undefined);
|
|
174
|
-
return { indexed, skipped: rows.length - indexed };
|
|
175
|
-
}
|
|
176
|
-
};
|
|
177
|
-
}
|
|
1
|
+
// OpenAI-compatible embedding client for vector search. DSH's LLM service is
|
|
2
|
+
// chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
|
|
3
|
+
// Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
|
|
4
|
+
// proxy) and any provider exposing the standard embeddings API.
|
|
5
|
+
const DEFAULT_TIMEOUT_MS = 15000;
|
|
6
|
+
|
|
7
|
+
/** djb2 — stable, fast fingerprint for a provider/model string. Mirrors the
|
|
8
|
+
* hash used by the local embedders so all backends share one fingerprint
|
|
9
|
+
* format (model#hex) for vector_meta consistency checks. */
|
|
10
|
+
function hashString(s) {
|
|
11
|
+
let h = 5381;
|
|
12
|
+
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
13
|
+
return h.toString(16);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Full provider+model fingerprint used for index-consistency checks. */
|
|
17
|
+
function modelHashOf(model) {
|
|
18
|
+
return `${model}#${hashString(model)}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
|
|
22
|
+
function embeddingsUrl(baseUrl) {
|
|
23
|
+
const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
|
|
24
|
+
if (!base) return "";
|
|
25
|
+
// Accept both "https://host/v1" and a full path ending in /embeddings.
|
|
26
|
+
if (/\/embeddings$/i.test(base)) return base;
|
|
27
|
+
return `${base}/embeddings`;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Call the embeddings API for one text. Resolves to a Float64 array, or null
|
|
32
|
+
* when the provider is not configured, the call fails, or the response is
|
|
33
|
+
* unusable. Never throws: failures degrade to keyword search.
|
|
34
|
+
*/
|
|
35
|
+
export async function embedText({ baseUrl, apiKey, model }, text) {
|
|
36
|
+
const url = embeddingsUrl(baseUrl);
|
|
37
|
+
if (!url || !apiKey || !model || !text) return null;
|
|
38
|
+
let res;
|
|
39
|
+
try {
|
|
40
|
+
res = await fetch(url, {
|
|
41
|
+
method: "POST",
|
|
42
|
+
headers: {
|
|
43
|
+
"Content-Type": "application/json",
|
|
44
|
+
"Authorization": `Bearer ${apiKey}`
|
|
45
|
+
},
|
|
46
|
+
body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
|
|
47
|
+
signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
|
|
48
|
+
});
|
|
49
|
+
} catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
if (!res.ok) return null;
|
|
53
|
+
let body;
|
|
54
|
+
try {
|
|
55
|
+
body = await res.json();
|
|
56
|
+
} catch {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
const vec = body?.data?.[0]?.embedding;
|
|
60
|
+
return Array.isArray(vec) && vec.length ? Array.from(vec) : null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Embedder bound to the current settings + store: on each write it re-embeds
|
|
65
|
+
* the row's title+content and stores the vector. Failures are swallowed so a
|
|
66
|
+
* flaky embedding endpoint never breaks memory writes.
|
|
67
|
+
*
|
|
68
|
+
* `vectorIndex` (optional) is the vector_meta fingerprint holder: after any
|
|
69
|
+
* successful embed the model that produced the vectors is recorded, so the
|
|
70
|
+
* index can detect drift and the auto-reindex backfill knows what to rebuild.
|
|
71
|
+
*/
|
|
72
|
+
export function createEmbedder({ store, settings, logger, vectorIndex }) {
|
|
73
|
+
// Dimension of the most recent successful embed, exposed for fingerprinting.
|
|
74
|
+
let _dimension = 0;
|
|
75
|
+
|
|
76
|
+
/** Record the producing model fingerprint in vector_meta (best-effort). */
|
|
77
|
+
function markModel(cfg, dimension) {
|
|
78
|
+
if (!vectorIndex || typeof vectorIndex.markModel !== "function") return;
|
|
79
|
+
try {
|
|
80
|
+
vectorIndex.markModel(modelHashOf(cfg.model), dimension);
|
|
81
|
+
} catch { /* metadata write is best-effort */ }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// issue #135: 「配置是否齐备」的唯一判据 —— embed()/embedFor() 的守卫与下面
|
|
85
|
+
// ready/configured 两个 getter 共用,避免以后改一处漏一处。
|
|
86
|
+
const configComplete = (cfg) => !!(cfg?.enabled && cfg.baseUrl && cfg.apiKey && cfg.model);
|
|
87
|
+
|
|
88
|
+
async function embedFor(id, title, content) {
|
|
89
|
+
const cfg = settings.getVectorConfig();
|
|
90
|
+
if (!configComplete(cfg)) return;
|
|
91
|
+
const text = [title, content].filter(Boolean).join("\n");
|
|
92
|
+
const vector = await embedText(cfg, text);
|
|
93
|
+
if (vector) {
|
|
94
|
+
store.setEmbedding(id, vector);
|
|
95
|
+
_dimension = vector.length;
|
|
96
|
+
// Bug3: record which model produced the current vectors so the index can
|
|
97
|
+
// detect drift and skip a redundant backfill when nothing changed.
|
|
98
|
+
markModel(cfg, vector.length);
|
|
99
|
+
logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
return {
|
|
104
|
+
// Display name for /semantic: a literal's constructor.name is "Object",
|
|
105
|
+
// which the status card would render verbatim.
|
|
106
|
+
name: "OpenAI",
|
|
107
|
+
/**
|
|
108
|
+
* issue #135: 这个 embedder 没有异步初始化,`"ready" in embedder` 的通用
|
|
109
|
+
* 分支会把它恒判为可用 —— 而 vector-config 四项有缺时它一条都嵌不出来。
|
|
110
|
+
* 对外报真实可用性,让 /semantic 的 ready 不再是「绿色假阳性」。
|
|
111
|
+
* 用 getter 而非快照字段:设置面板改完配置无需重启即生效(与 embed() 同源读取)。
|
|
112
|
+
*/
|
|
113
|
+
get ready() {
|
|
114
|
+
return configComplete(settings.getVectorConfig());
|
|
115
|
+
},
|
|
116
|
+
/**
|
|
117
|
+
* 与 ready 同源,分开两个名字是给 /semantic 用:local/ollama 的
|
|
118
|
+
* ready=false 表示「还在初始化」,而这里的 false 表示「根本没配」。
|
|
119
|
+
*/
|
|
120
|
+
get configured() {
|
|
121
|
+
return configComplete(settings.getVectorConfig());
|
|
122
|
+
},
|
|
123
|
+
/** Fire-and-forget re-embed of a memory after any write. */
|
|
124
|
+
schedule(memory) {
|
|
125
|
+
if (!memory?.id) return;
|
|
126
|
+
embedFor(memory.id, memory.title, memory.content).catch(() => {});
|
|
127
|
+
},
|
|
128
|
+
|
|
129
|
+
/** Embed one text and return its vector (null on failure/disabled). */
|
|
130
|
+
async embed(query) {
|
|
131
|
+
const cfg = settings.getVectorConfig();
|
|
132
|
+
if (!configComplete(cfg)) return null;
|
|
133
|
+
const vector = await embedText(cfg, query);
|
|
134
|
+
if (vector) _dimension = vector.length;
|
|
135
|
+
return vector;
|
|
136
|
+
},
|
|
137
|
+
|
|
138
|
+
// Bug1: single-text adapter. Local/ollama embedders expose embedSingle
|
|
139
|
+
// natively; the legacy OpenAI-compatible client only has embed. This
|
|
140
|
+
// adapter unifies the interface so vector-index rebuildIndex (which guards
|
|
141
|
+
// on `typeof embedder.embedSingle === "function"`) accepts this embedder.
|
|
142
|
+
async embedSingle(text) {
|
|
143
|
+
if (typeof this.embed === "function") return this.embed(text);
|
|
144
|
+
return null;
|
|
145
|
+
},
|
|
146
|
+
|
|
147
|
+
/** Model fingerprint (model#hex), or undefined when not configured. */
|
|
148
|
+
get modelHash() {
|
|
149
|
+
const cfg = settings.getVectorConfig();
|
|
150
|
+
return cfg?.enabled && cfg.model ? modelHashOf(cfg.model) : undefined;
|
|
151
|
+
},
|
|
152
|
+
|
|
153
|
+
/** Dimension of the last successful embed (0 when never embedded). */
|
|
154
|
+
get dimension() {
|
|
155
|
+
return _dimension || undefined;
|
|
156
|
+
},
|
|
157
|
+
|
|
158
|
+
/** Batch re-index rows still missing an embedding. */
|
|
159
|
+
async reindexMissing(limit = 50) {
|
|
160
|
+
const cfg = settings.getVectorConfig();
|
|
161
|
+
if (!cfg?.enabled) return { indexed: 0, skipped: 0 };
|
|
162
|
+
const rows = store.needsEmbedding(limit);
|
|
163
|
+
let indexed = 0;
|
|
164
|
+
for (const row of rows) {
|
|
165
|
+
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
166
|
+
const vector = await embedText(cfg, text);
|
|
167
|
+
if (vector) {
|
|
168
|
+
store.setEmbedding(row.id, vector);
|
|
169
|
+
_dimension = vector.length;
|
|
170
|
+
indexed++;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (indexed > 0) markModel(cfg, _dimension || undefined);
|
|
174
|
+
return { indexed, skipped: rows.length - indexed };
|
|
175
|
+
}
|
|
176
|
+
};
|
|
177
|
+
}
|