@modusensus/dsh-mneme 0.14.0-beta.1 → 0.14.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +830 -830
- package/assets/icon.svg +3 -3
- package/bin/cli.mjs +964 -964
- package/bin/dsh-mneme-choro.mjs +262 -262
- package/bin/dsh-mneme-mcp.mjs +510 -510
- package/bin/dsh-mneme-serve.mjs +151 -151
- package/cordis.patch.yml +21 -21
- package/docs/CONFIGURATION.md +407 -404
- package/docs/CONTINUITY.md +139 -139
- package/docs/DAEMON.md +57 -57
- package/docs/DIRECTORIES.md +114 -114
- package/docs/DISTILL.md +397 -371
- package/docs/DSH-GO.md +97 -97
- package/docs/ENTITIES.md +247 -247
- package/docs/FORGET.md +77 -77
- package/docs/KB.md +110 -110
- package/docs/LOCAL_MODEL.md +222 -222
- package/docs/SEMANTIC.md +256 -256
- package/docs/SHARE.md +552 -552
- package/docs/SLEEP.md +202 -202
- package/docs/STORAGE.md +122 -122
- package/docs/WORKFLOWS.md +366 -366
- package/lib/api-standalone.js +1007 -1007
- package/lib/api.js +3600 -3600
- package/lib/attachment-text.js +122 -122
- package/lib/bootstrap-http.js +105 -105
- package/lib/bootstrap.js +215 -215
- package/lib/candidates.js +454 -454
- package/lib/capabilities.js +819 -819
- package/lib/choro-command.js +111 -111
- package/lib/client.js +14420 -14322
- package/lib/commands.js +95 -95
- package/lib/config-keys.js +144 -144
- package/lib/config.js +1361 -1344
- package/lib/content-hash.js +54 -54
- package/lib/continuity.js +258 -258
- package/lib/date-hint.js +32 -0
- package/lib/diary/auto.js +190 -190
- package/lib/diary/compose.js +440 -440
- package/lib/diary/dir.js +59 -59
- package/lib/diary/export.js +97 -97
- package/lib/diary/index.js +26 -26
- package/lib/diary/library.js +189 -189
- package/lib/diary/qq-inbox.js +142 -142
- package/lib/diary/rollover.js +61 -61
- package/lib/diary/session-provider.js +122 -122
- package/lib/document-index.js +94 -94
- package/lib/document.js +322 -322
- package/lib/dream/clustering.js +123 -123
- package/lib/dream/decisions.js +945 -945
- package/lib/dream/narratives.js +44 -44
- package/lib/dream/sleep.js +2027 -2027
- package/lib/dream/triage.js +139 -139
- package/lib/dream.js +1879 -1879
- package/lib/dsh-go/host.js +162 -162
- package/lib/dsh-go/panel.js +2091 -2091
- package/lib/dsh-go/text.js +342 -342
- package/lib/dsh-go/views/common.js +68 -68
- package/lib/dsh-go/views/entity.js +48 -48
- package/lib/dsh-go/views/memory.js +198 -198
- package/lib/dsh-go/views/root.js +30 -30
- package/lib/dsh-go/views/settings.js +398 -398
- package/lib/dsh-go/views/status.js +51 -51
- package/lib/embedding.js +342 -342
- package/lib/entities/extractor.js +285 -285
- package/lib/fence.js +24 -24
- package/lib/graph/anchoring.js +184 -184
- package/lib/guide.js +74 -74
- package/lib/heat.js +144 -144
- package/lib/hot-memory.js +119 -119
- package/lib/image-describe.js +158 -0
- package/lib/index.js +852 -852
- package/lib/inject.js +656 -651
- package/lib/kb/index.js +309 -309
- package/lib/kb/routes.js +219 -219
- package/lib/kb/tools.js +148 -148
- package/lib/lang.js +607 -595
- package/lib/local-embedder.js +358 -358
- package/lib/maintenance.js +162 -162
- package/lib/memory-source.js +9 -9
- package/lib/mirror.js +330 -330
- package/lib/organize.js +456 -456
- package/lib/peak-hours.js +172 -172
- package/lib/pipeline-activities.js +178 -178
- package/lib/pipeline-attribution.js +165 -165
- package/lib/pipeline-command.js +279 -279
- package/lib/pipeline-confirm.js +142 -142
- package/lib/pipeline-exec.js +174 -174
- package/lib/pipeline-nudge-auto.js +129 -129
- package/lib/pipeline-nudge.js +1623 -1623
- package/lib/pipeline-observations.js +181 -181
- package/lib/pipeline-observe.js +133 -133
- package/lib/pipeline-runner.js +361 -361
- package/lib/pipeline-runs.js +442 -442
- package/lib/pipeline.js +716 -716
- package/lib/placement.js +13 -13
- package/lib/pre-inject-gate.js +354 -354
- package/lib/profile-extract.js +164 -164
- package/lib/profile-facts.js +356 -356
- package/lib/provenance-http.js +251 -251
- package/lib/quality-filter.js +162 -162
- package/lib/recall-stats.js +190 -190
- package/lib/reclaim-http.js +150 -150
- package/lib/reranker.js +269 -269
- package/lib/runtime/adopt.js +176 -176
- package/lib/runtime/closure.js +172 -172
- package/lib/runtime/download.js +247 -247
- package/lib/runtime/layout.js +265 -265
- package/lib/runtime/loader.js +252 -252
- package/lib/runtime/model-fetch.js +361 -361
- package/lib/runtime/provision.js +237 -237
- package/lib/runtime/tarball.js +130 -130
- package/lib/runtime/verify.js +223 -223
- package/lib/safe-slice.js +18 -18
- package/lib/scope-state.js +277 -277
- package/lib/scope.js +187 -187
- package/lib/search/adaptive.js +60 -60
- package/lib/search/bm25.js +96 -96
- package/lib/semantic.js +572 -572
- package/lib/sensitive-scan.js +209 -209
- package/lib/serve.js +228 -228
- package/lib/service.js +3268 -3268
- package/lib/session-gate.js +91 -91
- package/lib/session-usage.js +350 -350
- package/lib/settings.js +827 -817
- package/lib/share-http.js +281 -281
- package/lib/share-import.js +234 -234
- package/lib/share.js +335 -335
- package/lib/store.js +5895 -5895
- package/lib/summarize.js +1388 -1343
- package/lib/tools.js +1370 -1356
- package/lib/user-settings.js +230 -230
- package/lib/vector-index.js +180 -180
- package/lib/version-check.js +122 -122
- package/lib/wiki/claims.js +267 -267
- package/lib/wiki/cli-hint.js +90 -90
- package/lib/wiki/inject.js +164 -164
- package/lib/wiki/partition.js +200 -200
- package/lib/wiki/plan.js +214 -214
- package/lib/wiki/queries.js +640 -640
- package/lib/wiki/render.js +261 -261
- package/lib/wiki/scan.js +371 -371
- package/lib/wiki/t0.js +500 -500
- package/lib/wiki/t1-jobs.js +232 -232
- package/lib/wiki/t1.js +609 -609
- package/lib/write-admission.js +404 -404
- package/locale/en.json +6 -6
- package/locale/zh.json +6 -6
- package/package.json +1 -1
- package/runtime-manifest.json +526 -526
- package/scripts/benchmark-embed.js +201 -201
- package/scripts/benchmark-recall.js +249 -249
- package/scripts/benchmark-rerank.js +166 -166
- package/scripts/build-runtime-manifest.mjs +224 -224
- package/scripts/changelog-prep.mjs +91 -91
- package/scripts/check-sync.js +44 -44
- package/scripts/e2e-dsh.js +220 -220
- package/scripts/mneme-runtime.mjs +277 -277
- package/scripts/publish-gate.mjs +197 -197
- package/scripts/release-docs.mjs +210 -210
- package/scripts/release-notes.mjs +164 -164
- package/scripts/stress-dsh.js +255 -255
- package/scripts/sync-lib.js +47 -47
- package/scripts/sync-test-badge.mjs +69 -69
- package/scripts/test-count-sync.mjs +48 -48
- package/src/api-standalone.js +1007 -1007
- package/src/api.js +3600 -3600
- package/src/attachment-text.js +122 -122
- package/src/bootstrap-http.js +105 -105
- package/src/bootstrap.js +215 -215
- package/src/candidates.js +454 -454
- package/src/capabilities.js +819 -819
- package/src/choro-command.js +111 -111
- package/src/commands.js +95 -95
- package/src/config-keys.js +144 -144
- package/src/config.js +1361 -1344
- package/src/content-hash.js +54 -54
- package/src/continuity.js +258 -258
- package/src/date-hint.js +32 -0
- package/src/diary/auto.js +190 -190
- package/src/diary/compose.js +440 -440
- package/src/diary/dir.js +59 -59
- package/src/diary/export.js +97 -97
- package/src/diary/index.js +26 -26
- package/src/diary/library.js +189 -189
- package/src/diary/qq-inbox.js +142 -142
- package/src/diary/rollover.js +61 -61
- package/src/diary/session-provider.js +122 -122
- package/src/document-index.js +94 -94
- package/src/document.js +322 -322
- package/src/dream/clustering.js +123 -123
- package/src/dream/decisions.js +945 -945
- package/src/dream/narratives.js +44 -44
- package/src/dream/sleep.js +2027 -2027
- package/src/dream/triage.js +139 -139
- package/src/dream.js +1879 -1879
- package/src/dsh-go/host.js +162 -162
- package/src/dsh-go/panel.js +2091 -2091
- package/src/dsh-go/text.js +342 -342
- package/src/dsh-go/views/common.js +68 -68
- package/src/dsh-go/views/entity.js +48 -48
- package/src/dsh-go/views/memory.js +198 -198
- package/src/dsh-go/views/root.js +30 -30
- package/src/dsh-go/views/settings.js +398 -398
- package/src/dsh-go/views/status.js +51 -51
- package/src/embedding.js +342 -342
- package/src/entities/extractor.js +285 -285
- package/src/fence.js +24 -24
- package/src/graph/anchoring.js +184 -184
- package/src/guide.js +74 -74
- package/src/heat.js +144 -144
- package/src/hot-memory.js +119 -119
- package/src/image-describe.js +158 -0
- package/src/index.js +852 -852
- package/src/inject.js +656 -651
- package/src/kb/index.js +309 -309
- package/src/kb/routes.js +219 -219
- package/src/kb/tools.js +148 -148
- package/src/lang.js +607 -595
- package/src/local-embedder.js +358 -358
- package/src/maintenance.js +162 -162
- package/src/memory-source.js +9 -9
- package/src/mirror.js +330 -330
- package/src/organize.js +456 -456
- package/src/peak-hours.js +172 -172
- package/src/pipeline-activities.js +178 -178
- package/src/pipeline-attribution.js +165 -165
- package/src/pipeline-command.js +279 -279
- package/src/pipeline-confirm.js +142 -142
- package/src/pipeline-exec.js +174 -174
- package/src/pipeline-nudge-auto.js +129 -129
- package/src/pipeline-nudge.js +1623 -1623
- package/src/pipeline-observations.js +181 -181
- package/src/pipeline-observe.js +133 -133
- package/src/pipeline-runner.js +361 -361
- package/src/pipeline-runs.js +442 -442
- package/src/pipeline.js +716 -716
- package/src/placement.js +13 -13
- package/src/pre-inject-gate.js +354 -354
- package/src/profile-extract.js +164 -164
- package/src/profile-facts.js +356 -356
- package/src/provenance-http.js +251 -251
- package/src/quality-filter.js +162 -162
- package/src/recall-stats.js +190 -190
- package/src/reclaim-http.js +150 -150
- package/src/reranker.js +269 -269
- package/src/runtime/adopt.js +176 -176
- package/src/runtime/closure.js +172 -172
- package/src/runtime/download.js +247 -247
- package/src/runtime/layout.js +265 -265
- package/src/runtime/loader.js +252 -252
- package/src/runtime/model-fetch.js +361 -361
- package/src/runtime/provision.js +237 -237
- package/src/runtime/tarball.js +130 -130
- package/src/runtime/verify.js +223 -223
- package/src/safe-slice.js +18 -18
- package/src/scope-state.js +277 -277
- package/src/scope.js +187 -187
- package/src/search/adaptive.js +60 -60
- package/src/search/bm25.js +96 -96
- package/src/semantic.js +572 -572
- package/src/sensitive-scan.js +209 -209
- package/src/serve.js +228 -228
- package/src/service.js +3268 -3268
- package/src/session-gate.js +91 -91
- package/src/session-usage.js +350 -350
- package/src/settings.js +827 -817
- package/src/share-http.js +281 -281
- package/src/share-import.js +234 -234
- package/src/share.js +335 -335
- package/src/store.js +5895 -5895
- package/src/summarize.js +1388 -1343
- package/src/tools.js +1370 -1356
- package/src/user-settings.js +230 -230
- package/src/vector-index.js +180 -180
- package/src/version-check.js +122 -122
- package/src/wiki/claims.js +267 -267
- package/src/wiki/cli-hint.js +90 -90
- package/src/wiki/inject.js +164 -164
- package/src/wiki/partition.js +200 -200
- package/src/wiki/plan.js +214 -214
- package/src/wiki/queries.js +640 -640
- package/src/wiki/render.js +261 -261
- package/src/wiki/scan.js +371 -371
- package/src/wiki/t0.js +500 -500
- package/src/wiki/t1-jobs.js +232 -232
- package/src/wiki/t1.js +609 -609
- package/src/write-admission.js +404 -404
- package/test/adaptive-inject-budget.test.js +85 -85
- package/test/api-hardening.test.js +173 -173
- package/test/api-nudge-route.test.js +259 -259
- package/test/api-routes-gaps.test.js +1092 -1092
- package/test/api.test.js +3007 -2998
- package/test/attachment-text.test.js +139 -139
- package/test/audit.test.js +448 -448
- package/test/baseline-persist.test.js +248 -248
- package/test/benchmark.test.js +94 -94
- package/test/bm25-corpus-cache.test.js +145 -145
- package/test/bootstrap.test.js +228 -228
- package/test/candidates.test.js +345 -345
- package/test/capabilities.test.js +577 -577
- package/test/capacity-warning.test.js +262 -262
- package/test/changelog-prep.test.js +125 -125
- package/test/choro-command.test.js +218 -218
- package/test/cli-enums.test.js +37 -37
- package/test/cli-flag-value.test.js +86 -86
- package/test/cli-receive.test.js +390 -390
- package/test/cli-share.test.js +437 -437
- package/test/client-share.test.js +225 -225
- package/test/client.test.js +4595 -4425
- package/test/clustering.test.js +100 -100
- package/test/commands.test.js +106 -106
- package/test/config-keys-routes.test.js +233 -233
- package/test/config.test.js +153 -153
- package/test/conflict-freeze.test.js +344 -344
- package/test/conflict-ledger-enforce.test.js +452 -452
- package/test/conflict-ledger.test.js +242 -242
- package/test/conflict-reviewed.test.js +79 -79
- package/test/content-hash.test.js +204 -204
- package/test/continuity.test.js +452 -452
- package/test/contrast.test.js +432 -432
- package/test/date-hint.test.js +156 -0
- package/test/dedupe-candidate-window.test.js +180 -180
- package/test/dependency-overrides.test.js +78 -78
- package/test/diary-library.test.js +187 -187
- package/test/diary.test.js +576 -576
- package/test/differentiate-feedback.test.js +248 -248
- package/test/distill-tx-embed-gap.test.js +110 -110
- package/test/docs-config-keys.test.js +65 -65
- package/test/document-dir.test.js +233 -233
- package/test/document.test.js +703 -703
- package/test/dream-archive-guardrail.test.js +116 -116
- package/test/dream-baseline-clamp.test.js +174 -174
- package/test/dream-candidate-hybrid.test.js +281 -281
- package/test/dream-coverage-degrade.test.js +86 -86
- package/test/dream-decision-evidence.test.js +314 -314
- package/test/dream-failure-backoff.test.js +173 -173
- package/test/dream-fence.test.js +207 -207
- package/test/dream-maxtokens-fallback.test.js +132 -132
- package/test/dream-merge-guardrail.test.js +219 -219
- package/test/dream-peak-hours.test.js +194 -194
- package/test/dream.test.js +1528 -1528
- package/test/dsh-go-host.test.js +134 -134
- package/test/dsh-go-panel-actions.test.js +854 -854
- package/test/dsh-go-panel.test.js +251 -251
- package/test/embed-ready-poll.test.js +94 -94
- package/test/embed-text-reasons.test.js +112 -112
- package/test/embedding-cache-f32.test.js +276 -276
- package/test/embedding-ready.test.js +226 -226
- package/test/entities.test.js +711 -711
- package/test/entity-recall.test.js +179 -179
- package/test/epistemic.test.js +325 -325
- package/test/failure-query.test.js +147 -147
- package/test/failure-retention.test.js +105 -105
- package/test/fnew-0112.test.js +311 -311
- package/test/fnew-03.test.js +422 -422
- package/test/forget-corrections.test.js +70 -70
- package/test/graph-anchoring.test.js +184 -184
- package/test/graph-api.test.js +333 -333
- package/test/graph-boosted-relations.test.js +189 -189
- package/test/graph-inject-budget.test.js +103 -103
- package/test/graph-multipath.test.js +269 -269
- package/test/graph-passive-confirm.test.js +91 -91
- package/test/graph-weight-evolution.test.js +104 -104
- package/test/heat.test.js +175 -175
- package/test/helpers/dream-mock.js +86 -86
- package/test/helpers/peer-worker.mjs +25 -25
- package/test/helpers/runtime-source.js +60 -60
- package/test/helpers/tar-builder.js +80 -80
- package/test/helpers/write-admission-samples.js +312 -312
- package/test/hot-memory.test.js +259 -259
- package/test/image-description.test.js +271 -0
- package/test/image-memory.test.js +170 -170
- package/test/inject-content-cap.test.js +91 -91
- package/test/inject-first-round-bm25.test.js +89 -89
- package/test/inject-heat.test.js +77 -77
- package/test/inject-parent-gate.test.js +134 -109
- package/test/inject-pin-explicit.test.js +300 -300
- package/test/inject-pin-vector-crowd.test.js +225 -225
- package/test/inject-pools.test.js +195 -195
- package/test/inject-recall.test.js +172 -172
- package/test/inject-rotation.test.js +161 -161
- package/test/inject.test.js +553 -553
- package/test/issue-617-step1-query-and-fusion.test.js +361 -361
- package/test/kb-api-wiring.test.js +128 -128
- package/test/kb-collections.test.js +272 -272
- package/test/kb-routes.test.js +379 -379
- package/test/lang.test.js +98 -98
- package/test/lastrunat-persist.test.js +101 -101
- package/test/lib-smoke.test.js +116 -116
- package/test/llm-audit.test.js +815 -815
- package/test/llm-message-source.test.js +207 -207
- package/test/local-embedder.test.js +292 -292
- package/test/maintenance.test.js +501 -501
- package/test/mcp-stdio.test.js +385 -385
- package/test/mint-source.test.js +311 -311
- package/test/mirror-dirty.test.js +424 -424
- package/test/mirror-edit-digest.test.js +187 -187
- package/test/mirror-generation.test.js +499 -499
- package/test/mirror-types.test.js +208 -208
- package/test/mirror.test.js +249 -249
- package/test/narrative-bars.test.js +162 -162
- package/test/organize-tool.test.js +232 -232
- package/test/organize.test.js +418 -418
- package/test/panel-chart-scale.test.js +90 -90
- package/test/panel-tokens.test.js +245 -245
- package/test/peak-hours-observability.test.js +70 -70
- package/test/peer-blockers.test.js +193 -193
- package/test/persona-addressing.test.js +83 -83
- package/test/persona-api.test.js +172 -172
- package/test/persona-by-session.test.js +568 -568
- package/test/persona-flag.test.js +36 -36
- package/test/persona-inject.test.js +145 -145
- package/test/persona-settings.test.js +198 -198
- package/test/pipeline-activities.test.js +155 -155
- package/test/pipeline-attribution.test.js +175 -175
- package/test/pipeline-command.test.js +243 -243
- package/test/pipeline-confirm.test.js +148 -148
- package/test/pipeline-exec.test.js +172 -172
- package/test/pipeline-nudge-auto.test.js +248 -248
- package/test/pipeline-nudge.test.js +1278 -1278
- package/test/pipeline-observations.test.js +157 -157
- package/test/pipeline-observe.test.js +127 -127
- package/test/pipeline-r1r2.test.js +278 -278
- package/test/pipeline-runner.test.js +468 -468
- package/test/pipeline-runs.test.js +767 -767
- package/test/pipeline.test.js +636 -636
- package/test/placement.test.js +36 -36
- package/test/policy-epoch.test.js +259 -259
- package/test/pre-inject-gate.test.js +444 -444
- package/test/profile-extract.test.js +269 -269
- package/test/profile-facts.test.js +492 -492
- package/test/prompt-stance.test.js +128 -128
- package/test/provenance-http.test.js +367 -367
- package/test/publish-gate.test.js +272 -272
- package/test/quality-backfill.test.js +290 -290
- package/test/quality-filter.test.js +295 -295
- package/test/readme-claims.test.js +113 -113
- package/test/reasoning-effort.test.js +752 -752
- package/test/recall-anti-update.test.js +82 -82
- package/test/recall-evals.test.js +253 -253
- package/test/recall-layer.test.js +326 -326
- package/test/recall-retention.test.js +128 -128
- package/test/recall-stats.test.js +274 -274
- package/test/receipt-chain.test.js +451 -451
- package/test/reclaim-http.test.js +209 -209
- package/test/reflection-update-gate.test.js +129 -129
- package/test/reflection.test.js +226 -226
- package/test/reindex-backfill.test.js +475 -475
- package/test/reindex-scheduler.test.js +369 -369
- package/test/release-docs.test.js +156 -156
- package/test/release-notes.test.js +265 -265
- package/test/release-workflow.test.js +93 -93
- package/test/reranker-single-logit.test.js +73 -73
- package/test/reranker.test.js +197 -197
- package/test/resident-status.test.js +83 -83
- package/test/route-policy.test.js +113 -113
- package/test/runtime-adopt.test.js +206 -206
- package/test/runtime-cli.test.js +89 -89
- package/test/runtime-closure.test.js +193 -193
- package/test/runtime-download.test.js +302 -302
- package/test/runtime-layout.test.js +163 -163
- package/test/runtime-loader.test.js +255 -255
- package/test/runtime-manifest.test.js +159 -159
- package/test/runtime-model-fetch.test.js +348 -348
- package/test/runtime-provision.test.js +101 -101
- package/test/runtime-tarball.test.js +81 -81
- package/test/runtime-verify.test.js +319 -319
- package/test/scope-candidates.test.js +491 -491
- package/test/scope-distill-stamp.test.js +134 -134
- package/test/scope-explicit.test.js +452 -452
- package/test/scope-inject-soft.test.js +106 -106
- package/test/scope-optional-service.test.js +110 -110
- package/test/scope-retrieval.test.js +197 -197
- package/test/scope-state.test.js +610 -610
- package/test/scope-storage.test.js +255 -255
- package/test/scope-strict.test.js +367 -367
- package/test/search-fusion.test.js +90 -90
- package/test/security-doc.test.js +188 -188
- package/test/semantic.test.js +124 -124
- package/test/sensitive-scan.test.js +251 -251
- package/test/serve-bin.test.js +121 -121
- package/test/serve.test.js +258 -258
- package/test/service-search.test.js +174 -174
- package/test/service.test.js +410 -410
- package/test/session-gate.test.js +264 -264
- package/test/session-usage.test.js +317 -317
- package/test/settings.test.js +273 -273
- package/test/share-foreign-rows.test.js +144 -144
- package/test/share-http.test.js +144 -144
- package/test/share-import.test.js +577 -577
- package/test/share-imported-guard.test.js +311 -311
- package/test/share-imported-identity.test.js +111 -111
- package/test/share-pack.test.js +511 -511
- package/test/share-receive-http.test.js +756 -756
- package/test/share-routes.test.js +337 -337
- package/test/share-store.test.js +133 -133
- package/test/share-validate.test.js +101 -101
- package/test/skip-invalid-restore.test.js +262 -262
- package/test/sleep-action-set.test.js +441 -441
- package/test/sleep-catchup.test.js +315 -315
- package/test/sleep-failure-backoff.test.js +156 -156
- package/test/sleep-heat.test.js +174 -174
- package/test/sleep-scheduler-rearm.test.js +89 -89
- package/test/sleep.test.js +1455 -1455
- package/test/standalone-api.test.js +1344 -1344
- package/test/store-vector-cache.test.js +98 -98
- package/test/store-write-path-columns.test.js +191 -191
- package/test/store.test.js +744 -744
- package/test/stress.test.js +209 -209
- package/test/summarize-occurred-at.test.js +127 -127
- package/test/summarize-parse-diagnostics.test.js +75 -75
- package/test/summarize-reasoning-effort.test.js +189 -189
- package/test/summarize.test.js +2251 -2251
- package/test/supersede-state-tracking.test.js +394 -394
- package/test/test-count-sync.test.js +71 -71
- package/test/tool-state-ignore.test.js +65 -65
- package/test/tools-dto-schema-matrix.test.js +398 -398
- package/test/tools-memory-get-schema.test.js +65 -65
- package/test/tools.test.js +585 -585
- package/test/updated-at-semantics.test.js +113 -113
- package/test/user-settings-preview.test.js +127 -127
- package/test/user-settings-segments.test.js +527 -527
- package/test/vector-index.test.js +334 -334
- package/test/version-check.test.js +152 -152
- package/test/webServer-optional.test.js +82 -82
- package/test/wiki-api.test.js +734 -734
- package/test/wiki-claims.test.js +194 -194
- package/test/wiki-cli-hint.test.js +118 -118
- package/test/wiki-inject.test.js +265 -265
- package/test/wiki-partition.test.js +112 -112
- package/test/wiki-plan.test.js +80 -80
- package/test/wiki-provenance.test.js +202 -202
- package/test/wiki-queries.test.js +398 -398
- package/test/wiki-scan.test.js +208 -208
- package/test/wiki-store.test.js +190 -190
- package/test/wiki-t0.test.js +456 -456
- package/test/wiki-t1-jobs.test.js +125 -125
- package/test/wiki-t1.test.js +460 -460
- package/test/wiki-tools.test.js +176 -176
- package/test/workflow-guards.test.js +155 -155
- package/test/write-admission.test.js +668 -668
package/lib/embedding.js
CHANGED
|
@@ -1,342 +1,342 @@
|
|
|
1
|
-
// OpenAI-compatible embedding client for vector search. DSH's LLM service is
|
|
2
|
-
// chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
|
|
3
|
-
// Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
|
|
4
|
-
// proxy) and any provider exposing the standard embeddings API.
|
|
5
|
-
const DEFAULT_TIMEOUT_MS = 15000;
|
|
6
|
-
|
|
7
|
-
/** djb2 — stable, fast fingerprint for a provider/model string. Mirrors the
|
|
8
|
-
* hash used by the local embedders so all backends share one fingerprint
|
|
9
|
-
* format (model#hex) for vector_meta consistency checks. */
|
|
10
|
-
function hashString(s) {
|
|
11
|
-
let h = 5381;
|
|
12
|
-
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
13
|
-
return h.toString(16);
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
/** Full provider+model fingerprint used for index-consistency checks. */
|
|
17
|
-
function modelHashOf(model) {
|
|
18
|
-
return `${model}#${hashString(model)}`;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
|
|
22
|
-
function embeddingsUrl(baseUrl) {
|
|
23
|
-
const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
|
|
24
|
-
if (!base) return "";
|
|
25
|
-
// Accept both "https://host/v1" and a full path ending in /embeddings.
|
|
26
|
-
if (/\/embeddings$/i.test(base)) return base;
|
|
27
|
-
return `${base}/embeddings`;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Call the embeddings API for one text. Resolves to a Float64 array, or null
|
|
32
|
-
* when the provider is not configured, the call fails, or the response is
|
|
33
|
-
* unusable. Never throws: failures degrade to keyword search.
|
|
34
|
-
*
|
|
35
|
-
* The plain null return cannot tell the caller WHY — "model not loaded",
|
|
36
|
-
* "endpoint 404", "timeout" and "input rejected" all look identical, so the
|
|
37
|
-
* receipt ledger used to flatten them into one
|
|
38
|
-
* `embedding-request-returned-no-vector`. Use `embedTextDetailed` when the
|
|
39
|
-
* reason matters; this wrapper keeps the old contract for existing callers.
|
|
40
|
-
*/
|
|
41
|
-
export async function embedText(cfg, text) {
|
|
42
|
-
const out = await embedTextDetailed(cfg, text);
|
|
43
|
-
return out.vector;
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Same call, but the failure mode is distinguishable: returns
|
|
48
|
-
* { vector: number[]|null, reason: null|string } where reason is one of
|
|
49
|
-
* config-incomplete | request-failed:<err> | http-<status> |
|
|
50
|
-
* response-not-json | response-no-vector | empty-text
|
|
51
|
-
* Never throws. `reason: null` means success.
|
|
52
|
-
*/
|
|
53
|
-
export async function embedTextDetailed({ baseUrl, apiKey, model }, text) {
|
|
54
|
-
const url = embeddingsUrl(baseUrl);
|
|
55
|
-
if (!url || !apiKey || !model || !text) {
|
|
56
|
-
// 空输入不是端点故障:区分开,否则「text 为空」会被误报成网络问题。
|
|
57
|
-
const missing = !url ? "baseUrl" : !apiKey ? "apiKey" : !model ? "model" : "text";
|
|
58
|
-
return { vector: null, reason: `config-incomplete:${missing}` };
|
|
59
|
-
}
|
|
60
|
-
let res;
|
|
61
|
-
try {
|
|
62
|
-
res = await fetch(url, {
|
|
63
|
-
method: "POST",
|
|
64
|
-
headers: {
|
|
65
|
-
"Content-Type": "application/json",
|
|
66
|
-
"Authorization": `Bearer ${apiKey}`
|
|
67
|
-
},
|
|
68
|
-
body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
|
|
69
|
-
signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
|
|
70
|
-
});
|
|
71
|
-
} catch (error) {
|
|
72
|
-
// 超时是 AbortSignal 抛的 TimeoutError;网络层错误(DNS/拒连)是 TypeError。
|
|
73
|
-
// 两者都走 request-failed,但带上类型,日志里能分清「慢」和「不通」。
|
|
74
|
-
const kind = error?.name === "TimeoutError" || error?.name === "AbortError" ? "timeout" : "network";
|
|
75
|
-
return { vector: null, reason: `request-failed:${kind}:${String(error?.message ?? error).slice(0, 200)}` };
|
|
76
|
-
}
|
|
77
|
-
if (!res.ok) {
|
|
78
|
-
// 状态码本身就是最有用的诊断:401/403 是钥匙、404 是路径、429 是限流、
|
|
79
|
-
// 5xx 是服务端。文本截断且不含请求体,日志安全。
|
|
80
|
-
let detail = "";
|
|
81
|
-
try {
|
|
82
|
-
const t = await res.text();
|
|
83
|
-
detail = String(t).slice(0, 200);
|
|
84
|
-
} catch { /* body unreadable, status alone is already informative */ }
|
|
85
|
-
return { vector: null, reason: `http-${res.status}${detail ? `:${detail}` : ""}` };
|
|
86
|
-
}
|
|
87
|
-
let body;
|
|
88
|
-
try {
|
|
89
|
-
body = await res.json();
|
|
90
|
-
} catch {
|
|
91
|
-
return { vector: null, reason: "response-not-json" };
|
|
92
|
-
}
|
|
93
|
-
const vec = body?.data?.[0]?.embedding;
|
|
94
|
-
if (!Array.isArray(vec) || !vec.length) {
|
|
95
|
-
return { vector: null, reason: "response-no-vector" };
|
|
96
|
-
}
|
|
97
|
-
return { vector: Array.from(vec), reason: null };
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Embedder bound to the current settings + store: on each write it re-embeds
|
|
102
|
-
* the row's title+content and stores the vector. Failures are swallowed so a
|
|
103
|
-
* flaky embedding endpoint never breaks memory writes.
|
|
104
|
-
*
|
|
105
|
-
* `vectorIndex` (optional) is the vector_meta fingerprint holder: after any
|
|
106
|
-
* successful embed the model that produced the vectors is recorded, so the
|
|
107
|
-
* index can detect drift and the auto-reindex backfill knows what to rebuild.
|
|
108
|
-
*/
|
|
109
|
-
/**
|
|
110
|
-
* Bounded receipt ring for embedding attempts.
|
|
111
|
-
*
|
|
112
|
-
* Why a ring instead of llm_audit_logs: a full backfill is thousands of
|
|
113
|
-
* attempts, and one audit row each would bury the LLM calls that table exists
|
|
114
|
-
* to trace. What the user actually needs in order to answer "is the vector
|
|
115
|
-
* layer silently dead right now?" is the recent window plus running totals —
|
|
116
|
-
* which costs nothing and cannot flood.
|
|
117
|
-
*/
|
|
118
|
-
const RECEIPT_LIMIT = 200;
|
|
119
|
-
|
|
120
|
-
function createReceiptLedger() {
|
|
121
|
-
const recent = [];
|
|
122
|
-
const totals = { ok: 0, failed: 0, skipped: 0 };
|
|
123
|
-
let lastOkAt = null;
|
|
124
|
-
let lastFailAt = null;
|
|
125
|
-
let lastFailReason = null;
|
|
126
|
-
return {
|
|
127
|
-
record(entry) {
|
|
128
|
-
recent.push(entry);
|
|
129
|
-
if (recent.length > RECEIPT_LIMIT) recent.shift();
|
|
130
|
-
if (entry.status in totals) totals[entry.status]++;
|
|
131
|
-
if (entry.status === "ok") lastOkAt = entry.at;
|
|
132
|
-
else if (entry.status === "failed") {
|
|
133
|
-
lastFailAt = entry.at;
|
|
134
|
-
lastFailReason = entry.reason ?? null;
|
|
135
|
-
}
|
|
136
|
-
},
|
|
137
|
-
snapshot() {
|
|
138
|
-
return {
|
|
139
|
-
totals: { ...totals },
|
|
140
|
-
lastOkAt,
|
|
141
|
-
lastFailAt,
|
|
142
|
-
lastFailReason,
|
|
143
|
-
recent: recent.slice(-RECEIPT_LIMIT)
|
|
144
|
-
};
|
|
145
|
-
}
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
export function createEmbedder({ store, settings, logger, vectorIndex }) {
|
|
150
|
-
// Dimension of the most recent successful embed, exposed for fingerprinting.
|
|
151
|
-
let _dimension = 0;
|
|
152
|
-
// Per-write embed receipts (see createReceiptLedger above).
|
|
153
|
-
const ledger = createReceiptLedger();
|
|
154
|
-
|
|
155
|
-
/** Record the producing model fingerprint in vector_meta (best-effort). */
|
|
156
|
-
function markModel(cfg, dimension) {
|
|
157
|
-
if (!vectorIndex || typeof vectorIndex.markModel !== "function") return;
|
|
158
|
-
try {
|
|
159
|
-
vectorIndex.markModel(modelHashOf(cfg.model), dimension);
|
|
160
|
-
} catch { /* metadata write is best-effort */ }
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
// issue #135: 「配置是否齐备」的唯一判据 —— embed()/embedFor() 的守卫与下面
|
|
164
|
-
// ready/configured 两个 getter 共用,避免以后改一处漏一处。
|
|
165
|
-
const configComplete = (cfg) => !!(cfg?.enabled && cfg.baseUrl && cfg.apiKey && cfg.model);
|
|
166
|
-
|
|
167
|
-
/**
|
|
168
|
-
* Embed one memory and store the vector. Never throws — a flaky endpoint
|
|
169
|
-
* must not break memory writes — but it now REPORTS what happened instead
|
|
170
|
-
* of returning undefined on both success and failure. Callers that ignore
|
|
171
|
-
* the result are unaffected; callers that care get a receipt.
|
|
172
|
-
*
|
|
173
|
-
* @returns {Promise<{status:"ok"|"failed"|"skipped", id:string,
|
|
174
|
-
* at:string, reason?:string, dim?:number, model?:string}>}
|
|
175
|
-
*/
|
|
176
|
-
async function embedFor(id, title, content) {
|
|
177
|
-
const at = new Date().toISOString();
|
|
178
|
-
const cfg = settings.getVectorConfig();
|
|
179
|
-
if (!configComplete(cfg)) {
|
|
180
|
-
const entry = { status: "skipped", id, at, reason: "vector-config-incomplete" };
|
|
181
|
-
ledger.record(entry);
|
|
182
|
-
return entry;
|
|
183
|
-
}
|
|
184
|
-
const text = [title, content].filter(Boolean).join("\n");
|
|
185
|
-
const detailed = await embedTextDetailed(cfg, text);
|
|
186
|
-
if (detailed.reason) {
|
|
187
|
-
// 回执带上可区分的失败模式(http-404 / request-failed:timeout /
|
|
188
|
-
// config-incomplete:model / response-no-vector ...),不再把「模型没配」
|
|
189
|
-
// 「请求挂了」「响应没有向量」压成同一个词。状态页与日志按 reason 分诊。
|
|
190
|
-
const entry = { status: "failed", id, at, reason: detailed.reason };
|
|
191
|
-
ledger.record(entry);
|
|
192
|
-
logger?.warn?.(`[dsh-mneme] embed memory ${id} failed: ${detailed.reason}`);
|
|
193
|
-
return entry;
|
|
194
|
-
}
|
|
195
|
-
const vector = detailed.vector;
|
|
196
|
-
const model = modelHashOf(cfg.model);
|
|
197
|
-
store.setEmbedding(id, vector, model);
|
|
198
|
-
_dimension = vector.length;
|
|
199
|
-
// Bug3: record which model produced the current vectors so the index can
|
|
200
|
-
// detect drift and skip a redundant backfill when nothing changed.
|
|
201
|
-
markModel(cfg, vector.length);
|
|
202
|
-
const entry = { status: "ok", id, at, dim: vector.length, model };
|
|
203
|
-
ledger.record(entry);
|
|
204
|
-
logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
|
|
205
|
-
return entry;
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
return {
|
|
209
|
-
// Display name for /semantic: a literal's constructor.name is "Object",
|
|
210
|
-
// which the status card would render verbatim.
|
|
211
|
-
name: "OpenAI",
|
|
212
|
-
/**
|
|
213
|
-
* issue #135: 这个 embedder 没有异步初始化,`"ready" in embedder` 的通用
|
|
214
|
-
* 分支会把它恒判为可用 —— 而 vector-config 四项有缺时它一条都嵌不出来。
|
|
215
|
-
* 对外报真实可用性,让 /semantic 的 ready 不再是「绿色假阳性」。
|
|
216
|
-
* 用 getter 而非快照字段:设置面板改完配置无需重启即生效(与 embed() 同源读取)。
|
|
217
|
-
*/
|
|
218
|
-
get ready() {
|
|
219
|
-
return configComplete(settings.getVectorConfig());
|
|
220
|
-
},
|
|
221
|
-
/**
|
|
222
|
-
* 与 ready 同源,分开两个名字是给 /semantic 用:local/ollama 的
|
|
223
|
-
* ready=false 表示「还在初始化」,而这里的 false 表示「根本没配」。
|
|
224
|
-
*/
|
|
225
|
-
get configured() {
|
|
226
|
-
return configComplete(settings.getVectorConfig());
|
|
227
|
-
},
|
|
228
|
-
/**
|
|
229
|
-
* Re-embed a memory after any write. Still non-blocking — the caller does
|
|
230
|
-
* not have to await it — but it now RETURNS the receipt promise so a
|
|
231
|
-
* caller that wants to know can. Previously this returned undefined and
|
|
232
|
-
* swallowed the rejection, so "did this write get a vector?" had no answer
|
|
233
|
-
* anywhere in the process. Swallowing still applies to the rejection
|
|
234
|
-
* itself: a vector failure must never break the memory write.
|
|
235
|
-
*/
|
|
236
|
-
schedule(memory) {
|
|
237
|
-
if (!memory?.id) {
|
|
238
|
-
return Promise.resolve({ status: "skipped", id: null, at: new Date().toISOString(), reason: "no-id" });
|
|
239
|
-
}
|
|
240
|
-
return embedFor(memory.id, memory.title, memory.content).catch((error) => ({
|
|
241
|
-
status: "failed", id: memory.id, at: new Date().toISOString(), reason: String(error)
|
|
242
|
-
}));
|
|
243
|
-
},
|
|
244
|
-
|
|
245
|
-
/** Recent embed receipts + running totals (see createReceiptLedger). */
|
|
246
|
-
embedReceipts() {
|
|
247
|
-
return ledger.snapshot();
|
|
248
|
-
},
|
|
249
|
-
|
|
250
|
-
/** Embed one text and return its vector (null on failure/disabled). */
|
|
251
|
-
async embed(query) {
|
|
252
|
-
const cfg = settings.getVectorConfig();
|
|
253
|
-
if (!configComplete(cfg)) return null;
|
|
254
|
-
const vector = await embedText(cfg, query);
|
|
255
|
-
if (vector) _dimension = vector.length;
|
|
256
|
-
return vector;
|
|
257
|
-
},
|
|
258
|
-
|
|
259
|
-
// Bug1: single-text adapter. Local/ollama embedders expose embedSingle
|
|
260
|
-
// natively; the legacy OpenAI-compatible client only has embed. This
|
|
261
|
-
// adapter unifies the interface so vector-index rebuildIndex (which guards
|
|
262
|
-
// on `typeof embedder.embedSingle === "function"`) accepts this embedder.
|
|
263
|
-
async embedSingle(text) {
|
|
264
|
-
if (typeof this.embed === "function") return this.embed(text);
|
|
265
|
-
return null;
|
|
266
|
-
},
|
|
267
|
-
|
|
268
|
-
// Same as embedSingle, but keeps the reason a vector did not come back.
|
|
269
|
-
// embedSingle() drops it on purpose — its callers only care whether a
|
|
270
|
-
// vector exists — but the boot backfill has to tell "this row is broken"
|
|
271
|
-
// from "the provider was unreachable", and a bare null erases that
|
|
272
|
-
// distinction. Callers feature-detect this method and fall back to
|
|
273
|
-
// embedSingle() so older adapters keep working unchanged.
|
|
274
|
-
async embedSingleDetailed(text) {
|
|
275
|
-
const cfg = settings.getVectorConfig();
|
|
276
|
-
if (!configComplete(cfg)) return { vector: null, reason: "config-incomplete" };
|
|
277
|
-
const out = await embedTextDetailed(cfg, text);
|
|
278
|
-
if (out.vector) _dimension = out.vector.length;
|
|
279
|
-
return out;
|
|
280
|
-
},
|
|
281
|
-
|
|
282
|
-
/** Model fingerprint (model#hex), or undefined when not configured. */
|
|
283
|
-
get modelHash() {
|
|
284
|
-
const cfg = settings.getVectorConfig();
|
|
285
|
-
return cfg?.enabled && cfg.model ? modelHashOf(cfg.model) : undefined;
|
|
286
|
-
},
|
|
287
|
-
|
|
288
|
-
/** Dimension of the last successful embed (0 when never embedded). */
|
|
289
|
-
get dimension() {
|
|
290
|
-
return _dimension || undefined;
|
|
291
|
-
},
|
|
292
|
-
|
|
293
|
-
/**
|
|
294
|
-
* Batch re-index rows whose vector is missing OR came from another model.
|
|
295
|
-
* Uses needsEmbeddingFor (not needsEmbedding) so a model switch actually
|
|
296
|
-
* re-embeds the old vectors instead of leaving them eligible for "has a
|
|
297
|
-
* vector" forever while they poison similarity ranking.
|
|
298
|
-
*/
|
|
299
|
-
async reindexMissing(limit = 50) {
|
|
300
|
-
const cfg = settings.getVectorConfig();
|
|
301
|
-
if (!cfg?.enabled) return { indexed: 0, failed: 0, skipped: 0 };
|
|
302
|
-
const model = modelHashOf(cfg.model);
|
|
303
|
-
// Fall back to the plain selector for stores that predate the
|
|
304
|
-
// provenance column (older hosts, test doubles).
|
|
305
|
-
const rows = typeof store.needsEmbeddingFor === "function"
|
|
306
|
-
? store.needsEmbeddingFor(model, limit)
|
|
307
|
-
: store.needsEmbedding(limit);
|
|
308
|
-
let indexed = 0;
|
|
309
|
-
let failed = 0;
|
|
310
|
-
for (const row of rows) {
|
|
311
|
-
try {
|
|
312
|
-
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
313
|
-
// Prefer the reason-preserving call for the same reason the boot
|
|
314
|
-
// backfill does: a bare null cannot tell "this row is unembeddable"
|
|
315
|
-
// from "the provider was unreachable", and the receipt ledger files
|
|
316
|
-
// both under one indistinguishable entry.
|
|
317
|
-
const detailed = typeof this.embedSingleDetailed === "function"
|
|
318
|
-
? await this.embedSingleDetailed(text)
|
|
319
|
-
: null;
|
|
320
|
-
const vector = detailed ? detailed.vector : await embedText(cfg, text);
|
|
321
|
-
if (vector?.length) {
|
|
322
|
-
store.setEmbedding(row.id, vector, model);
|
|
323
|
-
_dimension = vector.length;
|
|
324
|
-
indexed++;
|
|
325
|
-
} else {
|
|
326
|
-
failed++;
|
|
327
|
-
}
|
|
328
|
-
} catch (error) {
|
|
329
|
-
// A throw used to abort the whole loop and surface as a rejected
|
|
330
|
-
// promise from the API route; one bad row must not stop the rest.
|
|
331
|
-
failed++;
|
|
332
|
-
logger?.warn?.(`[dsh-mneme] reindex embed failed for ${row.id}: ${String(error)}`);
|
|
333
|
-
}
|
|
334
|
-
}
|
|
335
|
-
if (indexed > 0) markModel(cfg, _dimension || undefined);
|
|
336
|
-
// #604: `skipped: rows.length - indexed` folded "tried and got nothing"
|
|
337
|
-
// into "never touched". Every selected row is attempted here, so nothing
|
|
338
|
-
// is skipped; what failed is reported as failed.
|
|
339
|
-
return { indexed, failed, skipped: 0 };
|
|
340
|
-
}
|
|
341
|
-
};
|
|
342
|
-
}
|
|
1
|
+
// OpenAI-compatible embedding client for vector search. DSH's LLM service is
|
|
2
|
+
// chat-only, so dsh-mneme calls an external `/embeddings` endpoint itself.
|
|
3
|
+
// Works with OpenAI, SiliconFlow, Zhipu, local Ollama (via OpenAI-compatible
|
|
4
|
+
// proxy) and any provider exposing the standard embeddings API.
|
|
5
|
+
const DEFAULT_TIMEOUT_MS = 15000;
|
|
6
|
+
|
|
7
|
+
/** djb2 — stable, fast fingerprint for a provider/model string. Mirrors the
|
|
8
|
+
* hash used by the local embedders so all backends share one fingerprint
|
|
9
|
+
* format (model#hex) for vector_meta consistency checks. */
|
|
10
|
+
function hashString(s) {
|
|
11
|
+
let h = 5381;
|
|
12
|
+
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) >>> 0;
|
|
13
|
+
return h.toString(16);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Full provider+model fingerprint used for index-consistency checks. */
|
|
17
|
+
function modelHashOf(model) {
|
|
18
|
+
return `${model}#${hashString(model)}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Normalize a configured baseUrl into the full embeddings endpoint URL. */
|
|
22
|
+
function embeddingsUrl(baseUrl) {
|
|
23
|
+
const base = String(baseUrl ?? "").trim().replace(/\/+$/, "");
|
|
24
|
+
if (!base) return "";
|
|
25
|
+
// Accept both "https://host/v1" and a full path ending in /embeddings.
|
|
26
|
+
if (/\/embeddings$/i.test(base)) return base;
|
|
27
|
+
return `${base}/embeddings`;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Call the embeddings API for one text. Resolves to a Float64 array, or null
|
|
32
|
+
* when the provider is not configured, the call fails, or the response is
|
|
33
|
+
* unusable. Never throws: failures degrade to keyword search.
|
|
34
|
+
*
|
|
35
|
+
* The plain null return cannot tell the caller WHY — "model not loaded",
|
|
36
|
+
* "endpoint 404", "timeout" and "input rejected" all look identical, so the
|
|
37
|
+
* receipt ledger used to flatten them into one
|
|
38
|
+
* `embedding-request-returned-no-vector`. Use `embedTextDetailed` when the
|
|
39
|
+
* reason matters; this wrapper keeps the old contract for existing callers.
|
|
40
|
+
*/
|
|
41
|
+
export async function embedText(cfg, text) {
|
|
42
|
+
const out = await embedTextDetailed(cfg, text);
|
|
43
|
+
return out.vector;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Same call, but the failure mode is distinguishable: returns
|
|
48
|
+
* { vector: number[]|null, reason: null|string } where reason is one of
|
|
49
|
+
* config-incomplete | request-failed:<err> | http-<status> |
|
|
50
|
+
* response-not-json | response-no-vector | empty-text
|
|
51
|
+
* Never throws. `reason: null` means success.
|
|
52
|
+
*/
|
|
53
|
+
export async function embedTextDetailed({ baseUrl, apiKey, model }, text) {
|
|
54
|
+
const url = embeddingsUrl(baseUrl);
|
|
55
|
+
if (!url || !apiKey || !model || !text) {
|
|
56
|
+
// 空输入不是端点故障:区分开,否则「text 为空」会被误报成网络问题。
|
|
57
|
+
const missing = !url ? "baseUrl" : !apiKey ? "apiKey" : !model ? "model" : "text";
|
|
58
|
+
return { vector: null, reason: `config-incomplete:${missing}` };
|
|
59
|
+
}
|
|
60
|
+
let res;
|
|
61
|
+
try {
|
|
62
|
+
res = await fetch(url, {
|
|
63
|
+
method: "POST",
|
|
64
|
+
headers: {
|
|
65
|
+
"Content-Type": "application/json",
|
|
66
|
+
"Authorization": `Bearer ${apiKey}`
|
|
67
|
+
},
|
|
68
|
+
body: JSON.stringify({ model, input: String(text).slice(0, 8000) }),
|
|
69
|
+
signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS)
|
|
70
|
+
});
|
|
71
|
+
} catch (error) {
|
|
72
|
+
// 超时是 AbortSignal 抛的 TimeoutError;网络层错误(DNS/拒连)是 TypeError。
|
|
73
|
+
// 两者都走 request-failed,但带上类型,日志里能分清「慢」和「不通」。
|
|
74
|
+
const kind = error?.name === "TimeoutError" || error?.name === "AbortError" ? "timeout" : "network";
|
|
75
|
+
return { vector: null, reason: `request-failed:${kind}:${String(error?.message ?? error).slice(0, 200)}` };
|
|
76
|
+
}
|
|
77
|
+
if (!res.ok) {
|
|
78
|
+
// 状态码本身就是最有用的诊断:401/403 是钥匙、404 是路径、429 是限流、
|
|
79
|
+
// 5xx 是服务端。文本截断且不含请求体,日志安全。
|
|
80
|
+
let detail = "";
|
|
81
|
+
try {
|
|
82
|
+
const t = await res.text();
|
|
83
|
+
detail = String(t).slice(0, 200);
|
|
84
|
+
} catch { /* body unreadable, status alone is already informative */ }
|
|
85
|
+
return { vector: null, reason: `http-${res.status}${detail ? `:${detail}` : ""}` };
|
|
86
|
+
}
|
|
87
|
+
let body;
|
|
88
|
+
try {
|
|
89
|
+
body = await res.json();
|
|
90
|
+
} catch {
|
|
91
|
+
return { vector: null, reason: "response-not-json" };
|
|
92
|
+
}
|
|
93
|
+
const vec = body?.data?.[0]?.embedding;
|
|
94
|
+
if (!Array.isArray(vec) || !vec.length) {
|
|
95
|
+
return { vector: null, reason: "response-no-vector" };
|
|
96
|
+
}
|
|
97
|
+
return { vector: Array.from(vec), reason: null };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Embedder bound to the current settings + store: on each write it re-embeds
|
|
102
|
+
* the row's title+content and stores the vector. Failures are swallowed so a
|
|
103
|
+
* flaky embedding endpoint never breaks memory writes.
|
|
104
|
+
*
|
|
105
|
+
* `vectorIndex` (optional) is the vector_meta fingerprint holder: after any
|
|
106
|
+
* successful embed the model that produced the vectors is recorded, so the
|
|
107
|
+
* index can detect drift and the auto-reindex backfill knows what to rebuild.
|
|
108
|
+
*/
|
|
109
|
+
/**
|
|
110
|
+
* Bounded receipt ring for embedding attempts.
|
|
111
|
+
*
|
|
112
|
+
* Why a ring instead of llm_audit_logs: a full backfill is thousands of
|
|
113
|
+
* attempts, and one audit row each would bury the LLM calls that table exists
|
|
114
|
+
* to trace. What the user actually needs in order to answer "is the vector
|
|
115
|
+
* layer silently dead right now?" is the recent window plus running totals —
|
|
116
|
+
* which costs nothing and cannot flood.
|
|
117
|
+
*/
|
|
118
|
+
const RECEIPT_LIMIT = 200;
|
|
119
|
+
|
|
120
|
+
function createReceiptLedger() {
|
|
121
|
+
const recent = [];
|
|
122
|
+
const totals = { ok: 0, failed: 0, skipped: 0 };
|
|
123
|
+
let lastOkAt = null;
|
|
124
|
+
let lastFailAt = null;
|
|
125
|
+
let lastFailReason = null;
|
|
126
|
+
return {
|
|
127
|
+
record(entry) {
|
|
128
|
+
recent.push(entry);
|
|
129
|
+
if (recent.length > RECEIPT_LIMIT) recent.shift();
|
|
130
|
+
if (entry.status in totals) totals[entry.status]++;
|
|
131
|
+
if (entry.status === "ok") lastOkAt = entry.at;
|
|
132
|
+
else if (entry.status === "failed") {
|
|
133
|
+
lastFailAt = entry.at;
|
|
134
|
+
lastFailReason = entry.reason ?? null;
|
|
135
|
+
}
|
|
136
|
+
},
|
|
137
|
+
snapshot() {
|
|
138
|
+
return {
|
|
139
|
+
totals: { ...totals },
|
|
140
|
+
lastOkAt,
|
|
141
|
+
lastFailAt,
|
|
142
|
+
lastFailReason,
|
|
143
|
+
recent: recent.slice(-RECEIPT_LIMIT)
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function createEmbedder({ store, settings, logger, vectorIndex }) {
|
|
150
|
+
// Dimension of the most recent successful embed, exposed for fingerprinting.
|
|
151
|
+
let _dimension = 0;
|
|
152
|
+
// Per-write embed receipts (see createReceiptLedger above).
|
|
153
|
+
const ledger = createReceiptLedger();
|
|
154
|
+
|
|
155
|
+
/** Record the producing model fingerprint in vector_meta (best-effort). */
|
|
156
|
+
function markModel(cfg, dimension) {
|
|
157
|
+
if (!vectorIndex || typeof vectorIndex.markModel !== "function") return;
|
|
158
|
+
try {
|
|
159
|
+
vectorIndex.markModel(modelHashOf(cfg.model), dimension);
|
|
160
|
+
} catch { /* metadata write is best-effort */ }
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// issue #135: 「配置是否齐备」的唯一判据 —— embed()/embedFor() 的守卫与下面
|
|
164
|
+
// ready/configured 两个 getter 共用,避免以后改一处漏一处。
|
|
165
|
+
const configComplete = (cfg) => !!(cfg?.enabled && cfg.baseUrl && cfg.apiKey && cfg.model);
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Embed one memory and store the vector. Never throws — a flaky endpoint
|
|
169
|
+
* must not break memory writes — but it now REPORTS what happened instead
|
|
170
|
+
* of returning undefined on both success and failure. Callers that ignore
|
|
171
|
+
* the result are unaffected; callers that care get a receipt.
|
|
172
|
+
*
|
|
173
|
+
* @returns {Promise<{status:"ok"|"failed"|"skipped", id:string,
|
|
174
|
+
* at:string, reason?:string, dim?:number, model?:string}>}
|
|
175
|
+
*/
|
|
176
|
+
async function embedFor(id, title, content) {
|
|
177
|
+
const at = new Date().toISOString();
|
|
178
|
+
const cfg = settings.getVectorConfig();
|
|
179
|
+
if (!configComplete(cfg)) {
|
|
180
|
+
const entry = { status: "skipped", id, at, reason: "vector-config-incomplete" };
|
|
181
|
+
ledger.record(entry);
|
|
182
|
+
return entry;
|
|
183
|
+
}
|
|
184
|
+
const text = [title, content].filter(Boolean).join("\n");
|
|
185
|
+
const detailed = await embedTextDetailed(cfg, text);
|
|
186
|
+
if (detailed.reason) {
|
|
187
|
+
// 回执带上可区分的失败模式(http-404 / request-failed:timeout /
|
|
188
|
+
// config-incomplete:model / response-no-vector ...),不再把「模型没配」
|
|
189
|
+
// 「请求挂了」「响应没有向量」压成同一个词。状态页与日志按 reason 分诊。
|
|
190
|
+
const entry = { status: "failed", id, at, reason: detailed.reason };
|
|
191
|
+
ledger.record(entry);
|
|
192
|
+
logger?.warn?.(`[dsh-mneme] embed memory ${id} failed: ${detailed.reason}`);
|
|
193
|
+
return entry;
|
|
194
|
+
}
|
|
195
|
+
const vector = detailed.vector;
|
|
196
|
+
const model = modelHashOf(cfg.model);
|
|
197
|
+
store.setEmbedding(id, vector, model);
|
|
198
|
+
_dimension = vector.length;
|
|
199
|
+
// Bug3: record which model produced the current vectors so the index can
|
|
200
|
+
// detect drift and skip a redundant backfill when nothing changed.
|
|
201
|
+
markModel(cfg, vector.length);
|
|
202
|
+
const entry = { status: "ok", id, at, dim: vector.length, model };
|
|
203
|
+
ledger.record(entry);
|
|
204
|
+
logger?.info?.(`[dsh-mneme] embedded memory ${id} (dim=${vector.length})`);
|
|
205
|
+
return entry;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
return {
|
|
209
|
+
// Display name for /semantic: a literal's constructor.name is "Object",
|
|
210
|
+
// which the status card would render verbatim.
|
|
211
|
+
name: "OpenAI",
|
|
212
|
+
/**
|
|
213
|
+
* issue #135: 这个 embedder 没有异步初始化,`"ready" in embedder` 的通用
|
|
214
|
+
* 分支会把它恒判为可用 —— 而 vector-config 四项有缺时它一条都嵌不出来。
|
|
215
|
+
* 对外报真实可用性,让 /semantic 的 ready 不再是「绿色假阳性」。
|
|
216
|
+
* 用 getter 而非快照字段:设置面板改完配置无需重启即生效(与 embed() 同源读取)。
|
|
217
|
+
*/
|
|
218
|
+
get ready() {
|
|
219
|
+
return configComplete(settings.getVectorConfig());
|
|
220
|
+
},
|
|
221
|
+
/**
|
|
222
|
+
* 与 ready 同源,分开两个名字是给 /semantic 用:local/ollama 的
|
|
223
|
+
* ready=false 表示「还在初始化」,而这里的 false 表示「根本没配」。
|
|
224
|
+
*/
|
|
225
|
+
get configured() {
|
|
226
|
+
return configComplete(settings.getVectorConfig());
|
|
227
|
+
},
|
|
228
|
+
/**
|
|
229
|
+
* Re-embed a memory after any write. Still non-blocking — the caller does
|
|
230
|
+
* not have to await it — but it now RETURNS the receipt promise so a
|
|
231
|
+
* caller that wants to know can. Previously this returned undefined and
|
|
232
|
+
* swallowed the rejection, so "did this write get a vector?" had no answer
|
|
233
|
+
* anywhere in the process. Swallowing still applies to the rejection
|
|
234
|
+
* itself: a vector failure must never break the memory write.
|
|
235
|
+
*/
|
|
236
|
+
schedule(memory) {
|
|
237
|
+
if (!memory?.id) {
|
|
238
|
+
return Promise.resolve({ status: "skipped", id: null, at: new Date().toISOString(), reason: "no-id" });
|
|
239
|
+
}
|
|
240
|
+
return embedFor(memory.id, memory.title, memory.content).catch((error) => ({
|
|
241
|
+
status: "failed", id: memory.id, at: new Date().toISOString(), reason: String(error)
|
|
242
|
+
}));
|
|
243
|
+
},
|
|
244
|
+
|
|
245
|
+
/** Recent embed receipts + running totals (see createReceiptLedger). */
|
|
246
|
+
embedReceipts() {
|
|
247
|
+
return ledger.snapshot();
|
|
248
|
+
},
|
|
249
|
+
|
|
250
|
+
/** Embed one text and return its vector (null on failure/disabled). */
|
|
251
|
+
async embed(query) {
|
|
252
|
+
const cfg = settings.getVectorConfig();
|
|
253
|
+
if (!configComplete(cfg)) return null;
|
|
254
|
+
const vector = await embedText(cfg, query);
|
|
255
|
+
if (vector) _dimension = vector.length;
|
|
256
|
+
return vector;
|
|
257
|
+
},
|
|
258
|
+
|
|
259
|
+
// Bug1: single-text adapter. Local/ollama embedders expose embedSingle
|
|
260
|
+
// natively; the legacy OpenAI-compatible client only has embed. This
|
|
261
|
+
// adapter unifies the interface so vector-index rebuildIndex (which guards
|
|
262
|
+
// on `typeof embedder.embedSingle === "function"`) accepts this embedder.
|
|
263
|
+
async embedSingle(text) {
|
|
264
|
+
if (typeof this.embed === "function") return this.embed(text);
|
|
265
|
+
return null;
|
|
266
|
+
},
|
|
267
|
+
|
|
268
|
+
// Same as embedSingle, but keeps the reason a vector did not come back.
|
|
269
|
+
// embedSingle() drops it on purpose — its callers only care whether a
|
|
270
|
+
// vector exists — but the boot backfill has to tell "this row is broken"
|
|
271
|
+
// from "the provider was unreachable", and a bare null erases that
|
|
272
|
+
// distinction. Callers feature-detect this method and fall back to
|
|
273
|
+
// embedSingle() so older adapters keep working unchanged.
|
|
274
|
+
async embedSingleDetailed(text) {
|
|
275
|
+
const cfg = settings.getVectorConfig();
|
|
276
|
+
if (!configComplete(cfg)) return { vector: null, reason: "config-incomplete" };
|
|
277
|
+
const out = await embedTextDetailed(cfg, text);
|
|
278
|
+
if (out.vector) _dimension = out.vector.length;
|
|
279
|
+
return out;
|
|
280
|
+
},
|
|
281
|
+
|
|
282
|
+
/** Model fingerprint (model#hex), or undefined when not configured. */
|
|
283
|
+
get modelHash() {
|
|
284
|
+
const cfg = settings.getVectorConfig();
|
|
285
|
+
return cfg?.enabled && cfg.model ? modelHashOf(cfg.model) : undefined;
|
|
286
|
+
},
|
|
287
|
+
|
|
288
|
+
/** Dimension of the last successful embed (0 when never embedded). */
|
|
289
|
+
get dimension() {
|
|
290
|
+
return _dimension || undefined;
|
|
291
|
+
},
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Batch re-index rows whose vector is missing OR came from another model.
|
|
295
|
+
* Uses needsEmbeddingFor (not needsEmbedding) so a model switch actually
|
|
296
|
+
* re-embeds the old vectors instead of leaving them eligible for "has a
|
|
297
|
+
* vector" forever while they poison similarity ranking.
|
|
298
|
+
*/
|
|
299
|
+
async reindexMissing(limit = 50) {
|
|
300
|
+
const cfg = settings.getVectorConfig();
|
|
301
|
+
if (!cfg?.enabled) return { indexed: 0, failed: 0, skipped: 0 };
|
|
302
|
+
const model = modelHashOf(cfg.model);
|
|
303
|
+
// Fall back to the plain selector for stores that predate the
|
|
304
|
+
// provenance column (older hosts, test doubles).
|
|
305
|
+
const rows = typeof store.needsEmbeddingFor === "function"
|
|
306
|
+
? store.needsEmbeddingFor(model, limit)
|
|
307
|
+
: store.needsEmbedding(limit);
|
|
308
|
+
let indexed = 0;
|
|
309
|
+
let failed = 0;
|
|
310
|
+
for (const row of rows) {
|
|
311
|
+
try {
|
|
312
|
+
const text = [row.title, row.content].filter(Boolean).join("\n");
|
|
313
|
+
// Prefer the reason-preserving call for the same reason the boot
|
|
314
|
+
// backfill does: a bare null cannot tell "this row is unembeddable"
|
|
315
|
+
// from "the provider was unreachable", and the receipt ledger files
|
|
316
|
+
// both under one indistinguishable entry.
|
|
317
|
+
const detailed = typeof this.embedSingleDetailed === "function"
|
|
318
|
+
? await this.embedSingleDetailed(text)
|
|
319
|
+
: null;
|
|
320
|
+
const vector = detailed ? detailed.vector : await embedText(cfg, text);
|
|
321
|
+
if (vector?.length) {
|
|
322
|
+
store.setEmbedding(row.id, vector, model);
|
|
323
|
+
_dimension = vector.length;
|
|
324
|
+
indexed++;
|
|
325
|
+
} else {
|
|
326
|
+
failed++;
|
|
327
|
+
}
|
|
328
|
+
} catch (error) {
|
|
329
|
+
// A throw used to abort the whole loop and surface as a rejected
|
|
330
|
+
// promise from the API route; one bad row must not stop the rest.
|
|
331
|
+
failed++;
|
|
332
|
+
logger?.warn?.(`[dsh-mneme] reindex embed failed for ${row.id}: ${String(error)}`);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
if (indexed > 0) markModel(cfg, _dimension || undefined);
|
|
336
|
+
// #604: `skipped: rows.length - indexed` folded "tried and got nothing"
|
|
337
|
+
// into "never touched". Every selected row is attempted here, so nothing
|
|
338
|
+
// is skipped; what failed is reported as failed.
|
|
339
|
+
return { indexed, failed, skipped: 0 };
|
|
340
|
+
}
|
|
341
|
+
};
|
|
342
|
+
}
|