dsh-memu-memory 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/AGENTS.md +1 -1
  2. package/README.md +19 -8
  3. package/README.zh.md +12 -6
  4. package/docs/contributing.md +16 -3
  5. package/docs/env-and-commands.md +8 -0
  6. package/docs/host-api.md +3 -1
  7. package/docs/memory-semantics.md +21 -2
  8. package/docs/troubleshooting.md +42 -3
  9. package/lib/activity-log.js +22 -5
  10. package/lib/backup.js +24 -2
  11. package/lib/binding-diagnostics.js +3 -3
  12. package/lib/client.js +1610 -117
  13. package/lib/consolidation-worker.js +68 -17
  14. package/lib/curator-evidence.js +131 -0
  15. package/lib/curator-policy.js +81 -0
  16. package/lib/curator.js +1299 -83
  17. package/lib/cycle.js +3 -0
  18. package/lib/embedding-download.js +39 -4
  19. package/lib/engine.js +6 -3
  20. package/lib/global-graph.js +282 -0
  21. package/lib/graph.js +190 -14
  22. package/lib/inbound.js +19 -0
  23. package/lib/index.js +33 -1
  24. package/lib/inject.js +277 -17
  25. package/lib/lexicon.js +1115 -0
  26. package/lib/local-vector-service.js +8 -6
  27. package/lib/managed-runtime.js +33 -2
  28. package/lib/memory.js +585 -76
  29. package/lib/memu.js +2 -2
  30. package/lib/migrate.js +83 -2
  31. package/lib/private-links.js +587 -0
  32. package/lib/private.js +128 -7
  33. package/lib/prompts/global-curator.md +76 -19
  34. package/lib/prompts/private-consolidation.coding.md +4 -1
  35. package/lib/prompts/private-consolidation.md +3 -1
  36. package/lib/prompts/private-consolidation.mixed.md +3 -1
  37. package/lib/prompts/private-consolidation.rp.md +3 -1
  38. package/lib/prompts.js +1 -1
  39. package/lib/rpc.js +622 -13
  40. package/lib/state.js +4 -1
  41. package/lib/text-retrieval.js +661 -0
  42. package/lib/text-similarity.js +124 -0
  43. package/lib/tools.js +230 -16
  44. package/lib/vector-migration.js +28 -3
  45. package/lib/vector-profile.js +24 -0
  46. package/lib/webui/standalone.css +17 -0
  47. package/lib/webui/standalone.js +2 -0
  48. package/package.json +3 -2
  49. package/scripts/acceptance-local.mjs +8 -5
  50. package/scripts/curator-pair-calibrate.mjs +259 -0
  51. package/scripts/lexicon-calibrate.mjs +230 -0
  52. package/scripts/lexicon-dedupe.mjs +318 -0
  53. package/scripts/replay.mjs +573 -0
  54. package/tests/absorb-review.test.mjs +353 -0
  55. package/tests/appendix-b.test.mjs +6 -2
  56. package/tests/appendix-b2.test.mjs +2 -2
  57. package/tests/client-export.test.mjs +520 -0
  58. package/tests/client-render.test.mjs +19 -3
  59. package/tests/consolidation-credential.test.mjs +16 -5
  60. package/tests/consolidation-feed.test.mjs +96 -0
  61. package/tests/consolidation-fork.test.mjs +144 -10
  62. package/tests/consolidation-snapshot.test.mjs +82 -4
  63. package/tests/curator-absorb.test.mjs +443 -0
  64. package/tests/curator-calibrate.test.mjs +173 -0
  65. package/tests/curator-clusters.test.mjs +281 -0
  66. package/tests/curator-commit.test.mjs +324 -0
  67. package/tests/curator-evidence.test.mjs +102 -0
  68. package/tests/curator-link.test.mjs +316 -0
  69. package/tests/curator-pairing.test.mjs +252 -0
  70. package/tests/curator-probe.test.mjs +15 -6
  71. package/tests/curator-review-mode.test.mjs +314 -0
  72. package/tests/disclosure.test.mjs +2 -1
  73. package/tests/fixtures/curator-pairs/labeled.v1.jsonl +90 -0
  74. package/tests/fixtures/lexicon-stub-hooks.mjs +15 -0
  75. package/tests/fixtures/lexicon-stub.mjs +284 -0
  76. package/tests/graph-cascade.test.mjs +223 -0
  77. package/tests/graph.test.mjs +113 -0
  78. package/tests/help-page.test.mjs +415 -0
  79. package/tests/hybrid-recall.test.mjs +32 -2
  80. package/tests/inbound.test.mjs +6 -3
  81. package/tests/inject-guide.test.mjs +28 -0
  82. package/tests/inject.test.mjs +157 -5
  83. package/tests/lexicon-calibrate.test.mjs +48 -0
  84. package/tests/lexicon-dedupe.test.mjs +171 -0
  85. package/tests/lexicon-frequency.test.mjs +179 -0
  86. package/tests/lexicon-inject.test.mjs +410 -0
  87. package/tests/lexicon-perf.test.mjs +436 -0
  88. package/tests/lexicon-ui.test.mjs +547 -0
  89. package/tests/lexicon.test.mjs +922 -0
  90. package/tests/managed-memory.test.mjs +55 -0
  91. package/tests/managed-runtime.test.mjs +32 -0
  92. package/tests/private-links.test.mjs +445 -0
  93. package/tests/prompts.test.mjs +35 -0
  94. package/tests/recall-dedup.test.mjs +446 -0
  95. package/tests/sidebar-beat.test.mjs +5 -1
  96. package/tests/tool-schema.test.mjs +1 -1
  97. package/tests/tools.test.mjs +112 -3
  98. package/tests/vector-migration.test.mjs +58 -1
  99. package/tests/webui-auth.test.mjs +348 -2
  100. package/tests/webui.test.mjs +191 -2
package/lib/memory.js CHANGED
@@ -15,10 +15,12 @@ import { record } from './activity-log.js'
15
15
  import { BASE_AROUSAL, derivedValence, dominanceBias, emotionActive, maxShare, moodCongruence, moodDecay, normalizeEmotion, overlap, readMood } from './mood.js'
16
16
  import { commit, retrieve, listFiles, forget } from './memu.js'
17
17
  import { bumpStats } from './stats.js'
18
- import { recallRelations } from './graph.js'
18
+ import { recallRelations, pruneOrphanEntities } from './graph.js'
19
19
  import { attachmentRefs } from './media.js'
20
20
  import { normalizeMemoryContext, contextVisible, sameFactContext } from './context.js'
21
- import { assertGeneration, assertVectorWriteAllowed, captureGeneration, enrollActiveLibrary, hasActiveVectorGeneration, isVectorStateError, readVectorState, withVectorWrite } from './vector-profile.js'
21
+ import { activeEntryStamp, assertGeneration, assertVectorWriteAllowed, captureGeneration, enrollActiveLibrary, hasActiveVectorGeneration, isVectorStateError, readVectorState, withVectorWrite } from './vector-profile.js'
22
+ import { bigrams, dice, similarTexts, DUP_DICE, DUP_REJECT_DICE, TOPIC_DICE } from './text-similarity.js'
23
+ import { applyAbsorbIndexTransaction } from './curator.js'
22
24
 
23
25
  const HALF_LIFE_DAYS = 14
24
26
  // A short-term entry is meant to live for the length of a conversation, not a
@@ -38,6 +40,11 @@ const HABITUATION_FLOOR = 0.5
38
40
  // library with three variants of the same decision would spend the whole block on it.
39
41
  // The top hit is always kept; later picks pay lambda x max-similarity-to-what-is-in.
40
42
  const MMR_LAMBDA = 0.72
43
+ // Absolute floor on the retrieval share itself . The relative
44
+ // relevanceThreshold cannot stop a near-empty result set from being padded to topK with
45
+ // junk that only matched a duplicate cluster; a floor cuts those before selection.
46
+ // 0 turns it off entirely — the calibrated default until §13.6 has run.
47
+ const RETRIEVAL_FLOOR = 0
41
48
  // Semantic fusion uses RRF: each route contributes a
42
49
  // rank term and the merged score is normalised to the best fused hit before the
43
50
  // existing mood/strength/MMR pipeline sees it. The candidate cap keeps a long
@@ -555,20 +562,8 @@ export function isTipOfTongue(entry) {
555
562
  // Not semantic entailment: two bigram sets and a negation test. It only has to be
556
563
  // good enough to stop "he likes tea" and "he hates tea" from sitting side by side
557
564
  // as equally true, and quiet enough that ordinary rewordings do not trip it.
558
- function bigrams(text) {
559
- const clean = String(text || '').toLowerCase().replace(/[\s\p{P}\p{S}]+/gu, '')
560
- const out = new Set()
561
- for (let i = 0; i < clean.length - 1; i += 1) out.add(clean.slice(i, i + 2))
562
- return out
563
- }
564
-
565
- function dice(a, b) {
566
- if (a.size === 0 || b.size === 0) return 0
567
- let shared = 0
568
- for (const gram of a) if (b.has(gram)) shared += 1
569
- return (2 * shared) / (a.size + b.size)
570
- }
571
-
565
+ // The bigrams/dice measure lives in lib/text-similarity.js so recall-side folding
566
+ // and the write-side duplicate gate measure with the exact same ruler.
572
567
  const NEGATION_RE = /不|没|别|无|never|not|no longer|doesn't|don't/i
573
568
 
574
569
  /**
@@ -601,6 +596,48 @@ export function findConflict(entries, candidate) {
601
596
  return null
602
597
  }
603
598
 
599
+ // Duplicate/conflict gates may only consult entries this conversation could
600
+ // actually surface. `secret` never auto-surfaces; knownBy/context-hidden rows are
601
+ // equally invisible. A hidden hit must not leak its id (5.1).
602
+ function visibleForDedupe(entry, sessionId) {
603
+ if (!entry || entry.visibility === 'secret') return false
604
+ return sessionId ? visibleToSession(entry, sessionId) : true
605
+ }
606
+
607
+ // Critical-section re-check for the duplicate gate (5.4): the pre-flight scan
608
+ // runs before the backend await, so a concurrent writer can land a twin while the
609
+ // commit is in flight. Same ruler, same visible scope, one pass inside the lock.
610
+ function findNearDuplicate(entries, { id, title, content, context, sessionId, excludeId = null }) {
611
+ const text = `${title || ''} ${content || ''}`
612
+ let duplicate = null
613
+ let hidden = false
614
+ for (const [otherId, other] of Object.entries(entries || {})) {
615
+ if (otherId === id || otherId === excludeId) continue
616
+ if (!isMemoryEligible(other)) continue
617
+ if (!sameFactContext(other.context, context)) continue
618
+ const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
619
+ if (!visibleForDedupe(other, sessionId)) {
620
+ if (similarity >= DUP_REJECT_DICE) hidden = true
621
+ continue
622
+ }
623
+ if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
624
+ }
625
+ return { duplicate, hidden }
626
+ }
627
+
628
+ // The visible tail that the rewrite adds/changes, for a rejection response. Only
629
+ // ever computed against a VISIBLE duplicate — a hidden twin's text must not leak.
630
+ function nearDuplicateDiff(existingText, candidateText, max = 180) {
631
+ const a = String(existingText || '')
632
+ const b = String(candidateText || '')
633
+ let start = 0
634
+ while (start < a.length && start < b.length && a[start] === b[start]) start += 1
635
+ let end = 0
636
+ while (end < a.length - start && end < b.length - start && a[a.length - 1 - end] === b[b.length - 1 - end]) end += 1
637
+ const added = b.slice(start, b.length - end).trim()
638
+ return clampText(added || b, max)
639
+ }
640
+
604
641
  function mirrorContent(file) {
605
642
  const raw = file?.content
606
643
  if (typeof raw === 'string' && raw.trim()) return raw
@@ -770,27 +807,47 @@ export function fuseRecallCandidates({ local = [], semantic = [], index, limit =
770
807
  if (automatic && entry.visibility === 'secret') return null
771
808
  return { id, entry, file, route, rank }
772
809
  }
773
- const take = (files, route) => {
810
+ const take = (files, route, collapsed) => {
774
811
  const picked = []
775
812
  // One route can name the same canonical entry more than once: a caller may
776
813
  // pass both `mem-x` and `mem-x.md`, and an engine can repeat a hit. Dedupe by
777
814
  // canonical id before scoring and truncating, keeping the first (best) rank.
778
815
  // Otherwise every duplicate adds another 1/(k+rank) and consumes a slot.
779
816
  const seen = new Set()
817
+ // Fact-level folding : id dedupe cannot see "same fact,
818
+ // different id", so a paraphrase cluster would spend one candidate slot per
819
+ // member and crowd real memories out before any ranking runs. A candidate that
820
+ // a accepted candidate already speaks for (dice ≥ DUP_DICE on title + body)
821
+ // is recorded and skipped: one cluster, one candidate slot.
822
+ const gramsCache = new Map()
823
+ const gramsOf = (item) => {
824
+ let set = gramsCache.get(item.id)
825
+ if (!set) {
826
+ set = bigrams(String(item.entry?.title || '') + ' ' + String(item.entry?.content || ''))
827
+ gramsCache.set(item.id, set)
828
+ }
829
+ return set
830
+ }
780
831
  for (const file of Array.isArray(files) ? files : []) {
781
832
  const item = eligible(file, route, picked.length + 1)
782
833
  if (!item || seen.has(item.id)) continue
783
834
  seen.add(item.id)
835
+ const twin = picked.find((accepted) => dice(gramsOf(item), gramsOf(accepted)) >= DUP_DICE)
836
+ if (twin) {
837
+ collapsed.push({ kept: twin.id, dropped: item.id })
838
+ continue
839
+ }
784
840
  picked.push(item)
785
841
  if (picked.length >= limit) break
786
842
  }
787
843
  return picked
788
844
  }
789
- const localPicked = take(local, 'local')
790
- const semanticPicked = take(semantic, 'semantic')
845
+ const routeCollapsed = []
846
+ const localPicked = take(local, 'local', routeCollapsed)
847
+ const semanticPicked = take(semantic, 'semantic', routeCollapsed)
791
848
  const merged = new Map()
792
849
  const add = (item) => {
793
- const current = merged.get(item.id) || { id: item.id, entry: item.entry, file: item.file, rrf: 0, bestRank: Infinity, localRank: null, semanticRank: null, routes: [] }
850
+ const current = merged.get(item.id) || { id: item.id, entry: item.entry, file: item.file, rrf: 0, bestRank: Infinity, localRank: null, semanticRank: null, routes: [], mergedTogether: [] }
794
851
  current.rrf += 1 / (HYBRID_RRF_K + item.rank)
795
852
  current.bestRank = Math.min(current.bestRank, item.rank)
796
853
  if (item.route === 'local') current.localRank = item.rank
@@ -804,14 +861,51 @@ export function fuseRecallCandidates({ local = [], semantic = [], index, limit =
804
861
  for (const item of localPicked) add(item)
805
862
  for (const item of semanticPicked) add(item)
806
863
  const rows = [...merged.values()]
864
+ // Cross-leg folding . Each leg folded within itself, but the
865
+ // two legs can still meet the same fact under two ids. Keep the highest-rrf member
866
+ // of each cross-leg cluster and let the rest ride along as mergedTogether metadata
867
+ // instead of spending recall slots; this must happen BEFORE mmrSelect, so the
868
+ // diversity ceiling is set by a real spread of memories rather than a cluster.
869
+ const fusionCollapsed = []
870
+ const absorbed = new Set()
871
+ const rowGrams = new Map()
872
+ const gramsOfRow = (row) => {
873
+ let set = rowGrams.get(row.id)
874
+ if (!set) {
875
+ set = bigrams(String(row.entry?.title || '') + ' ' + String(row.entry?.content || ''))
876
+ rowGrams.set(row.id, set)
877
+ }
878
+ return set
879
+ }
880
+ const hosts = []
881
+ for (const row of rows.slice().sort((a, b) => b.rrf - a.rrf || a.id.localeCompare(b.id))) {
882
+ const host = hosts.find((kept) => dice(gramsOfRow(row), gramsOfRow(kept)) >= DUP_DICE)
883
+ if (host) {
884
+ host.mergedTogether.push(row.id)
885
+ absorbed.add(row.id)
886
+ fusionCollapsed.push({ kept: host.id, dropped: row.id })
887
+ } else {
888
+ hosts.push(row)
889
+ }
890
+ }
891
+ const collapsedGroups = new Map()
892
+ for (const { kept, dropped } of [...routeCollapsed, ...fusionCollapsed]) {
893
+ if (!collapsedGroups.has(kept)) collapsedGroups.set(kept, { kept, dropped: [] })
894
+ const group = collapsedGroups.get(kept)
895
+ // The same pair can meet on both routes; report it once.
896
+ if (!group.dropped.includes(dropped)) group.dropped.push(dropped)
897
+ }
898
+ const collapsed = [...collapsedGroups.values()]
807
899
  const maxRrf = rows.reduce((max, row) => Math.max(max, row.rrf), 0) || 1
808
900
  const files = rows
901
+ .filter((row) => !absorbed.has(row.id))
809
902
  .map((row) => ({
810
903
  ...row.file,
811
904
  name: row.id,
812
905
  entry: row.entry,
813
906
  retrievalScore: row.rrf / maxRrf,
814
907
  fusion: { rrf: row.rrf, bestRank: row.bestRank, localRank: row.localRank, semanticRank: row.semanticRank, routes: row.routes.slice() },
908
+ ...(row.mergedTogether.length ? { mergedTogether: row.mergedTogether.slice() } : {}),
815
909
  }))
816
910
  .sort((a, b) => b.retrievalScore - a.retrievalScore
817
911
  || a.fusion.bestRank - b.fusion.bestRank
@@ -830,10 +924,21 @@ export function fuseRecallCandidates({ local = [], semantic = [], index, limit =
830
924
  localOnly: [...localIds].filter((id) => !semanticIds.has(id)).length,
831
925
  semanticOnly: [...semanticIds].filter((id) => !localIds.has(id)).length,
832
926
  shared: [...localIds].filter((id) => semanticIds.has(id)).length,
927
+ collapsed,
928
+ clusterCount: collapsed.length,
833
929
  },
834
930
  }
835
931
  }
836
932
 
933
+ // The retrieval component of recallScore: a fused file carries its own normalised
934
+ // RRF share; falling back to the old rank decay keeps every lexical caller
935
+ // byte-for-byte compatible. Shared with the retrieval-floor cut so the two can
936
+ // never disagree about what "retrieval" means.
937
+ function retrievalShare(file, rank) {
938
+ const fused = Number(file?.retrievalScore)
939
+ return Number.isFinite(fused) ? Math.max(0, Math.min(1, fused)) : 1 / (rank + 1)
940
+ }
941
+
837
942
  // Human-like recall ordering. Retrieval answers first (semantic match is still
838
943
  // the primary signal, which is why it stays a rank decay), then these biases
839
944
  // re-order within that set:
@@ -843,10 +948,7 @@ export function fuseRecallCandidates({ local = [], semantic = [], index, limit =
843
948
  // Every factor is bounded so no bias can outrank a good semantic hit by luck.
844
949
  function recallScore(file, rank, context) {
845
950
  const entry = file.entry || {}
846
- // A fused file carries its own normalised RRF share. Falling back to the old
847
- // rank decay keeps every lexical caller byte-for-byte compatible.
848
- const fused = Number(file.retrievalScore)
849
- const retrieval = Number.isFinite(fused) ? Math.max(0, Math.min(1, fused)) : 1 / (rank + 1)
951
+ const retrieval = retrievalShare(file, rank)
850
952
  const strengthTerm = 0.6 + 0.4 * file.strength
851
953
  const marginal = entry.marginalized ? 0.35 : 1
852
954
 
@@ -882,8 +984,13 @@ function recallScore(file, rank, context) {
882
984
  // a trump card: 1.25 still loses to a clearly better semantic hit.
883
985
  const openLoopBias = entry.openLoop ? 1.25 : 1
884
986
  const habituation = habituationFactor(entry, context.now)
987
+ // Fact-level dedup down-weight . A near-copy that arrived
988
+ // later is tagged duplicateOf at write time; without this factor the copy would
989
+ // ride the same touch/decay loop as the original and could outlive it in rank —
990
+ // the "duplicates age better than the truth" loop this closes.
991
+ const duplicateFactor = entry.duplicateOf ? 0.7 : 1
885
992
 
886
- return retrieval * strengthTerm * marginal * moodBias * peakEnd * primacy * openLoopBias * habituation * arousalTerm * dominanceTerm
993
+ return retrieval * strengthTerm * marginal * moodBias * peakEnd * primacy * openLoopBias * habituation * arousalTerm * dominanceTerm * duplicateFactor
887
994
  }
888
995
 
889
996
  /**
@@ -940,12 +1047,19 @@ function habituationFactor(entry, now = Date.now()) {
940
1047
  // bounded biases, so its scale depends on the library and a fixed cutoff would mean
941
1048
  // different things in different conversations. "Keep what scores within N of the best
942
1049
  // hit" is scale-free, and 0 turns the filter off entirely.
943
- function cutWeakScores(files, threshold) {
1050
+ // `retrievalFloor` is the absolute complement : whatever the
1051
+ // library looks like, a hit whose retrieval share never cleared the floor must not
1052
+ // pad a topK that a duplicate cluster emptied. 0 turns it off entirely.
1053
+ function cutWeakScores(files, threshold, retrievalFloor = 0) {
1054
+ let out = files
944
1055
  const cut = Number(threshold)
945
- if (!Number.isFinite(cut) || cut <= 0) return files
946
- const best = Number(files[0]?.score || 0)
947
- if (best <= 0) return files
948
- return files.filter((f) => Number(f.score || 0) >= best * cut)
1056
+ if (Number.isFinite(cut) && cut > 0) {
1057
+ const best = Number(out[0]?.score || 0)
1058
+ if (best > 0) out = out.filter((f) => Number(f.score || 0) >= best * cut)
1059
+ }
1060
+ const floor = Number(retrievalFloor)
1061
+ if (Number.isFinite(floor) && floor > 0) out = out.filter((f) => Number(f.retrieval ?? retrievalShare(f, 0)) >= floor)
1062
+ return out
949
1063
  }
950
1064
 
951
1065
  function privateWeighted(result, max, context = {}) {
@@ -959,6 +1073,7 @@ function privateWeighted(result, max, context = {}) {
959
1073
  lastCreatedAt: null,
960
1074
  sessionId: null,
961
1075
  now: Date.now(),
1076
+ retrievalFloor: RETRIEVAL_FLOOR,
962
1077
  ...context,
963
1078
  }
964
1079
  const items = (result.files || [])
@@ -969,55 +1084,79 @@ function privateWeighted(result, max, context = {}) {
969
1084
  // memories. Filtering here keeps the anchor and the recall block in agreement.
970
1085
  .filter((f) => visibleToSession(f.entry, ctx.sessionId))
971
1086
  .filter((f) => !ctx.automatic || f.entry?.visibility !== 'secret')
972
- .map((f, i) => ({ ...f, score: recallScore(f, i, ctx) }))
1087
+ .map((f, i) => ({ ...f, score: recallScore(f, i, ctx), retrieval: retrievalShare(f, i) }))
973
1088
  .sort((a, b) => b.score - a.score)
974
- return { ...result, files: mmrSelect(cutWeakScores(items, ctx.relevanceThreshold), max) }
1089
+ const { chosen, skipped } = mmrSelect(cutWeakScores(items, ctx.relevanceThreshold, ctx.retrievalFloor), max)
1090
+ // Observation for the dedup thresholds : candidate folding
1091
+ // (①②) reports through result.fusion, the ranking-level hard skips (③) come back
1092
+ // from mmrSelect. One merged list — without it, a mistuned threshold eats real
1093
+ // memories invisibly. Fail-open: metadata only, never a recall failure.
1094
+ const fusionCollapsed = Array.isArray(result.fusion?.collapsed) ? result.fusion.collapsed : []
1095
+ const collapsed = [...fusionCollapsed, ...skipped]
1096
+ return { ...result, files: chosen, dedup: { collapsed, clusterCount: collapsed.length } }
975
1097
  }
976
1098
 
977
1099
  /**
978
1100
  * Greedy maximal marginal relevance over already-scored files.
979
1101
  *
980
- * Similarity is lexical (queryTokens + Jaccard) on purpose: it must be cheap enough
981
- * to run on every recall and predictable enough to explain. An embedding version
982
- * would need a second engine round trip just to de-duplicate four lines.
1102
+ * Similarity is lexical on purpose (dice over title+body bigrams, the same shared
1103
+ * ruler the write-side duplicate gate uses): it must be cheap enough to run on every
1104
+ * recall and predictable enough to explain. An embedding version would need a second
1105
+ * engine round trip just to de-duplicate four lines.
1106
+ *
1107
+ * Two bands :
1108
+ * · dice >= DUP_DICE — the candidate describes a fact something already chosen
1109
+ * describes: it is never selected, only reported as `skipped`. A soft penalty
1110
+ * cannot bound how many seats one fact takes; this can.
1111
+ * · TOPIC_DICE <= dice < DUP_DICE — same topic, different fact: the existing soft
1112
+ * penalty (lambda x overlap x ceiling) still applies.
1113
+ * Below TOPIC_DICE the two memories do not interact.
983
1114
  */
984
1115
  function mmrSelect(items, max, lambda = MMR_LAMBDA) {
985
1116
  const limit = Number(max || 0)
986
- if (!(limit > 0)) return items
987
- if (items.length <= 1) return items.slice(0, limit)
1117
+ if (!(limit > 0)) return { chosen: items, skipped: [] }
1118
+ if (items.length <= 1) return { chosen: items.slice(0, limit), skipped: [] }
988
1119
  const chosen = []
989
1120
  const pool = items.slice()
990
- const tokenCache = new Map()
991
- const tokensOf = (file) => {
1121
+ const gramsCache = new Map()
1122
+ const gramsOf = (file) => {
992
1123
  const key = file.name
993
- if (tokenCache.has(key)) return tokenCache.get(key)
994
- const set = new Set(queryTokens(String(file.entry?.title || '') + ' ' + String(file.entry?.content || '')))
995
- tokenCache.set(key, set)
1124
+ if (gramsCache.has(key)) return gramsCache.get(key)
1125
+ const set = bigrams(String(file.entry?.title || '') + ' ' + String(file.entry?.content || ''))
1126
+ gramsCache.set(key, set)
996
1127
  return set
997
1128
  }
998
1129
  // Anchor the penalty to the best raw score so the diversity term is on the same
999
1130
  // scale as relevance instead of being dominated by it.
1000
1131
  const ceiling = Number(items[0]?.score || 0) || 1
1132
+ // dropped id -> kept id. Recorded across rounds; an item hard-blocked in one round
1133
+ // stays blocked (chosen only grows), and the map keeps it reported exactly once.
1134
+ const blocked = new Map()
1001
1135
  while (pool.length > 0 && chosen.length < limit) {
1002
- let bestIdx = 0
1136
+ let bestIdx = -1
1003
1137
  let bestScore = -Infinity
1004
1138
  for (let i = 0; i < pool.length; i += 1) {
1005
1139
  let overlap = 0
1006
- for (const picked of chosen) overlap = Math.max(overlap, jaccard(tokensOf(pool[i]), tokensOf(picked)))
1140
+ let twin = null
1141
+ for (const picked of chosen) {
1142
+ const similarity = dice(gramsOf(pool[i]), gramsOf(picked))
1143
+ if (similarity >= DUP_DICE) { twin = picked.name; break }
1144
+ if (similarity >= TOPIC_DICE) overlap = Math.max(overlap, similarity)
1145
+ }
1146
+ if (twin) { blocked.set(pool[i].name, twin); continue }
1007
1147
  const adjusted = Number(pool[i].score || 0) - lambda * overlap * ceiling
1008
1148
  if (adjusted > bestScore) { bestScore = adjusted; bestIdx = i }
1009
1149
  }
1150
+ if (bestIdx === -1) break // nothing selectable is left: only hard duplicates remain
1010
1151
  chosen.push(pool.splice(bestIdx, 1)[0])
1011
1152
  }
1012
- return chosen
1013
- }
1014
-
1015
- /** Jaccard overlap of two token sets; 0 when either side is empty. */
1016
- function jaccard(a, b) {
1017
- if (!a || !b || a.size === 0 || b.size === 0) return 0
1018
- let shared = 0
1019
- for (const token of a) if (b.has(token)) shared += 1
1020
- return shared / (a.size + b.size - shared)
1153
+ const skipped = []
1154
+ for (const [dropped, kept] of blocked) {
1155
+ let group = skipped.find((g) => g.kept === kept)
1156
+ if (!group) { group = { kept, dropped: [] }; skipped.push(group) }
1157
+ group.dropped.push(dropped)
1158
+ }
1159
+ return { chosen, skipped }
1021
1160
  }
1022
1161
 
1023
1162
  /**
@@ -1110,6 +1249,10 @@ async function privateRememberImpl({
1110
1249
  emotion = undefined, dominance = undefined, emotionSource = null,
1111
1250
  attachments, track = 'memory',
1112
1251
  context,
1252
+ // Escape hatch for the duplicate gate : two genuine
1253
+ // observations of the same fact on the same day are a legal write. Off by
1254
+ // default — the model must say it looked, not default past the gate.
1255
+ allowDuplicate = false,
1113
1256
  }) {
1114
1257
  const scope = resolveLibrary({ exec, libraryId })
1115
1258
  const binding = scope.binding
@@ -1250,7 +1393,10 @@ async function privateRememberImpl({
1250
1393
  // the library is not silently appended: it lands as pending_review with the id it
1251
1394
  // collides with, so the conflict surfaces instead of both copies being read as
1252
1395
  // true. Same mechanism the global curator uses; until now the private side had none.
1253
- const conflict = findConflict(Object.fromEntries(Object.entries(index.entries).filter(([key]) => key !== id)), entry)
1396
+ // Hidden rows are excluded (5.1): otherwise a visible write could learn an id
1397
+ // through conflictingWith, and a hidden claim could hold an unrelated new one.
1398
+ const memorySession = memorySessionId(exec)
1399
+ const conflict = findConflict(Object.fromEntries(Object.entries(index.entries).filter(([key, other]) => key !== id && visibleForDedupe(other, memorySession))), entry)
1254
1400
  if (conflict) {
1255
1401
  entry.status = 'pending_review'
1256
1402
  entry.conflictingWith = conflict.id
@@ -1258,6 +1404,69 @@ async function privateRememberImpl({
1258
1404
  ? `与已有记忆「${clampText(conflict.entry.title || conflict.id, 60)}」同题但内容不同`
1259
1405
  : `与已有记忆「${clampText(conflict.entry.title || conflict.id, 60)}」说法相反`
1260
1406
  }
1407
+ // Fact-level duplicate gate . Strictly AFTER the conflict
1408
+ // check — in the same-title band a contradictory rewording must land as
1409
+ // pending_review, not be swallowed as a duplicate — and strictly AFTER
1410
+ // redactSecrets, so key material cannot leak into the similarity metric. Measures
1411
+ // with the shared dice the recall side folds with: one ruler for "same fact".
1412
+ // Never deletes or rewrites anything: a hit either refuses the new entry, or tags
1413
+ // it duplicateOf so recall can keep one seat per fact.
1414
+ let duplicate = null
1415
+ let hiddenDuplicate = null
1416
+ if (!conflict && allowDuplicate !== true) {
1417
+ const text = `${entry.title || ''} ${content || ''}`
1418
+ for (const [otherId, other] of Object.entries(index.entries)) {
1419
+ if (otherId === id || !isMemoryEligible(other)) continue
1420
+ if (!sameFactContext(other.context, entry.context)) continue
1421
+ const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
1422
+ if (!visibleForDedupe(other, memorySession)) {
1423
+ // Hidden near-duplicate: refuse generically, never return its id/similarity.
1424
+ if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
1425
+ continue
1426
+ }
1427
+ if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
1428
+ }
1429
+ }
1430
+ if (hiddenDuplicate || (duplicate && duplicate.similarity >= DUP_REJECT_DICE)) {
1431
+ // 5.3:拒写不丢文本——新正文作为 pending_review 暂存,进入现有复核队列;
1432
+ // 可见重复给出 id 与差异片段,隐藏重复只说“有重复”,不泄露 id/正文。
1433
+ const visibleTarget = duplicate ? index.entries[duplicate.id] || null : null
1434
+ const pendingEntry = {
1435
+ ...entry,
1436
+ status: 'pending_review',
1437
+ reviewReason: 'near_duplicate',
1438
+ conflictingWith: duplicate ? duplicate.id : null,
1439
+ ...(duplicate ? { duplicateOf: duplicate.id } : {}),
1440
+ }
1441
+ await retryStoreWrite(() => updatePrivateIndex(scope.libraryId, (fresh) => {
1442
+ if (fresh.entries[id]) throw new Error('memory changed concurrently; retry')
1443
+ fresh.entries[id] = pendingEntry
1444
+ }))
1445
+ if (duplicate) {
1446
+ const diff = nearDuplicateDiff(
1447
+ `${visibleTarget?.title || ''} ${visibleTarget?.content || ''}`,
1448
+ `${entry.title || ''} ${entry.content || ''}`,
1449
+ )
1450
+ return {
1451
+ id,
1452
+ entry: pendingEntry,
1453
+ pending: true,
1454
+ deduped: true,
1455
+ existingId: duplicate.id,
1456
+ similarity: duplicate.similarity,
1457
+ hint: `库内已有一条几乎相同的记忆;新正文已暂存待复核(near_duplicate)。差异片段:${diff || '(无可见差异)'}。若确属两次独立观测,带 allowDuplicate=true 重写;要合并请改用 memu_private_update。`,
1458
+ }
1459
+ }
1460
+ return {
1461
+ id,
1462
+ entry: pendingEntry,
1463
+ pending: true,
1464
+ deduped: true,
1465
+ similarityBand: '0.90+',
1466
+ hint: '库内已有一条几乎相同的记忆,但它不在当前会话的可见范围内;新正文已暂存待复核(near_duplicate)。',
1467
+ }
1468
+ }
1469
+ if (duplicate && duplicate.similarity >= TOPIC_DICE) entry.duplicateOf = duplicate.id
1261
1470
  const payload = {
1262
1471
  recall_files: [{
1263
1472
  name: id,
@@ -1280,16 +1489,7 @@ async function privateRememberImpl({
1280
1489
  // 记下"这条进没进向量库":只有真的带代际提交成功才写,冲突暂扣/降级都不写。
1281
1490
  // 之前没有任何字段能回答"哪条记忆进向量库了",只能靠猜。
1282
1491
  if (route !== 'lexical' && !conflict && result?.ok !== false && !result?.parseWarning) {
1283
- const active = readVectorState({ tolerateMissing: true }).state?.active
1284
- if (active) {
1285
- entry.vector = {
1286
- route,
1287
- model: active.profile?.catalogId || null,
1288
- generation: active.generation,
1289
- fingerprint: String(active.fingerprint || '').slice(0, 8),
1290
- at: stamp(),
1291
- }
1292
- }
1492
+ entry.vector = activeEntryStamp(route) || entry.vector
1293
1493
  }
1294
1494
  // Re-read after the await: a curator job or another turn may have written in
1295
1495
  // the meantime, and saving the pre-await snapshot would erase it.
@@ -1298,13 +1498,27 @@ async function privateRememberImpl({
1298
1498
  if (sessionIdOf(exec) && (!currentBinding || currentBinding.libraryId !== scope.libraryId)) throw new Error('memory scope changed during write; retry')
1299
1499
  if (currentBinding) assertScopeEnabled(currentBinding)
1300
1500
  if (previousHabit && JSON.stringify(fresh.entries[id]) !== JSON.stringify(previousHabit[1])) throw new Error('habit changed concurrently; recall and retry')
1301
- const latestConflict = findConflict(Object.fromEntries(Object.entries(fresh.entries).filter(([key]) => key !== id)), entry)
1501
+ const latestConflict = findConflict(Object.fromEntries(Object.entries(fresh.entries).filter(([key, other]) => key !== id && visibleForDedupe(other, memorySession))), entry)
1302
1502
  if (latestConflict && !conflict) {
1303
1503
  if (db) throw new Error('memory conflict changed during write; retry for review')
1304
1504
  entry.status = 'pending_review'
1305
1505
  entry.conflictingWith = latestConflict.id
1306
1506
  entry.reviewReason = 'concurrent conflicting memory; review required'
1307
1507
  }
1508
+ // 5.4:预检在 await 之前,提交期间另一个 turn 可能刚写入同事实。临界区用同一
1509
+ // 尺子重扫;命中就 retry(有向量则 throw 让 compensation 撤掉刚提交的行),
1510
+ // lexical 路由直接暂扣为 near_duplicate,不双写也不丢文本。
1511
+ const latestDuplicate = allowDuplicate
1512
+ ? { duplicate: null, hidden: false }
1513
+ : findNearDuplicate(fresh.entries, {
1514
+ id, title: entry.title, content: entry.content, context: entry.context, sessionId: memorySession,
1515
+ })
1516
+ if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
1517
+ if (db) throw new Error('memory duplicate changed during write; retry')
1518
+ entry.status = 'pending_review'
1519
+ entry.reviewReason = 'near_duplicate'
1520
+ entry.conflictingWith = latestDuplicate.duplicate?.id || null
1521
+ }
1308
1522
  if (layer === 'must_remember' && pinSource === 'self') {
1309
1523
  const pinned = Object.values(fresh.entries).filter(e => e.layer === 'must_remember' && isMemoryEligible(e)).length
1310
1524
  if (pinned >= Number(getSettings().mustRememberCap || 12)) throw new Error('must_remember soft cap reached; retry as long_term')
@@ -1478,6 +1692,12 @@ async function importMemoriesImpl({ libraryId, memories, maxFiles = 2000, now =
1478
1692
  engineWarning = String(error?.message || error)
1479
1693
  }
1480
1694
  }
1695
+ // 批量恢复与单条写入同口径:提交成功就在条目上落"已入向量库"戳,否则这些条目
1696
+ // 在向量计数(membership 的戳腿)里永远缺失。受管提交失败直接抛,不会走到这里。
1697
+ if (route !== 'lexical' && engineWarning === null) {
1698
+ const vector = activeEntryStamp(route)
1699
+ if (vector) for (const item of entries) item.entry.vector = vector
1700
+ }
1481
1701
  const idMap = Object.fromEntries(entries.filter(item => item.entry.importedFrom).map(item => [item.entry.importedFrom, item.id]))
1482
1702
  await retryStoreWrite(() => updatePrivateIndex(libraryId, (fresh) => {
1483
1703
  for (const { id, entry } of entries) {
@@ -1536,6 +1756,21 @@ export async function privateRecall({ exec, query, topK, relevanceThreshold, pur
1536
1756
  // like one nobody ever touches. Batched into a single index write.
1537
1757
  if (purpose !== 'automatic') touchAccessMany({ libraryId: binding.libraryId, ids: (scored.files || []).map((f) => f.name) })
1538
1758
  bumpStats(binding.libraryId, 'recalled', (scored.files || []).length)
1759
+ // Fact-level dedup observation . Fail-open signal, not
1760
+ // bookkeeping: without it a mistuned DUP_DICE silently eats real memories and
1761
+ // nobody can tell "recalled N" from "recalled N members of one fact".
1762
+ if (scored.dedup?.collapsed?.length) {
1763
+ const folded = scored.dedup.collapsed.reduce((n, group) => n + group.dropped.length, 0)
1764
+ record({
1765
+ kind: 'recall',
1766
+ scope: 'private',
1767
+ sessionId: memorySessionId(exec),
1768
+ libraryId: binding.libraryId,
1769
+ message: `去重折叠 ${folded} 条候选(${scored.dedup.clusterCount} 簇)`,
1770
+ detail: { collapsed: scored.dedup.collapsed, clusterCount: scored.dedup.clusterCount },
1771
+ refs: scored.dedup.collapsed.map((group) => group.kept).slice(0, 20),
1772
+ })
1773
+ }
1539
1774
  for (const file of scored.files || []) {
1540
1775
  if (file.entry?.attachments?.length) {
1541
1776
  file.entry = { ...file.entry, attachments: file.entry.attachments.map(item => {
@@ -1604,6 +1839,7 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1604
1839
  const scope = resolveLibrary({ exec, libraryId })
1605
1840
  if (scope.binding) assertScopeEnabled(scope.binding)
1606
1841
  const index = loadPrivateIndex(scope.libraryId)
1842
+ const memorySession = memorySessionId(exec)
1607
1843
  const id_ = resolveEntryId(index, id)
1608
1844
  const entry = index.entries[id_]
1609
1845
  assertEntryAccess(entry, exec)
@@ -1626,10 +1862,56 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1626
1862
  // the same way. Capture before the update: `entry` is the pre-update snapshot.
1627
1863
  const wasArchived = entry.status === 'archived'
1628
1864
  const nextContent = content === undefined ? undefined : redactSecrets(content)
1629
- if (nextContent !== undefined && entry.status !== 'pending_review') {
1630
- const others = Object.fromEntries(Object.entries(index.entries).filter(([key]) => key !== id_))
1631
- const conflict = findConflict(others, { ...entry, title: title ?? entry.title, content: nextContent })
1865
+ // 改写文本(title 或 content 任一变化)要过与 remember 写入门同口径的两道门
1866
+ // :先矛盾、后重复,度量是同一把 dice 尺。此前只有新建
1867
+ // 被拦——把一条记忆改写成与另一条几乎相同就能从更新路径绕开门,这里补上。
1868
+ // 门在 backend commit 之前跑:拒绝时引擎里什么都没写,compensation 路径不受影响。
1869
+ const textChanged = nextContent !== undefined || title !== undefined
1870
+ const nextTitle = title !== undefined ? clampText(title, 120) : (entry.title || '')
1871
+ const nextContext = context === undefined ? entry.context : normalizeMemoryContext(context)
1872
+ const scanned = textChanged && entry.status !== 'pending_review'
1873
+ let duplicate = null
1874
+ let hiddenDuplicate = null
1875
+ let text = null
1876
+ if (scanned) {
1877
+ const allOthers = Object.fromEntries(Object.entries(index.entries).filter(([key]) => key !== id_))
1878
+ const visibleOthers = Object.fromEntries(Object.entries(allOthers).filter(([, other]) => visibleForDedupe(other, memorySession)))
1879
+ const conflict = findConflict(visibleOthers, { ...entry, title: nextTitle, content: nextContent ?? entry.content })
1632
1880
  if (conflict) throw new Error(`content conflicts with ${conflict.id}; write a new memory for review`)
1881
+ text = `${nextTitle} ${nextContent ?? entry.content}`
1882
+ for (const [otherId, other] of Object.entries(allOthers)) {
1883
+ // 与 remember 的门同一过滤集(isMemoryEligible + sameFactContext),再排除它原本的
1884
+ // duplicateOf 关系——那对条目是写入时就获准共存的,向它继续靠拢不该被拦。
1885
+ if (!isMemoryEligible(other)) continue
1886
+ if (otherId === entry.duplicateOf) continue
1887
+ if (!sameFactContext(other.context, nextContext)) continue
1888
+ const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
1889
+ if (!visibleForDedupe(other, memorySession)) {
1890
+ if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
1891
+ continue
1892
+ }
1893
+ if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
1894
+ }
1895
+ if (hiddenDuplicate) {
1896
+ record({
1897
+ kind: 'write', scope: 'private', sessionId: sessionIdOf(exec) || null, libraryId: scope.libraryId,
1898
+ message: '拒绝重复更新:改写后与当前会话不可见的记忆几乎相同',
1899
+ detail: { id: id_, deduped: true, hidden: true, similarityBand: '0.90+' },
1900
+ refs: [id_],
1901
+ })
1902
+ throw new Error('update refused: a nearly identical memory exists outside this conversation\'s visible scope; rewrite the text or archive one of the two instead')
1903
+ }
1904
+ if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
1905
+ const target = index.entries[duplicate.id]
1906
+ const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
1907
+ record({
1908
+ kind: 'write', scope: 'private', sessionId: sessionIdOf(exec) || null, libraryId: scope.libraryId,
1909
+ message: '拒绝重复更新:' + id_ + ' 改写后与 ' + duplicate.id + ' 几乎相同',
1910
+ detail: { id: id_, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
1911
+ refs: [id_, duplicate.id],
1912
+ })
1913
+ throw new Error(`update refused: the rewritten text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; merge into ${duplicate.id} or archive one of the two instead`)
1914
+ }
1633
1915
  }
1634
1916
  const route = engineRoute()
1635
1917
  const managed = route === 'managed-local'
@@ -1661,8 +1943,30 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1661
1943
  const currentBinding = peekBinding(exec)
1662
1944
  if (sessionIdOf(exec) && (!currentBinding || currentBinding.libraryId !== scope.libraryId)) throw new Error('memory scope changed during update; retry')
1663
1945
  if (currentBinding) assertScopeEnabled(currentBinding)
1946
+ if (scanned) {
1947
+ const latestDuplicate = findNearDuplicate(fresh.entries, {
1948
+ id: id_, title: nextTitle, content: nextContent ?? entry.content, context: nextContext,
1949
+ sessionId: memorySession, excludeId: entry.duplicateOf,
1950
+ })
1951
+ if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
1952
+ throw new Error('duplicate changed during update; retry')
1953
+ }
1954
+ if (latestDuplicate.duplicate) duplicate = latestDuplicate.duplicate
1955
+ }
1664
1956
  if (title !== undefined) target.title = clampText(title, 120)
1665
1957
  if (nextContent !== undefined) target.content = nextContent
1958
+ // 打标带与 remember 同口径:[0.55, 0.90) 改写后仍与另一条同主题 → duplicateOf。
1959
+ // 旧打标的去留也按同一把尺重新裁定:文本改过、且与新文本最相似的条目不在打标带,
1960
+ // 但原 duplicateOf 对象仍与改写后文本同主题(它被排除在拒绝扫描之外,不代表相似
1961
+ // 消失)→ 保留;否则(相似依据消失,或原指向对象已不可召回)摘除——留着会错误
1962
+ // 压低这条记忆的召回权重。未扫(无文本变化)不动。
1963
+ if (scanned) {
1964
+ const previousHost = target.duplicateOf ? fresh.entries[target.duplicateOf] : null
1965
+ const stillHosted = previousHost && isMemoryEligible(previousHost)
1966
+ && similarTexts(text, `${previousHost.title || ''} ${previousHost.content || ''}`) >= TOPIC_DICE
1967
+ if (duplicate && duplicate.similarity >= TOPIC_DICE) target.duplicateOf = duplicate.id
1968
+ else if (!stillHosted) delete target.duplicateOf
1969
+ }
1666
1970
  if (tags !== undefined) target.tags = Array.isArray(tags) ? tags.slice(0, 20) : target.tags
1667
1971
  if (status !== undefined) target.status = status
1668
1972
  if (openLoop !== undefined) target.openLoop = !!openLoop
@@ -1712,6 +2016,12 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1712
2016
  }
1713
2017
  if (dominance !== undefined) target.dominance = dominance === null ? 0 : Math.max(-1, Math.min(1, Number(dominance)))
1714
2018
  if (nextAttachments !== undefined) target.attachments = nextAttachments
2019
+ // 受管路由下这次更新把这一行重新提交进了当前向量空间(managed 时 commit 无条件发生)。
2020
+ // 戳要跟着刷新:它现在描述的是"这条在当前代际的空间里",membership 的戳腿靠它计数。
2021
+ if (committedDb) {
2022
+ const vector = activeEntryStamp(route)
2023
+ if (vector) target.vector = vector
2024
+ }
1715
2025
  // Vividness and accuracy are derived, not typed in: correcting the peak flag or
1716
2026
  // the arousal reading has to move `accuracy` too, otherwise the repair would only
1717
2027
  // change how the entry is stored and not how much it is trusted when recalled.
@@ -1759,18 +2069,63 @@ async function privateUpdateImpl({ exec, libraryId, id, content, title, tags, st
1759
2069
  }
1760
2070
 
1761
2071
  /** Review activation and invalidation of the previous claim are one index commit. */
1762
- async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolution = 'replace', reason = '', content, title }) {
2072
+ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolution = 'replace', reason = '', content, title, sessionId = null }) {
1763
2073
  if (!['accept', 'reject'].includes(action)) throw new Error('action must be accept or reject')
1764
- if (!['replace', 'coexist'].includes(resolution)) throw new Error('resolution must be replace or coexist')
1765
- if (resolution === 'coexist' && String(reason).trim().length < 8) throw new Error('coexist needs a reason describing the distinct conditions (at least 8 characters)')
1766
2074
  if (libraryId && !getLibrary(libraryId)) throw new Error('library not found')
1767
2075
  const index = libraryId ? loadPrivateIndex(libraryId) : loadGlobalIndex()
1768
2076
  const entry = index.entries[id]
1769
2077
  if (!entry || entry.status !== 'pending_review') throw new Error('no such pending review')
2078
+ // pendingAbsorb(spec §4.11 review-all / 安全门触发的 pending absorb):走专用复核
2079
+ // 分支——提案是"整簇合并稿",accept/reject 的落库动作与普通复核完全不同。
2080
+ if (!libraryId && entry.pendingAbsorb) {
2081
+ return resolveAbsorbReviewImpl({ id, entry, index, action, resolution, reason, content, title, sessionId })
2082
+ }
2083
+ if (!['replace', 'coexist'].includes(resolution)) throw new Error('resolution must be replace or coexist')
2084
+ if (resolution === 'coexist' && String(reason).trim().length < 8) throw new Error('coexist needs a reason describing the distinct conditions (at least 8 characters)')
1770
2085
  const next = { ...entry }
1771
2086
  if (action === 'accept') {
1772
2087
  if (typeof content === 'string' && content.trim()) next.content = content
1773
2088
  if (typeof title === 'string' && title.trim()) next.title = title
2089
+ // 接受即写入(accept 会把这条提交进向量库并转为 active),所以它与 remember /
2090
+ // privateUpdate 过同一道重复门:≥ 0.90 拒绝接受(复核保持
2091
+ // 暂扣,不丢数据——换内容或先归档对方后可重试),[0.55, 0.90) 接受但打 duplicateOf。
2092
+ // 排除它自己和 conflictingWith:后者正是这次复核要替代/共存的对象,与它相似是
2093
+ // 复核的本来含义,不是重复。
2094
+ const text = `${next.title || ''} ${next.content || ''}`
2095
+ let duplicate = null
2096
+ let hiddenDuplicate = null
2097
+ for (const [otherId, other] of Object.entries(index.entries || {})) {
2098
+ if (otherId === id || otherId === entry.conflictingWith) continue
2099
+ if (!isMemoryEligible(other)) continue
2100
+ if (!sameFactContext(other.context, next.context)) continue
2101
+ const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
2102
+ if (!visibleForDedupe(other, sessionId)) {
2103
+ if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
2104
+ continue
2105
+ }
2106
+ if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
2107
+ }
2108
+ if (hiddenDuplicate) {
2109
+ record({
2110
+ kind: 'write', scope: libraryId ? 'private' : 'global', libraryId: libraryId || null,
2111
+ message: '拒绝复核接受:接受稿与当前会话不可见的记忆几乎相同',
2112
+ detail: { id, deduped: true, hidden: true, similarityBand: '0.90+' },
2113
+ refs: [id],
2114
+ })
2115
+ throw new Error('review accept refused: a nearly identical memory exists outside the visible scope; pass different content or change the review scope first')
2116
+ }
2117
+ if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
2118
+ const target = index.entries[duplicate.id]
2119
+ const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
2120
+ record({
2121
+ kind: 'write', scope: libraryId ? 'private' : 'global', libraryId: libraryId || null,
2122
+ message: '拒绝复核接受:' + id + ' 与 ' + duplicate.id + ' 几乎相同',
2123
+ detail: { id, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
2124
+ refs: [id, duplicate.id],
2125
+ })
2126
+ throw new Error(`review accept refused: the accepted text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; pass different content or archive ${duplicate.id} first`)
2127
+ }
2128
+ if (duplicate && duplicate.similarity >= TOPIC_DICE) next.duplicateOf = duplicate.id
1774
2129
  const reviewRoute = engineRoute()
1775
2130
  if (reviewRoute !== 'lexical') {
1776
2131
  await commit({
@@ -1778,6 +2133,9 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
1778
2133
  route: reviewRoute,
1779
2134
  db: libraryId ? memoryDbPath(libraryId) : globalDbPath(),
1780
2135
  })
2136
+ // pending_review 从不提交,所以被接受的那一刻才第一次真正进向量库:补上戳,
2137
+ // 否则这条在向量计数里永远缺失。
2138
+ next.vector = activeEntryStamp(reviewRoute) || next.vector
1781
2139
  }
1782
2140
  }
1783
2141
  const mutate = fresh => {
@@ -1785,6 +2143,15 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
1785
2143
  || JSON.stringify(fresh.entries[entry.conflictingWith]) !== JSON.stringify(index.entries[entry.conflictingWith])) {
1786
2144
  throw new Error('review changed; reload and retry')
1787
2145
  }
2146
+ if (action === 'accept') {
2147
+ const latestDuplicate = findNearDuplicate(fresh.entries, {
2148
+ id, title: next.title, content: next.content, context: next.context,
2149
+ sessionId, excludeId: entry.conflictingWith,
2150
+ })
2151
+ if (latestDuplicate.hidden || (latestDuplicate.duplicate && latestDuplicate.duplicate.similarity >= DUP_REJECT_DICE)) {
2152
+ throw new Error('review changed; reload and retry')
2153
+ }
2154
+ }
1788
2155
  const at = stamp()
1789
2156
  next.status = action === 'accept' ? 'active' : 'archived'
1790
2157
  next.reviewedAt = at
@@ -1809,6 +2176,119 @@ async function resolveMemoryReviewImpl({ libraryId = null, id, action, resolutio
1809
2176
  return { id, status: next.status, resolution }
1810
2177
  }
1811
2178
 
2179
+ /**
2180
+ * pendingAbsorb 复核分支(spec §3.3/§4.11/§6.7)。只作用于全局队列(私有索引里
2181
+ * 不存在 pendingAbsorb)。
2182
+ *
2183
+ * accept:resolution 必须 replace——合并稿不能与自己的来源"并存",coexist 直接抛错;
2184
+ * 重复门排除 [id, conflictingWith, ...sourceIds],否则合并稿会被自己的来源判重;
2185
+ * 快照校验 target/sources 仍 active 且 updatedAt === expectedUpdatedAt,否则抛
2186
+ * 'review changed; reload and retry';向量库只提交提案 id 本身(绝不提交 sources,
2187
+ * §6.7 复核);然后单事务落 §4.11 步骤 3c-6 并清 pendingAbsorb/reviewReason/
2188
+ * conflictingWith。索引写失败时 pending 保持、已提交向量行未激活(现有行为,
2189
+ * 本版不额外补偿)。
2190
+ *
2191
+ * reject:提案 archived;target/sources 原样。
2192
+ */
2193
+ async function resolveAbsorbReviewImpl({ id, entry, index, action, resolution, reason, content, title, sessionId = null }) {
2194
+ const pa = entry.pendingAbsorb || {}
2195
+ const targetId = String(pa.targetId || '')
2196
+ const sourceIds = (Array.isArray(pa.sourceIds) ? pa.sourceIds : []).map(String)
2197
+ if (action === 'reject') {
2198
+ await retryStoreWrite(() => updateGlobalIndex((fresh) => {
2199
+ if (JSON.stringify(fresh.entries[id]) !== JSON.stringify(entry)) throw new Error('review changed; reload and retry')
2200
+ const at = stamp()
2201
+ const next = { ...entry, status: 'archived', reviewedAt: at, updatedAt: at, reviewResolution: resolution }
2202
+ if (reason) next.reviewNote = String(reason).trim().slice(0, 1000)
2203
+ fresh.entries[id] = next
2204
+ }))
2205
+ return { id, status: 'archived', resolution }
2206
+ }
2207
+ // —— accept ——
2208
+ if (resolution !== 'replace') {
2209
+ throw new Error('absorb review accept requires the replace resolution; a merged entry cannot coexist with its own sources')
2210
+ }
2211
+ const next = { ...entry }
2212
+ if (typeof content === 'string' && content.trim()) next.content = content
2213
+ if (typeof title === 'string' && title.trim()) next.title = title
2214
+ const exclude = new Set([id, entry.conflictingWith, targetId, ...sourceIds].filter(Boolean))
2215
+ const text = `${next.title || ''} ${next.content || ''}`
2216
+ let duplicate = null
2217
+ let hiddenDuplicate = null
2218
+ for (const [otherId, other] of Object.entries(index.entries || {})) {
2219
+ if (exclude.has(otherId)) continue
2220
+ if (!isMemoryEligible(other)) continue
2221
+ if (!sameFactContext(other.context, next.context)) continue
2222
+ const similarity = similarTexts(text, `${other.title || ''} ${other.content || ''}`)
2223
+ if (!visibleForDedupe(other, sessionId)) {
2224
+ if (similarity >= DUP_REJECT_DICE) hiddenDuplicate = { similarity }
2225
+ continue
2226
+ }
2227
+ if (similarity > (duplicate?.similarity ?? 0)) duplicate = { id: otherId, similarity }
2228
+ }
2229
+ if (hiddenDuplicate) {
2230
+ record({
2231
+ kind: 'write', scope: 'global', libraryId: null,
2232
+ message: '拒绝集合吸收接受:接受稿与当前会话不可见的记忆几乎相同',
2233
+ detail: { id, deduped: true, hidden: true, similarityBand: '0.90+' },
2234
+ refs: [id],
2235
+ })
2236
+ throw new Error('review accept refused: a nearly identical memory exists outside the visible scope; pass different content or change the review scope first')
2237
+ }
2238
+ if (duplicate && duplicate.similarity >= DUP_REJECT_DICE) {
2239
+ const target = index.entries[duplicate.id]
2240
+ const diff = nearDuplicateDiff(`${target?.title || ''} ${target?.content || ''}`, text)
2241
+ record({
2242
+ kind: 'write', scope: 'global', libraryId: null,
2243
+ message: '拒绝集合吸收接受:' + id + ' 与 ' + duplicate.id + ' 几乎相同',
2244
+ detail: { id, deduped: true, existingId: duplicate.id, similarity: duplicate.similarity },
2245
+ refs: [id, duplicate.id],
2246
+ })
2247
+ throw new Error(`review accept refused: the accepted text is nearly identical to ${duplicate.id} (dice ${duplicate.similarity.toFixed(2)}); differences: ${diff || '(none visible)'}; pass different content or archive ${duplicate.id} first`)
2248
+ }
2249
+ if (duplicate && duplicate.similarity >= TOPIC_DICE) next.duplicateOf = duplicate.id
2250
+ // 快照校验(§4.11 accept 3):target/sources 仍 active 且 updatedAt 未被并发改写。
2251
+ const expected = pa.expectedUpdatedAt || {}
2252
+ for (const memberId of [targetId, ...sourceIds].filter(Boolean)) {
2253
+ const current = index.entries[memberId]
2254
+ if (!current || current.status !== 'active'
2255
+ || (expected[memberId] !== undefined && current.updatedAt !== expected[memberId])) {
2256
+ throw new Error('review changed; reload and retry')
2257
+ }
2258
+ }
2259
+ // 向量提交:只提交提案 id,绝不提交 sources(§6.7 复核)。
2260
+ const reviewRoute = engineRoute()
2261
+ if (reviewRoute !== 'lexical') {
2262
+ await commit({
2263
+ payload: { recall_files: [{ name: id, track: next.track || 'memory', description: clampText(next.title, 180), content: next.content }], resource: [], user: {} },
2264
+ route: reviewRoute,
2265
+ db: globalDbPath(),
2266
+ })
2267
+ next.vector = activeEntryStamp(reviewRoute) || next.vector
2268
+ }
2269
+ // 单事务:§4.11 步骤 3c-6。merged 入索引、target superseded、sources archived、
2270
+ // rewire links、写 absorbSnapshot,然后清 pendingAbsorb/reviewReason/conflictingWith。
2271
+ await retryStoreWrite(() => updateGlobalIndex((fresh) => {
2272
+ if (JSON.stringify(fresh.entries[id]) !== JSON.stringify(entry)) throw new Error('review changed; reload and retry')
2273
+ const at = stamp()
2274
+ next.status = 'active'
2275
+ next.updatedAt = at
2276
+ next.reviewedAt = at
2277
+ next.reviewResolution = resolution
2278
+ next.validFrom = at
2279
+ if (reason) next.reviewNote = String(reason).trim().slice(0, 1000)
2280
+ const outcome = applyAbsorbIndexTransaction(fresh, {
2281
+ mergedId: id, merged: next, targetId, sourceIds, expectedUpdatedAt: expected, now: at,
2282
+ })
2283
+ if (!outcome.landed) throw new Error('review changed; reload and retry')
2284
+ const stored = fresh.entries[id]
2285
+ delete stored.pendingAbsorb
2286
+ delete stored.reviewReason
2287
+ delete stored.conflictingWith
2288
+ }))
2289
+ return { id, status: 'active', resolution }
2290
+ }
2291
+
1812
2292
  /**
1813
2293
  * Short-term entries expire instead of being deleted. Deleting would throw away the
1814
2294
  * only copy before the consolidation pass could decide what it was worth; flipping
@@ -1897,7 +2377,27 @@ async function privateDeleteImpl({ exec, libraryId, id }) {
1897
2377
  await retryStoreWrite(() => updatePrivateIndex(scope.libraryId, (fresh) => {
1898
2378
  if (JSON.stringify(fresh.entries[id_]) !== JSON.stringify(entry)) throw new Error('memory changed during deletion; reload and retry')
1899
2379
  delete fresh.entries[id_]
2380
+ // Cascade: inbound edges live in *other* entries' links[], so dropping the
2381
+ // row without sweeping them leaves links that point at a memory nothing can
2382
+ // resolve any more (outbound links die with the row itself). In place, O(N
2383
+ // over links) — same style as the unlink mutator, no deep clone. Archived /
2384
+ // suppressed entries keep their links on purpose: only this permanent path
2385
+ // cascades, so archiving stays reversible.
2386
+ for (const other of Object.values(fresh.entries)) {
2387
+ if (Array.isArray(other.links) && other.links.some(link => link.to === id_)) {
2388
+ other.links = other.links.filter(link => link.to !== id_)
2389
+ }
2390
+ }
1900
2391
  }))
2392
+ // Collect entities this row was the last thing keeping alive. Cleanup after
2393
+ // the fact, not a correctness step: if it fails (lock contention, a store
2394
+ // another writer holds) the delete stands, and the startup sweep gets the
2395
+ // orphans on the next launch.
2396
+ try {
2397
+ pruneOrphanEntities(scope.libraryId)
2398
+ } catch (error) {
2399
+ warn(`orphan entity prune after permanent delete skipped: ${error?.message || error}`)
2400
+ }
1901
2401
  return { id: id_, libraryId: scope.libraryId, deleted: true, backend, engine }
1902
2402
  }
1903
2403
 
@@ -1964,14 +2464,19 @@ export function privateUnpin({ exec, id, reason }) {
1964
2464
  export async function privateRemember(args) {
1965
2465
  const libraryId = resolveLibrary(args).libraryId
1966
2466
  const result = await withManagedPrivateWrite(libraryId, () => privateRememberImpl({ ...args, libraryId }))
2467
+ const deduped = result?.deduped === true
1967
2468
  record({
1968
2469
  kind: 'write',
1969
2470
  scope: 'private',
1970
2471
  sessionId: sessionIdOf(args.exec) || null,
1971
2472
  libraryId,
1972
- message: '写入 ' + (result?.id || '?') + ':' + clampText(args.title || args.content, 60) + (result?.conflict ? '(待复核)' : ''),
1973
- detail: { id: result?.id || null, title: args.title || null, layer: args.layer || null, conflict: result?.conflict || null },
1974
- refs: result?.id ? [result.id] : [],
2473
+ message: deduped
2474
+ ? '拒绝重复写入:与 ' + (result.existingId || '?') + ' 几乎相同'
2475
+ : '写入 ' + (result?.id || '?') + ':' + clampText(args.title || args.content, 60) + (result?.conflict ? '(待复核)' : ''),
2476
+ detail: deduped
2477
+ ? { deduped: true, existingId: result.existingId || null, similarity: result.similarity ?? null }
2478
+ : { id: result?.id || null, title: args.title || null, layer: args.layer || null, conflict: result?.conflict || null },
2479
+ refs: deduped ? [result.existingId] : (result?.id ? [result.id] : []),
1975
2480
  })
1976
2481
  return result
1977
2482
  }
@@ -2086,15 +2591,19 @@ export async function globalRecall({ exec, query, topK, relevanceThreshold, retr
2086
2591
  ...f,
2087
2592
  // Local lexical rank is already meaningful; confidence still nudges ties, and
2088
2593
  // a cross-session fact that has been injected three turns running gets the same
2089
- // wear penalty as a private one.
2594
+ // wear penalty as a private one. `retrieval` mirrors the retrieval term of the
2595
+ // score so the absolute floor cut sees the same number the score was built from.
2596
+ retrieval: Number.isFinite(f.retrievalScore) ? f.retrievalScore : f.lexical > 0 ? f.lexical : 1 / (i + 1),
2090
2597
  score: (Number.isFinite(f.retrievalScore) ? f.retrievalScore : f.lexical > 0 ? f.lexical : 1 / (i + 1))
2091
2598
  * (0.85 + 0.15 * Number(f.entry.confidence ?? 0.5))
2092
2599
  * habituationFactor(f.entry),
2093
2600
  }))
2094
2601
  .sort((a, b) => b.score - a.score)
2602
+ const ranked = mmrSelect(cutWeakScores(filtered, threshold, RETRIEVAL_FLOOR), Number(topK || settings.globalTopK || 3))
2095
2603
  return {
2096
2604
  ...enriched,
2097
- files: mmrSelect(cutWeakScores(filtered, threshold), Number(topK || settings.globalTopK || 3)),
2605
+ files: ranked.chosen,
2606
+ dedup: { collapsed: ranked.skipped, clusterCount: ranked.skipped.length },
2098
2607
  domain: domainOf(binding),
2099
2608
  unbound: !peeked,
2100
2609
  }