memhtml 0.2.4 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { $ as MEMORY_EXTENSION, B as ModelUnavailable, Ct as slugify, H as StorageFailure, J as relClassFor, L as DirtyTree, Q as INBOX_DIR, R as InvalidMemory, St as filenameFor, U as WriteConflict, V as PathNotFound, W as EdgeRel, X as relTokenFor, Y as relForToken, Z as ARCS_DIR, _t as TaskStatus, at as memoryPathFor, bt as parseEntity, ct as placementFor, dt as MEMORY_TYPES, et as PEOPLE_DIR, ft as MemoryStatus, gt as TASK_STATUSES, ht as PERSON_ENTITY_PREFIX, it as isValidMemoryPath, lt as Confidence, mt as PARA_BUCKETS, nt as archivePathFor, ot as normalizePath, pt as MemoryType, q as isEdgeRel, rt as isArchivePath, st as paraBucketOf, ut as Importance, vt as WRITABLE_MEMORY_TYPES, wt as withCollisionOrdinal, xt as SLUG_FALLBACK, yt as isTaskStatus, z as LlmContractViolation } from "./dist-t84Q_98w.mjs";
1
+ import { $ as INBOX_DIR, B as ModelUnavailable, Ct as filenameFor, H as StorageFailure, J as relClassFor, L as DirtyTree, Q as ARCS_DIR, R as InvalidMemory, St as SLUG_FALLBACK, Tt as withCollisionOrdinal, U as WriteConflict, V as PathNotFound, W as EdgeRel, X as relTokenFor, Y as relForToken, Z as ARCHIVE_BUCKET, _t as TASK_STATUSES, at as isValidMemoryPath, bt as isTaskStatus, ct as paraBucketOf, dt as Importance, et as MEMORY_EXTENSION, ft as MEMORY_TYPES, gt as PERSON_ENTITY_PREFIX, ht as PARA_BUCKETS, it as isArchivePath, lt as placementFor, mt as MemoryType, ot as memoryPathFor, pt as MemoryStatus, q as isEdgeRel, rt as archivePathFor, st as normalizePath, tt as PEOPLE_DIR, ut as Confidence, vt as TaskStatus, wt as slugify, xt as parseEntity, yt as WRITABLE_MEMORY_TYPES, z as LlmContractViolation } from "./dist-CSo_XRfz.mjs";
2
2
  import { createRequire } from "node:module";
3
3
  import { Config, Context, Effect, Layer, Result, Schedule, Schema } from "effect";
4
4
  import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
@@ -429,6 +429,12 @@ const bridgeCounts = (nodes, edges, communities) => {
429
429
  * correction was written to fix. These guards are a deterministic veto. A divergent pair
430
430
  * becomes a candidate contradiction for the conflict phase instead of a merge, no matter how
431
431
  * high its cosine runs.
432
+ *
433
+ * The guards are a POST-FILTER over every proposal, including a model's. `dedup-merge` asks a model
434
+ * to partition a connected component into merge groups, and each pair that partition implies is
435
+ * routed through {@link mergeCandidates} before anything is written. A model that groups a claim with
436
+ * its own negation is refused by the same predicate that refuses a blind cosine, so the set of pairs
437
+ * that can be committed does not widen when a model is bound.
432
438
  */
433
439
  /** Cosine similarity above which two bodies are the same content. Strict. */
434
440
  const NEAR_DUPLICATE_THRESHOLD = .92;
@@ -589,6 +595,59 @@ const mergeCandidates = (pairs, options = {}) => {
589
595
  return decisions;
590
596
  };
591
597
  /**
598
+ * Connected components over an undirected edge list, as sorted member lists.
599
+ *
600
+ * The near-duplicate graph's components are dedup's units of work. A component is what "these
601
+ * memories might all be one memory" looks like before anything has judged them, and it is the right
602
+ * unit because near-duplication is transitive in practice: three rewordings of one fact produce
603
+ * three edges, and folding them one pair at a time would ask the same question three times and could
604
+ * answer it three different ways.
605
+ *
606
+ * **The partition is order-INVARIANT, not merely order-stable.** A union always keeps the
607
+ * lexicographically smaller root, so every set's root is the smallest key it holds no matter which
608
+ * order the edges arrive in. Members come back sorted, and components come back ordered by root,
609
+ * which is each component's own smallest member. So the same edge SET produces the same output
610
+ * whether it arrives mined-first, frame-first, mirrored, or shuffled. That is stronger than sorting
611
+ * the input would be, and it is why no sort happens here: a caller cannot make this disagree with
612
+ * itself by changing how it enumerates.
613
+ *
614
+ * A "larger root wins" or "first root seen wins" rule would break exactly that, because both make
615
+ * the surviving root a fact about arrival order rather than about the set.
616
+ *
617
+ * Each pair is normalized before it is unioned, so `(a, b)` and `(b, a)` are one edge. A self-edge
618
+ * introduces its key and joins nothing. Cost is near-linear in the edge count, and no step here ever
619
+ * enumerates a pair the caller did not hand over.
620
+ */
621
+ const connectedComponents = (edges) => {
622
+ const parent = /* @__PURE__ */ new Map();
623
+ const find = (key) => {
624
+ let current = key;
625
+ while ((parent.get(current) ?? current) !== current) {
626
+ const next = parent.get(current);
627
+ parent.set(current, parent.get(next) ?? next);
628
+ current = parent.get(current);
629
+ }
630
+ return current;
631
+ };
632
+ for (const [left, right] of edges) {
633
+ if (!parent.has(left)) parent.set(left, left);
634
+ if (!parent.has(right)) parent.set(right, right);
635
+ const rootLeft = find(left);
636
+ const rootRight = find(right);
637
+ if (rootLeft === rootRight) continue;
638
+ if (rootLeft < rootRight) parent.set(rootRight, rootLeft);
639
+ else parent.set(rootLeft, rootRight);
640
+ }
641
+ const byRoot = /* @__PURE__ */ new Map();
642
+ for (const key of parent.keys()) {
643
+ const root = find(key);
644
+ const bucket = byRoot.get(root);
645
+ if (bucket === void 0) byRoot.set(root, [key]);
646
+ else bucket.push(key);
647
+ }
648
+ return [...byRoot.entries()].sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([, members]) => members.sort());
649
+ };
650
+ /**
592
651
  * The compress-path exclusion: the members of a batch to supersede and archive, with the
593
652
  * canonical removed and order preserved. When a batch folds into a pre-existing canonical, a
594
653
  * member can *be* that canonical, and archiving it would destroy the file just folded into.
@@ -644,6 +703,139 @@ const applyMmr = (candidates, limit, lambda = MMR_LAMBDA) => {
644
703
  return selected;
645
704
  };
646
705
 
706
+ //#endregion
707
+ //#region packages/domain/dist/neighbors.js
708
+ /**
709
+ * A `Float32Array` over stored bytes, copying only when it must.
710
+ *
711
+ * `Float32Array` requires a 4-byte-aligned `byteOffset`, and a driver row's `Uint8Array` may be
712
+ * a view into a pooled buffer at any offset. Viewing in place is the common case and costs
713
+ * nothing. A misaligned or ragged blob is copied rather than rejected, because the vector arm's
714
+ * job is to rank and a throw here would fail a whole search over one row.
715
+ */
716
+ const float32View = (bytes) => {
717
+ if (bytes.byteLength === 0 || bytes.byteLength % 4 !== 0) return void 0;
718
+ const aligned = bytes.byteOffset % 4 === 0 ? bytes : Uint8Array.from(bytes);
719
+ return new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);
720
+ };
721
+ /** `sqrt(Σ x²)` accumulated in index order — the same order {@link cosine} accumulates its norms. */
722
+ const sqrtNorm = (vec) => {
723
+ let sum = 0;
724
+ for (let index = 0; index < vec.length; index += 1) {
725
+ const x = vec[index];
726
+ sum += x * x;
727
+ }
728
+ return Math.sqrt(sum);
729
+ };
730
+ /**
731
+ * Cosine similarity from precomputed norms: {@link cosine}'s operations with the square roots
732
+ * hoisted out of the pair loop, so the result is bit-identical for equal-length vectors. The
733
+ * zero-norm rule and the `[-1, 1]` clamp are the same ones, for the same reasons. Mismatched
734
+ * lengths fall back to {@link cosine}, whose min-length walk defines that case; equal lengths are
735
+ * what the `embed_model` watermark guarantees for stored vectors.
736
+ */
737
+ const pairSimilarity = (a, aNorm, b, bNorm) => {
738
+ if (a.length !== b.length) return cosine(a, b);
739
+ if (aNorm === 0 || bNorm === 0) return 0;
740
+ let dot = 0;
741
+ for (let index = 0; index < a.length; index += 1) dot += a[index] * b[index];
742
+ return Math.max(-1, Math.min(1, dot / (aNorm * bNorm)));
743
+ };
744
+ /**
745
+ * Insert into a per-source list ordered `sim` DESC then `dst` ASC, bounded at `k`. Linear
746
+ * insertion, because `k` is single-digit everywhere this runs and a heap's constant factors lose
747
+ * at that size. Memory across the whole selection is O(n·k), never the pair space.
748
+ */
749
+ const insertBounded = (list, k, dst, sim) => {
750
+ let at = list.length;
751
+ for (let index = 0; index < list.length; index += 1) {
752
+ const held = list[index];
753
+ if (sim > held.sim || sim === held.sim && dst < held.dst) {
754
+ at = index;
755
+ break;
756
+ }
757
+ }
758
+ if (at >= k) return;
759
+ list.splice(at, 0, {
760
+ dst,
761
+ sim
762
+ });
763
+ if (list.length > k) list.pop();
764
+ };
765
+ /** The final ordering every consumer sees: `sim` DESC, then `src` ASC, then `dst` ASC, then cap. */
766
+ const collectRanked = (bySource, limit) => {
767
+ const rows = [];
768
+ for (const [src, list] of bySource) for (const held of list) rows.push({
769
+ src,
770
+ dst: held.dst,
771
+ sim: held.sim
772
+ });
773
+ rows.sort((left, right) => {
774
+ if (left.sim !== right.sim) return left.sim < right.sim ? 1 : -1;
775
+ if (left.src !== right.src) return left.src < right.src ? -1 : 1;
776
+ return left.dst < right.dst ? -1 : left.dst > right.dst ? 1 : 0;
777
+ });
778
+ return rows.slice(0, limit);
779
+ };
780
+ /**
781
+ * Per-source top-`k` nearest neighbors above a similarity floor, over every unordered pair.
782
+ *
783
+ * Each pair's similarity is computed ONCE and offered to BOTH endpoints' neighborhoods, so the
784
+ * output can hold `(a, b)` and `(b, a)` — each is a fact about a different source's neighborhood,
785
+ * and a consumer folding pairs must dedup the mirror itself (dedup-merge does, with its `seen`
786
+ * set). A floor comparison a NaN similarity cannot pass keeps a vector carrying NaN bytes out of
787
+ * every neighborhood rather than poisoning an ordering.
788
+ */
789
+ const topNeighborPairs = (vectors, options) => {
790
+ const norms = vectors.map((entry) => sqrtNorm(entry.vec));
791
+ const bySource = /* @__PURE__ */ new Map();
792
+ for (const entry of vectors) bySource.set(entry.key, []);
793
+ for (let i = 0; i < vectors.length; i += 1) {
794
+ const left = vectors[i];
795
+ const leftNorm = norms[i];
796
+ const leftList = bySource.get(left.key);
797
+ for (let j = i + 1; j < vectors.length; j += 1) {
798
+ const right = vectors[j];
799
+ const sim = pairSimilarity(left.vec, leftNorm, right.vec, norms[j]);
800
+ if (!(sim >= options.floor)) continue;
801
+ insertBounded(leftList, options.perSourceK, right.key, sim);
802
+ insertBounded(bySource.get(right.key), options.perSourceK, left.key, sim);
803
+ }
804
+ }
805
+ return collectRanked(bySource, options.limit);
806
+ };
807
+ /**
808
+ * Rank an ENUMERATED pair set: similarity, floor, per-source top-`k`, final ordering, cap.
809
+ *
810
+ * This is the shape for a consumer whose candidate pairs come from a selective predicate — the
811
+ * conflict scan's shared-entity join — rather than from the whole pair space. The predicate runs
812
+ * BEFORE ranking, exactly as a `WHERE` inside the ranking CTE would, so per-source top-`k` is
813
+ * computed over passing pairs only. A pair naming a key with no vector contributes nothing, the
814
+ * same outcome the SQL join's missing-embedding row produces.
815
+ */
816
+ const rankCandidatePairs = (pairs, vectors, options) => {
817
+ const byKey = /* @__PURE__ */ new Map();
818
+ for (const entry of vectors) byKey.set(entry.key, {
819
+ vec: entry.vec,
820
+ norm: sqrtNorm(entry.vec)
821
+ });
822
+ const bySource = /* @__PURE__ */ new Map();
823
+ for (const pair of pairs) {
824
+ const left = byKey.get(pair.src);
825
+ const right = byKey.get(pair.dst);
826
+ if (left === void 0 || right === void 0) continue;
827
+ const sim = pairSimilarity(left.vec, left.norm, right.vec, right.norm);
828
+ if (!(sim >= options.floor)) continue;
829
+ let list = bySource.get(pair.src);
830
+ if (list === void 0) {
831
+ list = [];
832
+ bySource.set(pair.src, list);
833
+ }
834
+ insertBounded(list, options.perSourceK, pair.dst, sim);
835
+ }
836
+ return collectRanked(bySource, options.limit);
837
+ };
838
+
647
839
  //#endregion
648
840
  //#region packages/domain/dist/reinforce.js
649
841
  /** The signal a reinforcement carries. `negative` is what drives the outcome EWMA down. */
@@ -2181,8 +2373,18 @@ const META_PREFIX = "memhtml-";
2181
2373
  /**
2182
2374
  * Meta keys that may appear more than once. Each value is its own `<meta>` element
2183
2375
  * rather than a comma-joined string, so correcting one tag is a one-line diff.
2376
+ *
2377
+ * `memhtml-alias` is the third: a person file declares the other names the same person is recorded
2378
+ * under, so `laith al-saadoon` can state that `laith` is them. Sleep's entity resolution reads it as
2379
+ * EVIDENCE and auto-merges an alias-backed pair whatever the character distance says, which no
2380
+ * string similarity can supply — `laith` against `laith al-saadoon` scores 0.476, below even the
2381
+ * review band. One meta per alias, for the same one-line-diff reason.
2184
2382
  */
2185
- const REPEATABLE_META = ["memhtml-entity", "memhtml-tag"];
2383
+ const REPEATABLE_META = [
2384
+ "memhtml-entity",
2385
+ "memhtml-tag",
2386
+ "memhtml-alias"
2387
+ ];
2186
2388
  /** True when a metadata name may legitimately appear more than once in one head. */
2187
2389
  const isRepeatableMeta = (name) => REPEATABLE_META.includes(name);
2188
2390
  /**
@@ -2224,7 +2426,8 @@ const META_ORDER = [
2224
2426
  "memhtml-task-status",
2225
2427
  "memhtml-due",
2226
2428
  "memhtml-entity",
2227
- "memhtml-tag"
2429
+ "memhtml-tag",
2430
+ "memhtml-alias"
2228
2431
  ];
2229
2432
  /** True when a `memhtml-`-prefixed metadata name is in the closed vocabulary. */
2230
2433
  const isMemoryMetaName = (name) => META_ORDER.includes(name);
@@ -3283,6 +3486,17 @@ const MemoryDoc = Schema.Struct({
3283
3486
  entities: Schema.Array(Schema.String),
3284
3487
  /** `memhtml-tag` values as authored, in document order. Open vocabulary. */
3285
3488
  tags: Schema.Array(Schema.String),
3489
+ /**
3490
+ * `memhtml-alias` values as authored, in document order. The other names this file's subject is
3491
+ * recorded under — a person file's declaration that `laith` and `l.alsaadoon` are the same person
3492
+ * as its own `person:` entity.
3493
+ *
3494
+ * Read as EVIDENCE by sleep's entity resolution, which auto-merges an alias-backed pair whatever
3495
+ * the name similarity says. That is why the field is a bare string list rather than a parsed
3496
+ * `type:name`: the declaring file's own `memhtml-entity` already carries the type, and an alias
3497
+ * that restated it could disagree with it.
3498
+ */
3499
+ aliases: Schema.Array(Schema.String),
3286
3500
  links: Schema.Array(MemoryLink),
3287
3501
  article: ArticleExtractions,
3288
3502
  /**
@@ -3694,6 +3908,7 @@ const parseMemory = (html) => Effect.suspend(() => {
3694
3908
  metas: metaResult.metas,
3695
3909
  entities: repeated(metas, "memhtml-entity"),
3696
3910
  tags: repeated(metas, "memhtml-tag"),
3911
+ aliases: repeated(metas, "memhtml-alias"),
3697
3912
  links: readLinks(document),
3698
3913
  article: readArticle(article),
3699
3914
  warnings: structural.warnings
@@ -3774,7 +3989,11 @@ const metaPairs = (doc) => {
3774
3989
  put("memhtml-needs-revision", metas.needsRevision);
3775
3990
  put("memhtml-task-status", metas.taskStatus);
3776
3991
  put("memhtml-due", metas.dueAt);
3777
- const repeatables = /* @__PURE__ */ new Map([["memhtml-entity", doc.entities], ["memhtml-tag", doc.tags]]);
3992
+ const repeatables = /* @__PURE__ */ new Map([
3993
+ ["memhtml-entity", doc.entities],
3994
+ ["memhtml-tag", doc.tags],
3995
+ ["memhtml-alias", doc.aliases]
3996
+ ]);
3778
3997
  const pairs = [];
3779
3998
  for (const name of META_ORDER) {
3780
3999
  if (isRepeatableMeta(name)) {
@@ -3906,6 +4125,7 @@ const newMemoryDoc = (input) => {
3906
4125
  },
3907
4126
  entities: input.entities ?? [],
3908
4127
  tags: input.tags ?? [],
4128
+ aliases: input.aliases ?? [],
3909
4129
  links: input.links ?? [],
3910
4130
  article: {
3911
4131
  html,
@@ -5487,19 +5707,6 @@ const packSentences = (units, maxChars) => {
5487
5707
  */
5488
5708
  const BUSY_TIMEOUT_MS = 5e3;
5489
5709
  /**
5490
- * A `Float32Array` over stored bytes, copying only when it must.
5491
- *
5492
- * `Float32Array` requires a 4-byte-aligned `byteOffset`, and a driver row's `Uint8Array` may be
5493
- * a view into a pooled buffer at any offset. Viewing in place is the common case and costs
5494
- * nothing. A misaligned or ragged blob is copied rather than rejected, because the vector arm's
5495
- * job is to rank and a throw here would fail a whole search over one row.
5496
- */
5497
- const float32View = (bytes) => {
5498
- if (bytes.byteLength === 0 || bytes.byteLength % 4 !== 0) return void 0;
5499
- const aligned = bytes.byteOffset % 4 === 0 ? bytes : Uint8Array.from(bytes);
5500
- return new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);
5501
- };
5502
- /**
5503
5710
  * Register `vector_distance_cos(a, b)`, the cosine distance over two float32 blobs.
5504
5711
  *
5505
5712
  * SQLite ships no vector functions, so the vector retrieval arm's distance is this. It calls
@@ -5513,6 +5720,12 @@ const float32View = (bytes) => {
5513
5720
  * vectors at top-40 (probed 2026-08-12 on node 24.19.0), against a Bedrock query-embedding round
5514
5721
  * trip of a few hundred milliseconds that every vector search pays first. An approximate index buys
5515
5722
  * nothing until the corpus is an order of magnitude larger.
5723
+ *
5724
+ * That measurement is the 1×n shape — one bound query vector against the table, n calls, n blob
5725
+ * copies — and it is the ONLY shape this function serves. Each invocation materializes a fresh
5726
+ * `Uint8Array` per blob argument, so an n×n consumer pays the corpus re-copied n times (probed
5727
+ * 2026-08-18, issue #40: 8.45M calls and an OOM at n = 2,907). The sleep pair scans decode once
5728
+ * and rank in `@memhtml/domain`'s neighbors module instead.
5516
5729
  */
5517
5730
  const registerVectorDistance = (db) => {
5518
5731
  db.function("vector_distance_cos", { deterministic: true }, (a, b) => {
@@ -8169,7 +8382,9 @@ const clampTokens = (requested) => Math.min(requested ?? 16384, MAX_TOKENS_CEILI
8169
8382
  * `emit` call, and that is the whole structured-output mechanism.
8170
8383
  *
8171
8384
  * `system` is omitted rather than sent empty, because an empty system block is a distinct
8172
- * (and rejected) input from no system block at all.
8385
+ * (and rejected) input from no system block at all. An omitted system also has nothing to cache, so
8386
+ * `cacheSystem` over an absent or empty system emits no `system` key at all instead of an empty
8387
+ * cached block.
8173
8388
  */
8174
8389
  const buildInvokeBody = (key, prompt, options, tool) => {
8175
8390
  const body = {
@@ -8181,7 +8396,11 @@ const buildInvokeBody = (key, prompt, options, tool) => {
8181
8396
  }],
8182
8397
  output_config: { effort: options.effort }
8183
8398
  };
8184
- if (options.system !== void 0 && options.system.length > 0) body.system = options.system;
8399
+ if (options.system !== void 0 && options.system.length > 0) body.system = options.cacheSystem === true ? [{
8400
+ type: "text",
8401
+ text: options.system,
8402
+ cache_control: { type: "ephemeral" }
8403
+ }] : options.system;
8185
8404
  const thinking = thinkingFor(key);
8186
8405
  if (thinking !== null) body.thinking = thinking;
8187
8406
  if (tool !== void 0) {
@@ -8273,7 +8492,8 @@ const makeModelClient = (client) => {
8273
8492
  const { parsed } = yield* invoke(request.modelKey, request.prompt, {
8274
8493
  system: request.system,
8275
8494
  maxTokens: request.maxTokens,
8276
- effort: request.effort
8495
+ effort: request.effort,
8496
+ cacheSystem: request.cacheSystem
8277
8497
  }, {
8278
8498
  inputSchema: request.inputSchema ?? toInputSchema(request.schema),
8279
8499
  ...request.toolDescription === void 0 ? {} : { description: request.toolDescription }
@@ -8562,6 +8782,163 @@ var DiscriminationFailed = class {
8562
8782
  */
8563
8783
  const discriminationGate = (options = {}) => runDiscrimination(options).pipe(Effect.flatMap((outcome) => outcome.passed ? Effect.succeed(outcome) : Effect.fail(new DiscriminationFailed(outcome))));
8564
8784
 
8785
+ //#endregion
8786
+ //#region packages/sleep/dist/batch.js
8787
+ /**
8788
+ * Mint `m1`..`mN` over `items` in the order given, and index each key back to its item.
8789
+ *
8790
+ * The keys carry no information beyond position, which is the point. A key that encoded a path or a
8791
+ * title would let a model answer with a target it inferred rather than one it was offered, and this
8792
+ * corpus stores instructions, so a member's own text can read as a directive about naming.
8793
+ *
8794
+ * `charBudget` slices each text after `textOf` builds it. The budget is per member and is applied
8795
+ * here so every phase slices at the same boundary, where the text stops being a row and becomes a
8796
+ * prompt.
8797
+ */
8798
+ const keyMembers = (items, textOf, options) => {
8799
+ const budget = options?.charBudget;
8800
+ const keyed = [];
8801
+ const itemForKey = /* @__PURE__ */ new Map();
8802
+ for (const [offset, item] of items.entries()) {
8803
+ const key = `m${offset + 1}`;
8804
+ const text = textOf(item);
8805
+ keyed.push({
8806
+ key,
8807
+ text: budget === void 0 ? text : text.slice(0, budget)
8808
+ });
8809
+ itemForKey.set(key, item);
8810
+ }
8811
+ return {
8812
+ keyed,
8813
+ itemForKey
8814
+ };
8815
+ };
8816
+ /**
8817
+ * Resolve the keys a model named back to items: unknown keys are dropped, repeats collapse.
8818
+ *
8819
+ * A key the batch never offered is a member the model invented, and every phase on this kernel turns
8820
+ * a named member into a write, so an unresolvable key must not reach that write. Dropping it leaves
8821
+ * the corresponding file untouched, which is the safe outcome for every one of the five phases.
8822
+ *
8823
+ * The result keeps the order the model named the keys in, and a key named twice appears once.
8824
+ * De-duplication is on the KEY rather than on the resolved item, so the count a phase gates on
8825
+ * ("at least two members absorbed") counts distinct offered members.
8826
+ */
8827
+ const resolveKeys = (batch, keys) => [...new Set(keys)].flatMap((key) => {
8828
+ const item = batch.itemForKey.get(key);
8829
+ return item === void 0 ? [] : [item];
8830
+ });
8831
+ /**
8832
+ * Slice each pre-sorted group into batches of at most `maxMembers`, dropping any batch that falls
8833
+ * below `minMembers`.
8834
+ *
8835
+ * The groups arrive in the order the caller wants them called in, and each group's members arrive in
8836
+ * the caller's own stable order. This walks both in that order, so the boundaries are reproducible.
8837
+ *
8838
+ * `minMembers` defaults to 1, which keeps every slice. A phase whose question is meaningless for a
8839
+ * lone member raises it: compress passes 2, because folding one memory into a "canonical" rewrites it
8840
+ * under a new path and archives the original for no gain.
8841
+ */
8842
+ const assembleBatches = (groups, options) => {
8843
+ const floor = options.minMembers ?? 1;
8844
+ const batches = [];
8845
+ for (const group of groups) for (let at = 0; at < group.length; at += options.maxMembers) {
8846
+ const slice = group.slice(at, at + options.maxMembers);
8847
+ if (slice.length >= floor) batches.push(slice);
8848
+ }
8849
+ return batches;
8850
+ };
8851
+ /**
8852
+ * Pack whole groups into shared batches, bounded by a member count and a character budget together.
8853
+ *
8854
+ * For a phase whose groups are mostly tiny. `dedup-merge` works over connected components of the
8855
+ * near-duplicate graph, where a typical component is a pair, so one call per component would spend a
8856
+ * model call on two memories. Ten or twenty components in one call cost one.
8857
+ *
8858
+ * Group boundaries survive into the result, because they are evidence. Two members in different
8859
+ * components are known NOT to be near-duplicates, and a prompt that flattened the pack into one list
8860
+ * would ask the model to rediscover that.
8861
+ *
8862
+ * A group longer than `maxMembers` is sliced on the same stride {@link assembleBatches} uses, so no
8863
+ * returned batch breaches either cap. Packing is greedy in the order given and closes a batch when the
8864
+ * next unit would breach a cap, which makes the packing a function of the input order. Filtering out
8865
+ * groups the phase does not want called (a singleton component, for dedup) is the caller's step,
8866
+ * because a floor applied after packing would measure the pack instead of the group.
8867
+ */
8868
+ const packGroups = (groups, options) => {
8869
+ const batches = [];
8870
+ let current = [];
8871
+ let members = 0;
8872
+ let chars = 0;
8873
+ const close = () => {
8874
+ if (current.length > 0) batches.push(current);
8875
+ current = [];
8876
+ members = 0;
8877
+ chars = 0;
8878
+ };
8879
+ for (const group of groups) {
8880
+ if (group.length === 0) continue;
8881
+ const units = group.length > options.maxMembers ? assembleBatches([group], { maxMembers: options.maxMembers }) : [group];
8882
+ for (const unit of units) {
8883
+ const cost = unit.reduce((total, item) => total + options.charsOf(item), 0);
8884
+ const breaches = members + unit.length > options.maxMembers || chars + cost > options.maxChars;
8885
+ if (current.length > 0 && breaches) close();
8886
+ current.push(unit);
8887
+ members += unit.length;
8888
+ chars += cost;
8889
+ }
8890
+ }
8891
+ close();
8892
+ return batches;
8893
+ };
8894
+ /**
8895
+ * The numbered member list as one prompt block: each member wrapped as data under `<label>_<key>`,
8896
+ * blocks separated by a blank line.
8897
+ *
8898
+ * Every member goes through `wrapAsData`, which is the prompt-injection boundary. This corpus stores
8899
+ * instructions, so a procedural memory about a deploy step reads exactly like a directive to the
8900
+ * model, and un-delimited member text in a user turn would be an injection surface the system built
8901
+ * for itself.
8902
+ */
8903
+ const memberList = (keyed, options) => {
8904
+ const label = options?.label ?? "member";
8905
+ return keyed.map((member) => wrapAsData(`${label}_${member.key}`, member.text)).join("\n\n");
8906
+ };
8907
+ /**
8908
+ * A batch's user turn: the member list first, the instruction that closes it last.
8909
+ *
8910
+ * The phase's own instruction sentence is the tail rather than the head, matching the order every
8911
+ * copy of this pattern already used. The stable half of a batch prompt is `system` and the tool
8912
+ * schema, which {@link batchCall} marks cacheable; the user turn is new bytes on every call whatever
8913
+ * order its parts sit in.
8914
+ */
8915
+ const batchPrompt = (keyed, instruction, options) => `${memberList(keyed, options)}\n\n${instruction}`;
8916
+ /**
8917
+ * Run one model call in isolation: a failure becomes `undefined` and a counted skip.
8918
+ *
8919
+ * This is the per-item posture the packet's §4 requires, expressed with `Effect.result` because
8920
+ * `Effect.either` does not exist in this beta. One violation skips its item and leaves the phase
8921
+ * running. A night that judged 199 pairs and lost the 200th to a malformed tool payload has done 199
8922
+ * pairs of work, and failing the phase would throw all of it away.
8923
+ */
8924
+ const isolate = (label, call) => Effect.gen(function* () {
8925
+ const outcome = yield* Effect.result(call);
8926
+ if (Result.isSuccess(outcome)) return outcome.success;
8927
+ yield* Effect.logWarning(`sleep.llm ${label} skipped: ${outcome.failure.reason}`);
8928
+ });
8929
+ /**
8930
+ * Run one batch's model call: prompt-cache the stable prefix, and isolate the failure.
8931
+ *
8932
+ * `cacheSystem` is set here instead of at each call site, because every phase on this kernel has the
8933
+ * same shape: one system prompt and one tool schema repeated across every batch of the night, with
8934
+ * only the member list changing. A phase that forgot the flag would re-bill its whole prefix on every
8935
+ * batch, and the omission would be invisible in the phase's output.
8936
+ */
8937
+ const batchCall = (model, label, request) => isolate(label, model.generateObject({
8938
+ ...request,
8939
+ cacheSystem: true
8940
+ }));
8941
+
8565
8942
  //#endregion
8566
8943
  //#region packages/sleep/dist/contract.js
8567
8944
  /**
@@ -8585,7 +8962,7 @@ const SLEEP_PHASES = [
8585
8962
  "entity-resolution",
8586
8963
  "person-links",
8587
8964
  "relationship-mining",
8588
- "conflict-detection",
8965
+ "edge-typing",
8589
8966
  "confidence-decay",
8590
8967
  "arc-synthesis",
8591
8968
  "retention-triage",
@@ -8612,6 +8989,10 @@ const phaseIndexOf = (phase) => SLEEP_PHASES.indexOf(phase) + 1;
8612
8989
  * `dedup-merge` is the one hard prerequisite, for `compress` and `retention-triage`: both operate
8613
8990
  * on the post-merge set, and running them over a corpus that still holds the duplicates would
8614
8991
  * compress a near-duplicate pair into a canonical while a merge later archives one of its members.
8992
+ *
8993
+ * That is why `dedup-merge` isolates each of its model calls instead of failing on one. It batches
8994
+ * components and a batch whose call comes back malformed is counted and skipped, so a single bad tool
8995
+ * payload cannot take two later phases down with it.
8615
8996
  */
8616
8997
  const HARD_PREREQUISITES = [["dedup-merge", "compress"], ["dedup-merge", "retention-triage"]];
8617
8998
  /** The phases blocked by `phase` failing. */
@@ -8849,7 +9230,7 @@ const datePlusDays = (date, days) => {
8849
9230
  //#endregion
8850
9231
  //#region packages/sleep/dist/env.js
8851
9232
  /**
8852
- * Model assignments per LLM phase: the cheap judge for stance, the strong one for synthesis.
9233
+ * Model assignments per LLM phase: the cheap judge for classification, the strong one for synthesis.
8853
9234
  *
8854
9235
  * `trace-consolidation` names `opus-5` and does not thereby choose it. The consolidator is an eve
8855
9236
  * agent that pins its own model in `apps/consolidator/agent/agent.ts`, and this map cannot reach that
@@ -8858,7 +9239,22 @@ const datePlusDays = (date, days) => {
8858
9239
  * the Bedrock global endpoint, high reasoning effort, no cost ceiling.)
8859
9240
  */
8860
9241
  const DEFAULT_MODELS = {
8861
- "conflict-detection": "sonnet-5",
9242
+ /**
9243
+ * `dedup-merge` names sonnet for the same reason the edge-typing judge does: the question is a
9244
+ * classification over text the model is shown, not a synthesis it has to write. It partitions a
9245
+ * component into "these are the same memory" groups, and every consequence of that answer — which
9246
+ * file survives, whether the pair diverges, whether either path is already claimed — is decided by
9247
+ * code afterwards. The strong model is spent where prose gets written.
9248
+ */
9249
+ "dedup-merge": "sonnet-5",
9250
+ /**
9251
+ * Sonnet, and one or two calls a night: the whole of one entity type's name list goes in one call.
9252
+ * The question is a partition over short strings with their evidence inline, not a synthesis, so
9253
+ * the strong model would buy nothing the deterministic floors around the answer do not already
9254
+ * supply.
9255
+ */
9256
+ "entity-resolution": "sonnet-5",
9257
+ "edge-typing": "sonnet-5",
8862
9258
  "arc-synthesis": "opus-5",
8863
9259
  compress: "sonnet-5",
8864
9260
  "trace-consolidation": "opus-5"
@@ -8875,7 +9271,7 @@ const emptyOutcome = (counts = {}) => ({
8875
9271
  //#endregion
8876
9272
  //#region packages/sleep/dist/llm.js
8877
9273
  /**
8878
- * The structured-output schemas the four LLM phases share, and the per-item isolation wrapper.
9274
+ * The structured-output schemas the four LLM phases share, and their prompts.
8879
9275
  *
8880
9276
  * Two rules govern everything here, both of them found by hitting the failure:
8881
9277
  *
@@ -8889,32 +9285,87 @@ const emptyOutcome = (counts = {}) => ({
8889
9285
  * itself. The prompts are also blind by construction. None names a path, a score, or a decision the
8890
9286
  * caller has already made, so the model cannot agree with a verdict it was shown.
8891
9287
  */
8892
- /** A stance judgment over one candidate pair. What conflict-detection asks for. */
8893
- const StanceVerdict = Schema.Literals([
8894
- "contradicts",
8895
- "entails",
8896
- "neutral"
8897
- ]);
8898
- const StanceJudgment = Schema.Struct({
8899
- verdict: StanceVerdict,
8900
- /** Unitless in `[0, 1]`. The assertion gate is deterministic and reads this, not the prose. */
9288
+ /**
9289
+ * The rels edge typing may propose, plus `none`. A closed subset of `MEMORY_RELS`, and the
9290
+ * omissions are deliberate.
9291
+ *
9292
+ * `supersedes` is out because it is a one-way door on stored belief: it says one memory REPLACES
9293
+ * another, which is dedup-merge's and compress's business and rides with an archive. `relates_to`
9294
+ * is out because it is what the pair already carries as a derived edge, so proposing it is a no-op
9295
+ * with a model call attached. `laterally_related` is out for the same reason one notch weaker.
9296
+ * A pair the model cannot type answers `none` and stays a mined suspicion.
9297
+ */
9298
+ const EDGE_TYPED_RELS = [
9299
+ "caused_by",
9300
+ "leads_to",
9301
+ "example_of",
9302
+ "supports",
9303
+ "part_of",
9304
+ "contradicts"
9305
+ ];
9306
+ /**
9307
+ * The five DIRECTIONAL rels: the ones whose meaning depends on which endpoint is the subject.
9308
+ *
9309
+ * `contradicts` is excluded and that is the whole distinction this list draws. A contradiction is
9310
+ * symmetric — a reader arriving at either file must see it — so it is promoted into BOTH files and
9311
+ * its `direction` field is ignored. A directional rel is promoted into ONE file, the subject's, and
9312
+ * the direction decides which one.
9313
+ */
9314
+ const EDGE_DIRECTIONAL_RELS = [
9315
+ "caused_by",
9316
+ "leads_to",
9317
+ "example_of",
9318
+ "supports",
9319
+ "part_of"
9320
+ ];
9321
+ /** The rel vocabulary the model answers over: the typed rels plus the refusal. */
9322
+ const EdgeVerdictRel = Schema.Literals([...EDGE_TYPED_RELS, "none"]);
9323
+ /**
9324
+ * Which endpoint is the rel's subject. Meaningful only for {@link EDGE_DIRECTIONAL_RELS}.
9325
+ *
9326
+ * Required rather than optional, because a model allowed to omit it would omit it on the rels where
9327
+ * it matters. `contradicts` and `none` carry a value the phase does not read.
9328
+ */
9329
+ const EdgeDirection = Schema.Literals(["src_to_dst", "dst_to_src"]);
9330
+ /** One pair's verdict, under the opaque key the pair was offered as. */
9331
+ const EdgeVerdict = Schema.Struct({
9332
+ /** The offered key, e.g. `m3`. A key the batch never held resolves to nothing and is dropped. */
9333
+ pairKey: Schema.String,
9334
+ rel: EdgeVerdictRel,
9335
+ direction: EdgeDirection,
9336
+ /** Unitless in `[0, 1]`. The promotion gate is deterministic and reads this, not the prose. */
8901
9337
  confidence: Schema.Finite.check(Schema.isBetween({
8902
9338
  minimum: 0,
8903
9339
  maximum: 1
8904
9340
  })),
8905
- /** One or two sentences naming the specific claims that conflict, or why they are compatible. */
8906
- rationale: Schema.String
9341
+ /** One or two sentences naming the specific claims that carry the rel. Optional: `none` has none. */
9342
+ rationale: Schema.optional(Schema.String)
8907
9343
  });
9344
+ /** One batch's whole answer: a verdict LIST, never one call per pair. */
9345
+ const EdgeTyping = Schema.Struct({ verdicts: Schema.Array(EdgeVerdict) });
9346
+ /**
9347
+ * The confidence a verdict must clear before the phase writes anything.
9348
+ *
9349
+ * A `contradicts` feeds a retention penalty that can eventually evict a memory, so a false one is
9350
+ * worse than a missed one; the floor and the `detections >= 2` corroboration gate are two
9351
+ * independent guards on that one-way door. A directional rel is milder but still an authored edge in
9352
+ * a file a human reads, so it clears the same floor. One number, because a second one would be a
9353
+ * knob nobody could say the meaning of.
9354
+ */
9355
+ const EDGE_CONFIDENCE_FLOOR = .7;
9356
+ /** True when a verdict is a proposal at all, above the floor. Computed here, never by the model. */
9357
+ const assertsEdge = (verdict) => verdict.rel !== "none" && verdict.confidence >= .7;
9358
+ /** True when a verdict earns a `contradicts` edge. The corroboration gate's precondition. */
9359
+ const assertsContradiction = (verdict) => verdict.rel === "contradicts" && verdict.confidence >= .7;
8908
9360
  /**
8909
- * The confidence a `contradicts` verdict must clear before the phase asserts an edge.
9361
+ * True when a rel's meaning depends on which endpoint is its subject.
8910
9362
  *
8911
- * A detected contradiction feeds a retention penalty that can eventually evict a memory, so a
8912
- * false `contradicts` is worse than a missed one. The floor and the `detections >= 2`
8913
- * corroboration gate are two independent guards on the same one-way door.
9363
+ * A NARROWING predicate, not a boolean, so the caller's `rel` becomes an `EdgeDirectionalRel` and
9364
+ * therefore a `MemoryRel` inside the branch that writes a `<link>`. A plain boolean would leave the
9365
+ * write site casting `"none"`-inclusive union to `EdgeRel`, which is the cast that would silently
9366
+ * survive someone adding a non-rel member to the verdict vocabulary.
8914
9367
  */
8915
- const STANCE_CONFIDENCE_FLOOR = .7;
8916
- /** True when a judgment earns a `contradicts` edge. Computed here, not decided by the model. */
8917
- const assertsContradiction = (judgment) => judgment.verdict === "contradicts" && judgment.confidence >= .7;
9368
+ const isDirectionalRel = (rel) => EDGE_DIRECTIONAL_RELS.includes(rel);
8918
9369
  /** One arc the triage call proposes to act on. */
8919
9370
  const ArcAction = Schema.Literals([
8920
9371
  "update",
@@ -8957,21 +9408,98 @@ const CompressSynthesis = Schema.Struct({
8957
9408
  */
8958
9409
  absorbedKeys: Schema.Array(Schema.String)
8959
9410
  });
8960
- /** The stance judge's system prompt. */
8961
- const STANCE_SYSTEM = `You are a natural-language-inference stance judge for an AI agent's long-term memory system.
8962
- You are given two memories, A and B, that are embedding-near and about the same entity or topic.
8963
- Decide the stance of B relative to A in one pass:
9411
+ /** One proposed identity cluster over the entity names a batch offered. */
9412
+ const EntityCluster = Schema.Struct({
9413
+ /**
9414
+ * The member key the cluster's canonical name was offered under. The phase re-derives the canonical
9415
+ * from ITS OWN weight-then-lexicographic rule, so this names which member the model considers the
9416
+ * fullest form and never which file gets rewritten. A key the batch did not offer resolves to
9417
+ * nothing and drops the cluster.
9418
+ */
9419
+ canonicalKey: Schema.String,
9420
+ /**
9421
+ * Every member key in the cluster, canonical included. A cluster of one is a valid answer meaning
9422
+ * "this name stands alone", and it produces no merge.
9423
+ */
9424
+ memberKeys: Schema.Array(Schema.String),
9425
+ /** Unitless in `[0, 1]`. The merge gate is deterministic and reads this, not the prose. */
9426
+ confidence: Schema.Finite.check(Schema.isBetween({
9427
+ minimum: 0,
9428
+ maximum: 1
9429
+ })),
9430
+ /** One sentence naming what makes these one subject: a declared alias, a shared neighborhood. */
9431
+ evidence: Schema.String
9432
+ });
9433
+ /**
9434
+ * The whole clustering answer for one batch: a partition of the offered names.
9435
+ *
9436
+ * `clusters: []` is a refusal and a valid answer. A model that cannot tell two short names apart must
9437
+ * be able to say so, because the alternative — inventing a cluster to fill the field — reaches a
9438
+ * permanent rewrite of stored identity.
9439
+ */
9440
+ const EntityClustering = Schema.Struct({ clusters: Schema.Array(EntityCluster) });
9441
+ /**
9442
+ * One merge group: the members the model says are the same memory, by their offered keys.
9443
+ *
9444
+ * **No canonical field, deliberately.** The keeper is the OLDER file, decided from corpus order in
9445
+ * the phase, and a model-chosen canonical would be a model-chosen write target: the file that
9446
+ * survives and the files that get archived. The model's whole job here is the partition — which
9447
+ * members are one memory — and orientation is arithmetic over `created_at` that needs no judgment.
9448
+ *
9449
+ * A group of fewer than two keys is meaningless and the phase drops it. That is the shape a model
9450
+ * produces when it wants to say "this one is on its own", which is a valid answer.
9451
+ */
9452
+ const MergeGroup = Schema.Struct({ memberKeys: Schema.Array(Schema.String) });
9453
+ /**
9454
+ * The dedup partition for one packed batch: every merge group the model found, across every
9455
+ * component in the batch.
9456
+ *
9457
+ * **The groups are FLAT, not nested per component, and the phase re-derives which component each one
9458
+ * came from.** A nested answer would need the model to keep a component index aligned with its
9459
+ * groups, which is bookkeeping a model gets wrong under load, and a mis-aligned index would attach a
9460
+ * group to the wrong component's files. A flat list of member keys carries the same information,
9461
+ * because a key already identifies its member and therefore its component. So the phase can check
9462
+ * containment itself instead of trusting a label.
9463
+ *
9464
+ * `groups: []` is a full refusal: every member stays where it is, which is the safe outcome and the
9465
+ * behavior a night with no model already has.
9466
+ */
9467
+ const MergePartition = Schema.Struct({ groups: Schema.Array(MergeGroup) });
9468
+ /**
9469
+ * The edge-typing system prompt: the whole rel vocabulary, one pass, one answer per pair.
9470
+ *
9471
+ * The same conservative posture the per-pair stance judge carried, generalized. An unsure pair
9472
+ * answers `none` and keeps the machine-mined `relates_to` it already has, which costs the corpus
9473
+ * nothing; a wrong `contradicts` starts a memory down the eviction path and a wrong directional rel
9474
+ * writes a claim about causality into a file a human reads.
9475
+ */
9476
+ const EDGE_TYPING_SYSTEM = `You type relationships between memories in an AI agent's long-term memory system. You are given a
9477
+ NUMBERED LIST of candidate pairs. Each pair holds two memories, src and dst, that are embedding-near
9478
+ or share an entity. Return ONE verdict per pair, naming the pair by the key it was offered under.
9479
+
9480
+ Choose the rel that holds between the two memories:
8964
9481
 
8965
- - contradicts: A and B make claims about the same thing that CANNOT both be true at the same time
8966
- (negation, opposite outcomes, mutually exclusive values).
8967
- - entails: B restates, paraphrases, or is fully implied by A redundant, not conflicting.
8968
- - neutral: A and B are about the same entity but make compatible, complementary, or simply
8969
- unrelated claims that can both hold.
9482
+ - caused_by: one memory's fact is the CAUSE of the other's. The subject is the effect.
9483
+ - leads_to: one memory's fact leads to, triggers, or produces the other's. The subject is the cause.
9484
+ - example_of: one memory is a concrete instance of the other's general claim. The subject is the instance.
9485
+ - supports: one memory is evidence FOR the other's claim, without restating it. The subject is the evidence.
9486
+ - part_of: one memory is a component, step, or subtopic of the other's larger whole. The subject is the part.
9487
+ - contradicts: the two make claims about the same thing that CANNOT both be true at the same time
9488
+ (negation, opposite outcomes, mutually exclusive values). SYMMETRIC: direction is ignored.
9489
+ - none: the two are merely about the same topic, restate each other, or carry no relationship you can
9490
+ name from the text. This is the correct answer whenever you are unsure.
8970
9491
 
8971
- Be conservative. A detected contradiction feeds a retention penalty that can eventually evict a
8972
- memory, so when the two claims COULD both be true different scope, different time, different
8973
- aspect answer neutral, not contradicts. Rate your confidence honestly and name the specific
8974
- conflicting or compatible claims in the rationale.`;
9492
+ direction says which endpoint is the rel's SUBJECT, as described per rel above: src_to_dst means src
9493
+ is the subject and dst the object; dst_to_src is the reverse. Answer src_to_dst on contradicts and
9494
+ none, where it is not read.
9495
+
9496
+ Be conservative. A verdict above the confidence floor is written into the memory files as an authored
9497
+ edge, and a false contradicts feeds a retention penalty that can eventually evict a memory. When the
9498
+ relationship COULD be something else — different scope, different time, mere topical adjacency —
9499
+ answer none. Two memories being similar is not a relationship. Rate confidence honestly and name the
9500
+ specific claims that carry the rel in the rationale.
9501
+
9502
+ Omitting a pair is allowed: an omitted pair is simply left untyped.`;
8975
9503
  /** The arc-triage system prompt: plan only, no content. */
8976
9504
  const ARC_TRIAGE_SYSTEM = `You triage behavioral arcs for an AI agent's long-term memory system. This is the planning pass:
8977
9505
  a second pass writes each arc's content, so your output is only the plan.
@@ -9010,34 +9538,151 @@ them, and each member you list in absorbedKeys is archived once the canonical is
9010
9538
  member you omit stays active, which is the safe outcome — never list one to be tidy.
9011
9539
  - If the members do not actually describe one thing, return an empty absorbedKeys and say so in the
9012
9540
  claim. Refusing to fold is a valid answer.`;
9541
+ /**
9542
+ * The entity-clustering system prompt: partition one type's names into subjects.
9543
+ *
9544
+ * Names the three evidence kinds a member block carries, because each one supports a different
9545
+ * inference and a model told only "decide if these are the same" would weigh the name string — the
9546
+ * signal that is measurably wrong here. `laith` against `laith al-saadoon` is 0.476 by character
9547
+ * overlap, below even the review band, while their memory centroids are near-identical.
9548
+ *
9549
+ * The refusal instruction is load-bearing rather than polite. A cluster this phase acts on rewrites
9550
+ * every `memhtml-entity` meta naming the alias across the corpus, and no later commit separates two
9551
+ * subjects whose memories were fused.
9552
+ */
9553
+ const ENTITY_CLUSTER_SYSTEM = `You group entity names for an AI agent's long-term memory system. Every name below is the same KIND of
9554
+ thing — all people, or all services, or all concepts — and several may be different ways of writing one
9555
+ subject. Partition them into subjects.
9556
+
9557
+ Each member gives you:
9558
+ - the name as the corpus records it, and how many active memories claim it;
9559
+ - up to three titles of memories claiming it, which say what that name is ABOUT;
9560
+ - its nearest neighbors by MEMORY CENTROID with a cosine — the centroid is the average of the vectors
9561
+ of every memory claiming the name, so a high cosine means two names are written about in the same
9562
+ terms. Two spellings of one person have near-identical centroids; two different services in one
9563
+ domain do not;
9564
+ - for a person, aliases DECLARED in that person's own file, which are an authoritative statement of
9565
+ identity rather than a guess.
9566
+
9567
+ Rules:
9568
+ - Every cluster lists canonicalKey plus every other member key that names the same subject. Set
9569
+ canonicalKey to the fullest, most complete form of the name.
9570
+ - A name that stands alone is its own cluster of one, or you may leave it out. Both mean "no merge".
9571
+ - Return an empty clusters list when nothing here is the same subject. Refusing to group is a valid
9572
+ and often correct answer.
9573
+ - Short name against long name is the case to look for: 'laith' and 'laith al-saadoon' are one person
9574
+ when the evidence supports it. Shared prefix is NOT: 'checkout-api' and 'payments-api' are two
9575
+ services, and 'metrics-api' and 'metrics-cli' are a service and a tool.
9576
+ - Never group two names because their strings are similar. Group them because the evidence says one
9577
+ subject, and cite that evidence.
9578
+ - Rate confidence honestly. A merge fuses two subjects' memories permanently and nothing separates
9579
+ them again, so answer low when you are unsure and the system will hold the merge back.`;
9580
+ /** The instruction that closes an entity-clustering batch's user turn, after the member list. */
9581
+ const ENTITY_CLUSTER_INSTRUCTION = "Partition these names into subjects. Return one cluster per subject with its canonicalKey, every member key it covers, your confidence, and the specific evidence that makes them one subject.";
9582
+ /**
9583
+ * The entity-clustering user turn for one batch: every member's evidence block under its offered key.
9584
+ *
9585
+ * `batchPrompt` from the kernel builds the list and appends the instruction, so the stable half of the
9586
+ * call is {@link ENTITY_CLUSTER_SYSTEM} plus the tool schema and only the member list is new bytes per
9587
+ * batch. Kept as a named function because the instruction belongs beside the system prompt.
9588
+ */
9589
+ const entityClusterPrompt = (members) => batchPrompt(members, ENTITY_CLUSTER_INSTRUCTION, { label: "entity" });
9590
+ /**
9591
+ * The dedup-partition system prompt.
9592
+ *
9593
+ * The stable prefix for every dedup call of a {@link batchCall} marks it cacheable, so only
9594
+ * the member list is new bytes per batch.
9595
+ *
9596
+ * It tells the model that a group is a claim about SAMENESS and nothing else. Every other decision
9597
+ * the fold needs — which file survives, whether the pair diverges in polarity or in a number,
9598
+ * whether either path is already spoken for — is made by code after the answer comes back, and the
9599
+ * prompt says so, because a model told it is choosing what gets deleted answers more conservatively
9600
+ * than the question deserves.
9601
+ */
9602
+ const DEDUP_SYSTEM = `You partition groups of near-duplicate memories for an AI agent's long-term memory system.
9603
+ Each component below holds memories that are near neighbors in vector space, or that state the same
9604
+ relation. Within EACH component, group the memories that are THE SAME MEMORY — one fact stored more
9605
+ than once, in different words.
9606
+
9607
+ - A group means: these state one fact, and keeping all of them stores it repeatedly. Two memories
9608
+ about the same topic that carry DIFFERENT facts are not a group.
9609
+ - Group only members of the SAME component. Members of different components are already known not to
9610
+ be near-duplicates.
9611
+ - A member belongs to at most one group. Leave a member out of every group when it is on its own.
9612
+ - Return groups: [] when no component holds a duplicate. Refusing to group is a valid answer and is
9613
+ the right one whenever you are unsure.
9614
+ - You are not choosing what to delete. Which memory survives a fold is decided from the memories'
9615
+ own dates afterwards, and a proposed group is still checked for contradicting claims, differing
9616
+ numbers, and differing product variants before anything is written. Answer only the question of
9617
+ sameness.`;
9013
9618
  /** One labelled corpus block, delimited so its prose cannot be read as an instruction. */
9014
9619
  const dataBlock = (label, text) => wrapAsData(label, text);
9015
9620
  /**
9016
- * The stance judge's user turn for one pair. Both texts are wrapped; neither carries a path.
9621
+ * One pair as the model sees it: both memories inline, delimited, under `src` and `dst` headings.
9622
+ *
9623
+ * This is a MEMBER's text, not a whole prompt: the kernel's `keyMembers` slices it to the phase's
9624
+ * per-member budget and `memberList` wraps the whole thing again under the pair's opaque key, so a
9625
+ * batch of thirty pairs is one nesting of sixty delimited memories. Neither half carries a path, a
9626
+ * cosine, or a prior verdict, so the model cannot infer which answer the caller is hoping for and
9627
+ * cannot recognize a pair it judged last night.
9628
+ *
9629
+ * The inner headings are plain lines rather than another `wrapAsData` block, because the outer wrap
9630
+ * already carries the "this is data" instruction and a second copy per member would repeat that
9631
+ * sentence sixty times in one prompt for no added guard.
9632
+ */
9633
+ const pairText = (srcText, dstText) => `src:\n${srcText}\n\ndst:\n${dstText}`;
9634
+ /** The instruction that closes an edge-typing batch's user turn, after the pair list. */
9635
+ const EDGE_TYPING_INSTRUCTION = "Type each pair above. Return one verdict per pair, naming the pair by its offered key, with the rel, the direction, your confidence, and a rationale naming the claims that carry the rel. Answer none whenever you are unsure.";
9636
+ /**
9637
+ * One edge-typing batch's user turn: every pair's two memories under its offered key, then the
9638
+ * instruction.
9017
9639
  *
9018
- * The prompt names no path, no cosine, and no prior verdict, so the model cannot infer which answer
9019
- * the caller is hoping for and cannot recognize a pair it judged last night.
9640
+ * `batchPrompt` from the kernel builds the list and appends the instruction, so the framing is the
9641
+ * same bytes compress's batches use. Kept as a named function because the instruction belongs beside
9642
+ * {@link EDGE_TYPING_SYSTEM}, which is the other half of what the model is told.
9020
9643
  */
9021
- const stancePrompt = (textA, textB) => `${dataBlock("memory_a", textA)}\n\n${dataBlock("memory_b", textB)}\n\nDo these two memories contradict each other? Give your verdict, your confidence, and a rationale naming the specific claims that conflict or why they are compatible.`;
9644
+ const edgeTypingPrompt = (pairs) => batchPrompt(pairs, EDGE_TYPING_INSTRUCTION, { label: "pair" });
9022
9645
  /** The arc-triage user turn: the live arcs and the recent evidence, both wrapped. */
9023
9646
  const arcTriagePrompt = (arcsText, evidenceText) => `${dataBlock("current_arcs", arcsText)}\n\n${dataBlock("evidence", evidenceText)}\n\nProduce a triage plan. Assign update or skip to every existing arc, and add a create entry for any genuinely new behavioral pattern the existing arcs do not cover.`;
9024
9647
  /** The arc-execute user turn for one arc. `current` is absent on a create. */
9025
9648
  const arcExecutePrompt = (input) => (input.current === void 0 ? `${dataBlock("new_arc_title", input.title)}\n\n` : `${dataBlock("existing_arc", input.current)}\n\n`) + `${dataBlock("evidence", input.evidenceText)}\n\n${dataBlock("triage_rationale", input.rationale)}\n\n` + (input.current === void 0 ? "Synthesize a new behavioral principal from this evidence." : "Update the arc to incorporate the new evidence, preserving existing knowledge that holds.");
9026
- /** The compress user turn for one batch: every member's text, wrapped, under its offered key. */
9027
- const compressPrompt = (members) => `${members.map((member) => dataBlock(`member_${member.key}`, member.text)).join("\n\n")}\n\nFold these memories into one canonical memory. List in absorbedKeys exactly the members whose content the canonical carries forward.`;
9649
+ /** The instruction that closes a compress batch's user turn, after the member list. */
9650
+ const COMPRESS_INSTRUCTION = "Fold these memories into one canonical memory. List in absorbedKeys exactly the members whose content the canonical carries forward.";
9028
9651
  /**
9029
- * Run one model call in isolation: a failure becomes `undefined` and a counted skip.
9652
+ * The compress user turn for one batch: every member's text, wrapped, under its offered key.
9030
9653
  *
9031
- * This is the per-item posture the packet's §4 requires, expressed with `Effect.result` because
9032
- * `Effect.either` does not exist in this beta. One violation skips its item and leaves
9033
- * the phase running. A night that judged 199 pairs and lost the 200th to a malformed tool payload has
9034
- * done 199 pairs of work, and failing the phase would throw all of it away.
9654
+ * `batchPrompt` from the kernel builds the member list and appends the instruction, so this produces
9655
+ * the same bytes it did when the framing was inline here. Kept as a named function because the
9656
+ * instruction belongs beside {@link COMPRESS_SYSTEM}, which is the other half of what the model is
9657
+ * told.
9035
9658
  */
9036
- const isolate = (label, call) => Effect.gen(function* () {
9037
- const outcome = yield* Effect.result(call);
9038
- if (Result.isSuccess(outcome)) return outcome.success;
9039
- yield* Effect.logWarning(`sleep.llm ${label} skipped: ${outcome.failure.reason}`);
9040
- });
9659
+ const compressPrompt = (members) => batchPrompt(members, COMPRESS_INSTRUCTION);
9660
+ /** The instruction that closes a dedup batch's user turn, after the components. */
9661
+ const DEDUP_INSTRUCTION = "Within each component above, group the members that are the same memory stated more than once. Name each group's members by the keys they were offered under. Return groups: [] if no component holds a duplicate.";
9662
+ /**
9663
+ * The dedup user turn for one packed batch: each component's members, wrapped, under a header that
9664
+ * names which keys sit in that component.
9665
+ *
9666
+ * **The component boundary is in the prompt because it is EVIDENCE.** Two members in different
9667
+ * components have already been measured as not near-duplicates, by a cosine floor and a frame-key
9668
+ * lookup, and a flat member list would throw that away and ask the model to rediscover it across the
9669
+ * whole batch. Packing ten components into one call is a cost decision; letting them blur into one
9670
+ * list would make it a correctness one.
9671
+ *
9672
+ * The headers are built from the OFFERED KEYS alone, never from a path or a title, so a header
9673
+ * carries nothing a member's own text could have chosen. `memberList` still wraps every member's
9674
+ * text, so the injection boundary is per member and the framing around it holds no corpus bytes.
9675
+ *
9676
+ * A containment claim in the prompt is not a containment guarantee: the phase re-checks that every
9677
+ * group the model returns sits inside ONE component, because the prompt is an instruction and the
9678
+ * post-pass is the enforcement.
9679
+ */
9680
+ const dedupPrompt = (components) => {
9681
+ return `${components.map((members, offset) => {
9682
+ const keys = members.map((member) => member.key).join(", ");
9683
+ return `component_${offset + 1} holds ${keys}.\n\n${memberList(members)}`;
9684
+ }).join("\n\n")}\n\n${DEDUP_INSTRUCTION}`;
9685
+ };
9041
9686
 
9042
9687
  //#endregion
9043
9688
  //#region packages/sleep/dist/sql.js
@@ -9054,8 +9699,8 @@ const isolate = (label, call) => Effect.gen(function* () {
9054
9699
  * The memory type no phase of a sleep cycle touches.
9055
9700
  *
9056
9701
  * A task is live working state, and every one of the fifteen phases is a judgment about REMEMBERED
9057
- * FACTS: decay says a claim is fading, dedup says two claims are one, conflict detection says two
9058
- * claims disagree, retention says a claim has stopped earning its place. None of those hold for
9702
+ * FACTS: decay says a claim is fading, dedup says two claims are one, edge typing says one claim
9703
+ * caused or contradicts another, retention says a claim has stopped earning its place. None of those hold for
9059
9704
  * a thing an agent intends to do, and each would be wrong applied to one. A task the agent has
9060
9705
  * not got to yet is not a claim losing confidence, and two open tasks with the same body are two
9061
9706
  * things to do, not one fact stored twice.
@@ -9080,8 +9725,10 @@ const isSleepExcluded = (memoryType) => SLEEP_EXCLUDED_TYPES.includes(memoryType
9080
9725
  const activeCorpus = (db) => db.all(`SELECT path, memory_type, title, gist, body_text, content_hash, confidence, importance,
9081
9726
  word_count, created_at, updated_at, valid_until, reprieves
9082
9727
  FROM files WHERE archived = 0 ORDER BY created_at ASC, path ASC`);
9728
+ /** A `memory_type NOT IN (…)` clause against alias `f`, or nothing when nothing is excluded. */
9729
+ const typeFilterFor = (alias, excluded) => excluded.length === 0 ? "" : ` AND ${alias}.memory_type NOT IN (${excluded.map(() => "?").join(", ")})`;
9083
9730
  /**
9084
- * Per-source top-`k` nearest neighbors above a similarity floor, over first-chunk vectors.
9731
+ * Every active file's first-chunk vector, decoded ONCE into the shape the pair kernel ranks.
9085
9732
  *
9086
9733
  * `ordinal = 0` collapses a file to its first chunk, not its best chunk. The format is one
9087
9734
  * fact per file, so almost every file is a single chunk. Taking the first keeps the pair set
@@ -9089,85 +9736,166 @@ const activeCorpus = (db) => db.all(`SELECT path, memory_type, title, gist, body
9089
9736
  * would make `(a, b)` a candidate while `(b, a)` is not, so which of two files was read first
9090
9737
  * would decide whether they merge.
9091
9738
  *
9092
- * `ROW_NUMBER() OVER (PARTITION BY src ...)` is the per-source cap. `vector_distance_cos` takes two
9093
- * STORED blobs here instead of a blob and a bound parameter. It can, because it is a registered
9094
- * SQL function over two `Uint8Array` arguments (`packages/index/src/database.ts`) and not a driver
9095
- * builtin with a fixed calling shape.
9739
+ * A row whose blob does not decode (empty or ragged) is dropped, the same exclusion the SQL
9740
+ * UDF's NULL produces for it in the retrieval arm.
9096
9741
  */
9097
- const neighborPairs = (db, options) => {
9098
- const excluded = options.excludeTypes ?? [];
9099
- const typeFilter = excluded.length === 0 ? "" : ` AND f.memory_type NOT IN (${excluded.map(() => "?").join(", ")})`;
9100
- return db.all(`WITH vecs AS (
9101
- SELECT f.path AS path, e.vec AS vec
9742
+ const firstChunkVectors = (db, excluded) => db.all(`SELECT f.path AS path, e.vec AS vec
9102
9743
  FROM files f
9103
9744
  JOIN chunks c ON c.path = f.path AND c.ordinal = 0
9104
9745
  JOIN embeddings e ON e.chunk_id = c.chunk_id
9105
- WHERE f.archived = 0${typeFilter}
9106
- ),
9107
- pairs AS (
9108
- SELECT l.path AS src, r.path AS dst, 1 - vector_distance_cos(l.vec, r.vec) AS sim
9109
- FROM vecs l JOIN vecs r ON r.path <> l.path
9110
- ),
9111
- ranked AS (
9112
- SELECT src, dst, sim, ROW_NUMBER() OVER (PARTITION BY src ORDER BY sim DESC, dst ASC) AS k
9113
- FROM pairs WHERE sim >= ?
9114
- )
9115
- SELECT src, dst, sim FROM ranked WHERE k <= ? ORDER BY sim DESC, src ASC, dst ASC LIMIT ?`, [
9116
- ...excluded,
9117
- options.floor,
9118
- options.perSourceK,
9119
- options.limit
9120
- ]);
9121
- };
9746
+ WHERE f.archived = 0${typeFilterFor("f", excluded)}`, [...excluded]).pipe(Effect.map((rows) => rows.flatMap((row) => {
9747
+ const vec = float32View(row.vec);
9748
+ return vec === void 0 ? [] : [{
9749
+ key: row.path,
9750
+ vec
9751
+ }];
9752
+ })));
9753
+ /**
9754
+ * Per-source top-`k` nearest neighbors above a similarity floor, over first-chunk vectors.
9755
+ *
9756
+ * The corpus filter is SQL, because that is where the index's reading semantics live. The pair
9757
+ * space is n² and ranks in TypeScript (`topNeighborPairs`), because a pair routed through the
9758
+ * `vector_distance_cos` UDF pays a fresh decode of BOTH 4 KB blobs per call — at a ~3k corpus
9759
+ * that is 8.45M calls and an OOM before the first phase records (issue #40), against ~12 MB
9760
+ * decoded once. The kernel reproduces this ordering exactly: floor, then per-source `sim` DESC /
9761
+ * `dst` ASC, then global `sim` DESC / `src` ASC / `dst` ASC, then the cap.
9762
+ */
9763
+ const neighborPairs = (db, options) => firstChunkVectors(db, options.excludeTypes ?? []).pipe(Effect.map((vectors) => topNeighborPairs(vectors, {
9764
+ floor: options.floor,
9765
+ perSourceK: options.perSourceK,
9766
+ limit: options.limit
9767
+ })));
9122
9768
  /**
9123
- * Candidate pairs for conflict detection: embedding-near, sharing an entity, and carrying no
9769
+ * Active non-task pairs that occupy the SAME frame key. Dedup's component seeds.
9770
+ *
9771
+ * A frame key is a claim's slot as surface grammar states it, so two active memories sharing one are
9772
+ * making a claim about the same thing by the corpus's own indexed evidence — no cosine, no model.
9773
+ * That is signal the vector floor can miss: "the owner of the deploy runbook is Priya" and "the owner
9774
+ * of the deploy runbook is Priya Raman" share a slot while their bodies share almost no vocabulary,
9775
+ * and their measured cosine under the fixture embedder is 0.59, far under any floor a night could
9776
+ * afford to mine at. Seeding components with these pairs puts them in front of the model, which is
9777
+ * the only reader that can say whether one is a rewording of the other.
9778
+ *
9779
+ * **The statement is OUTPUT-SENSITIVE: its cost follows the frame sharing that exists, not the pair
9780
+ * space.** The self-join is an equality on `frame_key`, which `files_frame_key_active` indexes under
9781
+ * exactly this predicate (`archived = 0 AND memory_type <> 'task' AND frame_key IS NOT NULL`,
9782
+ * migration 0009). So each row seeks its own key's bucket and emits one row per co-occupant, and a
9783
+ * corpus where no two memories share a slot emits nothing having read no pairs. `frame_key IS NOT
9784
+ * NULL` is stated even though the join equality already excludes NULL, because it is what makes the
9785
+ * partial index usable rather than leaving the planner to prove it.
9786
+ *
9787
+ * `r.path < l.path` orients each unordered pair once, which keeps the seed set the same size as the
9788
+ * edge set the component builder wants.
9789
+ *
9790
+ * **`memory_type <> 'task'` is written as the LITERAL the index uses, not as this module's
9791
+ * {@link SLEEP_EXCLUDED_TYPES} binding.** It is the same exclusion for the same reason — two open
9792
+ * tasks phrased alike are two things to do — but `NOT IN (?)` and `<> 'task'` are different
9793
+ * expressions to the planner, and only the second one matches `files_frame_key_active`'s predicate.
9794
+ * A bound form here would read as more general while quietly turning the seek into a scan.
9795
+ * `activeFramesFor` writes the literal for the same reason. `tests/units.test.ts` holds the two in
9796
+ * agreement, so a change to the excluded set cannot leave this statement behind silently.
9797
+ *
9798
+ * Measured plan (2026-08-19, node 24.19.0 against the shipped migrations): `SCAN l` then
9799
+ * `SEARCH r USING INDEX files_frame_key_active (frame_key=?)`. One side walks the KEYED rows, which
9800
+ * the partial index confines to the rows with a frame at all, and the other seeks.
9801
+ */
9802
+ const frameKeyPairs = (db) => db.all(`SELECT l.path AS src, r.path AS dst
9803
+ FROM files l
9804
+ JOIN files r ON r.frame_key = l.frame_key AND r.path < l.path
9805
+ AND r.archived = 0 AND r.memory_type <> 'task' AND r.frame_key IS NOT NULL
9806
+ WHERE l.archived = 0 AND l.memory_type <> 'task' AND l.frame_key IS NOT NULL
9807
+ ORDER BY l.path ASC, r.path ASC`);
9808
+ /**
9809
+ * Candidate pairs for edge typing: embedding-near, sharing an entity, and carrying no
9124
9810
  * AUTHORED edge between them in either direction.
9125
9811
  *
9126
9812
  * The shared-entity requirement is what keeps the model budget on pairs that could actually be about
9127
- * one thing. The anti-join keeps the phase from re-judging a pair an agent already linked. An
9128
- * authored `contradicts` is a settled fact, and re-asking the model about it would let a `neutral`
9129
- * answer look like new information.
9813
+ * one thing. The anti-join keeps the phase from re-typing a pair an agent already linked. An
9814
+ * authored `contradicts` or `caused_by` is a settled fact, and re-asking the model about it would let
9815
+ * a `none` answer look like new information.
9130
9816
  *
9131
9817
  * **`derived = 0` is what makes the anti-join correct.** Relationship mining runs one phase EARLIER and
9132
9818
  * writes a derived `relates_to` for every pair above 0.85 cosine, a strict superset of the
9133
- * pairs above the 0.80 conflict floor. An anti-join over ALL edges therefore excludes every candidate
9134
- * this phase exists to find, and the phase reports `candidates: 0` forever with no error anywhere.
9819
+ * pairs above the 0.80 typing floor. An anti-join over ALL edges therefore excludes every candidate
9820
+ * this scan exists to find, and the phase reports `candidates: 0` forever with no error anywhere.
9135
9821
  * A mined edge is a machine suspicion, not a settled relationship; only an authored one closes a pair.
9822
+ * {@link minedPairs} reads that same mined set as the OTHER arm of edge typing's candidate union, and
9823
+ * carries the identical anti-join for the identical reason.
9824
+ *
9825
+ * The statement ENUMERATES pairs from the shared-entity join instead of filtering an n×n vector
9826
+ * self-join, so its cost follows the entity sharing that actually exists. Similarity then ranks in
9827
+ * TypeScript over vectors decoded once (`rankCandidatePairs`), with the enumerated set standing
9828
+ * where the ranking CTE's `WHERE` stood: the predicates run BEFORE per-source top-`k`. `re.path <
9829
+ * le.path` orients each pair once, dst below src.
9136
9830
  */
9137
- const conflictCandidates = (db, options) => {
9831
+ const sharedEntityPairs = (db, options) => {
9138
9832
  const excluded = options.excludeTypes ?? [];
9139
- const typeFilter = excluded.length === 0 ? "" : ` AND f.memory_type NOT IN (${excluded.map(() => "?").join(", ")})`;
9140
- return db.all(`WITH vecs AS (
9141
- SELECT f.path AS path, e.vec AS vec
9142
- FROM files f
9143
- JOIN chunks c ON c.path = f.path AND c.ordinal = 0
9144
- JOIN embeddings e ON e.chunk_id = c.chunk_id
9145
- WHERE f.archived = 0${typeFilter}
9146
- ),
9147
- pairs AS (
9148
- SELECT l.path AS src, r.path AS dst, 1 - vector_distance_cos(l.vec, r.vec) AS sim
9149
- FROM vecs l JOIN vecs r ON r.path < l.path
9150
- WHERE EXISTS (
9151
- SELECT 1 FROM file_entities le
9152
- JOIN file_entities re ON re.entity_type = le.entity_type AND re.entity_name = le.entity_name
9153
- WHERE le.path = l.path AND re.path = r.path
9154
- )
9833
+ const pairs = db.all(`SELECT DISTINCT le.path AS src, re.path AS dst
9834
+ FROM file_entities le
9835
+ JOIN file_entities re ON re.entity_type = le.entity_type
9836
+ AND re.entity_name = le.entity_name AND re.path < le.path
9837
+ JOIN files fl ON fl.path = le.path AND fl.archived = 0${typeFilterFor("fl", excluded)}
9838
+ JOIN files fr ON fr.path = re.path AND fr.archived = 0${typeFilterFor("fr", excluded)}
9839
+ WHERE NOT EXISTS (
9840
+ SELECT 1 FROM edges e
9841
+ WHERE e.derived = 0
9842
+ AND ((e.src_path = le.path AND e.dst_path = re.path)
9843
+ OR (e.src_path = re.path AND e.dst_path = le.path))
9844
+ )`, [...excluded, ...excluded]);
9845
+ return Effect.all([pairs, firstChunkVectors(db, excluded)]).pipe(Effect.map(([candidatePairs, vectors]) => rankCandidatePairs(candidatePairs, vectors, {
9846
+ floor: options.floor,
9847
+ perSourceK: options.perSourceK,
9848
+ limit: options.limit
9849
+ })));
9850
+ };
9851
+ /**
9852
+ * The MINED edges of one rel, as candidate pairs: edge typing's second arm.
9853
+ *
9854
+ * Relationship mining runs one phase earlier and writes a derived `relates_to` for every pair above
9855
+ * its cosine floor, index-only. Those pairs are the corpus's own answer to "which memories look
9856
+ * related", and they are NOT a subset of {@link sharedEntityPairs}: two memories about one incident
9857
+ * that name no entity in common are invisible to the shared-entity join and obvious to the embedder.
9858
+ * Reading them here is what makes edge typing's recall the union of both signals rather than the
9859
+ * entity-authoring habits of whoever wrote the memories.
9860
+ *
9861
+ * `strength` is the mined edge's own cosine (`replaceMinedEdges` clamps it into `[0, 1]`), so the
9862
+ * caller can rank both arms of the union on one scale without re-decoding a vector. The statement
9863
+ * ORDERS BY it, descending, for the same reason {@link sharedEntityPairs} hands back a ranked list:
9864
+ * the caller's cap is a model-cost bound, and a cap over a path-ordered read would spend the night on
9865
+ * whichever pairs sort alphabetically first. `src_path` then `dst_path` break a tie, which is
9866
+ * `collectRanked`'s ordering, so both arms of the union arrive in one ordering.
9867
+ *
9868
+ * **Deliberately unbounded**, unlike the other arm: the caller ranks the UNION and caps that, so a
9869
+ * limit here would cut candidates before the two arms have been compared. The mined set is one row per
9870
+ * pair above mining's cosine floor (measured 1,498 on the production corpus), which is a read this
9871
+ * phase already performs once a night.
9872
+ *
9873
+ * Three filters, each load-bearing:
9874
+ *
9875
+ * - `derived = 1` restricts this to the machine-mined set. An authored `relates_to` is an agent's
9876
+ * assertion, and re-typing it would let a nightly job overwrite a human judgment with a narrower rel.
9877
+ * - `edge_class = 'memory'` is the same firewall every graph read carries.
9878
+ * - The `derived = 0` anti-join drops a pair that already carries ANY authored edge either way, which
9879
+ * is exactly {@link sharedEntityPairs}' rule. Without it a pair typed last night would be re-judged
9880
+ * every night, because promoting a typed edge does not delete the mined `relates_to` underneath it.
9881
+ */
9882
+ const minedPairs = (db, options) => {
9883
+ const excluded = options.excludeTypes ?? [];
9884
+ return db.all(`SELECT e.src_path AS src, e.dst_path AS dst, e.strength AS sim
9885
+ FROM edges e
9886
+ JOIN files fs ON fs.path = e.src_path AND fs.archived = 0${typeFilterFor("fs", excluded)}
9887
+ JOIN files fd ON fd.path = e.dst_path AND fd.archived = 0${typeFilterFor("fd", excluded)}
9888
+ WHERE e.derived = 1 AND e.edge_class = 'memory' AND e.rel = ?
9155
9889
  AND NOT EXISTS (
9156
- SELECT 1 FROM edges e
9157
- WHERE e.derived = 0
9158
- AND ((e.src_path = l.path AND e.dst_path = r.path)
9159
- OR (e.src_path = r.path AND e.dst_path = l.path))
9890
+ SELECT 1 FROM edges a
9891
+ WHERE a.derived = 0
9892
+ AND ((a.src_path = e.src_path AND a.dst_path = e.dst_path)
9893
+ OR (a.src_path = e.dst_path AND a.dst_path = e.src_path))
9160
9894
  )
9161
- ),
9162
- ranked AS (
9163
- SELECT src, dst, sim, ROW_NUMBER() OVER (PARTITION BY src ORDER BY sim DESC, dst ASC) AS k
9164
- FROM pairs WHERE sim >= ?
9165
- )
9166
- SELECT src, dst, sim FROM ranked WHERE k <= ? ORDER BY sim DESC, src ASC, dst ASC LIMIT ?`, [
9895
+ ORDER BY e.strength DESC, e.src_path ASC, e.dst_path ASC`, [
9896
+ ...excluded,
9167
9897
  ...excluded,
9168
- options.floor,
9169
- options.perSourceK,
9170
- options.limit
9898
+ options.rel
9171
9899
  ]);
9172
9900
  };
9173
9901
  /**
@@ -9204,6 +9932,62 @@ const pathsForEntity = (db, entityType, entityName) => db.all(`SELECT e.path AS
9204
9932
  /** `?` per excluded type, so the exclusion binds instead of interpolating a value into SQL. */
9205
9933
  const typePlaceholders = () => SLEEP_EXCLUDED_TYPES.map(() => "?").join(", ");
9206
9934
  /**
9935
+ * Every (entity, claiming active non-task file) pair in ONE statement, entity-ordered then path.
9936
+ *
9937
+ * The same corpus {@link activeEntities} counts, enumerated instead of aggregated. Entity resolution
9938
+ * needs both a per-entity memory centroid and a few sample titles per entity, and deriving either from
9939
+ * {@link pathsForEntity} would be one query per entity — 59 entities on the measured corpus, and one
9940
+ * round trip each for a join the database performs once.
9941
+ *
9942
+ * **The `ORDER BY` is for a reader, NOT for the centroid's determinism.** A centroid is a sum over its
9943
+ * members' vectors and float addition is not associative, so the summation order decides the bytes —
9944
+ * but `entityCentroids` re-sorts each entity's paths itself and does not inherit this order. That is
9945
+ * deliberate: the guarantee has to live where the sum happens, so a future caller reading these rows
9946
+ * through a different statement cannot silently lose it. (Confirmed by mutation: replacing this clause
9947
+ * with `ORDER BY e.path DESC` leaves the whole sleep suite green, while dropping the phase's own sort
9948
+ * fails it.)
9949
+ */
9950
+ const entityClaims = (db) => db.all(`SELECT e.entity_type AS entity_type, e.entity_name AS entity_name,
9951
+ e.path AS path, f.title AS title
9952
+ FROM file_entities e JOIN files f ON f.path = e.path
9953
+ WHERE f.archived = 0 AND f.memory_type NOT IN (${typePlaceholders()})
9954
+ ORDER BY e.entity_type ASC, e.entity_name ASC, e.path ASC`, [...SLEEP_EXCLUDED_TYPES]);
9955
+ /**
9956
+ * Every active file's first-chunk vector, path-keyed and decoded once. The centroid pass's input.
9957
+ *
9958
+ * Exported wrapper over the module-private statement the pair arms use, so entity resolution reads the
9959
+ * SAME vector space they do — `ordinal = 0`, the same drop of a blob that does not decode — instead of
9960
+ * a second SELECT free to disagree about which chunk represents a file.
9961
+ *
9962
+ * Tasks are excluded, matching {@link entityClaims}: a centroid built partly from working state would
9963
+ * describe what the agent intends to do about a subject rather than what it knows about one.
9964
+ */
9965
+ const entityVectors = (db) => firstChunkVectors(db, SLEEP_EXCLUDED_TYPES);
9966
+ /**
9967
+ * Every indexed person file, path-ordered. The alias oracle's file list.
9968
+ *
9969
+ * Selected by DIRECTORY, because that is what a person file is: `person-links` mints one per
9970
+ * `person:` entity under `PEOPLE_DIR`, and a hand-authored one placed there by an operator is just as
9971
+ * authoritative. Selecting by entity instead would miss a file whose subject the corpus has since
9972
+ * stopped mentioning, whose declaration is still the truth about those names.
9973
+ *
9974
+ * Archived files are included. Archiving a person file records that the corpus moved on from the
9975
+ * person, not that two of their names stopped being the same name, and an alias declaration losing its
9976
+ * force on archival would silently re-split a person the phase had already merged.
9977
+ *
9978
+ * **Which is why the archive prefix is matched too, and not just `archived = 0` left off.** Eviction is
9979
+ * the `git mv` into `archive/<YYYY>/<original-path>`, so an archived person file's PATH is
9980
+ * `archive/2026/resources/people/…` and no longer matches `resources/people/%` at all. A single
9981
+ * `LIKE` here would have said "archived files are included" while excluding every one of them, and the
9982
+ * re-split above is exactly what would have followed. The second pattern mirrors
9983
+ * `archivePathFor`'s shape (`%` for the year segment, which is four digits the statement need not
9984
+ * verify — a false match would be another file under `resources/people/`, which is a person file).
9985
+ *
9986
+ * The phase reads the BYTES of each of these; this statement only says which paths to open, because
9987
+ * `memhtml-alias` is repeatable and lives in the file rather than in any projection.
9988
+ */
9989
+ const peoplePaths = (db) => db.all("SELECT path FROM files WHERE path LIKE ? OR path LIKE ? ORDER BY path ASC", [`${PEOPLE_DIR}/%`, `${ARCHIVE_BUCKET}/%/${PEOPLE_DIR}/%`]);
9990
+ /**
9207
9991
  * The memory-class edge list over active files, both authored and derived.
9208
9992
  *
9209
9993
  * `edge_class = 'memory'` is the firewall. A person or provenance edge cannot enter PageRank, label
@@ -9241,7 +10025,7 @@ const accessRows = (db) => db.hasState ? db.all(`SELECT path, access_count, rein
9241
10025
  * one pair cannot both read `detections = 1` and both decline to promote.
9242
10026
  *
9243
10027
  * **The bump is idempotent WITHIN one run's instant.** `detections` advances only when `updated_at`
9244
- * differs from `at`. Corroboration means "two DIFFERENT nights saw this", and conflict detection
10028
+ * differs from `at`. Corroboration means "two DIFFERENT nights saw this", and edge typing
9245
10029
  * commits only when something is promoted, so a run that judged pairs and promoted nothing leaves no
9246
10030
  * trailer and `memhtml sleep resume` re-executes it. Without the guard that second pass would count as a
9247
10031
  * second detection and promote a contradiction one night's evidence had not earned. That puts a machine
@@ -9269,6 +10053,47 @@ const markPromoted = (db, input) => db.run(`UPDATE ${STATE_SCHEMA}.edge_corrobor
9269
10053
  input.rel,
9270
10054
  input.dstPath
9271
10055
  ]);
10056
+ /**
10057
+ * Bump an entity merge's detection counter and read the result back.
10058
+ *
10059
+ * The same `RETURNING` upsert {@link bumpCorroboration} uses, for the same reason: the promotion
10060
+ * decision is made in the database at the instant of the write, so two runs racing on one merge cannot
10061
+ * both read `detections = 1` and both decline to apply it, leaving a genuinely corroborated merge
10062
+ * pending forever.
10063
+ *
10064
+ * **And the bump is idempotent WITHIN one run's instant**, which entity resolution needs even more than
10065
+ * conflict detection does. This phase commits whenever it rewrites ANY file, so a night whose only work
10066
+ * was a deterministic normalization commits and leaves a trailer, while a night that only bumped
10067
+ * counters does not. `memhtml sleep resume` therefore re-executes this phase on the second pass, and
10068
+ * without the `updated_at` guard that pass would count as a second night's independent sighting and
10069
+ * apply a merge one night's evidence had not earned. `at` comes from the run's own date, so a resume of
10070
+ * the same run reuses it and a genuinely later night does not.
10071
+ *
10072
+ * Names are the NORMALIZED forms, which is what makes one merge one counter: `Checkout API` and
10073
+ * `checkout api` would otherwise be two rows for one merge and neither would reach two detections.
10074
+ */
10075
+ const bumpEntityCorroboration = (db, input) => db.all(`INSERT INTO ${STATE_SCHEMA}.entity_corroboration
10076
+ (entity_type, alias_name, canonical_name, detections, updated_at)
10077
+ VALUES (?, ?, ?, 1, ?)
10078
+ ON CONFLICT(entity_type, alias_name, canonical_name) DO UPDATE SET
10079
+ detections = detections + CASE
10080
+ WHEN entity_corroboration.updated_at = excluded.updated_at THEN 0 ELSE 1 END,
10081
+ updated_at = excluded.updated_at
10082
+ RETURNING entity_type, alias_name, canonical_name, detections, promoted`, [
10083
+ input.entityType,
10084
+ input.aliasName,
10085
+ input.canonicalName,
10086
+ input.at
10087
+ ]);
10088
+ /** Mark a corroborated merge applied, so a later night reads it as done instead of pending. */
10089
+ const markEntityPromoted = (db, input) => db.run(`UPDATE ${STATE_SCHEMA}.entity_corroboration
10090
+ SET promoted = 1, confirmed = 1, updated_at = ?
10091
+ WHERE entity_type = ? AND alias_name = ? AND canonical_name = ?`, [
10092
+ input.at,
10093
+ input.entityType,
10094
+ input.aliasName,
10095
+ input.canonicalName
10096
+ ]);
9272
10097
  /** Sessions with no memory linked to them, which is what trace-consolidation counts in v1. */
9273
10098
  const unlinkedSessionCount = (db) => db.get(`SELECT count(*) AS n FROM traces t
9274
10099
  WHERE NOT EXISTS (SELECT 1 FROM memory_session_links l WHERE l.session_id = t.session_id)`).pipe(Effect.map((row) => row?.n ?? 0));
@@ -9731,6 +10556,12 @@ const arcSynthesis = (env) => Effect.gen(function* () {
9731
10556
  * Phase 10, compress. COMPRESS-band memories grouped by community, folded into a synthesized
9732
10557
  * canonical in batches. ONE COMMIT PER BATCH.
9733
10558
  *
10559
+ * The batching runs on the shared kernel in `batch.ts`: this phase sorts the communities and their
10560
+ * members, and `assembleBatches`, `keyMembers`, `compressPrompt`, and `resolveKeys` do the slicing,
10561
+ * the opaque keying, the prompt framing, and the key resolution that four other phases also need. The
10562
+ * kernel preserves the order it is handed and does no sorting of its own, so the two sorts below are
10563
+ * what make a night's batch boundaries and member keys reproducible.
10564
+ *
9734
10565
  * Grouped by community instead of by similarity, because a community is the graph's own answer to
9735
10566
  * "what belongs together". A similarity group folds two memories that happen to share vocabulary,
9736
10567
  * while a community folds memories the corpus itself has linked. Communities below the minimum size
@@ -9752,6 +10583,11 @@ const arcSynthesis = (env) => Effect.gen(function* () {
9752
10583
  */
9753
10584
  /** Members per model call. Small enough that every member's facts fit the answer's attention. */
9754
10585
  const COMPRESS_BATCH_SIZE = 8;
10586
+ /**
10587
+ * Members a batch needs before it is worth a call. A batch of one is not a fold: it would rewrite a
10588
+ * lone memory into a "canonical" saying the same thing under a new path, and archive the original.
10589
+ */
10590
+ const COMPRESS_MIN_BATCH = 2;
9755
10591
  /** COMPRESS-band candidates considered per cycle. The model-cost guard. */
9756
10592
  const COMPRESS_CANDIDATE_LIMIT = 2e3;
9757
10593
  /** Characters of each member shown. A fold must see the facts, so this is wider than arc evidence. */
@@ -9776,14 +10612,16 @@ const compress = (env) => Effect.gen(function* () {
9776
10612
  if (bucket === void 0) byCommunity.set(label, [entry]);
9777
10613
  else bucket.push(entry);
9778
10614
  }
9779
- const batches = [];
9780
- for (const [, members] of [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
9781
- const ordered = [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0);
9782
- for (let at = 0; at < ordered.length; at += 8) {
9783
- const slice = ordered.slice(at, at + 8);
9784
- if (slice.length >= 2) batches.push(slice);
9785
- }
9786
- }
10615
+ /**
10616
+ * Both sorts are this phase's, and the kernel keeps the order they produce. Communities are
10617
+ * walked lexicographically by label so a night's call order is fixed, and each community's members
10618
+ * by `row.path` so the `m1`..`mN` keys land on the same files twice over.
10619
+ */
10620
+ const groups = [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1).map(([, members]) => [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0));
10621
+ const batches = assembleBatches(groups, {
10622
+ maxMembers: 8,
10623
+ minMembers: 2
10624
+ });
9787
10625
  const counts = {
9788
10626
  candidates: candidates.length,
9789
10627
  communities: byCommunity.size,
@@ -9802,33 +10640,22 @@ const compress = (env) => Effect.gen(function* () {
9802
10640
  let lastCommit = null;
9803
10641
  for (const batch of batches) {
9804
10642
  /** Opaque keys again, so `absorbedKeys` cannot name a path. */
9805
- const keyed = batch.map((entry, offset) => ({
9806
- key: `m${offset + 1}`,
9807
- path: entry.row.path,
9808
- title: entry.row.title,
9809
- text: `${entry.row.title}\n${entry.row.gist}\n${entry.row.body_text}`.slice(0, COMPRESS_MEMBER_CHARS)
9810
- }));
9811
- const pathForKey = new Map(keyed.map((entry) => [entry.key, entry.path]));
10643
+ const keyed = keyMembers(batch, (entry) => `${entry.row.title}\n${entry.row.gist}\n${entry.row.body_text}`, { charBudget: COMPRESS_MEMBER_CHARS });
9812
10644
  llmCalls += 1;
9813
- const synthesis = yield* isolate(`compress batch of ${batch.length}`, model.generateObject({
10645
+ const synthesis = yield* batchCall(model, `compress batch of ${batch.length}`, {
9814
10646
  schema: CompressSynthesis,
9815
10647
  system: COMPRESS_SYSTEM,
9816
- prompt: compressPrompt(keyed.map((entry) => ({
9817
- key: entry.key,
9818
- text: entry.text
9819
- }))),
10648
+ prompt: compressPrompt(keyed.keyed),
9820
10649
  modelKey,
9821
10650
  effort: "high",
9822
10651
  toolDescription: "Emit the canonical memory and the members whose content it absorbs."
9823
- }));
10652
+ });
9824
10653
  if (synthesis === void 0) {
9825
10654
  skipped += 1;
9826
10655
  continue;
9827
10656
  }
9828
- const absorbed = [...new Set(synthesis.absorbedKeys.flatMap((key) => {
9829
- const path = pathForKey.get(key);
9830
- return path === void 0 ? [] : [path];
9831
- }))];
10657
+ /** A key the batch never offered resolves to nothing, so a fold reaches only offered files. */
10658
+ const absorbed = resolveKeys(keyed, synthesis.absorbedKeys).map((entry) => entry.row.path);
9832
10659
  if (absorbed.length < 2 || synthesis.title.trim() === "" || synthesis.claim.trim() === "") {
9833
10660
  skipped += 1;
9834
10661
  continue;
@@ -9976,190 +10803,190 @@ const confidenceDecay = (env) => Effect.gen(function* () {
9976
10803
  };
9977
10804
  });
9978
10805
 
9979
- //#endregion
9980
- //#region packages/sleep/dist/phases/conflict-detection.js
9981
- /**
9982
- * Phase 6, conflict detection. An NLI stance judge over embedding-near same-entity pairs; a
9983
- * corroborated contradiction is promoted into BOTH files and committed.
9984
- *
9985
- * Three stages, and keeping them separate is what makes the phase safe:
9986
- *
9987
- * 1. **Scan (SQL, no model).** Same-entity active pairs above {@link CONFLICT_COSINE_FLOOR} carrying
9988
- * no edge in either direction, capped at {@link CONFLICT_CANDIDATE_LIMIT}.
9989
- * 2. **Judge (one model call per pair, isolated).** Each call is wrapped so one malformed tool
9990
- * payload skips its pair and is counted. A night that judged 199 pairs and lost the 200th has
9991
- * done 199 pairs of work; failing the phase would discard all of it.
9992
- * 3. **Assert (deterministic, decided here and not by the model).** Only `verdict: "contradicts"` above
9993
- * the confidence floor bumps the corroboration counter, and only `detections >= 2` promotes the edge
9994
- * into the files. A single machine detection therefore cannot reach the retention penalty. The
9995
- * counter lives in the state plane and the penalty counts only `derived = 0` file-borne edges.
9996
- *
9997
- * **Detection only.** The phase asserts the contradiction and stops. It does not supersede, close a
9998
- * `memhtml-valid-until`, or archive either side. Choosing the winner of a contradiction is a one-way
9999
- * door on stored belief, and it belongs to an agent or a human, not to a nightly job.
10000
- */
10001
- /** The moderate similarity floor a pair must clear to be worth a model call. */
10002
- const CONFLICT_COSINE_FLOOR = .8;
10003
- /** Nearest same-entity neighbors considered per source. */
10004
- const CONFLICT_PER_SOURCE_K = 5;
10005
- /** Pairs judged per cycle. The model-cost guard. */
10006
- const CONFLICT_CANDIDATE_LIMIT = 200;
10007
- /** Detections a machine-found contradiction needs before it is written into the files. */
10008
- const PROMOTION_DETECTIONS = 2;
10009
- const conflictDetection = (env) => Effect.gen(function* () {
10010
- const model = env.deps.model;
10011
- if (model === void 0) return {
10012
- ...emptyOutcome({
10013
- candidates: 0,
10014
- judged: 0
10015
- }),
10016
- detail: "no model bound"
10017
- };
10018
- /**
10019
- * Tasks are out of the candidate set. "These two contradict" is a judgment about asserted
10020
- * facts, and a task asserts nothing. A model asked about two tasks would answer a question
10021
- * that has no true answer, and a promoted `contradicts` between them would be a memory-class
10022
- * edge with task endpoints written into both files.
10023
- */
10024
- const candidates = yield* conflictCandidates(env.deps.db, {
10025
- floor: CONFLICT_COSINE_FLOOR,
10026
- perSourceK: 5,
10027
- limit: 200,
10028
- excludeTypes: SLEEP_EXCLUDED_TYPES
10029
- });
10030
- if (candidates.length === 0) return emptyOutcome({
10031
- candidates: 0,
10032
- judged: 0,
10033
- contradictions: 0,
10034
- promoted: 0,
10035
- skipped: 0
10036
- });
10037
- if (env.dryRun) return emptyOutcome({
10038
- candidates: candidates.length,
10039
- judged: 0,
10040
- contradictions: 0,
10041
- promoted: 0,
10042
- skipped: 0
10043
- });
10044
- const corpus = yield* activeCorpus(env.deps.db);
10045
- const textOf = new Map(corpus.map((row) => [row.path, `${row.gist}\n${row.body_text}`]));
10046
- const modelKey = modelFor(env.deps, "conflict-detection");
10047
- let judged = 0;
10048
- let contradictions = 0;
10049
- let promoted = 0;
10050
- let skipped = 0;
10051
- let llmCalls = 0;
10052
- for (const candidate of candidates) {
10053
- const textA = textOf.get(candidate.src);
10054
- const textB = textOf.get(candidate.dst);
10055
- if (textA === void 0 || textB === void 0) {
10056
- skipped += 1;
10057
- continue;
10058
- }
10059
- llmCalls += 1;
10060
- const judgment = yield* isolate(`conflict-detection pair ${judged + skipped}`, model.generateObject({
10061
- schema: StanceJudgment,
10062
- system: STANCE_SYSTEM,
10063
- prompt: stancePrompt(textA, textB),
10064
- modelKey,
10065
- effort: "medium",
10066
- toolDescription: "Emit the stance of memory B relative to memory A."
10067
- }));
10068
- if (judgment === void 0) {
10069
- skipped += 1;
10070
- continue;
10071
- }
10072
- judged += 1;
10073
- if (!assertsContradiction(judgment)) continue;
10074
- contradictions += 1;
10075
- const row = (yield* bumpCorroboration(env.deps.db, {
10076
- srcPath: candidate.src,
10077
- rel: "contradicts",
10078
- dstPath: candidate.dst,
10079
- at: env.at
10080
- }))[0];
10081
- if (row === void 0 || row.detections < 2 || row.promoted === 1) continue;
10082
- yield* stampFile(env, candidate.src, [link("contradicts", hrefFor(candidate.dst)), meta("memhtml-updated", env.at)]);
10083
- yield* stampFile(env, candidate.dst, [link("contradicts", hrefFor(candidate.src)), meta("memhtml-updated", env.at)]);
10084
- yield* markPromoted(env.deps.db, {
10085
- srcPath: candidate.src,
10086
- rel: "contradicts",
10087
- dstPath: candidate.dst,
10088
- at: env.at
10089
- });
10090
- promoted += 1;
10091
- }
10092
- const counts = {
10093
- candidates: candidates.length,
10094
- judged,
10095
- contradictions,
10096
- promoted,
10097
- skipped
10098
- };
10099
- if (promoted === 0) return {
10100
- counts,
10101
- commitSha: null,
10102
- llmCalls
10103
- };
10104
- return {
10105
- counts,
10106
- commitSha: yield* commitPhase(env, "conflict-detection", `promote ${promoted} corroborated contradictions`, counts),
10107
- llmCalls
10108
- };
10109
- });
10110
-
10111
10806
  //#endregion
10112
10807
  //#region packages/sleep/dist/phases/dedup-merge.js
10113
10808
  /**
10114
10809
  * Phase 2, dedup-merge. Fold near-duplicates: the keeper gains `memhtml-supersedes`, the dropped
10115
10810
  * files `git mv` into the archive. ONE commit.
10116
10811
  *
10812
+ * ## The model partitions; code decides
10813
+ *
10814
+ * With a model bound the phase mines a RECALL-oriented candidate set at {@link DEDUP_COMPONENT_FLOOR},
10815
+ * unions it with the frame-key exact matches, builds connected components over the union, and asks the
10816
+ * model to partition each component into merge groups. The model answers one question: which of these
10817
+ * memories are the same memory. It does not choose the canonical, it does not name a write target, and
10818
+ * it is never asked an n² pair question — a component of five is one entry in one batch's member list,
10819
+ * not ten pair calls.
10820
+ *
10821
+ * Everything the fold writes is derived afterwards. Orientation is arithmetic over corpus order, and
10822
+ * every pair a group implies is routed through `mergeCandidates`, which applies the divergence veto,
10823
+ * the self-merge check, the both-roles guard, and the per-night cap. So the set of pairs that CAN be
10824
+ * committed does not widen when a model is bound: it is the same predicate over a different candidate
10825
+ * set.
10826
+ *
10117
10827
  * **Orientation keeps the OLDER file.** That is why the divergence veto changes outcomes instead of
10118
10828
  * being cosmetic. A blind high-cosine merge of a newer correction into an older wrong memory does not
10119
10829
  * merely lose information, it restores the error the correction was written to fix. `activeCorpus`
10120
10830
  * reads oldest-first, so the older path is the keeper by construction and the choice is reproducible.
10831
+ * Inside a model-proposed group the keeper is the member with the lowest corpus offset, which is the
10832
+ * same rule applied to more than two files at once.
10121
10833
  *
10122
10834
  * **The veto and the in-batch role guard both live in `@memhtml/domain`.** `mergeCandidates` claims BOTH
10123
10835
  * roles for every committed pair. A path that was a keeper cannot later be dropped, and a path that
10124
10836
  * was dropped cannot later become a keeper. The predecessor memory system recorded only the drop side, so given
10125
10837
  * `(gf → a)` then `(b → gf)` both decisions committed: `gf` absorbed `a` and was then archived into
10126
- * `b`, superseding `a`'s content into a file the same batch destroyed.
10838
+ * `b`, superseding `a`'s content into a file the same batch destroyed. Batching makes that guard carry
10839
+ * MORE, not less: one model answer names several groups, and two groups overlapping on one path is
10840
+ * exactly that chain, arriving from one call instead of from two nights.
10841
+ *
10842
+ * **With no model bound the phase is the deterministic floor, unchanged.** It mines at
10843
+ * {@link NEAR_DUPLICATE_THRESHOLD}, orients, and hands the pairs to `mergeCandidates`. That is not a
10844
+ * degraded mode to be repaired later: a night with no credentials still folds every duplicate a cosine
10845
+ * can prove, and every count it reports is what this phase reported before it could call a model.
10846
+ *
10847
+ * ## Precedence between the two candidate sets
10848
+ *
10849
+ * Model groups are offered to `mergeCandidates` FIRST, then the mined pairs above the deterministic
10850
+ * floor that no group already claimed. Two properties follow, and both are the reason for the order:
10851
+ *
10852
+ * - The deterministic floor never regresses. Every pair the no-model path would have merged is still
10853
+ * in the list, so binding a model cannot make a night fold less than it did.
10854
+ * - Where the two disagree the semantic answer wins the path. A pair above 0.92 whose two files the
10855
+ * model instead grouped with a third folds as the model's group, because the both-roles guard gives
10856
+ * a path to whichever decision claims it first. The model read both files; the cosine read neither.
10857
+ *
10858
+ * Within each half the order is fixed: groups follow the batch, component, and group order they were
10859
+ * packed and answered in, and mined pairs stay in the kernel's `sim` DESC ordering.
10127
10860
  *
10128
10861
  * One commit for the whole batch, not one per pair. A keeper's `memhtml-supersedes` points at its
10129
10862
  * dropped file's ARCHIVE path, which is where that file lives only after this commit lands.
10130
10863
  * Splitting them would create a dangling href in the commit that made it dangle.
10131
10864
  */
10865
+ /**
10866
+ * The mining floor when a model is bound. RECALL-oriented, and deliberately below the merge floor.
10867
+ *
10868
+ * A pair between this and {@link NEAR_DUPLICATE_THRESHOLD} is one no cosine can settle: high enough
10869
+ * that the two memories are about one thing, not high enough that they are provably one claim. That
10870
+ * band is what a semantic reader is for, and the deterministic path cannot see into it at all. Issue
10871
+ * #43 measured ~800 pairs at 0.86 on the 2,907-memory production corpus against 77 at 0.92, and those
10872
+ * 800 collapse into components small enough that {@link DEDUP_MAX_COMPONENTS} bounds the night at tens
10873
+ * of calls rather than hundreds.
10874
+ *
10875
+ * The floor is only a floor. A pair that clears it still has to survive the model's partition and then
10876
+ * the veto, so more recall here cannot lower the bar on what gets written.
10877
+ */
10878
+ const DEDUP_COMPONENT_FLOOR = .86;
10879
+ /**
10880
+ * Mined pairs considered per night at the recall floor.
10881
+ *
10882
+ * `MAX_MERGE_PAIRS * 8`, twice the deterministic path's `* 4`, because the floor moved down and the
10883
+ * pair count grows with the band while the commit cap does not move: no more than `MAX_MERGE_PAIRS`
10884
+ * folds land whatever this admits, so the multiplier buys candidate COVERAGE and cannot buy extra
10885
+ * writes. Issue #43's measurement is the sizing — ~800 pairs at 0.86 against 800 here — so a corpus of
10886
+ * that shape is mined whole and a larger one is truncated at a bound that is stated rather than
10887
+ * emergent.
10888
+ */
10889
+ const DEDUP_PAIR_LIMIT = 100 * 8;
10890
+ /**
10891
+ * Files of one component that reach a model call. A larger component is TRUNCATED to its lowest paths.
10892
+ *
10893
+ * Eight is above the size a real near-duplicate family reaches — a fact restated eight times is
10894
+ * already pathological — so a component past it is almost always the recall floor having chained
10895
+ * several distinct facts through shared vocabulary. Handing all of it over would spend one call's whole
10896
+ * attention budget on the component least likely to hold a clean duplicate, and a model asked to
10897
+ * partition thirty loosely-related memories answers with a few large groups, which is the answer shape
10898
+ * the veto is least able to correct.
10899
+ *
10900
+ * Truncation keeps the LOWEST PATHS rather than the highest cosines, so which members are considered
10901
+ * is a property of the corpus and not of the floor. The remainder is deferred, not lost: the night's
10902
+ * folds change the graph, so tomorrow's components over the same corpus are smaller.
10903
+ */
10904
+ const DEDUP_MAX_COMPONENT = 8;
10905
+ /**
10906
+ * Components handed to a model per night. The cost bound.
10907
+ *
10908
+ * Set so a night lands in issue #43's measured envelope of ~15-25 calls: at
10909
+ * {@link DEDUP_BATCH_MEMBERS} members per call and a typical component of two, 300 components pack
10910
+ * into roughly 15 calls, and the per-call character budget closes some earlier. Components are taken
10911
+ * in component order, which is lowest-path first, so which ones a capped night considers is
10912
+ * reproducible.
10913
+ */
10914
+ const DEDUP_MAX_COMPONENTS = 300;
10915
+ /**
10916
+ * Members per dedup call. Wider than compress's 8 because the question is cheaper per member.
10917
+ *
10918
+ * compress asks the model to WRITE one canonical carrying every member's facts, so each member has to
10919
+ * fit the answer's generative attention. Dedup asks only which members restate each other and the
10920
+ * answer is a list of keys, so one batch can hold several components' worth.
10921
+ */
10922
+ const DEDUP_BATCH_MEMBERS = 40;
10923
+ /** Characters of each member shown. The house member budget, the same 1200 compress uses. */
10924
+ const DEDUP_MEMBER_CHARS = 1200;
10925
+ /**
10926
+ * Characters per dedup call: the member budget times the member cap.
10927
+ *
10928
+ * Derived rather than chosen, so the two caps cannot drift into a call that honors one and breaches
10929
+ * the other. It is a CEILING and normally slack, because most members are far shorter than their
10930
+ * budget, which is why `packGroups` takes both bounds and closes on whichever binds first.
10931
+ */
10932
+ const DEDUP_BATCH_CHARS = DEDUP_MEMBER_CHARS * 40;
10933
+ /**
10934
+ * The threshold the batched arm hands `mergeCandidates`, which must NOT re-gate on similarity.
10935
+ *
10936
+ * Admission on that arm is already decided when the filter runs. A group pair got there because the
10937
+ * model grouped it, and a mined pair got there because it cleared {@link NEAR_DUPLICATE_THRESHOLD} in
10938
+ * the phase's own filter. What is left for `mergeCandidates` to apply is the veto, the self check, the
10939
+ * both-roles guard, and the cap — the four that are about safety rather than about a number.
10940
+ *
10941
+ * Zero rather than {@link DEDUP_COMPONENT_FLOOR} because that comparison is STRICT (`<= threshold`
10942
+ * skips), and a group pair the corpus never mined carries the floor itself as its similarity. A
10943
+ * threshold of the floor would drop exactly the frame-seeded pairs that seeding exists to find, and it
10944
+ * would do it silently: the count would read as a veto. Nothing negative can reach here, since every
10945
+ * mined similarity is at or above the floor and the synthetic value IS the floor.
10946
+ */
10947
+ const DEDUP_ADMIT_FLOOR = 0;
10948
+ /** The text a member is offered under: its claim and its body, the same join compress uses. */
10949
+ const textFor = (row) => `${row.gist}\n${row.body_text}`;
10950
+ /**
10951
+ * `arc` is excluded from the candidate set. An arc is a synthesis of many memories, so it is
10952
+ * embedding-near everything it summarizes, and merging one into a member would replace the
10953
+ * conclusion with one of its premises.
10954
+ *
10955
+ * `task` is excluded for the opposite reason: two open tasks with the same body are two things
10956
+ * to do, not one fact stored twice. Folding them would archive real work an agent still owes.
10957
+ * The `files_content_hash_active` index carves tasks out for the same reason, so structural and
10958
+ * semantic dedup agree about them.
10959
+ */
10960
+ const EXCLUDED_TYPES = ["arc", ...SLEEP_EXCLUDED_TYPES];
10132
10961
  const dedupMerge = (env) => Effect.gen(function* () {
10133
10962
  const corpus = yield* activeCorpus(env.deps.db);
10134
10963
  const order = new Map(corpus.map((row, offset) => [row.path, offset]));
10135
- const textOf = new Map(corpus.map((row) => [row.path, `${row.gist}\n${row.body_text}`]));
10136
- /**
10137
- * `arc` is excluded from the candidate set. An arc is a synthesis of many memories, so it is
10138
- * embedding-near everything it summarizes, and merging one into a member would replace the
10139
- * conclusion with one of its premises.
10140
- *
10141
- * `task` is excluded for the opposite reason: two open tasks with the same body are two things
10142
- * to do, not one fact stored twice. Folding them would archive real work an agent still owes.
10143
- * The `files_content_hash_active` index carves tasks out for the same reason, so structural and
10144
- * semantic dedup agree about them.
10145
- */
10964
+ const rowFor = new Map(corpus.map((row) => [row.path, row]));
10965
+ const textOf = new Map(corpus.map((row) => [row.path, textFor(row)]));
10966
+ const model = env.deps.model;
10146
10967
  const pairs = yield* neighborPairs(env.deps.db, {
10147
- floor: NEAR_DUPLICATE_THRESHOLD,
10968
+ floor: model === void 0 ? NEAR_DUPLICATE_THRESHOLD : DEDUP_COMPONENT_FLOOR,
10148
10969
  perSourceK: 5,
10149
- limit: 100 * 4,
10150
- excludeTypes: ["arc", ...SLEEP_EXCLUDED_TYPES]
10970
+ limit: model === void 0 ? 100 * 4 : DEDUP_PAIR_LIMIT,
10971
+ excludeTypes: EXCLUDED_TYPES
10151
10972
  });
10152
- /** Orient each unordered pair once, older path as keeper, and drop the mirrored duplicate. */
10973
+ /**
10974
+ * Orient each unordered pair once, older path as keeper, and drop the mirrored duplicate. The
10975
+ * kernel offers each pair to BOTH endpoints' neighborhoods, so `(a, b)` and `(b, a)` both arrive.
10976
+ */
10153
10977
  const seen = /* @__PURE__ */ new Set();
10154
10978
  const oriented = [];
10979
+ /** `keepPath dropPath` -> the mined similarity, so a group can report a measured value. */
10980
+ const simFor = /* @__PURE__ */ new Map();
10155
10981
  for (const pair of pairs) {
10156
10982
  const left = order.get(pair.src);
10157
10983
  const right = order.get(pair.dst);
10158
10984
  if (left === void 0 || right === void 0) continue;
10159
10985
  const [keepPath, dropPath] = left <= right ? [pair.src, pair.dst] : [pair.dst, pair.src];
10160
- const key = `${keepPath}${dropPath}`;
10986
+ const key = `${keepPath} ${dropPath}`;
10161
10987
  if (seen.has(key)) continue;
10162
10988
  seen.add(key);
10989
+ simFor.set(key, pair.sim);
10163
10990
  oriented.push({
10164
10991
  keepPath,
10165
10992
  dropPath,
@@ -10168,18 +10995,180 @@ const dedupMerge = (env) => Effect.gen(function* () {
10168
10995
  dropText: textOf.get(dropPath)
10169
10996
  });
10170
10997
  }
10171
- const decisions = mergeCandidates(oriented);
10172
- const vetoed = oriented.length - decisions.length;
10998
+ if (model === void 0) {
10999
+ /**
11000
+ * The deterministic path: the mined pairs at 0.92, oriented, through the same filter under its
11001
+ * own default threshold. Every count and every write here is what this phase produced before it
11002
+ * could call a model, which is what makes the existing dedup tests an oracle for the rest.
11003
+ */
11004
+ const decisions = mergeCandidates(oriented);
11005
+ return yield* commitMerges(env, decisions, {
11006
+ candidates: oriented.length,
11007
+ components: 0,
11008
+ llmGroups: 0,
11009
+ vetoed: oriented.length - decisions.length
11010
+ });
11011
+ }
11012
+ /**
11013
+ * The component graph: the mined edges at the recall floor, unioned with the frame-key exact
11014
+ * matches. A frame seed is an edge no cosine produced, so the union is what puts a slot collision
11015
+ * in front of the model even when the two bodies share little vocabulary.
11016
+ */
11017
+ const frameSeeds = yield* frameKeyPairs(env.deps.db);
11018
+ const edges = [...oriented.map((pair) => [pair.keepPath, pair.dropPath]), ...frameSeeds.flatMap((pair) => {
11019
+ /**
11020
+ * A seed is filtered by the SAME type exclusion the mining arm passes to SQL. The frame index
11021
+ * already carves out tasks, but not `arc` — and an arc shares a slot with any member it
11022
+ * summarizes, so an unfiltered seed would put the conclusion in a component with its premise
11023
+ * and invite the model to fold one into the other.
11024
+ */
11025
+ const src = rowFor.get(pair.src);
11026
+ const dst = rowFor.get(pair.dst);
11027
+ if (src === void 0 || dst === void 0) return [];
11028
+ if (EXCLUDED_TYPES.includes(src.memory_type) || EXCLUDED_TYPES.includes(dst.memory_type)) return [];
11029
+ return [[pair.src, pair.dst]];
11030
+ })];
11031
+ /**
11032
+ * Components of two or more are the units of work, truncated to {@link DEDUP_MAX_COMPONENT} at
11033
+ * their lowest paths and capped at {@link DEDUP_MAX_COMPONENTS} per night. `connectedComponents`
11034
+ * returns members sorted and components ordered by their smallest member, so both cuts are a
11035
+ * function of the corpus rather than of how the edges were enumerated.
11036
+ */
11037
+ const components = connectedComponents(edges).filter((members) => members.length >= 2).slice(0, 300).map((members) => members.slice(0, 8).flatMap((path) => {
11038
+ const row = rowFor.get(path);
11039
+ return row === void 0 ? [] : [row];
11040
+ })).filter((members) => members.length >= 2);
11041
+ /**
11042
+ * Whole components per call, so no group's members are split across two answers. Splitting one
11043
+ * would ask each half whether it holds a duplicate having hidden the other half.
11044
+ */
11045
+ const batches = packGroups(components, {
11046
+ maxMembers: 40,
11047
+ maxChars: DEDUP_BATCH_CHARS,
11048
+ charsOf: (row) => Math.min(textFor(row).length, DEDUP_MEMBER_CHARS)
11049
+ });
11050
+ const modelKey = modelFor(env.deps, "dedup-merge");
11051
+ let llmCalls = 0;
11052
+ let llmGroups = 0;
11053
+ let skipped = 0;
11054
+ /** Group-implied pairs, in batch then component then group order. */
11055
+ const groupPairs = [];
11056
+ /** Every path a surviving group claimed, so the mined arm cannot re-propose one. */
11057
+ const grouped = /* @__PURE__ */ new Set();
11058
+ for (const batch of batches) {
11059
+ /**
11060
+ * ONE keying across the whole batch, not one per component. Keys have to be unique inside the
11061
+ * answer's namespace, and per-component keying would mint `m1` several times over — so a model
11062
+ * naming `m1` would name several files and `resolveKeys` could not say which.
11063
+ */
11064
+ const keyed = keyMembers(batch.flat(), textFor, { charBudget: DEDUP_MEMBER_CHARS });
11065
+ /** Which component each offered key sits in. The containment check below reads this. */
11066
+ const componentOfKey = /* @__PURE__ */ new Map();
11067
+ const framed = [];
11068
+ let cursor = 0;
11069
+ for (const [offset, members] of batch.entries()) {
11070
+ const slice = keyed.keyed.slice(cursor, cursor + members.length);
11071
+ for (const member of slice) componentOfKey.set(member.key, offset);
11072
+ framed.push(slice);
11073
+ cursor += members.length;
11074
+ }
11075
+ llmCalls += 1;
11076
+ const partition = yield* batchCall(model, `dedup batch of ${batch.length} components`, {
11077
+ schema: MergePartition,
11078
+ system: DEDUP_SYSTEM,
11079
+ prompt: dedupPrompt(framed),
11080
+ modelKey,
11081
+ effort: "high",
11082
+ toolDescription: "Emit the merge groups: within each component, the members that are the same memory."
11083
+ });
11084
+ if (partition === void 0) {
11085
+ /**
11086
+ * One call's failure costs its own components and nothing else. `dedup-merge` is a HARD
11087
+ * prerequisite of compress and retention-triage, so failing the phase over one malformed tool
11088
+ * payload would cancel two later phases as well as this one's whole night.
11089
+ */
11090
+ skipped += 1;
11091
+ continue;
11092
+ }
11093
+ for (const group of partition.groups) {
11094
+ const members = resolveKeys(keyed, group.memberKeys);
11095
+ if (members.length < 2) continue;
11096
+ if (new Set(group.memberKeys.flatMap((key) => {
11097
+ const id = componentOfKey.get(key);
11098
+ return id === void 0 ? [] : [id];
11099
+ })).size !== 1) continue;
11100
+ /** The keeper is the OLDEST member: the lowest corpus offset, the same rule a pair uses. */
11101
+ const sorted = [...members].sort((left, right) => (order.get(left.path) ?? 0) - (order.get(right.path) ?? 0));
11102
+ const keeper = sorted[0];
11103
+ if (keeper === void 0) continue;
11104
+ llmGroups += 1;
11105
+ for (const member of sorted.slice(1)) {
11106
+ groupPairs.push({
11107
+ keepPath: keeper.path,
11108
+ dropPath: member.path,
11109
+ /**
11110
+ * The mined similarity when this pair was itself mined, else the floor. A frame-seeded
11111
+ * pair and a transitive pair inside a component were never scored, and the floor is the
11112
+ * honest value for "at least this near, never measured closer" — see
11113
+ * {@link DEDUP_ADMIT_FLOOR} for why the filter must not compare against it.
11114
+ */
11115
+ similarity: simFor.get(`${keeper.path} ${member.path}`) ?? .86,
11116
+ keepText: textOf.get(keeper.path),
11117
+ dropText: textOf.get(member.path)
11118
+ });
11119
+ grouped.add(member.path);
11120
+ }
11121
+ grouped.add(keeper.path);
11122
+ }
11123
+ }
11124
+ /**
11125
+ * Groups first, then the mined pairs above the DETERMINISTIC floor that no group claimed. The
11126
+ * comparison is against 0.92 and not against the recall floor, so a pair in the recall band the
11127
+ * model declined to group is not folded: the model's silence about it is the answer, and folding it
11128
+ * anyway would make the recall floor the merge floor.
11129
+ */
11130
+ const remaining = oriented.filter((pair) => pair.similarity > .92 && !grouped.has(pair.keepPath) && !grouped.has(pair.dropPath));
11131
+ const proposed = [...groupPairs, ...remaining];
11132
+ const decisions = mergeCandidates(proposed, { threshold: 0 });
11133
+ return {
11134
+ ...yield* commitMerges(env, decisions, {
11135
+ candidates: proposed.length,
11136
+ components: components.length,
11137
+ llmGroups,
11138
+ vetoed: proposed.length - decisions.length,
11139
+ skipped
11140
+ }),
11141
+ llmCalls
11142
+ };
11143
+ });
11144
+ /**
11145
+ * Archive each drop, stamp each keeper, and commit once.
11146
+ *
11147
+ * Shared by both arms so the WRITES do not fork on whether a model was bound: the two differ in how
11148
+ * they choose pairs and in nothing else. A pair that reached here has already passed the veto, the self
11149
+ * check, the both-roles guard, and the cap, whichever arm proposed it.
11150
+ *
11151
+ * The counts are real on a dry run — including the veto — because an operator sizing a night needs to
11152
+ * know what it would have folded. Only the writes are withheld.
11153
+ *
11154
+ * **A dry run here DOES spend model calls, and `entity-resolution`'s deliberately does not.** The two
11155
+ * choices differ because what a preview is worth differs. The number an operator wants from this phase
11156
+ * is how many folds a real night would make, and the model's partition is what decides that — a dry run
11157
+ * that skipped the call would report only the deterministic floor's folds and understate the night it is
11158
+ * previewing. `entity-resolution` refuses because its writes are identity rewrites, the one-way door
11159
+ * this codebase guards hardest: its dry run would have to bump `entity_corroboration` to be honest about
11160
+ * night two, and a counter bumped by a run that wrote nothing is a night of evidence the corpus never
11161
+ * saw.
11162
+ */
11163
+ const commitMerges = (env, decisions, base) => Effect.gen(function* () {
10173
11164
  if (decisions.length === 0) return emptyOutcome({
10174
- candidates: oriented.length,
11165
+ ...base,
10175
11166
  merged: 0,
10176
- vetoed,
10177
11167
  vanished: 0
10178
11168
  });
10179
11169
  if (env.dryRun) return emptyOutcome({
10180
- candidates: oriented.length,
11170
+ ...base,
10181
11171
  merged: decisions.length,
10182
- vetoed,
10183
11172
  vanished: 0
10184
11173
  });
10185
11174
  let merged = 0;
@@ -10193,45 +11182,528 @@ const dedupMerge = (env) => Effect.gen(function* () {
10193
11182
  yield* stampFile(env, decision.keepPath, [link("supersedes", hrefFor(archived)), meta("memhtml-updated", env.at)]);
10194
11183
  merged += 1;
10195
11184
  }
10196
- const counts = {
10197
- candidates: oriented.length,
11185
+ const final = {
11186
+ ...base,
10198
11187
  merged,
10199
- vetoed,
10200
11188
  vanished
10201
11189
  };
10202
11190
  return {
10203
- counts,
10204
- commitSha: yield* commitPhase(env, "dedup-merge", `fold ${merged} near-duplicates into canonicals`, counts),
11191
+ counts: final,
11192
+ commitSha: yield* commitPhase(env, "dedup-merge", `fold ${merged} near-duplicates into canonicals`, final),
10205
11193
  llmCalls: 0
10206
11194
  };
10207
11195
  });
10208
11196
 
10209
11197
  //#endregion
10210
- //#region packages/sleep/dist/phases/entity-resolution.js
11198
+ //#region packages/sleep/dist/phases/edge-typing.js
11199
+ /**
11200
+ * Phase 6, edge typing. Candidate pairs grouped and BATCHED, one structured verdict list per call
11201
+ * over the whole memory-rel vocabulary, then a deterministic promotion. One commit for the night's
11202
+ * promotions.
11203
+ *
11204
+ * Four stages, and keeping them separate is what makes the phase safe:
11205
+ *
11206
+ * 1. **Scan (SQL, no model).** The union of two candidate arms, deduplicated by unordered pair and
11207
+ * RANKED BY SIMILARITY before the cap: relationship mining's derived `relates_to` edges
11208
+ * ({@link minedPairs}) and the shared-entity scan ({@link sharedEntityPairs}). Neither arm
11209
+ * subsumes the other — two memories about one incident naming no common entity are invisible to
11210
+ * the join and obvious to the embedder, and a same-entity pair below the mining floor is the
11211
+ * reverse — so recall is the union rather than whichever signal happens to be stronger in a
11212
+ * corpus. Both arms exclude tasks and anti-join pairs that already carry an AUTHORED edge either
11213
+ * way.
11214
+ * 2. **Batch (deterministic).** Pairs sorted by the directory both endpoints share, then sliced at
11215
+ * {@link EDGE_PAIRS_PER_CALL} on the shared kernel, so topically related pairs land in one call.
11216
+ * One call per batch, never one per pair: at the measured 1,498 mined pairs a night, per-pair
11217
+ * judging is 1,498 calls and does not scale.
11218
+ * 3. **Judge (one model call per batch, isolated).** Each call is wrapped so one malformed tool
11219
+ * payload skips its BATCH and is counted. A night that typed nine batches and lost the tenth has
11220
+ * done nine batches of work; failing the phase would discard all of it.
11221
+ * 4. **Promote (deterministic, decided here and not by the model).** The model proposes a rel, a
11222
+ * direction, and a confidence; code decides what is written:
11223
+ * - `contradicts` above `EDGE_CONFIDENCE_FLOOR` bumps the corroboration counter and is
11224
+ * written into BOTH files only at `detections >= 2`. A single machine detection therefore
11225
+ * cannot reach the retention penalty, which counts only `derived = 0` file-borne edges. Both
11226
+ * endpoints must still be in the TREE, checked before either write, and the counter is marked
11227
+ * promoted only when both sides actually gained the link — otherwise the pair is left
11228
+ * re-eligible for a later night rather than recorded as half done.
11229
+ * - A DIRECTIONAL rel above the floor is written into the SUBJECT's file alone, per the
11230
+ * direction the model named. No corroboration gate: a `part_of` carries no penalty and is
11231
+ * cheap for a reviewer to delete, so a second night's wait would buy nothing.
11232
+ * - `none`, or anything below the floor, writes nothing at all and leaves the pair a mined
11233
+ * `relates_to`. That is the safe outcome and the one an unsure model is told to pick.
11234
+ * - A verdict naming a key the batch never offered resolves to nothing and is dropped, so a
11235
+ * hallucinated key cannot become a write. A pair the model omits is simply not typed tonight.
11236
+ *
11237
+ * **Determinism is the phase's, not the kernel's.** Both sorts below fix the batch boundaries and
11238
+ * the `m1`..`mN` keys, and the kernel preserves the order it is handed. Two runs over an unchanged
11239
+ * corpus therefore send the same prompt bytes in the same order.
11240
+ *
11241
+ * **Deferred: a `none` pair is re-judged on every later night, bounded by the candidate cap.** Neither
11242
+ * arm records that a pair was judged and answered `none`, so the pair stays a mined `relates_to` and
11243
+ * re-enters the union tomorrow. The cap is what bounds that cost — a night judges
11244
+ * {@link EDGE_TYPING_CANDIDATE_LIMIT} pairs whatever their history — and a judged-`none` watermark is
11245
+ * new durable state-plane surface, so it stays out of this change.
11246
+ *
11247
+ * **Detection only, still.** A promoted `contradicts` asserts the conflict and stops: nothing is
11248
+ * superseded, no `memhtml-valid-until` is closed, neither side is archived. Choosing the winner of a
11249
+ * contradiction is a one-way door on stored belief, and it belongs to an agent or a human, not to a
11250
+ * nightly job.
11251
+ *
11252
+ * This phase replaced `conflict-detection`, which asked one `generateObject` per pair for a stance
11253
+ * verdict over `{contradicts, entails, neutral}`. Contradiction is now one more verdict in the same
11254
+ * list, with the same corroboration gate. A run whose commits predate the rename carries
11255
+ * `Memhtml-Phase: conflict-detection` trailers, and `memhtml sleep resume` matches trailers by name,
11256
+ * so resuming a pre-rename run re-executes this phase; that is out of scope and costs a re-judge, not
11257
+ * a wrong write, because every write below is idempotent on its pair.
11258
+ */
11259
+ /**
11260
+ * Pairs offered per model call.
11261
+ *
11262
+ * Sized for the answer's attention rather than for the context window: thirty pairs is sixty
11263
+ * memories, each sliced to {@link EDGE_PAIR_SIDE_CHARS}, and the model has to hold a distinct
11264
+ * judgment for each one. A batch twice this size buys half the calls and invites the model to answer
11265
+ * the first ten pairs carefully and the rest by pattern.
11266
+ */
11267
+ const EDGE_PAIRS_PER_CALL = 30;
11268
+ /** The similarity floor a shared-entity pair must clear to be worth including. */
11269
+ const EDGE_COSINE_FLOOR = .8;
11270
+ /** Nearest same-entity neighbors considered per source, on the shared-entity arm. */
11271
+ const EDGE_PER_SOURCE_K = 5;
11272
+ /**
11273
+ * Pairs typed per cycle. The model-cost guard, unchanged from the per-pair phase's 200 even though
11274
+ * the calls are now ~7 instead of 200: the cap bounds how many AUTHORED EDGES one night can write
11275
+ * into the corpus, and that budget did not get cheaper because the judging did.
11276
+ */
11277
+ const EDGE_TYPING_CANDIDATE_LIMIT = 200;
11278
+ /**
11279
+ * Characters of EACH SIDE of a pair shown. The house per-member budget, applied per side.
11280
+ *
11281
+ * A pair's member text holds two memories, so the kernel's per-member slice would cut the whole
11282
+ * `src` + `dst` block at one budget and could truncate `dst` away entirely on a long `src` — a
11283
+ * verdict about a pair whose second half the model never saw. Slicing each side first bounds the
11284
+ * pair at twice this and guarantees both halves are present.
11285
+ */
11286
+ const EDGE_PAIR_SIDE_CHARS = 1200;
11287
+ /** Detections a machine-found contradiction needs before it is written into the files. */
11288
+ const PROMOTION_DETECTIONS = 2;
10211
11289
  /**
10212
- * Phase 3, entity resolution. Normalize entity names, then fuzzy-merge transitive alias clusters.
10213
- * ONE commit rewriting `memhtml-entity` values in place.
11290
+ * Authored edges one night may promote, across every batch and both kinds.
10214
11291
  *
10215
- * Two passes. The first lowercases and collapses whitespace, and is idempotent, so a second run
10216
- * touches nothing. The second is a union-find over pairs above {@link AUTO_MERGE_THRESHOLD}, so
10217
- * `A~B` and `B~C` land in one cluster; the name held by the most active files wins the root, ties
10218
- * broken lexicographically so a corpus that did not change resolves the same way twice.
11292
+ * The candidate cap bounds what is JUDGED and this bounds what is WRITTEN, and they are different
11293
+ * guards: a model that answered `caused_by` at confidence 1.0 for all 200 candidates would otherwise
11294
+ * add 200 `<link>` lines to the corpus in one commit, which is not a diff a human reviews. Hitting
11295
+ * the cap is visible as `capped` in the counts.
11296
+ */
11297
+ const EDGE_PROMOTION_CAP = 50;
11298
+ /**
11299
+ * The union of both candidate arms, deduplicated by UNORDERED pair and ranked `sim` DESC.
10219
11300
  *
10220
- * **Similarity is a normalized-string ratio, not an embedding cosine.** Entity names are short
10221
- * identifiers such as `checkout-api`, `checkout_api`, and `Checkout API`, where the whole signal is
10222
- * character overlap. An embedding of a two-token name is dominated by whatever domain the tokens evoke:
10223
- * `checkout-api` and `payments-api` sit high in vector space because both are payment services, and
10224
- * merging them would fuse two services' memories permanently. A character ratio cannot make that
10225
- * mistake. This is the packet's documented choice between the two options it offered.
11301
+ * The two arms orient their pairs differently — the shared-entity join emits `dst < src` and a mined
11302
+ * edge carries whichever orientation mining wrote so the dedup key sorts the endpoints. Without
11303
+ * that, one pair reaching both arms would be typed twice in one night, and the two verdicts could
11304
+ * disagree.
10226
11305
  *
10227
- * The 0.75-0.85 band is COUNTED, not merged. A review candidate is a human's call, because entity
10228
- * merges are a one-way door on stored identity and the failure mode of an over-eager threshold is
10229
- * silent and permanent.
11306
+ * The kept row is the FIRST one seen, and the mined arm is walked first, so a pair in both arms
11307
+ * carries mining's own orientation AND mining's own `sim`. That choice is arbitrary, because the
11308
+ * direction the phase writes comes from the model's `direction` field relative to this orientation,
11309
+ * so it must only be STABLE, which the sort makes it.
11310
+ *
11311
+ * **`sim` DESC is what makes the candidate cap select rather than truncate.** Both arms carry a
11312
+ * similarity on one scale — `sharedEntityPairs` reports the cosine `rankCandidatePairs` computed and
11313
+ * `minedPairs` reports the mined edge's own clamped cosine — so the union is rankable without
11314
+ * re-decoding a vector. A path-ordered union capped at {@link EDGE_TYPING_CANDIDATE_LIMIT} would spend
11315
+ * the whole night's model budget on whichever pairs sort alphabetically first, so a corpus whose
11316
+ * strongest candidates live under `services/` or `team/` would never have them judged at all, however
11317
+ * many nights ran. The tie-break is `src` ASC then `dst` ASC, which is `collectRanked`'s ordering in
11318
+ * `@memhtml/domain` — the house rule every other pair consumer already follows, so two runs over an
11319
+ * unchanged corpus select and batch the same pairs.
10230
11320
  */
10231
- /** At or above this ratio two names are the same entity. Auto-merged. */
11321
+ const unionPairs$1 = (arms) => {
11322
+ const seen = /* @__PURE__ */ new Set();
11323
+ const out = [];
11324
+ for (const arm of arms) for (const pair of arm) {
11325
+ const key = pair.src < pair.dst ? `${pair.src} ${pair.dst}` : `${pair.dst} ${pair.src}`;
11326
+ if (seen.has(key)) continue;
11327
+ seen.add(key);
11328
+ out.push(pair);
11329
+ }
11330
+ return out.sort((left, right) => {
11331
+ if (left.sim !== right.sim) return left.sim < right.sim ? 1 : -1;
11332
+ if (left.src !== right.src) return left.src < right.src ? -1 : 1;
11333
+ return left.dst < right.dst ? -1 : left.dst > right.dst ? 1 : 0;
11334
+ });
11335
+ };
11336
+ /**
11337
+ * The night's candidate pairs: the union of both arms, ranked `sim` DESC, then capped.
11338
+ *
11339
+ * The rank is {@link unionPairs}' and the cap is applied AFTER it, so the cap selects the strongest
11340
+ * {@link EDGE_TYPING_CANDIDATE_LIMIT} pairs the corpus offers rather than the alphabetically first
11341
+ * ones. That ordering is also the batch order's first input, so a night's strongest pairs are judged
11342
+ * even when the cap bites.
11343
+ *
11344
+ * Its own function so the SCAN is separable from the judging, and exported so a test asserting on a
11345
+ * batch boundary, a cap, or a skip count reads the same set the phase will type instead of
11346
+ * reconstructing it. A test that rebuilt this by hand would be a second implementation free to
11347
+ * disagree, and every count below is stated relative to it. The scan's own correctness has an
11348
+ * independent all-SQL oracle in `tests/neighbor-pairs.test.ts`; this is the composition of two
11349
+ * already-tested reads.
11350
+ */
11351
+ const edgeTypingCandidates = (db) => Effect.gen(function* () {
11352
+ const mined = yield* minedPairs(db, {
11353
+ rel: "relates_to",
11354
+ excludeTypes: SLEEP_EXCLUDED_TYPES
11355
+ });
11356
+ const shared = yield* sharedEntityPairs(db, {
11357
+ floor: EDGE_COSINE_FLOOR,
11358
+ perSourceK: 5,
11359
+ limit: 200,
11360
+ excludeTypes: SLEEP_EXCLUDED_TYPES
11361
+ });
11362
+ return unionPairs$1([mined, shared]).slice(0, 200);
11363
+ });
11364
+ /**
11365
+ * The grouping key for batching: the deepest directory both endpoints share, or `""` when they share
11366
+ * none.
11367
+ *
11368
+ * Deliberately NOT the graph community. `runRetentionPass` computes label propagation over the whole
11369
+ * memory-edge list plus PageRank plus the access plane, and this phase needs none of that — it would
11370
+ * be a second corpus-wide pass for a grouping hint, and it answers `undefined` for every pair in a
11371
+ * community below the size floor, which is most pairs in a small corpus. The shared directory is
11372
+ * already the corpus's own topical partition (`areas/deploy`, `areas/oncall`), it is a pure function
11373
+ * of two paths, and it puts related pairs in one call, which is all the batching needs from it. A
11374
+ * model shown thirty pairs from one area also has the area's context, which is the substantive half
11375
+ * of what community grouping was for.
11376
+ */
11377
+ const pairGroupKey = (pair) => {
11378
+ const left = pair.src.split("/");
11379
+ const right = pair.dst.split("/");
11380
+ const shared = [];
11381
+ for (let at = 0; at < Math.min(left.length, right.length) - 1; at += 1) {
11382
+ if (left[at] !== right[at]) break;
11383
+ shared.push(left[at]);
11384
+ }
11385
+ return shared.join("/");
11386
+ };
11387
+ const edgeTyping = (env) => Effect.gen(function* () {
11388
+ const model = env.deps.model;
11389
+ if (model === void 0) return {
11390
+ ...emptyOutcome({
11391
+ candidates: 0,
11392
+ judged: 0
11393
+ }),
11394
+ detail: "no model bound"
11395
+ };
11396
+ /**
11397
+ * Tasks are out of both candidate arms, inside {@link edgeTypingCandidates}. Every rel in the
11398
+ * vocabulary is a judgment about asserted facts, and a task asserts nothing: "these two
11399
+ * contradict" and "this one caused that one" have no true answer about intended work. A promoted
11400
+ * edge between two tasks would also be a memory-class edge with task endpoints written into both
11401
+ * files.
11402
+ */
11403
+ const candidates = yield* edgeTypingCandidates(env.deps.db);
11404
+ /**
11405
+ * The full path's count SHAPE, at zero. Every key the phase can report is present, because a
11406
+ * report reader comparing two nights reads a missing key as a phase that does not have that
11407
+ * concept rather than as a night that did none of it.
11408
+ */
11409
+ const zero = {
11410
+ candidates: 0,
11411
+ judged: 0,
11412
+ typed: 0,
11413
+ contradictions: 0,
11414
+ promoted: 0,
11415
+ skipped: 0,
11416
+ capped: 0,
11417
+ duplicates: 0
11418
+ };
11419
+ if (candidates.length === 0) return emptyOutcome(zero);
11420
+ if (env.dryRun) return emptyOutcome({
11421
+ ...zero,
11422
+ candidates: candidates.length
11423
+ });
11424
+ const corpus = yield* activeCorpus(env.deps.db);
11425
+ const textOf = new Map(corpus.map((row) => [row.path, `${row.gist}\n${row.body_text}`]));
11426
+ /**
11427
+ * A pair whose endpoint the corpus no longer holds is dropped before batching rather than inside
11428
+ * the loop. An earlier phase's archive is the normal case here (the index is refreshed once, in
11429
+ * preflight), and dropping it later would leave a hole in the numbered list the model is asked
11430
+ * about.
11431
+ */
11432
+ const withText = [];
11433
+ let skipped = 0;
11434
+ for (const pair of candidates) {
11435
+ const srcText = textOf.get(pair.src);
11436
+ const dstText = textOf.get(pair.dst);
11437
+ if (srcText === void 0 || dstText === void 0) {
11438
+ skipped += 1;
11439
+ continue;
11440
+ }
11441
+ withText.push({
11442
+ pair,
11443
+ srcText,
11444
+ dstText
11445
+ });
11446
+ }
11447
+ /**
11448
+ * The group is a SORT KEY, not a batch boundary, and that distinction is the phase's cost model.
11449
+ *
11450
+ * `dedup-merge` packs whole groups with the boundaries preserved because there a boundary is
11451
+ * EVIDENCE: two members in different components are known not to be near-duplicates. Here every
11452
+ * pair is judged on its own two memories, so a boundary carries no information the verdict needs —
11453
+ * and honoring it would cost a model call per group. On a corpus whose pairs spread over a dozen
11454
+ * directories that is a dozen calls for thirty pairs, which is the per-pair shape this phase
11455
+ * exists to replace. Sorting by the group instead keeps related pairs ADJACENT, so they land in
11456
+ * one call whenever they fit, and the call count is `ceil(pairs / EDGE_PAIRS_PER_CALL)`.
11457
+ *
11458
+ * The sort is this phase's and the kernel keeps the order it produces: group key first, then the
11459
+ * `src`/`dst` order `unionPairs` already fixed, so a night's batch boundaries and `m1`..`mN` keys
11460
+ * are a function of the corpus alone.
11461
+ */
11462
+ const sorted = [...withText].sort((left, right) => {
11463
+ const leftKey = pairGroupKey(left.pair);
11464
+ const rightKey = pairGroupKey(right.pair);
11465
+ if (leftKey !== rightKey) return leftKey < rightKey ? -1 : 1;
11466
+ if (left.pair.src !== right.pair.src) return left.pair.src < right.pair.src ? -1 : 1;
11467
+ return left.pair.dst < right.pair.dst ? -1 : left.pair.dst > right.pair.dst ? 1 : 0;
11468
+ });
11469
+ const batches = assembleBatches([sorted], { maxMembers: 30 });
11470
+ const modelKey = modelFor(env.deps, "edge-typing");
11471
+ let judged = 0;
11472
+ let typed = 0;
11473
+ let contradictions = 0;
11474
+ let promoted = 0;
11475
+ let capped = 0;
11476
+ /** Second-and-later verdicts naming a key their batch had already answered for. */
11477
+ let duplicates = 0;
11478
+ let llmCalls = 0;
11479
+ for (const batch of batches) {
11480
+ /** Opaque keys again, so a verdict cannot name a path. Each SIDE is sliced to its budget. */
11481
+ const keyed = keyMembers(batch, (candidate) => pairText(candidate.srcText.slice(0, EDGE_PAIR_SIDE_CHARS), candidate.dstText.slice(0, EDGE_PAIR_SIDE_CHARS)));
11482
+ llmCalls += 1;
11483
+ const answer = yield* batchCall(model, `edge-typing batch of ${batch.length}`, {
11484
+ schema: EdgeTyping,
11485
+ system: EDGE_TYPING_SYSTEM,
11486
+ prompt: edgeTypingPrompt(keyed.keyed),
11487
+ modelKey,
11488
+ effort: "medium",
11489
+ toolDescription: "Emit one relationship verdict per candidate pair."
11490
+ });
11491
+ if (answer === void 0) {
11492
+ skipped += batch.length;
11493
+ continue;
11494
+ }
11495
+ /**
11496
+ * The keys this batch has already answered for, so a SECOND verdict naming one is dropped.
11497
+ *
11498
+ * A verdict is one pair's answer, and nothing in the schema stops a model from emitting two for
11499
+ * one key. Acting on both would write two authored edges from one relationship — and since the
11500
+ * two are free to disagree about `direction`, `caused_by` could land in BOTH files, which says
11501
+ * each memory caused the other. `resolveKeys` does not help: it is called one key at a time
11502
+ * here, because a verdict names one pair, so its own repeat-collapsing never sees the pair.
11503
+ *
11504
+ * FIRST wins rather than last, and the choice is the same one {@link unionPairs} makes: the
11505
+ * batch's order is deterministic, so which verdict is first is reproducible, and a later verdict
11506
+ * cannot revise a write already committed to the tree. Repeats are counted in `duplicates`
11507
+ * rather than silently swallowed, so a model doing this is visible in a night's report.
11508
+ */
11509
+ const answered = /* @__PURE__ */ new Set();
11510
+ for (const verdict of answer.verdicts) {
11511
+ /**
11512
+ * The key is resolved through the kernel, so an invented key yields no candidate and no
11513
+ * write.
11514
+ */
11515
+ const [candidate] = resolveKeys(keyed, [verdict.pairKey]);
11516
+ if (candidate === void 0) continue;
11517
+ if (answered.has(verdict.pairKey)) {
11518
+ duplicates += 1;
11519
+ continue;
11520
+ }
11521
+ answered.add(verdict.pairKey);
11522
+ judged += 1;
11523
+ if (!assertsEdge(verdict)) continue;
11524
+ if (assertsContradiction(verdict)) {
11525
+ contradictions += 1;
11526
+ const row = (yield* bumpCorroboration(env.deps.db, {
11527
+ srcPath: candidate.pair.src,
11528
+ rel: "contradicts",
11529
+ dstPath: candidate.pair.dst,
11530
+ at: env.at
11531
+ }))[0];
11532
+ if (row === void 0 || row.detections < 2 || row.promoted === 1) continue;
11533
+ if (promoted + typed >= 50) {
11534
+ capped += 1;
11535
+ continue;
11536
+ }
11537
+ /**
11538
+ * **BOTH endpoints, or nothing at all — checked BEFORE either write.**
11539
+ *
11540
+ * A `contradicts` is symmetric, and the phase's own promotion rule is that a reader arriving
11541
+ * at either file sees it. So the pair is all-or-nothing, and the check has to come first
11542
+ * because the alternative is not recoverable: stamping `src` and then finding `dst` gone
11543
+ * leaves a `<link>` pointing at a path the tree does not hold — a dangling href committed by
11544
+ * the commit that created it — while the other half of the conflict is invisible.
11545
+ *
11546
+ * A missing endpoint is ORDINARY here, not exceptional. Every phase reads its candidates from
11547
+ * an index refreshed once in preflight and not again, so a file an earlier phase archived is
11548
+ * still listed active at its old path when this phase reads it. `readFileBytes` answers
11549
+ * `undefined` for exactly that case, and the TREE is the system of record.
11550
+ */
11551
+ /**
11552
+ * **BOTH endpoints, or nothing at all — checked BEFORE either write.**
11553
+ *
11554
+ * A `contradicts` is symmetric, and the phase's own promotion rule is that a reader arriving
11555
+ * at either file sees it. So the pair is all-or-nothing, and the check has to come first
11556
+ * because the alternative is not recoverable: stamping `src` and then finding `dst` gone
11557
+ * leaves a `<link>` pointing at a path the tree does not hold — a dangling href committed by
11558
+ * the commit that created it — while the other half of the conflict is invisible.
11559
+ *
11560
+ * A missing endpoint is ORDINARY here, not exceptional. Every phase reads its candidates from
11561
+ * an index refreshed once in preflight and not again, so a file an earlier phase archived is
11562
+ * still listed active at its old path when this phase reads it. `readFileBytes` answers
11563
+ * `undefined` for exactly that case, and the TREE is the system of record.
11564
+ */
11565
+ const haveSrc = yield* readFileBytes(env, candidate.pair.src);
11566
+ const haveDst = yield* readFileBytes(env, candidate.pair.dst);
11567
+ if (haveSrc === void 0 || haveDst === void 0) continue;
11568
+ const wroteSrc = yield* stampFile(env, candidate.pair.src, [link("contradicts", hrefFor(candidate.pair.dst)), meta("memhtml-updated", env.at)]);
11569
+ const wroteDst = yield* stampFile(env, candidate.pair.dst, [link("contradicts", hrefFor(candidate.pair.src)), meta("memhtml-updated", env.at)]);
11570
+ /**
11571
+ * The counter is promoted only when BOTH sides gained the edge on this run. `stampFile`'s
11572
+ * `false` also covers "the head already said this", so a pair whose files were somehow
11573
+ * stamped without the counter being promoted stays un-promoted — and therefore RE-ELIGIBLE,
11574
+ * which is the outcome that lets a later night with a refreshed index finish the job rather
11575
+ * than record a half-written edge as done.
11576
+ */
11577
+ /**
11578
+ * The counter is promoted only when BOTH sides gained the edge on this run. `stampFile`'s
11579
+ * `false` also covers "the head already said this", so a pair whose files were somehow
11580
+ * stamped without the counter being promoted stays un-promoted — and therefore RE-ELIGIBLE,
11581
+ * which is the outcome that lets a later night with a refreshed index finish the job rather
11582
+ * than record a half-written edge as done.
11583
+ */
11584
+ if (!wroteSrc || !wroteDst) continue;
11585
+ yield* markPromoted(env.deps.db, {
11586
+ srcPath: candidate.pair.src,
11587
+ rel: "contradicts",
11588
+ dstPath: candidate.pair.dst,
11589
+ at: env.at
11590
+ });
11591
+ promoted += 1;
11592
+ continue;
11593
+ }
11594
+ if (!isDirectionalRel(verdict.rel)) continue;
11595
+ if (promoted + typed >= 50) {
11596
+ capped += 1;
11597
+ continue;
11598
+ }
11599
+ /**
11600
+ * ONE file, the subject's. A directional rel read from the wrong end inverts its meaning —
11601
+ * `caused_by` written into the cause instead of the effect says the opposite of what the
11602
+ * model answered — so the direction decides the file and the href together, from one
11603
+ * statement, and cannot disagree with itself.
11604
+ */
11605
+ const [subject, object] = verdict.direction === "src_to_dst" ? [candidate.pair.src, candidate.pair.dst] : [candidate.pair.dst, candidate.pair.src];
11606
+ if (yield* stampFile(env, subject, [link(verdict.rel, hrefFor(object)), meta("memhtml-updated", env.at)])) typed += 1;
11607
+ }
11608
+ }
11609
+ const counts = {
11610
+ candidates: candidates.length,
11611
+ judged,
11612
+ typed,
11613
+ contradictions,
11614
+ promoted,
11615
+ skipped,
11616
+ capped,
11617
+ duplicates
11618
+ };
11619
+ if (promoted === 0 && typed === 0) return {
11620
+ counts,
11621
+ commitSha: null,
11622
+ llmCalls
11623
+ };
11624
+ return {
11625
+ counts,
11626
+ commitSha: yield* commitPhase(env, "edge-typing", `promote ${typed} typed edges and ${promoted} corroborated contradictions`, counts),
11627
+ llmCalls
11628
+ };
11629
+ });
11630
+
11631
+ //#endregion
11632
+ //#region packages/sleep/dist/phases/entity-resolution.js
11633
+ /**
11634
+ * Phase 3, entity resolution. Cluster one entity type's names into subjects, then rewrite each alias
11635
+ * onto its canonical. ONE commit rewriting `memhtml-entity` values in place.
11636
+ *
11637
+ * Three stages, and the separation is what makes the phase safe:
11638
+ *
11639
+ * 1. **Pre (deterministic, cheap).** Normalize every name, exact-merge the ones that normalize
11640
+ * together, auto-merge pairs at or above {@link AUTO_MERGE_THRESHOLD} character overlap, and merge
11641
+ * every pair a person file DECLARES ({@link aliasPairs}). This pass alone is the whole phase when no
11642
+ * model is bound, so a credential-free run still collapses `Checkout API` onto `checkout api` and
11643
+ * still applies a seeded declaration. The same pass computes one MEMORY CENTROID per name, in
11644
+ * O(files) and never per pair.
11645
+ * 2. **Core (one model call per entity type, sharded at {@link ENTITY_BATCH_SIZE}).** The model sees
11646
+ * every name of one type as a numbered member list and returns a PARTITION into subjects. Never one
11647
+ * call per pair: 59 entities on the measured corpus is one call, and the pair space is 1,711.
11648
+ * 3. **Post (deterministic, the one-way-door guards).** Which name survives a merge is decided by
11649
+ * {@link unionPairs}'s weight-then-lexicographic rule and never by the model. All THREE pair sources
11650
+ * — the character pass, the declarations, and the model — feed that ONE union-find, so no two of them
11651
+ * can disagree about a canonical. A merge backed by a DECLARED alias applies at once and is never
11652
+ * counted; a merge the model alone proposes is counted in `state.entity_corroboration` and applies
11653
+ * only once {@link ENTITY_PROMOTION_DETECTIONS} different nights have reached it.
11654
+ *
11655
+ * **Why the model, and why centroids as its evidence.** Character overlap is measurably wrong on the
11656
+ * case this phase exists for. Measured on the live corpus: `laith` against `laith al-saadoon` scores
11657
+ * 0.476 and `sanju` against `sanju kumar` 0.625 — below even the 0.75 review band — so a short name and
11658
+ * its full form are structurally invisible to a character ratio, and the phase minted two person files
11659
+ * for one person. The signal that does separate them is not the name string but WHAT IS WRITTEN under
11660
+ * each name: the centroid of the vectors of every memory claiming a name. Two spellings of one person
11661
+ * have near-identical centroids; `checkout-api` and `payments-api` do not, however close their strings
11662
+ * or their domain.
11663
+ *
11664
+ * The centroid is EVIDENCE HANDED TO THE MODEL, not a threshold. A cosine floor over centroids would
11665
+ * make exactly the mistake a bare character ratio avoids, because two services in one domain are
11666
+ * written about in the same terms. What the deterministic code keeps is the part a threshold is good
11667
+ * at: the confidence floor, the corroboration count, and the choice of which name survives.
11668
+ *
11669
+ * **Every band that does not merge is COUNTED, not merged.** The 0.75-0.85 character band the model did
11670
+ * not cluster, and a cluster below {@link ENTITY_CONFIDENCE_FLOOR}, both land in `reviewCandidates`. An
11671
+ * entity merge is a one-way door on stored identity: no later commit separates two subjects whose
11672
+ * memories were fused, and the failure mode of an over-eager gate is silent and permanent.
11673
+ */
11674
+ /** At or above this ratio two names are the same entity. Auto-merged with no model call. */
10232
11675
  const AUTO_MERGE_THRESHOLD = .85;
10233
11676
  /** At or above this ratio, below the auto threshold: counted for review, left unmerged. */
10234
11677
  const REVIEW_THRESHOLD = .75;
11678
+ /**
11679
+ * Confidence a model-proposed cluster must clear before it is even counted toward a merge.
11680
+ *
11681
+ * The same floor {@link STANCE_CONFIDENCE_FLOOR} sets for a contradiction, for the same reason: a false
11682
+ * merge is worse than a missed one, and this floor and the corroboration gate are two independent
11683
+ * guards on one door. A cluster below it is reported as a review candidate and nothing else.
11684
+ */
11685
+ const ENTITY_CONFIDENCE_FLOOR = .7;
11686
+ /** Nights a model-only merge must be proposed on before it is written into the files. */
11687
+ const ENTITY_PROMOTION_DETECTIONS = 2;
11688
+ /**
11689
+ * Names offered per model call. One type's whole name list fits one call at the measured corpus size
11690
+ * (59 entities); this is the shard boundary for a corpus that outgrows that.
11691
+ */
11692
+ const ENTITY_BATCH_SIZE = 500;
11693
+ /** Memory titles shown per name. Enough to say what a name is about, few enough to stay cheap. */
11694
+ const ENTITY_SAMPLE_TITLES = 3;
11695
+ /** Centroid neighbors shown per name, nearest first. */
11696
+ const ENTITY_NEIGHBORS = 3;
11697
+ /** Characters of each member's evidence block shown. A name plus three titles fits comfortably. */
11698
+ const ENTITY_MEMBER_CHARS = 600;
11699
+ /**
11700
+ * The one entity type the alias oracle speaks for, derived from the prefix rather than retyped.
11701
+ *
11702
+ * A declaration lives in a person file, and `resources/people/` is the only directory the format gives
11703
+ * a hand-edited identity surface. A service has no equivalent file to declare from, so offering an
11704
+ * `aliases` line for one would show the model a field that is always empty.
11705
+ */
11706
+ const PERSON_TYPE = PERSON_ENTITY_PREFIX.slice(0, -1);
10235
11707
  /** Lowercase, NFC-normalize, collapse internal whitespace, trim. The pre-compare form. */
10236
11708
  const normalizeEntityName = (name) => name.normalize("NFC").toLowerCase().replace(/\s+/g, " ").trim();
10237
11709
  /**
@@ -10241,6 +11713,9 @@ const normalizeEntityName = (name) => name.normalize("NFC").toLowerCase().replac
10241
11713
  * separator or casing change actually is. `checkout-api` against `checkout api` differs in one
10242
11714
  * character and scores 0.92, while `checkout-api` against `payments-api` shares only the suffix and
10243
11715
  * scores 0.67. That sits below both thresholds, so two distinct services stay separate.
11716
+ *
11717
+ * It is the pre-pass and not the decision core. Its blind spot is short-name-against-full-name, which
11718
+ * is what the model call exists for.
10244
11719
  */
10245
11720
  const nameSimilarity = (left, right) => {
10246
11721
  if (left === right) return 1;
@@ -10258,14 +11733,24 @@ const nameSimilarity = (left, right) => {
10258
11733
  }
10259
11734
  return 2 * (previous[columns - 1] ?? 0) / (left.length + right.length);
10260
11735
  };
11736
+ /** A pair as a stable key, so a set of pairs is order-independent. */
11737
+ const pairKey = (left, right) => left < right ? `${left}\u0000${right}` : `${right}\u0000${left}`;
10261
11738
  /**
10262
- * Union-find over the auto-merge pairs. The higher-count name wins the root; a tie goes to the
10263
- * lexicographically smaller name, so the partition is a function of the input alone.
11739
+ * Union-find over an explicit pair list. The higher-count name wins the root; a tie goes to the
11740
+ * lexicographically smaller name.
11741
+ *
11742
+ * **This is the one place a canonical name is chosen, and every merge routes through it.** The
11743
+ * character pass, the alias oracle, and the model all contribute PAIRS to one call, so `A~B` from the
11744
+ * character ratio and `B~C` from the model land in one cluster with one root. Two separate union-finds
11745
+ * would let the two passes disagree about which name survives, and the rewrite would then depend on
11746
+ * which pass ran first.
11747
+ *
11748
+ * `names` is walked in sorted order and the pairs in the order given, so the partition is a function of
11749
+ * the input alone and a corpus that did not change resolves the same way twice.
10264
11750
  */
10265
- const resolveClusters = (counts) => {
11751
+ const unionPairs = (counts, pairs) => {
10266
11752
  const names = [...counts.keys()].sort();
10267
11753
  const parent = /* @__PURE__ */ new Map();
10268
- let reviewCandidates = 0;
10269
11754
  const find = (name) => {
10270
11755
  let current = name;
10271
11756
  while ((parent.get(current) ?? current) !== current) {
@@ -10275,33 +11760,262 @@ const resolveClusters = (counts) => {
10275
11760
  }
10276
11761
  return current;
10277
11762
  };
10278
- const union = (left, right) => {
11763
+ for (const [left, right] of pairs) {
10279
11764
  const rootLeft = find(left);
10280
11765
  const rootRight = find(right);
10281
- if (rootLeft === rootRight) return;
11766
+ if (rootLeft === rootRight) continue;
10282
11767
  const weightLeft = counts.get(rootLeft) ?? 0;
10283
11768
  const weightRight = counts.get(rootRight) ?? 0;
10284
11769
  if (weightLeft > weightRight || weightLeft === weightRight && rootLeft < rootRight) parent.set(rootRight, rootLeft);
10285
11770
  else parent.set(rootLeft, rootRight);
10286
- };
10287
- for (let outer = 0; outer < names.length; outer += 1) for (let inner = outer + 1; inner < names.length; inner += 1) {
10288
- const left = names[outer];
10289
- const right = names[inner];
10290
- if (left === void 0 || right === void 0) continue;
10291
- const similarity = nameSimilarity(left, right);
10292
- if (similarity >= .85) union(left, right);
10293
- else if (similarity >= .75) reviewCandidates += 1;
10294
11771
  }
10295
11772
  const aliasToCanonical = /* @__PURE__ */ new Map();
10296
11773
  for (const name of names) {
10297
11774
  const root = find(name);
10298
11775
  if (root !== name) aliasToCanonical.set(name, root);
10299
11776
  }
11777
+ return aliasToCanonical;
11778
+ };
11779
+ /**
11780
+ * Every pair of one type's names, split at the two thresholds. Names are walked in sorted order, so
11781
+ * the pair list is a function of the name set alone.
11782
+ */
11783
+ const characterPairs = (names) => {
11784
+ const sorted = [...names].sort();
11785
+ const auto = [];
11786
+ const review = [];
11787
+ for (let outer = 0; outer < sorted.length; outer += 1) for (let inner = outer + 1; inner < sorted.length; inner += 1) {
11788
+ const left = sorted[outer];
11789
+ const right = sorted[inner];
11790
+ if (left === void 0 || right === void 0) continue;
11791
+ const similarity = nameSimilarity(left, right);
11792
+ if (similarity >= .85) auto.push([left, right]);
11793
+ else if (similarity >= .75) review.push([left, right]);
11794
+ }
10300
11795
  return {
10301
- aliasToCanonical,
10302
- reviewCandidates
11796
+ auto,
11797
+ review
10303
11798
  };
10304
11799
  };
11800
+ /**
11801
+ * One memory centroid per normalized name, per entity type, in ONE pass over the claims.
11802
+ *
11803
+ * **Members are summed in SORTED PATH order, and that is a determinism requirement rather than tidiness.**
11804
+ * Floating-point addition is not associative — `1 + 1e-16 + 1e-16` is `1` and `1e-16 + 1e-16 + 1` is
11805
+ * `1.0000000000000002` (probed on node 24.19.0, in float64) — so a centroid summed in a different order
11806
+ * is different bytes, and different bytes reorder the nearest-neighbor list the model is shown. Two
11807
+ * nights over an unchanged corpus have to produce the same prompt, so the order is fixed here rather
11808
+ * than inherited from whatever order the rows arrived in.
11809
+ *
11810
+ * **A path claiming one name twice contributes its vector ONCE.** A file may carry both
11811
+ * `Service:Checkout-API` and `service:checkout-api`, two `file_entities` rows that normalize together,
11812
+ * and summing that memory twice would let one file's authoring quirk double its own weight in the
11813
+ * centroid.
11814
+ *
11815
+ * Accumulated in float64 over float32 inputs, because the sum of n unit vectors is not a unit vector
11816
+ * and float32 would round each partial sum. Cost is O(files), never O(names²).
11817
+ */
11818
+ const entityCentroids = (claims, vectorForPath, options) => {
11819
+ const sampleTitles = options?.sampleTitles ?? 3;
11820
+ /** `type` -> normalized name -> its distinct claiming paths, and each path's title. */
11821
+ const byType = /* @__PURE__ */ new Map();
11822
+ for (const claim of claims) {
11823
+ const name = normalizeEntityName(claim.entity_name);
11824
+ if (name === "") continue;
11825
+ let names = byType.get(claim.entity_type);
11826
+ if (names === void 0) {
11827
+ names = /* @__PURE__ */ new Map();
11828
+ byType.set(claim.entity_type, names);
11829
+ }
11830
+ let paths = names.get(name);
11831
+ if (paths === void 0) {
11832
+ paths = /* @__PURE__ */ new Map();
11833
+ names.set(name, paths);
11834
+ }
11835
+ paths.set(claim.path, claim.title);
11836
+ }
11837
+ const out = /* @__PURE__ */ new Map();
11838
+ for (const [entityType, names] of byType) {
11839
+ const centroids = [];
11840
+ for (const name of [...names.keys()].sort()) {
11841
+ const paths = names.get(name);
11842
+ if (paths === void 0) continue;
11843
+ const sorted = [...paths.keys()].sort();
11844
+ centroids.push({
11845
+ name,
11846
+ memories: sorted.length,
11847
+ titles: sorted.slice(0, sampleTitles).flatMap((path) => {
11848
+ const title = paths.get(path);
11849
+ return title === void 0 || title.trim() === "" ? [] : [title.trim()];
11850
+ }),
11851
+ vec: meanVector(sorted.flatMap((path) => vectorForPath.get(path) ?? []))
11852
+ });
11853
+ }
11854
+ out.set(entityType, centroids);
11855
+ }
11856
+ return out;
11857
+ };
11858
+ /**
11859
+ * The L2-normalized mean of vectors summed in the order given, or absent for an empty or zero set.
11860
+ *
11861
+ * Normalized so a cosine between two centroids does not depend on how many memories each was built
11862
+ * from, and so a one-memory name and a fifty-memory name are comparable at all.
11863
+ */
11864
+ const meanVector = (vectors) => {
11865
+ const first = vectors[0];
11866
+ if (first === void 0) return void 0;
11867
+ const sum = new Float64Array(first.length);
11868
+ for (const vector of vectors) {
11869
+ const width = Math.min(sum.length, vector.length);
11870
+ for (let at = 0; at < width; at += 1) sum[at] = sum[at] + vector[at];
11871
+ }
11872
+ let norm = 0;
11873
+ for (const component of sum) norm += component * component;
11874
+ if (norm === 0) return void 0;
11875
+ const scale = 1 / Math.sqrt(norm);
11876
+ for (let at = 0; at < sum.length; at += 1) sum[at] = sum[at] * scale;
11877
+ return sum;
11878
+ };
11879
+ /**
11880
+ * The `k` nearest same-type centroids to `of`, ordered `sim` DESC then `name` ASC.
11881
+ *
11882
+ * The tie-break matches the pair kernel's (`sim` DESC, then the other key ASC), so two names whose
11883
+ * centroids are equidistant are listed in one fixed order and the prompt's bytes do not depend on the
11884
+ * input order. A name with no centroid has no neighbors, and a candidate with no centroid is not one.
11885
+ *
11886
+ * `cosine` from the domain rather than a dot product over the already-normalized vectors, so this
11887
+ * similarity is the same arithmetic every other reader of this vector space performs. Cost is O(n²) per
11888
+ * type, bounded by {@link ENTITY_BATCH_SIZE} being the point at which a type is sharded for the CALL —
11889
+ * at the measured 59 entities the pair space is 1,711 dot products, which is the work the phase used to
11890
+ * do with character ratios.
11891
+ */
11892
+ const nearestCentroids = (centroids, of, k) => {
11893
+ const subjectVec = centroids.find((candidate) => candidate.name === of)?.vec;
11894
+ if (subjectVec === void 0) return [];
11895
+ const scored = [];
11896
+ for (const candidate of centroids) {
11897
+ if (candidate.name === of || candidate.vec === void 0) continue;
11898
+ scored.push({
11899
+ name: candidate.name,
11900
+ sim: cosine(subjectVec, candidate.vec)
11901
+ });
11902
+ }
11903
+ scored.sort((left, right) => left.sim !== right.sim ? left.sim < right.sim ? 1 : -1 : left.name < right.name ? -1 : 1);
11904
+ return scored.slice(0, k);
11905
+ };
11906
+ /**
11907
+ * One name's evidence block, as the model reads it.
11908
+ *
11909
+ * Neighbors are named by NAME and not by member key. A key names a member of THIS batch, and a
11910
+ * centroid neighbor may sit in another shard, so offering its key would invite an answer referencing a
11911
+ * member the batch never contained. The similarity is rendered at two decimals so a corpus whose
11912
+ * vectors moved in the sixteenth place does not change the prompt's bytes.
11913
+ */
11914
+ const entityMemberText = (input) => {
11915
+ const lines = [`name: ${input.centroid.name}`, `memories: ${input.centroid.memories}`];
11916
+ if (input.centroid.titles.length > 0) lines.push("titles:", ...input.centroid.titles.map((title) => `- ${title}`));
11917
+ if (input.neighbors.length > 0) lines.push("nearest by memory centroid:", ...input.neighbors.map((one) => `- ${one.name} (${one.sim.toFixed(2)})`));
11918
+ if (input.aliases.length > 0) lines.push(`declared aliases: ${input.aliases.join(", ")}`);
11919
+ return lines.join("\n");
11920
+ };
11921
+ /**
11922
+ * Decompose one cluster of member names into oriented merges: the highest-count name survives, ties
11923
+ * broken lexicographically, and every other member rewrites onto it.
11924
+ *
11925
+ * **The model's `canonicalKey` does not decide this**, and the reason is worth stating. The canonical
11926
+ * name is what every `memhtml-entity` meta in the corpus is rewritten TO, and it becomes a person file's
11927
+ * path once person-links runs. Letting the model choose it would make a nightly job's write target a
11928
+ * model's answer. What `canonicalKey` is for is validation: a cluster whose canonical is not one of its
11929
+ * own members is a self-contradicting answer, and the caller drops it.
11930
+ *
11931
+ * A cluster of fewer than two names produces no merges, which is how a model refuses.
11932
+ */
11933
+ const decomposeCluster = (members, counts) => {
11934
+ const distinct = [...new Set(members)].sort();
11935
+ if (distinct.length < 2) return [];
11936
+ let canonical = distinct[0];
11937
+ for (const name of distinct.slice(1)) {
11938
+ const held = counts.get(canonical) ?? 0;
11939
+ const weight = counts.get(name) ?? 0;
11940
+ if (weight > held || weight === held && name < canonical) canonical = name;
11941
+ }
11942
+ return distinct.flatMap((name) => name === canonical ? [] : [{
11943
+ alias: name,
11944
+ canonical
11945
+ }]);
11946
+ };
11947
+ /** True when some declaration names both. The alias oracle's whole question. */
11948
+ const aliasBacked = (groups, left, right) => groups.some((group) => group.has(left) && group.has(right));
11949
+ /**
11950
+ * Every pair of one type's names that a DECLARATION backs: the alias oracle as a pair source of its
11951
+ * own, answerable with no model and no corroboration.
11952
+ *
11953
+ * **This is what makes the oracle an oracle.** Issue #43 states that entity resolution consults
11954
+ * declared aliases FIRST and that an alias-backed merge auto-commits regardless of string distance. A
11955
+ * declaration read only where the model core reads it would deliver neither half: a credential-free
11956
+ * night would leave `laith` and `laith al-saadoon` split with a person file sitting in the corpus
11957
+ * saying they are one person, and even a night WITH credentials would apply the declaration only if
11958
+ * the model happened to propose that pair — so the operator surface the format invites someone to
11959
+ * hand-edit would work or not work depending on a model's attention.
11960
+ *
11961
+ * A pair the character pass already merges is left out, because counting it as an alias merge as well
11962
+ * would report one merge twice. Names are walked in sorted order, so the pair list is a function of the
11963
+ * name set and the declarations alone.
11964
+ */
11965
+ const aliasPairs = (groups, counts) => {
11966
+ const names = [...counts.keys()].sort();
11967
+ const out = [];
11968
+ for (let outer = 0; outer < names.length; outer += 1) for (let inner = outer + 1; inner < names.length; inner += 1) {
11969
+ const left = names[outer];
11970
+ const right = names[inner];
11971
+ if (left === void 0 || right === void 0) continue;
11972
+ if (!aliasBacked(groups, left, right)) continue;
11973
+ if (nameSimilarity(left, right) >= .85) continue;
11974
+ out.push([left, right]);
11975
+ }
11976
+ return out;
11977
+ };
11978
+ /**
11979
+ * Read the alias declarations out of the person files.
11980
+ *
11981
+ * Parsed with the production parser rather than scanned for meta lines, because `memhtml-alias` is
11982
+ * repeatable and the surgical `readMeta` reads only the first of a name. A file that does not parse is
11983
+ * skipped: it is not indexed either, so it has no entities for a merge to be about.
11984
+ *
11985
+ * **Read from the FILES at phase time, and deliberately not projected to SQL.** The whole point of the
11986
+ * oracle is that a person file is hand-editable and operator-seedable — someone with an authoritative
11987
+ * directory writes the aliases in and the phase converges to auto-merge. A projection would put the
11988
+ * declaration behind an index refresh, so an alias written and committed during a session would not be
11989
+ * evidence until the next rebuild, and the one surface an operator is invited to edit would be the one
11990
+ * with a stale read. There are as many person files as there are people, and the phase reads each once.
11991
+ *
11992
+ * Read on EVERY run, including a credential-free one and a dry run, because the declarations are a
11993
+ * deterministic pair source rather than the model core's evidence. Reading them is pure, so a dry run
11994
+ * can count what they would merge without writing anything.
11995
+ */
11996
+ const readAliasGroups = (env) => Effect.gen(function* () {
11997
+ const paths = yield* peoplePaths(env.deps.db).pipe(Effect.orElseSucceed(() => []));
11998
+ const groups = [];
11999
+ for (const row of paths) {
12000
+ const html = yield* readFileBytes(env, row.path).pipe(Effect.orElseSucceed(() => void 0));
12001
+ if (html === void 0) continue;
12002
+ const doc = yield* parseMemory(html).pipe(Effect.orElseSucceed(() => void 0));
12003
+ if (doc === void 0) continue;
12004
+ const group = /* @__PURE__ */ new Set();
12005
+ for (const entity of doc.entities) {
12006
+ if (!entity.startsWith(PERSON_ENTITY_PREFIX)) continue;
12007
+ const name = normalizeEntityName(entity.slice(PERSON_ENTITY_PREFIX.length));
12008
+ if (name !== "") group.add(name);
12009
+ }
12010
+ if (group.size === 0) continue;
12011
+ for (const alias of doc.aliases) {
12012
+ const name = normalizeEntityName(alias);
12013
+ if (name !== "") group.add(name);
12014
+ }
12015
+ if (group.size > 1) groups.push(group);
12016
+ }
12017
+ return groups;
12018
+ });
10305
12019
  /** The `type:name` form a `memhtml-entity` meta carries. */
10306
12020
  const entityRef = (entityType, entityName) => `${entityType}:${entityName}`;
10307
12021
  const entityResolution = (env) => Effect.gen(function* () {
@@ -10316,7 +12030,7 @@ const entityResolution = (env) => Effect.gen(function* () {
10316
12030
  if (bucket === void 0) byType.set(entity.entity_type, [entity]);
10317
12031
  else bucket.push(entity);
10318
12032
  }
10319
- /** `path -> [(oldRef, newRef)]`, accumulated across the normalize and merge passes. */
12033
+ /** `path -> [(oldRef, newRef)]`, accumulated across every pass. */
10320
12034
  const rewrites = /* @__PURE__ */ new Map();
10321
12035
  const addRewrite = (path, from, to) => {
10322
12036
  if (from === to) return;
@@ -10326,7 +12040,44 @@ const entityResolution = (env) => Effect.gen(function* () {
10326
12040
  };
10327
12041
  let normalized = 0;
10328
12042
  let fuzzyMerges = 0;
12043
+ let llmMerges = 0;
12044
+ let aliasMerges = 0;
12045
+ let pendingCorroboration = 0;
10329
12046
  let reviewCandidates = 0;
12047
+ let llmCalls = 0;
12048
+ /**
12049
+ * The model core is skipped entirely on a dry run and when no model is bound, and both leave the
12050
+ * deterministic passes running. A dry run must make no model call and bump no counter, because a
12051
+ * counter bumped by a run that wrote nothing would be a night of corroboration the corpus never
12052
+ * saw. An absent model is a credential-free run, not a broken one.
12053
+ *
12054
+ * **`dedup-merge`'s dry run makes the opposite choice and DOES spend its calls**, because there the
12055
+ * model's partition is the number an operator is asking for and a call costs nothing but money. Here
12056
+ * an honest preview would have to bump the corroboration counter — the merge count for night two
12057
+ * depends on it — and this phase's writes are identity rewrites, which is the one-way door where
12058
+ * manufacturing a night of evidence is worse than declining to preview.
12059
+ */
12060
+ const model = env.dryRun ? void 0 : env.deps.model;
12061
+ const modelKey = modelFor(env.deps, "entity-resolution");
12062
+ /**
12063
+ * The declarations, read UNCONDITIONALLY — before the model core, and whether or not one exists.
12064
+ *
12065
+ * The oracle is a deterministic pair source like the character pass, not evidence the model core
12066
+ * owns. Reading it here is what makes a person file an operator surface: seed one, and the merge it
12067
+ * declares lands on the next night with no credentials, no cosine, and no second night. Gathering it
12068
+ * under `model !== undefined` made the declaration effective only where a model had already proposed
12069
+ * the same pair, which is the narrower behavior issue #43 names as the defect.
12070
+ *
12071
+ * It is also the model core's evidence, unchanged: `aliasesFor` reads these same groups to render
12072
+ * the `declared aliases` line, so the model sees what the code already decided rather than being
12073
+ * asked about it.
12074
+ *
12075
+ * A read, never a write, so a DRY RUN performs it too. Its merges are counted like every other dry
12076
+ * run count and nothing is written, which is what an operator sizing a night needs.
12077
+ */
12078
+ const aliasGroups = yield* readAliasGroups(env);
12079
+ /** The centroids the model core needs, gathered once for every type rather than per type. */
12080
+ const centroidsByType = model === void 0 ? void 0 : yield* Effect.all([entityClaims(env.deps.db), entityVectors(env.deps.db)]).pipe(Effect.map(([claims, vectors]) => entityCentroids(claims, new Map(vectors.map((entry) => [entry.key, entry.vec])), { sampleTitles: 3 })));
10330
12081
  for (const [entityType, bucket] of [...byType.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
10331
12082
  /** Pass one: normalization, folding counts of names that normalize together. */
10332
12083
  const counts = /* @__PURE__ */ new Map();
@@ -10337,12 +12088,124 @@ const entityResolution = (env) => Effect.gen(function* () {
10337
12088
  counts.set(canonical, (counts.get(canonical) ?? 0) + entity.files);
10338
12089
  if (canonical !== entity.entity_name) normalized += 1;
10339
12090
  }
10340
- /** Pass two: the fuzzy clusters over the normalized names. */
10341
- const clusters = resolveClusters(counts);
10342
- reviewCandidates += clusters.reviewCandidates;
12091
+ /** Pass two: the character pass. Its auto pairs merge; its band pairs await a later stage. */
12092
+ const character = characterPairs([...counts.keys()]);
12093
+ const accepted = [...character.auto];
12094
+ /**
12095
+ * Pass two-and-a-half: the DECLARED aliases, accepted straight into the union.
12096
+ *
12097
+ * Only for {@link PERSON_TYPE}, because that is the only type the format gives a declaration
12098
+ * surface — `resources/people/` — so an alias group can only ever be about a person, and running
12099
+ * this for `service` would compare names against groups that cannot hold them.
12100
+ *
12101
+ * These merges are recorded in `aliasMerges` and never in `entity_corroboration`. A declaration is
12102
+ * a human's assertion of identity rather than a machine's suspicion, so a second night would add no
12103
+ * evidence, and a counter row for it would tell a reader of that table there is a decision still
12104
+ * waiting.
12105
+ */
12106
+ const declared = entityType === PERSON_TYPE ? aliasPairs(aliasGroups, counts) : [];
12107
+ accepted.push(...declared);
12108
+ aliasMerges += declared.length;
12109
+ /** So the model core does not count a declared pair a second time. */
12110
+ const declaredKeys = new Set(declared.map(([left, right]) => pairKey(left, right)));
12111
+ /**
12112
+ * Pass three: the model core. One call per shard of one type, then a deterministic decision per
12113
+ * proposed merge. Every merge the model contributes is either alias-backed and immediate, or
12114
+ * corroborated across nights, or counted for review — the model never writes.
12115
+ */
12116
+ const clusteredPairs = /* @__PURE__ */ new Set();
12117
+ if (model !== void 0 && centroidsByType !== void 0) {
12118
+ const centroids = centroidsByType.get(entityType) ?? [];
12119
+ const members = centroids.filter((centroid) => counts.has(centroid.name));
12120
+ const aliasesFor = (name) => entityType === PERSON_TYPE ? [...new Set(aliasGroups.filter((group) => group.has(name)).flatMap((group) => [...group].filter((other) => other !== name)))].sort() : [];
12121
+ for (const shard of assembleBatches([members], {
12122
+ maxMembers: 500,
12123
+ minMembers: 2
12124
+ })) {
12125
+ const keyed = keyMembers(shard, (centroid) => entityMemberText({
12126
+ centroid,
12127
+ neighbors: nearestCentroids(centroids, centroid.name, 3),
12128
+ aliases: aliasesFor(centroid.name)
12129
+ }), { charBudget: 600 });
12130
+ llmCalls += 1;
12131
+ const clustering = yield* batchCall(model, `entity-resolution ${entityType} batch of ${shard.length}`, {
12132
+ schema: EntityClustering,
12133
+ system: ENTITY_CLUSTER_SYSTEM,
12134
+ prompt: entityClusterPrompt(keyed.keyed),
12135
+ modelKey,
12136
+ effort: "medium",
12137
+ toolDescription: "Emit one cluster per subject, naming the members that are the same subject."
12138
+ });
12139
+ if (clustering === void 0) continue;
12140
+ for (const cluster of clustering.clusters) {
12141
+ /**
12142
+ * A key the batch never offered resolves to nothing, so an invented member cannot become a
12143
+ * rewrite. The canonical must be one of the cluster's own members: a cluster whose stated
12144
+ * canonical is outside it contradicts itself, and guessing which half was meant would be
12145
+ * the caller inventing a merge.
12146
+ */
12147
+ const memberNames = resolveKeys(keyed, cluster.memberKeys).map((centroid) => centroid.name);
12148
+ const [canonicalMember] = resolveKeys(keyed, [cluster.canonicalKey]);
12149
+ if (canonicalMember === void 0 || !memberNames.includes(canonicalMember.name)) continue;
12150
+ for (const merge of decomposeCluster(memberNames, counts)) {
12151
+ const key = pairKey(merge.alias, merge.canonical);
12152
+ clusteredPairs.add(key);
12153
+ if (nameSimilarity(merge.alias, merge.canonical) >= .85) continue;
12154
+ /**
12155
+ * Already accepted by the declaration pass above, so the merge is happening and only the
12156
+ * counting is at stake: adding it again would report one merge as two, and corroborating
12157
+ * it would count a night of evidence toward a decision already made. The model agreeing
12158
+ * with a declaration is not new information — the declaration is the stronger evidence.
12159
+ */
12160
+ if (declaredKeys.has(key)) continue;
12161
+ /**
12162
+ * A declaration the pass above could not have seen: the model named a pair whose two names
12163
+ * are in one alias group, but at least one of them is not in `counts` for this type — a
12164
+ * name the batch offered whose entity rows this type's bucket does not hold. Rare, and the
12165
+ * rule is the same one, so it is applied here rather than left to the corroboration path.
12166
+ */
12167
+ if (aliasBacked(aliasGroups, merge.alias, merge.canonical)) {
12168
+ accepted.push([merge.alias, merge.canonical]);
12169
+ aliasMerges += 1;
12170
+ continue;
12171
+ }
12172
+ if (cluster.confidence < .7) {
12173
+ reviewCandidates += 1;
12174
+ continue;
12175
+ }
12176
+ const row = (yield* bumpEntityCorroboration(env.deps.db, {
12177
+ entityType,
12178
+ aliasName: merge.alias,
12179
+ canonicalName: merge.canonical,
12180
+ at: env.at
12181
+ }))[0];
12182
+ if (row === void 0 || row.detections < 2) {
12183
+ pendingCorroboration += 1;
12184
+ continue;
12185
+ }
12186
+ accepted.push([merge.alias, merge.canonical]);
12187
+ llmMerges += 1;
12188
+ if (row.promoted === 0) yield* markEntityPromoted(env.deps.db, {
12189
+ entityType,
12190
+ aliasName: merge.alias,
12191
+ canonicalName: merge.canonical,
12192
+ at: env.at
12193
+ });
12194
+ }
12195
+ }
12196
+ }
12197
+ }
12198
+ /**
12199
+ * A band pair the model went on to cluster is no longer awaiting a human: it was decided, and
12200
+ * the decision is recorded either as a merge or as a below-floor review candidate already counted
12201
+ * above. Counting it here as well would report one pair twice.
12202
+ */
12203
+ reviewCandidates += character.review.filter(([left, right]) => !clusteredPairs.has(pairKey(left, right))).length;
12204
+ /** One union-find over every accepted pair, so the three sources cannot disagree on a root. */
12205
+ const aliasToCanonical = unionPairs(counts, accepted);
10343
12206
  for (const entity of bucket) {
10344
12207
  const afterNormalize = normalizedOf.get(entity.entity_name) ?? entity.entity_name;
10345
- const afterMerge = clusters.aliasToCanonical.get(afterNormalize) ?? afterNormalize;
12208
+ const afterMerge = aliasToCanonical.get(afterNormalize) ?? afterNormalize;
10346
12209
  if (afterMerge === entity.entity_name) continue;
10347
12210
  if (afterMerge !== afterNormalize) fuzzyMerges += 1;
10348
12211
  const paths = yield* pathsForEntity(env.deps.db, entityType, entity.entity_name);
@@ -10353,10 +12216,16 @@ const entityResolution = (env) => Effect.gen(function* () {
10353
12216
  entities: entities.length,
10354
12217
  namesNormalized: normalized,
10355
12218
  fuzzyMerges,
12219
+ llmMerges,
12220
+ aliasMerges,
12221
+ pendingCorroboration,
10356
12222
  reviewCandidates,
10357
12223
  filesRewritten: rewrites.size
10358
12224
  };
10359
- if (rewrites.size === 0 || env.dryRun) return emptyOutcome(counts);
12225
+ if (rewrites.size === 0 || env.dryRun) return {
12226
+ ...emptyOutcome(counts),
12227
+ llmCalls
12228
+ };
10360
12229
  let rewritten = 0;
10361
12230
  for (const [path, pairs] of [...rewrites.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
10362
12231
  const html = yield* readFileBytes(env, path);
@@ -10376,7 +12245,7 @@ const entityResolution = (env) => Effect.gen(function* () {
10376
12245
  return {
10377
12246
  counts: final,
10378
12247
  commitSha: yield* commitPhase(env, "entity-resolution", `normalize ${normalized} entity names, merge ${fuzzyMerges} aliases`, final),
10379
- llmCalls: 0
12248
+ llmCalls
10380
12249
  };
10381
12250
  });
10382
12251
 
@@ -10716,7 +12585,11 @@ const preflight = (env) => Effect.gen(function* () {
10716
12585
  const MINING_COSINE_FLOOR = .85;
10717
12586
  /** Nearest neighbors considered per source file. */
10718
12587
  const MINING_PER_SOURCE_K = 5;
10719
- /** Pairs mined per cycle. The cost guard on a corpus whose pair space is quadratic. */
12588
+ /**
12589
+ * Pairs mined per cycle: a cap on what {@link replaceMinedEdges} writes, not on the scan — the
12590
+ * kernel's arithmetic is O(n²·d) whatever this says, and it bounds the edge table so one dense
12591
+ * neighborhood cannot flood the graph the lateral arm and PageRank read.
12592
+ */
10720
12593
  const MINING_SAMPLE_LIMIT = 2e3;
10721
12594
  const relationshipMining = (env) => Effect.gen(function* () {
10722
12595
  /**
@@ -11426,8 +13299,8 @@ const traceConsolidation = (env) => Effect.gen(function* () {
11426
13299
  * change.
11427
13300
  *
11428
13301
  * Nor does the conflict become an authored `<link>`, and the reason is mechanical. Any authored
11429
- * edge between two paths permanently closes that pair to the NLI phase's scan: `derived = 0` is
11430
- * the anti-join in `conflictCandidates` (`sql.ts:176-181`), so stamping one here would silence
13302
+ * edge between two paths permanently closes that pair to edge typing's scans: `derived = 0` is
13303
+ * the anti-join in BOTH `sharedEntityPairs` and `minedPairs`, so stamping one here would silence
11431
13304
  * the very disagreement this lookup surfaced. The conflict lives in the counts, in the
11432
13305
  * `Memhtml-Counts` trailer, and in the commit message's context, where a reviewer sees it at merge
11433
13306
  * review and decides.
@@ -11683,7 +13556,7 @@ const PHASE_BODIES = {
11683
13556
  "entity-resolution": entityResolution,
11684
13557
  "person-links": personLinks,
11685
13558
  "relationship-mining": relationshipMining,
11686
- "conflict-detection": conflictDetection,
13559
+ "edge-typing": edgeTyping,
11687
13560
  "confidence-decay": confidenceDecay,
11688
13561
  "arc-synthesis": arcSynthesis,
11689
13562
  "retention-triage": retentionTriage,
@@ -12847,4 +14720,4 @@ const latest = (left, right) => left === null ? right : right === null ? left :
12847
14720
 
12848
14721
  //#endregion
12849
14722
  export { STATE_SIDECAR_PATH as $, makeIndexRecorder as A, makeGitPort as B, ModelClientLive as C, EMBED_DIM as D, EmbeddingsLive as E, reinforce as F, STATE_MIGRATIONS_DIR as G, DatabaseService as H, Indexer as I, expandRoot as J, STATE_SCHEMA as K, makeIndexer as L, readWatermark as M, Retrieval as N, EMBED_WATERMARK as O, makeRetrieval as P, STATE_DB_PATH as Q, readIndexState as R, ModelClient as S, Embeddings as T, makeDatabase as U, sanitizeFtsQuery as V, MIGRATIONS_DIR as W, INDEX_DB_PATH as X, makeStore as Y, SLEEP_REPORTS_DIR as Z, unlink as _, parseSidecar as a, commitSubject as at, discriminationGate as b, generateArtifacts as c, isValidDatetime as ct, danglingEdges as d, REINFORCE_SIGNALS as dt, attemptIo as et, publishRows as f, frameKeyOf as ft, meta as g, link as h, makeSleep as i, makeGit as it, persistScanned as j, IndexRecorder as k, accessRows as l, closesFence as lt, hrefFor as m, scanTraceRoot as n, readFileOrNull as nt, renderSidecar as o, checkMemory as ot, applyHeadEdits as p, Store as q, Sleep as r, Git as rt, archivedFormOf as s, setMeta as st, mergeTailExtract as t, initRepo as tt, allPaths as u, fenceOpeningOf as ut, SLEEP_PHASES as v, wrapAsData as w, runDiscrimination as x, isSleepPhase as y, IndexGit as z };
12850
- //# sourceMappingURL=dist-dfrcDFad.mjs.map
14723
+ //# sourceMappingURL=dist-B3yDga97.mjs.map