memhtml 0.2.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/{dist-Uj47oBRC.mjs → dist-B3yDga97.mjs} +2049 -297
- package/dist/dist-B3yDga97.mjs.map +1 -0
- package/dist/{dist-t84Q_98w.mjs → dist-CSo_XRfz.mjs} +2 -2
- package/dist/{dist-t84Q_98w.mjs.map → dist-CSo_XRfz.mjs.map} +1 -1
- package/dist/{dist-Dj-MYf9q.mjs → dist-FgNX_rxc.mjs} +1 -1
- package/dist/memhtml-mcp.mjs +5 -5
- package/dist/memhtml-mcp.mjs.map +1 -1
- package/dist/memhtml.mjs +7 -7
- package/dist/memhtml.mjs.map +1 -1
- package/package.json +1 -1
- package/state-migrations/S0002_entity_corroboration.sql +55 -0
- package/dist/dist-Uj47oBRC.mjs.map +0 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as INBOX_DIR, B as ModelUnavailable, Ct as filenameFor, H as StorageFailure, J as relClassFor, L as DirtyTree, Q as ARCS_DIR, R as InvalidMemory, St as SLUG_FALLBACK, Tt as withCollisionOrdinal, U as WriteConflict, V as PathNotFound, W as EdgeRel, X as relTokenFor, Y as relForToken, Z as ARCHIVE_BUCKET, _t as TASK_STATUSES, at as isValidMemoryPath, bt as isTaskStatus, ct as paraBucketOf, dt as Importance, et as MEMORY_EXTENSION, ft as MEMORY_TYPES, gt as PERSON_ENTITY_PREFIX, ht as PARA_BUCKETS, it as isArchivePath, lt as placementFor, mt as MemoryType, ot as memoryPathFor, pt as MemoryStatus, q as isEdgeRel, rt as archivePathFor, st as normalizePath, tt as PEOPLE_DIR, ut as Confidence, vt as TaskStatus, wt as slugify, xt as parseEntity, yt as WRITABLE_MEMORY_TYPES, z as LlmContractViolation } from "./dist-CSo_XRfz.mjs";
|
|
2
2
|
import { createRequire } from "node:module";
|
|
3
3
|
import { Config, Context, Effect, Layer, Result, Schedule, Schema } from "effect";
|
|
4
4
|
import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
@@ -429,6 +429,12 @@ const bridgeCounts = (nodes, edges, communities) => {
|
|
|
429
429
|
* correction was written to fix. These guards are a deterministic veto. A divergent pair
|
|
430
430
|
* becomes a candidate contradiction for the conflict phase instead of a merge, no matter how
|
|
431
431
|
* high its cosine runs.
|
|
432
|
+
*
|
|
433
|
+
* The guards are a POST-FILTER over every proposal, including a model's. `dedup-merge` asks a model
|
|
434
|
+
* to partition a connected component into merge groups, and each pair that partition implies is
|
|
435
|
+
* routed through {@link mergeCandidates} before anything is written. A model that groups a claim with
|
|
436
|
+
* its own negation is refused by the same predicate that refuses a blind cosine, so the set of pairs
|
|
437
|
+
* that can be committed does not widen when a model is bound.
|
|
432
438
|
*/
|
|
433
439
|
/** Cosine similarity above which two bodies are the same content. Strict. */
|
|
434
440
|
const NEAR_DUPLICATE_THRESHOLD = .92;
|
|
@@ -589,6 +595,59 @@ const mergeCandidates = (pairs, options = {}) => {
|
|
|
589
595
|
return decisions;
|
|
590
596
|
};
|
|
591
597
|
/**
|
|
598
|
+
* Connected components over an undirected edge list, as sorted member lists.
|
|
599
|
+
*
|
|
600
|
+
* The near-duplicate graph's components are dedup's units of work. A component is what "these
|
|
601
|
+
* memories might all be one memory" looks like before anything has judged them, and it is the right
|
|
602
|
+
* unit because near-duplication is transitive in practice: three rewordings of one fact produce
|
|
603
|
+
* three edges, and folding them one pair at a time would ask the same question three times and could
|
|
604
|
+
* answer it three different ways.
|
|
605
|
+
*
|
|
606
|
+
* **The partition is order-INVARIANT, not merely order-stable.** A union always keeps the
|
|
607
|
+
* lexicographically smaller root, so every set's root is the smallest key it holds no matter which
|
|
608
|
+
* order the edges arrive in. Members come back sorted, and components come back ordered by root,
|
|
609
|
+
* which is each component's own smallest member. So the same edge SET produces the same output
|
|
610
|
+
* whether it arrives mined-first, frame-first, mirrored, or shuffled. That is stronger than sorting
|
|
611
|
+
* the input would be, and it is why no sort happens here: a caller cannot make this disagree with
|
|
612
|
+
* itself by changing how it enumerates.
|
|
613
|
+
*
|
|
614
|
+
* A "larger root wins" or "first root seen wins" rule would break exactly that, because both make
|
|
615
|
+
* the surviving root a fact about arrival order rather than about the set.
|
|
616
|
+
*
|
|
617
|
+
* Each pair is normalized before it is unioned, so `(a, b)` and `(b, a)` are one edge. A self-edge
|
|
618
|
+
* introduces its key and joins nothing. Cost is near-linear in the edge count, and no step here ever
|
|
619
|
+
* enumerates a pair the caller did not hand over.
|
|
620
|
+
*/
|
|
621
|
+
const connectedComponents = (edges) => {
|
|
622
|
+
const parent = /* @__PURE__ */ new Map();
|
|
623
|
+
const find = (key) => {
|
|
624
|
+
let current = key;
|
|
625
|
+
while ((parent.get(current) ?? current) !== current) {
|
|
626
|
+
const next = parent.get(current);
|
|
627
|
+
parent.set(current, parent.get(next) ?? next);
|
|
628
|
+
current = parent.get(current);
|
|
629
|
+
}
|
|
630
|
+
return current;
|
|
631
|
+
};
|
|
632
|
+
for (const [left, right] of edges) {
|
|
633
|
+
if (!parent.has(left)) parent.set(left, left);
|
|
634
|
+
if (!parent.has(right)) parent.set(right, right);
|
|
635
|
+
const rootLeft = find(left);
|
|
636
|
+
const rootRight = find(right);
|
|
637
|
+
if (rootLeft === rootRight) continue;
|
|
638
|
+
if (rootLeft < rootRight) parent.set(rootRight, rootLeft);
|
|
639
|
+
else parent.set(rootLeft, rootRight);
|
|
640
|
+
}
|
|
641
|
+
const byRoot = /* @__PURE__ */ new Map();
|
|
642
|
+
for (const key of parent.keys()) {
|
|
643
|
+
const root = find(key);
|
|
644
|
+
const bucket = byRoot.get(root);
|
|
645
|
+
if (bucket === void 0) byRoot.set(root, [key]);
|
|
646
|
+
else bucket.push(key);
|
|
647
|
+
}
|
|
648
|
+
return [...byRoot.entries()].sort(([left], [right]) => left < right ? -1 : left > right ? 1 : 0).map(([, members]) => members.sort());
|
|
649
|
+
};
|
|
650
|
+
/**
|
|
592
651
|
* The compress-path exclusion: the members of a batch to supersede and archive, with the
|
|
593
652
|
* canonical removed and order preserved. When a batch folds into a pre-existing canonical, a
|
|
594
653
|
* member can *be* that canonical, and archiving it would destroy the file just folded into.
|
|
@@ -2314,8 +2373,18 @@ const META_PREFIX = "memhtml-";
|
|
|
2314
2373
|
/**
|
|
2315
2374
|
* Meta keys that may appear more than once. Each value is its own `<meta>` element
|
|
2316
2375
|
* rather than a comma-joined string, so correcting one tag is a one-line diff.
|
|
2376
|
+
*
|
|
2377
|
+
* `memhtml-alias` is the third: a person file declares the other names the same person is recorded
|
|
2378
|
+
* under, so `laith al-saadoon` can state that `laith` is them. Sleep's entity resolution reads it as
|
|
2379
|
+
* EVIDENCE and auto-merges an alias-backed pair whatever the character distance says, which no
|
|
2380
|
+
* string similarity can supply — `laith` against `laith al-saadoon` scores 0.476, below even the
|
|
2381
|
+
* review band. One meta per alias, for the same one-line-diff reason.
|
|
2317
2382
|
*/
|
|
2318
|
-
const REPEATABLE_META = [
|
|
2383
|
+
const REPEATABLE_META = [
|
|
2384
|
+
"memhtml-entity",
|
|
2385
|
+
"memhtml-tag",
|
|
2386
|
+
"memhtml-alias"
|
|
2387
|
+
];
|
|
2319
2388
|
/** True when a metadata name may legitimately appear more than once in one head. */
|
|
2320
2389
|
const isRepeatableMeta = (name) => REPEATABLE_META.includes(name);
|
|
2321
2390
|
/**
|
|
@@ -2357,7 +2426,8 @@ const META_ORDER = [
|
|
|
2357
2426
|
"memhtml-task-status",
|
|
2358
2427
|
"memhtml-due",
|
|
2359
2428
|
"memhtml-entity",
|
|
2360
|
-
"memhtml-tag"
|
|
2429
|
+
"memhtml-tag",
|
|
2430
|
+
"memhtml-alias"
|
|
2361
2431
|
];
|
|
2362
2432
|
/** True when a `memhtml-`-prefixed metadata name is in the closed vocabulary. */
|
|
2363
2433
|
const isMemoryMetaName = (name) => META_ORDER.includes(name);
|
|
@@ -3416,6 +3486,17 @@ const MemoryDoc = Schema.Struct({
|
|
|
3416
3486
|
entities: Schema.Array(Schema.String),
|
|
3417
3487
|
/** `memhtml-tag` values as authored, in document order. Open vocabulary. */
|
|
3418
3488
|
tags: Schema.Array(Schema.String),
|
|
3489
|
+
/**
|
|
3490
|
+
* `memhtml-alias` values as authored, in document order. The other names this file's subject is
|
|
3491
|
+
* recorded under — a person file's declaration that `laith` and `l.alsaadoon` are the same person
|
|
3492
|
+
* as its own `person:` entity.
|
|
3493
|
+
*
|
|
3494
|
+
* Read as EVIDENCE by sleep's entity resolution, which auto-merges an alias-backed pair whatever
|
|
3495
|
+
* the name similarity says. That is why the field is a bare string list rather than a parsed
|
|
3496
|
+
* `type:name`: the declaring file's own `memhtml-entity` already carries the type, and an alias
|
|
3497
|
+
* that restated it could disagree with it.
|
|
3498
|
+
*/
|
|
3499
|
+
aliases: Schema.Array(Schema.String),
|
|
3419
3500
|
links: Schema.Array(MemoryLink),
|
|
3420
3501
|
article: ArticleExtractions,
|
|
3421
3502
|
/**
|
|
@@ -3827,6 +3908,7 @@ const parseMemory = (html) => Effect.suspend(() => {
|
|
|
3827
3908
|
metas: metaResult.metas,
|
|
3828
3909
|
entities: repeated(metas, "memhtml-entity"),
|
|
3829
3910
|
tags: repeated(metas, "memhtml-tag"),
|
|
3911
|
+
aliases: repeated(metas, "memhtml-alias"),
|
|
3830
3912
|
links: readLinks(document),
|
|
3831
3913
|
article: readArticle(article),
|
|
3832
3914
|
warnings: structural.warnings
|
|
@@ -3907,7 +3989,11 @@ const metaPairs = (doc) => {
|
|
|
3907
3989
|
put("memhtml-needs-revision", metas.needsRevision);
|
|
3908
3990
|
put("memhtml-task-status", metas.taskStatus);
|
|
3909
3991
|
put("memhtml-due", metas.dueAt);
|
|
3910
|
-
const repeatables = /* @__PURE__ */ new Map([
|
|
3992
|
+
const repeatables = /* @__PURE__ */ new Map([
|
|
3993
|
+
["memhtml-entity", doc.entities],
|
|
3994
|
+
["memhtml-tag", doc.tags],
|
|
3995
|
+
["memhtml-alias", doc.aliases]
|
|
3996
|
+
]);
|
|
3911
3997
|
const pairs = [];
|
|
3912
3998
|
for (const name of META_ORDER) {
|
|
3913
3999
|
if (isRepeatableMeta(name)) {
|
|
@@ -4039,6 +4125,7 @@ const newMemoryDoc = (input) => {
|
|
|
4039
4125
|
},
|
|
4040
4126
|
entities: input.entities ?? [],
|
|
4041
4127
|
tags: input.tags ?? [],
|
|
4128
|
+
aliases: input.aliases ?? [],
|
|
4042
4129
|
links: input.links ?? [],
|
|
4043
4130
|
article: {
|
|
4044
4131
|
html,
|
|
@@ -8295,7 +8382,9 @@ const clampTokens = (requested) => Math.min(requested ?? 16384, MAX_TOKENS_CEILI
|
|
|
8295
8382
|
* `emit` call, and that is the whole structured-output mechanism.
|
|
8296
8383
|
*
|
|
8297
8384
|
* `system` is omitted rather than sent empty, because an empty system block is a distinct
|
|
8298
|
-
* (and rejected) input from no system block at all.
|
|
8385
|
+
* (and rejected) input from no system block at all. An omitted system also has nothing to cache, so
|
|
8386
|
+
* `cacheSystem` over an absent or empty system emits no `system` key at all instead of an empty
|
|
8387
|
+
* cached block.
|
|
8299
8388
|
*/
|
|
8300
8389
|
const buildInvokeBody = (key, prompt, options, tool) => {
|
|
8301
8390
|
const body = {
|
|
@@ -8307,7 +8396,11 @@ const buildInvokeBody = (key, prompt, options, tool) => {
|
|
|
8307
8396
|
}],
|
|
8308
8397
|
output_config: { effort: options.effort }
|
|
8309
8398
|
};
|
|
8310
|
-
if (options.system !== void 0 && options.system.length > 0) body.system = options.
|
|
8399
|
+
if (options.system !== void 0 && options.system.length > 0) body.system = options.cacheSystem === true ? [{
|
|
8400
|
+
type: "text",
|
|
8401
|
+
text: options.system,
|
|
8402
|
+
cache_control: { type: "ephemeral" }
|
|
8403
|
+
}] : options.system;
|
|
8311
8404
|
const thinking = thinkingFor(key);
|
|
8312
8405
|
if (thinking !== null) body.thinking = thinking;
|
|
8313
8406
|
if (tool !== void 0) {
|
|
@@ -8399,7 +8492,8 @@ const makeModelClient = (client) => {
|
|
|
8399
8492
|
const { parsed } = yield* invoke(request.modelKey, request.prompt, {
|
|
8400
8493
|
system: request.system,
|
|
8401
8494
|
maxTokens: request.maxTokens,
|
|
8402
|
-
effort: request.effort
|
|
8495
|
+
effort: request.effort,
|
|
8496
|
+
cacheSystem: request.cacheSystem
|
|
8403
8497
|
}, {
|
|
8404
8498
|
inputSchema: request.inputSchema ?? toInputSchema(request.schema),
|
|
8405
8499
|
...request.toolDescription === void 0 ? {} : { description: request.toolDescription }
|
|
@@ -8688,6 +8782,163 @@ var DiscriminationFailed = class {
|
|
|
8688
8782
|
*/
|
|
8689
8783
|
const discriminationGate = (options = {}) => runDiscrimination(options).pipe(Effect.flatMap((outcome) => outcome.passed ? Effect.succeed(outcome) : Effect.fail(new DiscriminationFailed(outcome))));
|
|
8690
8784
|
|
|
8785
|
+
//#endregion
|
|
8786
|
+
//#region packages/sleep/dist/batch.js
|
|
8787
|
+
/**
|
|
8788
|
+
* Mint `m1`..`mN` over `items` in the order given, and index each key back to its item.
|
|
8789
|
+
*
|
|
8790
|
+
* The keys carry no information beyond position, which is the point. A key that encoded a path or a
|
|
8791
|
+
* title would let a model answer with a target it inferred rather than one it was offered, and this
|
|
8792
|
+
* corpus stores instructions, so a member's own text can read as a directive about naming.
|
|
8793
|
+
*
|
|
8794
|
+
* `charBudget` slices each text after `textOf` builds it. The budget is per member and is applied
|
|
8795
|
+
* here so every phase slices at the same boundary, where the text stops being a row and becomes a
|
|
8796
|
+
* prompt.
|
|
8797
|
+
*/
|
|
8798
|
+
const keyMembers = (items, textOf, options) => {
|
|
8799
|
+
const budget = options?.charBudget;
|
|
8800
|
+
const keyed = [];
|
|
8801
|
+
const itemForKey = /* @__PURE__ */ new Map();
|
|
8802
|
+
for (const [offset, item] of items.entries()) {
|
|
8803
|
+
const key = `m${offset + 1}`;
|
|
8804
|
+
const text = textOf(item);
|
|
8805
|
+
keyed.push({
|
|
8806
|
+
key,
|
|
8807
|
+
text: budget === void 0 ? text : text.slice(0, budget)
|
|
8808
|
+
});
|
|
8809
|
+
itemForKey.set(key, item);
|
|
8810
|
+
}
|
|
8811
|
+
return {
|
|
8812
|
+
keyed,
|
|
8813
|
+
itemForKey
|
|
8814
|
+
};
|
|
8815
|
+
};
|
|
8816
|
+
/**
|
|
8817
|
+
* Resolve the keys a model named back to items: unknown keys are dropped, repeats collapse.
|
|
8818
|
+
*
|
|
8819
|
+
* A key the batch never offered is a member the model invented, and every phase on this kernel turns
|
|
8820
|
+
* a named member into a write, so an unresolvable key must not reach that write. Dropping it leaves
|
|
8821
|
+
* the corresponding file untouched, which is the safe outcome for every one of the five phases.
|
|
8822
|
+
*
|
|
8823
|
+
* The result keeps the order the model named the keys in, and a key named twice appears once.
|
|
8824
|
+
* De-duplication is on the KEY rather than on the resolved item, so the count a phase gates on
|
|
8825
|
+
* ("at least two members absorbed") counts distinct offered members.
|
|
8826
|
+
*/
|
|
8827
|
+
const resolveKeys = (batch, keys) => [...new Set(keys)].flatMap((key) => {
|
|
8828
|
+
const item = batch.itemForKey.get(key);
|
|
8829
|
+
return item === void 0 ? [] : [item];
|
|
8830
|
+
});
|
|
8831
|
+
/**
|
|
8832
|
+
* Slice each pre-sorted group into batches of at most `maxMembers`, dropping any batch that falls
|
|
8833
|
+
* below `minMembers`.
|
|
8834
|
+
*
|
|
8835
|
+
* The groups arrive in the order the caller wants them called in, and each group's members arrive in
|
|
8836
|
+
* the caller's own stable order. This walks both in that order, so the boundaries are reproducible.
|
|
8837
|
+
*
|
|
8838
|
+
* `minMembers` defaults to 1, which keeps every slice. A phase whose question is meaningless for a
|
|
8839
|
+
* lone member raises it: compress passes 2, because folding one memory into a "canonical" rewrites it
|
|
8840
|
+
* under a new path and archives the original for no gain.
|
|
8841
|
+
*/
|
|
8842
|
+
const assembleBatches = (groups, options) => {
|
|
8843
|
+
const floor = options.minMembers ?? 1;
|
|
8844
|
+
const batches = [];
|
|
8845
|
+
for (const group of groups) for (let at = 0; at < group.length; at += options.maxMembers) {
|
|
8846
|
+
const slice = group.slice(at, at + options.maxMembers);
|
|
8847
|
+
if (slice.length >= floor) batches.push(slice);
|
|
8848
|
+
}
|
|
8849
|
+
return batches;
|
|
8850
|
+
};
|
|
8851
|
+
/**
|
|
8852
|
+
* Pack whole groups into shared batches, bounded by a member count and a character budget together.
|
|
8853
|
+
*
|
|
8854
|
+
* For a phase whose groups are mostly tiny. `dedup-merge` works over connected components of the
|
|
8855
|
+
* near-duplicate graph, where a typical component is a pair, so one call per component would spend a
|
|
8856
|
+
* model call on two memories. Ten or twenty components in one call cost one.
|
|
8857
|
+
*
|
|
8858
|
+
* Group boundaries survive into the result, because they are evidence. Two members in different
|
|
8859
|
+
* components are known NOT to be near-duplicates, and a prompt that flattened the pack into one list
|
|
8860
|
+
* would ask the model to rediscover that.
|
|
8861
|
+
*
|
|
8862
|
+
* A group longer than `maxMembers` is sliced on the same stride {@link assembleBatches} uses, so no
|
|
8863
|
+
* returned batch breaches either cap. Packing is greedy in the order given and closes a batch when the
|
|
8864
|
+
* next unit would breach a cap, which makes the packing a function of the input order. Filtering out
|
|
8865
|
+
* groups the phase does not want called (a singleton component, for dedup) is the caller's step,
|
|
8866
|
+
* because a floor applied after packing would measure the pack instead of the group.
|
|
8867
|
+
*/
|
|
8868
|
+
const packGroups = (groups, options) => {
|
|
8869
|
+
const batches = [];
|
|
8870
|
+
let current = [];
|
|
8871
|
+
let members = 0;
|
|
8872
|
+
let chars = 0;
|
|
8873
|
+
const close = () => {
|
|
8874
|
+
if (current.length > 0) batches.push(current);
|
|
8875
|
+
current = [];
|
|
8876
|
+
members = 0;
|
|
8877
|
+
chars = 0;
|
|
8878
|
+
};
|
|
8879
|
+
for (const group of groups) {
|
|
8880
|
+
if (group.length === 0) continue;
|
|
8881
|
+
const units = group.length > options.maxMembers ? assembleBatches([group], { maxMembers: options.maxMembers }) : [group];
|
|
8882
|
+
for (const unit of units) {
|
|
8883
|
+
const cost = unit.reduce((total, item) => total + options.charsOf(item), 0);
|
|
8884
|
+
const breaches = members + unit.length > options.maxMembers || chars + cost > options.maxChars;
|
|
8885
|
+
if (current.length > 0 && breaches) close();
|
|
8886
|
+
current.push(unit);
|
|
8887
|
+
members += unit.length;
|
|
8888
|
+
chars += cost;
|
|
8889
|
+
}
|
|
8890
|
+
}
|
|
8891
|
+
close();
|
|
8892
|
+
return batches;
|
|
8893
|
+
};
|
|
8894
|
+
/**
|
|
8895
|
+
* The numbered member list as one prompt block: each member wrapped as data under `<label>_<key>`,
|
|
8896
|
+
* blocks separated by a blank line.
|
|
8897
|
+
*
|
|
8898
|
+
* Every member goes through `wrapAsData`, which is the prompt-injection boundary. This corpus stores
|
|
8899
|
+
* instructions, so a procedural memory about a deploy step reads exactly like a directive to the
|
|
8900
|
+
* model, and un-delimited member text in a user turn would be an injection surface the system built
|
|
8901
|
+
* for itself.
|
|
8902
|
+
*/
|
|
8903
|
+
const memberList = (keyed, options) => {
|
|
8904
|
+
const label = options?.label ?? "member";
|
|
8905
|
+
return keyed.map((member) => wrapAsData(`${label}_${member.key}`, member.text)).join("\n\n");
|
|
8906
|
+
};
|
|
8907
|
+
/**
|
|
8908
|
+
* A batch's user turn: the member list first, the instruction that closes it last.
|
|
8909
|
+
*
|
|
8910
|
+
* The phase's own instruction sentence is the tail rather than the head, matching the order every
|
|
8911
|
+
* copy of this pattern already used. The stable half of a batch prompt is `system` and the tool
|
|
8912
|
+
* schema, which {@link batchCall} marks cacheable; the user turn is new bytes on every call whatever
|
|
8913
|
+
* order its parts sit in.
|
|
8914
|
+
*/
|
|
8915
|
+
const batchPrompt = (keyed, instruction, options) => `${memberList(keyed, options)}\n\n${instruction}`;
|
|
8916
|
+
/**
|
|
8917
|
+
* Run one model call in isolation: a failure becomes `undefined` and a counted skip.
|
|
8918
|
+
*
|
|
8919
|
+
* This is the per-item posture the packet's §4 requires, expressed with `Effect.result` because
|
|
8920
|
+
* `Effect.either` does not exist in this beta. One violation skips its item and leaves the phase
|
|
8921
|
+
* running. A night that judged 199 pairs and lost the 200th to a malformed tool payload has done 199
|
|
8922
|
+
* pairs of work, and failing the phase would throw all of it away.
|
|
8923
|
+
*/
|
|
8924
|
+
const isolate = (label, call) => Effect.gen(function* () {
|
|
8925
|
+
const outcome = yield* Effect.result(call);
|
|
8926
|
+
if (Result.isSuccess(outcome)) return outcome.success;
|
|
8927
|
+
yield* Effect.logWarning(`sleep.llm ${label} skipped: ${outcome.failure.reason}`);
|
|
8928
|
+
});
|
|
8929
|
+
/**
|
|
8930
|
+
* Run one batch's model call: prompt-cache the stable prefix, and isolate the failure.
|
|
8931
|
+
*
|
|
8932
|
+
* `cacheSystem` is set here instead of at each call site, because every phase on this kernel has the
|
|
8933
|
+
* same shape: one system prompt and one tool schema repeated across every batch of the night, with
|
|
8934
|
+
* only the member list changing. A phase that forgot the flag would re-bill its whole prefix on every
|
|
8935
|
+
* batch, and the omission would be invisible in the phase's output.
|
|
8936
|
+
*/
|
|
8937
|
+
const batchCall = (model, label, request) => isolate(label, model.generateObject({
|
|
8938
|
+
...request,
|
|
8939
|
+
cacheSystem: true
|
|
8940
|
+
}));
|
|
8941
|
+
|
|
8691
8942
|
//#endregion
|
|
8692
8943
|
//#region packages/sleep/dist/contract.js
|
|
8693
8944
|
/**
|
|
@@ -8711,7 +8962,7 @@ const SLEEP_PHASES = [
|
|
|
8711
8962
|
"entity-resolution",
|
|
8712
8963
|
"person-links",
|
|
8713
8964
|
"relationship-mining",
|
|
8714
|
-
"
|
|
8965
|
+
"edge-typing",
|
|
8715
8966
|
"confidence-decay",
|
|
8716
8967
|
"arc-synthesis",
|
|
8717
8968
|
"retention-triage",
|
|
@@ -8738,6 +8989,10 @@ const phaseIndexOf = (phase) => SLEEP_PHASES.indexOf(phase) + 1;
|
|
|
8738
8989
|
* `dedup-merge` is the one hard prerequisite, for `compress` and `retention-triage`: both operate
|
|
8739
8990
|
* on the post-merge set, and running them over a corpus that still holds the duplicates would
|
|
8740
8991
|
* compress a near-duplicate pair into a canonical while a merge later archives one of its members.
|
|
8992
|
+
*
|
|
8993
|
+
* That is why `dedup-merge` isolates each of its model calls instead of failing on one. It batches
|
|
8994
|
+
* components and a batch whose call comes back malformed is counted and skipped, so a single bad tool
|
|
8995
|
+
* payload cannot take two later phases down with it.
|
|
8741
8996
|
*/
|
|
8742
8997
|
const HARD_PREREQUISITES = [["dedup-merge", "compress"], ["dedup-merge", "retention-triage"]];
|
|
8743
8998
|
/** The phases blocked by `phase` failing. */
|
|
@@ -8975,7 +9230,7 @@ const datePlusDays = (date, days) => {
|
|
|
8975
9230
|
//#endregion
|
|
8976
9231
|
//#region packages/sleep/dist/env.js
|
|
8977
9232
|
/**
|
|
8978
|
-
* Model assignments per LLM phase: the cheap judge for
|
|
9233
|
+
* Model assignments per LLM phase: the cheap judge for classification, the strong one for synthesis.
|
|
8979
9234
|
*
|
|
8980
9235
|
* `trace-consolidation` names `opus-5` and does not thereby choose it. The consolidator is an eve
|
|
8981
9236
|
* agent that pins its own model in `apps/consolidator/agent/agent.ts`, and this map cannot reach that
|
|
@@ -8984,7 +9239,22 @@ const datePlusDays = (date, days) => {
|
|
|
8984
9239
|
* the Bedrock global endpoint, high reasoning effort, no cost ceiling.)
|
|
8985
9240
|
*/
|
|
8986
9241
|
const DEFAULT_MODELS = {
|
|
8987
|
-
|
|
9242
|
+
/**
|
|
9243
|
+
* `dedup-merge` names sonnet for the same reason the edge-typing judge does: the question is a
|
|
9244
|
+
* classification over text the model is shown, not a synthesis it has to write. It partitions a
|
|
9245
|
+
* component into "these are the same memory" groups, and every consequence of that answer — which
|
|
9246
|
+
* file survives, whether the pair diverges, whether either path is already claimed — is decided by
|
|
9247
|
+
* code afterwards. The strong model is spent where prose gets written.
|
|
9248
|
+
*/
|
|
9249
|
+
"dedup-merge": "sonnet-5",
|
|
9250
|
+
/**
|
|
9251
|
+
* Sonnet, and one or two calls a night: the whole of one entity type's name list goes in one call.
|
|
9252
|
+
* The question is a partition over short strings with their evidence inline, not a synthesis, so
|
|
9253
|
+
* the strong model would buy nothing the deterministic floors around the answer do not already
|
|
9254
|
+
* supply.
|
|
9255
|
+
*/
|
|
9256
|
+
"entity-resolution": "sonnet-5",
|
|
9257
|
+
"edge-typing": "sonnet-5",
|
|
8988
9258
|
"arc-synthesis": "opus-5",
|
|
8989
9259
|
compress: "sonnet-5",
|
|
8990
9260
|
"trace-consolidation": "opus-5"
|
|
@@ -9001,7 +9271,7 @@ const emptyOutcome = (counts = {}) => ({
|
|
|
9001
9271
|
//#endregion
|
|
9002
9272
|
//#region packages/sleep/dist/llm.js
|
|
9003
9273
|
/**
|
|
9004
|
-
* The structured-output schemas the four LLM phases share, and
|
|
9274
|
+
* The structured-output schemas the four LLM phases share, and their prompts.
|
|
9005
9275
|
*
|
|
9006
9276
|
* Two rules govern everything here, both of them found by hitting the failure:
|
|
9007
9277
|
*
|
|
@@ -9015,32 +9285,87 @@ const emptyOutcome = (counts = {}) => ({
|
|
|
9015
9285
|
* itself. The prompts are also blind by construction. None names a path, a score, or a decision the
|
|
9016
9286
|
* caller has already made, so the model cannot agree with a verdict it was shown.
|
|
9017
9287
|
*/
|
|
9018
|
-
/**
|
|
9019
|
-
|
|
9020
|
-
|
|
9021
|
-
|
|
9022
|
-
|
|
9023
|
-
|
|
9024
|
-
|
|
9025
|
-
|
|
9026
|
-
|
|
9288
|
+
/**
|
|
9289
|
+
* The rels edge typing may propose, plus `none`. A closed subset of `MEMORY_RELS`, and the
|
|
9290
|
+
* omissions are deliberate.
|
|
9291
|
+
*
|
|
9292
|
+
* `supersedes` is out because it is a one-way door on stored belief: it says one memory REPLACES
|
|
9293
|
+
* another, which is dedup-merge's and compress's business and rides with an archive. `relates_to`
|
|
9294
|
+
* is out because it is what the pair already carries as a derived edge, so proposing it is a no-op
|
|
9295
|
+
* with a model call attached. `laterally_related` is out for the same reason one notch weaker.
|
|
9296
|
+
* A pair the model cannot type answers `none` and stays a mined suspicion.
|
|
9297
|
+
*/
|
|
9298
|
+
const EDGE_TYPED_RELS = [
|
|
9299
|
+
"caused_by",
|
|
9300
|
+
"leads_to",
|
|
9301
|
+
"example_of",
|
|
9302
|
+
"supports",
|
|
9303
|
+
"part_of",
|
|
9304
|
+
"contradicts"
|
|
9305
|
+
];
|
|
9306
|
+
/**
|
|
9307
|
+
* The five DIRECTIONAL rels: the ones whose meaning depends on which endpoint is the subject.
|
|
9308
|
+
*
|
|
9309
|
+
* `contradicts` is excluded and that is the whole distinction this list draws. A contradiction is
|
|
9310
|
+
* symmetric — a reader arriving at either file must see it — so it is promoted into BOTH files and
|
|
9311
|
+
* its `direction` field is ignored. A directional rel is promoted into ONE file, the subject's, and
|
|
9312
|
+
* the direction decides which one.
|
|
9313
|
+
*/
|
|
9314
|
+
const EDGE_DIRECTIONAL_RELS = [
|
|
9315
|
+
"caused_by",
|
|
9316
|
+
"leads_to",
|
|
9317
|
+
"example_of",
|
|
9318
|
+
"supports",
|
|
9319
|
+
"part_of"
|
|
9320
|
+
];
|
|
9321
|
+
/** The rel vocabulary the model answers over: the typed rels plus the refusal. */
|
|
9322
|
+
const EdgeVerdictRel = Schema.Literals([...EDGE_TYPED_RELS, "none"]);
|
|
9323
|
+
/**
|
|
9324
|
+
* Which endpoint is the rel's subject. Meaningful only for {@link EDGE_DIRECTIONAL_RELS}.
|
|
9325
|
+
*
|
|
9326
|
+
* Required rather than optional, because a model allowed to omit it would omit it on the rels where
|
|
9327
|
+
* it matters. `contradicts` and `none` carry a value the phase does not read.
|
|
9328
|
+
*/
|
|
9329
|
+
const EdgeDirection = Schema.Literals(["src_to_dst", "dst_to_src"]);
|
|
9330
|
+
/** One pair's verdict, under the opaque key the pair was offered as. */
|
|
9331
|
+
const EdgeVerdict = Schema.Struct({
|
|
9332
|
+
/** The offered key, e.g. `m3`. A key the batch never held resolves to nothing and is dropped. */
|
|
9333
|
+
pairKey: Schema.String,
|
|
9334
|
+
rel: EdgeVerdictRel,
|
|
9335
|
+
direction: EdgeDirection,
|
|
9336
|
+
/** Unitless in `[0, 1]`. The promotion gate is deterministic and reads this, not the prose. */
|
|
9027
9337
|
confidence: Schema.Finite.check(Schema.isBetween({
|
|
9028
9338
|
minimum: 0,
|
|
9029
9339
|
maximum: 1
|
|
9030
9340
|
})),
|
|
9031
|
-
/** One or two sentences naming the specific claims that
|
|
9032
|
-
rationale: Schema.String
|
|
9341
|
+
/** One or two sentences naming the specific claims that carry the rel. Optional: `none` has none. */
|
|
9342
|
+
rationale: Schema.optional(Schema.String)
|
|
9033
9343
|
});
|
|
9344
|
+
/** One batch's whole answer: a verdict LIST, never one call per pair. */
|
|
9345
|
+
const EdgeTyping = Schema.Struct({ verdicts: Schema.Array(EdgeVerdict) });
|
|
9034
9346
|
/**
|
|
9035
|
-
* The confidence a
|
|
9347
|
+
* The confidence a verdict must clear before the phase writes anything.
|
|
9036
9348
|
*
|
|
9037
|
-
* A
|
|
9038
|
-
*
|
|
9039
|
-
*
|
|
9349
|
+
* A `contradicts` feeds a retention penalty that can eventually evict a memory, so a false one is
|
|
9350
|
+
* worse than a missed one; the floor and the `detections >= 2` corroboration gate are two
|
|
9351
|
+
* independent guards on that one-way door. A directional rel is milder but still an authored edge in
|
|
9352
|
+
* a file a human reads, so it clears the same floor. One number, because a second one would be a
|
|
9353
|
+
* knob nobody could say the meaning of.
|
|
9040
9354
|
*/
|
|
9041
|
-
const
|
|
9042
|
-
/** True when a
|
|
9043
|
-
const
|
|
9355
|
+
const EDGE_CONFIDENCE_FLOOR = .7;
|
|
9356
|
+
/** True when a verdict is a proposal at all, above the floor. Computed here, never by the model. */
|
|
9357
|
+
const assertsEdge = (verdict) => verdict.rel !== "none" && verdict.confidence >= .7;
|
|
9358
|
+
/** True when a verdict earns a `contradicts` edge. The corroboration gate's precondition. */
|
|
9359
|
+
const assertsContradiction = (verdict) => verdict.rel === "contradicts" && verdict.confidence >= .7;
|
|
9360
|
+
/**
|
|
9361
|
+
* True when a rel's meaning depends on which endpoint is its subject.
|
|
9362
|
+
*
|
|
9363
|
+
* A NARROWING predicate, not a boolean, so the caller's `rel` becomes an `EdgeDirectionalRel` — and
|
|
9364
|
+
* therefore a `MemoryRel` — inside the branch that writes a `<link>`. A plain boolean would leave the
|
|
9365
|
+
* write site casting `"none"`-inclusive union to `EdgeRel`, which is the cast that would silently
|
|
9366
|
+
* survive someone adding a non-rel member to the verdict vocabulary.
|
|
9367
|
+
*/
|
|
9368
|
+
const isDirectionalRel = (rel) => EDGE_DIRECTIONAL_RELS.includes(rel);
|
|
9044
9369
|
/** One arc the triage call proposes to act on. */
|
|
9045
9370
|
const ArcAction = Schema.Literals([
|
|
9046
9371
|
"update",
|
|
@@ -9083,21 +9408,98 @@ const CompressSynthesis = Schema.Struct({
|
|
|
9083
9408
|
*/
|
|
9084
9409
|
absorbedKeys: Schema.Array(Schema.String)
|
|
9085
9410
|
});
|
|
9086
|
-
/**
|
|
9087
|
-
const
|
|
9088
|
-
|
|
9089
|
-
|
|
9411
|
+
/** One proposed identity cluster over the entity names a batch offered. */
|
|
9412
|
+
const EntityCluster = Schema.Struct({
|
|
9413
|
+
/**
|
|
9414
|
+
* The member key the cluster's canonical name was offered under. The phase re-derives the canonical
|
|
9415
|
+
* from ITS OWN weight-then-lexicographic rule, so this names which member the model considers the
|
|
9416
|
+
* fullest form and never which file gets rewritten. A key the batch did not offer resolves to
|
|
9417
|
+
* nothing and drops the cluster.
|
|
9418
|
+
*/
|
|
9419
|
+
canonicalKey: Schema.String,
|
|
9420
|
+
/**
|
|
9421
|
+
* Every member key in the cluster, canonical included. A cluster of one is a valid answer meaning
|
|
9422
|
+
* "this name stands alone", and it produces no merge.
|
|
9423
|
+
*/
|
|
9424
|
+
memberKeys: Schema.Array(Schema.String),
|
|
9425
|
+
/** Unitless in `[0, 1]`. The merge gate is deterministic and reads this, not the prose. */
|
|
9426
|
+
confidence: Schema.Finite.check(Schema.isBetween({
|
|
9427
|
+
minimum: 0,
|
|
9428
|
+
maximum: 1
|
|
9429
|
+
})),
|
|
9430
|
+
/** One sentence naming what makes these one subject: a declared alias, a shared neighborhood. */
|
|
9431
|
+
evidence: Schema.String
|
|
9432
|
+
});
|
|
9433
|
+
/**
|
|
9434
|
+
* The whole clustering answer for one batch: a partition of the offered names.
|
|
9435
|
+
*
|
|
9436
|
+
* `clusters: []` is a refusal and a valid answer. A model that cannot tell two short names apart must
|
|
9437
|
+
* be able to say so, because the alternative — inventing a cluster to fill the field — reaches a
|
|
9438
|
+
* permanent rewrite of stored identity.
|
|
9439
|
+
*/
|
|
9440
|
+
const EntityClustering = Schema.Struct({ clusters: Schema.Array(EntityCluster) });
|
|
9441
|
+
/**
|
|
9442
|
+
* One merge group: the members the model says are the same memory, by their offered keys.
|
|
9443
|
+
*
|
|
9444
|
+
* **No canonical field, deliberately.** The keeper is the OLDER file, decided from corpus order in
|
|
9445
|
+
* the phase, and a model-chosen canonical would be a model-chosen write target: the file that
|
|
9446
|
+
* survives and the files that get archived. The model's whole job here is the partition — which
|
|
9447
|
+
* members are one memory — and orientation is arithmetic over `created_at` that needs no judgment.
|
|
9448
|
+
*
|
|
9449
|
+
* A group of fewer than two keys is meaningless and the phase drops it. That is the shape a model
|
|
9450
|
+
* produces when it wants to say "this one is on its own", which is a valid answer.
|
|
9451
|
+
*/
|
|
9452
|
+
const MergeGroup = Schema.Struct({ memberKeys: Schema.Array(Schema.String) });
|
|
9453
|
+
/**
|
|
9454
|
+
* The dedup partition for one packed batch: every merge group the model found, across every
|
|
9455
|
+
* component in the batch.
|
|
9456
|
+
*
|
|
9457
|
+
* **The groups are FLAT, not nested per component, and the phase re-derives which component each one
|
|
9458
|
+
* came from.** A nested answer would need the model to keep a component index aligned with its
|
|
9459
|
+
* groups, which is bookkeeping a model gets wrong under load, and a mis-aligned index would attach a
|
|
9460
|
+
* group to the wrong component's files. A flat list of member keys carries the same information,
|
|
9461
|
+
* because a key already identifies its member and therefore its component. So the phase can check
|
|
9462
|
+
* containment itself instead of trusting a label.
|
|
9463
|
+
*
|
|
9464
|
+
* `groups: []` is a full refusal: every member stays where it is, which is the safe outcome and the
|
|
9465
|
+
* behavior a night with no model already has.
|
|
9466
|
+
*/
|
|
9467
|
+
const MergePartition = Schema.Struct({ groups: Schema.Array(MergeGroup) });
|
|
9468
|
+
/**
|
|
9469
|
+
* The edge-typing system prompt: the whole rel vocabulary, one pass, one answer per pair.
|
|
9470
|
+
*
|
|
9471
|
+
* The same conservative posture the per-pair stance judge carried, generalized. An unsure pair
|
|
9472
|
+
* answers `none` and keeps the machine-mined `relates_to` it already has, which costs the corpus
|
|
9473
|
+
* nothing; a wrong `contradicts` starts a memory down the eviction path and a wrong directional rel
|
|
9474
|
+
* writes a claim about causality into a file a human reads.
|
|
9475
|
+
*/
|
|
9476
|
+
const EDGE_TYPING_SYSTEM = `You type relationships between memories in an AI agent's long-term memory system. You are given a
|
|
9477
|
+
NUMBERED LIST of candidate pairs. Each pair holds two memories, src and dst, that are embedding-near
|
|
9478
|
+
or share an entity. Return ONE verdict per pair, naming the pair by the key it was offered under.
|
|
9479
|
+
|
|
9480
|
+
Choose the rel that holds between the two memories:
|
|
9481
|
+
|
|
9482
|
+
- caused_by: one memory's fact is the CAUSE of the other's. The subject is the effect.
|
|
9483
|
+
- leads_to: one memory's fact leads to, triggers, or produces the other's. The subject is the cause.
|
|
9484
|
+
- example_of: one memory is a concrete instance of the other's general claim. The subject is the instance.
|
|
9485
|
+
- supports: one memory is evidence FOR the other's claim, without restating it. The subject is the evidence.
|
|
9486
|
+
- part_of: one memory is a component, step, or subtopic of the other's larger whole. The subject is the part.
|
|
9487
|
+
- contradicts: the two make claims about the same thing that CANNOT both be true at the same time
|
|
9488
|
+
(negation, opposite outcomes, mutually exclusive values). SYMMETRIC: direction is ignored.
|
|
9489
|
+
- none: the two are merely about the same topic, restate each other, or carry no relationship you can
|
|
9490
|
+
name from the text. This is the correct answer whenever you are unsure.
|
|
9090
9491
|
|
|
9091
|
-
|
|
9092
|
-
|
|
9093
|
-
|
|
9094
|
-
- neutral: A and B are about the same entity but make compatible, complementary, or simply
|
|
9095
|
-
unrelated claims that can both hold.
|
|
9492
|
+
direction says which endpoint is the rel's SUBJECT, as described per rel above: src_to_dst means src
|
|
9493
|
+
is the subject and dst the object; dst_to_src is the reverse. Answer src_to_dst on contradicts and
|
|
9494
|
+
none, where it is not read.
|
|
9096
9495
|
|
|
9097
|
-
Be conservative. A
|
|
9098
|
-
|
|
9099
|
-
|
|
9100
|
-
|
|
9496
|
+
Be conservative. A verdict above the confidence floor is written into the memory files as an authored
|
|
9497
|
+
edge, and a false contradicts feeds a retention penalty that can eventually evict a memory. When the
|
|
9498
|
+
relationship COULD be something else — different scope, different time, mere topical adjacency —
|
|
9499
|
+
answer none. Two memories being similar is not a relationship. Rate confidence honestly and name the
|
|
9500
|
+
specific claims that carry the rel in the rationale.
|
|
9501
|
+
|
|
9502
|
+
Omitting a pair is allowed: an omitted pair is simply left untyped.`;
|
|
9101
9503
|
/** The arc-triage system prompt: plan only, no content. */
|
|
9102
9504
|
const ARC_TRIAGE_SYSTEM = `You triage behavioral arcs for an AI agent's long-term memory system. This is the planning pass:
|
|
9103
9505
|
a second pass writes each arc's content, so your output is only the plan.
|
|
@@ -9136,34 +9538,151 @@ them, and each member you list in absorbedKeys is archived once the canonical is
|
|
|
9136
9538
|
member you omit stays active, which is the safe outcome — never list one to be tidy.
|
|
9137
9539
|
- If the members do not actually describe one thing, return an empty absorbedKeys and say so in the
|
|
9138
9540
|
claim. Refusing to fold is a valid answer.`;
|
|
9541
|
+
/**
|
|
9542
|
+
* The entity-clustering system prompt: partition one type's names into subjects.
|
|
9543
|
+
*
|
|
9544
|
+
* Names the three evidence kinds a member block carries, because each one supports a different
|
|
9545
|
+
* inference and a model told only "decide if these are the same" would weigh the name string — the
|
|
9546
|
+
* signal that is measurably wrong here. `laith` against `laith al-saadoon` is 0.476 by character
|
|
9547
|
+
* overlap, below even the review band, while their memory centroids are near-identical.
|
|
9548
|
+
*
|
|
9549
|
+
* The refusal instruction is load-bearing rather than polite. A cluster this phase acts on rewrites
|
|
9550
|
+
* every `memhtml-entity` meta naming the alias across the corpus, and no later commit separates two
|
|
9551
|
+
* subjects whose memories were fused.
|
|
9552
|
+
*/
|
|
9553
|
+
const ENTITY_CLUSTER_SYSTEM = `You group entity names for an AI agent's long-term memory system. Every name below is the same KIND of
|
|
9554
|
+
thing — all people, or all services, or all concepts — and several may be different ways of writing one
|
|
9555
|
+
subject. Partition them into subjects.
|
|
9556
|
+
|
|
9557
|
+
Each member gives you:
|
|
9558
|
+
- the name as the corpus records it, and how many active memories claim it;
|
|
9559
|
+
- up to three titles of memories claiming it, which say what that name is ABOUT;
|
|
9560
|
+
- its nearest neighbors by MEMORY CENTROID with a cosine — the centroid is the average of the vectors
|
|
9561
|
+
of every memory claiming the name, so a high cosine means two names are written about in the same
|
|
9562
|
+
terms. Two spellings of one person have near-identical centroids; two different services in one
|
|
9563
|
+
domain do not;
|
|
9564
|
+
- for a person, aliases DECLARED in that person's own file, which are an authoritative statement of
|
|
9565
|
+
identity rather than a guess.
|
|
9566
|
+
|
|
9567
|
+
Rules:
|
|
9568
|
+
- Every cluster lists canonicalKey plus every other member key that names the same subject. Set
|
|
9569
|
+
canonicalKey to the fullest, most complete form of the name.
|
|
9570
|
+
- A name that stands alone is its own cluster of one, or you may leave it out. Both mean "no merge".
|
|
9571
|
+
- Return an empty clusters list when nothing here is the same subject. Refusing to group is a valid
|
|
9572
|
+
and often correct answer.
|
|
9573
|
+
- Short name against long name is the case to look for: 'laith' and 'laith al-saadoon' are one person
|
|
9574
|
+
when the evidence supports it. Shared prefix is NOT: 'checkout-api' and 'payments-api' are two
|
|
9575
|
+
services, and 'metrics-api' and 'metrics-cli' are a service and a tool.
|
|
9576
|
+
- Never group two names because their strings are similar. Group them because the evidence says one
|
|
9577
|
+
subject, and cite that evidence.
|
|
9578
|
+
- Rate confidence honestly. A merge fuses two subjects' memories permanently and nothing separates
|
|
9579
|
+
them again, so answer low when you are unsure and the system will hold the merge back.`;
|
|
9580
|
+
/** The instruction that closes an entity-clustering batch's user turn, after the member list. */
|
|
9581
|
+
const ENTITY_CLUSTER_INSTRUCTION = "Partition these names into subjects. Return one cluster per subject with its canonicalKey, every member key it covers, your confidence, and the specific evidence that makes them one subject.";
|
|
9582
|
+
/**
|
|
9583
|
+
* The entity-clustering user turn for one batch: every member's evidence block under its offered key.
|
|
9584
|
+
*
|
|
9585
|
+
* `batchPrompt` from the kernel builds the list and appends the instruction, so the stable half of the
|
|
9586
|
+
* call is {@link ENTITY_CLUSTER_SYSTEM} plus the tool schema and only the member list is new bytes per
|
|
9587
|
+
* batch. Kept as a named function because the instruction belongs beside the system prompt.
|
|
9588
|
+
*/
|
|
9589
|
+
const entityClusterPrompt = (members) => batchPrompt(members, ENTITY_CLUSTER_INSTRUCTION, { label: "entity" });
|
|
9590
|
+
/**
|
|
9591
|
+
* The dedup-partition system prompt.
|
|
9592
|
+
*
|
|
9593
|
+
* The stable prefix for every dedup call of a {@link batchCall} marks it cacheable, so only
|
|
9594
|
+
* the member list is new bytes per batch.
|
|
9595
|
+
*
|
|
9596
|
+
* It tells the model that a group is a claim about SAMENESS and nothing else. Every other decision
|
|
9597
|
+
* the fold needs — which file survives, whether the pair diverges in polarity or in a number,
|
|
9598
|
+
* whether either path is already spoken for — is made by code after the answer comes back, and the
|
|
9599
|
+
* prompt says so, because a model told it is choosing what gets deleted answers more conservatively
|
|
9600
|
+
* than the question deserves.
|
|
9601
|
+
*/
|
|
9602
|
+
const DEDUP_SYSTEM = `You partition groups of near-duplicate memories for an AI agent's long-term memory system.
|
|
9603
|
+
Each component below holds memories that are near neighbors in vector space, or that state the same
|
|
9604
|
+
relation. Within EACH component, group the memories that are THE SAME MEMORY — one fact stored more
|
|
9605
|
+
than once, in different words.
|
|
9606
|
+
|
|
9607
|
+
- A group means: these state one fact, and keeping all of them stores it repeatedly. Two memories
|
|
9608
|
+
about the same topic that carry DIFFERENT facts are not a group.
|
|
9609
|
+
- Group only members of the SAME component. Members of different components are already known not to
|
|
9610
|
+
be near-duplicates.
|
|
9611
|
+
- A member belongs to at most one group. Leave a member out of every group when it is on its own.
|
|
9612
|
+
- Return groups: [] when no component holds a duplicate. Refusing to group is a valid answer and is
|
|
9613
|
+
the right one whenever you are unsure.
|
|
9614
|
+
- You are not choosing what to delete. Which memory survives a fold is decided from the memories'
|
|
9615
|
+
own dates afterwards, and a proposed group is still checked for contradicting claims, differing
|
|
9616
|
+
numbers, and differing product variants before anything is written. Answer only the question of
|
|
9617
|
+
sameness.`;
|
|
9139
9618
|
/** One labelled corpus block, delimited so its prose cannot be read as an instruction. */
|
|
9140
9619
|
const dataBlock = (label, text) => wrapAsData(label, text);
|
|
9141
9620
|
/**
|
|
9142
|
-
*
|
|
9621
|
+
* One pair as the model sees it: both memories inline, delimited, under `src` and `dst` headings.
|
|
9622
|
+
*
|
|
9623
|
+
* This is a MEMBER's text, not a whole prompt: the kernel's `keyMembers` slices it to the phase's
|
|
9624
|
+
* per-member budget and `memberList` wraps the whole thing again under the pair's opaque key, so a
|
|
9625
|
+
* batch of thirty pairs is one nesting of sixty delimited memories. Neither half carries a path, a
|
|
9626
|
+
* cosine, or a prior verdict, so the model cannot infer which answer the caller is hoping for and
|
|
9627
|
+
* cannot recognize a pair it judged last night.
|
|
9628
|
+
*
|
|
9629
|
+
* The inner headings are plain lines rather than another `wrapAsData` block, because the outer wrap
|
|
9630
|
+
* already carries the "this is data" instruction and a second copy per member would repeat that
|
|
9631
|
+
* sentence sixty times in one prompt for no added guard.
|
|
9632
|
+
*/
|
|
9633
|
+
const pairText = (srcText, dstText) => `src:\n${srcText}\n\ndst:\n${dstText}`;
|
|
9634
|
+
/** The instruction that closes an edge-typing batch's user turn, after the pair list. */
|
|
9635
|
+
const EDGE_TYPING_INSTRUCTION = "Type each pair above. Return one verdict per pair, naming the pair by its offered key, with the rel, the direction, your confidence, and a rationale naming the claims that carry the rel. Answer none whenever you are unsure.";
|
|
9636
|
+
/**
|
|
9637
|
+
* One edge-typing batch's user turn: every pair's two memories under its offered key, then the
|
|
9638
|
+
* instruction.
|
|
9143
9639
|
*
|
|
9144
|
-
*
|
|
9145
|
-
*
|
|
9640
|
+
* `batchPrompt` from the kernel builds the list and appends the instruction, so the framing is the
|
|
9641
|
+
* same bytes compress's batches use. Kept as a named function because the instruction belongs beside
|
|
9642
|
+
* {@link EDGE_TYPING_SYSTEM}, which is the other half of what the model is told.
|
|
9146
9643
|
*/
|
|
9147
|
-
const
|
|
9644
|
+
const edgeTypingPrompt = (pairs) => batchPrompt(pairs, EDGE_TYPING_INSTRUCTION, { label: "pair" });
|
|
9148
9645
|
/** The arc-triage user turn: the live arcs and the recent evidence, both wrapped. */
|
|
9149
9646
|
const arcTriagePrompt = (arcsText, evidenceText) => `${dataBlock("current_arcs", arcsText)}\n\n${dataBlock("evidence", evidenceText)}\n\nProduce a triage plan. Assign update or skip to every existing arc, and add a create entry for any genuinely new behavioral pattern the existing arcs do not cover.`;
|
|
9150
9647
|
/** The arc-execute user turn for one arc. `current` is absent on a create. */
|
|
9151
9648
|
const arcExecutePrompt = (input) => (input.current === void 0 ? `${dataBlock("new_arc_title", input.title)}\n\n` : `${dataBlock("existing_arc", input.current)}\n\n`) + `${dataBlock("evidence", input.evidenceText)}\n\n${dataBlock("triage_rationale", input.rationale)}\n\n` + (input.current === void 0 ? "Synthesize a new behavioral principal from this evidence." : "Update the arc to incorporate the new evidence, preserving existing knowledge that holds.");
|
|
9152
|
-
/** The
|
|
9153
|
-
const
|
|
9649
|
+
/** The instruction that closes a compress batch's user turn, after the member list. */
|
|
9650
|
+
const COMPRESS_INSTRUCTION = "Fold these memories into one canonical memory. List in absorbedKeys exactly the members whose content the canonical carries forward.";
|
|
9154
9651
|
/**
|
|
9155
|
-
*
|
|
9652
|
+
* The compress user turn for one batch: every member's text, wrapped, under its offered key.
|
|
9156
9653
|
*
|
|
9157
|
-
*
|
|
9158
|
-
*
|
|
9159
|
-
*
|
|
9160
|
-
*
|
|
9654
|
+
* `batchPrompt` from the kernel builds the member list and appends the instruction, so this produces
|
|
9655
|
+
* the same bytes it did when the framing was inline here. Kept as a named function because the
|
|
9656
|
+
* instruction belongs beside {@link COMPRESS_SYSTEM}, which is the other half of what the model is
|
|
9657
|
+
* told.
|
|
9161
9658
|
*/
|
|
9162
|
-
const
|
|
9163
|
-
|
|
9164
|
-
|
|
9165
|
-
|
|
9166
|
-
|
|
9659
|
+
const compressPrompt = (members) => batchPrompt(members, COMPRESS_INSTRUCTION);
|
|
9660
|
+
/** The instruction that closes a dedup batch's user turn, after the components. */
|
|
9661
|
+
const DEDUP_INSTRUCTION = "Within each component above, group the members that are the same memory stated more than once. Name each group's members by the keys they were offered under. Return groups: [] if no component holds a duplicate.";
|
|
9662
|
+
/**
|
|
9663
|
+
* The dedup user turn for one packed batch: each component's members, wrapped, under a header that
|
|
9664
|
+
* names which keys sit in that component.
|
|
9665
|
+
*
|
|
9666
|
+
* **The component boundary is in the prompt because it is EVIDENCE.** Two members in different
|
|
9667
|
+
* components have already been measured as not near-duplicates, by a cosine floor and a frame-key
|
|
9668
|
+
* lookup, and a flat member list would throw that away and ask the model to rediscover it across the
|
|
9669
|
+
* whole batch. Packing ten components into one call is a cost decision; letting them blur into one
|
|
9670
|
+
* list would make it a correctness one.
|
|
9671
|
+
*
|
|
9672
|
+
* The headers are built from the OFFERED KEYS alone, never from a path or a title, so a header
|
|
9673
|
+
* carries nothing a member's own text could have chosen. `memberList` still wraps every member's
|
|
9674
|
+
* text, so the injection boundary is per member and the framing around it holds no corpus bytes.
|
|
9675
|
+
*
|
|
9676
|
+
* A containment claim in the prompt is not a containment guarantee: the phase re-checks that every
|
|
9677
|
+
* group the model returns sits inside ONE component, because the prompt is an instruction and the
|
|
9678
|
+
* post-pass is the enforcement.
|
|
9679
|
+
*/
|
|
9680
|
+
const dedupPrompt = (components) => {
|
|
9681
|
+
return `${components.map((members, offset) => {
|
|
9682
|
+
const keys = members.map((member) => member.key).join(", ");
|
|
9683
|
+
return `component_${offset + 1} holds ${keys}.\n\n${memberList(members)}`;
|
|
9684
|
+
}).join("\n\n")}\n\n${DEDUP_INSTRUCTION}`;
|
|
9685
|
+
};
|
|
9167
9686
|
|
|
9168
9687
|
//#endregion
|
|
9169
9688
|
//#region packages/sleep/dist/sql.js
|
|
@@ -9180,8 +9699,8 @@ const isolate = (label, call) => Effect.gen(function* () {
|
|
|
9180
9699
|
* The memory type no phase of a sleep cycle touches.
|
|
9181
9700
|
*
|
|
9182
9701
|
* A task is live working state, and every one of the fifteen phases is a judgment about REMEMBERED
|
|
9183
|
-
* FACTS: decay says a claim is fading, dedup says two claims are one,
|
|
9184
|
-
*
|
|
9702
|
+
* FACTS: decay says a claim is fading, dedup says two claims are one, edge typing says one claim
|
|
9703
|
+
* caused or contradicts another, retention says a claim has stopped earning its place. None of those hold for
|
|
9185
9704
|
* a thing an agent intends to do, and each would be wrong applied to one. A task the agent has
|
|
9186
9705
|
* not got to yet is not a claim losing confidence, and two open tasks with the same body are two
|
|
9187
9706
|
* things to do, not one fact stored twice.
|
|
@@ -9247,19 +9766,61 @@ const neighborPairs = (db, options) => firstChunkVectors(db, options.excludeType
|
|
|
9247
9766
|
limit: options.limit
|
|
9248
9767
|
})));
|
|
9249
9768
|
/**
|
|
9250
|
-
*
|
|
9769
|
+
* Active non-task pairs that occupy the SAME frame key. Dedup's component seeds.
|
|
9770
|
+
*
|
|
9771
|
+
* A frame key is a claim's slot as surface grammar states it, so two active memories sharing one are
|
|
9772
|
+
* making a claim about the same thing by the corpus's own indexed evidence — no cosine, no model.
|
|
9773
|
+
* That is signal the vector floor can miss: "the owner of the deploy runbook is Priya" and "the owner
|
|
9774
|
+
* of the deploy runbook is Priya Raman" share a slot while their bodies share almost no vocabulary,
|
|
9775
|
+
* and their measured cosine under the fixture embedder is 0.59, far under any floor a night could
|
|
9776
|
+
* afford to mine at. Seeding components with these pairs puts them in front of the model, which is
|
|
9777
|
+
* the only reader that can say whether one is a rewording of the other.
|
|
9778
|
+
*
|
|
9779
|
+
* **The statement is OUTPUT-SENSITIVE: its cost follows the frame sharing that exists, not the pair
|
|
9780
|
+
* space.** The self-join is an equality on `frame_key`, which `files_frame_key_active` indexes under
|
|
9781
|
+
* exactly this predicate (`archived = 0 AND memory_type <> 'task' AND frame_key IS NOT NULL`,
|
|
9782
|
+
* migration 0009). So each row seeks its own key's bucket and emits one row per co-occupant, and a
|
|
9783
|
+
* corpus where no two memories share a slot emits nothing having read no pairs. `frame_key IS NOT
|
|
9784
|
+
* NULL` is stated even though the join equality already excludes NULL, because it is what makes the
|
|
9785
|
+
* partial index usable rather than leaving the planner to prove it.
|
|
9786
|
+
*
|
|
9787
|
+
* `r.path < l.path` orients each unordered pair once, which keeps the seed set the same size as the
|
|
9788
|
+
* edge set the component builder wants.
|
|
9789
|
+
*
|
|
9790
|
+
* **`memory_type <> 'task'` is written as the LITERAL the index uses, not as this module's
|
|
9791
|
+
* {@link SLEEP_EXCLUDED_TYPES} binding.** It is the same exclusion for the same reason — two open
|
|
9792
|
+
* tasks phrased alike are two things to do — but `NOT IN (?)` and `<> 'task'` are different
|
|
9793
|
+
* expressions to the planner, and only the second one matches `files_frame_key_active`'s predicate.
|
|
9794
|
+
* A bound form here would read as more general while quietly turning the seek into a scan.
|
|
9795
|
+
* `activeFramesFor` writes the literal for the same reason. `tests/units.test.ts` holds the two in
|
|
9796
|
+
* agreement, so a change to the excluded set cannot leave this statement behind silently.
|
|
9797
|
+
*
|
|
9798
|
+
* Measured plan (2026-08-19, node 24.19.0 against the shipped migrations): `SCAN l` then
|
|
9799
|
+
* `SEARCH r USING INDEX files_frame_key_active (frame_key=?)`. One side walks the KEYED rows, which
|
|
9800
|
+
* the partial index confines to the rows with a frame at all, and the other seeks.
|
|
9801
|
+
*/
|
|
9802
|
+
const frameKeyPairs = (db) => db.all(`SELECT l.path AS src, r.path AS dst
|
|
9803
|
+
FROM files l
|
|
9804
|
+
JOIN files r ON r.frame_key = l.frame_key AND r.path < l.path
|
|
9805
|
+
AND r.archived = 0 AND r.memory_type <> 'task' AND r.frame_key IS NOT NULL
|
|
9806
|
+
WHERE l.archived = 0 AND l.memory_type <> 'task' AND l.frame_key IS NOT NULL
|
|
9807
|
+
ORDER BY l.path ASC, r.path ASC`);
|
|
9808
|
+
/**
|
|
9809
|
+
* Candidate pairs for edge typing: embedding-near, sharing an entity, and carrying no
|
|
9251
9810
|
* AUTHORED edge between them in either direction.
|
|
9252
9811
|
*
|
|
9253
9812
|
* The shared-entity requirement is what keeps the model budget on pairs that could actually be about
|
|
9254
|
-
* one thing. The anti-join keeps the phase from re-
|
|
9255
|
-
* authored `contradicts` is a settled fact, and re-asking the model about it would let
|
|
9256
|
-
* answer look like new information.
|
|
9813
|
+
* one thing. The anti-join keeps the phase from re-typing a pair an agent already linked. An
|
|
9814
|
+
* authored `contradicts` or `caused_by` is a settled fact, and re-asking the model about it would let
|
|
9815
|
+
* a `none` answer look like new information.
|
|
9257
9816
|
*
|
|
9258
9817
|
* **`derived = 0` is what makes the anti-join correct.** Relationship mining runs one phase EARLIER and
|
|
9259
9818
|
* writes a derived `relates_to` for every pair above 0.85 cosine, a strict superset of the
|
|
9260
|
-
* pairs above the 0.80
|
|
9261
|
-
* this
|
|
9819
|
+
* pairs above the 0.80 typing floor. An anti-join over ALL edges therefore excludes every candidate
|
|
9820
|
+
* this scan exists to find, and the phase reports `candidates: 0` forever with no error anywhere.
|
|
9262
9821
|
* A mined edge is a machine suspicion, not a settled relationship; only an authored one closes a pair.
|
|
9822
|
+
* {@link minedPairs} reads that same mined set as the OTHER arm of edge typing's candidate union, and
|
|
9823
|
+
* carries the identical anti-join for the identical reason.
|
|
9263
9824
|
*
|
|
9264
9825
|
* The statement ENUMERATES pairs from the shared-entity join instead of filtering an n×n vector
|
|
9265
9826
|
* self-join, so its cost follows the entity sharing that actually exists. Similarity then ranks in
|
|
@@ -9267,7 +9828,7 @@ const neighborPairs = (db, options) => firstChunkVectors(db, options.excludeType
|
|
|
9267
9828
|
* where the ranking CTE's `WHERE` stood: the predicates run BEFORE per-source top-`k`. `re.path <
|
|
9268
9829
|
* le.path` orients each pair once, dst below src.
|
|
9269
9830
|
*/
|
|
9270
|
-
const
|
|
9831
|
+
const sharedEntityPairs = (db, options) => {
|
|
9271
9832
|
const excluded = options.excludeTypes ?? [];
|
|
9272
9833
|
const pairs = db.all(`SELECT DISTINCT le.path AS src, re.path AS dst
|
|
9273
9834
|
FROM file_entities le
|
|
@@ -9288,6 +9849,56 @@ const conflictCandidates = (db, options) => {
|
|
|
9288
9849
|
})));
|
|
9289
9850
|
};
|
|
9290
9851
|
/**
|
|
9852
|
+
* The MINED edges of one rel, as candidate pairs: edge typing's second arm.
|
|
9853
|
+
*
|
|
9854
|
+
* Relationship mining runs one phase earlier and writes a derived `relates_to` for every pair above
|
|
9855
|
+
* its cosine floor, index-only. Those pairs are the corpus's own answer to "which memories look
|
|
9856
|
+
* related", and they are NOT a subset of {@link sharedEntityPairs}: two memories about one incident
|
|
9857
|
+
* that name no entity in common are invisible to the shared-entity join and obvious to the embedder.
|
|
9858
|
+
* Reading them here is what makes edge typing's recall the union of both signals rather than the
|
|
9859
|
+
* entity-authoring habits of whoever wrote the memories.
|
|
9860
|
+
*
|
|
9861
|
+
* `strength` is the mined edge's own cosine (`replaceMinedEdges` clamps it into `[0, 1]`), so the
|
|
9862
|
+
* caller can rank both arms of the union on one scale without re-decoding a vector. The statement
|
|
9863
|
+
* ORDERS BY it, descending, for the same reason {@link sharedEntityPairs} hands back a ranked list:
|
|
9864
|
+
* the caller's cap is a model-cost bound, and a cap over a path-ordered read would spend the night on
|
|
9865
|
+
* whichever pairs sort alphabetically first. `src_path` then `dst_path` break a tie, which is
|
|
9866
|
+
* `collectRanked`'s ordering, so both arms of the union arrive in one ordering.
|
|
9867
|
+
*
|
|
9868
|
+
* **Deliberately unbounded**, unlike the other arm: the caller ranks the UNION and caps that, so a
|
|
9869
|
+
* limit here would cut candidates before the two arms have been compared. The mined set is one row per
|
|
9870
|
+
* pair above mining's cosine floor (measured 1,498 on the production corpus), which is a read this
|
|
9871
|
+
* phase already performs once a night.
|
|
9872
|
+
*
|
|
9873
|
+
* Three filters, each load-bearing:
|
|
9874
|
+
*
|
|
9875
|
+
* - `derived = 1` restricts this to the machine-mined set. An authored `relates_to` is an agent's
|
|
9876
|
+
* assertion, and re-typing it would let a nightly job overwrite a human judgment with a narrower rel.
|
|
9877
|
+
* - `edge_class = 'memory'` is the same firewall every graph read carries.
|
|
9878
|
+
* - The `derived = 0` anti-join drops a pair that already carries ANY authored edge either way, which
|
|
9879
|
+
* is exactly {@link sharedEntityPairs}' rule. Without it a pair typed last night would be re-judged
|
|
9880
|
+
* every night, because promoting a typed edge does not delete the mined `relates_to` underneath it.
|
|
9881
|
+
*/
|
|
9882
|
+
const minedPairs = (db, options) => {
|
|
9883
|
+
const excluded = options.excludeTypes ?? [];
|
|
9884
|
+
return db.all(`SELECT e.src_path AS src, e.dst_path AS dst, e.strength AS sim
|
|
9885
|
+
FROM edges e
|
|
9886
|
+
JOIN files fs ON fs.path = e.src_path AND fs.archived = 0${typeFilterFor("fs", excluded)}
|
|
9887
|
+
JOIN files fd ON fd.path = e.dst_path AND fd.archived = 0${typeFilterFor("fd", excluded)}
|
|
9888
|
+
WHERE e.derived = 1 AND e.edge_class = 'memory' AND e.rel = ?
|
|
9889
|
+
AND NOT EXISTS (
|
|
9890
|
+
SELECT 1 FROM edges a
|
|
9891
|
+
WHERE a.derived = 0
|
|
9892
|
+
AND ((a.src_path = e.src_path AND a.dst_path = e.dst_path)
|
|
9893
|
+
OR (a.src_path = e.dst_path AND a.dst_path = e.src_path))
|
|
9894
|
+
)
|
|
9895
|
+
ORDER BY e.strength DESC, e.src_path ASC, e.dst_path ASC`, [
|
|
9896
|
+
...excluded,
|
|
9897
|
+
...excluded,
|
|
9898
|
+
options.rel
|
|
9899
|
+
]);
|
|
9900
|
+
};
|
|
9901
|
+
/**
|
|
9291
9902
|
* Every entity on an active NON-TASK file, with its file count. The union-find's input.
|
|
9292
9903
|
*
|
|
9293
9904
|
* Tasks are excluded here instead of in the two phases that read this, so both get the exclusion
|
|
@@ -9321,6 +9932,62 @@ const pathsForEntity = (db, entityType, entityName) => db.all(`SELECT e.path AS
|
|
|
9321
9932
|
/** `?` per excluded type, so the exclusion binds instead of interpolating a value into SQL. */
|
|
9322
9933
|
const typePlaceholders = () => SLEEP_EXCLUDED_TYPES.map(() => "?").join(", ");
|
|
9323
9934
|
/**
|
|
9935
|
+
* Every (entity, claiming active non-task file) pair in ONE statement, entity-ordered then path.
|
|
9936
|
+
*
|
|
9937
|
+
* The same corpus {@link activeEntities} counts, enumerated instead of aggregated. Entity resolution
|
|
9938
|
+
* needs both a per-entity memory centroid and a few sample titles per entity, and deriving either from
|
|
9939
|
+
* {@link pathsForEntity} would be one query per entity — 59 entities on the measured corpus, and one
|
|
9940
|
+
* round trip each for a join the database performs once.
|
|
9941
|
+
*
|
|
9942
|
+
* **The `ORDER BY` is for a reader, NOT for the centroid's determinism.** A centroid is a sum over its
|
|
9943
|
+
* members' vectors and float addition is not associative, so the summation order decides the bytes —
|
|
9944
|
+
* but `entityCentroids` re-sorts each entity's paths itself and does not inherit this order. That is
|
|
9945
|
+
* deliberate: the guarantee has to live where the sum happens, so a future caller reading these rows
|
|
9946
|
+
* through a different statement cannot silently lose it. (Confirmed by mutation: replacing this clause
|
|
9947
|
+
* with `ORDER BY e.path DESC` leaves the whole sleep suite green, while dropping the phase's own sort
|
|
9948
|
+
* fails it.)
|
|
9949
|
+
*/
|
|
9950
|
+
const entityClaims = (db) => db.all(`SELECT e.entity_type AS entity_type, e.entity_name AS entity_name,
|
|
9951
|
+
e.path AS path, f.title AS title
|
|
9952
|
+
FROM file_entities e JOIN files f ON f.path = e.path
|
|
9953
|
+
WHERE f.archived = 0 AND f.memory_type NOT IN (${typePlaceholders()})
|
|
9954
|
+
ORDER BY e.entity_type ASC, e.entity_name ASC, e.path ASC`, [...SLEEP_EXCLUDED_TYPES]);
|
|
9955
|
+
/**
|
|
9956
|
+
* Every active file's first-chunk vector, path-keyed and decoded once. The centroid pass's input.
|
|
9957
|
+
*
|
|
9958
|
+
* Exported wrapper over the module-private statement the pair arms use, so entity resolution reads the
|
|
9959
|
+
* SAME vector space they do — `ordinal = 0`, the same drop of a blob that does not decode — instead of
|
|
9960
|
+
* a second SELECT free to disagree about which chunk represents a file.
|
|
9961
|
+
*
|
|
9962
|
+
* Tasks are excluded, matching {@link entityClaims}: a centroid built partly from working state would
|
|
9963
|
+
* describe what the agent intends to do about a subject rather than what it knows about one.
|
|
9964
|
+
*/
|
|
9965
|
+
const entityVectors = (db) => firstChunkVectors(db, SLEEP_EXCLUDED_TYPES);
|
|
9966
|
+
/**
|
|
9967
|
+
* Every indexed person file, path-ordered. The alias oracle's file list.
|
|
9968
|
+
*
|
|
9969
|
+
* Selected by DIRECTORY, because that is what a person file is: `person-links` mints one per
|
|
9970
|
+
* `person:` entity under `PEOPLE_DIR`, and a hand-authored one placed there by an operator is just as
|
|
9971
|
+
* authoritative. Selecting by entity instead would miss a file whose subject the corpus has since
|
|
9972
|
+
* stopped mentioning, whose declaration is still the truth about those names.
|
|
9973
|
+
*
|
|
9974
|
+
* Archived files are included. Archiving a person file records that the corpus moved on from the
|
|
9975
|
+
* person, not that two of their names stopped being the same name, and an alias declaration losing its
|
|
9976
|
+
* force on archival would silently re-split a person the phase had already merged.
|
|
9977
|
+
*
|
|
9978
|
+
* **Which is why the archive prefix is matched too, and not just `archived = 0` left off.** Eviction is
|
|
9979
|
+
* the `git mv` into `archive/<YYYY>/<original-path>`, so an archived person file's PATH is
|
|
9980
|
+
* `archive/2026/resources/people/…` and no longer matches `resources/people/%` at all. A single
|
|
9981
|
+
* `LIKE` here would have said "archived files are included" while excluding every one of them, and the
|
|
9982
|
+
* re-split above is exactly what would have followed. The second pattern mirrors
|
|
9983
|
+
* `archivePathFor`'s shape (`%` for the year segment, which is four digits the statement need not
|
|
9984
|
+
* verify — a false match would be another file under `resources/people/`, which is a person file).
|
|
9985
|
+
*
|
|
9986
|
+
* The phase reads the BYTES of each of these; this statement only says which paths to open, because
|
|
9987
|
+
* `memhtml-alias` is repeatable and lives in the file rather than in any projection.
|
|
9988
|
+
*/
|
|
9989
|
+
const peoplePaths = (db) => db.all("SELECT path FROM files WHERE path LIKE ? OR path LIKE ? ORDER BY path ASC", [`${PEOPLE_DIR}/%`, `${ARCHIVE_BUCKET}/%/${PEOPLE_DIR}/%`]);
|
|
9990
|
+
/**
|
|
9324
9991
|
* The memory-class edge list over active files, both authored and derived.
|
|
9325
9992
|
*
|
|
9326
9993
|
* `edge_class = 'memory'` is the firewall. A person or provenance edge cannot enter PageRank, label
|
|
@@ -9358,7 +10025,7 @@ const accessRows = (db) => db.hasState ? db.all(`SELECT path, access_count, rein
|
|
|
9358
10025
|
* one pair cannot both read `detections = 1` and both decline to promote.
|
|
9359
10026
|
*
|
|
9360
10027
|
* **The bump is idempotent WITHIN one run's instant.** `detections` advances only when `updated_at`
|
|
9361
|
-
* differs from `at`. Corroboration means "two DIFFERENT nights saw this", and
|
|
10028
|
+
* differs from `at`. Corroboration means "two DIFFERENT nights saw this", and edge typing
|
|
9362
10029
|
* commits only when something is promoted, so a run that judged pairs and promoted nothing leaves no
|
|
9363
10030
|
* trailer and `memhtml sleep resume` re-executes it. Without the guard that second pass would count as a
|
|
9364
10031
|
* second detection and promote a contradiction one night's evidence had not earned. That puts a machine
|
|
@@ -9386,6 +10053,47 @@ const markPromoted = (db, input) => db.run(`UPDATE ${STATE_SCHEMA}.edge_corrobor
|
|
|
9386
10053
|
input.rel,
|
|
9387
10054
|
input.dstPath
|
|
9388
10055
|
]);
|
|
10056
|
+
/**
|
|
10057
|
+
* Bump an entity merge's detection counter and read the result back.
|
|
10058
|
+
*
|
|
10059
|
+
* The same `RETURNING` upsert {@link bumpCorroboration} uses, for the same reason: the promotion
|
|
10060
|
+
* decision is made in the database at the instant of the write, so two runs racing on one merge cannot
|
|
10061
|
+
* both read `detections = 1` and both decline to apply it, leaving a genuinely corroborated merge
|
|
10062
|
+
* pending forever.
|
|
10063
|
+
*
|
|
10064
|
+
* **And the bump is idempotent WITHIN one run's instant**, which entity resolution needs even more than
|
|
10065
|
+
* conflict detection does. This phase commits whenever it rewrites ANY file, so a night whose only work
|
|
10066
|
+
* was a deterministic normalization commits and leaves a trailer, while a night that only bumped
|
|
10067
|
+
* counters does not. `memhtml sleep resume` therefore re-executes this phase on the second pass, and
|
|
10068
|
+
* without the `updated_at` guard that pass would count as a second night's independent sighting and
|
|
10069
|
+
* apply a merge one night's evidence had not earned. `at` comes from the run's own date, so a resume of
|
|
10070
|
+
* the same run reuses it and a genuinely later night does not.
|
|
10071
|
+
*
|
|
10072
|
+
* Names are the NORMALIZED forms, which is what makes one merge one counter: `Checkout API` and
|
|
10073
|
+
* `checkout api` would otherwise be two rows for one merge and neither would reach two detections.
|
|
10074
|
+
*/
|
|
10075
|
+
const bumpEntityCorroboration = (db, input) => db.all(`INSERT INTO ${STATE_SCHEMA}.entity_corroboration
|
|
10076
|
+
(entity_type, alias_name, canonical_name, detections, updated_at)
|
|
10077
|
+
VALUES (?, ?, ?, 1, ?)
|
|
10078
|
+
ON CONFLICT(entity_type, alias_name, canonical_name) DO UPDATE SET
|
|
10079
|
+
detections = detections + CASE
|
|
10080
|
+
WHEN entity_corroboration.updated_at = excluded.updated_at THEN 0 ELSE 1 END,
|
|
10081
|
+
updated_at = excluded.updated_at
|
|
10082
|
+
RETURNING entity_type, alias_name, canonical_name, detections, promoted`, [
|
|
10083
|
+
input.entityType,
|
|
10084
|
+
input.aliasName,
|
|
10085
|
+
input.canonicalName,
|
|
10086
|
+
input.at
|
|
10087
|
+
]);
|
|
10088
|
+
/** Mark a corroborated merge applied, so a later night reads it as done instead of pending. */
|
|
10089
|
+
const markEntityPromoted = (db, input) => db.run(`UPDATE ${STATE_SCHEMA}.entity_corroboration
|
|
10090
|
+
SET promoted = 1, confirmed = 1, updated_at = ?
|
|
10091
|
+
WHERE entity_type = ? AND alias_name = ? AND canonical_name = ?`, [
|
|
10092
|
+
input.at,
|
|
10093
|
+
input.entityType,
|
|
10094
|
+
input.aliasName,
|
|
10095
|
+
input.canonicalName
|
|
10096
|
+
]);
|
|
9389
10097
|
/** Sessions with no memory linked to them, which is what trace-consolidation counts in v1. */
|
|
9390
10098
|
const unlinkedSessionCount = (db) => db.get(`SELECT count(*) AS n FROM traces t
|
|
9391
10099
|
WHERE NOT EXISTS (SELECT 1 FROM memory_session_links l WHERE l.session_id = t.session_id)`).pipe(Effect.map((row) => row?.n ?? 0));
|
|
@@ -9848,6 +10556,12 @@ const arcSynthesis = (env) => Effect.gen(function* () {
|
|
|
9848
10556
|
* Phase 10, compress. COMPRESS-band memories grouped by community, folded into a synthesized
|
|
9849
10557
|
* canonical in batches. ONE COMMIT PER BATCH.
|
|
9850
10558
|
*
|
|
10559
|
+
* The batching runs on the shared kernel in `batch.ts`: this phase sorts the communities and their
|
|
10560
|
+
* members, and `assembleBatches`, `keyMembers`, `compressPrompt`, and `resolveKeys` do the slicing,
|
|
10561
|
+
* the opaque keying, the prompt framing, and the key resolution that four other phases also need. The
|
|
10562
|
+
* kernel preserves the order it is handed and does no sorting of its own, so the two sorts below are
|
|
10563
|
+
* what make a night's batch boundaries and member keys reproducible.
|
|
10564
|
+
*
|
|
9851
10565
|
* Grouped by community instead of by similarity, because a community is the graph's own answer to
|
|
9852
10566
|
* "what belongs together". A similarity group folds two memories that happen to share vocabulary,
|
|
9853
10567
|
* while a community folds memories the corpus itself has linked. Communities below the minimum size
|
|
@@ -9869,6 +10583,11 @@ const arcSynthesis = (env) => Effect.gen(function* () {
|
|
|
9869
10583
|
*/
|
|
9870
10584
|
/** Members per model call. Small enough that every member's facts fit the answer's attention. */
|
|
9871
10585
|
const COMPRESS_BATCH_SIZE = 8;
|
|
10586
|
+
/**
|
|
10587
|
+
* Members a batch needs before it is worth a call. A batch of one is not a fold: it would rewrite a
|
|
10588
|
+
* lone memory into a "canonical" saying the same thing under a new path, and archive the original.
|
|
10589
|
+
*/
|
|
10590
|
+
const COMPRESS_MIN_BATCH = 2;
|
|
9872
10591
|
/** COMPRESS-band candidates considered per cycle. The model-cost guard. */
|
|
9873
10592
|
const COMPRESS_CANDIDATE_LIMIT = 2e3;
|
|
9874
10593
|
/** Characters of each member shown. A fold must see the facts, so this is wider than arc evidence. */
|
|
@@ -9893,14 +10612,16 @@ const compress = (env) => Effect.gen(function* () {
|
|
|
9893
10612
|
if (bucket === void 0) byCommunity.set(label, [entry]);
|
|
9894
10613
|
else bucket.push(entry);
|
|
9895
10614
|
}
|
|
9896
|
-
|
|
9897
|
-
|
|
9898
|
-
|
|
9899
|
-
|
|
9900
|
-
|
|
9901
|
-
|
|
9902
|
-
|
|
9903
|
-
|
|
10615
|
+
/**
|
|
10616
|
+
* Both sorts are this phase's, and the kernel keeps the order they produce. Communities are
|
|
10617
|
+
* walked lexicographically by label so a night's call order is fixed, and each community's members
|
|
10618
|
+
* by `row.path` so the `m1`..`mN` keys land on the same files twice over.
|
|
10619
|
+
*/
|
|
10620
|
+
const groups = [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1).map(([, members]) => [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0));
|
|
10621
|
+
const batches = assembleBatches(groups, {
|
|
10622
|
+
maxMembers: 8,
|
|
10623
|
+
minMembers: 2
|
|
10624
|
+
});
|
|
9904
10625
|
const counts = {
|
|
9905
10626
|
candidates: candidates.length,
|
|
9906
10627
|
communities: byCommunity.size,
|
|
@@ -9919,33 +10640,22 @@ const compress = (env) => Effect.gen(function* () {
|
|
|
9919
10640
|
let lastCommit = null;
|
|
9920
10641
|
for (const batch of batches) {
|
|
9921
10642
|
/** Opaque keys again, so `absorbedKeys` cannot name a path. */
|
|
9922
|
-
const keyed = batch
|
|
9923
|
-
key: `m${offset + 1}`,
|
|
9924
|
-
path: entry.row.path,
|
|
9925
|
-
title: entry.row.title,
|
|
9926
|
-
text: `${entry.row.title}\n${entry.row.gist}\n${entry.row.body_text}`.slice(0, COMPRESS_MEMBER_CHARS)
|
|
9927
|
-
}));
|
|
9928
|
-
const pathForKey = new Map(keyed.map((entry) => [entry.key, entry.path]));
|
|
10643
|
+
const keyed = keyMembers(batch, (entry) => `${entry.row.title}\n${entry.row.gist}\n${entry.row.body_text}`, { charBudget: COMPRESS_MEMBER_CHARS });
|
|
9929
10644
|
llmCalls += 1;
|
|
9930
|
-
const synthesis = yield*
|
|
10645
|
+
const synthesis = yield* batchCall(model, `compress batch of ${batch.length}`, {
|
|
9931
10646
|
schema: CompressSynthesis,
|
|
9932
10647
|
system: COMPRESS_SYSTEM,
|
|
9933
|
-
prompt: compressPrompt(keyed.
|
|
9934
|
-
key: entry.key,
|
|
9935
|
-
text: entry.text
|
|
9936
|
-
}))),
|
|
10648
|
+
prompt: compressPrompt(keyed.keyed),
|
|
9937
10649
|
modelKey,
|
|
9938
10650
|
effort: "high",
|
|
9939
10651
|
toolDescription: "Emit the canonical memory and the members whose content it absorbs."
|
|
9940
|
-
})
|
|
10652
|
+
});
|
|
9941
10653
|
if (synthesis === void 0) {
|
|
9942
10654
|
skipped += 1;
|
|
9943
10655
|
continue;
|
|
9944
10656
|
}
|
|
9945
|
-
|
|
9946
|
-
|
|
9947
|
-
return path === void 0 ? [] : [path];
|
|
9948
|
-
}))];
|
|
10657
|
+
/** A key the batch never offered resolves to nothing, so a fold reaches only offered files. */
|
|
10658
|
+
const absorbed = resolveKeys(keyed, synthesis.absorbedKeys).map((entry) => entry.row.path);
|
|
9949
10659
|
if (absorbed.length < 2 || synthesis.title.trim() === "" || synthesis.claim.trim() === "") {
|
|
9950
10660
|
skipped += 1;
|
|
9951
10661
|
continue;
|
|
@@ -10093,190 +10803,190 @@ const confidenceDecay = (env) => Effect.gen(function* () {
|
|
|
10093
10803
|
};
|
|
10094
10804
|
});
|
|
10095
10805
|
|
|
10096
|
-
//#endregion
|
|
10097
|
-
//#region packages/sleep/dist/phases/conflict-detection.js
|
|
10098
|
-
/**
|
|
10099
|
-
* Phase 6, conflict detection. An NLI stance judge over embedding-near same-entity pairs; a
|
|
10100
|
-
* corroborated contradiction is promoted into BOTH files and committed.
|
|
10101
|
-
*
|
|
10102
|
-
* Three stages, and keeping them separate is what makes the phase safe:
|
|
10103
|
-
*
|
|
10104
|
-
* 1. **Scan (SQL, no model).** Same-entity active pairs above {@link CONFLICT_COSINE_FLOOR} carrying
|
|
10105
|
-
* no edge in either direction, capped at {@link CONFLICT_CANDIDATE_LIMIT}.
|
|
10106
|
-
* 2. **Judge (one model call per pair, isolated).** Each call is wrapped so one malformed tool
|
|
10107
|
-
* payload skips its pair and is counted. A night that judged 199 pairs and lost the 200th has
|
|
10108
|
-
* done 199 pairs of work; failing the phase would discard all of it.
|
|
10109
|
-
* 3. **Assert (deterministic, decided here and not by the model).** Only `verdict: "contradicts"` above
|
|
10110
|
-
* the confidence floor bumps the corroboration counter, and only `detections >= 2` promotes the edge
|
|
10111
|
-
* into the files. A single machine detection therefore cannot reach the retention penalty. The
|
|
10112
|
-
* counter lives in the state plane and the penalty counts only `derived = 0` file-borne edges.
|
|
10113
|
-
*
|
|
10114
|
-
* **Detection only.** The phase asserts the contradiction and stops. It does not supersede, close a
|
|
10115
|
-
* `memhtml-valid-until`, or archive either side. Choosing the winner of a contradiction is a one-way
|
|
10116
|
-
* door on stored belief, and it belongs to an agent or a human, not to a nightly job.
|
|
10117
|
-
*/
|
|
10118
|
-
/** The moderate similarity floor a pair must clear to be worth a model call. */
|
|
10119
|
-
const CONFLICT_COSINE_FLOOR = .8;
|
|
10120
|
-
/** Nearest same-entity neighbors considered per source. */
|
|
10121
|
-
const CONFLICT_PER_SOURCE_K = 5;
|
|
10122
|
-
/** Pairs judged per cycle. The model-cost guard. */
|
|
10123
|
-
const CONFLICT_CANDIDATE_LIMIT = 200;
|
|
10124
|
-
/** Detections a machine-found contradiction needs before it is written into the files. */
|
|
10125
|
-
const PROMOTION_DETECTIONS = 2;
|
|
10126
|
-
const conflictDetection = (env) => Effect.gen(function* () {
|
|
10127
|
-
const model = env.deps.model;
|
|
10128
|
-
if (model === void 0) return {
|
|
10129
|
-
...emptyOutcome({
|
|
10130
|
-
candidates: 0,
|
|
10131
|
-
judged: 0
|
|
10132
|
-
}),
|
|
10133
|
-
detail: "no model bound"
|
|
10134
|
-
};
|
|
10135
|
-
/**
|
|
10136
|
-
* Tasks are out of the candidate set. "These two contradict" is a judgment about asserted
|
|
10137
|
-
* facts, and a task asserts nothing. A model asked about two tasks would answer a question
|
|
10138
|
-
* that has no true answer, and a promoted `contradicts` between them would be a memory-class
|
|
10139
|
-
* edge with task endpoints written into both files.
|
|
10140
|
-
*/
|
|
10141
|
-
const candidates = yield* conflictCandidates(env.deps.db, {
|
|
10142
|
-
floor: CONFLICT_COSINE_FLOOR,
|
|
10143
|
-
perSourceK: 5,
|
|
10144
|
-
limit: 200,
|
|
10145
|
-
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
10146
|
-
});
|
|
10147
|
-
if (candidates.length === 0) return emptyOutcome({
|
|
10148
|
-
candidates: 0,
|
|
10149
|
-
judged: 0,
|
|
10150
|
-
contradictions: 0,
|
|
10151
|
-
promoted: 0,
|
|
10152
|
-
skipped: 0
|
|
10153
|
-
});
|
|
10154
|
-
if (env.dryRun) return emptyOutcome({
|
|
10155
|
-
candidates: candidates.length,
|
|
10156
|
-
judged: 0,
|
|
10157
|
-
contradictions: 0,
|
|
10158
|
-
promoted: 0,
|
|
10159
|
-
skipped: 0
|
|
10160
|
-
});
|
|
10161
|
-
const corpus = yield* activeCorpus(env.deps.db);
|
|
10162
|
-
const textOf = new Map(corpus.map((row) => [row.path, `${row.gist}\n${row.body_text}`]));
|
|
10163
|
-
const modelKey = modelFor(env.deps, "conflict-detection");
|
|
10164
|
-
let judged = 0;
|
|
10165
|
-
let contradictions = 0;
|
|
10166
|
-
let promoted = 0;
|
|
10167
|
-
let skipped = 0;
|
|
10168
|
-
let llmCalls = 0;
|
|
10169
|
-
for (const candidate of candidates) {
|
|
10170
|
-
const textA = textOf.get(candidate.src);
|
|
10171
|
-
const textB = textOf.get(candidate.dst);
|
|
10172
|
-
if (textA === void 0 || textB === void 0) {
|
|
10173
|
-
skipped += 1;
|
|
10174
|
-
continue;
|
|
10175
|
-
}
|
|
10176
|
-
llmCalls += 1;
|
|
10177
|
-
const judgment = yield* isolate(`conflict-detection pair ${judged + skipped}`, model.generateObject({
|
|
10178
|
-
schema: StanceJudgment,
|
|
10179
|
-
system: STANCE_SYSTEM,
|
|
10180
|
-
prompt: stancePrompt(textA, textB),
|
|
10181
|
-
modelKey,
|
|
10182
|
-
effort: "medium",
|
|
10183
|
-
toolDescription: "Emit the stance of memory B relative to memory A."
|
|
10184
|
-
}));
|
|
10185
|
-
if (judgment === void 0) {
|
|
10186
|
-
skipped += 1;
|
|
10187
|
-
continue;
|
|
10188
|
-
}
|
|
10189
|
-
judged += 1;
|
|
10190
|
-
if (!assertsContradiction(judgment)) continue;
|
|
10191
|
-
contradictions += 1;
|
|
10192
|
-
const row = (yield* bumpCorroboration(env.deps.db, {
|
|
10193
|
-
srcPath: candidate.src,
|
|
10194
|
-
rel: "contradicts",
|
|
10195
|
-
dstPath: candidate.dst,
|
|
10196
|
-
at: env.at
|
|
10197
|
-
}))[0];
|
|
10198
|
-
if (row === void 0 || row.detections < 2 || row.promoted === 1) continue;
|
|
10199
|
-
yield* stampFile(env, candidate.src, [link("contradicts", hrefFor(candidate.dst)), meta("memhtml-updated", env.at)]);
|
|
10200
|
-
yield* stampFile(env, candidate.dst, [link("contradicts", hrefFor(candidate.src)), meta("memhtml-updated", env.at)]);
|
|
10201
|
-
yield* markPromoted(env.deps.db, {
|
|
10202
|
-
srcPath: candidate.src,
|
|
10203
|
-
rel: "contradicts",
|
|
10204
|
-
dstPath: candidate.dst,
|
|
10205
|
-
at: env.at
|
|
10206
|
-
});
|
|
10207
|
-
promoted += 1;
|
|
10208
|
-
}
|
|
10209
|
-
const counts = {
|
|
10210
|
-
candidates: candidates.length,
|
|
10211
|
-
judged,
|
|
10212
|
-
contradictions,
|
|
10213
|
-
promoted,
|
|
10214
|
-
skipped
|
|
10215
|
-
};
|
|
10216
|
-
if (promoted === 0) return {
|
|
10217
|
-
counts,
|
|
10218
|
-
commitSha: null,
|
|
10219
|
-
llmCalls
|
|
10220
|
-
};
|
|
10221
|
-
return {
|
|
10222
|
-
counts,
|
|
10223
|
-
commitSha: yield* commitPhase(env, "conflict-detection", `promote ${promoted} corroborated contradictions`, counts),
|
|
10224
|
-
llmCalls
|
|
10225
|
-
};
|
|
10226
|
-
});
|
|
10227
|
-
|
|
10228
10806
|
//#endregion
|
|
10229
10807
|
//#region packages/sleep/dist/phases/dedup-merge.js
|
|
10230
10808
|
/**
|
|
10231
10809
|
* Phase 2, dedup-merge. Fold near-duplicates: the keeper gains `memhtml-supersedes`, the dropped
|
|
10232
10810
|
* files `git mv` into the archive. ONE commit.
|
|
10233
10811
|
*
|
|
10812
|
+
* ## The model partitions; code decides
|
|
10813
|
+
*
|
|
10814
|
+
* With a model bound the phase mines a RECALL-oriented candidate set at {@link DEDUP_COMPONENT_FLOOR},
|
|
10815
|
+
* unions it with the frame-key exact matches, builds connected components over the union, and asks the
|
|
10816
|
+
* model to partition each component into merge groups. The model answers one question: which of these
|
|
10817
|
+
* memories are the same memory. It does not choose the canonical, it does not name a write target, and
|
|
10818
|
+
* it is never asked an n² pair question — a component of five is one entry in one batch's member list,
|
|
10819
|
+
* not ten pair calls.
|
|
10820
|
+
*
|
|
10821
|
+
* Everything the fold writes is derived afterwards. Orientation is arithmetic over corpus order, and
|
|
10822
|
+
* every pair a group implies is routed through `mergeCandidates`, which applies the divergence veto,
|
|
10823
|
+
* the self-merge check, the both-roles guard, and the per-night cap. So the set of pairs that CAN be
|
|
10824
|
+
* committed does not widen when a model is bound: it is the same predicate over a different candidate
|
|
10825
|
+
* set.
|
|
10826
|
+
*
|
|
10234
10827
|
* **Orientation keeps the OLDER file.** That is why the divergence veto changes outcomes instead of
|
|
10235
10828
|
* being cosmetic. A blind high-cosine merge of a newer correction into an older wrong memory does not
|
|
10236
10829
|
* merely lose information, it restores the error the correction was written to fix. `activeCorpus`
|
|
10237
10830
|
* reads oldest-first, so the older path is the keeper by construction and the choice is reproducible.
|
|
10831
|
+
* Inside a model-proposed group the keeper is the member with the lowest corpus offset, which is the
|
|
10832
|
+
* same rule applied to more than two files at once.
|
|
10238
10833
|
*
|
|
10239
10834
|
* **The veto and the in-batch role guard both live in `@memhtml/domain`.** `mergeCandidates` claims BOTH
|
|
10240
10835
|
* roles for every committed pair. A path that was a keeper cannot later be dropped, and a path that
|
|
10241
10836
|
* was dropped cannot later become a keeper. The predecessor memory system recorded only the drop side, so given
|
|
10242
10837
|
* `(gf → a)` then `(b → gf)` both decisions committed: `gf` absorbed `a` and was then archived into
|
|
10243
|
-
* `b`, superseding `a`'s content into a file the same batch destroyed.
|
|
10838
|
+
* `b`, superseding `a`'s content into a file the same batch destroyed. Batching makes that guard carry
|
|
10839
|
+
* MORE, not less: one model answer names several groups, and two groups overlapping on one path is
|
|
10840
|
+
* exactly that chain, arriving from one call instead of from two nights.
|
|
10841
|
+
*
|
|
10842
|
+
* **With no model bound the phase is the deterministic floor, unchanged.** It mines at
|
|
10843
|
+
* {@link NEAR_DUPLICATE_THRESHOLD}, orients, and hands the pairs to `mergeCandidates`. That is not a
|
|
10844
|
+
* degraded mode to be repaired later: a night with no credentials still folds every duplicate a cosine
|
|
10845
|
+
* can prove, and every count it reports is what this phase reported before it could call a model.
|
|
10846
|
+
*
|
|
10847
|
+
* ## Precedence between the two candidate sets
|
|
10848
|
+
*
|
|
10849
|
+
* Model groups are offered to `mergeCandidates` FIRST, then the mined pairs above the deterministic
|
|
10850
|
+
* floor that no group already claimed. Two properties follow, and both are the reason for the order:
|
|
10851
|
+
*
|
|
10852
|
+
* - The deterministic floor never regresses. Every pair the no-model path would have merged is still
|
|
10853
|
+
* in the list, so binding a model cannot make a night fold less than it did.
|
|
10854
|
+
* - Where the two disagree the semantic answer wins the path. A pair above 0.92 whose two files the
|
|
10855
|
+
* model instead grouped with a third folds as the model's group, because the both-roles guard gives
|
|
10856
|
+
* a path to whichever decision claims it first. The model read both files; the cosine read neither.
|
|
10857
|
+
*
|
|
10858
|
+
* Within each half the order is fixed: groups follow the batch, component, and group order they were
|
|
10859
|
+
* packed and answered in, and mined pairs stay in the kernel's `sim` DESC ordering.
|
|
10244
10860
|
*
|
|
10245
10861
|
* One commit for the whole batch, not one per pair. A keeper's `memhtml-supersedes` points at its
|
|
10246
10862
|
* dropped file's ARCHIVE path, which is where that file lives only after this commit lands.
|
|
10247
10863
|
* Splitting them would create a dangling href in the commit that made it dangle.
|
|
10248
10864
|
*/
|
|
10865
|
+
/**
|
|
10866
|
+
* The mining floor when a model is bound. RECALL-oriented, and deliberately below the merge floor.
|
|
10867
|
+
*
|
|
10868
|
+
* A pair between this and {@link NEAR_DUPLICATE_THRESHOLD} is one no cosine can settle: high enough
|
|
10869
|
+
* that the two memories are about one thing, not high enough that they are provably one claim. That
|
|
10870
|
+
* band is what a semantic reader is for, and the deterministic path cannot see into it at all. Issue
|
|
10871
|
+
* #43 measured ~800 pairs at 0.86 on the 2,907-memory production corpus against 77 at 0.92, and those
|
|
10872
|
+
* 800 collapse into components small enough that {@link DEDUP_MAX_COMPONENTS} bounds the night at tens
|
|
10873
|
+
* of calls rather than hundreds.
|
|
10874
|
+
*
|
|
10875
|
+
* The floor is only a floor. A pair that clears it still has to survive the model's partition and then
|
|
10876
|
+
* the veto, so more recall here cannot lower the bar on what gets written.
|
|
10877
|
+
*/
|
|
10878
|
+
const DEDUP_COMPONENT_FLOOR = .86;
|
|
10879
|
+
/**
|
|
10880
|
+
* Mined pairs considered per night at the recall floor.
|
|
10881
|
+
*
|
|
10882
|
+
* `MAX_MERGE_PAIRS * 8`, twice the deterministic path's `* 4`, because the floor moved down and the
|
|
10883
|
+
* pair count grows with the band while the commit cap does not move: no more than `MAX_MERGE_PAIRS`
|
|
10884
|
+
* folds land whatever this admits, so the multiplier buys candidate COVERAGE and cannot buy extra
|
|
10885
|
+
* writes. Issue #43's measurement is the sizing — ~800 pairs at 0.86 against 800 here — so a corpus of
|
|
10886
|
+
* that shape is mined whole and a larger one is truncated at a bound that is stated rather than
|
|
10887
|
+
* emergent.
|
|
10888
|
+
*/
|
|
10889
|
+
const DEDUP_PAIR_LIMIT = 100 * 8;
|
|
10890
|
+
/**
|
|
10891
|
+
* Files of one component that reach a model call. A larger component is TRUNCATED to its lowest paths.
|
|
10892
|
+
*
|
|
10893
|
+
* Eight is above the size a real near-duplicate family reaches — a fact restated eight times is
|
|
10894
|
+
* already pathological — so a component past it is almost always the recall floor having chained
|
|
10895
|
+
* several distinct facts through shared vocabulary. Handing all of it over would spend one call's whole
|
|
10896
|
+
* attention budget on the component least likely to hold a clean duplicate, and a model asked to
|
|
10897
|
+
* partition thirty loosely-related memories answers with a few large groups, which is the answer shape
|
|
10898
|
+
* the veto is least able to correct.
|
|
10899
|
+
*
|
|
10900
|
+
* Truncation keeps the LOWEST PATHS rather than the highest cosines, so which members are considered
|
|
10901
|
+
* is a property of the corpus and not of the floor. The remainder is deferred, not lost: the night's
|
|
10902
|
+
* folds change the graph, so tomorrow's components over the same corpus are smaller.
|
|
10903
|
+
*/
|
|
10904
|
+
const DEDUP_MAX_COMPONENT = 8;
|
|
10905
|
+
/**
|
|
10906
|
+
* Components handed to a model per night. The cost bound.
|
|
10907
|
+
*
|
|
10908
|
+
* Set so a night lands in issue #43's measured envelope of ~15-25 calls: at
|
|
10909
|
+
* {@link DEDUP_BATCH_MEMBERS} members per call and a typical component of two, 300 components pack
|
|
10910
|
+
* into roughly 15 calls, and the per-call character budget closes some earlier. Components are taken
|
|
10911
|
+
* in component order, which is lowest-path first, so which ones a capped night considers is
|
|
10912
|
+
* reproducible.
|
|
10913
|
+
*/
|
|
10914
|
+
const DEDUP_MAX_COMPONENTS = 300;
|
|
10915
|
+
/**
|
|
10916
|
+
* Members per dedup call. Wider than compress's 8 because the question is cheaper per member.
|
|
10917
|
+
*
|
|
10918
|
+
* compress asks the model to WRITE one canonical carrying every member's facts, so each member has to
|
|
10919
|
+
* fit the answer's generative attention. Dedup asks only which members restate each other and the
|
|
10920
|
+
* answer is a list of keys, so one batch can hold several components' worth.
|
|
10921
|
+
*/
|
|
10922
|
+
const DEDUP_BATCH_MEMBERS = 40;
|
|
10923
|
+
/** Characters of each member shown. The house member budget, the same 1200 compress uses. */
|
|
10924
|
+
const DEDUP_MEMBER_CHARS = 1200;
|
|
10925
|
+
/**
|
|
10926
|
+
* Characters per dedup call: the member budget times the member cap.
|
|
10927
|
+
*
|
|
10928
|
+
* Derived rather than chosen, so the two caps cannot drift into a call that honors one and breaches
|
|
10929
|
+
* the other. It is a CEILING and normally slack, because most members are far shorter than their
|
|
10930
|
+
* budget, which is why `packGroups` takes both bounds and closes on whichever binds first.
|
|
10931
|
+
*/
|
|
10932
|
+
const DEDUP_BATCH_CHARS = DEDUP_MEMBER_CHARS * 40;
|
|
10933
|
+
/**
|
|
10934
|
+
* The threshold the batched arm hands `mergeCandidates`, which must NOT re-gate on similarity.
|
|
10935
|
+
*
|
|
10936
|
+
* Admission on that arm is already decided when the filter runs. A group pair got there because the
|
|
10937
|
+
* model grouped it, and a mined pair got there because it cleared {@link NEAR_DUPLICATE_THRESHOLD} in
|
|
10938
|
+
* the phase's own filter. What is left for `mergeCandidates` to apply is the veto, the self check, the
|
|
10939
|
+
* both-roles guard, and the cap — the four that are about safety rather than about a number.
|
|
10940
|
+
*
|
|
10941
|
+
* Zero rather than {@link DEDUP_COMPONENT_FLOOR} because that comparison is STRICT (`<= threshold`
|
|
10942
|
+
* skips), and a group pair the corpus never mined carries the floor itself as its similarity. A
|
|
10943
|
+
* threshold of the floor would drop exactly the frame-seeded pairs that seeding exists to find, and it
|
|
10944
|
+
* would do it silently: the count would read as a veto. Nothing negative can reach here, since every
|
|
10945
|
+
* mined similarity is at or above the floor and the synthetic value IS the floor.
|
|
10946
|
+
*/
|
|
10947
|
+
const DEDUP_ADMIT_FLOOR = 0;
|
|
10948
|
+
/** The text a member is offered under: its claim and its body, the same join compress uses. */
|
|
10949
|
+
const textFor = (row) => `${row.gist}\n${row.body_text}`;
|
|
10950
|
+
/**
|
|
10951
|
+
* `arc` is excluded from the candidate set. An arc is a synthesis of many memories, so it is
|
|
10952
|
+
* embedding-near everything it summarizes, and merging one into a member would replace the
|
|
10953
|
+
* conclusion with one of its premises.
|
|
10954
|
+
*
|
|
10955
|
+
* `task` is excluded for the opposite reason: two open tasks with the same body are two things
|
|
10956
|
+
* to do, not one fact stored twice. Folding them would archive real work an agent still owes.
|
|
10957
|
+
* The `files_content_hash_active` index carves tasks out for the same reason, so structural and
|
|
10958
|
+
* semantic dedup agree about them.
|
|
10959
|
+
*/
|
|
10960
|
+
const EXCLUDED_TYPES = ["arc", ...SLEEP_EXCLUDED_TYPES];
|
|
10249
10961
|
const dedupMerge = (env) => Effect.gen(function* () {
|
|
10250
10962
|
const corpus = yield* activeCorpus(env.deps.db);
|
|
10251
10963
|
const order = new Map(corpus.map((row, offset) => [row.path, offset]));
|
|
10252
|
-
const
|
|
10253
|
-
|
|
10254
|
-
|
|
10255
|
-
* embedding-near everything it summarizes, and merging one into a member would replace the
|
|
10256
|
-
* conclusion with one of its premises.
|
|
10257
|
-
*
|
|
10258
|
-
* `task` is excluded for the opposite reason: two open tasks with the same body are two things
|
|
10259
|
-
* to do, not one fact stored twice. Folding them would archive real work an agent still owes.
|
|
10260
|
-
* The `files_content_hash_active` index carves tasks out for the same reason, so structural and
|
|
10261
|
-
* semantic dedup agree about them.
|
|
10262
|
-
*/
|
|
10964
|
+
const rowFor = new Map(corpus.map((row) => [row.path, row]));
|
|
10965
|
+
const textOf = new Map(corpus.map((row) => [row.path, textFor(row)]));
|
|
10966
|
+
const model = env.deps.model;
|
|
10263
10967
|
const pairs = yield* neighborPairs(env.deps.db, {
|
|
10264
|
-
floor: NEAR_DUPLICATE_THRESHOLD,
|
|
10968
|
+
floor: model === void 0 ? NEAR_DUPLICATE_THRESHOLD : DEDUP_COMPONENT_FLOOR,
|
|
10265
10969
|
perSourceK: 5,
|
|
10266
|
-
limit: 100 * 4,
|
|
10267
|
-
excludeTypes:
|
|
10970
|
+
limit: model === void 0 ? 100 * 4 : DEDUP_PAIR_LIMIT,
|
|
10971
|
+
excludeTypes: EXCLUDED_TYPES
|
|
10268
10972
|
});
|
|
10269
|
-
/**
|
|
10973
|
+
/**
|
|
10974
|
+
* Orient each unordered pair once, older path as keeper, and drop the mirrored duplicate. The
|
|
10975
|
+
* kernel offers each pair to BOTH endpoints' neighborhoods, so `(a, b)` and `(b, a)` both arrive.
|
|
10976
|
+
*/
|
|
10270
10977
|
const seen = /* @__PURE__ */ new Set();
|
|
10271
10978
|
const oriented = [];
|
|
10979
|
+
/** `keepPath dropPath` -> the mined similarity, so a group can report a measured value. */
|
|
10980
|
+
const simFor = /* @__PURE__ */ new Map();
|
|
10272
10981
|
for (const pair of pairs) {
|
|
10273
10982
|
const left = order.get(pair.src);
|
|
10274
10983
|
const right = order.get(pair.dst);
|
|
10275
10984
|
if (left === void 0 || right === void 0) continue;
|
|
10276
10985
|
const [keepPath, dropPath] = left <= right ? [pair.src, pair.dst] : [pair.dst, pair.src];
|
|
10277
|
-
const key = `${keepPath}
|
|
10986
|
+
const key = `${keepPath} ${dropPath}`;
|
|
10278
10987
|
if (seen.has(key)) continue;
|
|
10279
10988
|
seen.add(key);
|
|
10989
|
+
simFor.set(key, pair.sim);
|
|
10280
10990
|
oriented.push({
|
|
10281
10991
|
keepPath,
|
|
10282
10992
|
dropPath,
|
|
@@ -10285,18 +10995,180 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
10285
10995
|
dropText: textOf.get(dropPath)
|
|
10286
10996
|
});
|
|
10287
10997
|
}
|
|
10288
|
-
|
|
10289
|
-
|
|
10998
|
+
if (model === void 0) {
|
|
10999
|
+
/**
|
|
11000
|
+
* The deterministic path: the mined pairs at 0.92, oriented, through the same filter under its
|
|
11001
|
+
* own default threshold. Every count and every write here is what this phase produced before it
|
|
11002
|
+
* could call a model, which is what makes the existing dedup tests an oracle for the rest.
|
|
11003
|
+
*/
|
|
11004
|
+
const decisions = mergeCandidates(oriented);
|
|
11005
|
+
return yield* commitMerges(env, decisions, {
|
|
11006
|
+
candidates: oriented.length,
|
|
11007
|
+
components: 0,
|
|
11008
|
+
llmGroups: 0,
|
|
11009
|
+
vetoed: oriented.length - decisions.length
|
|
11010
|
+
});
|
|
11011
|
+
}
|
|
11012
|
+
/**
|
|
11013
|
+
* The component graph: the mined edges at the recall floor, unioned with the frame-key exact
|
|
11014
|
+
* matches. A frame seed is an edge no cosine produced, so the union is what puts a slot collision
|
|
11015
|
+
* in front of the model even when the two bodies share little vocabulary.
|
|
11016
|
+
*/
|
|
11017
|
+
const frameSeeds = yield* frameKeyPairs(env.deps.db);
|
|
11018
|
+
const edges = [...oriented.map((pair) => [pair.keepPath, pair.dropPath]), ...frameSeeds.flatMap((pair) => {
|
|
11019
|
+
/**
|
|
11020
|
+
* A seed is filtered by the SAME type exclusion the mining arm passes to SQL. The frame index
|
|
11021
|
+
* already carves out tasks, but not `arc` — and an arc shares a slot with any member it
|
|
11022
|
+
* summarizes, so an unfiltered seed would put the conclusion in a component with its premise
|
|
11023
|
+
* and invite the model to fold one into the other.
|
|
11024
|
+
*/
|
|
11025
|
+
const src = rowFor.get(pair.src);
|
|
11026
|
+
const dst = rowFor.get(pair.dst);
|
|
11027
|
+
if (src === void 0 || dst === void 0) return [];
|
|
11028
|
+
if (EXCLUDED_TYPES.includes(src.memory_type) || EXCLUDED_TYPES.includes(dst.memory_type)) return [];
|
|
11029
|
+
return [[pair.src, pair.dst]];
|
|
11030
|
+
})];
|
|
11031
|
+
/**
|
|
11032
|
+
* Components of two or more are the units of work, truncated to {@link DEDUP_MAX_COMPONENT} at
|
|
11033
|
+
* their lowest paths and capped at {@link DEDUP_MAX_COMPONENTS} per night. `connectedComponents`
|
|
11034
|
+
* returns members sorted and components ordered by their smallest member, so both cuts are a
|
|
11035
|
+
* function of the corpus rather than of how the edges were enumerated.
|
|
11036
|
+
*/
|
|
11037
|
+
const components = connectedComponents(edges).filter((members) => members.length >= 2).slice(0, 300).map((members) => members.slice(0, 8).flatMap((path) => {
|
|
11038
|
+
const row = rowFor.get(path);
|
|
11039
|
+
return row === void 0 ? [] : [row];
|
|
11040
|
+
})).filter((members) => members.length >= 2);
|
|
11041
|
+
/**
|
|
11042
|
+
* Whole components per call, so no group's members are split across two answers. Splitting one
|
|
11043
|
+
* would ask each half whether it holds a duplicate having hidden the other half.
|
|
11044
|
+
*/
|
|
11045
|
+
const batches = packGroups(components, {
|
|
11046
|
+
maxMembers: 40,
|
|
11047
|
+
maxChars: DEDUP_BATCH_CHARS,
|
|
11048
|
+
charsOf: (row) => Math.min(textFor(row).length, DEDUP_MEMBER_CHARS)
|
|
11049
|
+
});
|
|
11050
|
+
const modelKey = modelFor(env.deps, "dedup-merge");
|
|
11051
|
+
let llmCalls = 0;
|
|
11052
|
+
let llmGroups = 0;
|
|
11053
|
+
let skipped = 0;
|
|
11054
|
+
/** Group-implied pairs, in batch then component then group order. */
|
|
11055
|
+
const groupPairs = [];
|
|
11056
|
+
/** Every path a surviving group claimed, so the mined arm cannot re-propose one. */
|
|
11057
|
+
const grouped = /* @__PURE__ */ new Set();
|
|
11058
|
+
for (const batch of batches) {
|
|
11059
|
+
/**
|
|
11060
|
+
* ONE keying across the whole batch, not one per component. Keys have to be unique inside the
|
|
11061
|
+
* answer's namespace, and per-component keying would mint `m1` several times over — so a model
|
|
11062
|
+
* naming `m1` would name several files and `resolveKeys` could not say which.
|
|
11063
|
+
*/
|
|
11064
|
+
const keyed = keyMembers(batch.flat(), textFor, { charBudget: DEDUP_MEMBER_CHARS });
|
|
11065
|
+
/** Which component each offered key sits in. The containment check below reads this. */
|
|
11066
|
+
const componentOfKey = /* @__PURE__ */ new Map();
|
|
11067
|
+
const framed = [];
|
|
11068
|
+
let cursor = 0;
|
|
11069
|
+
for (const [offset, members] of batch.entries()) {
|
|
11070
|
+
const slice = keyed.keyed.slice(cursor, cursor + members.length);
|
|
11071
|
+
for (const member of slice) componentOfKey.set(member.key, offset);
|
|
11072
|
+
framed.push(slice);
|
|
11073
|
+
cursor += members.length;
|
|
11074
|
+
}
|
|
11075
|
+
llmCalls += 1;
|
|
11076
|
+
const partition = yield* batchCall(model, `dedup batch of ${batch.length} components`, {
|
|
11077
|
+
schema: MergePartition,
|
|
11078
|
+
system: DEDUP_SYSTEM,
|
|
11079
|
+
prompt: dedupPrompt(framed),
|
|
11080
|
+
modelKey,
|
|
11081
|
+
effort: "high",
|
|
11082
|
+
toolDescription: "Emit the merge groups: within each component, the members that are the same memory."
|
|
11083
|
+
});
|
|
11084
|
+
if (partition === void 0) {
|
|
11085
|
+
/**
|
|
11086
|
+
* One call's failure costs its own components and nothing else. `dedup-merge` is a HARD
|
|
11087
|
+
* prerequisite of compress and retention-triage, so failing the phase over one malformed tool
|
|
11088
|
+
* payload would cancel two later phases as well as this one's whole night.
|
|
11089
|
+
*/
|
|
11090
|
+
skipped += 1;
|
|
11091
|
+
continue;
|
|
11092
|
+
}
|
|
11093
|
+
for (const group of partition.groups) {
|
|
11094
|
+
const members = resolveKeys(keyed, group.memberKeys);
|
|
11095
|
+
if (members.length < 2) continue;
|
|
11096
|
+
if (new Set(group.memberKeys.flatMap((key) => {
|
|
11097
|
+
const id = componentOfKey.get(key);
|
|
11098
|
+
return id === void 0 ? [] : [id];
|
|
11099
|
+
})).size !== 1) continue;
|
|
11100
|
+
/** The keeper is the OLDEST member: the lowest corpus offset, the same rule a pair uses. */
|
|
11101
|
+
const sorted = [...members].sort((left, right) => (order.get(left.path) ?? 0) - (order.get(right.path) ?? 0));
|
|
11102
|
+
const keeper = sorted[0];
|
|
11103
|
+
if (keeper === void 0) continue;
|
|
11104
|
+
llmGroups += 1;
|
|
11105
|
+
for (const member of sorted.slice(1)) {
|
|
11106
|
+
groupPairs.push({
|
|
11107
|
+
keepPath: keeper.path,
|
|
11108
|
+
dropPath: member.path,
|
|
11109
|
+
/**
|
|
11110
|
+
* The mined similarity when this pair was itself mined, else the floor. A frame-seeded
|
|
11111
|
+
* pair and a transitive pair inside a component were never scored, and the floor is the
|
|
11112
|
+
* honest value for "at least this near, never measured closer" — see
|
|
11113
|
+
* {@link DEDUP_ADMIT_FLOOR} for why the filter must not compare against it.
|
|
11114
|
+
*/
|
|
11115
|
+
similarity: simFor.get(`${keeper.path} ${member.path}`) ?? .86,
|
|
11116
|
+
keepText: textOf.get(keeper.path),
|
|
11117
|
+
dropText: textOf.get(member.path)
|
|
11118
|
+
});
|
|
11119
|
+
grouped.add(member.path);
|
|
11120
|
+
}
|
|
11121
|
+
grouped.add(keeper.path);
|
|
11122
|
+
}
|
|
11123
|
+
}
|
|
11124
|
+
/**
|
|
11125
|
+
* Groups first, then the mined pairs above the DETERMINISTIC floor that no group claimed. The
|
|
11126
|
+
* comparison is against 0.92 and not against the recall floor, so a pair in the recall band the
|
|
11127
|
+
* model declined to group is not folded: the model's silence about it is the answer, and folding it
|
|
11128
|
+
* anyway would make the recall floor the merge floor.
|
|
11129
|
+
*/
|
|
11130
|
+
const remaining = oriented.filter((pair) => pair.similarity > .92 && !grouped.has(pair.keepPath) && !grouped.has(pair.dropPath));
|
|
11131
|
+
const proposed = [...groupPairs, ...remaining];
|
|
11132
|
+
const decisions = mergeCandidates(proposed, { threshold: 0 });
|
|
11133
|
+
return {
|
|
11134
|
+
...yield* commitMerges(env, decisions, {
|
|
11135
|
+
candidates: proposed.length,
|
|
11136
|
+
components: components.length,
|
|
11137
|
+
llmGroups,
|
|
11138
|
+
vetoed: proposed.length - decisions.length,
|
|
11139
|
+
skipped
|
|
11140
|
+
}),
|
|
11141
|
+
llmCalls
|
|
11142
|
+
};
|
|
11143
|
+
});
|
|
11144
|
+
/**
|
|
11145
|
+
* Archive each drop, stamp each keeper, and commit once.
|
|
11146
|
+
*
|
|
11147
|
+
* Shared by both arms so the WRITES do not fork on whether a model was bound: the two differ in how
|
|
11148
|
+
* they choose pairs and in nothing else. A pair that reached here has already passed the veto, the self
|
|
11149
|
+
* check, the both-roles guard, and the cap, whichever arm proposed it.
|
|
11150
|
+
*
|
|
11151
|
+
* The counts are real on a dry run — including the veto — because an operator sizing a night needs to
|
|
11152
|
+
* know what it would have folded. Only the writes are withheld.
|
|
11153
|
+
*
|
|
11154
|
+
* **A dry run here DOES spend model calls, and `entity-resolution`'s deliberately does not.** The two
|
|
11155
|
+
* choices differ because what a preview is worth differs. The number an operator wants from this phase
|
|
11156
|
+
* is how many folds a real night would make, and the model's partition is what decides that — a dry run
|
|
11157
|
+
* that skipped the call would report only the deterministic floor's folds and understate the night it is
|
|
11158
|
+
* previewing. `entity-resolution` refuses because its writes are identity rewrites, the one-way door
|
|
11159
|
+
* this codebase guards hardest: its dry run would have to bump `entity_corroboration` to be honest about
|
|
11160
|
+
* night two, and a counter bumped by a run that wrote nothing is a night of evidence the corpus never
|
|
11161
|
+
* saw.
|
|
11162
|
+
*/
|
|
11163
|
+
const commitMerges = (env, decisions, base) => Effect.gen(function* () {
|
|
10290
11164
|
if (decisions.length === 0) return emptyOutcome({
|
|
10291
|
-
|
|
11165
|
+
...base,
|
|
10292
11166
|
merged: 0,
|
|
10293
|
-
vetoed,
|
|
10294
11167
|
vanished: 0
|
|
10295
11168
|
});
|
|
10296
11169
|
if (env.dryRun) return emptyOutcome({
|
|
10297
|
-
|
|
11170
|
+
...base,
|
|
10298
11171
|
merged: decisions.length,
|
|
10299
|
-
vetoed,
|
|
10300
11172
|
vanished: 0
|
|
10301
11173
|
});
|
|
10302
11174
|
let merged = 0;
|
|
@@ -10310,45 +11182,528 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
10310
11182
|
yield* stampFile(env, decision.keepPath, [link("supersedes", hrefFor(archived)), meta("memhtml-updated", env.at)]);
|
|
10311
11183
|
merged += 1;
|
|
10312
11184
|
}
|
|
10313
|
-
const
|
|
10314
|
-
|
|
11185
|
+
const final = {
|
|
11186
|
+
...base,
|
|
10315
11187
|
merged,
|
|
10316
|
-
vetoed,
|
|
10317
11188
|
vanished
|
|
10318
11189
|
};
|
|
10319
11190
|
return {
|
|
10320
|
-
counts,
|
|
10321
|
-
commitSha: yield* commitPhase(env, "dedup-merge", `fold ${merged} near-duplicates into canonicals`,
|
|
11191
|
+
counts: final,
|
|
11192
|
+
commitSha: yield* commitPhase(env, "dedup-merge", `fold ${merged} near-duplicates into canonicals`, final),
|
|
10322
11193
|
llmCalls: 0
|
|
10323
11194
|
};
|
|
10324
11195
|
});
|
|
10325
11196
|
|
|
10326
11197
|
//#endregion
|
|
10327
|
-
//#region packages/sleep/dist/phases/
|
|
11198
|
+
//#region packages/sleep/dist/phases/edge-typing.js
|
|
11199
|
+
/**
|
|
11200
|
+
* Phase 6, edge typing. Candidate pairs grouped and BATCHED, one structured verdict list per call
|
|
11201
|
+
* over the whole memory-rel vocabulary, then a deterministic promotion. One commit for the night's
|
|
11202
|
+
* promotions.
|
|
11203
|
+
*
|
|
11204
|
+
* Four stages, and keeping them separate is what makes the phase safe:
|
|
11205
|
+
*
|
|
11206
|
+
* 1. **Scan (SQL, no model).** The union of two candidate arms, deduplicated by unordered pair and
|
|
11207
|
+
* RANKED BY SIMILARITY before the cap: relationship mining's derived `relates_to` edges
|
|
11208
|
+
* ({@link minedPairs}) and the shared-entity scan ({@link sharedEntityPairs}). Neither arm
|
|
11209
|
+
* subsumes the other — two memories about one incident naming no common entity are invisible to
|
|
11210
|
+
* the join and obvious to the embedder, and a same-entity pair below the mining floor is the
|
|
11211
|
+
* reverse — so recall is the union rather than whichever signal happens to be stronger in a
|
|
11212
|
+
* corpus. Both arms exclude tasks and anti-join pairs that already carry an AUTHORED edge either
|
|
11213
|
+
* way.
|
|
11214
|
+
* 2. **Batch (deterministic).** Pairs sorted by the directory both endpoints share, then sliced at
|
|
11215
|
+
* {@link EDGE_PAIRS_PER_CALL} on the shared kernel, so topically related pairs land in one call.
|
|
11216
|
+
* One call per batch, never one per pair: at the measured 1,498 mined pairs a night, per-pair
|
|
11217
|
+
* judging is 1,498 calls and does not scale.
|
|
11218
|
+
* 3. **Judge (one model call per batch, isolated).** Each call is wrapped so one malformed tool
|
|
11219
|
+
* payload skips its BATCH and is counted. A night that typed nine batches and lost the tenth has
|
|
11220
|
+
* done nine batches of work; failing the phase would discard all of it.
|
|
11221
|
+
* 4. **Promote (deterministic, decided here and not by the model).** The model proposes a rel, a
|
|
11222
|
+
* direction, and a confidence; code decides what is written:
|
|
11223
|
+
* - `contradicts` above `EDGE_CONFIDENCE_FLOOR` bumps the corroboration counter and is
|
|
11224
|
+
* written into BOTH files only at `detections >= 2`. A single machine detection therefore
|
|
11225
|
+
* cannot reach the retention penalty, which counts only `derived = 0` file-borne edges. Both
|
|
11226
|
+
* endpoints must still be in the TREE, checked before either write, and the counter is marked
|
|
11227
|
+
* promoted only when both sides actually gained the link — otherwise the pair is left
|
|
11228
|
+
* re-eligible for a later night rather than recorded as half done.
|
|
11229
|
+
* - A DIRECTIONAL rel above the floor is written into the SUBJECT's file alone, per the
|
|
11230
|
+
* direction the model named. No corroboration gate: a `part_of` carries no penalty and is
|
|
11231
|
+
* cheap for a reviewer to delete, so a second night's wait would buy nothing.
|
|
11232
|
+
* - `none`, or anything below the floor, writes nothing at all and leaves the pair a mined
|
|
11233
|
+
* `relates_to`. That is the safe outcome and the one an unsure model is told to pick.
|
|
11234
|
+
* - A verdict naming a key the batch never offered resolves to nothing and is dropped, so a
|
|
11235
|
+
* hallucinated key cannot become a write. A pair the model omits is simply not typed tonight.
|
|
11236
|
+
*
|
|
11237
|
+
* **Determinism is the phase's, not the kernel's.** Both sorts below fix the batch boundaries and
|
|
11238
|
+
* the `m1`..`mN` keys, and the kernel preserves the order it is handed. Two runs over an unchanged
|
|
11239
|
+
* corpus therefore send the same prompt bytes in the same order.
|
|
11240
|
+
*
|
|
11241
|
+
* **Deferred: a `none` pair is re-judged on every later night, bounded by the candidate cap.** Neither
|
|
11242
|
+
* arm records that a pair was judged and answered `none`, so the pair stays a mined `relates_to` and
|
|
11243
|
+
* re-enters the union tomorrow. The cap is what bounds that cost — a night judges
|
|
11244
|
+
* {@link EDGE_TYPING_CANDIDATE_LIMIT} pairs whatever their history — and a judged-`none` watermark is
|
|
11245
|
+
* new durable state-plane surface, so it stays out of this change.
|
|
11246
|
+
*
|
|
11247
|
+
* **Detection only, still.** A promoted `contradicts` asserts the conflict and stops: nothing is
|
|
11248
|
+
* superseded, no `memhtml-valid-until` is closed, neither side is archived. Choosing the winner of a
|
|
11249
|
+
* contradiction is a one-way door on stored belief, and it belongs to an agent or a human, not to a
|
|
11250
|
+
* nightly job.
|
|
11251
|
+
*
|
|
11252
|
+
* This phase replaced `conflict-detection`, which asked one `generateObject` per pair for a stance
|
|
11253
|
+
* verdict over `{contradicts, entails, neutral}`. Contradiction is now one more verdict in the same
|
|
11254
|
+
* list, with the same corroboration gate. A run whose commits predate the rename carries
|
|
11255
|
+
* `Memhtml-Phase: conflict-detection` trailers, and `memhtml sleep resume` matches trailers by name,
|
|
11256
|
+
* so resuming a pre-rename run re-executes this phase; that is out of scope and costs a re-judge, not
|
|
11257
|
+
* a wrong write, because every write below is idempotent on its pair.
|
|
11258
|
+
*/
|
|
11259
|
+
/**
|
|
11260
|
+
* Pairs offered per model call.
|
|
11261
|
+
*
|
|
11262
|
+
* Sized for the answer's attention rather than for the context window: thirty pairs is sixty
|
|
11263
|
+
* memories, each sliced to {@link EDGE_PAIR_SIDE_CHARS}, and the model has to hold a distinct
|
|
11264
|
+
* judgment for each one. A batch twice this size buys half the calls and invites the model to answer
|
|
11265
|
+
* the first ten pairs carefully and the rest by pattern.
|
|
11266
|
+
*/
|
|
11267
|
+
const EDGE_PAIRS_PER_CALL = 30;
|
|
11268
|
+
/** The similarity floor a shared-entity pair must clear to be worth including. */
|
|
11269
|
+
const EDGE_COSINE_FLOOR = .8;
|
|
11270
|
+
/** Nearest same-entity neighbors considered per source, on the shared-entity arm. */
|
|
11271
|
+
const EDGE_PER_SOURCE_K = 5;
|
|
11272
|
+
/**
|
|
11273
|
+
* Pairs typed per cycle. The model-cost guard, unchanged from the per-pair phase's 200 even though
|
|
11274
|
+
* the calls are now ~7 instead of 200: the cap bounds how many AUTHORED EDGES one night can write
|
|
11275
|
+
* into the corpus, and that budget did not get cheaper because the judging did.
|
|
11276
|
+
*/
|
|
11277
|
+
const EDGE_TYPING_CANDIDATE_LIMIT = 200;
|
|
11278
|
+
/**
|
|
11279
|
+
* Characters of EACH SIDE of a pair shown. The house per-member budget, applied per side.
|
|
11280
|
+
*
|
|
11281
|
+
* A pair's member text holds two memories, so the kernel's per-member slice would cut the whole
|
|
11282
|
+
* `src` + `dst` block at one budget and could truncate `dst` away entirely on a long `src` — a
|
|
11283
|
+
* verdict about a pair whose second half the model never saw. Slicing each side first bounds the
|
|
11284
|
+
* pair at twice this and guarantees both halves are present.
|
|
11285
|
+
*/
|
|
11286
|
+
const EDGE_PAIR_SIDE_CHARS = 1200;
|
|
11287
|
+
/** Detections a machine-found contradiction needs before it is written into the files. */
|
|
11288
|
+
const PROMOTION_DETECTIONS = 2;
|
|
11289
|
+
/**
|
|
11290
|
+
* Authored edges one night may promote, across every batch and both kinds.
|
|
11291
|
+
*
|
|
11292
|
+
* The candidate cap bounds what is JUDGED and this bounds what is WRITTEN, and they are different
|
|
11293
|
+
* guards: a model that answered `caused_by` at confidence 1.0 for all 200 candidates would otherwise
|
|
11294
|
+
* add 200 `<link>` lines to the corpus in one commit, which is not a diff a human reviews. Hitting
|
|
11295
|
+
* the cap is visible as `capped` in the counts.
|
|
11296
|
+
*/
|
|
11297
|
+
const EDGE_PROMOTION_CAP = 50;
|
|
10328
11298
|
/**
|
|
10329
|
-
*
|
|
10330
|
-
* ONE commit rewriting `memhtml-entity` values in place.
|
|
11299
|
+
* The union of both candidate arms, deduplicated by UNORDERED pair and ranked `sim` DESC.
|
|
10331
11300
|
*
|
|
10332
|
-
*
|
|
10333
|
-
*
|
|
10334
|
-
*
|
|
10335
|
-
*
|
|
11301
|
+
* The two arms orient their pairs differently — the shared-entity join emits `dst < src` and a mined
|
|
11302
|
+
* edge carries whichever orientation mining wrote — so the dedup key sorts the endpoints. Without
|
|
11303
|
+
* that, one pair reaching both arms would be typed twice in one night, and the two verdicts could
|
|
11304
|
+
* disagree.
|
|
10336
11305
|
*
|
|
10337
|
-
*
|
|
10338
|
-
*
|
|
10339
|
-
*
|
|
10340
|
-
*
|
|
10341
|
-
* merging them would fuse two services' memories permanently. A character ratio cannot make that
|
|
10342
|
-
* mistake. This is the packet's documented choice between the two options it offered.
|
|
11306
|
+
* The kept row is the FIRST one seen, and the mined arm is walked first, so a pair in both arms
|
|
11307
|
+
* carries mining's own orientation AND mining's own `sim`. That choice is arbitrary, because the
|
|
11308
|
+
* direction the phase writes comes from the model's `direction` field relative to this orientation,
|
|
11309
|
+
* so it must only be STABLE, which the sort makes it.
|
|
10343
11310
|
*
|
|
10344
|
-
*
|
|
10345
|
-
*
|
|
10346
|
-
*
|
|
11311
|
+
* **`sim` DESC is what makes the candidate cap select rather than truncate.** Both arms carry a
|
|
11312
|
+
* similarity on one scale — `sharedEntityPairs` reports the cosine `rankCandidatePairs` computed and
|
|
11313
|
+
* `minedPairs` reports the mined edge's own clamped cosine — so the union is rankable without
|
|
11314
|
+
* re-decoding a vector. A path-ordered union capped at {@link EDGE_TYPING_CANDIDATE_LIMIT} would spend
|
|
11315
|
+
* the whole night's model budget on whichever pairs sort alphabetically first, so a corpus whose
|
|
11316
|
+
* strongest candidates live under `services/` or `team/` would never have them judged at all, however
|
|
11317
|
+
* many nights ran. The tie-break is `src` ASC then `dst` ASC, which is `collectRanked`'s ordering in
|
|
11318
|
+
* `@memhtml/domain` — the house rule every other pair consumer already follows, so two runs over an
|
|
11319
|
+
* unchanged corpus select and batch the same pairs.
|
|
10347
11320
|
*/
|
|
10348
|
-
|
|
11321
|
+
const unionPairs$1 = (arms) => {
|
|
11322
|
+
const seen = /* @__PURE__ */ new Set();
|
|
11323
|
+
const out = [];
|
|
11324
|
+
for (const arm of arms) for (const pair of arm) {
|
|
11325
|
+
const key = pair.src < pair.dst ? `${pair.src} ${pair.dst}` : `${pair.dst} ${pair.src}`;
|
|
11326
|
+
if (seen.has(key)) continue;
|
|
11327
|
+
seen.add(key);
|
|
11328
|
+
out.push(pair);
|
|
11329
|
+
}
|
|
11330
|
+
return out.sort((left, right) => {
|
|
11331
|
+
if (left.sim !== right.sim) return left.sim < right.sim ? 1 : -1;
|
|
11332
|
+
if (left.src !== right.src) return left.src < right.src ? -1 : 1;
|
|
11333
|
+
return left.dst < right.dst ? -1 : left.dst > right.dst ? 1 : 0;
|
|
11334
|
+
});
|
|
11335
|
+
};
|
|
11336
|
+
/**
|
|
11337
|
+
* The night's candidate pairs: the union of both arms, ranked `sim` DESC, then capped.
|
|
11338
|
+
*
|
|
11339
|
+
* The rank is {@link unionPairs}' and the cap is applied AFTER it, so the cap selects the strongest
|
|
11340
|
+
* {@link EDGE_TYPING_CANDIDATE_LIMIT} pairs the corpus offers rather than the alphabetically first
|
|
11341
|
+
* ones. That ordering is also the batch order's first input, so a night's strongest pairs are judged
|
|
11342
|
+
* even when the cap bites.
|
|
11343
|
+
*
|
|
11344
|
+
* Its own function so the SCAN is separable from the judging, and exported so a test asserting on a
|
|
11345
|
+
* batch boundary, a cap, or a skip count reads the same set the phase will type instead of
|
|
11346
|
+
* reconstructing it. A test that rebuilt this by hand would be a second implementation free to
|
|
11347
|
+
* disagree, and every count below is stated relative to it. The scan's own correctness has an
|
|
11348
|
+
* independent all-SQL oracle in `tests/neighbor-pairs.test.ts`; this is the composition of two
|
|
11349
|
+
* already-tested reads.
|
|
11350
|
+
*/
|
|
11351
|
+
const edgeTypingCandidates = (db) => Effect.gen(function* () {
|
|
11352
|
+
const mined = yield* minedPairs(db, {
|
|
11353
|
+
rel: "relates_to",
|
|
11354
|
+
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
11355
|
+
});
|
|
11356
|
+
const shared = yield* sharedEntityPairs(db, {
|
|
11357
|
+
floor: EDGE_COSINE_FLOOR,
|
|
11358
|
+
perSourceK: 5,
|
|
11359
|
+
limit: 200,
|
|
11360
|
+
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
11361
|
+
});
|
|
11362
|
+
return unionPairs$1([mined, shared]).slice(0, 200);
|
|
11363
|
+
});
|
|
11364
|
+
/**
|
|
11365
|
+
* The grouping key for batching: the deepest directory both endpoints share, or `""` when they share
|
|
11366
|
+
* none.
|
|
11367
|
+
*
|
|
11368
|
+
* Deliberately NOT the graph community. `runRetentionPass` computes label propagation over the whole
|
|
11369
|
+
* memory-edge list plus PageRank plus the access plane, and this phase needs none of that — it would
|
|
11370
|
+
* be a second corpus-wide pass for a grouping hint, and it answers `undefined` for every pair in a
|
|
11371
|
+
* community below the size floor, which is most pairs in a small corpus. The shared directory is
|
|
11372
|
+
* already the corpus's own topical partition (`areas/deploy`, `areas/oncall`), it is a pure function
|
|
11373
|
+
* of two paths, and it puts related pairs in one call, which is all the batching needs from it. A
|
|
11374
|
+
* model shown thirty pairs from one area also has the area's context, which is the substantive half
|
|
11375
|
+
* of what community grouping was for.
|
|
11376
|
+
*/
|
|
11377
|
+
const pairGroupKey = (pair) => {
|
|
11378
|
+
const left = pair.src.split("/");
|
|
11379
|
+
const right = pair.dst.split("/");
|
|
11380
|
+
const shared = [];
|
|
11381
|
+
for (let at = 0; at < Math.min(left.length, right.length) - 1; at += 1) {
|
|
11382
|
+
if (left[at] !== right[at]) break;
|
|
11383
|
+
shared.push(left[at]);
|
|
11384
|
+
}
|
|
11385
|
+
return shared.join("/");
|
|
11386
|
+
};
|
|
11387
|
+
const edgeTyping = (env) => Effect.gen(function* () {
|
|
11388
|
+
const model = env.deps.model;
|
|
11389
|
+
if (model === void 0) return {
|
|
11390
|
+
...emptyOutcome({
|
|
11391
|
+
candidates: 0,
|
|
11392
|
+
judged: 0
|
|
11393
|
+
}),
|
|
11394
|
+
detail: "no model bound"
|
|
11395
|
+
};
|
|
11396
|
+
/**
|
|
11397
|
+
* Tasks are out of both candidate arms, inside {@link edgeTypingCandidates}. Every rel in the
|
|
11398
|
+
* vocabulary is a judgment about asserted facts, and a task asserts nothing: "these two
|
|
11399
|
+
* contradict" and "this one caused that one" have no true answer about intended work. A promoted
|
|
11400
|
+
* edge between two tasks would also be a memory-class edge with task endpoints written into both
|
|
11401
|
+
* files.
|
|
11402
|
+
*/
|
|
11403
|
+
const candidates = yield* edgeTypingCandidates(env.deps.db);
|
|
11404
|
+
/**
|
|
11405
|
+
* The full path's count SHAPE, at zero. Every key the phase can report is present, because a
|
|
11406
|
+
* report reader comparing two nights reads a missing key as a phase that does not have that
|
|
11407
|
+
* concept rather than as a night that did none of it.
|
|
11408
|
+
*/
|
|
11409
|
+
const zero = {
|
|
11410
|
+
candidates: 0,
|
|
11411
|
+
judged: 0,
|
|
11412
|
+
typed: 0,
|
|
11413
|
+
contradictions: 0,
|
|
11414
|
+
promoted: 0,
|
|
11415
|
+
skipped: 0,
|
|
11416
|
+
capped: 0,
|
|
11417
|
+
duplicates: 0
|
|
11418
|
+
};
|
|
11419
|
+
if (candidates.length === 0) return emptyOutcome(zero);
|
|
11420
|
+
if (env.dryRun) return emptyOutcome({
|
|
11421
|
+
...zero,
|
|
11422
|
+
candidates: candidates.length
|
|
11423
|
+
});
|
|
11424
|
+
const corpus = yield* activeCorpus(env.deps.db);
|
|
11425
|
+
const textOf = new Map(corpus.map((row) => [row.path, `${row.gist}\n${row.body_text}`]));
|
|
11426
|
+
/**
|
|
11427
|
+
* A pair whose endpoint the corpus no longer holds is dropped before batching rather than inside
|
|
11428
|
+
* the loop. An earlier phase's archive is the normal case here (the index is refreshed once, in
|
|
11429
|
+
* preflight), and dropping it later would leave a hole in the numbered list the model is asked
|
|
11430
|
+
* about.
|
|
11431
|
+
*/
|
|
11432
|
+
const withText = [];
|
|
11433
|
+
let skipped = 0;
|
|
11434
|
+
for (const pair of candidates) {
|
|
11435
|
+
const srcText = textOf.get(pair.src);
|
|
11436
|
+
const dstText = textOf.get(pair.dst);
|
|
11437
|
+
if (srcText === void 0 || dstText === void 0) {
|
|
11438
|
+
skipped += 1;
|
|
11439
|
+
continue;
|
|
11440
|
+
}
|
|
11441
|
+
withText.push({
|
|
11442
|
+
pair,
|
|
11443
|
+
srcText,
|
|
11444
|
+
dstText
|
|
11445
|
+
});
|
|
11446
|
+
}
|
|
11447
|
+
/**
|
|
11448
|
+
* The group is a SORT KEY, not a batch boundary, and that distinction is the phase's cost model.
|
|
11449
|
+
*
|
|
11450
|
+
* `dedup-merge` packs whole groups with the boundaries preserved because there a boundary is
|
|
11451
|
+
* EVIDENCE: two members in different components are known not to be near-duplicates. Here every
|
|
11452
|
+
* pair is judged on its own two memories, so a boundary carries no information the verdict needs —
|
|
11453
|
+
* and honoring it would cost a model call per group. On a corpus whose pairs spread over a dozen
|
|
11454
|
+
* directories that is a dozen calls for thirty pairs, which is the per-pair shape this phase
|
|
11455
|
+
* exists to replace. Sorting by the group instead keeps related pairs ADJACENT, so they land in
|
|
11456
|
+
* one call whenever they fit, and the call count is `ceil(pairs / EDGE_PAIRS_PER_CALL)`.
|
|
11457
|
+
*
|
|
11458
|
+
* The sort is this phase's and the kernel keeps the order it produces: group key first, then the
|
|
11459
|
+
* `src`/`dst` order `unionPairs` already fixed, so a night's batch boundaries and `m1`..`mN` keys
|
|
11460
|
+
* are a function of the corpus alone.
|
|
11461
|
+
*/
|
|
11462
|
+
const sorted = [...withText].sort((left, right) => {
|
|
11463
|
+
const leftKey = pairGroupKey(left.pair);
|
|
11464
|
+
const rightKey = pairGroupKey(right.pair);
|
|
11465
|
+
if (leftKey !== rightKey) return leftKey < rightKey ? -1 : 1;
|
|
11466
|
+
if (left.pair.src !== right.pair.src) return left.pair.src < right.pair.src ? -1 : 1;
|
|
11467
|
+
return left.pair.dst < right.pair.dst ? -1 : left.pair.dst > right.pair.dst ? 1 : 0;
|
|
11468
|
+
});
|
|
11469
|
+
const batches = assembleBatches([sorted], { maxMembers: 30 });
|
|
11470
|
+
const modelKey = modelFor(env.deps, "edge-typing");
|
|
11471
|
+
let judged = 0;
|
|
11472
|
+
let typed = 0;
|
|
11473
|
+
let contradictions = 0;
|
|
11474
|
+
let promoted = 0;
|
|
11475
|
+
let capped = 0;
|
|
11476
|
+
/** Second-and-later verdicts naming a key their batch had already answered for. */
|
|
11477
|
+
let duplicates = 0;
|
|
11478
|
+
let llmCalls = 0;
|
|
11479
|
+
for (const batch of batches) {
|
|
11480
|
+
/** Opaque keys again, so a verdict cannot name a path. Each SIDE is sliced to its budget. */
|
|
11481
|
+
const keyed = keyMembers(batch, (candidate) => pairText(candidate.srcText.slice(0, EDGE_PAIR_SIDE_CHARS), candidate.dstText.slice(0, EDGE_PAIR_SIDE_CHARS)));
|
|
11482
|
+
llmCalls += 1;
|
|
11483
|
+
const answer = yield* batchCall(model, `edge-typing batch of ${batch.length}`, {
|
|
11484
|
+
schema: EdgeTyping,
|
|
11485
|
+
system: EDGE_TYPING_SYSTEM,
|
|
11486
|
+
prompt: edgeTypingPrompt(keyed.keyed),
|
|
11487
|
+
modelKey,
|
|
11488
|
+
effort: "medium",
|
|
11489
|
+
toolDescription: "Emit one relationship verdict per candidate pair."
|
|
11490
|
+
});
|
|
11491
|
+
if (answer === void 0) {
|
|
11492
|
+
skipped += batch.length;
|
|
11493
|
+
continue;
|
|
11494
|
+
}
|
|
11495
|
+
/**
|
|
11496
|
+
* The keys this batch has already answered for, so a SECOND verdict naming one is dropped.
|
|
11497
|
+
*
|
|
11498
|
+
* A verdict is one pair's answer, and nothing in the schema stops a model from emitting two for
|
|
11499
|
+
* one key. Acting on both would write two authored edges from one relationship — and since the
|
|
11500
|
+
* two are free to disagree about `direction`, `caused_by` could land in BOTH files, which says
|
|
11501
|
+
* each memory caused the other. `resolveKeys` does not help: it is called one key at a time
|
|
11502
|
+
* here, because a verdict names one pair, so its own repeat-collapsing never sees the pair.
|
|
11503
|
+
*
|
|
11504
|
+
* FIRST wins rather than last, and the choice is the same one {@link unionPairs} makes: the
|
|
11505
|
+
* batch's order is deterministic, so which verdict is first is reproducible, and a later verdict
|
|
11506
|
+
* cannot revise a write already committed to the tree. Repeats are counted in `duplicates`
|
|
11507
|
+
* rather than silently swallowed, so a model doing this is visible in a night's report.
|
|
11508
|
+
*/
|
|
11509
|
+
const answered = /* @__PURE__ */ new Set();
|
|
11510
|
+
for (const verdict of answer.verdicts) {
|
|
11511
|
+
/**
|
|
11512
|
+
* The key is resolved through the kernel, so an invented key yields no candidate and no
|
|
11513
|
+
* write.
|
|
11514
|
+
*/
|
|
11515
|
+
const [candidate] = resolveKeys(keyed, [verdict.pairKey]);
|
|
11516
|
+
if (candidate === void 0) continue;
|
|
11517
|
+
if (answered.has(verdict.pairKey)) {
|
|
11518
|
+
duplicates += 1;
|
|
11519
|
+
continue;
|
|
11520
|
+
}
|
|
11521
|
+
answered.add(verdict.pairKey);
|
|
11522
|
+
judged += 1;
|
|
11523
|
+
if (!assertsEdge(verdict)) continue;
|
|
11524
|
+
if (assertsContradiction(verdict)) {
|
|
11525
|
+
contradictions += 1;
|
|
11526
|
+
const row = (yield* bumpCorroboration(env.deps.db, {
|
|
11527
|
+
srcPath: candidate.pair.src,
|
|
11528
|
+
rel: "contradicts",
|
|
11529
|
+
dstPath: candidate.pair.dst,
|
|
11530
|
+
at: env.at
|
|
11531
|
+
}))[0];
|
|
11532
|
+
if (row === void 0 || row.detections < 2 || row.promoted === 1) continue;
|
|
11533
|
+
if (promoted + typed >= 50) {
|
|
11534
|
+
capped += 1;
|
|
11535
|
+
continue;
|
|
11536
|
+
}
|
|
11537
|
+
/**
|
|
11538
|
+
* **BOTH endpoints, or nothing at all — checked BEFORE either write.**
|
|
11539
|
+
*
|
|
11540
|
+
* A `contradicts` is symmetric, and the phase's own promotion rule is that a reader arriving
|
|
11541
|
+
* at either file sees it. So the pair is all-or-nothing, and the check has to come first
|
|
11542
|
+
* because the alternative is not recoverable: stamping `src` and then finding `dst` gone
|
|
11543
|
+
* leaves a `<link>` pointing at a path the tree does not hold — a dangling href committed by
|
|
11544
|
+
* the commit that created it — while the other half of the conflict is invisible.
|
|
11545
|
+
*
|
|
11546
|
+
* A missing endpoint is ORDINARY here, not exceptional. Every phase reads its candidates from
|
|
11547
|
+
* an index refreshed once in preflight and not again, so a file an earlier phase archived is
|
|
11548
|
+
* still listed active at its old path when this phase reads it. `readFileBytes` answers
|
|
11549
|
+
* `undefined` for exactly that case, and the TREE is the system of record.
|
|
11550
|
+
*/
|
|
11551
|
+
/**
|
|
11552
|
+
* **BOTH endpoints, or nothing at all — checked BEFORE either write.**
|
|
11553
|
+
*
|
|
11554
|
+
* A `contradicts` is symmetric, and the phase's own promotion rule is that a reader arriving
|
|
11555
|
+
* at either file sees it. So the pair is all-or-nothing, and the check has to come first
|
|
11556
|
+
* because the alternative is not recoverable: stamping `src` and then finding `dst` gone
|
|
11557
|
+
* leaves a `<link>` pointing at a path the tree does not hold — a dangling href committed by
|
|
11558
|
+
* the commit that created it — while the other half of the conflict is invisible.
|
|
11559
|
+
*
|
|
11560
|
+
* A missing endpoint is ORDINARY here, not exceptional. Every phase reads its candidates from
|
|
11561
|
+
* an index refreshed once in preflight and not again, so a file an earlier phase archived is
|
|
11562
|
+
* still listed active at its old path when this phase reads it. `readFileBytes` answers
|
|
11563
|
+
* `undefined` for exactly that case, and the TREE is the system of record.
|
|
11564
|
+
*/
|
|
11565
|
+
const haveSrc = yield* readFileBytes(env, candidate.pair.src);
|
|
11566
|
+
const haveDst = yield* readFileBytes(env, candidate.pair.dst);
|
|
11567
|
+
if (haveSrc === void 0 || haveDst === void 0) continue;
|
|
11568
|
+
const wroteSrc = yield* stampFile(env, candidate.pair.src, [link("contradicts", hrefFor(candidate.pair.dst)), meta("memhtml-updated", env.at)]);
|
|
11569
|
+
const wroteDst = yield* stampFile(env, candidate.pair.dst, [link("contradicts", hrefFor(candidate.pair.src)), meta("memhtml-updated", env.at)]);
|
|
11570
|
+
/**
|
|
11571
|
+
* The counter is promoted only when BOTH sides gained the edge on this run. `stampFile`'s
|
|
11572
|
+
* `false` also covers "the head already said this", so a pair whose files were somehow
|
|
11573
|
+
* stamped without the counter being promoted stays un-promoted — and therefore RE-ELIGIBLE,
|
|
11574
|
+
* which is the outcome that lets a later night with a refreshed index finish the job rather
|
|
11575
|
+
* than record a half-written edge as done.
|
|
11576
|
+
*/
|
|
11577
|
+
/**
|
|
11578
|
+
* The counter is promoted only when BOTH sides gained the edge on this run. `stampFile`'s
|
|
11579
|
+
* `false` also covers "the head already said this", so a pair whose files were somehow
|
|
11580
|
+
* stamped without the counter being promoted stays un-promoted — and therefore RE-ELIGIBLE,
|
|
11581
|
+
* which is the outcome that lets a later night with a refreshed index finish the job rather
|
|
11582
|
+
* than record a half-written edge as done.
|
|
11583
|
+
*/
|
|
11584
|
+
if (!wroteSrc || !wroteDst) continue;
|
|
11585
|
+
yield* markPromoted(env.deps.db, {
|
|
11586
|
+
srcPath: candidate.pair.src,
|
|
11587
|
+
rel: "contradicts",
|
|
11588
|
+
dstPath: candidate.pair.dst,
|
|
11589
|
+
at: env.at
|
|
11590
|
+
});
|
|
11591
|
+
promoted += 1;
|
|
11592
|
+
continue;
|
|
11593
|
+
}
|
|
11594
|
+
if (!isDirectionalRel(verdict.rel)) continue;
|
|
11595
|
+
if (promoted + typed >= 50) {
|
|
11596
|
+
capped += 1;
|
|
11597
|
+
continue;
|
|
11598
|
+
}
|
|
11599
|
+
/**
|
|
11600
|
+
* ONE file, the subject's. A directional rel read from the wrong end inverts its meaning —
|
|
11601
|
+
* `caused_by` written into the cause instead of the effect says the opposite of what the
|
|
11602
|
+
* model answered — so the direction decides the file and the href together, from one
|
|
11603
|
+
* statement, and cannot disagree with itself.
|
|
11604
|
+
*/
|
|
11605
|
+
const [subject, object] = verdict.direction === "src_to_dst" ? [candidate.pair.src, candidate.pair.dst] : [candidate.pair.dst, candidate.pair.src];
|
|
11606
|
+
if (yield* stampFile(env, subject, [link(verdict.rel, hrefFor(object)), meta("memhtml-updated", env.at)])) typed += 1;
|
|
11607
|
+
}
|
|
11608
|
+
}
|
|
11609
|
+
const counts = {
|
|
11610
|
+
candidates: candidates.length,
|
|
11611
|
+
judged,
|
|
11612
|
+
typed,
|
|
11613
|
+
contradictions,
|
|
11614
|
+
promoted,
|
|
11615
|
+
skipped,
|
|
11616
|
+
capped,
|
|
11617
|
+
duplicates
|
|
11618
|
+
};
|
|
11619
|
+
if (promoted === 0 && typed === 0) return {
|
|
11620
|
+
counts,
|
|
11621
|
+
commitSha: null,
|
|
11622
|
+
llmCalls
|
|
11623
|
+
};
|
|
11624
|
+
return {
|
|
11625
|
+
counts,
|
|
11626
|
+
commitSha: yield* commitPhase(env, "edge-typing", `promote ${typed} typed edges and ${promoted} corroborated contradictions`, counts),
|
|
11627
|
+
llmCalls
|
|
11628
|
+
};
|
|
11629
|
+
});
|
|
11630
|
+
|
|
11631
|
+
//#endregion
|
|
11632
|
+
//#region packages/sleep/dist/phases/entity-resolution.js
|
|
11633
|
+
/**
|
|
11634
|
+
* Phase 3, entity resolution. Cluster one entity type's names into subjects, then rewrite each alias
|
|
11635
|
+
* onto its canonical. ONE commit rewriting `memhtml-entity` values in place.
|
|
11636
|
+
*
|
|
11637
|
+
* Three stages, and the separation is what makes the phase safe:
|
|
11638
|
+
*
|
|
11639
|
+
* 1. **Pre (deterministic, cheap).** Normalize every name, exact-merge the ones that normalize
|
|
11640
|
+
* together, auto-merge pairs at or above {@link AUTO_MERGE_THRESHOLD} character overlap, and merge
|
|
11641
|
+
* every pair a person file DECLARES ({@link aliasPairs}). This pass alone is the whole phase when no
|
|
11642
|
+
* model is bound, so a credential-free run still collapses `Checkout API` onto `checkout api` and
|
|
11643
|
+
* still applies a seeded declaration. The same pass computes one MEMORY CENTROID per name, in
|
|
11644
|
+
* O(files) and never per pair.
|
|
11645
|
+
* 2. **Core (one model call per entity type, sharded at {@link ENTITY_BATCH_SIZE}).** The model sees
|
|
11646
|
+
* every name of one type as a numbered member list and returns a PARTITION into subjects. Never one
|
|
11647
|
+
* call per pair: 59 entities on the measured corpus is one call, and the pair space is 1,711.
|
|
11648
|
+
* 3. **Post (deterministic, the one-way-door guards).** Which name survives a merge is decided by
|
|
11649
|
+
* {@link unionPairs}'s weight-then-lexicographic rule and never by the model. All THREE pair sources
|
|
11650
|
+
* — the character pass, the declarations, and the model — feed that ONE union-find, so no two of them
|
|
11651
|
+
* can disagree about a canonical. A merge backed by a DECLARED alias applies at once and is never
|
|
11652
|
+
* counted; a merge the model alone proposes is counted in `state.entity_corroboration` and applies
|
|
11653
|
+
* only once {@link ENTITY_PROMOTION_DETECTIONS} different nights have reached it.
|
|
11654
|
+
*
|
|
11655
|
+
* **Why the model, and why centroids as its evidence.** Character overlap is measurably wrong on the
|
|
11656
|
+
* case this phase exists for. Measured on the live corpus: `laith` against `laith al-saadoon` scores
|
|
11657
|
+
* 0.476 and `sanju` against `sanju kumar` 0.625 — below even the 0.75 review band — so a short name and
|
|
11658
|
+
* its full form are structurally invisible to a character ratio, and the phase minted two person files
|
|
11659
|
+
* for one person. The signal that does separate them is not the name string but WHAT IS WRITTEN under
|
|
11660
|
+
* each name: the centroid of the vectors of every memory claiming a name. Two spellings of one person
|
|
11661
|
+
* have near-identical centroids; `checkout-api` and `payments-api` do not, however close their strings
|
|
11662
|
+
* or their domain.
|
|
11663
|
+
*
|
|
11664
|
+
* The centroid is EVIDENCE HANDED TO THE MODEL, not a threshold. A cosine floor over centroids would
|
|
11665
|
+
* make exactly the mistake a bare character ratio avoids, because two services in one domain are
|
|
11666
|
+
* written about in the same terms. What the deterministic code keeps is the part a threshold is good
|
|
11667
|
+
* at: the confidence floor, the corroboration count, and the choice of which name survives.
|
|
11668
|
+
*
|
|
11669
|
+
* **Every band that does not merge is COUNTED, not merged.** The 0.75-0.85 character band the model did
|
|
11670
|
+
* not cluster, and a cluster below {@link ENTITY_CONFIDENCE_FLOOR}, both land in `reviewCandidates`. An
|
|
11671
|
+
* entity merge is a one-way door on stored identity: no later commit separates two subjects whose
|
|
11672
|
+
* memories were fused, and the failure mode of an over-eager gate is silent and permanent.
|
|
11673
|
+
*/
|
|
11674
|
+
/** At or above this ratio two names are the same entity. Auto-merged with no model call. */
|
|
10349
11675
|
const AUTO_MERGE_THRESHOLD = .85;
|
|
10350
11676
|
/** At or above this ratio, below the auto threshold: counted for review, left unmerged. */
|
|
10351
11677
|
const REVIEW_THRESHOLD = .75;
|
|
11678
|
+
/**
|
|
11679
|
+
* Confidence a model-proposed cluster must clear before it is even counted toward a merge.
|
|
11680
|
+
*
|
|
11681
|
+
* The same floor {@link STANCE_CONFIDENCE_FLOOR} sets for a contradiction, for the same reason: a false
|
|
11682
|
+
* merge is worse than a missed one, and this floor and the corroboration gate are two independent
|
|
11683
|
+
* guards on one door. A cluster below it is reported as a review candidate and nothing else.
|
|
11684
|
+
*/
|
|
11685
|
+
const ENTITY_CONFIDENCE_FLOOR = .7;
|
|
11686
|
+
/** Nights a model-only merge must be proposed on before it is written into the files. */
|
|
11687
|
+
const ENTITY_PROMOTION_DETECTIONS = 2;
|
|
11688
|
+
/**
|
|
11689
|
+
* Names offered per model call. One type's whole name list fits one call at the measured corpus size
|
|
11690
|
+
* (59 entities); this is the shard boundary for a corpus that outgrows that.
|
|
11691
|
+
*/
|
|
11692
|
+
const ENTITY_BATCH_SIZE = 500;
|
|
11693
|
+
/** Memory titles shown per name. Enough to say what a name is about, few enough to stay cheap. */
|
|
11694
|
+
const ENTITY_SAMPLE_TITLES = 3;
|
|
11695
|
+
/** Centroid neighbors shown per name, nearest first. */
|
|
11696
|
+
const ENTITY_NEIGHBORS = 3;
|
|
11697
|
+
/** Characters of each member's evidence block shown. A name plus three titles fits comfortably. */
|
|
11698
|
+
const ENTITY_MEMBER_CHARS = 600;
|
|
11699
|
+
/**
|
|
11700
|
+
* The one entity type the alias oracle speaks for, derived from the prefix rather than retyped.
|
|
11701
|
+
*
|
|
11702
|
+
* A declaration lives in a person file, and `resources/people/` is the only directory the format gives
|
|
11703
|
+
* a hand-edited identity surface. A service has no equivalent file to declare from, so offering an
|
|
11704
|
+
* `aliases` line for one would show the model a field that is always empty.
|
|
11705
|
+
*/
|
|
11706
|
+
const PERSON_TYPE = PERSON_ENTITY_PREFIX.slice(0, -1);
|
|
10352
11707
|
/** Lowercase, NFC-normalize, collapse internal whitespace, trim. The pre-compare form. */
|
|
10353
11708
|
const normalizeEntityName = (name) => name.normalize("NFC").toLowerCase().replace(/\s+/g, " ").trim();
|
|
10354
11709
|
/**
|
|
@@ -10358,6 +11713,9 @@ const normalizeEntityName = (name) => name.normalize("NFC").toLowerCase().replac
|
|
|
10358
11713
|
* separator or casing change actually is. `checkout-api` against `checkout api` differs in one
|
|
10359
11714
|
* character and scores 0.92, while `checkout-api` against `payments-api` shares only the suffix and
|
|
10360
11715
|
* scores 0.67. That sits below both thresholds, so two distinct services stay separate.
|
|
11716
|
+
*
|
|
11717
|
+
* It is the pre-pass and not the decision core. Its blind spot is short-name-against-full-name, which
|
|
11718
|
+
* is what the model call exists for.
|
|
10361
11719
|
*/
|
|
10362
11720
|
const nameSimilarity = (left, right) => {
|
|
10363
11721
|
if (left === right) return 1;
|
|
@@ -10375,14 +11733,24 @@ const nameSimilarity = (left, right) => {
|
|
|
10375
11733
|
}
|
|
10376
11734
|
return 2 * (previous[columns - 1] ?? 0) / (left.length + right.length);
|
|
10377
11735
|
};
|
|
11736
|
+
/** A pair as a stable key, so a set of pairs is order-independent. */
|
|
11737
|
+
const pairKey = (left, right) => left < right ? `${left}\u0000${right}` : `${right}\u0000${left}`;
|
|
10378
11738
|
/**
|
|
10379
|
-
* Union-find over
|
|
10380
|
-
* lexicographically smaller name
|
|
11739
|
+
* Union-find over an explicit pair list. The higher-count name wins the root; a tie goes to the
|
|
11740
|
+
* lexicographically smaller name.
|
|
11741
|
+
*
|
|
11742
|
+
* **This is the one place a canonical name is chosen, and every merge routes through it.** The
|
|
11743
|
+
* character pass, the alias oracle, and the model all contribute PAIRS to one call, so `A~B` from the
|
|
11744
|
+
* character ratio and `B~C` from the model land in one cluster with one root. Two separate union-finds
|
|
11745
|
+
* would let the two passes disagree about which name survives, and the rewrite would then depend on
|
|
11746
|
+
* which pass ran first.
|
|
11747
|
+
*
|
|
11748
|
+
* `names` is walked in sorted order and the pairs in the order given, so the partition is a function of
|
|
11749
|
+
* the input alone and a corpus that did not change resolves the same way twice.
|
|
10381
11750
|
*/
|
|
10382
|
-
const
|
|
11751
|
+
const unionPairs = (counts, pairs) => {
|
|
10383
11752
|
const names = [...counts.keys()].sort();
|
|
10384
11753
|
const parent = /* @__PURE__ */ new Map();
|
|
10385
|
-
let reviewCandidates = 0;
|
|
10386
11754
|
const find = (name) => {
|
|
10387
11755
|
let current = name;
|
|
10388
11756
|
while ((parent.get(current) ?? current) !== current) {
|
|
@@ -10392,33 +11760,262 @@ const resolveClusters = (counts) => {
|
|
|
10392
11760
|
}
|
|
10393
11761
|
return current;
|
|
10394
11762
|
};
|
|
10395
|
-
const
|
|
11763
|
+
for (const [left, right] of pairs) {
|
|
10396
11764
|
const rootLeft = find(left);
|
|
10397
11765
|
const rootRight = find(right);
|
|
10398
|
-
if (rootLeft === rootRight)
|
|
11766
|
+
if (rootLeft === rootRight) continue;
|
|
10399
11767
|
const weightLeft = counts.get(rootLeft) ?? 0;
|
|
10400
11768
|
const weightRight = counts.get(rootRight) ?? 0;
|
|
10401
11769
|
if (weightLeft > weightRight || weightLeft === weightRight && rootLeft < rootRight) parent.set(rootRight, rootLeft);
|
|
10402
11770
|
else parent.set(rootLeft, rootRight);
|
|
10403
|
-
};
|
|
10404
|
-
for (let outer = 0; outer < names.length; outer += 1) for (let inner = outer + 1; inner < names.length; inner += 1) {
|
|
10405
|
-
const left = names[outer];
|
|
10406
|
-
const right = names[inner];
|
|
10407
|
-
if (left === void 0 || right === void 0) continue;
|
|
10408
|
-
const similarity = nameSimilarity(left, right);
|
|
10409
|
-
if (similarity >= .85) union(left, right);
|
|
10410
|
-
else if (similarity >= .75) reviewCandidates += 1;
|
|
10411
11771
|
}
|
|
10412
11772
|
const aliasToCanonical = /* @__PURE__ */ new Map();
|
|
10413
11773
|
for (const name of names) {
|
|
10414
11774
|
const root = find(name);
|
|
10415
11775
|
if (root !== name) aliasToCanonical.set(name, root);
|
|
10416
11776
|
}
|
|
11777
|
+
return aliasToCanonical;
|
|
11778
|
+
};
|
|
11779
|
+
/**
|
|
11780
|
+
* Every pair of one type's names, split at the two thresholds. Names are walked in sorted order, so
|
|
11781
|
+
* the pair list is a function of the name set alone.
|
|
11782
|
+
*/
|
|
11783
|
+
const characterPairs = (names) => {
|
|
11784
|
+
const sorted = [...names].sort();
|
|
11785
|
+
const auto = [];
|
|
11786
|
+
const review = [];
|
|
11787
|
+
for (let outer = 0; outer < sorted.length; outer += 1) for (let inner = outer + 1; inner < sorted.length; inner += 1) {
|
|
11788
|
+
const left = sorted[outer];
|
|
11789
|
+
const right = sorted[inner];
|
|
11790
|
+
if (left === void 0 || right === void 0) continue;
|
|
11791
|
+
const similarity = nameSimilarity(left, right);
|
|
11792
|
+
if (similarity >= .85) auto.push([left, right]);
|
|
11793
|
+
else if (similarity >= .75) review.push([left, right]);
|
|
11794
|
+
}
|
|
10417
11795
|
return {
|
|
10418
|
-
|
|
10419
|
-
|
|
11796
|
+
auto,
|
|
11797
|
+
review
|
|
10420
11798
|
};
|
|
10421
11799
|
};
|
|
11800
|
+
/**
|
|
11801
|
+
* One memory centroid per normalized name, per entity type, in ONE pass over the claims.
|
|
11802
|
+
*
|
|
11803
|
+
* **Members are summed in SORTED PATH order, and that is a determinism requirement rather than tidiness.**
|
|
11804
|
+
* Floating-point addition is not associative — `1 + 1e-16 + 1e-16` is `1` and `1e-16 + 1e-16 + 1` is
|
|
11805
|
+
* `1.0000000000000002` (probed on node 24.19.0, in float64) — so a centroid summed in a different order
|
|
11806
|
+
* is different bytes, and different bytes reorder the nearest-neighbor list the model is shown. Two
|
|
11807
|
+
* nights over an unchanged corpus have to produce the same prompt, so the order is fixed here rather
|
|
11808
|
+
* than inherited from whatever order the rows arrived in.
|
|
11809
|
+
*
|
|
11810
|
+
* **A path claiming one name twice contributes its vector ONCE.** A file may carry both
|
|
11811
|
+
* `Service:Checkout-API` and `service:checkout-api`, two `file_entities` rows that normalize together,
|
|
11812
|
+
* and summing that memory twice would let one file's authoring quirk double its own weight in the
|
|
11813
|
+
* centroid.
|
|
11814
|
+
*
|
|
11815
|
+
* Accumulated in float64 over float32 inputs, because the sum of n unit vectors is not a unit vector
|
|
11816
|
+
* and float32 would round each partial sum. Cost is O(files), never O(names²).
|
|
11817
|
+
*/
|
|
11818
|
+
const entityCentroids = (claims, vectorForPath, options) => {
|
|
11819
|
+
const sampleTitles = options?.sampleTitles ?? 3;
|
|
11820
|
+
/** `type` -> normalized name -> its distinct claiming paths, and each path's title. */
|
|
11821
|
+
const byType = /* @__PURE__ */ new Map();
|
|
11822
|
+
for (const claim of claims) {
|
|
11823
|
+
const name = normalizeEntityName(claim.entity_name);
|
|
11824
|
+
if (name === "") continue;
|
|
11825
|
+
let names = byType.get(claim.entity_type);
|
|
11826
|
+
if (names === void 0) {
|
|
11827
|
+
names = /* @__PURE__ */ new Map();
|
|
11828
|
+
byType.set(claim.entity_type, names);
|
|
11829
|
+
}
|
|
11830
|
+
let paths = names.get(name);
|
|
11831
|
+
if (paths === void 0) {
|
|
11832
|
+
paths = /* @__PURE__ */ new Map();
|
|
11833
|
+
names.set(name, paths);
|
|
11834
|
+
}
|
|
11835
|
+
paths.set(claim.path, claim.title);
|
|
11836
|
+
}
|
|
11837
|
+
const out = /* @__PURE__ */ new Map();
|
|
11838
|
+
for (const [entityType, names] of byType) {
|
|
11839
|
+
const centroids = [];
|
|
11840
|
+
for (const name of [...names.keys()].sort()) {
|
|
11841
|
+
const paths = names.get(name);
|
|
11842
|
+
if (paths === void 0) continue;
|
|
11843
|
+
const sorted = [...paths.keys()].sort();
|
|
11844
|
+
centroids.push({
|
|
11845
|
+
name,
|
|
11846
|
+
memories: sorted.length,
|
|
11847
|
+
titles: sorted.slice(0, sampleTitles).flatMap((path) => {
|
|
11848
|
+
const title = paths.get(path);
|
|
11849
|
+
return title === void 0 || title.trim() === "" ? [] : [title.trim()];
|
|
11850
|
+
}),
|
|
11851
|
+
vec: meanVector(sorted.flatMap((path) => vectorForPath.get(path) ?? []))
|
|
11852
|
+
});
|
|
11853
|
+
}
|
|
11854
|
+
out.set(entityType, centroids);
|
|
11855
|
+
}
|
|
11856
|
+
return out;
|
|
11857
|
+
};
|
|
11858
|
+
/**
|
|
11859
|
+
* The L2-normalized mean of vectors summed in the order given, or absent for an empty or zero set.
|
|
11860
|
+
*
|
|
11861
|
+
* Normalized so a cosine between two centroids does not depend on how many memories each was built
|
|
11862
|
+
* from, and so a one-memory name and a fifty-memory name are comparable at all.
|
|
11863
|
+
*/
|
|
11864
|
+
const meanVector = (vectors) => {
|
|
11865
|
+
const first = vectors[0];
|
|
11866
|
+
if (first === void 0) return void 0;
|
|
11867
|
+
const sum = new Float64Array(first.length);
|
|
11868
|
+
for (const vector of vectors) {
|
|
11869
|
+
const width = Math.min(sum.length, vector.length);
|
|
11870
|
+
for (let at = 0; at < width; at += 1) sum[at] = sum[at] + vector[at];
|
|
11871
|
+
}
|
|
11872
|
+
let norm = 0;
|
|
11873
|
+
for (const component of sum) norm += component * component;
|
|
11874
|
+
if (norm === 0) return void 0;
|
|
11875
|
+
const scale = 1 / Math.sqrt(norm);
|
|
11876
|
+
for (let at = 0; at < sum.length; at += 1) sum[at] = sum[at] * scale;
|
|
11877
|
+
return sum;
|
|
11878
|
+
};
|
|
11879
|
+
/**
|
|
11880
|
+
* The `k` nearest same-type centroids to `of`, ordered `sim` DESC then `name` ASC.
|
|
11881
|
+
*
|
|
11882
|
+
* The tie-break matches the pair kernel's (`sim` DESC, then the other key ASC), so two names whose
|
|
11883
|
+
* centroids are equidistant are listed in one fixed order and the prompt's bytes do not depend on the
|
|
11884
|
+
* input order. A name with no centroid has no neighbors, and a candidate with no centroid is not one.
|
|
11885
|
+
*
|
|
11886
|
+
* `cosine` from the domain rather than a dot product over the already-normalized vectors, so this
|
|
11887
|
+
* similarity is the same arithmetic every other reader of this vector space performs. Cost is O(n²) per
|
|
11888
|
+
* type, bounded by {@link ENTITY_BATCH_SIZE} being the point at which a type is sharded for the CALL —
|
|
11889
|
+
* at the measured 59 entities the pair space is 1,711 dot products, which is the work the phase used to
|
|
11890
|
+
* do with character ratios.
|
|
11891
|
+
*/
|
|
11892
|
+
const nearestCentroids = (centroids, of, k) => {
|
|
11893
|
+
const subjectVec = centroids.find((candidate) => candidate.name === of)?.vec;
|
|
11894
|
+
if (subjectVec === void 0) return [];
|
|
11895
|
+
const scored = [];
|
|
11896
|
+
for (const candidate of centroids) {
|
|
11897
|
+
if (candidate.name === of || candidate.vec === void 0) continue;
|
|
11898
|
+
scored.push({
|
|
11899
|
+
name: candidate.name,
|
|
11900
|
+
sim: cosine(subjectVec, candidate.vec)
|
|
11901
|
+
});
|
|
11902
|
+
}
|
|
11903
|
+
scored.sort((left, right) => left.sim !== right.sim ? left.sim < right.sim ? 1 : -1 : left.name < right.name ? -1 : 1);
|
|
11904
|
+
return scored.slice(0, k);
|
|
11905
|
+
};
|
|
11906
|
+
/**
|
|
11907
|
+
* One name's evidence block, as the model reads it.
|
|
11908
|
+
*
|
|
11909
|
+
* Neighbors are named by NAME and not by member key. A key names a member of THIS batch, and a
|
|
11910
|
+
* centroid neighbor may sit in another shard, so offering its key would invite an answer referencing a
|
|
11911
|
+
* member the batch never contained. The similarity is rendered at two decimals so a corpus whose
|
|
11912
|
+
* vectors moved in the sixteenth place does not change the prompt's bytes.
|
|
11913
|
+
*/
|
|
11914
|
+
const entityMemberText = (input) => {
|
|
11915
|
+
const lines = [`name: ${input.centroid.name}`, `memories: ${input.centroid.memories}`];
|
|
11916
|
+
if (input.centroid.titles.length > 0) lines.push("titles:", ...input.centroid.titles.map((title) => `- ${title}`));
|
|
11917
|
+
if (input.neighbors.length > 0) lines.push("nearest by memory centroid:", ...input.neighbors.map((one) => `- ${one.name} (${one.sim.toFixed(2)})`));
|
|
11918
|
+
if (input.aliases.length > 0) lines.push(`declared aliases: ${input.aliases.join(", ")}`);
|
|
11919
|
+
return lines.join("\n");
|
|
11920
|
+
};
|
|
11921
|
+
/**
|
|
11922
|
+
* Decompose one cluster of member names into oriented merges: the highest-count name survives, ties
|
|
11923
|
+
* broken lexicographically, and every other member rewrites onto it.
|
|
11924
|
+
*
|
|
11925
|
+
* **The model's `canonicalKey` does not decide this**, and the reason is worth stating. The canonical
|
|
11926
|
+
* name is what every `memhtml-entity` meta in the corpus is rewritten TO, and it becomes a person file's
|
|
11927
|
+
* path once person-links runs. Letting the model choose it would make a nightly job's write target a
|
|
11928
|
+
* model's answer. What `canonicalKey` is for is validation: a cluster whose canonical is not one of its
|
|
11929
|
+
* own members is a self-contradicting answer, and the caller drops it.
|
|
11930
|
+
*
|
|
11931
|
+
* A cluster of fewer than two names produces no merges, which is how a model refuses.
|
|
11932
|
+
*/
|
|
11933
|
+
const decomposeCluster = (members, counts) => {
|
|
11934
|
+
const distinct = [...new Set(members)].sort();
|
|
11935
|
+
if (distinct.length < 2) return [];
|
|
11936
|
+
let canonical = distinct[0];
|
|
11937
|
+
for (const name of distinct.slice(1)) {
|
|
11938
|
+
const held = counts.get(canonical) ?? 0;
|
|
11939
|
+
const weight = counts.get(name) ?? 0;
|
|
11940
|
+
if (weight > held || weight === held && name < canonical) canonical = name;
|
|
11941
|
+
}
|
|
11942
|
+
return distinct.flatMap((name) => name === canonical ? [] : [{
|
|
11943
|
+
alias: name,
|
|
11944
|
+
canonical
|
|
11945
|
+
}]);
|
|
11946
|
+
};
|
|
11947
|
+
/** True when some declaration names both. The alias oracle's whole question. */
|
|
11948
|
+
const aliasBacked = (groups, left, right) => groups.some((group) => group.has(left) && group.has(right));
|
|
11949
|
+
/**
|
|
11950
|
+
* Every pair of one type's names that a DECLARATION backs: the alias oracle as a pair source of its
|
|
11951
|
+
* own, answerable with no model and no corroboration.
|
|
11952
|
+
*
|
|
11953
|
+
* **This is what makes the oracle an oracle.** Issue #43 states that entity resolution consults
|
|
11954
|
+
* declared aliases FIRST and that an alias-backed merge auto-commits regardless of string distance. A
|
|
11955
|
+
* declaration read only where the model core reads it would deliver neither half: a credential-free
|
|
11956
|
+
* night would leave `laith` and `laith al-saadoon` split with a person file sitting in the corpus
|
|
11957
|
+
* saying they are one person, and even a night WITH credentials would apply the declaration only if
|
|
11958
|
+
* the model happened to propose that pair — so the operator surface the format invites someone to
|
|
11959
|
+
* hand-edit would work or not work depending on a model's attention.
|
|
11960
|
+
*
|
|
11961
|
+
* A pair the character pass already merges is left out, because counting it as an alias merge as well
|
|
11962
|
+
* would report one merge twice. Names are walked in sorted order, so the pair list is a function of the
|
|
11963
|
+
* name set and the declarations alone.
|
|
11964
|
+
*/
|
|
11965
|
+
const aliasPairs = (groups, counts) => {
|
|
11966
|
+
const names = [...counts.keys()].sort();
|
|
11967
|
+
const out = [];
|
|
11968
|
+
for (let outer = 0; outer < names.length; outer += 1) for (let inner = outer + 1; inner < names.length; inner += 1) {
|
|
11969
|
+
const left = names[outer];
|
|
11970
|
+
const right = names[inner];
|
|
11971
|
+
if (left === void 0 || right === void 0) continue;
|
|
11972
|
+
if (!aliasBacked(groups, left, right)) continue;
|
|
11973
|
+
if (nameSimilarity(left, right) >= .85) continue;
|
|
11974
|
+
out.push([left, right]);
|
|
11975
|
+
}
|
|
11976
|
+
return out;
|
|
11977
|
+
};
|
|
11978
|
+
/**
|
|
11979
|
+
* Read the alias declarations out of the person files.
|
|
11980
|
+
*
|
|
11981
|
+
* Parsed with the production parser rather than scanned for meta lines, because `memhtml-alias` is
|
|
11982
|
+
* repeatable and the surgical `readMeta` reads only the first of a name. A file that does not parse is
|
|
11983
|
+
* skipped: it is not indexed either, so it has no entities for a merge to be about.
|
|
11984
|
+
*
|
|
11985
|
+
* **Read from the FILES at phase time, and deliberately not projected to SQL.** The whole point of the
|
|
11986
|
+
* oracle is that a person file is hand-editable and operator-seedable — someone with an authoritative
|
|
11987
|
+
* directory writes the aliases in and the phase converges to auto-merge. A projection would put the
|
|
11988
|
+
* declaration behind an index refresh, so an alias written and committed during a session would not be
|
|
11989
|
+
* evidence until the next rebuild, and the one surface an operator is invited to edit would be the one
|
|
11990
|
+
* with a stale read. There are as many person files as there are people, and the phase reads each once.
|
|
11991
|
+
*
|
|
11992
|
+
* Read on EVERY run, including a credential-free one and a dry run, because the declarations are a
|
|
11993
|
+
* deterministic pair source rather than the model core's evidence. Reading them is pure, so a dry run
|
|
11994
|
+
* can count what they would merge without writing anything.
|
|
11995
|
+
*/
|
|
11996
|
+
const readAliasGroups = (env) => Effect.gen(function* () {
|
|
11997
|
+
const paths = yield* peoplePaths(env.deps.db).pipe(Effect.orElseSucceed(() => []));
|
|
11998
|
+
const groups = [];
|
|
11999
|
+
for (const row of paths) {
|
|
12000
|
+
const html = yield* readFileBytes(env, row.path).pipe(Effect.orElseSucceed(() => void 0));
|
|
12001
|
+
if (html === void 0) continue;
|
|
12002
|
+
const doc = yield* parseMemory(html).pipe(Effect.orElseSucceed(() => void 0));
|
|
12003
|
+
if (doc === void 0) continue;
|
|
12004
|
+
const group = /* @__PURE__ */ new Set();
|
|
12005
|
+
for (const entity of doc.entities) {
|
|
12006
|
+
if (!entity.startsWith(PERSON_ENTITY_PREFIX)) continue;
|
|
12007
|
+
const name = normalizeEntityName(entity.slice(PERSON_ENTITY_PREFIX.length));
|
|
12008
|
+
if (name !== "") group.add(name);
|
|
12009
|
+
}
|
|
12010
|
+
if (group.size === 0) continue;
|
|
12011
|
+
for (const alias of doc.aliases) {
|
|
12012
|
+
const name = normalizeEntityName(alias);
|
|
12013
|
+
if (name !== "") group.add(name);
|
|
12014
|
+
}
|
|
12015
|
+
if (group.size > 1) groups.push(group);
|
|
12016
|
+
}
|
|
12017
|
+
return groups;
|
|
12018
|
+
});
|
|
10422
12019
|
/** The `type:name` form a `memhtml-entity` meta carries. */
|
|
10423
12020
|
const entityRef = (entityType, entityName) => `${entityType}:${entityName}`;
|
|
10424
12021
|
const entityResolution = (env) => Effect.gen(function* () {
|
|
@@ -10433,7 +12030,7 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
10433
12030
|
if (bucket === void 0) byType.set(entity.entity_type, [entity]);
|
|
10434
12031
|
else bucket.push(entity);
|
|
10435
12032
|
}
|
|
10436
|
-
/** `path -> [(oldRef, newRef)]`, accumulated across
|
|
12033
|
+
/** `path -> [(oldRef, newRef)]`, accumulated across every pass. */
|
|
10437
12034
|
const rewrites = /* @__PURE__ */ new Map();
|
|
10438
12035
|
const addRewrite = (path, from, to) => {
|
|
10439
12036
|
if (from === to) return;
|
|
@@ -10443,7 +12040,44 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
10443
12040
|
};
|
|
10444
12041
|
let normalized = 0;
|
|
10445
12042
|
let fuzzyMerges = 0;
|
|
12043
|
+
let llmMerges = 0;
|
|
12044
|
+
let aliasMerges = 0;
|
|
12045
|
+
let pendingCorroboration = 0;
|
|
10446
12046
|
let reviewCandidates = 0;
|
|
12047
|
+
let llmCalls = 0;
|
|
12048
|
+
/**
|
|
12049
|
+
* The model core is skipped entirely on a dry run and when no model is bound, and both leave the
|
|
12050
|
+
* deterministic passes running. A dry run must make no model call and bump no counter, because a
|
|
12051
|
+
* counter bumped by a run that wrote nothing would be a night of corroboration the corpus never
|
|
12052
|
+
* saw. An absent model is a credential-free run, not a broken one.
|
|
12053
|
+
*
|
|
12054
|
+
* **`dedup-merge`'s dry run makes the opposite choice and DOES spend its calls**, because there the
|
|
12055
|
+
* model's partition is the number an operator is asking for and a call costs nothing but money. Here
|
|
12056
|
+
* an honest preview would have to bump the corroboration counter — the merge count for night two
|
|
12057
|
+
* depends on it — and this phase's writes are identity rewrites, which is the one-way door where
|
|
12058
|
+
* manufacturing a night of evidence is worse than declining to preview.
|
|
12059
|
+
*/
|
|
12060
|
+
const model = env.dryRun ? void 0 : env.deps.model;
|
|
12061
|
+
const modelKey = modelFor(env.deps, "entity-resolution");
|
|
12062
|
+
/**
|
|
12063
|
+
* The declarations, read UNCONDITIONALLY — before the model core, and whether or not one exists.
|
|
12064
|
+
*
|
|
12065
|
+
* The oracle is a deterministic pair source like the character pass, not evidence the model core
|
|
12066
|
+
* owns. Reading it here is what makes a person file an operator surface: seed one, and the merge it
|
|
12067
|
+
* declares lands on the next night with no credentials, no cosine, and no second night. Gathering it
|
|
12068
|
+
* under `model !== undefined` made the declaration effective only where a model had already proposed
|
|
12069
|
+
* the same pair, which is the narrower behavior issue #43 names as the defect.
|
|
12070
|
+
*
|
|
12071
|
+
* It is also the model core's evidence, unchanged: `aliasesFor` reads these same groups to render
|
|
12072
|
+
* the `declared aliases` line, so the model sees what the code already decided rather than being
|
|
12073
|
+
* asked about it.
|
|
12074
|
+
*
|
|
12075
|
+
* A read, never a write, so a DRY RUN performs it too. Its merges are counted like every other dry
|
|
12076
|
+
* run count and nothing is written, which is what an operator sizing a night needs.
|
|
12077
|
+
*/
|
|
12078
|
+
const aliasGroups = yield* readAliasGroups(env);
|
|
12079
|
+
/** The centroids the model core needs, gathered once for every type rather than per type. */
|
|
12080
|
+
const centroidsByType = model === void 0 ? void 0 : yield* Effect.all([entityClaims(env.deps.db), entityVectors(env.deps.db)]).pipe(Effect.map(([claims, vectors]) => entityCentroids(claims, new Map(vectors.map((entry) => [entry.key, entry.vec])), { sampleTitles: 3 })));
|
|
10447
12081
|
for (const [entityType, bucket] of [...byType.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
|
|
10448
12082
|
/** Pass one: normalization, folding counts of names that normalize together. */
|
|
10449
12083
|
const counts = /* @__PURE__ */ new Map();
|
|
@@ -10454,12 +12088,124 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
10454
12088
|
counts.set(canonical, (counts.get(canonical) ?? 0) + entity.files);
|
|
10455
12089
|
if (canonical !== entity.entity_name) normalized += 1;
|
|
10456
12090
|
}
|
|
10457
|
-
/** Pass two: the
|
|
10458
|
-
const
|
|
10459
|
-
|
|
12091
|
+
/** Pass two: the character pass. Its auto pairs merge; its band pairs await a later stage. */
|
|
12092
|
+
const character = characterPairs([...counts.keys()]);
|
|
12093
|
+
const accepted = [...character.auto];
|
|
12094
|
+
/**
|
|
12095
|
+
* Pass two-and-a-half: the DECLARED aliases, accepted straight into the union.
|
|
12096
|
+
*
|
|
12097
|
+
* Only for {@link PERSON_TYPE}, because that is the only type the format gives a declaration
|
|
12098
|
+
* surface — `resources/people/` — so an alias group can only ever be about a person, and running
|
|
12099
|
+
* this for `service` would compare names against groups that cannot hold them.
|
|
12100
|
+
*
|
|
12101
|
+
* These merges are recorded in `aliasMerges` and never in `entity_corroboration`. A declaration is
|
|
12102
|
+
* a human's assertion of identity rather than a machine's suspicion, so a second night would add no
|
|
12103
|
+
* evidence, and a counter row for it would tell a reader of that table there is a decision still
|
|
12104
|
+
* waiting.
|
|
12105
|
+
*/
|
|
12106
|
+
const declared = entityType === PERSON_TYPE ? aliasPairs(aliasGroups, counts) : [];
|
|
12107
|
+
accepted.push(...declared);
|
|
12108
|
+
aliasMerges += declared.length;
|
|
12109
|
+
/** So the model core does not count a declared pair a second time. */
|
|
12110
|
+
const declaredKeys = new Set(declared.map(([left, right]) => pairKey(left, right)));
|
|
12111
|
+
/**
|
|
12112
|
+
* Pass three: the model core. One call per shard of one type, then a deterministic decision per
|
|
12113
|
+
* proposed merge. Every merge the model contributes is either alias-backed and immediate, or
|
|
12114
|
+
* corroborated across nights, or counted for review — the model never writes.
|
|
12115
|
+
*/
|
|
12116
|
+
const clusteredPairs = /* @__PURE__ */ new Set();
|
|
12117
|
+
if (model !== void 0 && centroidsByType !== void 0) {
|
|
12118
|
+
const centroids = centroidsByType.get(entityType) ?? [];
|
|
12119
|
+
const members = centroids.filter((centroid) => counts.has(centroid.name));
|
|
12120
|
+
const aliasesFor = (name) => entityType === PERSON_TYPE ? [...new Set(aliasGroups.filter((group) => group.has(name)).flatMap((group) => [...group].filter((other) => other !== name)))].sort() : [];
|
|
12121
|
+
for (const shard of assembleBatches([members], {
|
|
12122
|
+
maxMembers: 500,
|
|
12123
|
+
minMembers: 2
|
|
12124
|
+
})) {
|
|
12125
|
+
const keyed = keyMembers(shard, (centroid) => entityMemberText({
|
|
12126
|
+
centroid,
|
|
12127
|
+
neighbors: nearestCentroids(centroids, centroid.name, 3),
|
|
12128
|
+
aliases: aliasesFor(centroid.name)
|
|
12129
|
+
}), { charBudget: 600 });
|
|
12130
|
+
llmCalls += 1;
|
|
12131
|
+
const clustering = yield* batchCall(model, `entity-resolution ${entityType} batch of ${shard.length}`, {
|
|
12132
|
+
schema: EntityClustering,
|
|
12133
|
+
system: ENTITY_CLUSTER_SYSTEM,
|
|
12134
|
+
prompt: entityClusterPrompt(keyed.keyed),
|
|
12135
|
+
modelKey,
|
|
12136
|
+
effort: "medium",
|
|
12137
|
+
toolDescription: "Emit one cluster per subject, naming the members that are the same subject."
|
|
12138
|
+
});
|
|
12139
|
+
if (clustering === void 0) continue;
|
|
12140
|
+
for (const cluster of clustering.clusters) {
|
|
12141
|
+
/**
|
|
12142
|
+
* A key the batch never offered resolves to nothing, so an invented member cannot become a
|
|
12143
|
+
* rewrite. The canonical must be one of the cluster's own members: a cluster whose stated
|
|
12144
|
+
* canonical is outside it contradicts itself, and guessing which half was meant would be
|
|
12145
|
+
* the caller inventing a merge.
|
|
12146
|
+
*/
|
|
12147
|
+
const memberNames = resolveKeys(keyed, cluster.memberKeys).map((centroid) => centroid.name);
|
|
12148
|
+
const [canonicalMember] = resolveKeys(keyed, [cluster.canonicalKey]);
|
|
12149
|
+
if (canonicalMember === void 0 || !memberNames.includes(canonicalMember.name)) continue;
|
|
12150
|
+
for (const merge of decomposeCluster(memberNames, counts)) {
|
|
12151
|
+
const key = pairKey(merge.alias, merge.canonical);
|
|
12152
|
+
clusteredPairs.add(key);
|
|
12153
|
+
if (nameSimilarity(merge.alias, merge.canonical) >= .85) continue;
|
|
12154
|
+
/**
|
|
12155
|
+
* Already accepted by the declaration pass above, so the merge is happening and only the
|
|
12156
|
+
* counting is at stake: adding it again would report one merge as two, and corroborating
|
|
12157
|
+
* it would count a night of evidence toward a decision already made. The model agreeing
|
|
12158
|
+
* with a declaration is not new information — the declaration is the stronger evidence.
|
|
12159
|
+
*/
|
|
12160
|
+
if (declaredKeys.has(key)) continue;
|
|
12161
|
+
/**
|
|
12162
|
+
* A declaration the pass above could not have seen: the model named a pair whose two names
|
|
12163
|
+
* are in one alias group, but at least one of them is not in `counts` for this type — a
|
|
12164
|
+
* name the batch offered whose entity rows this type's bucket does not hold. Rare, and the
|
|
12165
|
+
* rule is the same one, so it is applied here rather than left to the corroboration path.
|
|
12166
|
+
*/
|
|
12167
|
+
if (aliasBacked(aliasGroups, merge.alias, merge.canonical)) {
|
|
12168
|
+
accepted.push([merge.alias, merge.canonical]);
|
|
12169
|
+
aliasMerges += 1;
|
|
12170
|
+
continue;
|
|
12171
|
+
}
|
|
12172
|
+
if (cluster.confidence < .7) {
|
|
12173
|
+
reviewCandidates += 1;
|
|
12174
|
+
continue;
|
|
12175
|
+
}
|
|
12176
|
+
const row = (yield* bumpEntityCorroboration(env.deps.db, {
|
|
12177
|
+
entityType,
|
|
12178
|
+
aliasName: merge.alias,
|
|
12179
|
+
canonicalName: merge.canonical,
|
|
12180
|
+
at: env.at
|
|
12181
|
+
}))[0];
|
|
12182
|
+
if (row === void 0 || row.detections < 2) {
|
|
12183
|
+
pendingCorroboration += 1;
|
|
12184
|
+
continue;
|
|
12185
|
+
}
|
|
12186
|
+
accepted.push([merge.alias, merge.canonical]);
|
|
12187
|
+
llmMerges += 1;
|
|
12188
|
+
if (row.promoted === 0) yield* markEntityPromoted(env.deps.db, {
|
|
12189
|
+
entityType,
|
|
12190
|
+
aliasName: merge.alias,
|
|
12191
|
+
canonicalName: merge.canonical,
|
|
12192
|
+
at: env.at
|
|
12193
|
+
});
|
|
12194
|
+
}
|
|
12195
|
+
}
|
|
12196
|
+
}
|
|
12197
|
+
}
|
|
12198
|
+
/**
|
|
12199
|
+
* A band pair the model went on to cluster is no longer awaiting a human: it was decided, and
|
|
12200
|
+
* the decision is recorded either as a merge or as a below-floor review candidate already counted
|
|
12201
|
+
* above. Counting it here as well would report one pair twice.
|
|
12202
|
+
*/
|
|
12203
|
+
reviewCandidates += character.review.filter(([left, right]) => !clusteredPairs.has(pairKey(left, right))).length;
|
|
12204
|
+
/** One union-find over every accepted pair, so the three sources cannot disagree on a root. */
|
|
12205
|
+
const aliasToCanonical = unionPairs(counts, accepted);
|
|
10460
12206
|
for (const entity of bucket) {
|
|
10461
12207
|
const afterNormalize = normalizedOf.get(entity.entity_name) ?? entity.entity_name;
|
|
10462
|
-
const afterMerge =
|
|
12208
|
+
const afterMerge = aliasToCanonical.get(afterNormalize) ?? afterNormalize;
|
|
10463
12209
|
if (afterMerge === entity.entity_name) continue;
|
|
10464
12210
|
if (afterMerge !== afterNormalize) fuzzyMerges += 1;
|
|
10465
12211
|
const paths = yield* pathsForEntity(env.deps.db, entityType, entity.entity_name);
|
|
@@ -10470,10 +12216,16 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
10470
12216
|
entities: entities.length,
|
|
10471
12217
|
namesNormalized: normalized,
|
|
10472
12218
|
fuzzyMerges,
|
|
12219
|
+
llmMerges,
|
|
12220
|
+
aliasMerges,
|
|
12221
|
+
pendingCorroboration,
|
|
10473
12222
|
reviewCandidates,
|
|
10474
12223
|
filesRewritten: rewrites.size
|
|
10475
12224
|
};
|
|
10476
|
-
if (rewrites.size === 0 || env.dryRun) return
|
|
12225
|
+
if (rewrites.size === 0 || env.dryRun) return {
|
|
12226
|
+
...emptyOutcome(counts),
|
|
12227
|
+
llmCalls
|
|
12228
|
+
};
|
|
10477
12229
|
let rewritten = 0;
|
|
10478
12230
|
for (const [path, pairs] of [...rewrites.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
|
|
10479
12231
|
const html = yield* readFileBytes(env, path);
|
|
@@ -10493,7 +12245,7 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
10493
12245
|
return {
|
|
10494
12246
|
counts: final,
|
|
10495
12247
|
commitSha: yield* commitPhase(env, "entity-resolution", `normalize ${normalized} entity names, merge ${fuzzyMerges} aliases`, final),
|
|
10496
|
-
llmCalls
|
|
12248
|
+
llmCalls
|
|
10497
12249
|
};
|
|
10498
12250
|
});
|
|
10499
12251
|
|
|
@@ -11547,8 +13299,8 @@ const traceConsolidation = (env) => Effect.gen(function* () {
|
|
|
11547
13299
|
* change.
|
|
11548
13300
|
*
|
|
11549
13301
|
* Nor does the conflict become an authored `<link>`, and the reason is mechanical. Any authored
|
|
11550
|
-
* edge between two paths permanently closes that pair to
|
|
11551
|
-
* the anti-join in `
|
|
13302
|
+
* edge between two paths permanently closes that pair to edge typing's scans: `derived = 0` is
|
|
13303
|
+
* the anti-join in BOTH `sharedEntityPairs` and `minedPairs`, so stamping one here would silence
|
|
11552
13304
|
* the very disagreement this lookup surfaced. The conflict lives in the counts, in the
|
|
11553
13305
|
* `Memhtml-Counts` trailer, and in the commit message's context, where a reviewer sees it at merge
|
|
11554
13306
|
* review and decides.
|
|
@@ -11804,7 +13556,7 @@ const PHASE_BODIES = {
|
|
|
11804
13556
|
"entity-resolution": entityResolution,
|
|
11805
13557
|
"person-links": personLinks,
|
|
11806
13558
|
"relationship-mining": relationshipMining,
|
|
11807
|
-
"
|
|
13559
|
+
"edge-typing": edgeTyping,
|
|
11808
13560
|
"confidence-decay": confidenceDecay,
|
|
11809
13561
|
"arc-synthesis": arcSynthesis,
|
|
11810
13562
|
"retention-triage": retentionTriage,
|
|
@@ -12968,4 +14720,4 @@ const latest = (left, right) => left === null ? right : right === null ? left :
|
|
|
12968
14720
|
|
|
12969
14721
|
//#endregion
|
|
12970
14722
|
export { STATE_SIDECAR_PATH as $, makeIndexRecorder as A, makeGitPort as B, ModelClientLive as C, EMBED_DIM as D, EmbeddingsLive as E, reinforce as F, STATE_MIGRATIONS_DIR as G, DatabaseService as H, Indexer as I, expandRoot as J, STATE_SCHEMA as K, makeIndexer as L, readWatermark as M, Retrieval as N, EMBED_WATERMARK as O, makeRetrieval as P, STATE_DB_PATH as Q, readIndexState as R, ModelClient as S, Embeddings as T, makeDatabase as U, sanitizeFtsQuery as V, MIGRATIONS_DIR as W, INDEX_DB_PATH as X, makeStore as Y, SLEEP_REPORTS_DIR as Z, unlink as _, parseSidecar as a, commitSubject as at, discriminationGate as b, generateArtifacts as c, isValidDatetime as ct, danglingEdges as d, REINFORCE_SIGNALS as dt, attemptIo as et, publishRows as f, frameKeyOf as ft, meta as g, link as h, makeSleep as i, makeGit as it, persistScanned as j, IndexRecorder as k, accessRows as l, closesFence as lt, hrefFor as m, scanTraceRoot as n, readFileOrNull as nt, renderSidecar as o, checkMemory as ot, applyHeadEdits as p, Store as q, Sleep as r, Git as rt, archivedFormOf as s, setMeta as st, mergeTailExtract as t, initRepo as tt, allPaths as u, fenceOpeningOf as ut, SLEEP_PHASES as v, wrapAsData as w, runDiscrimination as x, isSleepPhase as y, IndexGit as z };
|
|
12971
|
-
//# sourceMappingURL=dist-
|
|
14723
|
+
//# sourceMappingURL=dist-B3yDga97.mjs.map
|