memhtml 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{dist-D5DlgqH2.mjs → dist-CHoz5uHd.mjs} +895 -187
- package/dist/dist-CHoz5uHd.mjs.map +1 -0
- package/dist/memhtml-mcp.mjs +13 -2
- package/dist/memhtml-mcp.mjs.map +1 -1
- package/dist/memhtml.mjs +17 -3
- package/dist/memhtml.mjs.map +1 -1
- package/package.json +1 -1
- package/dist/dist-D5DlgqH2.mjs.map +0 -1
|
@@ -9021,17 +9021,42 @@ const keyMembers = (items, textOf, options) => {
|
|
|
9021
9021
|
};
|
|
9022
9022
|
};
|
|
9023
9023
|
/**
|
|
9024
|
+
* The offered key a model's answer denotes, or `undefined` when it denotes none.
|
|
9025
|
+
*
|
|
9026
|
+
* A model shown `<member_m3>` answers `member_m3` at least as readily as `m3`: the wrapper tag is
|
|
9027
|
+
* the only place most batch prompts DISPLAY a key, so the label-prefixed form is the one the prompt
|
|
9028
|
+
* itself teaches (measured live 2026-08-23: `gpt-5.6-sol` answers the prefixed form on every call,
|
|
9029
|
+
* Claude Sonnet 5 on most). So a key that does not match directly is retried once with everything
|
|
9030
|
+
* up to its last `_` stripped, and only a suffix the batch actually offered resolves — `member_m9`
|
|
9031
|
+
* in a batch of three still denotes nothing, and a path or an invented name still drops. The
|
|
9032
|
+
* canonical form is returned so two spellings of one member collapse to one key everywhere a phase
|
|
9033
|
+
* keeps per-key state.
|
|
9034
|
+
*/
|
|
9035
|
+
const offeredKeyFor = (batch, key) => {
|
|
9036
|
+
if (batch.itemForKey.has(key)) return key;
|
|
9037
|
+
const at = key.lastIndexOf("_");
|
|
9038
|
+
if (at === -1) return void 0;
|
|
9039
|
+
const suffix = key.slice(at + 1);
|
|
9040
|
+
return batch.itemForKey.has(suffix) ? suffix : void 0;
|
|
9041
|
+
};
|
|
9042
|
+
/**
|
|
9024
9043
|
* Resolve the keys a model named back to items: unknown keys are dropped, repeats collapse.
|
|
9025
9044
|
*
|
|
9026
9045
|
* A key the batch never offered is a member the model invented, and every phase on this kernel turns
|
|
9027
9046
|
* a named member into a write, so an unresolvable key must not reach that write. Dropping it leaves
|
|
9028
9047
|
* the corresponding file untouched, which is the safe outcome for every one of the five phases.
|
|
9048
|
+
* Resolution goes through {@link offeredKeyFor}, so the label-prefixed spelling of an offered key
|
|
9049
|
+
* (`member_m3` for `m3`) resolves rather than reading as an invention.
|
|
9029
9050
|
*
|
|
9030
9051
|
* The result keeps the order the model named the keys in, and a key named twice appears once.
|
|
9031
|
-
* De-duplication is on the
|
|
9032
|
-
*
|
|
9033
|
-
|
|
9034
|
-
|
|
9052
|
+
* De-duplication is on the CANONICAL key rather than on the spelling or the resolved item, so `m1`
|
|
9053
|
+
* and `member_m1` in one answer count as one member, and the count a phase gates on ("at least two
|
|
9054
|
+
* members absorbed") counts distinct offered members.
|
|
9055
|
+
*/
|
|
9056
|
+
const resolveKeys = (batch, keys) => [...new Set(keys.flatMap((key) => {
|
|
9057
|
+
const canonical = offeredKeyFor(batch, key);
|
|
9058
|
+
return canonical === void 0 ? [] : [canonical];
|
|
9059
|
+
}))].flatMap((key) => {
|
|
9035
9060
|
const item = batch.itemForKey.get(key);
|
|
9036
9061
|
return item === void 0 ? [] : [item];
|
|
9037
9062
|
});
|
|
@@ -9156,21 +9181,31 @@ const batchCall = (model, label, request) => isolate(label, model.generateObject
|
|
|
9156
9181
|
* `sleep_phases` row, and in a `--phases` flag, and three copies of the string would drift.
|
|
9157
9182
|
*/
|
|
9158
9183
|
/**
|
|
9159
|
-
* The
|
|
9184
|
+
* The seventeen phases, in execution order.
|
|
9160
9185
|
*
|
|
9161
9186
|
* The order encodes the predecessor memory system's dependencies (design §6): entity resolution precedes person
|
|
9162
9187
|
* links so aliases have already merged, confidence decay precedes retention triage so triage
|
|
9163
9188
|
* scores the decayed value, and dedup-merge precedes compress and retention because both operate
|
|
9164
9189
|
* on the post-merge set.
|
|
9165
9190
|
*
|
|
9166
|
-
* `task-detection`
|
|
9167
|
-
*
|
|
9191
|
+
* `task-detection` sits after `trace-consolidation` and before `integrity`, and both edges are
|
|
9192
|
+
* deliberate. It scans the ACTIVE corpus for unresolved
|
|
9168
9193
|
* commitments, so it has to run after every phase that changes what is active — after dedup's folds,
|
|
9169
9194
|
* after retention's evictions, after compress's canonicals, and after trace consolidation's newly
|
|
9170
9195
|
* distilled memories, which are the freshest text of the night and the likeliest to carry one. And it
|
|
9171
9196
|
* WRITES files, so it must precede `integrity`, which repairs dangling hrefs and regenerates the
|
|
9172
9197
|
* directory artifacts: a task minted afterwards would be absent from its directory's `index.html`
|
|
9173
9198
|
* until the next night.
|
|
9199
|
+
*
|
|
9200
|
+
* `placement-triage` is deep-only (issue #63): on a run without `--deep` it returns immediately,
|
|
9201
|
+
* writes nothing, and commits nothing, so the nightly cycle's behavior is unchanged by its presence
|
|
9202
|
+
* in this list. Its slot has the same two edges task-detection's does, plus one more on each side:
|
|
9203
|
+
* it must run after `compress` (it re-files only what even deep grouping could not fold, so folding
|
|
9204
|
+
* has to have had its chance first) and after `task-detection` (a move mid-scan would hand the
|
|
9205
|
+
* detector paths that no longer hold files), and it must precede `integrity` because it MOVES files —
|
|
9206
|
+
* integrity regenerates each directory's `index.html`, and it rewrites inbound hrefs itself because
|
|
9207
|
+
* integrity's dangling-href repair only knows how to chase a target into the ARCHIVE, not into a
|
|
9208
|
+
* topic directory.
|
|
9174
9209
|
*/
|
|
9175
9210
|
const SLEEP_PHASES = [
|
|
9176
9211
|
"preflight",
|
|
@@ -9186,6 +9221,7 @@ const SLEEP_PHASES = [
|
|
|
9186
9221
|
"reprieve",
|
|
9187
9222
|
"trace-consolidation",
|
|
9188
9223
|
"task-detection",
|
|
9224
|
+
"placement-triage",
|
|
9189
9225
|
"integrity",
|
|
9190
9226
|
"state-export",
|
|
9191
9227
|
"report"
|
|
@@ -9552,7 +9588,8 @@ const DEFAULT_MODELS = {
|
|
|
9552
9588
|
"arc-synthesis": "gpt-5.6-sol",
|
|
9553
9589
|
compress: "gpt-5.6-sol",
|
|
9554
9590
|
"trace-consolidation": "opus-5",
|
|
9555
|
-
"task-detection": "gpt-5.6-sol"
|
|
9591
|
+
"task-detection": "gpt-5.6-sol",
|
|
9592
|
+
"placement-triage": "gpt-5.6-sol"
|
|
9556
9593
|
};
|
|
9557
9594
|
/** The model a phase calls: the caller's override, else {@link DEFAULT_MODELS}, else sonnet. */
|
|
9558
9595
|
const modelFor = (deps, phase) => deps.models?.[phase] ?? DEFAULT_MODELS[phase] ?? "sonnet-5";
|
|
@@ -9562,6 +9599,25 @@ const emptyOutcome = (counts = {}) => ({
|
|
|
9562
9599
|
commitSha: null,
|
|
9563
9600
|
llmCalls: 0
|
|
9564
9601
|
});
|
|
9602
|
+
/** A fresh budget. A non-positive or fractional cap clamps to a usable whole number. */
|
|
9603
|
+
const makeLlmBudget = (maxCalls) => ({
|
|
9604
|
+
maxCalls: Math.max(0, Math.trunc(maxCalls)),
|
|
9605
|
+
spent: 0
|
|
9606
|
+
});
|
|
9607
|
+
/**
|
|
9608
|
+
* Take one call from the budget, or report exhaustion.
|
|
9609
|
+
*
|
|
9610
|
+
* `true` means the caller may make the call and the budget has been charged. Charging BEFORE the
|
|
9611
|
+
* call rather than after means a crash mid-call cannot under-count, which errs on the side the
|
|
9612
|
+
* budget exists for. No budget bound means every call is allowed.
|
|
9613
|
+
*/
|
|
9614
|
+
const takeLlmCall = (deep) => {
|
|
9615
|
+
const budget = deep?.budget;
|
|
9616
|
+
if (budget === void 0) return true;
|
|
9617
|
+
if (budget.spent >= budget.maxCalls) return false;
|
|
9618
|
+
budget.spent += 1;
|
|
9619
|
+
return true;
|
|
9620
|
+
};
|
|
9565
9621
|
|
|
9566
9622
|
//#endregion
|
|
9567
9623
|
//#region packages/sleep/dist/llm.js
|
|
@@ -10029,6 +10085,71 @@ const edgeTypingPrompt = (pairs) => batchPrompt(pairs, EDGE_TYPING_INSTRUCTION,
|
|
|
10029
10085
|
const arcTriagePrompt = (arcsText, evidenceText) => `${dataBlock("current_arcs", arcsText)}\n\n${dataBlock("evidence", evidenceText)}\n\nProduce a triage plan. Assign update or skip to every existing arc, and add a create entry for any genuinely new behavioral pattern the existing arcs do not cover.`;
|
|
10030
10086
|
/** The arc-execute user turn for one arc. `current` is absent on a create. */
|
|
10031
10087
|
const arcExecutePrompt = (input) => (input.current === void 0 ? `${dataBlock("new_arc_title", input.title)}\n\n` : `${dataBlock("existing_arc", input.current)}\n\n`) + `${dataBlock("evidence", input.evidenceText)}\n\n${dataBlock("triage_rationale", input.rationale)}\n\n` + (input.current === void 0 ? "Synthesize a new behavioral principal from this evidence." : "Update the arc to incorporate the new evidence, preserving existing knowledge that holds.");
|
|
10088
|
+
/**
|
|
10089
|
+
* One inbox memory's proposed destination, under the opaque key it was offered as (issue #63).
|
|
10090
|
+
*
|
|
10091
|
+
* `destination` is a STRING the phase validates, never trusts: only `keep-inbox`, a directory the
|
|
10092
|
+
* corpus already has, or one of a capped handful of new topic directories survives the code gate.
|
|
10093
|
+
* A model-invented path outside those is a refused placement, counted and logged, because a
|
|
10094
|
+
* destination is a write target and the model does not choose write targets anywhere in this
|
|
10095
|
+
* pipeline.
|
|
10096
|
+
*/
|
|
10097
|
+
const Placement = Schema.Struct({
|
|
10098
|
+
/** The offered key, e.g. `m3`. A key the batch never held resolves to nothing and is dropped. */
|
|
10099
|
+
memberKey: Schema.String,
|
|
10100
|
+
/**
|
|
10101
|
+
* `keep-inbox`, or a bucket-rooted directory like `areas/deploys` or `resources/sqlite`.
|
|
10102
|
+
* `keep-inbox` is the refusal and the ordinary answer for a memory whose topic is not clear.
|
|
10103
|
+
*/
|
|
10104
|
+
destination: Schema.String,
|
|
10105
|
+
/** Unitless in `[0, 1]`. The move gate is deterministic and reads this, not the prose. */
|
|
10106
|
+
confidence: Schema.Finite.check(Schema.isBetween({
|
|
10107
|
+
minimum: 0,
|
|
10108
|
+
maximum: 1
|
|
10109
|
+
}))
|
|
10110
|
+
});
|
|
10111
|
+
/** One batch's whole answer: a placement list. An omitted member keeps its inbox path. */
|
|
10112
|
+
const PlacementTriage = Schema.Struct({ placements: Schema.Array(Placement) });
|
|
10113
|
+
/**
|
|
10114
|
+
* The confidence a placement must clear before the phase moves a file. The same 0.7 the edge and
|
|
10115
|
+
* entity gates use, for the same reason one number serves them: a second knob would be a threshold
|
|
10116
|
+
* nobody could state the meaning of. A move is cheap to reverse (`git mv` back, reviewable on the
|
|
10117
|
+
* branch), so the floor matches the mild gates rather than the corroboration-backed ones.
|
|
10118
|
+
*/
|
|
10119
|
+
const PLACEMENT_CONFIDENCE_FLOOR = .7;
|
|
10120
|
+
/** The literal a placement answer uses to decline. */
|
|
10121
|
+
const PLACEMENT_KEEP = "keep-inbox";
|
|
10122
|
+
/**
|
|
10123
|
+
* The placement-triage system prompt (issue #63).
|
|
10124
|
+
*
|
|
10125
|
+
* The refusal is stated as the ordinary answer, because for a bulk-imported inbox it is: most
|
|
10126
|
+
* singletons are distinct facts whose topic directory does not exist yet, and a model that felt
|
|
10127
|
+
* obliged to place everything would scatter the inbox across invented directories nobody asked for.
|
|
10128
|
+
*/
|
|
10129
|
+
const PLACEMENT_SYSTEM = `You file inbox memories into topic directories for an AI agent's long-term memory system. You are
|
|
10130
|
+
given the list of directories the corpus already has, and a NUMBERED LIST of memories currently
|
|
10131
|
+
sitting in the inbox. For each memory, propose where it belongs.
|
|
10132
|
+
|
|
10133
|
+
- destination is one of the existing directories, verbatim from the list, when the memory clearly
|
|
10134
|
+
belongs to that topic.
|
|
10135
|
+
- destination may be a NEW directory (areas/<topic> for ongoing concerns, resources/<topic> for
|
|
10136
|
+
reference material) when several of THESE memories share a topic no existing directory covers.
|
|
10137
|
+
Use a short lowercase hyphenated topic name. New directories are rationed, so propose one only
|
|
10138
|
+
when it would hold more than a stray file.
|
|
10139
|
+
- destination is keep-inbox when you are unsure, when the memory's topic is ambiguous, or when no
|
|
10140
|
+
directory fits. This is the ordinary answer, not a failure.
|
|
10141
|
+
- Never propose archive/, areas/inbox itself, areas/arcs, or resources/people. Those are managed
|
|
10142
|
+
surfaces.
|
|
10143
|
+
- Rate confidence honestly. A placement above the floor moves the file on a review branch.
|
|
10144
|
+
- Omitting a member means keep-inbox.`;
|
|
10145
|
+
/** The instruction that closes a placement batch's user turn, after the member list. */
|
|
10146
|
+
const PLACEMENT_INSTRUCTION = "Propose a destination for each memory above: an existing directory from the list, a new areas/<topic> or resources/<topic> directory, or keep-inbox. Name each memory by its offered key and rate your confidence.";
|
|
10147
|
+
/**
|
|
10148
|
+
* The placement user turn for one batch: the existing directories, then every member under its
|
|
10149
|
+
* offered key, then the instruction. The directory list is corpus-derived text, so it is wrapped as
|
|
10150
|
+
* data exactly as the member texts are.
|
|
10151
|
+
*/
|
|
10152
|
+
const placementPrompt = (directories, members) => `${dataBlock("existing_directories", directories.join("\n"))}\n\n` + batchPrompt(members, PLACEMENT_INSTRUCTION, { label: "memory" });
|
|
10032
10153
|
/** The instruction that closes a compress batch's user turn, after the member list. */
|
|
10033
10154
|
const COMPRESS_INSTRUCTION = "Fold these memories into one canonical memory. List in absorbedKeys exactly the members whose content the canonical carries forward.";
|
|
10034
10155
|
/**
|
|
@@ -10081,7 +10202,7 @@ const dedupPrompt = (components) => {
|
|
|
10081
10202
|
/**
|
|
10082
10203
|
* The memory type no phase of a sleep cycle touches.
|
|
10083
10204
|
*
|
|
10084
|
-
* A task is live working state, and every one of the
|
|
10205
|
+
* A task is live working state, and every one of the seventeen phases is a judgment about REMEMBERED
|
|
10085
10206
|
* FACTS: decay says a claim is fading, dedup says two claims are one, edge typing says one claim
|
|
10086
10207
|
* caused or contradicts another, retention says a claim has stopped earning its place. None of those hold for
|
|
10087
10208
|
* a thing an agent intends to do, and each would be wrong applied to one. A task the agent has
|
|
@@ -10403,6 +10524,16 @@ const peoplePaths = (db) => db.all("SELECT path FROM files WHERE path LIKE ? OR
|
|
|
10403
10524
|
* `edge_class = 'memory'` is the firewall. A person or provenance edge cannot enter PageRank, label
|
|
10404
10525
|
* propagation, or the retention bridge count, and this query is what makes that true. The CHECK
|
|
10405
10526
|
* constraint alone does not.
|
|
10527
|
+
*
|
|
10528
|
+
* **The deep grouping band is excluded, and the exclusion is what keeps nightly scoring stable
|
|
10529
|
+
* across deep runs (issue #63).** Deep mining writes machine-mined `laterally_related` edges at a
|
|
10530
|
+
* floor (0.72) far below the nightly one, purely so label propagation can partition the inbox tail
|
|
10531
|
+
* for compress. Those edges persist in the index after the deep run, so without this predicate the
|
|
10532
|
+
* FIRST deep run would permanently change every subsequent nightly run's PageRank, communities, and
|
|
10533
|
+
* bridge counts — and therefore its eviction decisions — on a corpus whose files did not change.
|
|
10534
|
+
* The filter names `derived = 1` as well as the rel, so an AUTHORED `laterally_related` (an agent's
|
|
10535
|
+
* own assertion, addable through `memhtml link`) keeps exactly the graph standing it had.
|
|
10536
|
+
* {@link deepGroupingEdges} is the one reader of the band.
|
|
10406
10537
|
*/
|
|
10407
10538
|
const memoryEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel, e.dst_path AS dst_path,
|
|
10408
10539
|
e.strength AS strength, e.derived AS derived
|
|
@@ -10410,6 +10541,24 @@ const memoryEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel,
|
|
|
10410
10541
|
JOIN files s ON s.path = e.src_path AND s.archived = 0
|
|
10411
10542
|
JOIN files d ON d.path = e.dst_path AND d.archived = 0
|
|
10412
10543
|
WHERE e.edge_class = 'memory'
|
|
10544
|
+
AND NOT (e.derived = 1 AND e.rel = 'laterally_related')
|
|
10545
|
+
ORDER BY e.src_path ASC, e.rel ASC, e.dst_path ASC`);
|
|
10546
|
+
/**
|
|
10547
|
+
* The deep grouping band: machine-mined `laterally_related` edges over active files (issue #63).
|
|
10548
|
+
*
|
|
10549
|
+
* The mirror of {@link memoryEdges}' exclusion, and deliberately a separate read instead of a flag on
|
|
10550
|
+
* it: the band has exactly one consumer intent — widening label propagation's partition for the deep
|
|
10551
|
+
* compress and placement phases — and a parameterized `memoryEdges` would let any caller widen the
|
|
10552
|
+
* retention graph by passing a boolean. `provenance = 'sleep'` is implied by `derived = 1` (the
|
|
10553
|
+
* table CHECK pins the pair) and stated anyway so the statement reads as what it is.
|
|
10554
|
+
*/
|
|
10555
|
+
const deepGroupingEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel, e.dst_path AS dst_path,
|
|
10556
|
+
e.strength AS strength, e.derived AS derived
|
|
10557
|
+
FROM edges e
|
|
10558
|
+
JOIN files s ON s.path = e.src_path AND s.archived = 0
|
|
10559
|
+
JOIN files d ON d.path = e.dst_path AND d.archived = 0
|
|
10560
|
+
WHERE e.edge_class = 'memory' AND e.derived = 1 AND e.rel = 'laterally_related'
|
|
10561
|
+
AND e.provenance = 'sleep'
|
|
10413
10562
|
ORDER BY e.src_path ASC, e.rel ASC, e.dst_path ASC`);
|
|
10414
10563
|
const retentionEdgeCounts = (db) => db.all(`SELECT f.path AS path,
|
|
10415
10564
|
sum(CASE WHEN e.rel = 'supports' THEN 1 ELSE 0 END) AS reinforcements,
|
|
@@ -10614,6 +10763,19 @@ const markSessionsConsolidated = (db, input) => db.writeAll(input.sessionIds.map
|
|
|
10614
10763
|
const linkedSessionCount = (db) => db.get(`SELECT count(DISTINCT t.session_id) AS n FROM traces t
|
|
10615
10764
|
JOIN memory_session_links l ON l.session_id = t.session_id`).pipe(Effect.map((row) => row?.n ?? 0));
|
|
10616
10765
|
/**
|
|
10766
|
+
* Authored memory-class edges POINTING AT one path: which files hold a `<link>` a move must rewrite.
|
|
10767
|
+
*
|
|
10768
|
+
* Placement triage's read (issue #63). `derived = 0` because a mined edge lives only in the index
|
|
10769
|
+
* and the next re-mine follows the moved vectors on its own; only a file-borne link needs a splice.
|
|
10770
|
+
* Both memory and task classes are included — a task may `blocks`-link a memory it waits on, and a
|
|
10771
|
+
* move that left that href dangling would break the task surface — but provenance and person edges
|
|
10772
|
+
* cannot point at an inbox memory by construction.
|
|
10773
|
+
*/
|
|
10774
|
+
const inboundAuthoredEdges = (db, dstPath) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel
|
|
10775
|
+
FROM edges e
|
|
10776
|
+
WHERE e.derived = 0 AND e.dst_path = ?
|
|
10777
|
+
ORDER BY e.src_path ASC, e.rel ASC`, [dstPath]);
|
|
10778
|
+
/**
|
|
10617
10779
|
* Authored edges pointing at a path the index does not hold.
|
|
10618
10780
|
*
|
|
10619
10781
|
* `derived = 0` only: a mined edge lives in the index and nowhere else, so a dangling one is
|
|
@@ -10960,6 +11122,129 @@ const arcSynthesis = (env) => Effect.gen(function* () {
|
|
|
10960
11122
|
};
|
|
10961
11123
|
});
|
|
10962
11124
|
|
|
11125
|
+
//#endregion
|
|
11126
|
+
//#region packages/sleep/dist/phases/relationship-mining.js
|
|
11127
|
+
/**
|
|
11128
|
+
* Phase 5, relationship mining. Derived `relates_to` edges in the index only. NO COMMIT.
|
|
11129
|
+
*
|
|
11130
|
+
* A mined edge is a re-derivable function of the corpus and the embedder. `index rebuild` plus the
|
|
11131
|
+
* next night's mining regenerates the identical set, so committing thousands of them would bury every
|
|
11132
|
+
* real diff in machine noise for zero recoverable information. The `derived` column is the firewall
|
|
11133
|
+
* that makes losing them cheap. The retention penalty counts only `derived = 0`, so an
|
|
11134
|
+
* uncorroborated machine suspicion cannot evict a memory.
|
|
11135
|
+
*
|
|
11136
|
+
* The insert is scoped to `provenance = 'sleep'` and `derived = 1` and the whole replace is one
|
|
11137
|
+
* atomic batch, so an authored edge is unreachable from here and the corpus is never left with the
|
|
11138
|
+
* old mined set deleted and the new one not yet written. In that window the lateral arm would
|
|
11139
|
+
* silently return nothing.
|
|
11140
|
+
*
|
|
11141
|
+
* **Under `--deep` the phase additionally mines a GROUPING band (issue #63).** On a measured 3,079-file
|
|
11142
|
+
* inbox, 8% of files have a neighbor at the 0.85 floor and 84% touch no edge at all — no community, so
|
|
11143
|
+
* compress is structurally unable to reach them at any frequency. The deep band mines
|
|
11144
|
+
* [{@link DEEP_MINING_COSINE_FLOOR}, {@link MINING_COSINE_FLOOR}) as `laterally_related` edges whose
|
|
11145
|
+
* only consumer intent is giving label propagation a partition; the existing compress model then
|
|
11146
|
+
* judges the folds, and `absorbedKeys: []` is its refusal, so a looser grouping floor costs calls and
|
|
11147
|
+
* never correctness.
|
|
11148
|
+
*
|
|
11149
|
+
* **The two bands are separate rels, and that is the isolation mechanism.** {@link replaceMinedEdges}
|
|
11150
|
+
* scopes its atomic delete by rel, so the deep replace cannot clobber the nightly `relates_to` set and
|
|
11151
|
+
* the nightly replace cannot clobber the deep band. The issue sketched a distinct PROVENANCE instead;
|
|
11152
|
+
* the `edges` CHECKs pin `derived = 1` to `provenance = 'sleep'` (migration 0008), so that spelling
|
|
11153
|
+
* needs a table recreate while a distinct rel needs nothing: `laterally_related` is already in the
|
|
11154
|
+
* memory-rel vocabulary with no producer, and edge typing's candidate scan reads `rel = 'relates_to'`
|
|
11155
|
+
* alone, so the deep band never spends nightly edge-typing calls. A nightly run never touches the
|
|
11156
|
+
* deep rel, so a deep band persists until the next deep run re-mines it or `index rebuild` drops it —
|
|
11157
|
+
* both re-derivable, which is the property that makes an index-only edge safe to hold.
|
|
11158
|
+
*/
|
|
11159
|
+
/** The similarity floor a pair must clear to become a mined `relates_to`. */
|
|
11160
|
+
const MINING_COSINE_FLOOR = .85;
|
|
11161
|
+
/**
|
|
11162
|
+
* The GROUPING-band floor deep mining reaches down to (issue #63). At 0.72, between 43% and 61% of
|
|
11163
|
+
* the measured bulk-import inbox has a neighbor (43% at 0.75, 61% at 0.70, median best-neighbor
|
|
11164
|
+
* cosine 0.731) — reach enough to partition most of the tail while staying above the ~0.5-0.6
|
|
11165
|
+
* cross-topic baseline the fixture corpora measure, so a community is still a topic and not noise.
|
|
11166
|
+
* The compress model judging every proposed fold is what makes this floor a cost knob rather than a
|
|
11167
|
+
* correctness one.
|
|
11168
|
+
*/
|
|
11169
|
+
const DEEP_MINING_COSINE_FLOOR = .72;
|
|
11170
|
+
/** Nearest neighbors considered per source file. */
|
|
11171
|
+
const MINING_PER_SOURCE_K = 5;
|
|
11172
|
+
/**
|
|
11173
|
+
* Pairs mined per cycle: a cap on what {@link replaceMinedEdges} writes, not on the scan — the
|
|
11174
|
+
* kernel's arithmetic is O(n²·d) whatever this says, and it bounds the edge table so one dense
|
|
11175
|
+
* neighborhood cannot flood the graph the lateral arm and PageRank read.
|
|
11176
|
+
*/
|
|
11177
|
+
const MINING_SAMPLE_LIMIT = 2e3;
|
|
11178
|
+
/**
|
|
11179
|
+
* The deep band's own write cap. Separate from {@link MINING_SAMPLE_LIMIT} because the band is wider
|
|
11180
|
+
* by construction — it exists to reach the 84% the nightly band cannot — and sharing the nightly cap
|
|
11181
|
+
* would make the deep run's reach a function of how crowded the nightly band happens to be.
|
|
11182
|
+
*/
|
|
11183
|
+
const DEEP_MINING_SAMPLE_LIMIT = 1e4;
|
|
11184
|
+
/** The rel the deep grouping band is written under. The band separator; see the module header. */
|
|
11185
|
+
const DEEP_GROUPING_REL = "laterally_related";
|
|
11186
|
+
/**
|
|
11187
|
+
* Mine every band the run is entitled to and replace the index's mined sets: the whole phase minus
|
|
11188
|
+
* its counts. Exported because deep compress re-runs it BETWEEN passes (issue #63's
|
|
11189
|
+
* iterate-until-quiet): a fold's canonical is a new neighbor only after it is indexed and re-mined,
|
|
11190
|
+
* and a second implementation of the scan here would be free to disagree with the nightly one about
|
|
11191
|
+
* floors, caps, and exclusions.
|
|
11192
|
+
*/
|
|
11193
|
+
const mineAllBands = (env) => Effect.gen(function* () {
|
|
11194
|
+
/**
|
|
11195
|
+
* Tasks are excluded, and here the exclusion is the graph firewall, not a cost guard.
|
|
11196
|
+
* Every mined edge is written with `edge_class = 'memory'`, so a pair with a task endpoint
|
|
11197
|
+
* would put a task INTO the memory graph, reaching PageRank, MMR, and the retention bridge
|
|
11198
|
+
* count. The `edges` CHECK cannot refuse it, because `relates_to` under `memory` is a
|
|
11199
|
+
* well-formed edge whatever files sit at its ends.
|
|
11200
|
+
*/
|
|
11201
|
+
const pairs = yield* neighborPairs(env.deps.db, {
|
|
11202
|
+
floor: MINING_COSINE_FLOOR,
|
|
11203
|
+
perSourceK: 5,
|
|
11204
|
+
limit: MINING_SAMPLE_LIMIT,
|
|
11205
|
+
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
11206
|
+
});
|
|
11207
|
+
/**
|
|
11208
|
+
* The deep grouping band: everything in [deep floor, nightly floor). Mined only under `--deep`,
|
|
11209
|
+
* so a nightly run's writes — and therefore the graph every nightly consumer reads on a corpus
|
|
11210
|
+
* that has never run deep — are byte-identical to what they were before this branch existed.
|
|
11211
|
+
* The nightly floor is the band's EXCLUSIVE ceiling: a pair at or above it is the nightly
|
|
11212
|
+
* band's, and holding it in both would double its label-propagation weight.
|
|
11213
|
+
*/
|
|
11214
|
+
const deepPairs = env.deep === void 0 ? [] : (yield* neighborPairs(env.deps.db, {
|
|
11215
|
+
floor: DEEP_MINING_COSINE_FLOOR,
|
|
11216
|
+
perSourceK: 5,
|
|
11217
|
+
limit: DEEP_MINING_SAMPLE_LIMIT,
|
|
11218
|
+
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
11219
|
+
})).filter((pair) => pair.sim < MINING_COSINE_FLOOR);
|
|
11220
|
+
if (!env.dryRun) {
|
|
11221
|
+
yield* replaceMinedEdges(env.deps.db, {
|
|
11222
|
+
runId: env.runId,
|
|
11223
|
+
at: env.at,
|
|
11224
|
+
rel: "relates_to",
|
|
11225
|
+
pairs
|
|
11226
|
+
});
|
|
11227
|
+
if (env.deep !== void 0) yield* replaceMinedEdges(env.deps.db, {
|
|
11228
|
+
runId: env.runId,
|
|
11229
|
+
at: env.at,
|
|
11230
|
+
rel: DEEP_GROUPING_REL,
|
|
11231
|
+
pairs: deepPairs
|
|
11232
|
+
});
|
|
11233
|
+
}
|
|
11234
|
+
return {
|
|
11235
|
+
mined: pairs.length,
|
|
11236
|
+
deepMined: deepPairs.length
|
|
11237
|
+
};
|
|
11238
|
+
});
|
|
11239
|
+
const relationshipMining = (env) => Effect.gen(function* () {
|
|
11240
|
+
const mined = yield* mineAllBands(env);
|
|
11241
|
+
return emptyOutcome({
|
|
11242
|
+
candidates: mined.mined,
|
|
11243
|
+
mined: mined.mined,
|
|
11244
|
+
...env.deep === void 0 ? {} : { deepMined: mined.deepMined }
|
|
11245
|
+
});
|
|
11246
|
+
});
|
|
11247
|
+
|
|
10963
11248
|
//#endregion
|
|
10964
11249
|
//#region packages/sleep/dist/phases/compress.js
|
|
10965
11250
|
/**
|
|
@@ -10990,6 +11275,28 @@ const arcSynthesis = (env) => Effect.gen(function* () {
|
|
|
10990
11275
|
*
|
|
10991
11276
|
* `dedup-merge` is a HARD prerequisite. Compressing before duplicates are folded would synthesize a
|
|
10992
11277
|
* canonical over a pair the merge phase then archives one half of.
|
|
11278
|
+
*
|
|
11279
|
+
* ## Deep mode (issue #63)
|
|
11280
|
+
*
|
|
11281
|
+
* Under `--deep` three things change about WHICH memories a fold can reach, and nothing about what
|
|
11282
|
+
* happens to a fold:
|
|
11283
|
+
*
|
|
11284
|
+
* 1. **Communities come from the widened graph.** Label propagation runs over the nightly memory
|
|
11285
|
+
* edges PLUS the deep grouping band (`laterally_related`, mined at 0.72), so inbox files whose
|
|
11286
|
+
* best neighbor sits under the nightly 0.85 floor get a partition. Retention SCORES stay the
|
|
11287
|
+
* nightly function — the band decides grouping, never eviction.
|
|
11288
|
+
* 2. **Entity groups are a second community source.** Candidates the widened graph still leaves
|
|
11289
|
+
* communityless are grouped by shared `file_entities` reference under a synthetic
|
|
11290
|
+
* `entity:<type>:<name>` label, because two memories about one subject can share no vocabulary at
|
|
11291
|
+
* all. Hub entities (more than {@link DEEP_ENTITY_HUB_LIMIT} active claimants) are stop-words and
|
|
11292
|
+
* are skipped.
|
|
11293
|
+
* 3. **The phase iterates until quiet.** A pass that produced canonicals re-indexes the branch,
|
|
11294
|
+
* re-mines both bands, recomputes retention, and folds again — a canonical is a new neighbor and
|
|
11295
|
+
* a new community member — until a pass folds nothing or {@link DEEP_COMPRESS_MAX_PASSES} hits.
|
|
11296
|
+
*
|
|
11297
|
+
* Every model call in deep mode is charged against the run's shared `--max-llm-calls` budget first;
|
|
11298
|
+
* an exhausted budget skips the batch with reason `budget` and the run stays green, the same
|
|
11299
|
+
* degradation posture a model outage has.
|
|
10993
11300
|
*/
|
|
10994
11301
|
/** Members per model call. Small enough that every member's facts fit the answer's attention. */
|
|
10995
11302
|
const COMPRESS_BATCH_SIZE = 8;
|
|
@@ -11002,6 +11309,83 @@ const COMPRESS_MIN_BATCH = 2;
|
|
|
11002
11309
|
const COMPRESS_CANDIDATE_LIMIT = 2e3;
|
|
11003
11310
|
/** Characters of each member shown. A fold must see the facts, so this is wider than arc evidence. */
|
|
11004
11311
|
const COMPRESS_MEMBER_CHARS = 1200;
|
|
11312
|
+
/**
|
|
11313
|
+
* Deep compress passes before the loop stops regardless of yield (issue #63). Each extra pass costs
|
|
11314
|
+
* a full re-index, a re-mine, and another round of model calls, and the third pass's yield on any
|
|
11315
|
+
* real corpus is folds of folds — past that the loop is spending calls to rename its own output.
|
|
11316
|
+
*/
|
|
11317
|
+
const DEEP_COMPRESS_MAX_PASSES = 3;
|
|
11318
|
+
/**
|
|
11319
|
+
* Active files an entity may be claimed by before deep grouping treats it as a stop-word
|
|
11320
|
+
* (issue #63). An entity on sixty-plus files ("service:api", a person who appears everywhere) says
|
|
11321
|
+
* which corpus this is, not which memories are one topic, and batching its claimants would fold
|
|
11322
|
+
* unrelated facts for sharing a byline. Groups under the limit still slice to
|
|
11323
|
+
* {@link COMPRESS_BATCH_SIZE} through the same kernel as every other group.
|
|
11324
|
+
*/
|
|
11325
|
+
const DEEP_ENTITY_HUB_LIMIT = 64;
|
|
11326
|
+
/** The synthetic community-label prefix entity groups use. No git path starts with this. */
|
|
11327
|
+
const ENTITY_LABEL_PREFIX = "entity:";
|
|
11328
|
+
/**
|
|
11329
|
+
* The community label of every active path under the WIDENED graph: nightly memory edges plus the
|
|
11330
|
+
* deep grouping band, one label-propagation pass (issue #63).
|
|
11331
|
+
*
|
|
11332
|
+
* Computed here rather than inside `runRetentionPass`, deliberately: retention's partition feeds
|
|
11333
|
+
* eviction scoring and must not move when a deep band is present in the index, so the widened
|
|
11334
|
+
* partition is a second, compress-only computation over reads this module already owns.
|
|
11335
|
+
*/
|
|
11336
|
+
const deepCommunityLabels = (env) => Effect.gen(function* () {
|
|
11337
|
+
const corpus = yield* activeCorpus(env.deps.db);
|
|
11338
|
+
const nightly = yield* memoryEdges(env.deps.db);
|
|
11339
|
+
const band = yield* deepGroupingEdges(env.deps.db);
|
|
11340
|
+
return labelPropagation(corpus.map((row) => row.path), [...nightly, ...band].map((edge) => ({
|
|
11341
|
+
src: edge.src_path,
|
|
11342
|
+
dst: edge.dst_path,
|
|
11343
|
+
strength: edge.strength
|
|
11344
|
+
})));
|
|
11345
|
+
}).pipe(Effect.orElseSucceed(() => /* @__PURE__ */ new Map()));
|
|
11346
|
+
/**
|
|
11347
|
+
* Assign an `entity:<type>:<name>` label to every candidate the graph left communityless, greedily,
|
|
11348
|
+
* one label per file (issue #63). PURE, so the hub cap and the one-label rule are unit-assertable
|
|
11349
|
+
* without a corpus.
|
|
11350
|
+
*
|
|
11351
|
+
* Entities are walked in lexicographic (type, name) order and each claims its unclaimed candidates,
|
|
11352
|
+
* so the assignment is a pure function of the corpus: a file naming two entities lands with the
|
|
11353
|
+
* lexicographically first, and the run makes the same batches twice over. A group needs two members
|
|
11354
|
+
* to mean anything, matching {@link COMPRESS_MIN_BATCH}.
|
|
11355
|
+
*
|
|
11356
|
+
* **The hub cap reads the entity's ACTIVE claimant count, not its needing count.** An entity on a
|
|
11357
|
+
* hundred files whose tail happens to leave only three in the inbox band is still a stop-word:
|
|
11358
|
+
* what those three share is a byline, not a topic.
|
|
11359
|
+
*/
|
|
11360
|
+
const assignEntityLabels = (claims, needing) => {
|
|
11361
|
+
const byEntity = /* @__PURE__ */ new Map();
|
|
11362
|
+
for (const claim of claims) {
|
|
11363
|
+
const key = `${claim.entity_type}:${claim.entity_name}`;
|
|
11364
|
+
const bucket = byEntity.get(key);
|
|
11365
|
+
if (bucket === void 0) byEntity.set(key, [claim.path]);
|
|
11366
|
+
else bucket.push(claim.path);
|
|
11367
|
+
}
|
|
11368
|
+
const labels = /* @__PURE__ */ new Map();
|
|
11369
|
+
let hubsSkipped = 0;
|
|
11370
|
+
for (const [key, paths] of [...byEntity.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
|
|
11371
|
+
if (paths.length > 64) {
|
|
11372
|
+
hubsSkipped += 1;
|
|
11373
|
+
continue;
|
|
11374
|
+
}
|
|
11375
|
+
const members = paths.filter((path) => needing.has(path) && !labels.has(path));
|
|
11376
|
+
if (members.length < 2) continue;
|
|
11377
|
+
for (const member of members) labels.set(member, `${ENTITY_LABEL_PREFIX}${key}`);
|
|
11378
|
+
}
|
|
11379
|
+
return {
|
|
11380
|
+
labels,
|
|
11381
|
+
hubsSkipped
|
|
11382
|
+
};
|
|
11383
|
+
};
|
|
11384
|
+
/** {@link assignEntityLabels} over the index's own claim rows. */
|
|
11385
|
+
const entityLabelsFor = (env, needing) => entityClaims(env.deps.db).pipe(Effect.map((claims) => assignEntityLabels(claims, needing)), Effect.orElseSucceed(() => ({
|
|
11386
|
+
labels: /* @__PURE__ */ new Map(),
|
|
11387
|
+
hubsSkipped: 0
|
|
11388
|
+
})));
|
|
11005
11389
|
const compress = (env) => Effect.gen(function* () {
|
|
11006
11390
|
const model = env.deps.model;
|
|
11007
11391
|
if (model === void 0) return {
|
|
@@ -11012,121 +11396,187 @@ const compress = (env) => Effect.gen(function* () {
|
|
|
11012
11396
|
}),
|
|
11013
11397
|
detail: "no model bound"
|
|
11014
11398
|
};
|
|
11015
|
-
const candidates = (yield* runRetentionPass(env.deps.db, env.at)).scored.filter((entry) => entry.score.action === "compress" && entry.row.memory_type !== "arc" && !isSleepExcluded(entry.row.memory_type) && entry.community !== void 0).slice(0, COMPRESS_CANDIDATE_LIMIT);
|
|
11016
|
-
/** Community -> its COMPRESS-band members, both orders fixed so batching is reproducible. */
|
|
11017
|
-
const byCommunity = /* @__PURE__ */ new Map();
|
|
11018
|
-
for (const entry of candidates) {
|
|
11019
|
-
const label = entry.community;
|
|
11020
|
-
if (label === void 0) continue;
|
|
11021
|
-
const bucket = byCommunity.get(label);
|
|
11022
|
-
if (bucket === void 0) byCommunity.set(label, [entry]);
|
|
11023
|
-
else bucket.push(entry);
|
|
11024
|
-
}
|
|
11025
|
-
/**
|
|
11026
|
-
* Both sorts are this phase's, and the kernel keeps the order they produce. Communities are
|
|
11027
|
-
* walked lexicographically by label so a night's call order is fixed, and each community's members
|
|
11028
|
-
* by `row.path` so the `m1`..`mN` keys land on the same files twice over.
|
|
11029
|
-
*/
|
|
11030
|
-
const groups = [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1).map(([, members]) => [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0));
|
|
11031
|
-
const batches = assembleBatches(groups, {
|
|
11032
|
-
maxMembers: 8,
|
|
11033
|
-
minMembers: 2
|
|
11034
|
-
});
|
|
11035
|
-
const counts = {
|
|
11036
|
-
candidates: candidates.length,
|
|
11037
|
-
communities: byCommunity.size,
|
|
11038
|
-
batches: batches.length,
|
|
11039
|
-
canonicals: 0,
|
|
11040
|
-
archived: 0,
|
|
11041
|
-
skipped: 0
|
|
11042
|
-
};
|
|
11043
|
-
if (batches.length === 0) return emptyOutcome(counts);
|
|
11044
|
-
if (env.dryRun) return emptyOutcome(counts);
|
|
11045
11399
|
const modelKey = modelFor(env.deps, "compress");
|
|
11046
|
-
|
|
11047
|
-
let canonicals = 0;
|
|
11048
|
-
let archived = 0;
|
|
11049
|
-
let skipped = 0;
|
|
11400
|
+
const passes = [];
|
|
11050
11401
|
let lastCommit = null;
|
|
11051
|
-
for (
|
|
11052
|
-
|
|
11053
|
-
const
|
|
11054
|
-
llmCalls += 1;
|
|
11055
|
-
const synthesis = yield* batchCall(model, `compress batch of ${batch.length}`, {
|
|
11056
|
-
schema: CompressSynthesis,
|
|
11057
|
-
system: COMPRESS_SYSTEM,
|
|
11058
|
-
prompt: compressPrompt(keyed.keyed),
|
|
11059
|
-
modelKey,
|
|
11060
|
-
effort: "high",
|
|
11061
|
-
toolDescription: "Emit the canonical memory and the members whose content it absorbs."
|
|
11062
|
-
});
|
|
11063
|
-
if (synthesis === void 0) {
|
|
11064
|
-
skipped += 1;
|
|
11065
|
-
continue;
|
|
11066
|
-
}
|
|
11067
|
-
/** A key the batch never offered resolves to nothing, so a fold reaches only offered files. */
|
|
11068
|
-
const absorbed = resolveKeys(keyed, synthesis.absorbedKeys).map((entry) => entry.row.path);
|
|
11069
|
-
if (absorbed.length < 2 || synthesis.title.trim() === "" || synthesis.claim.trim() === "") {
|
|
11070
|
-
skipped += 1;
|
|
11071
|
-
continue;
|
|
11072
|
-
}
|
|
11402
|
+
for (let passAt = 0; passAt < (env.deep === void 0 ? 1 : 3);) {
|
|
11403
|
+
passAt += 1;
|
|
11404
|
+
const banded = (yield* runRetentionPass(env.deps.db, env.at)).scored.filter((entry) => entry.score.action === "compress" && entry.row.memory_type !== "arc" && !isSleepExcluded(entry.row.memory_type));
|
|
11073
11405
|
/**
|
|
11074
|
-
*
|
|
11075
|
-
*
|
|
11076
|
-
*
|
|
11406
|
+
* Which label a candidate folds under. Nightly: the retention pass's own community, and a
|
|
11407
|
+
* memory without one is not a candidate — the exact selection this phase has always made.
|
|
11408
|
+
* Deep: the widened partition first, then an entity label for what the graph still missed.
|
|
11077
11409
|
*/
|
|
11078
|
-
const
|
|
11079
|
-
const
|
|
11080
|
-
const
|
|
11081
|
-
|
|
11082
|
-
|
|
11083
|
-
|
|
11410
|
+
const widened = env.deep === void 0 ? void 0 : yield* deepCommunityLabels(env);
|
|
11411
|
+
const communityOf = (entry) => widened === void 0 ? entry.community : widened.get(entry.row.path);
|
|
11412
|
+
const graphLabelled = banded.filter((entry) => communityOf(entry) !== void 0);
|
|
11413
|
+
const needing = env.deep === void 0 ? /* @__PURE__ */ new Set() : new Set(banded.filter((entry) => communityOf(entry) === void 0).map((entry) => entry.row.path));
|
|
11414
|
+
const entities = env.deep === void 0 ? {
|
|
11415
|
+
labels: /* @__PURE__ */ new Map(),
|
|
11416
|
+
hubsSkipped: 0
|
|
11417
|
+
} : yield* entityLabelsFor(env, needing);
|
|
11418
|
+
const candidates = [...graphLabelled, ...banded.filter((entry) => entities.labels.has(entry.row.path))].slice(0, COMPRESS_CANDIDATE_LIMIT);
|
|
11419
|
+
/** Community -> its COMPRESS-band members, both orders fixed so batching is reproducible. */
|
|
11420
|
+
const byCommunity = /* @__PURE__ */ new Map();
|
|
11421
|
+
for (const entry of candidates) {
|
|
11422
|
+
const label = communityOf(entry) ?? entities.labels.get(entry.row.path);
|
|
11423
|
+
if (label === void 0) continue;
|
|
11424
|
+
const bucket = byCommunity.get(label);
|
|
11425
|
+
if (bucket === void 0) byCommunity.set(label, [entry]);
|
|
11426
|
+
else bucket.push(entry);
|
|
11084
11427
|
}
|
|
11085
11428
|
/**
|
|
11086
|
-
*
|
|
11087
|
-
*
|
|
11088
|
-
*
|
|
11429
|
+
* Both sorts are this phase's, and the kernel keeps the order they produce. Communities are
|
|
11430
|
+
* walked lexicographically by label so a night's call order is fixed, and each community's members
|
|
11431
|
+
* by `row.path` so the `m1`..`mN` keys land on the same files twice over.
|
|
11089
11432
|
*/
|
|
11090
|
-
const
|
|
11091
|
-
|
|
11092
|
-
|
|
11093
|
-
|
|
11094
|
-
}
|
|
11095
|
-
if (archivedPaths.length === 0) {
|
|
11096
|
-
skipped += 1;
|
|
11097
|
-
continue;
|
|
11098
|
-
}
|
|
11099
|
-
yield* writeFileBytes(env, canonicalPath, renderTemplate({
|
|
11100
|
-
title: synthesis.title.trim(),
|
|
11101
|
-
claim: synthesis.claim,
|
|
11102
|
-
body: synthesis.paragraphs,
|
|
11103
|
-
memoryType: "semantic",
|
|
11104
|
-
at: env.at,
|
|
11105
|
-
author: "agent:sleep"
|
|
11106
|
-
}));
|
|
11107
|
-
for (const archivedPath of archivedPaths) yield* stampFile(env, canonicalPath, [link("supersedes", hrefFor(archivedPath))]);
|
|
11108
|
-
yield* env.deps.git.add([canonicalPath]);
|
|
11109
|
-
archived += archivedPaths.length;
|
|
11110
|
-
canonicals += 1;
|
|
11111
|
-
const commitSha = yield* commitPhase(env, "compress", `fold ${members.length} memories into ${synthesis.title}`, {
|
|
11112
|
-
...counts,
|
|
11113
|
-
canonicals,
|
|
11114
|
-
archived,
|
|
11115
|
-
skipped
|
|
11433
|
+
const groups = [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1).map(([, members]) => [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0));
|
|
11434
|
+
const batches = assembleBatches(groups, {
|
|
11435
|
+
maxMembers: 8,
|
|
11436
|
+
minMembers: 2
|
|
11116
11437
|
});
|
|
11117
|
-
|
|
11438
|
+
const tally = {
|
|
11439
|
+
candidates: candidates.length,
|
|
11440
|
+
communities: byCommunity.size,
|
|
11441
|
+
entityGroups: new Set(entities.labels.values()).size,
|
|
11442
|
+
batches: batches.length,
|
|
11443
|
+
canonicals: 0,
|
|
11444
|
+
archived: 0,
|
|
11445
|
+
skipped: 0,
|
|
11446
|
+
failed: 0,
|
|
11447
|
+
refused: 0,
|
|
11448
|
+
budget: 0,
|
|
11449
|
+
llmCalls: 0
|
|
11450
|
+
};
|
|
11451
|
+
passes.push(tally);
|
|
11452
|
+
if (batches.length === 0 || env.dryRun) break;
|
|
11453
|
+
for (const batch of batches) {
|
|
11454
|
+
/** Opaque keys again, so `absorbedKeys` cannot name a path. */
|
|
11455
|
+
const keyed = keyMembers(batch, (entry) => `${entry.row.title}\n${entry.row.gist}\n${entry.row.body_text}`, { charBudget: COMPRESS_MEMBER_CHARS });
|
|
11456
|
+
/**
|
|
11457
|
+
* The deep budget is charged BEFORE the call, and an exhausted budget is its own skip
|
|
11458
|
+
* reason: a night that stopped because the operator's cap ran out and a night whose model
|
|
11459
|
+
* fell over need different mornings-after, and `skipped` alone cannot say which this was.
|
|
11460
|
+
*/
|
|
11461
|
+
if (!takeLlmCall(env.deep)) {
|
|
11462
|
+
tally.skipped += 1;
|
|
11463
|
+
tally.budget += 1;
|
|
11464
|
+
continue;
|
|
11465
|
+
}
|
|
11466
|
+
tally.llmCalls += 1;
|
|
11467
|
+
const synthesis = yield* batchCall(model, `compress batch of ${batch.length}`, {
|
|
11468
|
+
schema: CompressSynthesis,
|
|
11469
|
+
system: COMPRESS_SYSTEM,
|
|
11470
|
+
prompt: compressPrompt(keyed.keyed),
|
|
11471
|
+
modelKey,
|
|
11472
|
+
effort: "high",
|
|
11473
|
+
toolDescription: "Emit the canonical memory and the members whose content it absorbs."
|
|
11474
|
+
});
|
|
11475
|
+
if (synthesis === void 0) {
|
|
11476
|
+
tally.skipped += 1;
|
|
11477
|
+
tally.failed += 1;
|
|
11478
|
+
continue;
|
|
11479
|
+
}
|
|
11480
|
+
/** A key the batch never offered resolves to nothing, so a fold reaches only offered files. */
|
|
11481
|
+
const absorbed = resolveKeys(keyed, synthesis.absorbedKeys).map((entry) => entry.row.path);
|
|
11482
|
+
if (absorbed.length < 2 || synthesis.title.trim() === "" || synthesis.claim.trim() === "") {
|
|
11483
|
+
tally.skipped += 1;
|
|
11484
|
+
tally.refused += 1;
|
|
11485
|
+
yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: the model absorbed ${absorbed.length} of ${synthesis.absorbedKeys.length} named keys` + (synthesis.absorbedKeys.length > 0 && absorbed.length === 0 ? ` (none of the named keys resolved: ${synthesis.absorbedKeys.slice(0, 3).join(", ")}${synthesis.absorbedKeys.length > 3 ? ", …" : ""})` : ""));
|
|
11486
|
+
continue;
|
|
11487
|
+
}
|
|
11488
|
+
/**
|
|
11489
|
+
* The canonical is placed in the batch's own directory when the members agree on one, and in the
|
|
11490
|
+
* inbox otherwise. Placing it under a member's directory keeps a compressed group where a reader
|
|
11491
|
+
* would look for it, and `memhtml doctor` reports inbox depth so a disagreeing batch is visible.
|
|
11492
|
+
*/
|
|
11493
|
+
const directories = new Set(absorbed.map((path) => path.slice(0, path.lastIndexOf("/"))));
|
|
11494
|
+
const canonicalPath = `${(directories.size === 1 ? [...directories][0] : "areas/inbox") ?? "areas/inbox"}/${slugify(synthesis.title)}.html`;
|
|
11495
|
+
const members = excludeSelfSupersede(canonicalPath, absorbed);
|
|
11496
|
+
if (members.length === 0) {
|
|
11497
|
+
tally.skipped += 1;
|
|
11498
|
+
tally.refused += 1;
|
|
11499
|
+
yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every absorbed member was the canonical itself`);
|
|
11500
|
+
continue;
|
|
11501
|
+
}
|
|
11502
|
+
/**
|
|
11503
|
+
* The members are archived FIRST, and the canonical is written only if at least one member was
|
|
11504
|
+
* actually moved. A batch whose members an earlier phase already evicted would otherwise leave a
|
|
11505
|
+
* canonical behind claiming to supersede files it never absorbed.
|
|
11506
|
+
*/
|
|
11507
|
+
const archivedPaths = [];
|
|
11508
|
+
for (const member of members) {
|
|
11509
|
+
const archivedPath = yield* archiveFile(env, member, [meta("memhtml-superseded-by", hrefFor(canonicalPath))]);
|
|
11510
|
+
if (archivedPath !== null) archivedPaths.push(archivedPath);
|
|
11511
|
+
}
|
|
11512
|
+
if (archivedPaths.length === 0) {
|
|
11513
|
+
tally.skipped += 1;
|
|
11514
|
+
tally.refused += 1;
|
|
11515
|
+
yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every member was already gone from the tree`);
|
|
11516
|
+
continue;
|
|
11517
|
+
}
|
|
11518
|
+
yield* writeFileBytes(env, canonicalPath, renderTemplate({
|
|
11519
|
+
title: synthesis.title.trim(),
|
|
11520
|
+
claim: synthesis.claim,
|
|
11521
|
+
body: synthesis.paragraphs,
|
|
11522
|
+
memoryType: "semantic",
|
|
11523
|
+
at: env.at,
|
|
11524
|
+
author: "agent:sleep"
|
|
11525
|
+
}));
|
|
11526
|
+
for (const archivedPath of archivedPaths) yield* stampFile(env, canonicalPath, [link("supersedes", hrefFor(archivedPath))]);
|
|
11527
|
+
yield* env.deps.git.add([canonicalPath]);
|
|
11528
|
+
tally.archived += archivedPaths.length;
|
|
11529
|
+
tally.canonicals += 1;
|
|
11530
|
+
const commitSha = yield* commitPhase(env, "compress", `fold ${members.length} memories into ${synthesis.title}`, totalsOf(passes));
|
|
11531
|
+
if (commitSha !== null) lastCommit = commitSha;
|
|
11532
|
+
}
|
|
11533
|
+
/**
|
|
11534
|
+
* Iterate-until-quiet, deep only (issue #63). A pass that folded nothing has reached the
|
|
11535
|
+
* fixed point; one that folded something changed the neighbor structure, so the branch is
|
|
11536
|
+
* re-indexed (renames tracked, new canonicals embedded), both bands are re-mined over the new
|
|
11537
|
+
* vectors, and the next pass sees the canonicals as members. The nightly cycle exits here
|
|
11538
|
+
* unconditionally, which is what keeps its behavior single-pass and byte-identical.
|
|
11539
|
+
*/
|
|
11540
|
+
if (env.deep === void 0 || tally.canonicals === 0) break;
|
|
11541
|
+
if (passAt >= 3) break;
|
|
11542
|
+
yield* env.deps.indexer.update({ embed: true });
|
|
11543
|
+
yield* mineAllBands(env);
|
|
11118
11544
|
}
|
|
11119
11545
|
return {
|
|
11120
|
-
counts:
|
|
11121
|
-
...counts,
|
|
11122
|
-
canonicals,
|
|
11123
|
-
archived,
|
|
11124
|
-
skipped
|
|
11125
|
-
},
|
|
11546
|
+
counts: totalsOf(passes),
|
|
11126
11547
|
commitSha: lastCommit,
|
|
11127
|
-
llmCalls
|
|
11548
|
+
llmCalls: passes.reduce((total, tally) => total + tally.llmCalls, 0)
|
|
11128
11549
|
};
|
|
11129
11550
|
});
|
|
11551
|
+
/**
|
|
11552
|
+
* The pass tallies as one counts record. Work counters SUM across passes; `candidates` and
|
|
11553
|
+
* `communities` describe the corpus the run STARTED from (the first pass), because a sum of
|
|
11554
|
+
* re-scans of one shrinking corpus counts nothing a reviewer can reconcile. On a nightly
|
|
11555
|
+
* single-pass run every key and every number is byte-identical to what this phase has always
|
|
11556
|
+
* reported — the deep-only keys appear only when a deep quantity is nonzero or a second pass ran,
|
|
11557
|
+
* which cannot happen without the flag. Response counts are append-only, so the deep keys are
|
|
11558
|
+
* additions and no shipped key changes meaning.
|
|
11559
|
+
*/
|
|
11560
|
+
const totalsOf = (passes) => {
|
|
11561
|
+
const first = passes[0];
|
|
11562
|
+
const sum = (of) => passes.reduce((total, tally) => total + of(tally), 0);
|
|
11563
|
+
const deepish = passes.length > 1 || (first?.entityGroups ?? 0) > 0 || sum((tally) => tally.budget) > 0;
|
|
11564
|
+
return {
|
|
11565
|
+
candidates: first?.candidates ?? 0,
|
|
11566
|
+
communities: first?.communities ?? 0,
|
|
11567
|
+
batches: sum((tally) => tally.batches),
|
|
11568
|
+
canonicals: sum((tally) => tally.canonicals),
|
|
11569
|
+
archived: sum((tally) => tally.archived),
|
|
11570
|
+
skipped: sum((tally) => tally.skipped),
|
|
11571
|
+
failed: sum((tally) => tally.failed),
|
|
11572
|
+
refused: sum((tally) => tally.refused),
|
|
11573
|
+
...deepish ? {
|
|
11574
|
+
passes: passes.length,
|
|
11575
|
+
entityGroups: first?.entityGroups ?? 0,
|
|
11576
|
+
budgetSkipped: sum((tally) => tally.budget)
|
|
11577
|
+
} : {}
|
|
11578
|
+
};
|
|
11579
|
+
};
|
|
11130
11580
|
|
|
11131
11581
|
//#endregion
|
|
11132
11582
|
//#region packages/sleep/dist/phases/confidence-decay.js
|
|
@@ -12032,7 +12482,9 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
12032
12482
|
candidates: oriented.length,
|
|
12033
12483
|
components: 0,
|
|
12034
12484
|
llmGroups: 0,
|
|
12035
|
-
vetoed: oriented.length - decisions.length
|
|
12485
|
+
vetoed: oriented.length - decisions.length,
|
|
12486
|
+
skipped: 0,
|
|
12487
|
+
unresolved: 0
|
|
12036
12488
|
},
|
|
12037
12489
|
/**
|
|
12038
12490
|
* Every mined pair on this arm cleared 0.92, so a vetoed one here is a near-certain duplicate
|
|
@@ -12103,6 +12555,12 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
12103
12555
|
let llmCalls = 0;
|
|
12104
12556
|
let llmGroups = 0;
|
|
12105
12557
|
let skipped = 0;
|
|
12558
|
+
/**
|
|
12559
|
+
* Model-named member keys that resolved to no offered member, dropped unacted. The drop is the
|
|
12560
|
+
* safe outcome and stays; the count makes a systematic naming pattern visible instead of reading
|
|
12561
|
+
* as a night in which the model proposed no merges (issue #58).
|
|
12562
|
+
*/
|
|
12563
|
+
let unresolved = 0;
|
|
12106
12564
|
/** Group-implied pairs, in batch then component then group order. */
|
|
12107
12565
|
const groupPairs = [];
|
|
12108
12566
|
/** Every path a surviving group claimed, so the mined arm cannot re-propose one. */
|
|
@@ -12142,11 +12600,17 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
12142
12600
|
skipped += 1;
|
|
12143
12601
|
continue;
|
|
12144
12602
|
}
|
|
12603
|
+
/** This batch's unresolvable keys, so the warning below can name the spellings that failed. */
|
|
12604
|
+
const unresolvedKeys = [];
|
|
12145
12605
|
for (const group of partition.groups) {
|
|
12606
|
+
const dropped = group.memberKeys.filter((key) => offeredKeyFor(keyed, key) === void 0);
|
|
12607
|
+
unresolved += dropped.length;
|
|
12608
|
+
unresolvedKeys.push(...dropped);
|
|
12146
12609
|
const members = resolveKeys(keyed, group.memberKeys);
|
|
12147
12610
|
if (members.length < 2) continue;
|
|
12148
12611
|
if (new Set(group.memberKeys.flatMap((key) => {
|
|
12149
|
-
const
|
|
12612
|
+
const canonical = offeredKeyFor(keyed, key);
|
|
12613
|
+
const id = canonical === void 0 ? void 0 : componentOfKey.get(canonical);
|
|
12150
12614
|
return id === void 0 ? [] : [id];
|
|
12151
12615
|
})).size !== 1) continue;
|
|
12152
12616
|
/** The keeper is the OLDEST member: the lowest corpus offset, the same rule a pair uses. */
|
|
@@ -12172,6 +12636,7 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
12172
12636
|
}
|
|
12173
12637
|
grouped.add(keeper.path);
|
|
12174
12638
|
}
|
|
12639
|
+
if (unresolvedKeys.length > 0) yield* Effect.logWarning(`sleep.llm dedup batch of ${batch.length} components dropped ${unresolvedKeys.length} member keys naming no offered member (${unresolvedKeys.slice(0, 3).join(", ")}${unresolvedKeys.length > 3 ? ", …" : ""})`);
|
|
12175
12640
|
}
|
|
12176
12641
|
/**
|
|
12177
12642
|
* Groups first, then the mined pairs above the DETERMINISTIC floor that no group claimed. The
|
|
@@ -12191,7 +12656,8 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
12191
12656
|
components: components.length,
|
|
12192
12657
|
llmGroups,
|
|
12193
12658
|
vetoed: proposed.length - decisions.length,
|
|
12194
|
-
skipped
|
|
12659
|
+
skipped,
|
|
12660
|
+
unresolved
|
|
12195
12661
|
},
|
|
12196
12662
|
/**
|
|
12197
12663
|
* On this arm a vetoed pair is one the MODEL grouped as the same memory, or one that cleared
|
|
@@ -12202,11 +12668,13 @@ const dedupMerge = (env) => Effect.gen(function* () {
|
|
|
12202
12668
|
*
|
|
12203
12669
|
* `judged` is false when a batch's call failed, because those components were never partitioned:
|
|
12204
12670
|
* their pairs reach the veto only through the mined arm, so a night that lost a call cannot say
|
|
12205
|
-
* whether a pair it did not see is still a candidate.
|
|
12671
|
+
* whether a pair it did not see is still a candidate. An unresolved member key is the same
|
|
12672
|
+
* hazard from the answer side — a group the phase could not fully map was not fully judged —
|
|
12673
|
+
* so it holds the sweep back the same way.
|
|
12206
12674
|
*/
|
|
12207
12675
|
{
|
|
12208
12676
|
vetoed: vetoedPairs(proposed),
|
|
12209
|
-
judged: skipped === 0
|
|
12677
|
+
judged: skipped === 0 && unresolved === 0
|
|
12210
12678
|
}
|
|
12211
12679
|
),
|
|
12212
12680
|
llmCalls
|
|
@@ -12631,6 +13099,7 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
12631
13099
|
contradictions: 0,
|
|
12632
13100
|
promoted: 0,
|
|
12633
13101
|
skipped: 0,
|
|
13102
|
+
unresolved: 0,
|
|
12634
13103
|
capped: 0,
|
|
12635
13104
|
duplicates: 0,
|
|
12636
13105
|
tasksMinted: 0,
|
|
@@ -12697,6 +13166,13 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
12697
13166
|
let capped = 0;
|
|
12698
13167
|
/** Second-and-later verdicts naming a key their batch had already answered for. */
|
|
12699
13168
|
let duplicates = 0;
|
|
13169
|
+
/**
|
|
13170
|
+
* Verdicts whose key resolved to no offered pair, so they were dropped unacted. Dropping is the
|
|
13171
|
+
* safe outcome and stays; the count is what makes a SYSTEMATIC pattern visible — a model naming
|
|
13172
|
+
* keys in a spelling the resolver refuses drops every verdict of every batch, and a night that
|
|
13173
|
+
* judged nothing looked identical to a night whose model answered nothing (issue #58).
|
|
13174
|
+
*/
|
|
13175
|
+
let unresolved = 0;
|
|
12700
13176
|
let llmCalls = 0;
|
|
12701
13177
|
/**
|
|
12702
13178
|
* Contradictions this night detected for the FIRST time, so below the promotion gate.
|
|
@@ -12737,18 +13213,27 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
12737
13213
|
* rather than silently swallowed, so a model doing this is visible in a night's report.
|
|
12738
13214
|
*/
|
|
12739
13215
|
const answered = /* @__PURE__ */ new Set();
|
|
13216
|
+
/** This batch's dropped verdicts, so the warning below can name the spellings that failed. */
|
|
13217
|
+
const unresolvedKeys = [];
|
|
12740
13218
|
for (const verdict of answer.verdicts) {
|
|
12741
13219
|
/**
|
|
12742
13220
|
* The key is resolved through the kernel, so an invented key yields no candidate and no
|
|
12743
|
-
* write.
|
|
13221
|
+
* write. The CANONICAL key feeds the repeat guard, so `m1` and its label-prefixed spelling
|
|
13222
|
+
* `pair_m1` in one answer are one pair answered twice, not two pairs.
|
|
12744
13223
|
*/
|
|
12745
|
-
const
|
|
13224
|
+
const pairKey = offeredKeyFor(keyed, verdict.pairKey);
|
|
13225
|
+
if (pairKey === void 0) {
|
|
13226
|
+
unresolved += 1;
|
|
13227
|
+
unresolvedKeys.push(verdict.pairKey);
|
|
13228
|
+
continue;
|
|
13229
|
+
}
|
|
13230
|
+
const [candidate] = resolveKeys(keyed, [pairKey]);
|
|
12746
13231
|
if (candidate === void 0) continue;
|
|
12747
|
-
if (answered.has(
|
|
13232
|
+
if (answered.has(pairKey)) {
|
|
12748
13233
|
duplicates += 1;
|
|
12749
13234
|
continue;
|
|
12750
13235
|
}
|
|
12751
|
-
answered.add(
|
|
13236
|
+
answered.add(pairKey);
|
|
12752
13237
|
judged += 1;
|
|
12753
13238
|
if (!assertsEdge(verdict)) continue;
|
|
12754
13239
|
if (assertsContradiction(verdict)) {
|
|
@@ -12851,14 +13336,18 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
12851
13336
|
const [subject, object] = verdict.direction === "src_to_dst" ? [candidate.pair.src, candidate.pair.dst] : [candidate.pair.dst, candidate.pair.src];
|
|
12852
13337
|
if (yield* stampFile(env, subject, [link(verdict.rel, hrefFor(object)), meta("memhtml-updated", env.at)])) typed += 1;
|
|
12853
13338
|
}
|
|
13339
|
+
if (unresolvedKeys.length > 0) yield* Effect.logWarning(`sleep.llm edge-typing batch of ${batch.length} dropped ${unresolvedKeys.length} verdicts naming no offered pair (${unresolvedKeys.slice(0, 3).join(", ")}${unresolvedKeys.length > 3 ? ", …" : ""})`);
|
|
12854
13340
|
}
|
|
12855
13341
|
/**
|
|
12856
13342
|
* The single-detection contradictions become tasks in the SAME commit as the promotions. The
|
|
12857
13343
|
* sweep is gated on a night that judged its whole candidate set: `skipped` counts pairs whose
|
|
12858
13344
|
* batch's call failed as well as pairs whose endpoint the tree no longer holds, and a pair the
|
|
12859
13345
|
* model was never asked about must not read as a pair the model stopped contradicting.
|
|
13346
|
+
* `unresolved` is the same hazard from the answer side — a verdict the phase could not map to a
|
|
13347
|
+
* pair is a pair that was never judged, so its held-back contradiction must not close as "no
|
|
13348
|
+
* longer detected" over a misspelled key.
|
|
12860
13349
|
*/
|
|
12861
|
-
const tasks = yield* mintContradictionTasks(env, deferred, skipped === 0);
|
|
13350
|
+
const tasks = yield* mintContradictionTasks(env, deferred, skipped === 0 && unresolved === 0);
|
|
12862
13351
|
const counts = {
|
|
12863
13352
|
candidates: candidates.length,
|
|
12864
13353
|
judged,
|
|
@@ -12866,6 +13355,7 @@ const edgeTyping = (env) => Effect.gen(function* () {
|
|
|
12866
13355
|
contradictions,
|
|
12867
13356
|
promoted,
|
|
12868
13357
|
skipped,
|
|
13358
|
+
unresolved,
|
|
12869
13359
|
capped,
|
|
12870
13360
|
duplicates,
|
|
12871
13361
|
tasksMinted: tasks.minted,
|
|
@@ -13382,6 +13872,12 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
13382
13872
|
/** Model calls that came back malformed. The sweep's precondition reads this; see below. */
|
|
13383
13873
|
let callsFailed = 0;
|
|
13384
13874
|
/**
|
|
13875
|
+
* Cluster member keys (or canonical keys) resolving to no offered member, dropped unacted. The
|
|
13876
|
+
* drop is the safe outcome and stays; the count makes a systematic naming pattern visible
|
|
13877
|
+
* instead of reading as a night in which the model proposed no merges (issue #58).
|
|
13878
|
+
*/
|
|
13879
|
+
let unresolved = 0;
|
|
13880
|
+
/**
|
|
13385
13881
|
* Every pair this night deferred to a human, as a value rather than only a count.
|
|
13386
13882
|
*
|
|
13387
13883
|
* This is issue #44's motivating case in one variable. The phase used to report
|
|
@@ -13493,6 +13989,8 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
13493
13989
|
callsFailed += 1;
|
|
13494
13990
|
continue;
|
|
13495
13991
|
}
|
|
13992
|
+
/** This shard's unresolvable keys, so the warning below names the spellings that failed. */
|
|
13993
|
+
const unresolvedKeys = [];
|
|
13496
13994
|
for (const cluster of clustering.clusters) {
|
|
13497
13995
|
/**
|
|
13498
13996
|
* A key the batch never offered resolves to nothing, so an invented member cannot become a
|
|
@@ -13500,6 +13998,9 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
13500
13998
|
* canonical is outside it contradicts itself, and guessing which half was meant would be
|
|
13501
13999
|
* the caller inventing a merge.
|
|
13502
14000
|
*/
|
|
14001
|
+
const droppedKeys = [...cluster.memberKeys, cluster.canonicalKey].filter((key) => offeredKeyFor(keyed, key) === void 0);
|
|
14002
|
+
unresolved += droppedKeys.length;
|
|
14003
|
+
unresolvedKeys.push(...droppedKeys);
|
|
13503
14004
|
const memberNames = resolveKeys(keyed, cluster.memberKeys).map((centroid) => centroid.name);
|
|
13504
14005
|
const [canonicalMember] = resolveKeys(keyed, [cluster.canonicalKey]);
|
|
13505
14006
|
if (canonicalMember === void 0 || !memberNames.includes(canonicalMember.name)) continue;
|
|
@@ -13558,6 +14059,7 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
13558
14059
|
});
|
|
13559
14060
|
}
|
|
13560
14061
|
}
|
|
14062
|
+
if (unresolvedKeys.length > 0) yield* Effect.logWarning(`sleep.llm entity-resolution ${entityType} batch of ${shard.length} dropped ${unresolvedKeys.length} cluster keys naming no offered member (${unresolvedKeys.slice(0, 3).join(", ")}${unresolvedKeys.length > 3 ? ", …" : ""})`);
|
|
13561
14063
|
}
|
|
13562
14064
|
}
|
|
13563
14065
|
/**
|
|
@@ -13595,6 +14097,8 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
13595
14097
|
aliasMerges,
|
|
13596
14098
|
pendingCorroboration,
|
|
13597
14099
|
reviewCandidates,
|
|
14100
|
+
callsFailed,
|
|
14101
|
+
unresolved,
|
|
13598
14102
|
tasksMinted: 0,
|
|
13599
14103
|
tasksFramed: 0,
|
|
13600
14104
|
tasksDismissed: 0,
|
|
@@ -13620,7 +14124,12 @@ const entityResolution = (env) => Effect.gen(function* () {
|
|
|
13620
14124
|
* 1. The old early return on `rewrites.size === 0` would have skipped exactly the night this
|
|
13621
14125
|
* feature exists for: a night whose only outcome was deferrals is a night with no rewrites.
|
|
13622
14126
|
*/
|
|
13623
|
-
|
|
14127
|
+
/**
|
|
14128
|
+
* `unresolved === 0` joins the sweep gate for the reason `callsFailed === 0` is already in it: a
|
|
14129
|
+
* cluster key the phase could not map to a member is a name that was never judged, and sweeping
|
|
14130
|
+
* against a night that lost part of an answer closes reviews over a misspelling.
|
|
14131
|
+
*/
|
|
14132
|
+
const tasks = yield* mintReviewTasks(env, deferred, model !== void 0 && callsFailed === 0 && unresolved === 0);
|
|
13624
14133
|
let rewritten = 0;
|
|
13625
14134
|
for (const [path, pairs] of [...rewrites.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
|
|
13626
14135
|
const html = yield* readFileBytes(env, path);
|
|
@@ -14023,6 +14532,228 @@ const personLinks = (env) => Effect.gen(function* () {
|
|
|
14023
14532
|
};
|
|
14024
14533
|
});
|
|
14025
14534
|
|
|
14535
|
+
//#endregion
|
|
14536
|
+
//#region packages/sleep/dist/phases/placement-triage.js
|
|
14537
|
+
/**
|
|
14538
|
+
* Phase 14, placement triage. DEEP-ONLY (issue #63): propose a topic directory for each inbox
|
|
14539
|
+
* memory that even deep grouping left communityless, and `git mv` the ones the model places with
|
|
14540
|
+
* confidence. ONE COMMIT.
|
|
14541
|
+
*
|
|
14542
|
+
* The mechanism this phase exists for: a bulk-imported inbox is mostly NOT compressible — each file
|
|
14543
|
+
* is a distinct fact — and what keeps the inbox crowded is that nothing ever re-files them. `doctor`
|
|
14544
|
+
* reports `inboxCrowded` and no phase acts on it. This one does, under the same discipline as every
|
|
14545
|
+
* other mutation: on the review branch, in its own commit, reversible by discarding the branch.
|
|
14546
|
+
*
|
|
14547
|
+
* **On a nightly run the phase returns immediately**, before any read, with a reason — the same
|
|
14548
|
+
* shape `compress` has for a missing model. That is the whole no-flag contract: a run without
|
|
14549
|
+
* `--deep` cannot reach a single line of this phase's work.
|
|
14550
|
+
*
|
|
14551
|
+
* The deterministic guardrails, each enforced in code after the model answers:
|
|
14552
|
+
*
|
|
14553
|
+
* - a destination must be an EXISTING directory (as the index knows them) or one of at most
|
|
14554
|
+
* {@link PLACEMENT_NEW_DIR_CAP} new `areas/<topic>` or `resources/<topic>` directories per run —
|
|
14555
|
+
* first proposed, first minted, lexicographic order within one batch;
|
|
14556
|
+
* - the managed surfaces refuse: the inbox itself, `areas/arcs`, `resources/people`, anything under
|
|
14557
|
+
* `archive/`, and any path outside the PARA buckets;
|
|
14558
|
+
* - tasks never move (excluded from the scan AND re-checked per row, matching the double guard
|
|
14559
|
+
* task-detection carries);
|
|
14560
|
+
* - a file another phase already touched this run never moves, read from `git diff --name-only
|
|
14561
|
+
* base..HEAD` — a `mv` of a path compress just archived would fail, and one of a path an earlier
|
|
14562
|
+
* phase stamped would tear that phase's edit out of its own commit's diff;
|
|
14563
|
+
* - `keep-inbox`, an unknown key, a below-floor confidence, and an omitted member all leave the
|
|
14564
|
+
* file where it is.
|
|
14565
|
+
*
|
|
14566
|
+
* **Inbound hrefs are rewritten in the same commit.** Integrity's dangling-href repair chases a
|
|
14567
|
+
* target into the ARCHIVE by deriving `archivePathFor`; a placement move is not an archive, so this
|
|
14568
|
+
* phase rewrites `<link>` elements in the files that point at each moved path itself, the same
|
|
14569
|
+
* remove-then-add splice integrity uses. The indexer tracks the rename (`diff -M` + `movePath`), so
|
|
14570
|
+
* chunks and embeddings carry over and nothing re-embeds.
|
|
14571
|
+
*/
|
|
14572
|
+
/** Inbox memories offered per model call. Placement is a lighter question than a fold, so wider. */
|
|
14573
|
+
const PLACEMENT_BATCH_SIZE = 16;
|
|
14574
|
+
/** Inbox memories considered per run. The model-cost guard, in the spirit of compress's 2000. */
|
|
14575
|
+
const PLACEMENT_CANDIDATE_LIMIT = 2e3;
|
|
14576
|
+
/** Characters of each member shown. Placement needs the topic, not every fact. */
|
|
14577
|
+
const PLACEMENT_MEMBER_CHARS = 600;
|
|
14578
|
+
/**
|
|
14579
|
+
* New topic directories one run may mint. Small on purpose: a new directory is a new place every
|
|
14580
|
+
* future reader and `placementFor` caller has to know about, and a bulk import that genuinely needs
|
|
14581
|
+
* thirty new topics should earn them over several reviewed runs, not one.
|
|
14582
|
+
*/
|
|
14583
|
+
const PLACEMENT_NEW_DIR_CAP = 5;
|
|
14584
|
+
const placementTriage = (env) => Effect.gen(function* () {
|
|
14585
|
+
if (env.deep === void 0) return {
|
|
14586
|
+
...emptyOutcome({ candidates: 0 }),
|
|
14587
|
+
detail: "deep-only phase; run with --deep"
|
|
14588
|
+
};
|
|
14589
|
+
const model = env.deps.model;
|
|
14590
|
+
if (model === void 0) return {
|
|
14591
|
+
...emptyOutcome({ candidates: 0 }),
|
|
14592
|
+
detail: "no model bound"
|
|
14593
|
+
};
|
|
14594
|
+
/**
|
|
14595
|
+
* Candidates: active inbox memories with no community even under the WIDENED graph — nightly
|
|
14596
|
+
* edges plus the deep grouping band, the same partition deep compress groups by. Issue #63's
|
|
14597
|
+
* "true singleton" is exactly this: a file even the 0.72 band could not attach to anything.
|
|
14598
|
+
* A file the widened graph DID reach belongs to compress's pipeline (this run or a later one),
|
|
14599
|
+
* not to placement; an ENTITY-grouped file is deliberately still placeable, because entity
|
|
14600
|
+
* grouping is a compress-band mechanism and a KEEP-band inbox file it cannot fold still needs
|
|
14601
|
+
* a home.
|
|
14602
|
+
*/
|
|
14603
|
+
const widened = yield* deepCommunityLabels(env);
|
|
14604
|
+
const corpus = yield* activeCorpus(env.deps.db);
|
|
14605
|
+
const inInbox = (path) => path.startsWith(`${"areas/inbox"}/`) && !path.startsWith(`${"areas/inbox"}/tasks/`);
|
|
14606
|
+
const candidates = corpus.filter((row) => inInbox(row.path) && !isSleepExcluded(row.memory_type) && row.memory_type !== "arc" && widened.get(row.path) === void 0).sort((left, right) => left.path < right.path ? -1 : 1).slice(0, PLACEMENT_CANDIDATE_LIMIT);
|
|
14607
|
+
/**
|
|
14608
|
+
* The directories the corpus already has, from the index's own path set: every distinct
|
|
14609
|
+
* directory holding an active file under `areas/` or `resources/`, minus the managed surfaces.
|
|
14610
|
+
* Offered to the model verbatim and used as the validation set, so the question and the gate
|
|
14611
|
+
* cannot disagree.
|
|
14612
|
+
*/
|
|
14613
|
+
const managed = /* @__PURE__ */ new Set([
|
|
14614
|
+
INBOX_DIR,
|
|
14615
|
+
`${INBOX_DIR}/tasks`,
|
|
14616
|
+
ARCS_DIR,
|
|
14617
|
+
PEOPLE_DIR
|
|
14618
|
+
]);
|
|
14619
|
+
const existingDirs = [...new Set(corpus.map((row) => row.path.slice(0, row.path.lastIndexOf("/"))).filter((dir) => (dir.startsWith("areas/") || dir.startsWith("resources/")) && !managed.has(dir)))].sort();
|
|
14620
|
+
/** Paths this run already touched. A move of one would cross-contaminate another phase's diff. */
|
|
14621
|
+
const touched = new Set(env.baseSha === "" ? [] : yield* env.deps.git.run([
|
|
14622
|
+
"diff",
|
|
14623
|
+
"--name-only",
|
|
14624
|
+
`${env.baseSha}..HEAD`
|
|
14625
|
+
]).pipe(Effect.map((out) => out.split("\n").map((line) => line.trim()).filter(Boolean)), Effect.orElseSucceed(() => [])));
|
|
14626
|
+
const batches = assembleBatches([candidates], { maxMembers: 16 });
|
|
14627
|
+
const counts = {
|
|
14628
|
+
candidates: candidates.length,
|
|
14629
|
+
batches: batches.length,
|
|
14630
|
+
proposed: 0,
|
|
14631
|
+
applied: 0,
|
|
14632
|
+
refused: 0,
|
|
14633
|
+
keptInbox: 0,
|
|
14634
|
+
newDirs: 0,
|
|
14635
|
+
budgetSkipped: 0,
|
|
14636
|
+
failed: 0
|
|
14637
|
+
};
|
|
14638
|
+
if (batches.length === 0 || env.dryRun) return emptyOutcome(counts);
|
|
14639
|
+
const modelKey = modelFor(env.deps, "placement-triage");
|
|
14640
|
+
const mintedDirs = /* @__PURE__ */ new Set();
|
|
14641
|
+
const movedFrom = /* @__PURE__ */ new Map();
|
|
14642
|
+
let llmCalls = 0;
|
|
14643
|
+
for (const batch of batches) {
|
|
14644
|
+
const keyed = keyMembers(batch, (row) => `${row.title}\n${row.gist}\n${row.body_text}`, { charBudget: 600 });
|
|
14645
|
+
if (!takeLlmCall(env.deep)) {
|
|
14646
|
+
counts["budgetSkipped"] = (counts["budgetSkipped"] ?? 0) + 1;
|
|
14647
|
+
continue;
|
|
14648
|
+
}
|
|
14649
|
+
llmCalls += 1;
|
|
14650
|
+
const answer = yield* batchCall(model, `placement batch of ${batch.length}`, {
|
|
14651
|
+
schema: PlacementTriage,
|
|
14652
|
+
system: PLACEMENT_SYSTEM,
|
|
14653
|
+
prompt: placementPrompt(existingDirs, keyed.keyed),
|
|
14654
|
+
modelKey,
|
|
14655
|
+
effort: "high",
|
|
14656
|
+
toolDescription: "Propose a destination directory (or keep-inbox) per offered memory."
|
|
14657
|
+
});
|
|
14658
|
+
if (answer === void 0) {
|
|
14659
|
+
counts["failed"] = (counts["failed"] ?? 0) + 1;
|
|
14660
|
+
continue;
|
|
14661
|
+
}
|
|
14662
|
+
/**
|
|
14663
|
+
* Placements resolve through the kernel, so an invented key drops. Sorted by (destination,
|
|
14664
|
+
* key) so within one answer the minting order of new directories is a function of the answer's
|
|
14665
|
+
* CONTENT rather than of its field order.
|
|
14666
|
+
*/
|
|
14667
|
+
const resolvedRows = answer.placements.filter((placement) => placement.destination.trim() !== "").sort((left, right) => left.destination < right.destination ? -1 : left.destination > right.destination ? 1 : left.memberKey < right.memberKey ? -1 : 1);
|
|
14668
|
+
for (const placement of resolvedRows) {
|
|
14669
|
+
const row = resolveKeys(keyed, [placement.memberKey])[0];
|
|
14670
|
+
if (row === void 0) continue;
|
|
14671
|
+
counts["proposed"] = (counts["proposed"] ?? 0) + 1;
|
|
14672
|
+
const destination = normalizePath(placement.destination.trim());
|
|
14673
|
+
if (destination === "keep-inbox") {
|
|
14674
|
+
counts["keptInbox"] = (counts["keptInbox"] ?? 0) + 1;
|
|
14675
|
+
continue;
|
|
14676
|
+
}
|
|
14677
|
+
const refuse = (reason) => {
|
|
14678
|
+
counts["refused"] = (counts["refused"] ?? 0) + 1;
|
|
14679
|
+
return Effect.logWarning(`sleep.placement refused ${row.path} -> ${destination}: ${reason}`);
|
|
14680
|
+
};
|
|
14681
|
+
if (placement.confidence < .7) {
|
|
14682
|
+
yield* refuse("below the confidence floor");
|
|
14683
|
+
continue;
|
|
14684
|
+
}
|
|
14685
|
+
const bucket = paraBucketOf(destination);
|
|
14686
|
+
if (bucket !== "areas" && bucket !== "resources" || managed.has(destination) || destination === row.path.slice(0, row.path.lastIndexOf("/")) || destination.split("/").some((segment) => segment === "" || segment === "." || segment === "..")) {
|
|
14687
|
+
yield* refuse("not a placeable directory");
|
|
14688
|
+
continue;
|
|
14689
|
+
}
|
|
14690
|
+
if (isSleepExcluded(row.memory_type)) {
|
|
14691
|
+
yield* refuse("tasks never move");
|
|
14692
|
+
continue;
|
|
14693
|
+
}
|
|
14694
|
+
if (touched.has(row.path) || movedFrom.has(row.path)) {
|
|
14695
|
+
yield* refuse("another phase touched this file this run");
|
|
14696
|
+
continue;
|
|
14697
|
+
}
|
|
14698
|
+
const isNew = !existingDirs.includes(destination) && !mintedDirs.has(destination);
|
|
14699
|
+
if (isNew && mintedDirs.size >= 5) {
|
|
14700
|
+
yield* refuse("the new-directory cap for this run is spent");
|
|
14701
|
+
continue;
|
|
14702
|
+
}
|
|
14703
|
+
const target = `${destination}/${row.path.slice(row.path.lastIndexOf("/") + 1)}`;
|
|
14704
|
+
if (!isValidMemoryPath(target)) {
|
|
14705
|
+
yield* refuse("the destination path is not a valid memory path");
|
|
14706
|
+
continue;
|
|
14707
|
+
}
|
|
14708
|
+
if ((yield* readFileBytes(env, target)) !== void 0) {
|
|
14709
|
+
yield* refuse("the destination already holds a file by that name");
|
|
14710
|
+
continue;
|
|
14711
|
+
}
|
|
14712
|
+
if ((yield* readFileBytes(env, row.path)) === void 0) {
|
|
14713
|
+
yield* refuse("the source file is already gone from the tree");
|
|
14714
|
+
continue;
|
|
14715
|
+
}
|
|
14716
|
+
yield* attemptIo(`sleep.placement.mkdir:${target}`, async () => {
|
|
14717
|
+
const { mkdir } = await import("node:fs/promises");
|
|
14718
|
+
const { dirname } = await import("node:path");
|
|
14719
|
+
await mkdir(dirname(absoluteIn(env, target)), { recursive: true });
|
|
14720
|
+
});
|
|
14721
|
+
yield* env.deps.git.mv(row.path, target);
|
|
14722
|
+
yield* stampFile(env, target, [meta("memhtml-updated", env.at)]);
|
|
14723
|
+
if (isNew) {
|
|
14724
|
+
mintedDirs.add(destination);
|
|
14725
|
+
counts["newDirs"] = (counts["newDirs"] ?? 0) + 1;
|
|
14726
|
+
}
|
|
14727
|
+
movedFrom.set(row.path, target);
|
|
14728
|
+
counts["applied"] = (counts["applied"] ?? 0) + 1;
|
|
14729
|
+
}
|
|
14730
|
+
}
|
|
14731
|
+
/**
|
|
14732
|
+
* Inbound href repair, inside the same commit as the moves. Every authored `<link>` whose
|
|
14733
|
+
* target moved is respliced in ITS OWN file to the new path — remove-then-add, integrity's
|
|
14734
|
+
* idempotent shape — so the branch never carries a commit whose links dangle by construction.
|
|
14735
|
+
*/
|
|
14736
|
+
let rewritten = 0;
|
|
14737
|
+
for (const [from, to] of [...movedFrom.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
|
|
14738
|
+
const inbound = yield* inboundAuthoredEdges(env.deps.db, from);
|
|
14739
|
+
for (const edge of inbound) {
|
|
14740
|
+
if (!isEdgeRel(edge.rel)) continue;
|
|
14741
|
+
const holder = movedFrom.get(edge.src_path) ?? edge.src_path;
|
|
14742
|
+
if (yield* stampFile(env, holder, [
|
|
14743
|
+
unlink(edge.rel, hrefFor(from)),
|
|
14744
|
+
link(edge.rel, hrefFor(to)),
|
|
14745
|
+
meta("memhtml-updated", env.at)
|
|
14746
|
+
])) rewritten += 1;
|
|
14747
|
+
}
|
|
14748
|
+
}
|
|
14749
|
+
counts["hrefsRewritten"] = rewritten;
|
|
14750
|
+
return {
|
|
14751
|
+
counts,
|
|
14752
|
+
commitSha: yield* commitPhase(env, "placement-triage", `re-file ${counts["applied"] ?? 0} inbox memories into topic directories`, counts),
|
|
14753
|
+
llmCalls
|
|
14754
|
+
};
|
|
14755
|
+
});
|
|
14756
|
+
|
|
14026
14757
|
//#endregion
|
|
14027
14758
|
//#region packages/sleep/dist/phases/preflight.js
|
|
14028
14759
|
/**
|
|
@@ -14059,60 +14790,6 @@ const preflight = (env) => Effect.gen(function* () {
|
|
|
14059
14790
|
}) };
|
|
14060
14791
|
});
|
|
14061
14792
|
|
|
14062
|
-
//#endregion
|
|
14063
|
-
//#region packages/sleep/dist/phases/relationship-mining.js
|
|
14064
|
-
/**
|
|
14065
|
-
* Phase 5, relationship mining. Derived `relates_to` edges in the index only. NO COMMIT.
|
|
14066
|
-
*
|
|
14067
|
-
* A mined edge is a re-derivable function of the corpus and the embedder. `index rebuild` plus the
|
|
14068
|
-
* next night's mining regenerates the identical set, so committing thousands of them would bury every
|
|
14069
|
-
* real diff in machine noise for zero recoverable information. The `derived` column is the firewall
|
|
14070
|
-
* that makes losing them cheap. The retention penalty counts only `derived = 0`, so an
|
|
14071
|
-
* uncorroborated machine suspicion cannot evict a memory.
|
|
14072
|
-
*
|
|
14073
|
-
* The insert is scoped to `provenance = 'sleep'` and `derived = 1` and the whole replace is one
|
|
14074
|
-
* atomic batch, so an authored edge is unreachable from here and the corpus is never left with the
|
|
14075
|
-
* old mined set deleted and the new one not yet written. In that window the lateral arm would
|
|
14076
|
-
* silently return nothing.
|
|
14077
|
-
*/
|
|
14078
|
-
/** The similarity floor a pair must clear to become a mined `relates_to`. */
|
|
14079
|
-
const MINING_COSINE_FLOOR = .85;
|
|
14080
|
-
/** Nearest neighbors considered per source file. */
|
|
14081
|
-
const MINING_PER_SOURCE_K = 5;
|
|
14082
|
-
/**
|
|
14083
|
-
* Pairs mined per cycle: a cap on what {@link replaceMinedEdges} writes, not on the scan — the
|
|
14084
|
-
* kernel's arithmetic is O(n²·d) whatever this says, and it bounds the edge table so one dense
|
|
14085
|
-
* neighborhood cannot flood the graph the lateral arm and PageRank read.
|
|
14086
|
-
*/
|
|
14087
|
-
const MINING_SAMPLE_LIMIT = 2e3;
|
|
14088
|
-
const relationshipMining = (env) => Effect.gen(function* () {
|
|
14089
|
-
/**
|
|
14090
|
-
* Tasks are excluded, and here the exclusion is the graph firewall, not a cost guard.
|
|
14091
|
-
* Every mined edge is written with `edge_class = 'memory'`, so a pair with a task endpoint
|
|
14092
|
-
* would put a task INTO the memory graph, reaching PageRank, MMR, and the retention bridge
|
|
14093
|
-
* count. The `edges` CHECK cannot refuse it, because `relates_to` under `memory` is a
|
|
14094
|
-
* well-formed edge whatever files sit at its ends.
|
|
14095
|
-
*/
|
|
14096
|
-
const pairs = yield* neighborPairs(env.deps.db, {
|
|
14097
|
-
floor: MINING_COSINE_FLOOR,
|
|
14098
|
-
perSourceK: 5,
|
|
14099
|
-
limit: MINING_SAMPLE_LIMIT,
|
|
14100
|
-
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
14101
|
-
});
|
|
14102
|
-
const counts = {
|
|
14103
|
-
candidates: pairs.length,
|
|
14104
|
-
mined: pairs.length
|
|
14105
|
-
};
|
|
14106
|
-
if (env.dryRun) return emptyOutcome(counts);
|
|
14107
|
-
yield* replaceMinedEdges(env.deps.db, {
|
|
14108
|
-
runId: env.runId,
|
|
14109
|
-
at: env.at,
|
|
14110
|
-
rel: "relates_to",
|
|
14111
|
-
pairs
|
|
14112
|
-
});
|
|
14113
|
-
return emptyOutcome(counts);
|
|
14114
|
-
});
|
|
14115
|
-
|
|
14116
14793
|
//#endregion
|
|
14117
14794
|
//#region packages/sleep/dist/report.js
|
|
14118
14795
|
/**
|
|
@@ -14617,6 +15294,12 @@ const taskDetection = (env) => Effect.gen(function* () {
|
|
|
14617
15294
|
let dismissed = 0;
|
|
14618
15295
|
let skipped = 0;
|
|
14619
15296
|
/**
|
|
15297
|
+
* Findings whose key resolved to no offered member, dropped unacted. The drop is the safe
|
|
15298
|
+
* outcome and stays; the count makes a systematic pattern visible, because a night whose every
|
|
15299
|
+
* finding named an unresolvable key otherwise reads as a night with no open work (issue #58).
|
|
15300
|
+
*/
|
|
15301
|
+
let unresolved = 0;
|
|
15302
|
+
/**
|
|
14620
15303
|
* Every key this night's scan SAW, whether or not it minted and whether or not it cleared the floor.
|
|
14621
15304
|
* The sweep's input; see the `liveKeys.add` below for why the floor is not a filter here.
|
|
14622
15305
|
*/
|
|
@@ -14644,11 +15327,20 @@ const taskDetection = (env) => Effect.gen(function* () {
|
|
|
14644
15327
|
* because it is called one key at a time here — a finding names one member.
|
|
14645
15328
|
*/
|
|
14646
15329
|
const answered = /* @__PURE__ */ new Set();
|
|
15330
|
+
/** This batch's dropped findings, so the warning below can name the spellings that failed. */
|
|
15331
|
+
const unresolvedKeys = [];
|
|
14647
15332
|
for (const finding of answer.findings) {
|
|
14648
|
-
|
|
15333
|
+
/** The CANONICAL key feeds the repeat guard, same as edge-typing's, and for the same reason. */
|
|
15334
|
+
const memberKey = offeredKeyFor(keyed, finding.memberKey);
|
|
15335
|
+
if (memberKey === void 0) {
|
|
15336
|
+
unresolved += 1;
|
|
15337
|
+
unresolvedKeys.push(finding.memberKey);
|
|
15338
|
+
continue;
|
|
15339
|
+
}
|
|
15340
|
+
const [row] = resolveKeys(keyed, [memberKey]);
|
|
14649
15341
|
if (row === void 0) continue;
|
|
14650
|
-
if (answered.has(
|
|
14651
|
-
answered.add(
|
|
15342
|
+
if (answered.has(memberKey)) continue;
|
|
15343
|
+
answered.add(memberKey);
|
|
14652
15344
|
findings += 1;
|
|
14653
15345
|
/**
|
|
14654
15346
|
* The key is the SOURCE PATH plus the normalized sentence, so the same commitment found again
|
|
@@ -14694,13 +15386,17 @@ const taskDetection = (env) => Effect.gen(function* () {
|
|
|
14694
15386
|
else if (outcome === "framed") framed += 1;
|
|
14695
15387
|
else if (outcome === "dismissed") dismissed += 1;
|
|
14696
15388
|
}
|
|
15389
|
+
if (unresolvedKeys.length > 0) yield* Effect.logWarning(`sleep.llm task-detection batch of ${batch.length} dropped ${unresolvedKeys.length} findings naming no offered member (${unresolvedKeys.slice(0, 3).join(", ")}${unresolvedKeys.length > 3 ? ", …" : ""})`);
|
|
14697
15390
|
}
|
|
14698
15391
|
/**
|
|
14699
15392
|
* The sweep, only from a full-strength scan. `skipped > 0` means at least one batch's memories
|
|
14700
15393
|
* went unread, so a finding of theirs is missing from `liveKeys` because the phase could not look
|
|
14701
|
-
* rather than because it is gone.
|
|
15394
|
+
* rather than because it is gone. `unresolved > 0` is the same hazard from the answer side: the
|
|
15395
|
+
* model reported a finding the phase could not map to a member, so its detection key was never
|
|
15396
|
+
* constructed, and sweeping against that would close a live task because the model misspelled a
|
|
15397
|
+
* key rather than because the finding vanished.
|
|
14702
15398
|
*/
|
|
14703
|
-
const closed = skipped === 0 ? yield* closeVanishedDetections(env, TASK_DETECT_DETECTOR, liveKeys) : 0;
|
|
15399
|
+
const closed = skipped === 0 && unresolved === 0 ? yield* closeVanishedDetections(env, TASK_DETECT_DETECTOR, liveKeys) : 0;
|
|
14704
15400
|
const counts = {
|
|
14705
15401
|
candidates: candidates.length,
|
|
14706
15402
|
batches: batches.length,
|
|
@@ -14712,7 +15408,8 @@ const taskDetection = (env) => Effect.gen(function* () {
|
|
|
14712
15408
|
dismissed,
|
|
14713
15409
|
closed,
|
|
14714
15410
|
capped: budget.overflow,
|
|
14715
|
-
skipped
|
|
15411
|
+
skipped,
|
|
15412
|
+
unresolved
|
|
14716
15413
|
};
|
|
14717
15414
|
/**
|
|
14718
15415
|
* A refresh writes a `memhtml-updated` stamp, which is a staged file, so it commits — the queue's
|
|
@@ -14748,7 +15445,8 @@ const ZERO = {
|
|
|
14748
15445
|
dismissed: 0,
|
|
14749
15446
|
closed: 0,
|
|
14750
15447
|
capped: 0,
|
|
14751
|
-
skipped: 0
|
|
15448
|
+
skipped: 0,
|
|
15449
|
+
unresolved: 0
|
|
14752
15450
|
};
|
|
14753
15451
|
|
|
14754
15452
|
//#endregion
|
|
@@ -15685,6 +16383,7 @@ const PHASE_BODIES = {
|
|
|
15685
16383
|
reprieve,
|
|
15686
16384
|
"trace-consolidation": traceConsolidation,
|
|
15687
16385
|
"task-detection": taskDetection,
|
|
16386
|
+
"placement-triage": placementTriage,
|
|
15688
16387
|
integrity,
|
|
15689
16388
|
"state-export": stateExport,
|
|
15690
16389
|
report: reportPhase([])
|
|
@@ -15745,7 +16444,14 @@ const run = (deps, options) => Effect.gen(function* () {
|
|
|
15745
16444
|
* Created per run rather than held in a module, which is what keeps two runs in one process (and
|
|
15746
16445
|
* two tests in one file) from sharing a counter.
|
|
15747
16446
|
*/
|
|
15748
|
-
detectionBudget: makeDetectionBudget()
|
|
16447
|
+
detectionBudget: makeDetectionBudget(),
|
|
16448
|
+
/**
|
|
16449
|
+
* The deep switches, shaped once here so every phase reads one field. The LLM budget is the
|
|
16450
|
+
* same created-per-run discipline as the detection budget above, and it exists only when the
|
|
16451
|
+
* caller stated a cap: an uncapped deep run is a valid ask, and a phantom cap of zero (or of
|
|
16452
|
+
* some default nobody chose) would silently skip work the operator paid for the flag to reach.
|
|
16453
|
+
*/
|
|
16454
|
+
...options.deep === true ? { deep: { ...options.maxLlmCalls === void 0 ? {} : { budget: makeLlmBudget(options.maxLlmCalls) } } } : {}
|
|
15749
16455
|
};
|
|
15750
16456
|
if (!dryRun) yield* deps.git.checkoutBranch(runId, { create: true }).pipe(Effect.orElseSucceed(() => {}));
|
|
15751
16457
|
yield* ignoreFailure(recordRun(deps.db, {
|
|
@@ -15812,7 +16518,9 @@ const resume = (deps, runId, options = {}) => Effect.gen(function* () {
|
|
|
15812
16518
|
* finishing. The cost of a fresh one is bounded by the cap, and the mints a resume repeats are
|
|
15813
16519
|
* refreshes rather than duplicates, which cost no budget at all.
|
|
15814
16520
|
*/
|
|
15815
|
-
detectionBudget: makeDetectionBudget()
|
|
16521
|
+
detectionBudget: makeDetectionBudget(),
|
|
16522
|
+
/** A fresh LLM budget too, for the same reason the detection budget above is fresh. */
|
|
16523
|
+
...options.deep === true ? { deep: { ...options.maxLlmCalls === void 0 ? {} : { budget: makeLlmBudget(options.maxLlmCalls) } } } : {}
|
|
15816
16524
|
};
|
|
15817
16525
|
const remaining = SLEEP_PHASES.filter((phase) => !completed.has(phase));
|
|
15818
16526
|
const executed = yield* executePhases(env, remaining, /* @__PURE__ */ new Set());
|
|
@@ -15820,7 +16528,7 @@ const resume = (deps, runId, options = {}) => Effect.gen(function* () {
|
|
|
15820
16528
|
const ended = yield* nowIso;
|
|
15821
16529
|
/**
|
|
15822
16530
|
* Skipped-because-already-done rows are reported explicitly, so a resume's report accounts for all
|
|
15823
|
-
*
|
|
16531
|
+
* seventeen phases. A report that showed only the eight it ran would read as a partial run.
|
|
15824
16532
|
*/
|
|
15825
16533
|
const priorRows = yield* ignoreFailureWith(readPhases(deps.db, runId), []);
|
|
15826
16534
|
const already = [...completed].map((phase) => {
|
|
@@ -16861,4 +17569,4 @@ const latest = (left, right) => left === null ? right : right === null ? left :
|
|
|
16861
17569
|
|
|
16862
17570
|
//#endregion
|
|
16863
17571
|
export { STATE_DB_PATH as $, IndexRecorder as A, IndexGit as B, ModelClient as C, EmbeddingsLive as D, Embeddings as E, makeRetrieval as F, MIGRATIONS_DIR as G, sanitizeFtsQuery as H, reinforce as I, Store as J, STATE_MIGRATIONS_DIR as K, Indexer as L, persistScanned as M, readWatermark as N, EMBED_DIM as O, Retrieval as P, SLEEP_REPORTS_DIR as Q, makeIndexer as R, runDiscrimination as S, wrapAsData as T, DatabaseService as U, makeGitPort as V, makeDatabase as W, makeStore as X, expandRoot as Y, INDEX_DB_PATH as Z, meta as _, parseSidecar as a, makeGit as at, isSleepPhase as b, generateArtifacts as c, setMeta as ct, allPaths as d, fenceOpeningOf as dt, STATE_SIDECAR_PATH as et, danglingEdges as f, REINFORCE_SIGNALS as ft, link as g, hrefFor as h, makeSleep as i, Git as it, makeIndexRecorder as j, EMBED_WATERMARK as k, DETECTION_PREFIX as l, isValidDatetime as lt, applyHeadEdits as m, scanTraceRoot as n, initRepo as nt, renderSidecar as o, commitSubject as ot, publishRows as p, frameKeyOf as pt, STATE_SCHEMA as q, Sleep as r, readFileOrNull as rt, archivedFormOf as s, checkMemory as st, mergeTailExtract as t, attemptIo as tt, accessRows as u, closesFence as ut, unlink as v, ModelClientLive as w, discriminationGate as x, SLEEP_PHASES as y, readIndexState as z };
|
|
16864
|
-
//# sourceMappingURL=dist-
|
|
17572
|
+
//# sourceMappingURL=dist-CHoz5uHd.mjs.map
|