memhtml 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9181,21 +9181,31 @@ const batchCall = (model, label, request) => isolate(label, model.generateObject
9181
9181
  * `sleep_phases` row, and in a `--phases` flag, and three copies of the string would drift.
9182
9182
  */
9183
9183
  /**
9184
- * The sixteen phases, in execution order.
9184
+ * The seventeen phases, in execution order.
9185
9185
  *
9186
9186
  * The order encodes the predecessor memory system's dependencies (design §6): entity resolution precedes person
9187
9187
  * links so aliases have already merged, confidence decay precedes retention triage so triage
9188
9188
  * scores the decayed value, and dedup-merge precedes compress and retention because both operate
9189
9189
  * on the post-merge set.
9190
9190
  *
9191
- * `task-detection` is sixteenth-in-list and thirteenth-in-order, sitting after `trace-consolidation`
9192
- * and before `integrity`, and both edges are deliberate. It scans the ACTIVE corpus for unresolved
9191
+ * `task-detection` sits after `trace-consolidation` and before `integrity`, and both edges are
9192
+ * deliberate. It scans the ACTIVE corpus for unresolved
9193
9193
  * commitments, so it has to run after every phase that changes what is active — after dedup's folds,
9194
9194
  * after retention's evictions, after compress's canonicals, and after trace consolidation's newly
9195
9195
  * distilled memories, which are the freshest text of the night and the likeliest to carry one. And it
9196
9196
  * WRITES files, so it must precede `integrity`, which repairs dangling hrefs and regenerates the
9197
9197
  * directory artifacts: a task minted afterwards would be absent from its directory's `index.html`
9198
9198
  * until the next night.
9199
+ *
9200
+ * `placement-triage` is deep-only (issue #63): on a run without `--deep` it returns immediately,
9201
+ * writes nothing, and commits nothing, so the nightly cycle's behavior is unchanged by its presence
9202
+ * in this list. Its slot has the same two edges task-detection's does, plus one more on each side:
9203
+ * it must run after `compress` (it re-files only what even deep grouping could not fold, so folding
9204
+ * has to have had its chance first) and after `task-detection` (a move mid-scan would hand the
9205
+ * detector paths that no longer hold files), and it must precede `integrity` because it MOVES files —
9206
+ * integrity regenerates each directory's `index.html`, and it rewrites inbound hrefs itself because
9207
+ * integrity's dangling-href repair only knows how to chase a target into the ARCHIVE, not into a
9208
+ * topic directory.
9199
9209
  */
9200
9210
  const SLEEP_PHASES = [
9201
9211
  "preflight",
@@ -9211,6 +9221,7 @@ const SLEEP_PHASES = [
9211
9221
  "reprieve",
9212
9222
  "trace-consolidation",
9213
9223
  "task-detection",
9224
+ "placement-triage",
9214
9225
  "integrity",
9215
9226
  "state-export",
9216
9227
  "report"
@@ -9577,7 +9588,8 @@ const DEFAULT_MODELS = {
9577
9588
  "arc-synthesis": "gpt-5.6-sol",
9578
9589
  compress: "gpt-5.6-sol",
9579
9590
  "trace-consolidation": "opus-5",
9580
- "task-detection": "gpt-5.6-sol"
9591
+ "task-detection": "gpt-5.6-sol",
9592
+ "placement-triage": "gpt-5.6-sol"
9581
9593
  };
9582
9594
  /** The model a phase calls: the caller's override, else {@link DEFAULT_MODELS}, else sonnet. */
9583
9595
  const modelFor = (deps, phase) => deps.models?.[phase] ?? DEFAULT_MODELS[phase] ?? "sonnet-5";
@@ -9587,6 +9599,25 @@ const emptyOutcome = (counts = {}) => ({
9587
9599
  commitSha: null,
9588
9600
  llmCalls: 0
9589
9601
  });
9602
+ /** A fresh budget. A non-positive or fractional cap clamps to a usable whole number. */
9603
+ const makeLlmBudget = (maxCalls) => ({
9604
+ maxCalls: Math.max(0, Math.trunc(maxCalls)),
9605
+ spent: 0
9606
+ });
9607
+ /**
9608
+ * Take one call from the budget, or report exhaustion.
9609
+ *
9610
+ * `true` means the caller may make the call and the budget has been charged. Charging BEFORE the
9611
+ * call rather than after means a crash mid-call cannot under-count, which errs on the side the
9612
+ * budget exists for. No budget bound means every call is allowed.
9613
+ */
9614
+ const takeLlmCall = (deep) => {
9615
+ const budget = deep?.budget;
9616
+ if (budget === void 0) return true;
9617
+ if (budget.spent >= budget.maxCalls) return false;
9618
+ budget.spent += 1;
9619
+ return true;
9620
+ };
9590
9621
 
9591
9622
  //#endregion
9592
9623
  //#region packages/sleep/dist/llm.js
@@ -10054,6 +10085,71 @@ const edgeTypingPrompt = (pairs) => batchPrompt(pairs, EDGE_TYPING_INSTRUCTION,
10054
10085
  const arcTriagePrompt = (arcsText, evidenceText) => `${dataBlock("current_arcs", arcsText)}\n\n${dataBlock("evidence", evidenceText)}\n\nProduce a triage plan. Assign update or skip to every existing arc, and add a create entry for any genuinely new behavioral pattern the existing arcs do not cover.`;
10055
10086
  /** The arc-execute user turn for one arc. `current` is absent on a create. */
10056
10087
  const arcExecutePrompt = (input) => (input.current === void 0 ? `${dataBlock("new_arc_title", input.title)}\n\n` : `${dataBlock("existing_arc", input.current)}\n\n`) + `${dataBlock("evidence", input.evidenceText)}\n\n${dataBlock("triage_rationale", input.rationale)}\n\n` + (input.current === void 0 ? "Synthesize a new behavioral principal from this evidence." : "Update the arc to incorporate the new evidence, preserving existing knowledge that holds.");
10088
+ /**
10089
+ * One inbox memory's proposed destination, under the opaque key it was offered as (issue #63).
10090
+ *
10091
+ * `destination` is a STRING the phase validates, never trusts: only `keep-inbox`, a directory the
10092
+ * corpus already has, or one of a capped handful of new topic directories survives the code gate.
10093
+ * A model-invented path outside those is a refused placement, counted and logged, because a
10094
+ * destination is a write target and the model does not choose write targets anywhere in this
10095
+ * pipeline.
10096
+ */
10097
+ const Placement = Schema.Struct({
10098
+ /** The offered key, e.g. `m3`. A key the batch never held resolves to nothing and is dropped. */
10099
+ memberKey: Schema.String,
10100
+ /**
10101
+ * `keep-inbox`, or a bucket-rooted directory like `areas/deploys` or `resources/sqlite`.
10102
+ * `keep-inbox` is the refusal and the ordinary answer for a memory whose topic is not clear.
10103
+ */
10104
+ destination: Schema.String,
10105
+ /** Unitless in `[0, 1]`. The move gate is deterministic and reads this, not the prose. */
10106
+ confidence: Schema.Finite.check(Schema.isBetween({
10107
+ minimum: 0,
10108
+ maximum: 1
10109
+ }))
10110
+ });
10111
+ /** One batch's whole answer: a placement list. An omitted member keeps its inbox path. */
10112
+ const PlacementTriage = Schema.Struct({ placements: Schema.Array(Placement) });
10113
+ /**
10114
+ * The confidence a placement must clear before the phase moves a file. The same 0.7 the edge and
10115
+ * entity gates use, for the same reason one number serves them: a second knob would be a threshold
10116
+ * nobody could state the meaning of. A move is cheap to reverse (`git mv` back, reviewable on the
10117
+ * branch), so the floor matches the mild gates rather than the corroboration-backed ones.
10118
+ */
10119
+ const PLACEMENT_CONFIDENCE_FLOOR = .7;
10120
+ /** The literal a placement answer uses to decline. */
10121
+ const PLACEMENT_KEEP = "keep-inbox";
10122
+ /**
10123
+ * The placement-triage system prompt (issue #63).
10124
+ *
10125
+ * The refusal is stated as the ordinary answer, because for a bulk-imported inbox it is: most
10126
+ * singletons are distinct facts whose topic directory does not exist yet, and a model that felt
10127
+ * obliged to place everything would scatter the inbox across invented directories nobody asked for.
10128
+ */
10129
+ const PLACEMENT_SYSTEM = `You file inbox memories into topic directories for an AI agent's long-term memory system. You are
10130
+ given the list of directories the corpus already has, and a NUMBERED LIST of memories currently
10131
+ sitting in the inbox. For each memory, propose where it belongs.
10132
+
10133
+ - destination is one of the existing directories, verbatim from the list, when the memory clearly
10134
+ belongs to that topic.
10135
+ - destination may be a NEW directory (areas/<topic> for ongoing concerns, resources/<topic> for
10136
+ reference material) when several of THESE memories share a topic no existing directory covers.
10137
+ Use a short lowercase hyphenated topic name. New directories are rationed, so propose one only
10138
+ when it would hold more than a stray file.
10139
+ - destination is keep-inbox when you are unsure, when the memory's topic is ambiguous, or when no
10140
+ directory fits. This is the ordinary answer, not a failure.
10141
+ - Never propose archive/, areas/inbox itself, areas/arcs, or resources/people. Those are managed
10142
+ surfaces.
10143
+ - Rate confidence honestly. A placement above the floor moves the file on a review branch.
10144
+ - Omitting a member means keep-inbox.`;
10145
+ /** The instruction that closes a placement batch's user turn, after the member list. */
10146
+ const PLACEMENT_INSTRUCTION = "Propose a destination for each memory above: an existing directory from the list, a new areas/<topic> or resources/<topic> directory, or keep-inbox. Name each memory by its offered key and rate your confidence.";
10147
+ /**
10148
+ * The placement user turn for one batch: the existing directories, then every member under its
10149
+ * offered key, then the instruction. The directory list is corpus-derived text, so it is wrapped as
10150
+ * data exactly as the member texts are.
10151
+ */
10152
+ const placementPrompt = (directories, members) => `${dataBlock("existing_directories", directories.join("\n"))}\n\n` + batchPrompt(members, PLACEMENT_INSTRUCTION, { label: "memory" });
10057
10153
  /** The instruction that closes a compress batch's user turn, after the member list. */
10058
10154
  const COMPRESS_INSTRUCTION = "Fold these memories into one canonical memory. List in absorbedKeys exactly the members whose content the canonical carries forward.";
10059
10155
  /**
@@ -10106,7 +10202,7 @@ const dedupPrompt = (components) => {
10106
10202
  /**
10107
10203
  * The memory type no phase of a sleep cycle touches.
10108
10204
  *
10109
- * A task is live working state, and every one of the sixteen phases is a judgment about REMEMBERED
10205
+ * A task is live working state, and every one of the seventeen phases is a judgment about REMEMBERED
10110
10206
  * FACTS: decay says a claim is fading, dedup says two claims are one, edge typing says one claim
10111
10207
  * caused or contradicts another, retention says a claim has stopped earning its place. None of those hold for
10112
10208
  * a thing an agent intends to do, and each would be wrong applied to one. A task the agent has
@@ -10428,6 +10524,16 @@ const peoplePaths = (db) => db.all("SELECT path FROM files WHERE path LIKE ? OR
10428
10524
  * `edge_class = 'memory'` is the firewall. A person or provenance edge cannot enter PageRank, label
10429
10525
  * propagation, or the retention bridge count, and this query is what makes that true. The CHECK
10430
10526
  * constraint alone does not.
10527
+ *
10528
+ * **The deep grouping band is excluded, and the exclusion is what keeps nightly scoring stable
10529
+ * across deep runs (issue #63).** Deep mining writes machine-mined `laterally_related` edges at a
10530
+ * floor (0.72) far below the nightly one, purely so label propagation can partition the inbox tail
10531
+ * for compress. Those edges persist in the index after the deep run, so without this predicate the
10532
+ * FIRST deep run would permanently change every subsequent nightly run's PageRank, communities, and
10533
+ * bridge counts — and therefore its eviction decisions — on a corpus whose files did not change.
10534
+ * The filter names `derived = 1` as well as the rel, so an AUTHORED `laterally_related` (an agent's
10535
+ * own assertion, addable through `memhtml link`) keeps exactly the graph standing it had.
10536
+ * {@link deepGroupingEdges} is the one reader of the band.
10431
10537
  */
10432
10538
  const memoryEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel, e.dst_path AS dst_path,
10433
10539
  e.strength AS strength, e.derived AS derived
@@ -10435,6 +10541,24 @@ const memoryEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel,
10435
10541
  JOIN files s ON s.path = e.src_path AND s.archived = 0
10436
10542
  JOIN files d ON d.path = e.dst_path AND d.archived = 0
10437
10543
  WHERE e.edge_class = 'memory'
10544
+ AND NOT (e.derived = 1 AND e.rel = 'laterally_related')
10545
+ ORDER BY e.src_path ASC, e.rel ASC, e.dst_path ASC`);
10546
+ /**
10547
+ * The deep grouping band: machine-mined `laterally_related` edges over active files (issue #63).
10548
+ *
10549
+ * The mirror of {@link memoryEdges}' exclusion, and deliberately a separate read instead of a flag on
10550
+ * it: the band has exactly one consumer intent — widening label propagation's partition for the deep
10551
+ * compress and placement phases — and a parameterized `memoryEdges` would let any caller widen the
10552
+ * retention graph by passing a boolean. `provenance = 'sleep'` is implied by `derived = 1` (the
10553
+ * table CHECK pins the pair) and stated anyway so the statement reads as what it is.
10554
+ */
10555
+ const deepGroupingEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel, e.dst_path AS dst_path,
10556
+ e.strength AS strength, e.derived AS derived
10557
+ FROM edges e
10558
+ JOIN files s ON s.path = e.src_path AND s.archived = 0
10559
+ JOIN files d ON d.path = e.dst_path AND d.archived = 0
10560
+ WHERE e.edge_class = 'memory' AND e.derived = 1 AND e.rel = 'laterally_related'
10561
+ AND e.provenance = 'sleep'
10438
10562
  ORDER BY e.src_path ASC, e.rel ASC, e.dst_path ASC`);
10439
10563
  const retentionEdgeCounts = (db) => db.all(`SELECT f.path AS path,
10440
10564
  sum(CASE WHEN e.rel = 'supports' THEN 1 ELSE 0 END) AS reinforcements,
@@ -10639,6 +10763,19 @@ const markSessionsConsolidated = (db, input) => db.writeAll(input.sessionIds.map
10639
10763
  const linkedSessionCount = (db) => db.get(`SELECT count(DISTINCT t.session_id) AS n FROM traces t
10640
10764
  JOIN memory_session_links l ON l.session_id = t.session_id`).pipe(Effect.map((row) => row?.n ?? 0));
10641
10765
  /**
10766
+ * Authored memory-class edges POINTING AT one path: which files hold a `<link>` a move must rewrite.
10767
+ *
10768
+ * Placement triage's read (issue #63). `derived = 0` because a mined edge lives only in the index
10769
+ * and the next re-mine follows the moved vectors on its own; only a file-borne link needs a splice.
10770
+ * Both memory and task classes are included — a task may `blocks`-link a memory it waits on, and a
10771
+ * move that left that href dangling would break the task surface — but provenance and person edges
10772
+ * cannot point at an inbox memory by construction.
10773
+ */
10774
+ const inboundAuthoredEdges = (db, dstPath) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel
10775
+ FROM edges e
10776
+ WHERE e.derived = 0 AND e.dst_path = ?
10777
+ ORDER BY e.src_path ASC, e.rel ASC`, [dstPath]);
10778
+ /**
10642
10779
  * Authored edges pointing at a path the index does not hold.
10643
10780
  *
10644
10781
  * `derived = 0` only: a mined edge lives in the index and nowhere else, so a dangling one is
@@ -10985,6 +11122,129 @@ const arcSynthesis = (env) => Effect.gen(function* () {
10985
11122
  };
10986
11123
  });
10987
11124
 
11125
+ //#endregion
11126
+ //#region packages/sleep/dist/phases/relationship-mining.js
11127
+ /**
11128
+ * Phase 5, relationship mining. Derived `relates_to` edges in the index only. NO COMMIT.
11129
+ *
11130
+ * A mined edge is a re-derivable function of the corpus and the embedder. `index rebuild` plus the
11131
+ * next night's mining regenerates the identical set, so committing thousands of them would bury every
11132
+ * real diff in machine noise for zero recoverable information. The `derived` column is the firewall
11133
+ * that makes losing them cheap. The retention penalty counts only `derived = 0`, so an
11134
+ * uncorroborated machine suspicion cannot evict a memory.
11135
+ *
11136
+ * The insert is scoped to `provenance = 'sleep'` and `derived = 1` and the whole replace is one
11137
+ * atomic batch, so an authored edge is unreachable from here and the corpus is never left with the
11138
+ * old mined set deleted and the new one not yet written. In that window the lateral arm would
11139
+ * silently return nothing.
11140
+ *
11141
+ * **Under `--deep` the phase additionally mines a GROUPING band (issue #63).** On a measured 3,079-file
11142
+ * inbox, 8% of files have a neighbor at the 0.85 floor and 84% touch no edge at all — no community, so
11143
+ * compress is structurally unable to reach them at any frequency. The deep band mines
11144
+ * [{@link DEEP_MINING_COSINE_FLOOR}, {@link MINING_COSINE_FLOOR}) as `laterally_related` edges whose
11145
+ * only consumer intent is giving label propagation a partition; the existing compress model then
11146
+ * judges the folds, and `absorbedKeys: []` is its refusal, so a looser grouping floor costs calls and
11147
+ * never correctness.
11148
+ *
11149
+ * **The two bands are separate rels, and that is the isolation mechanism.** {@link replaceMinedEdges}
11150
+ * scopes its atomic delete by rel, so the deep replace cannot clobber the nightly `relates_to` set and
11151
+ * the nightly replace cannot clobber the deep band. The issue sketched a distinct PROVENANCE instead;
11152
+ * the `edges` CHECKs pin `derived = 1` to `provenance = 'sleep'` (migration 0008), so that spelling
11153
+ * needs a table recreate while a distinct rel needs nothing: `laterally_related` is already in the
11154
+ * memory-rel vocabulary with no producer, and edge typing's candidate scan reads `rel = 'relates_to'`
11155
+ * alone, so the deep band never spends nightly edge-typing calls. A nightly run never touches the
11156
+ * deep rel, so a deep band persists until the next deep run re-mines it or `index rebuild` drops it —
11157
+ * both re-derivable, which is the property that makes an index-only edge safe to hold.
11158
+ */
11159
+ /** The similarity floor a pair must clear to become a mined `relates_to`. */
11160
+ const MINING_COSINE_FLOOR = .85;
11161
+ /**
11162
+ * The GROUPING-band floor deep mining reaches down to (issue #63). At 0.72, between 43% and 61% of
11163
+ * the measured bulk-import inbox has a neighbor (43% at 0.75, 61% at 0.70, median best-neighbor
11164
+ * cosine 0.731) — reach enough to partition most of the tail while staying above the ~0.5-0.6
11165
+ * cross-topic baseline the fixture corpora measure, so a community is still a topic and not noise.
11166
+ * The compress model judging every proposed fold is what makes this floor a cost knob rather than a
11167
+ * correctness one.
11168
+ */
11169
+ const DEEP_MINING_COSINE_FLOOR = .72;
11170
+ /** Nearest neighbors considered per source file. */
11171
+ const MINING_PER_SOURCE_K = 5;
11172
+ /**
11173
+ * Pairs mined per cycle: a cap on what {@link replaceMinedEdges} writes, not on the scan — the
11174
+ * kernel's arithmetic is O(n²·d) whatever this says, and it bounds the edge table so one dense
11175
+ * neighborhood cannot flood the graph the lateral arm and PageRank read.
11176
+ */
11177
+ const MINING_SAMPLE_LIMIT = 2e3;
11178
+ /**
11179
+ * The deep band's own write cap. Separate from {@link MINING_SAMPLE_LIMIT} because the band is wider
11180
+ * by construction — it exists to reach the 84% the nightly band cannot — and sharing the nightly cap
11181
+ * would make the deep run's reach a function of how crowded the nightly band happens to be.
11182
+ */
11183
+ const DEEP_MINING_SAMPLE_LIMIT = 1e4;
11184
+ /** The rel the deep grouping band is written under. The band separator; see the module header. */
11185
+ const DEEP_GROUPING_REL = "laterally_related";
11186
+ /**
11187
+ * Mine every band the run is entitled to and replace the index's mined sets: the whole phase minus
11188
+ * its counts. Exported because deep compress re-runs it BETWEEN passes (issue #63's
11189
+ * iterate-until-quiet): a fold's canonical is a new neighbor only after it is indexed and re-mined,
11190
+ * and a second implementation of the scan here would be free to disagree with the nightly one about
11191
+ * floors, caps, and exclusions.
11192
+ */
11193
+ const mineAllBands = (env) => Effect.gen(function* () {
11194
+ /**
11195
+ * Tasks are excluded, and here the exclusion is the graph firewall, not a cost guard.
11196
+ * Every mined edge is written with `edge_class = 'memory'`, so a pair with a task endpoint
11197
+ * would put a task INTO the memory graph, reaching PageRank, MMR, and the retention bridge
11198
+ * count. The `edges` CHECK cannot refuse it, because `relates_to` under `memory` is a
11199
+ * well-formed edge whatever files sit at its ends.
11200
+ */
11201
+ const pairs = yield* neighborPairs(env.deps.db, {
11202
+ floor: MINING_COSINE_FLOOR,
11203
+ perSourceK: 5,
11204
+ limit: MINING_SAMPLE_LIMIT,
11205
+ excludeTypes: SLEEP_EXCLUDED_TYPES
11206
+ });
11207
+ /**
11208
+ * The deep grouping band: everything in [deep floor, nightly floor). Mined only under `--deep`,
11209
+ * so a nightly run's writes — and therefore the graph every nightly consumer reads on a corpus
11210
+ * that has never run deep — are byte-identical to what they were before this branch existed.
11211
+ * The nightly floor is the band's EXCLUSIVE ceiling: a pair at or above it is the nightly
11212
+ * band's, and holding it in both would double its label-propagation weight.
11213
+ */
11214
+ const deepPairs = env.deep === void 0 ? [] : (yield* neighborPairs(env.deps.db, {
11215
+ floor: DEEP_MINING_COSINE_FLOOR,
11216
+ perSourceK: 5,
11217
+ limit: DEEP_MINING_SAMPLE_LIMIT,
11218
+ excludeTypes: SLEEP_EXCLUDED_TYPES
11219
+ })).filter((pair) => pair.sim < MINING_COSINE_FLOOR);
11220
+ if (!env.dryRun) {
11221
+ yield* replaceMinedEdges(env.deps.db, {
11222
+ runId: env.runId,
11223
+ at: env.at,
11224
+ rel: "relates_to",
11225
+ pairs
11226
+ });
11227
+ if (env.deep !== void 0) yield* replaceMinedEdges(env.deps.db, {
11228
+ runId: env.runId,
11229
+ at: env.at,
11230
+ rel: DEEP_GROUPING_REL,
11231
+ pairs: deepPairs
11232
+ });
11233
+ }
11234
+ return {
11235
+ mined: pairs.length,
11236
+ deepMined: deepPairs.length
11237
+ };
11238
+ });
11239
+ const relationshipMining = (env) => Effect.gen(function* () {
11240
+ const mined = yield* mineAllBands(env);
11241
+ return emptyOutcome({
11242
+ candidates: mined.mined,
11243
+ mined: mined.mined,
11244
+ ...env.deep === void 0 ? {} : { deepMined: mined.deepMined }
11245
+ });
11246
+ });
11247
+
10988
11248
  //#endregion
10989
11249
  //#region packages/sleep/dist/phases/compress.js
10990
11250
  /**
@@ -11015,6 +11275,28 @@ const arcSynthesis = (env) => Effect.gen(function* () {
11015
11275
  *
11016
11276
  * `dedup-merge` is a HARD prerequisite. Compressing before duplicates are folded would synthesize a
11017
11277
  * canonical over a pair the merge phase then archives one half of.
11278
+ *
11279
+ * ## Deep mode (issue #63)
11280
+ *
11281
+ * Under `--deep` three things change about WHICH memories a fold can reach, and nothing about what
11282
+ * happens to a fold:
11283
+ *
11284
+ * 1. **Communities come from the widened graph.** Label propagation runs over the nightly memory
11285
+ * edges PLUS the deep grouping band (`laterally_related`, mined at 0.72), so inbox files whose
11286
+ * best neighbor sits under the nightly 0.85 floor get a partition. Retention SCORES stay the
11287
+ * nightly function — the band decides grouping, never eviction.
11288
+ * 2. **Entity groups are a second community source.** Candidates the widened graph still leaves
11289
+ * communityless are grouped by shared `file_entities` reference under a synthetic
11290
+ * `entity:<type>:<name>` label, because two memories about one subject can share no vocabulary at
11291
+ * all. Hub entities (more than {@link DEEP_ENTITY_HUB_LIMIT} active claimants) are stop-words and
11292
+ * are skipped.
11293
+ * 3. **The phase iterates until quiet.** A pass that produced canonicals re-indexes the branch,
11294
+ * re-mines both bands, recomputes retention, and folds again — a canonical is a new neighbor and
11295
+ * a new community member — until a pass folds nothing or {@link DEEP_COMPRESS_MAX_PASSES} hits.
11296
+ *
11297
+ * Every model call in deep mode is charged against the run's shared `--max-llm-calls` budget first;
11298
+ * an exhausted budget skips the batch with reason `budget` and the run stays green, the same
11299
+ * degradation posture a model outage has.
11018
11300
  */
11019
11301
  /** Members per model call. Small enough that every member's facts fit the answer's attention. */
11020
11302
  const COMPRESS_BATCH_SIZE = 8;
@@ -11027,6 +11309,83 @@ const COMPRESS_MIN_BATCH = 2;
11027
11309
  const COMPRESS_CANDIDATE_LIMIT = 2e3;
11028
11310
  /** Characters of each member shown. A fold must see the facts, so this is wider than arc evidence. */
11029
11311
  const COMPRESS_MEMBER_CHARS = 1200;
11312
+ /**
11313
+ * Deep compress passes before the loop stops regardless of yield (issue #63). Each extra pass costs
11314
+ * a full re-index, a re-mine, and another round of model calls, and the third pass's yield on any
11315
+ * real corpus is folds of folds — past that the loop is spending calls to rename its own output.
11316
+ */
11317
+ const DEEP_COMPRESS_MAX_PASSES = 3;
11318
+ /**
11319
+ * Active files an entity may be claimed by before deep grouping treats it as a stop-word
11320
+ * (issue #63). An entity on sixty-plus files ("service:api", a person who appears everywhere) says
11321
+ * which corpus this is, not which memories are one topic, and batching its claimants would fold
11322
+ * unrelated facts for sharing a byline. Groups under the limit still slice to
11323
+ * {@link COMPRESS_BATCH_SIZE} through the same kernel as every other group.
11324
+ */
11325
+ const DEEP_ENTITY_HUB_LIMIT = 64;
11326
+ /** The synthetic community-label prefix entity groups use. No git path starts with this. */
11327
+ const ENTITY_LABEL_PREFIX = "entity:";
11328
+ /**
11329
+ * The community label of every active path under the WIDENED graph: nightly memory edges plus the
11330
+ * deep grouping band, one label-propagation pass (issue #63).
11331
+ *
11332
+ * Computed here rather than inside `runRetentionPass`, deliberately: retention's partition feeds
11333
+ * eviction scoring and must not move when a deep band is present in the index, so the widened
11334
+ * partition is a second, compress-only computation over reads this module already owns.
11335
+ */
11336
+ const deepCommunityLabels = (env) => Effect.gen(function* () {
11337
+ const corpus = yield* activeCorpus(env.deps.db);
11338
+ const nightly = yield* memoryEdges(env.deps.db);
11339
+ const band = yield* deepGroupingEdges(env.deps.db);
11340
+ return labelPropagation(corpus.map((row) => row.path), [...nightly, ...band].map((edge) => ({
11341
+ src: edge.src_path,
11342
+ dst: edge.dst_path,
11343
+ strength: edge.strength
11344
+ })));
11345
+ }).pipe(Effect.orElseSucceed(() => /* @__PURE__ */ new Map()));
11346
+ /**
11347
+ * Assign an `entity:<type>:<name>` label to every candidate the graph left communityless, greedily,
11348
+ * one label per file (issue #63). PURE, so the hub cap and the one-label rule are unit-assertable
11349
+ * without a corpus.
11350
+ *
11351
+ * Entities are walked in lexicographic (type, name) order and each claims its unclaimed candidates,
11352
+ * so the assignment is a pure function of the corpus: a file naming two entities lands with the
11353
+ * lexicographically first, and the run makes the same batches twice over. A group needs two members
11354
+ * to mean anything, matching {@link COMPRESS_MIN_BATCH}.
11355
+ *
11356
+ * **The hub cap reads the entity's ACTIVE claimant count, not its needing count.** An entity on a
11357
+ * hundred files whose tail happens to leave only three in the inbox band is still a stop-word:
11358
+ * what those three share is a byline, not a topic.
11359
+ */
11360
+ const assignEntityLabels = (claims, needing) => {
11361
+ const byEntity = /* @__PURE__ */ new Map();
11362
+ for (const claim of claims) {
11363
+ const key = `${claim.entity_type}:${claim.entity_name}`;
11364
+ const bucket = byEntity.get(key);
11365
+ if (bucket === void 0) byEntity.set(key, [claim.path]);
11366
+ else bucket.push(claim.path);
11367
+ }
11368
+ const labels = /* @__PURE__ */ new Map();
11369
+ let hubsSkipped = 0;
11370
+ for (const [key, paths] of [...byEntity.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
11371
+ if (paths.length > 64) {
11372
+ hubsSkipped += 1;
11373
+ continue;
11374
+ }
11375
+ const members = paths.filter((path) => needing.has(path) && !labels.has(path));
11376
+ if (members.length < 2) continue;
11377
+ for (const member of members) labels.set(member, `${ENTITY_LABEL_PREFIX}${key}`);
11378
+ }
11379
+ return {
11380
+ labels,
11381
+ hubsSkipped
11382
+ };
11383
+ };
11384
+ /** {@link assignEntityLabels} over the index's own claim rows. */
11385
+ const entityLabelsFor = (env, needing) => entityClaims(env.deps.db).pipe(Effect.map((claims) => assignEntityLabels(claims, needing)), Effect.orElseSucceed(() => ({
11386
+ labels: /* @__PURE__ */ new Map(),
11387
+ hubsSkipped: 0
11388
+ })));
11030
11389
  const compress = (env) => Effect.gen(function* () {
11031
11390
  const model = env.deps.model;
11032
11391
  if (model === void 0) return {
@@ -11037,143 +11396,187 @@ const compress = (env) => Effect.gen(function* () {
11037
11396
  }),
11038
11397
  detail: "no model bound"
11039
11398
  };
11040
- const candidates = (yield* runRetentionPass(env.deps.db, env.at)).scored.filter((entry) => entry.score.action === "compress" && entry.row.memory_type !== "arc" && !isSleepExcluded(entry.row.memory_type) && entry.community !== void 0).slice(0, COMPRESS_CANDIDATE_LIMIT);
11041
- /** Community -> its COMPRESS-band members, both orders fixed so batching is reproducible. */
11042
- const byCommunity = /* @__PURE__ */ new Map();
11043
- for (const entry of candidates) {
11044
- const label = entry.community;
11045
- if (label === void 0) continue;
11046
- const bucket = byCommunity.get(label);
11047
- if (bucket === void 0) byCommunity.set(label, [entry]);
11048
- else bucket.push(entry);
11049
- }
11050
- /**
11051
- * Both sorts are this phase's, and the kernel keeps the order they produce. Communities are
11052
- * walked lexicographically by label so a night's call order is fixed, and each community's members
11053
- * by `row.path` so the `m1`..`mN` keys land on the same files twice over.
11054
- */
11055
- const groups = [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1).map(([, members]) => [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0));
11056
- const batches = assembleBatches(groups, {
11057
- maxMembers: 8,
11058
- minMembers: 2
11059
- });
11060
- const counts = {
11061
- candidates: candidates.length,
11062
- communities: byCommunity.size,
11063
- batches: batches.length,
11064
- canonicals: 0,
11065
- archived: 0,
11066
- skipped: 0,
11067
- failed: 0,
11068
- refused: 0
11069
- };
11070
- if (batches.length === 0) return emptyOutcome(counts);
11071
- if (env.dryRun) return emptyOutcome(counts);
11072
11399
  const modelKey = modelFor(env.deps, "compress");
11073
- let llmCalls = 0;
11074
- let canonicals = 0;
11075
- let archived = 0;
11076
- /**
11077
- * `skipped` stays the total, and `failed` + `refused` partition it. The two are different
11078
- * diagnoses with different fixes — a failed call is the model or the wire (already logged by
11079
- * `isolate`), a refusal is an answer the phase declined to act on (logged below) — and a night
11080
- * reporting only their sum cannot say which one it had. 47 of 47 batches once skipped as
11081
- * refusals with nothing on stderr, and the sum read as flaky calls.
11082
- */
11083
- let skipped = 0;
11084
- let failed = 0;
11085
- let refused = 0;
11400
+ const passes = [];
11086
11401
  let lastCommit = null;
11087
- for (const batch of batches) {
11088
- /** Opaque keys again, so `absorbedKeys` cannot name a path. */
11089
- const keyed = keyMembers(batch, (entry) => `${entry.row.title}\n${entry.row.gist}\n${entry.row.body_text}`, { charBudget: COMPRESS_MEMBER_CHARS });
11090
- llmCalls += 1;
11091
- const synthesis = yield* batchCall(model, `compress batch of ${batch.length}`, {
11092
- schema: CompressSynthesis,
11093
- system: COMPRESS_SYSTEM,
11094
- prompt: compressPrompt(keyed.keyed),
11095
- modelKey,
11096
- effort: "high",
11097
- toolDescription: "Emit the canonical memory and the members whose content it absorbs."
11098
- });
11099
- if (synthesis === void 0) {
11100
- skipped += 1;
11101
- failed += 1;
11102
- continue;
11103
- }
11104
- /** A key the batch never offered resolves to nothing, so a fold reaches only offered files. */
11105
- const absorbed = resolveKeys(keyed, synthesis.absorbedKeys).map((entry) => entry.row.path);
11106
- if (absorbed.length < 2 || synthesis.title.trim() === "" || synthesis.claim.trim() === "") {
11107
- skipped += 1;
11108
- refused += 1;
11109
- yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: the model absorbed ${absorbed.length} of ${synthesis.absorbedKeys.length} named keys` + (synthesis.absorbedKeys.length > 0 && absorbed.length === 0 ? ` (none of the named keys resolved: ${synthesis.absorbedKeys.slice(0, 3).join(", ")}${synthesis.absorbedKeys.length > 3 ? ", …" : ""})` : ""));
11110
- continue;
11111
- }
11402
+ for (let passAt = 0; passAt < (env.deep === void 0 ? 1 : 3);) {
11403
+ passAt += 1;
11404
+ const banded = (yield* runRetentionPass(env.deps.db, env.at)).scored.filter((entry) => entry.score.action === "compress" && entry.row.memory_type !== "arc" && !isSleepExcluded(entry.row.memory_type));
11112
11405
  /**
11113
- * The canonical is placed in the batch's own directory when the members agree on one, and in the
11114
- * inbox otherwise. Placing it under a member's directory keeps a compressed group where a reader
11115
- * would look for it, and `memhtml doctor` reports inbox depth so a disagreeing batch is visible.
11406
+ * Which label a candidate folds under. Nightly: the retention pass's own community, and a
11407
+ * memory without one is not a candidate the exact selection this phase has always made.
11408
+ * Deep: the widened partition first, then an entity label for what the graph still missed.
11116
11409
  */
11117
- const directories = new Set(absorbed.map((path) => path.slice(0, path.lastIndexOf("/"))));
11118
- const canonicalPath = `${(directories.size === 1 ? [...directories][0] : "areas/inbox") ?? "areas/inbox"}/${slugify(synthesis.title)}.html`;
11119
- const members = excludeSelfSupersede(canonicalPath, absorbed);
11120
- if (members.length === 0) {
11121
- skipped += 1;
11122
- refused += 1;
11123
- yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every absorbed member was the canonical itself`);
11124
- continue;
11410
+ const widened = env.deep === void 0 ? void 0 : yield* deepCommunityLabels(env);
11411
+ const communityOf = (entry) => widened === void 0 ? entry.community : widened.get(entry.row.path);
11412
+ const graphLabelled = banded.filter((entry) => communityOf(entry) !== void 0);
11413
+ const needing = env.deep === void 0 ? /* @__PURE__ */ new Set() : new Set(banded.filter((entry) => communityOf(entry) === void 0).map((entry) => entry.row.path));
11414
+ const entities = env.deep === void 0 ? {
11415
+ labels: /* @__PURE__ */ new Map(),
11416
+ hubsSkipped: 0
11417
+ } : yield* entityLabelsFor(env, needing);
11418
+ const candidates = [...graphLabelled, ...banded.filter((entry) => entities.labels.has(entry.row.path))].slice(0, COMPRESS_CANDIDATE_LIMIT);
11419
+ /** Community -> its COMPRESS-band members, both orders fixed so batching is reproducible. */
11420
+ const byCommunity = /* @__PURE__ */ new Map();
11421
+ for (const entry of candidates) {
11422
+ const label = communityOf(entry) ?? entities.labels.get(entry.row.path);
11423
+ if (label === void 0) continue;
11424
+ const bucket = byCommunity.get(label);
11425
+ if (bucket === void 0) byCommunity.set(label, [entry]);
11426
+ else bucket.push(entry);
11125
11427
  }
11126
11428
  /**
11127
- * The members are archived FIRST, and the canonical is written only if at least one member was
11128
- * actually moved. A batch whose members an earlier phase already evicted would otherwise leave a
11129
- * canonical behind claiming to supersede files it never absorbed.
11429
+ * Both sorts are this phase's, and the kernel keeps the order they produce. Communities are
11430
+ * walked lexicographically by label so a night's call order is fixed, and each community's members
11431
+ * by `row.path` so the `m1`..`mN` keys land on the same files twice over.
11130
11432
  */
11131
- const archivedPaths = [];
11132
- for (const member of members) {
11133
- const archivedPath = yield* archiveFile(env, member, [meta("memhtml-superseded-by", hrefFor(canonicalPath))]);
11134
- if (archivedPath !== null) archivedPaths.push(archivedPath);
11135
- }
11136
- if (archivedPaths.length === 0) {
11137
- skipped += 1;
11138
- refused += 1;
11139
- yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every member was already gone from the tree`);
11140
- continue;
11141
- }
11142
- yield* writeFileBytes(env, canonicalPath, renderTemplate({
11143
- title: synthesis.title.trim(),
11144
- claim: synthesis.claim,
11145
- body: synthesis.paragraphs,
11146
- memoryType: "semantic",
11147
- at: env.at,
11148
- author: "agent:sleep"
11149
- }));
11150
- for (const archivedPath of archivedPaths) yield* stampFile(env, canonicalPath, [link("supersedes", hrefFor(archivedPath))]);
11151
- yield* env.deps.git.add([canonicalPath]);
11152
- archived += archivedPaths.length;
11153
- canonicals += 1;
11154
- const commitSha = yield* commitPhase(env, "compress", `fold ${members.length} memories into ${synthesis.title}`, {
11155
- ...counts,
11156
- canonicals,
11157
- archived,
11158
- skipped,
11159
- failed,
11160
- refused
11433
+ const groups = [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1).map(([, members]) => [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0));
11434
+ const batches = assembleBatches(groups, {
11435
+ maxMembers: 8,
11436
+ minMembers: 2
11161
11437
  });
11162
- if (commitSha !== null) lastCommit = commitSha;
11438
+ const tally = {
11439
+ candidates: candidates.length,
11440
+ communities: byCommunity.size,
11441
+ entityGroups: new Set(entities.labels.values()).size,
11442
+ batches: batches.length,
11443
+ canonicals: 0,
11444
+ archived: 0,
11445
+ skipped: 0,
11446
+ failed: 0,
11447
+ refused: 0,
11448
+ budget: 0,
11449
+ llmCalls: 0
11450
+ };
11451
+ passes.push(tally);
11452
+ if (batches.length === 0 || env.dryRun) break;
11453
+ for (const batch of batches) {
11454
+ /** Opaque keys again, so `absorbedKeys` cannot name a path. */
11455
+ const keyed = keyMembers(batch, (entry) => `${entry.row.title}\n${entry.row.gist}\n${entry.row.body_text}`, { charBudget: COMPRESS_MEMBER_CHARS });
11456
+ /**
11457
+ * The deep budget is charged BEFORE the call, and an exhausted budget is its own skip
11458
+ * reason: a night that stopped because the operator's cap ran out and a night whose model
11459
+ * fell over need different mornings-after, and `skipped` alone cannot say which this was.
11460
+ */
11461
+ if (!takeLlmCall(env.deep)) {
11462
+ tally.skipped += 1;
11463
+ tally.budget += 1;
11464
+ continue;
11465
+ }
11466
+ tally.llmCalls += 1;
11467
+ const synthesis = yield* batchCall(model, `compress batch of ${batch.length}`, {
11468
+ schema: CompressSynthesis,
11469
+ system: COMPRESS_SYSTEM,
11470
+ prompt: compressPrompt(keyed.keyed),
11471
+ modelKey,
11472
+ effort: "high",
11473
+ toolDescription: "Emit the canonical memory and the members whose content it absorbs."
11474
+ });
11475
+ if (synthesis === void 0) {
11476
+ tally.skipped += 1;
11477
+ tally.failed += 1;
11478
+ continue;
11479
+ }
11480
+ /** A key the batch never offered resolves to nothing, so a fold reaches only offered files. */
11481
+ const absorbed = resolveKeys(keyed, synthesis.absorbedKeys).map((entry) => entry.row.path);
11482
+ if (absorbed.length < 2 || synthesis.title.trim() === "" || synthesis.claim.trim() === "") {
11483
+ tally.skipped += 1;
11484
+ tally.refused += 1;
11485
+ yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: the model absorbed ${absorbed.length} of ${synthesis.absorbedKeys.length} named keys` + (synthesis.absorbedKeys.length > 0 && absorbed.length === 0 ? ` (none of the named keys resolved: ${synthesis.absorbedKeys.slice(0, 3).join(", ")}${synthesis.absorbedKeys.length > 3 ? ", …" : ""})` : ""));
11486
+ continue;
11487
+ }
11488
+ /**
11489
+ * The canonical is placed in the batch's own directory when the members agree on one, and in the
11490
+ * inbox otherwise. Placing it under a member's directory keeps a compressed group where a reader
11491
+ * would look for it, and `memhtml doctor` reports inbox depth so a disagreeing batch is visible.
11492
+ */
11493
+ const directories = new Set(absorbed.map((path) => path.slice(0, path.lastIndexOf("/"))));
11494
+ const canonicalPath = `${(directories.size === 1 ? [...directories][0] : "areas/inbox") ?? "areas/inbox"}/${slugify(synthesis.title)}.html`;
11495
+ const members = excludeSelfSupersede(canonicalPath, absorbed);
11496
+ if (members.length === 0) {
11497
+ tally.skipped += 1;
11498
+ tally.refused += 1;
11499
+ yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every absorbed member was the canonical itself`);
11500
+ continue;
11501
+ }
11502
+ /**
11503
+ * The members are archived FIRST, and the canonical is written only if at least one member was
11504
+ * actually moved. A batch whose members an earlier phase already evicted would otherwise leave a
11505
+ * canonical behind claiming to supersede files it never absorbed.
11506
+ */
11507
+ const archivedPaths = [];
11508
+ for (const member of members) {
11509
+ const archivedPath = yield* archiveFile(env, member, [meta("memhtml-superseded-by", hrefFor(canonicalPath))]);
11510
+ if (archivedPath !== null) archivedPaths.push(archivedPath);
11511
+ }
11512
+ if (archivedPaths.length === 0) {
11513
+ tally.skipped += 1;
11514
+ tally.refused += 1;
11515
+ yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every member was already gone from the tree`);
11516
+ continue;
11517
+ }
11518
+ yield* writeFileBytes(env, canonicalPath, renderTemplate({
11519
+ title: synthesis.title.trim(),
11520
+ claim: synthesis.claim,
11521
+ body: synthesis.paragraphs,
11522
+ memoryType: "semantic",
11523
+ at: env.at,
11524
+ author: "agent:sleep"
11525
+ }));
11526
+ for (const archivedPath of archivedPaths) yield* stampFile(env, canonicalPath, [link("supersedes", hrefFor(archivedPath))]);
11527
+ yield* env.deps.git.add([canonicalPath]);
11528
+ tally.archived += archivedPaths.length;
11529
+ tally.canonicals += 1;
11530
+ const commitSha = yield* commitPhase(env, "compress", `fold ${members.length} memories into ${synthesis.title}`, totalsOf(passes));
11531
+ if (commitSha !== null) lastCommit = commitSha;
11532
+ }
11533
+ /**
11534
+ * Iterate-until-quiet, deep only (issue #63). A pass that folded nothing has reached the
11535
+ * fixed point; one that folded something changed the neighbor structure, so the branch is
11536
+ * re-indexed (renames tracked, new canonicals embedded), both bands are re-mined over the new
11537
+ * vectors, and the next pass sees the canonicals as members. The nightly cycle exits here
11538
+ * unconditionally, which is what keeps its behavior single-pass and byte-identical.
11539
+ */
11540
+ if (env.deep === void 0 || tally.canonicals === 0) break;
11541
+ if (passAt >= 3) break;
11542
+ yield* env.deps.indexer.update({ embed: true });
11543
+ yield* mineAllBands(env);
11163
11544
  }
11164
11545
  return {
11165
- counts: {
11166
- ...counts,
11167
- canonicals,
11168
- archived,
11169
- skipped,
11170
- failed,
11171
- refused
11172
- },
11546
+ counts: totalsOf(passes),
11173
11547
  commitSha: lastCommit,
11174
- llmCalls
11548
+ llmCalls: passes.reduce((total, tally) => total + tally.llmCalls, 0)
11175
11549
  };
11176
11550
  });
11551
+ /**
11552
+ * The pass tallies as one counts record. Work counters SUM across passes; `candidates` and
11553
+ * `communities` describe the corpus the run STARTED from (the first pass), because a sum of
11554
+ * re-scans of one shrinking corpus counts nothing a reviewer can reconcile. On a nightly
11555
+ * single-pass run every key and every number is byte-identical to what this phase has always
11556
+ * reported — the deep-only keys appear only when a deep quantity is nonzero or a second pass ran,
11557
+ * which cannot happen without the flag. Response counts are append-only, so the deep keys are
11558
+ * additions and no shipped key changes meaning.
11559
+ */
11560
+ const totalsOf = (passes) => {
11561
+ const first = passes[0];
11562
+ const sum = (of) => passes.reduce((total, tally) => total + of(tally), 0);
11563
+ const deepish = passes.length > 1 || (first?.entityGroups ?? 0) > 0 || sum((tally) => tally.budget) > 0;
11564
+ return {
11565
+ candidates: first?.candidates ?? 0,
11566
+ communities: first?.communities ?? 0,
11567
+ batches: sum((tally) => tally.batches),
11568
+ canonicals: sum((tally) => tally.canonicals),
11569
+ archived: sum((tally) => tally.archived),
11570
+ skipped: sum((tally) => tally.skipped),
11571
+ failed: sum((tally) => tally.failed),
11572
+ refused: sum((tally) => tally.refused),
11573
+ ...deepish ? {
11574
+ passes: passes.length,
11575
+ entityGroups: first?.entityGroups ?? 0,
11576
+ budgetSkipped: sum((tally) => tally.budget)
11577
+ } : {}
11578
+ };
11579
+ };
11177
11580
 
11178
11581
  //#endregion
11179
11582
  //#region packages/sleep/dist/phases/confidence-decay.js
@@ -14129,6 +14532,228 @@ const personLinks = (env) => Effect.gen(function* () {
14129
14532
  };
14130
14533
  });
14131
14534
 
14535
+ //#endregion
14536
+ //#region packages/sleep/dist/phases/placement-triage.js
14537
+ /**
14538
+ * Phase 14, placement triage. DEEP-ONLY (issue #63): propose a topic directory for each inbox
14539
+ * memory that even deep grouping left communityless, and `git mv` the ones the model places with
14540
+ * confidence. ONE COMMIT.
14541
+ *
14542
+ * The mechanism this phase exists for: a bulk-imported inbox is mostly NOT compressible — each file
14543
+ * is a distinct fact — and what keeps the inbox crowded is that nothing ever re-files them. `doctor`
14544
+ * reports `inboxCrowded` and no phase acts on it. This one does, under the same discipline as every
14545
+ * other mutation: on the review branch, in its own commit, reversible by discarding the branch.
14546
+ *
14547
+ * **On a nightly run the phase returns immediately**, before any read, with a reason — the same
14548
+ * shape `compress` has for a missing model. That is the whole no-flag contract: a run without
14549
+ * `--deep` cannot reach a single line of this phase's work.
14550
+ *
14551
+ * The deterministic guardrails, each enforced in code after the model answers:
14552
+ *
14553
+ * - a destination must be an EXISTING directory (as the index knows them) or one of at most
14554
+ * {@link PLACEMENT_NEW_DIR_CAP} new `areas/<topic>` or `resources/<topic>` directories per run —
14555
+ * first proposed, first minted, lexicographic order within one batch;
14556
+ * - the managed surfaces refuse: the inbox itself, `areas/arcs`, `resources/people`, anything under
14557
+ * `archive/`, and any path outside the PARA buckets;
14558
+ * - tasks never move (excluded from the scan AND re-checked per row, matching the double guard
14559
+ * task-detection carries);
14560
+ * - a file another phase already touched this run never moves, read from `git diff --name-only
14561
+ * base..HEAD` — a `mv` of a path compress just archived would fail, and one of a path an earlier
14562
+ * phase stamped would tear that phase's edit out of its own commit's diff;
14563
+ * - `keep-inbox`, an unknown key, a below-floor confidence, and an omitted member all leave the
14564
+ * file where it is.
14565
+ *
14566
+ * **Inbound hrefs are rewritten in the same commit.** Integrity's dangling-href repair chases a
14567
+ * target into the ARCHIVE by deriving `archivePathFor`; a placement move is not an archive, so this
14568
+ * phase rewrites `<link>` elements in the files that point at each moved path itself, the same
14569
+ * remove-then-add splice integrity uses. The indexer tracks the rename (`diff -M` + `movePath`), so
14570
+ * chunks and embeddings carry over and nothing re-embeds.
14571
+ */
14572
+ /** Inbox memories offered per model call. Placement is a lighter question than a fold, so wider. */
14573
+ const PLACEMENT_BATCH_SIZE = 16;
14574
+ /** Inbox memories considered per run. The model-cost guard, in the spirit of compress's 2000. */
14575
+ const PLACEMENT_CANDIDATE_LIMIT = 2e3;
14576
+ /** Characters of each member shown. Placement needs the topic, not every fact. */
14577
+ const PLACEMENT_MEMBER_CHARS = 600;
14578
+ /**
14579
+ * New topic directories one run may mint. Small on purpose: a new directory is a new place every
14580
+ * future reader and `placementFor` caller has to know about, and a bulk import that genuinely needs
14581
+ * thirty new topics should earn them over several reviewed runs, not one.
14582
+ */
14583
+ const PLACEMENT_NEW_DIR_CAP = 5;
14584
+ const placementTriage = (env) => Effect.gen(function* () {
14585
+ if (env.deep === void 0) return {
14586
+ ...emptyOutcome({ candidates: 0 }),
14587
+ detail: "deep-only phase; run with --deep"
14588
+ };
14589
+ const model = env.deps.model;
14590
+ if (model === void 0) return {
14591
+ ...emptyOutcome({ candidates: 0 }),
14592
+ detail: "no model bound"
14593
+ };
14594
+ /**
14595
+ * Candidates: active inbox memories with no community even under the WIDENED graph — nightly
14596
+ * edges plus the deep grouping band, the same partition deep compress groups by. Issue #63's
14597
+ * "true singleton" is exactly this: a file even the 0.72 band could not attach to anything.
14598
+ * A file the widened graph DID reach belongs to compress's pipeline (this run or a later one),
14599
+ * not to placement; an ENTITY-grouped file is deliberately still placeable, because entity
14600
+ * grouping is a compress-band mechanism and a KEEP-band inbox file it cannot fold still needs
14601
+ * a home.
14602
+ */
14603
+ const widened = yield* deepCommunityLabels(env);
14604
+ const corpus = yield* activeCorpus(env.deps.db);
14605
+ const inInbox = (path) => path.startsWith(`${"areas/inbox"}/`) && !path.startsWith(`${"areas/inbox"}/tasks/`);
14606
+ const candidates = corpus.filter((row) => inInbox(row.path) && !isSleepExcluded(row.memory_type) && row.memory_type !== "arc" && widened.get(row.path) === void 0).sort((left, right) => left.path < right.path ? -1 : 1).slice(0, PLACEMENT_CANDIDATE_LIMIT);
14607
+ /**
14608
+ * The directories the corpus already has, from the index's own path set: every distinct
14609
+ * directory holding an active file under `areas/` or `resources/`, minus the managed surfaces.
14610
+ * Offered to the model verbatim and used as the validation set, so the question and the gate
14611
+ * cannot disagree.
14612
+ */
14613
+ const managed = /* @__PURE__ */ new Set([
14614
+ INBOX_DIR,
14615
+ `${INBOX_DIR}/tasks`,
14616
+ ARCS_DIR,
14617
+ PEOPLE_DIR
14618
+ ]);
14619
+ const existingDirs = [...new Set(corpus.map((row) => row.path.slice(0, row.path.lastIndexOf("/"))).filter((dir) => (dir.startsWith("areas/") || dir.startsWith("resources/")) && !managed.has(dir)))].sort();
14620
+ /** Paths this run already touched. A move of one would cross-contaminate another phase's diff. */
14621
+ const touched = new Set(env.baseSha === "" ? [] : yield* env.deps.git.run([
14622
+ "diff",
14623
+ "--name-only",
14624
+ `${env.baseSha}..HEAD`
14625
+ ]).pipe(Effect.map((out) => out.split("\n").map((line) => line.trim()).filter(Boolean)), Effect.orElseSucceed(() => [])));
14626
+ const batches = assembleBatches([candidates], { maxMembers: 16 });
14627
+ const counts = {
14628
+ candidates: candidates.length,
14629
+ batches: batches.length,
14630
+ proposed: 0,
14631
+ applied: 0,
14632
+ refused: 0,
14633
+ keptInbox: 0,
14634
+ newDirs: 0,
14635
+ budgetSkipped: 0,
14636
+ failed: 0
14637
+ };
14638
+ if (batches.length === 0 || env.dryRun) return emptyOutcome(counts);
14639
+ const modelKey = modelFor(env.deps, "placement-triage");
14640
+ const mintedDirs = /* @__PURE__ */ new Set();
14641
+ const movedFrom = /* @__PURE__ */ new Map();
14642
+ let llmCalls = 0;
14643
+ for (const batch of batches) {
14644
+ const keyed = keyMembers(batch, (row) => `${row.title}\n${row.gist}\n${row.body_text}`, { charBudget: 600 });
14645
+ if (!takeLlmCall(env.deep)) {
14646
+ counts["budgetSkipped"] = (counts["budgetSkipped"] ?? 0) + 1;
14647
+ continue;
14648
+ }
14649
+ llmCalls += 1;
14650
+ const answer = yield* batchCall(model, `placement batch of ${batch.length}`, {
14651
+ schema: PlacementTriage,
14652
+ system: PLACEMENT_SYSTEM,
14653
+ prompt: placementPrompt(existingDirs, keyed.keyed),
14654
+ modelKey,
14655
+ effort: "high",
14656
+ toolDescription: "Propose a destination directory (or keep-inbox) per offered memory."
14657
+ });
14658
+ if (answer === void 0) {
14659
+ counts["failed"] = (counts["failed"] ?? 0) + 1;
14660
+ continue;
14661
+ }
14662
+ /**
14663
+ * Placements resolve through the kernel, so an invented key drops. Sorted by (destination,
14664
+ * key) so within one answer the minting order of new directories is a function of the answer's
14665
+ * CONTENT rather than of its field order.
14666
+ */
14667
+ const resolvedRows = answer.placements.filter((placement) => placement.destination.trim() !== "").sort((left, right) => left.destination < right.destination ? -1 : left.destination > right.destination ? 1 : left.memberKey < right.memberKey ? -1 : 1);
14668
+ for (const placement of resolvedRows) {
14669
+ const row = resolveKeys(keyed, [placement.memberKey])[0];
14670
+ if (row === void 0) continue;
14671
+ counts["proposed"] = (counts["proposed"] ?? 0) + 1;
14672
+ const destination = normalizePath(placement.destination.trim());
14673
+ if (destination === "keep-inbox") {
14674
+ counts["keptInbox"] = (counts["keptInbox"] ?? 0) + 1;
14675
+ continue;
14676
+ }
14677
+ const refuse = (reason) => {
14678
+ counts["refused"] = (counts["refused"] ?? 0) + 1;
14679
+ return Effect.logWarning(`sleep.placement refused ${row.path} -> ${destination}: ${reason}`);
14680
+ };
14681
+ if (placement.confidence < .7) {
14682
+ yield* refuse("below the confidence floor");
14683
+ continue;
14684
+ }
14685
+ const bucket = paraBucketOf(destination);
14686
+ if (bucket !== "areas" && bucket !== "resources" || managed.has(destination) || destination === row.path.slice(0, row.path.lastIndexOf("/")) || destination.split("/").some((segment) => segment === "" || segment === "." || segment === "..")) {
14687
+ yield* refuse("not a placeable directory");
14688
+ continue;
14689
+ }
14690
+ if (isSleepExcluded(row.memory_type)) {
14691
+ yield* refuse("tasks never move");
14692
+ continue;
14693
+ }
14694
+ if (touched.has(row.path) || movedFrom.has(row.path)) {
14695
+ yield* refuse("another phase touched this file this run");
14696
+ continue;
14697
+ }
14698
+ const isNew = !existingDirs.includes(destination) && !mintedDirs.has(destination);
14699
+ if (isNew && mintedDirs.size >= 5) {
14700
+ yield* refuse("the new-directory cap for this run is spent");
14701
+ continue;
14702
+ }
14703
+ const target = `${destination}/${row.path.slice(row.path.lastIndexOf("/") + 1)}`;
14704
+ if (!isValidMemoryPath(target)) {
14705
+ yield* refuse("the destination path is not a valid memory path");
14706
+ continue;
14707
+ }
14708
+ if ((yield* readFileBytes(env, target)) !== void 0) {
14709
+ yield* refuse("the destination already holds a file by that name");
14710
+ continue;
14711
+ }
14712
+ if ((yield* readFileBytes(env, row.path)) === void 0) {
14713
+ yield* refuse("the source file is already gone from the tree");
14714
+ continue;
14715
+ }
14716
+ yield* attemptIo(`sleep.placement.mkdir:${target}`, async () => {
14717
+ const { mkdir } = await import("node:fs/promises");
14718
+ const { dirname } = await import("node:path");
14719
+ await mkdir(dirname(absoluteIn(env, target)), { recursive: true });
14720
+ });
14721
+ yield* env.deps.git.mv(row.path, target);
14722
+ yield* stampFile(env, target, [meta("memhtml-updated", env.at)]);
14723
+ if (isNew) {
14724
+ mintedDirs.add(destination);
14725
+ counts["newDirs"] = (counts["newDirs"] ?? 0) + 1;
14726
+ }
14727
+ movedFrom.set(row.path, target);
14728
+ counts["applied"] = (counts["applied"] ?? 0) + 1;
14729
+ }
14730
+ }
14731
+ /**
14732
+ * Inbound href repair, inside the same commit as the moves. Every authored `<link>` whose
14733
+ * target moved is respliced in ITS OWN file to the new path — remove-then-add, integrity's
14734
+ * idempotent shape — so the branch never carries a commit whose links dangle by construction.
14735
+ */
14736
+ let rewritten = 0;
14737
+ for (const [from, to] of [...movedFrom.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
14738
+ const inbound = yield* inboundAuthoredEdges(env.deps.db, from);
14739
+ for (const edge of inbound) {
14740
+ if (!isEdgeRel(edge.rel)) continue;
14741
+ const holder = movedFrom.get(edge.src_path) ?? edge.src_path;
14742
+ if (yield* stampFile(env, holder, [
14743
+ unlink(edge.rel, hrefFor(from)),
14744
+ link(edge.rel, hrefFor(to)),
14745
+ meta("memhtml-updated", env.at)
14746
+ ])) rewritten += 1;
14747
+ }
14748
+ }
14749
+ counts["hrefsRewritten"] = rewritten;
14750
+ return {
14751
+ counts,
14752
+ commitSha: yield* commitPhase(env, "placement-triage", `re-file ${counts["applied"] ?? 0} inbox memories into topic directories`, counts),
14753
+ llmCalls
14754
+ };
14755
+ });
14756
+
14132
14757
  //#endregion
14133
14758
  //#region packages/sleep/dist/phases/preflight.js
14134
14759
  /**
@@ -14165,60 +14790,6 @@ const preflight = (env) => Effect.gen(function* () {
14165
14790
  }) };
14166
14791
  });
14167
14792
 
14168
- //#endregion
14169
- //#region packages/sleep/dist/phases/relationship-mining.js
14170
- /**
14171
- * Phase 5, relationship mining. Derived `relates_to` edges in the index only. NO COMMIT.
14172
- *
14173
- * A mined edge is a re-derivable function of the corpus and the embedder. `index rebuild` plus the
14174
- * next night's mining regenerates the identical set, so committing thousands of them would bury every
14175
- * real diff in machine noise for zero recoverable information. The `derived` column is the firewall
14176
- * that makes losing them cheap. The retention penalty counts only `derived = 0`, so an
14177
- * uncorroborated machine suspicion cannot evict a memory.
14178
- *
14179
- * The insert is scoped to `provenance = 'sleep'` and `derived = 1` and the whole replace is one
14180
- * atomic batch, so an authored edge is unreachable from here and the corpus is never left with the
14181
- * old mined set deleted and the new one not yet written. In that window the lateral arm would
14182
- * silently return nothing.
14183
- */
14184
- /** The similarity floor a pair must clear to become a mined `relates_to`. */
14185
- const MINING_COSINE_FLOOR = .85;
14186
- /** Nearest neighbors considered per source file. */
14187
- const MINING_PER_SOURCE_K = 5;
14188
- /**
14189
- * Pairs mined per cycle: a cap on what {@link replaceMinedEdges} writes, not on the scan — the
14190
- * kernel's arithmetic is O(n²·d) whatever this says, and it bounds the edge table so one dense
14191
- * neighborhood cannot flood the graph the lateral arm and PageRank read.
14192
- */
14193
- const MINING_SAMPLE_LIMIT = 2e3;
14194
- const relationshipMining = (env) => Effect.gen(function* () {
14195
- /**
14196
- * Tasks are excluded, and here the exclusion is the graph firewall, not a cost guard.
14197
- * Every mined edge is written with `edge_class = 'memory'`, so a pair with a task endpoint
14198
- * would put a task INTO the memory graph, reaching PageRank, MMR, and the retention bridge
14199
- * count. The `edges` CHECK cannot refuse it, because `relates_to` under `memory` is a
14200
- * well-formed edge whatever files sit at its ends.
14201
- */
14202
- const pairs = yield* neighborPairs(env.deps.db, {
14203
- floor: MINING_COSINE_FLOOR,
14204
- perSourceK: 5,
14205
- limit: MINING_SAMPLE_LIMIT,
14206
- excludeTypes: SLEEP_EXCLUDED_TYPES
14207
- });
14208
- const counts = {
14209
- candidates: pairs.length,
14210
- mined: pairs.length
14211
- };
14212
- if (env.dryRun) return emptyOutcome(counts);
14213
- yield* replaceMinedEdges(env.deps.db, {
14214
- runId: env.runId,
14215
- at: env.at,
14216
- rel: "relates_to",
14217
- pairs
14218
- });
14219
- return emptyOutcome(counts);
14220
- });
14221
-
14222
14793
  //#endregion
14223
14794
  //#region packages/sleep/dist/report.js
14224
14795
  /**
@@ -15812,6 +16383,7 @@ const PHASE_BODIES = {
15812
16383
  reprieve,
15813
16384
  "trace-consolidation": traceConsolidation,
15814
16385
  "task-detection": taskDetection,
16386
+ "placement-triage": placementTriage,
15815
16387
  integrity,
15816
16388
  "state-export": stateExport,
15817
16389
  report: reportPhase([])
@@ -15872,7 +16444,14 @@ const run = (deps, options) => Effect.gen(function* () {
15872
16444
  * Created per run rather than held in a module, which is what keeps two runs in one process (and
15873
16445
  * two tests in one file) from sharing a counter.
15874
16446
  */
15875
- detectionBudget: makeDetectionBudget()
16447
+ detectionBudget: makeDetectionBudget(),
16448
+ /**
16449
+ * The deep switches, shaped once here so every phase reads one field. The LLM budget is the
16450
+ * same created-per-run discipline as the detection budget above, and it exists only when the
16451
+ * caller stated a cap: an uncapped deep run is a valid ask, and a phantom cap of zero (or of
16452
+ * some default nobody chose) would silently skip work the operator paid for the flag to reach.
16453
+ */
16454
+ ...options.deep === true ? { deep: { ...options.maxLlmCalls === void 0 ? {} : { budget: makeLlmBudget(options.maxLlmCalls) } } } : {}
15876
16455
  };
15877
16456
  if (!dryRun) yield* deps.git.checkoutBranch(runId, { create: true }).pipe(Effect.orElseSucceed(() => {}));
15878
16457
  yield* ignoreFailure(recordRun(deps.db, {
@@ -15939,7 +16518,9 @@ const resume = (deps, runId, options = {}) => Effect.gen(function* () {
15939
16518
  * finishing. The cost of a fresh one is bounded by the cap, and the mints a resume repeats are
15940
16519
  * refreshes rather than duplicates, which cost no budget at all.
15941
16520
  */
15942
- detectionBudget: makeDetectionBudget()
16521
+ detectionBudget: makeDetectionBudget(),
16522
+ /** A fresh LLM budget too, for the same reason the detection budget above is fresh. */
16523
+ ...options.deep === true ? { deep: { ...options.maxLlmCalls === void 0 ? {} : { budget: makeLlmBudget(options.maxLlmCalls) } } } : {}
15943
16524
  };
15944
16525
  const remaining = SLEEP_PHASES.filter((phase) => !completed.has(phase));
15945
16526
  const executed = yield* executePhases(env, remaining, /* @__PURE__ */ new Set());
@@ -15947,7 +16528,7 @@ const resume = (deps, runId, options = {}) => Effect.gen(function* () {
15947
16528
  const ended = yield* nowIso;
15948
16529
  /**
15949
16530
  * Skipped-because-already-done rows are reported explicitly, so a resume's report accounts for all
15950
- * sixteen phases. A report that showed only the eight it ran would read as a partial run.
16531
+ * seventeen phases. A report that showed only the eight it ran would read as a partial run.
15951
16532
  */
15952
16533
  const priorRows = yield* ignoreFailureWith(readPhases(deps.db, runId), []);
15953
16534
  const already = [...completed].map((phase) => {
@@ -16988,4 +17569,4 @@ const latest = (left, right) => left === null ? right : right === null ? left :
16988
17569
 
16989
17570
  //#endregion
16990
17571
  export { STATE_DB_PATH as $, IndexRecorder as A, IndexGit as B, ModelClient as C, EmbeddingsLive as D, Embeddings as E, makeRetrieval as F, MIGRATIONS_DIR as G, sanitizeFtsQuery as H, reinforce as I, Store as J, STATE_MIGRATIONS_DIR as K, Indexer as L, persistScanned as M, readWatermark as N, EMBED_DIM as O, Retrieval as P, SLEEP_REPORTS_DIR as Q, makeIndexer as R, runDiscrimination as S, wrapAsData as T, DatabaseService as U, makeGitPort as V, makeDatabase as W, makeStore as X, expandRoot as Y, INDEX_DB_PATH as Z, meta as _, parseSidecar as a, makeGit as at, isSleepPhase as b, generateArtifacts as c, setMeta as ct, allPaths as d, fenceOpeningOf as dt, STATE_SIDECAR_PATH as et, danglingEdges as f, REINFORCE_SIGNALS as ft, link as g, hrefFor as h, makeSleep as i, Git as it, makeIndexRecorder as j, EMBED_WATERMARK as k, DETECTION_PREFIX as l, isValidDatetime as lt, applyHeadEdits as m, scanTraceRoot as n, initRepo as nt, renderSidecar as o, commitSubject as ot, publishRows as p, frameKeyOf as pt, STATE_SCHEMA as q, Sleep as r, readFileOrNull as rt, archivedFormOf as s, checkMemory as st, mergeTailExtract as t, attemptIo as tt, accessRows as u, closesFence as ut, unlink as v, ModelClientLive as w, discriminationGate as x, SLEEP_PHASES as y, readIndexState as z };
16991
- //# sourceMappingURL=dist-DgKlozi6.mjs.map
17572
+ //# sourceMappingURL=dist-CHoz5uHd.mjs.map