memhtml 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{dist-DgKlozi6.mjs → dist-CHoz5uHd.mjs} +768 -187
- package/dist/dist-CHoz5uHd.mjs.map +1 -0
- package/dist/memhtml-mcp.mjs +13 -2
- package/dist/memhtml-mcp.mjs.map +1 -1
- package/dist/memhtml.mjs +17 -3
- package/dist/memhtml.mjs.map +1 -1
- package/package.json +1 -1
- package/dist/dist-DgKlozi6.mjs.map +0 -1
|
@@ -9181,21 +9181,31 @@ const batchCall = (model, label, request) => isolate(label, model.generateObject
|
|
|
9181
9181
|
* `sleep_phases` row, and in a `--phases` flag, and three copies of the string would drift.
|
|
9182
9182
|
*/
|
|
9183
9183
|
/**
|
|
9184
|
-
* The
|
|
9184
|
+
* The seventeen phases, in execution order.
|
|
9185
9185
|
*
|
|
9186
9186
|
* The order encodes the predecessor memory system's dependencies (design §6): entity resolution precedes person
|
|
9187
9187
|
* links so aliases have already merged, confidence decay precedes retention triage so triage
|
|
9188
9188
|
* scores the decayed value, and dedup-merge precedes compress and retention because both operate
|
|
9189
9189
|
* on the post-merge set.
|
|
9190
9190
|
*
|
|
9191
|
-
* `task-detection`
|
|
9192
|
-
*
|
|
9191
|
+
* `task-detection` sits after `trace-consolidation` and before `integrity`, and both edges are
|
|
9192
|
+
* deliberate. It scans the ACTIVE corpus for unresolved
|
|
9193
9193
|
* commitments, so it has to run after every phase that changes what is active — after dedup's folds,
|
|
9194
9194
|
* after retention's evictions, after compress's canonicals, and after trace consolidation's newly
|
|
9195
9195
|
* distilled memories, which are the freshest text of the night and the likeliest to carry one. And it
|
|
9196
9196
|
* WRITES files, so it must precede `integrity`, which repairs dangling hrefs and regenerates the
|
|
9197
9197
|
* directory artifacts: a task minted afterwards would be absent from its directory's `index.html`
|
|
9198
9198
|
* until the next night.
|
|
9199
|
+
*
|
|
9200
|
+
* `placement-triage` is deep-only (issue #63): on a run without `--deep` it returns immediately,
|
|
9201
|
+
* writes nothing, and commits nothing, so the nightly cycle's behavior is unchanged by its presence
|
|
9202
|
+
* in this list. Its slot has the same two edges task-detection's does, plus one more on each side:
|
|
9203
|
+
* it must run after `compress` (it re-files only what even deep grouping could not fold, so folding
|
|
9204
|
+
* has to have had its chance first) and after `task-detection` (a move mid-scan would hand the
|
|
9205
|
+
* detector paths that no longer hold files), and it must precede `integrity` because it MOVES files —
|
|
9206
|
+
* integrity regenerates each directory's `index.html`, and it rewrites inbound hrefs itself because
|
|
9207
|
+
* integrity's dangling-href repair only knows how to chase a target into the ARCHIVE, not into a
|
|
9208
|
+
* topic directory.
|
|
9199
9209
|
*/
|
|
9200
9210
|
const SLEEP_PHASES = [
|
|
9201
9211
|
"preflight",
|
|
@@ -9211,6 +9221,7 @@ const SLEEP_PHASES = [
|
|
|
9211
9221
|
"reprieve",
|
|
9212
9222
|
"trace-consolidation",
|
|
9213
9223
|
"task-detection",
|
|
9224
|
+
"placement-triage",
|
|
9214
9225
|
"integrity",
|
|
9215
9226
|
"state-export",
|
|
9216
9227
|
"report"
|
|
@@ -9577,7 +9588,8 @@ const DEFAULT_MODELS = {
|
|
|
9577
9588
|
"arc-synthesis": "gpt-5.6-sol",
|
|
9578
9589
|
compress: "gpt-5.6-sol",
|
|
9579
9590
|
"trace-consolidation": "opus-5",
|
|
9580
|
-
"task-detection": "gpt-5.6-sol"
|
|
9591
|
+
"task-detection": "gpt-5.6-sol",
|
|
9592
|
+
"placement-triage": "gpt-5.6-sol"
|
|
9581
9593
|
};
|
|
9582
9594
|
/** The model a phase calls: the caller's override, else {@link DEFAULT_MODELS}, else sonnet. */
|
|
9583
9595
|
const modelFor = (deps, phase) => deps.models?.[phase] ?? DEFAULT_MODELS[phase] ?? "sonnet-5";
|
|
@@ -9587,6 +9599,25 @@ const emptyOutcome = (counts = {}) => ({
|
|
|
9587
9599
|
commitSha: null,
|
|
9588
9600
|
llmCalls: 0
|
|
9589
9601
|
});
|
|
9602
|
+
/** A fresh budget. A non-positive or fractional cap clamps to a usable whole number. */
|
|
9603
|
+
const makeLlmBudget = (maxCalls) => ({
|
|
9604
|
+
maxCalls: Math.max(0, Math.trunc(maxCalls)),
|
|
9605
|
+
spent: 0
|
|
9606
|
+
});
|
|
9607
|
+
/**
|
|
9608
|
+
* Take one call from the budget, or report exhaustion.
|
|
9609
|
+
*
|
|
9610
|
+
* `true` means the caller may make the call and the budget has been charged. Charging BEFORE the
|
|
9611
|
+
* call rather than after means a crash mid-call cannot under-count, which errs on the side the
|
|
9612
|
+
* budget exists for. No budget bound means every call is allowed.
|
|
9613
|
+
*/
|
|
9614
|
+
const takeLlmCall = (deep) => {
|
|
9615
|
+
const budget = deep?.budget;
|
|
9616
|
+
if (budget === void 0) return true;
|
|
9617
|
+
if (budget.spent >= budget.maxCalls) return false;
|
|
9618
|
+
budget.spent += 1;
|
|
9619
|
+
return true;
|
|
9620
|
+
};
|
|
9590
9621
|
|
|
9591
9622
|
//#endregion
|
|
9592
9623
|
//#region packages/sleep/dist/llm.js
|
|
@@ -10054,6 +10085,71 @@ const edgeTypingPrompt = (pairs) => batchPrompt(pairs, EDGE_TYPING_INSTRUCTION,
|
|
|
10054
10085
|
const arcTriagePrompt = (arcsText, evidenceText) => `${dataBlock("current_arcs", arcsText)}\n\n${dataBlock("evidence", evidenceText)}\n\nProduce a triage plan. Assign update or skip to every existing arc, and add a create entry for any genuinely new behavioral pattern the existing arcs do not cover.`;
|
|
10055
10086
|
/** The arc-execute user turn for one arc. `current` is absent on a create. */
|
|
10056
10087
|
const arcExecutePrompt = (input) => (input.current === void 0 ? `${dataBlock("new_arc_title", input.title)}\n\n` : `${dataBlock("existing_arc", input.current)}\n\n`) + `${dataBlock("evidence", input.evidenceText)}\n\n${dataBlock("triage_rationale", input.rationale)}\n\n` + (input.current === void 0 ? "Synthesize a new behavioral principal from this evidence." : "Update the arc to incorporate the new evidence, preserving existing knowledge that holds.");
|
|
10088
|
+
/**
|
|
10089
|
+
* One inbox memory's proposed destination, under the opaque key it was offered as (issue #63).
|
|
10090
|
+
*
|
|
10091
|
+
* `destination` is a STRING the phase validates, never trusts: only `keep-inbox`, a directory the
|
|
10092
|
+
* corpus already has, or one of a capped handful of new topic directories survives the code gate.
|
|
10093
|
+
* A model-invented path outside those is a refused placement, counted and logged, because a
|
|
10094
|
+
* destination is a write target and the model does not choose write targets anywhere in this
|
|
10095
|
+
* pipeline.
|
|
10096
|
+
*/
|
|
10097
|
+
const Placement = Schema.Struct({
|
|
10098
|
+
/** The offered key, e.g. `m3`. A key the batch never held resolves to nothing and is dropped. */
|
|
10099
|
+
memberKey: Schema.String,
|
|
10100
|
+
/**
|
|
10101
|
+
* `keep-inbox`, or a bucket-rooted directory like `areas/deploys` or `resources/sqlite`.
|
|
10102
|
+
* `keep-inbox` is the refusal and the ordinary answer for a memory whose topic is not clear.
|
|
10103
|
+
*/
|
|
10104
|
+
destination: Schema.String,
|
|
10105
|
+
/** Unitless in `[0, 1]`. The move gate is deterministic and reads this, not the prose. */
|
|
10106
|
+
confidence: Schema.Finite.check(Schema.isBetween({
|
|
10107
|
+
minimum: 0,
|
|
10108
|
+
maximum: 1
|
|
10109
|
+
}))
|
|
10110
|
+
});
|
|
10111
|
+
/** One batch's whole answer: a placement list. An omitted member keeps its inbox path. */
|
|
10112
|
+
const PlacementTriage = Schema.Struct({ placements: Schema.Array(Placement) });
|
|
10113
|
+
/**
|
|
10114
|
+
* The confidence a placement must clear before the phase moves a file. The same 0.7 the edge and
|
|
10115
|
+
* entity gates use, for the same reason one number serves them: a second knob would be a threshold
|
|
10116
|
+
* nobody could state the meaning of. A move is cheap to reverse (`git mv` back, reviewable on the
|
|
10117
|
+
* branch), so the floor matches the mild gates rather than the corroboration-backed ones.
|
|
10118
|
+
*/
|
|
10119
|
+
const PLACEMENT_CONFIDENCE_FLOOR = .7;
|
|
10120
|
+
/** The literal a placement answer uses to decline. */
|
|
10121
|
+
const PLACEMENT_KEEP = "keep-inbox";
|
|
10122
|
+
/**
|
|
10123
|
+
* The placement-triage system prompt (issue #63).
|
|
10124
|
+
*
|
|
10125
|
+
* The refusal is stated as the ordinary answer, because for a bulk-imported inbox it is: most
|
|
10126
|
+
* singletons are distinct facts whose topic directory does not exist yet, and a model that felt
|
|
10127
|
+
* obliged to place everything would scatter the inbox across invented directories nobody asked for.
|
|
10128
|
+
*/
|
|
10129
|
+
const PLACEMENT_SYSTEM = `You file inbox memories into topic directories for an AI agent's long-term memory system. You are
|
|
10130
|
+
given the list of directories the corpus already has, and a NUMBERED LIST of memories currently
|
|
10131
|
+
sitting in the inbox. For each memory, propose where it belongs.
|
|
10132
|
+
|
|
10133
|
+
- destination is one of the existing directories, verbatim from the list, when the memory clearly
|
|
10134
|
+
belongs to that topic.
|
|
10135
|
+
- destination may be a NEW directory (areas/<topic> for ongoing concerns, resources/<topic> for
|
|
10136
|
+
reference material) when several of THESE memories share a topic no existing directory covers.
|
|
10137
|
+
Use a short lowercase hyphenated topic name. New directories are rationed, so propose one only
|
|
10138
|
+
when it would hold more than a stray file.
|
|
10139
|
+
- destination is keep-inbox when you are unsure, when the memory's topic is ambiguous, or when no
|
|
10140
|
+
directory fits. This is the ordinary answer, not a failure.
|
|
10141
|
+
- Never propose archive/, areas/inbox itself, areas/arcs, or resources/people. Those are managed
|
|
10142
|
+
surfaces.
|
|
10143
|
+
- Rate confidence honestly. A placement above the floor moves the file on a review branch.
|
|
10144
|
+
- Omitting a member means keep-inbox.`;
|
|
10145
|
+
/** The instruction that closes a placement batch's user turn, after the member list. */
|
|
10146
|
+
const PLACEMENT_INSTRUCTION = "Propose a destination for each memory above: an existing directory from the list, a new areas/<topic> or resources/<topic> directory, or keep-inbox. Name each memory by its offered key and rate your confidence.";
|
|
10147
|
+
/**
|
|
10148
|
+
* The placement user turn for one batch: the existing directories, then every member under its
|
|
10149
|
+
* offered key, then the instruction. The directory list is corpus-derived text, so it is wrapped as
|
|
10150
|
+
* data exactly as the member texts are.
|
|
10151
|
+
*/
|
|
10152
|
+
const placementPrompt = (directories, members) => `${dataBlock("existing_directories", directories.join("\n"))}\n\n` + batchPrompt(members, PLACEMENT_INSTRUCTION, { label: "memory" });
|
|
10057
10153
|
/** The instruction that closes a compress batch's user turn, after the member list. */
|
|
10058
10154
|
const COMPRESS_INSTRUCTION = "Fold these memories into one canonical memory. List in absorbedKeys exactly the members whose content the canonical carries forward.";
|
|
10059
10155
|
/**
|
|
@@ -10106,7 +10202,7 @@ const dedupPrompt = (components) => {
|
|
|
10106
10202
|
/**
|
|
10107
10203
|
* The memory type no phase of a sleep cycle touches.
|
|
10108
10204
|
*
|
|
10109
|
-
* A task is live working state, and every one of the
|
|
10205
|
+
* A task is live working state, and every one of the seventeen phases is a judgment about REMEMBERED
|
|
10110
10206
|
* FACTS: decay says a claim is fading, dedup says two claims are one, edge typing says one claim
|
|
10111
10207
|
* caused or contradicts another, retention says a claim has stopped earning its place. None of those hold for
|
|
10112
10208
|
* a thing an agent intends to do, and each would be wrong applied to one. A task the agent has
|
|
@@ -10428,6 +10524,16 @@ const peoplePaths = (db) => db.all("SELECT path FROM files WHERE path LIKE ? OR
|
|
|
10428
10524
|
* `edge_class = 'memory'` is the firewall. A person or provenance edge cannot enter PageRank, label
|
|
10429
10525
|
* propagation, or the retention bridge count, and this query is what makes that true. The CHECK
|
|
10430
10526
|
* constraint alone does not.
|
|
10527
|
+
*
|
|
10528
|
+
* **The deep grouping band is excluded, and the exclusion is what keeps nightly scoring stable
|
|
10529
|
+
* across deep runs (issue #63).** Deep mining writes machine-mined `laterally_related` edges at a
|
|
10530
|
+
* floor (0.72) far below the nightly one, purely so label propagation can partition the inbox tail
|
|
10531
|
+
* for compress. Those edges persist in the index after the deep run, so without this predicate the
|
|
10532
|
+
* FIRST deep run would permanently change every subsequent nightly run's PageRank, communities, and
|
|
10533
|
+
* bridge counts — and therefore its eviction decisions — on a corpus whose files did not change.
|
|
10534
|
+
* The filter names `derived = 1` as well as the rel, so an AUTHORED `laterally_related` (an agent's
|
|
10535
|
+
* own assertion, addable through `memhtml link`) keeps exactly the graph standing it had.
|
|
10536
|
+
* {@link deepGroupingEdges} is the one reader of the band.
|
|
10431
10537
|
*/
|
|
10432
10538
|
const memoryEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel, e.dst_path AS dst_path,
|
|
10433
10539
|
e.strength AS strength, e.derived AS derived
|
|
@@ -10435,6 +10541,24 @@ const memoryEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel,
|
|
|
10435
10541
|
JOIN files s ON s.path = e.src_path AND s.archived = 0
|
|
10436
10542
|
JOIN files d ON d.path = e.dst_path AND d.archived = 0
|
|
10437
10543
|
WHERE e.edge_class = 'memory'
|
|
10544
|
+
AND NOT (e.derived = 1 AND e.rel = 'laterally_related')
|
|
10545
|
+
ORDER BY e.src_path ASC, e.rel ASC, e.dst_path ASC`);
|
|
10546
|
+
/**
|
|
10547
|
+
* The deep grouping band: machine-mined `laterally_related` edges over active files (issue #63).
|
|
10548
|
+
*
|
|
10549
|
+
* The mirror of {@link memoryEdges}' exclusion, and deliberately a separate read instead of a flag on
|
|
10550
|
+
* it: the band has exactly one consumer intent — widening label propagation's partition for the deep
|
|
10551
|
+
* compress and placement phases — and a parameterized `memoryEdges` would let any caller widen the
|
|
10552
|
+
* retention graph by passing a boolean. `provenance = 'sleep'` is implied by `derived = 1` (the
|
|
10553
|
+
* table CHECK pins the pair) and stated anyway so the statement reads as what it is.
|
|
10554
|
+
*/
|
|
10555
|
+
const deepGroupingEdges = (db) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel, e.dst_path AS dst_path,
|
|
10556
|
+
e.strength AS strength, e.derived AS derived
|
|
10557
|
+
FROM edges e
|
|
10558
|
+
JOIN files s ON s.path = e.src_path AND s.archived = 0
|
|
10559
|
+
JOIN files d ON d.path = e.dst_path AND d.archived = 0
|
|
10560
|
+
WHERE e.edge_class = 'memory' AND e.derived = 1 AND e.rel = 'laterally_related'
|
|
10561
|
+
AND e.provenance = 'sleep'
|
|
10438
10562
|
ORDER BY e.src_path ASC, e.rel ASC, e.dst_path ASC`);
|
|
10439
10563
|
const retentionEdgeCounts = (db) => db.all(`SELECT f.path AS path,
|
|
10440
10564
|
sum(CASE WHEN e.rel = 'supports' THEN 1 ELSE 0 END) AS reinforcements,
|
|
@@ -10639,6 +10763,19 @@ const markSessionsConsolidated = (db, input) => db.writeAll(input.sessionIds.map
|
|
|
10639
10763
|
const linkedSessionCount = (db) => db.get(`SELECT count(DISTINCT t.session_id) AS n FROM traces t
|
|
10640
10764
|
JOIN memory_session_links l ON l.session_id = t.session_id`).pipe(Effect.map((row) => row?.n ?? 0));
|
|
10641
10765
|
/**
|
|
10766
|
+
* Authored memory-class edges POINTING AT one path: which files hold a `<link>` a move must rewrite.
|
|
10767
|
+
*
|
|
10768
|
+
* Placement triage's read (issue #63). `derived = 0` because a mined edge lives only in the index
|
|
10769
|
+
* and the next re-mine follows the moved vectors on its own; only a file-borne link needs a splice.
|
|
10770
|
+
* Both memory and task classes are included — a task may `blocks`-link a memory it waits on, and a
|
|
10771
|
+
* move that left that href dangling would break the task surface — but provenance and person edges
|
|
10772
|
+
* cannot point at an inbox memory by construction.
|
|
10773
|
+
*/
|
|
10774
|
+
const inboundAuthoredEdges = (db, dstPath) => db.all(`SELECT e.src_path AS src_path, e.rel AS rel
|
|
10775
|
+
FROM edges e
|
|
10776
|
+
WHERE e.derived = 0 AND e.dst_path = ?
|
|
10777
|
+
ORDER BY e.src_path ASC, e.rel ASC`, [dstPath]);
|
|
10778
|
+
/**
|
|
10642
10779
|
* Authored edges pointing at a path the index does not hold.
|
|
10643
10780
|
*
|
|
10644
10781
|
* `derived = 0` only: a mined edge lives in the index and nowhere else, so a dangling one is
|
|
@@ -10985,6 +11122,129 @@ const arcSynthesis = (env) => Effect.gen(function* () {
|
|
|
10985
11122
|
};
|
|
10986
11123
|
});
|
|
10987
11124
|
|
|
11125
|
+
//#endregion
|
|
11126
|
+
//#region packages/sleep/dist/phases/relationship-mining.js
|
|
11127
|
+
/**
|
|
11128
|
+
* Phase 5, relationship mining. Derived `relates_to` edges in the index only. NO COMMIT.
|
|
11129
|
+
*
|
|
11130
|
+
* A mined edge is a re-derivable function of the corpus and the embedder. `index rebuild` plus the
|
|
11131
|
+
* next night's mining regenerates the identical set, so committing thousands of them would bury every
|
|
11132
|
+
* real diff in machine noise for zero recoverable information. The `derived` column is the firewall
|
|
11133
|
+
* that makes losing them cheap. The retention penalty counts only `derived = 0`, so an
|
|
11134
|
+
* uncorroborated machine suspicion cannot evict a memory.
|
|
11135
|
+
*
|
|
11136
|
+
* The insert is scoped to `provenance = 'sleep'` and `derived = 1` and the whole replace is one
|
|
11137
|
+
* atomic batch, so an authored edge is unreachable from here and the corpus is never left with the
|
|
11138
|
+
* old mined set deleted and the new one not yet written. In that window the lateral arm would
|
|
11139
|
+
* silently return nothing.
|
|
11140
|
+
*
|
|
11141
|
+
* **Under `--deep` the phase additionally mines a GROUPING band (issue #63).** On a measured 3,079-file
|
|
11142
|
+
* inbox, 8% of files have a neighbor at the 0.85 floor and 84% touch no edge at all — no community, so
|
|
11143
|
+
* compress is structurally unable to reach them at any frequency. The deep band mines
|
|
11144
|
+
* [{@link DEEP_MINING_COSINE_FLOOR}, {@link MINING_COSINE_FLOOR}) as `laterally_related` edges whose
|
|
11145
|
+
* only consumer intent is giving label propagation a partition; the existing compress model then
|
|
11146
|
+
* judges the folds, and `absorbedKeys: []` is its refusal, so a looser grouping floor costs calls and
|
|
11147
|
+
* never correctness.
|
|
11148
|
+
*
|
|
11149
|
+
* **The two bands are separate rels, and that is the isolation mechanism.** {@link replaceMinedEdges}
|
|
11150
|
+
* scopes its atomic delete by rel, so the deep replace cannot clobber the nightly `relates_to` set and
|
|
11151
|
+
* the nightly replace cannot clobber the deep band. The issue sketched a distinct PROVENANCE instead;
|
|
11152
|
+
* the `edges` CHECKs pin `derived = 1` to `provenance = 'sleep'` (migration 0008), so that spelling
|
|
11153
|
+
* needs a table recreate while a distinct rel needs nothing: `laterally_related` is already in the
|
|
11154
|
+
* memory-rel vocabulary with no producer, and edge typing's candidate scan reads `rel = 'relates_to'`
|
|
11155
|
+
* alone, so the deep band never spends nightly edge-typing calls. A nightly run never touches the
|
|
11156
|
+
* deep rel, so a deep band persists until the next deep run re-mines it or `index rebuild` drops it —
|
|
11157
|
+
* both re-derivable, which is the property that makes an index-only edge safe to hold.
|
|
11158
|
+
*/
|
|
11159
|
+
/** The similarity floor a pair must clear to become a mined `relates_to`. */
|
|
11160
|
+
const MINING_COSINE_FLOOR = .85;
|
|
11161
|
+
/**
|
|
11162
|
+
* The GROUPING-band floor deep mining reaches down to (issue #63). At 0.72, between 43% and 61% of
|
|
11163
|
+
* the measured bulk-import inbox has a neighbor (43% at 0.75, 61% at 0.70, median best-neighbor
|
|
11164
|
+
* cosine 0.731) — reach enough to partition most of the tail while staying above the ~0.5-0.6
|
|
11165
|
+
* cross-topic baseline the fixture corpora measure, so a community is still a topic and not noise.
|
|
11166
|
+
* The compress model judging every proposed fold is what makes this floor a cost knob rather than a
|
|
11167
|
+
* correctness one.
|
|
11168
|
+
*/
|
|
11169
|
+
const DEEP_MINING_COSINE_FLOOR = .72;
|
|
11170
|
+
/** Nearest neighbors considered per source file. */
|
|
11171
|
+
const MINING_PER_SOURCE_K = 5;
|
|
11172
|
+
/**
|
|
11173
|
+
* Pairs mined per cycle: a cap on what {@link replaceMinedEdges} writes, not on the scan — the
|
|
11174
|
+
* kernel's arithmetic is O(n²·d) whatever this says, and it bounds the edge table so one dense
|
|
11175
|
+
* neighborhood cannot flood the graph the lateral arm and PageRank read.
|
|
11176
|
+
*/
|
|
11177
|
+
const MINING_SAMPLE_LIMIT = 2e3;
|
|
11178
|
+
/**
|
|
11179
|
+
* The deep band's own write cap. Separate from {@link MINING_SAMPLE_LIMIT} because the band is wider
|
|
11180
|
+
* by construction — it exists to reach the 84% the nightly band cannot — and sharing the nightly cap
|
|
11181
|
+
* would make the deep run's reach a function of how crowded the nightly band happens to be.
|
|
11182
|
+
*/
|
|
11183
|
+
const DEEP_MINING_SAMPLE_LIMIT = 1e4;
|
|
11184
|
+
/** The rel the deep grouping band is written under. The band separator; see the module header. */
|
|
11185
|
+
const DEEP_GROUPING_REL = "laterally_related";
|
|
11186
|
+
/**
|
|
11187
|
+
* Mine every band the run is entitled to and replace the index's mined sets: the whole phase minus
|
|
11188
|
+
* its counts. Exported because deep compress re-runs it BETWEEN passes (issue #63's
|
|
11189
|
+
* iterate-until-quiet): a fold's canonical is a new neighbor only after it is indexed and re-mined,
|
|
11190
|
+
* and a second implementation of the scan here would be free to disagree with the nightly one about
|
|
11191
|
+
* floors, caps, and exclusions.
|
|
11192
|
+
*/
|
|
11193
|
+
const mineAllBands = (env) => Effect.gen(function* () {
|
|
11194
|
+
/**
|
|
11195
|
+
* Tasks are excluded, and here the exclusion is the graph firewall, not a cost guard.
|
|
11196
|
+
* Every mined edge is written with `edge_class = 'memory'`, so a pair with a task endpoint
|
|
11197
|
+
* would put a task INTO the memory graph, reaching PageRank, MMR, and the retention bridge
|
|
11198
|
+
* count. The `edges` CHECK cannot refuse it, because `relates_to` under `memory` is a
|
|
11199
|
+
* well-formed edge whatever files sit at its ends.
|
|
11200
|
+
*/
|
|
11201
|
+
const pairs = yield* neighborPairs(env.deps.db, {
|
|
11202
|
+
floor: MINING_COSINE_FLOOR,
|
|
11203
|
+
perSourceK: 5,
|
|
11204
|
+
limit: MINING_SAMPLE_LIMIT,
|
|
11205
|
+
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
11206
|
+
});
|
|
11207
|
+
/**
|
|
11208
|
+
* The deep grouping band: everything in [deep floor, nightly floor). Mined only under `--deep`,
|
|
11209
|
+
* so a nightly run's writes — and therefore the graph every nightly consumer reads on a corpus
|
|
11210
|
+
* that has never run deep — are byte-identical to what they were before this branch existed.
|
|
11211
|
+
* The nightly floor is the band's EXCLUSIVE ceiling: a pair at or above it is the nightly
|
|
11212
|
+
* band's, and holding it in both would double its label-propagation weight.
|
|
11213
|
+
*/
|
|
11214
|
+
const deepPairs = env.deep === void 0 ? [] : (yield* neighborPairs(env.deps.db, {
|
|
11215
|
+
floor: DEEP_MINING_COSINE_FLOOR,
|
|
11216
|
+
perSourceK: 5,
|
|
11217
|
+
limit: DEEP_MINING_SAMPLE_LIMIT,
|
|
11218
|
+
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
11219
|
+
})).filter((pair) => pair.sim < MINING_COSINE_FLOOR);
|
|
11220
|
+
if (!env.dryRun) {
|
|
11221
|
+
yield* replaceMinedEdges(env.deps.db, {
|
|
11222
|
+
runId: env.runId,
|
|
11223
|
+
at: env.at,
|
|
11224
|
+
rel: "relates_to",
|
|
11225
|
+
pairs
|
|
11226
|
+
});
|
|
11227
|
+
if (env.deep !== void 0) yield* replaceMinedEdges(env.deps.db, {
|
|
11228
|
+
runId: env.runId,
|
|
11229
|
+
at: env.at,
|
|
11230
|
+
rel: DEEP_GROUPING_REL,
|
|
11231
|
+
pairs: deepPairs
|
|
11232
|
+
});
|
|
11233
|
+
}
|
|
11234
|
+
return {
|
|
11235
|
+
mined: pairs.length,
|
|
11236
|
+
deepMined: deepPairs.length
|
|
11237
|
+
};
|
|
11238
|
+
});
|
|
11239
|
+
const relationshipMining = (env) => Effect.gen(function* () {
|
|
11240
|
+
const mined = yield* mineAllBands(env);
|
|
11241
|
+
return emptyOutcome({
|
|
11242
|
+
candidates: mined.mined,
|
|
11243
|
+
mined: mined.mined,
|
|
11244
|
+
...env.deep === void 0 ? {} : { deepMined: mined.deepMined }
|
|
11245
|
+
});
|
|
11246
|
+
});
|
|
11247
|
+
|
|
10988
11248
|
//#endregion
|
|
10989
11249
|
//#region packages/sleep/dist/phases/compress.js
|
|
10990
11250
|
/**
|
|
@@ -11015,6 +11275,28 @@ const arcSynthesis = (env) => Effect.gen(function* () {
|
|
|
11015
11275
|
*
|
|
11016
11276
|
* `dedup-merge` is a HARD prerequisite. Compressing before duplicates are folded would synthesize a
|
|
11017
11277
|
* canonical over a pair the merge phase then archives one half of.
|
|
11278
|
+
*
|
|
11279
|
+
* ## Deep mode (issue #63)
|
|
11280
|
+
*
|
|
11281
|
+
* Under `--deep` three things change about WHICH memories a fold can reach, and nothing about what
|
|
11282
|
+
* happens to a fold:
|
|
11283
|
+
*
|
|
11284
|
+
* 1. **Communities come from the widened graph.** Label propagation runs over the nightly memory
|
|
11285
|
+
* edges PLUS the deep grouping band (`laterally_related`, mined at 0.72), so inbox files whose
|
|
11286
|
+
* best neighbor sits under the nightly 0.85 floor get a partition. Retention SCORES stay the
|
|
11287
|
+
* nightly function — the band decides grouping, never eviction.
|
|
11288
|
+
* 2. **Entity groups are a second community source.** Candidates the widened graph still leaves
|
|
11289
|
+
* communityless are grouped by shared `file_entities` reference under a synthetic
|
|
11290
|
+
* `entity:<type>:<name>` label, because two memories about one subject can share no vocabulary at
|
|
11291
|
+
* all. Hub entities (more than {@link DEEP_ENTITY_HUB_LIMIT} active claimants) are stop-words and
|
|
11292
|
+
* are skipped.
|
|
11293
|
+
* 3. **The phase iterates until quiet.** A pass that produced canonicals re-indexes the branch,
|
|
11294
|
+
* re-mines both bands, recomputes retention, and folds again — a canonical is a new neighbor and
|
|
11295
|
+
* a new community member — until a pass folds nothing or {@link DEEP_COMPRESS_MAX_PASSES} hits.
|
|
11296
|
+
*
|
|
11297
|
+
* Every model call in deep mode is charged against the run's shared `--max-llm-calls` budget first;
|
|
11298
|
+
* an exhausted budget skips the batch with reason `budget` and the run stays green, the same
|
|
11299
|
+
* degradation posture a model outage has.
|
|
11018
11300
|
*/
|
|
11019
11301
|
/** Members per model call. Small enough that every member's facts fit the answer's attention. */
|
|
11020
11302
|
const COMPRESS_BATCH_SIZE = 8;
|
|
@@ -11027,6 +11309,83 @@ const COMPRESS_MIN_BATCH = 2;
|
|
|
11027
11309
|
const COMPRESS_CANDIDATE_LIMIT = 2e3;
|
|
11028
11310
|
/** Characters of each member shown. A fold must see the facts, so this is wider than arc evidence. */
|
|
11029
11311
|
const COMPRESS_MEMBER_CHARS = 1200;
|
|
11312
|
+
/**
|
|
11313
|
+
* Deep compress passes before the loop stops regardless of yield (issue #63). Each extra pass costs
|
|
11314
|
+
* a full re-index, a re-mine, and another round of model calls, and the third pass's yield on any
|
|
11315
|
+
* real corpus is folds of folds — past that the loop is spending calls to rename its own output.
|
|
11316
|
+
*/
|
|
11317
|
+
const DEEP_COMPRESS_MAX_PASSES = 3;
|
|
11318
|
+
/**
|
|
11319
|
+
* Active files an entity may be claimed by before deep grouping treats it as a stop-word
|
|
11320
|
+
* (issue #63). An entity on sixty-plus files ("service:api", a person who appears everywhere) says
|
|
11321
|
+
* which corpus this is, not which memories are one topic, and batching its claimants would fold
|
|
11322
|
+
* unrelated facts for sharing a byline. Groups under the limit still slice to
|
|
11323
|
+
* {@link COMPRESS_BATCH_SIZE} through the same kernel as every other group.
|
|
11324
|
+
*/
|
|
11325
|
+
const DEEP_ENTITY_HUB_LIMIT = 64;
|
|
11326
|
+
/** The synthetic community-label prefix entity groups use. No git path starts with this. */
|
|
11327
|
+
const ENTITY_LABEL_PREFIX = "entity:";
|
|
11328
|
+
/**
|
|
11329
|
+
* The community label of every active path under the WIDENED graph: nightly memory edges plus the
|
|
11330
|
+
* deep grouping band, one label-propagation pass (issue #63).
|
|
11331
|
+
*
|
|
11332
|
+
* Computed here rather than inside `runRetentionPass`, deliberately: retention's partition feeds
|
|
11333
|
+
* eviction scoring and must not move when a deep band is present in the index, so the widened
|
|
11334
|
+
* partition is a second, compress-only computation over reads this module already owns.
|
|
11335
|
+
*/
|
|
11336
|
+
const deepCommunityLabels = (env) => Effect.gen(function* () {
|
|
11337
|
+
const corpus = yield* activeCorpus(env.deps.db);
|
|
11338
|
+
const nightly = yield* memoryEdges(env.deps.db);
|
|
11339
|
+
const band = yield* deepGroupingEdges(env.deps.db);
|
|
11340
|
+
return labelPropagation(corpus.map((row) => row.path), [...nightly, ...band].map((edge) => ({
|
|
11341
|
+
src: edge.src_path,
|
|
11342
|
+
dst: edge.dst_path,
|
|
11343
|
+
strength: edge.strength
|
|
11344
|
+
})));
|
|
11345
|
+
}).pipe(Effect.orElseSucceed(() => /* @__PURE__ */ new Map()));
|
|
11346
|
+
/**
|
|
11347
|
+
* Assign an `entity:<type>:<name>` label to every candidate the graph left communityless, greedily,
|
|
11348
|
+
* one label per file (issue #63). PURE, so the hub cap and the one-label rule are unit-assertable
|
|
11349
|
+
* without a corpus.
|
|
11350
|
+
*
|
|
11351
|
+
* Entities are walked in lexicographic (type, name) order and each claims its unclaimed candidates,
|
|
11352
|
+
* so the assignment is a pure function of the corpus: a file naming two entities lands with the
|
|
11353
|
+
* lexicographically first, and the run makes the same batches twice over. A group needs two members
|
|
11354
|
+
* to mean anything, matching {@link COMPRESS_MIN_BATCH}.
|
|
11355
|
+
*
|
|
11356
|
+
* **The hub cap reads the entity's ACTIVE claimant count, not its needing count.** An entity on a
|
|
11357
|
+
* hundred files whose tail happens to leave only three in the inbox band is still a stop-word:
|
|
11358
|
+
* what those three share is a byline, not a topic.
|
|
11359
|
+
*/
|
|
11360
|
+
const assignEntityLabels = (claims, needing) => {
|
|
11361
|
+
const byEntity = /* @__PURE__ */ new Map();
|
|
11362
|
+
for (const claim of claims) {
|
|
11363
|
+
const key = `${claim.entity_type}:${claim.entity_name}`;
|
|
11364
|
+
const bucket = byEntity.get(key);
|
|
11365
|
+
if (bucket === void 0) byEntity.set(key, [claim.path]);
|
|
11366
|
+
else bucket.push(claim.path);
|
|
11367
|
+
}
|
|
11368
|
+
const labels = /* @__PURE__ */ new Map();
|
|
11369
|
+
let hubsSkipped = 0;
|
|
11370
|
+
for (const [key, paths] of [...byEntity.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
|
|
11371
|
+
if (paths.length > 64) {
|
|
11372
|
+
hubsSkipped += 1;
|
|
11373
|
+
continue;
|
|
11374
|
+
}
|
|
11375
|
+
const members = paths.filter((path) => needing.has(path) && !labels.has(path));
|
|
11376
|
+
if (members.length < 2) continue;
|
|
11377
|
+
for (const member of members) labels.set(member, `${ENTITY_LABEL_PREFIX}${key}`);
|
|
11378
|
+
}
|
|
11379
|
+
return {
|
|
11380
|
+
labels,
|
|
11381
|
+
hubsSkipped
|
|
11382
|
+
};
|
|
11383
|
+
};
|
|
11384
|
+
/** {@link assignEntityLabels} over the index's own claim rows. */
|
|
11385
|
+
const entityLabelsFor = (env, needing) => entityClaims(env.deps.db).pipe(Effect.map((claims) => assignEntityLabels(claims, needing)), Effect.orElseSucceed(() => ({
|
|
11386
|
+
labels: /* @__PURE__ */ new Map(),
|
|
11387
|
+
hubsSkipped: 0
|
|
11388
|
+
})));
|
|
11030
11389
|
const compress = (env) => Effect.gen(function* () {
|
|
11031
11390
|
const model = env.deps.model;
|
|
11032
11391
|
if (model === void 0) return {
|
|
@@ -11037,143 +11396,187 @@ const compress = (env) => Effect.gen(function* () {
|
|
|
11037
11396
|
}),
|
|
11038
11397
|
detail: "no model bound"
|
|
11039
11398
|
};
|
|
11040
|
-
const candidates = (yield* runRetentionPass(env.deps.db, env.at)).scored.filter((entry) => entry.score.action === "compress" && entry.row.memory_type !== "arc" && !isSleepExcluded(entry.row.memory_type) && entry.community !== void 0).slice(0, COMPRESS_CANDIDATE_LIMIT);
|
|
11041
|
-
/** Community -> its COMPRESS-band members, both orders fixed so batching is reproducible. */
|
|
11042
|
-
const byCommunity = /* @__PURE__ */ new Map();
|
|
11043
|
-
for (const entry of candidates) {
|
|
11044
|
-
const label = entry.community;
|
|
11045
|
-
if (label === void 0) continue;
|
|
11046
|
-
const bucket = byCommunity.get(label);
|
|
11047
|
-
if (bucket === void 0) byCommunity.set(label, [entry]);
|
|
11048
|
-
else bucket.push(entry);
|
|
11049
|
-
}
|
|
11050
|
-
/**
|
|
11051
|
-
* Both sorts are this phase's, and the kernel keeps the order they produce. Communities are
|
|
11052
|
-
* walked lexicographically by label so a night's call order is fixed, and each community's members
|
|
11053
|
-
* by `row.path` so the `m1`..`mN` keys land on the same files twice over.
|
|
11054
|
-
*/
|
|
11055
|
-
const groups = [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1).map(([, members]) => [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0));
|
|
11056
|
-
const batches = assembleBatches(groups, {
|
|
11057
|
-
maxMembers: 8,
|
|
11058
|
-
minMembers: 2
|
|
11059
|
-
});
|
|
11060
|
-
const counts = {
|
|
11061
|
-
candidates: candidates.length,
|
|
11062
|
-
communities: byCommunity.size,
|
|
11063
|
-
batches: batches.length,
|
|
11064
|
-
canonicals: 0,
|
|
11065
|
-
archived: 0,
|
|
11066
|
-
skipped: 0,
|
|
11067
|
-
failed: 0,
|
|
11068
|
-
refused: 0
|
|
11069
|
-
};
|
|
11070
|
-
if (batches.length === 0) return emptyOutcome(counts);
|
|
11071
|
-
if (env.dryRun) return emptyOutcome(counts);
|
|
11072
11399
|
const modelKey = modelFor(env.deps, "compress");
|
|
11073
|
-
|
|
11074
|
-
let canonicals = 0;
|
|
11075
|
-
let archived = 0;
|
|
11076
|
-
/**
|
|
11077
|
-
* `skipped` stays the total, and `failed` + `refused` partition it. The two are different
|
|
11078
|
-
* diagnoses with different fixes — a failed call is the model or the wire (already logged by
|
|
11079
|
-
* `isolate`), a refusal is an answer the phase declined to act on (logged below) — and a night
|
|
11080
|
-
* reporting only their sum cannot say which one it had. 47 of 47 batches once skipped as
|
|
11081
|
-
* refusals with nothing on stderr, and the sum read as flaky calls.
|
|
11082
|
-
*/
|
|
11083
|
-
let skipped = 0;
|
|
11084
|
-
let failed = 0;
|
|
11085
|
-
let refused = 0;
|
|
11400
|
+
const passes = [];
|
|
11086
11401
|
let lastCommit = null;
|
|
11087
|
-
for (
|
|
11088
|
-
|
|
11089
|
-
const
|
|
11090
|
-
llmCalls += 1;
|
|
11091
|
-
const synthesis = yield* batchCall(model, `compress batch of ${batch.length}`, {
|
|
11092
|
-
schema: CompressSynthesis,
|
|
11093
|
-
system: COMPRESS_SYSTEM,
|
|
11094
|
-
prompt: compressPrompt(keyed.keyed),
|
|
11095
|
-
modelKey,
|
|
11096
|
-
effort: "high",
|
|
11097
|
-
toolDescription: "Emit the canonical memory and the members whose content it absorbs."
|
|
11098
|
-
});
|
|
11099
|
-
if (synthesis === void 0) {
|
|
11100
|
-
skipped += 1;
|
|
11101
|
-
failed += 1;
|
|
11102
|
-
continue;
|
|
11103
|
-
}
|
|
11104
|
-
/** A key the batch never offered resolves to nothing, so a fold reaches only offered files. */
|
|
11105
|
-
const absorbed = resolveKeys(keyed, synthesis.absorbedKeys).map((entry) => entry.row.path);
|
|
11106
|
-
if (absorbed.length < 2 || synthesis.title.trim() === "" || synthesis.claim.trim() === "") {
|
|
11107
|
-
skipped += 1;
|
|
11108
|
-
refused += 1;
|
|
11109
|
-
yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: the model absorbed ${absorbed.length} of ${synthesis.absorbedKeys.length} named keys` + (synthesis.absorbedKeys.length > 0 && absorbed.length === 0 ? ` (none of the named keys resolved: ${synthesis.absorbedKeys.slice(0, 3).join(", ")}${synthesis.absorbedKeys.length > 3 ? ", …" : ""})` : ""));
|
|
11110
|
-
continue;
|
|
11111
|
-
}
|
|
11402
|
+
for (let passAt = 0; passAt < (env.deep === void 0 ? 1 : 3);) {
|
|
11403
|
+
passAt += 1;
|
|
11404
|
+
const banded = (yield* runRetentionPass(env.deps.db, env.at)).scored.filter((entry) => entry.score.action === "compress" && entry.row.memory_type !== "arc" && !isSleepExcluded(entry.row.memory_type));
|
|
11112
11405
|
/**
|
|
11113
|
-
*
|
|
11114
|
-
*
|
|
11115
|
-
*
|
|
11406
|
+
* Which label a candidate folds under. Nightly: the retention pass's own community, and a
|
|
11407
|
+
* memory without one is not a candidate — the exact selection this phase has always made.
|
|
11408
|
+
* Deep: the widened partition first, then an entity label for what the graph still missed.
|
|
11116
11409
|
*/
|
|
11117
|
-
const
|
|
11118
|
-
const
|
|
11119
|
-
const
|
|
11120
|
-
|
|
11121
|
-
|
|
11122
|
-
|
|
11123
|
-
|
|
11124
|
-
|
|
11410
|
+
const widened = env.deep === void 0 ? void 0 : yield* deepCommunityLabels(env);
|
|
11411
|
+
const communityOf = (entry) => widened === void 0 ? entry.community : widened.get(entry.row.path);
|
|
11412
|
+
const graphLabelled = banded.filter((entry) => communityOf(entry) !== void 0);
|
|
11413
|
+
const needing = env.deep === void 0 ? /* @__PURE__ */ new Set() : new Set(banded.filter((entry) => communityOf(entry) === void 0).map((entry) => entry.row.path));
|
|
11414
|
+
const entities = env.deep === void 0 ? {
|
|
11415
|
+
labels: /* @__PURE__ */ new Map(),
|
|
11416
|
+
hubsSkipped: 0
|
|
11417
|
+
} : yield* entityLabelsFor(env, needing);
|
|
11418
|
+
const candidates = [...graphLabelled, ...banded.filter((entry) => entities.labels.has(entry.row.path))].slice(0, COMPRESS_CANDIDATE_LIMIT);
|
|
11419
|
+
/** Community -> its COMPRESS-band members, both orders fixed so batching is reproducible. */
|
|
11420
|
+
const byCommunity = /* @__PURE__ */ new Map();
|
|
11421
|
+
for (const entry of candidates) {
|
|
11422
|
+
const label = communityOf(entry) ?? entities.labels.get(entry.row.path);
|
|
11423
|
+
if (label === void 0) continue;
|
|
11424
|
+
const bucket = byCommunity.get(label);
|
|
11425
|
+
if (bucket === void 0) byCommunity.set(label, [entry]);
|
|
11426
|
+
else bucket.push(entry);
|
|
11125
11427
|
}
|
|
11126
11428
|
/**
|
|
11127
|
-
*
|
|
11128
|
-
*
|
|
11129
|
-
*
|
|
11429
|
+
* Both sorts are this phase's, and the kernel keeps the order they produce. Communities are
|
|
11430
|
+
* walked lexicographically by label so a night's call order is fixed, and each community's members
|
|
11431
|
+
* by `row.path` so the `m1`..`mN` keys land on the same files twice over.
|
|
11130
11432
|
*/
|
|
11131
|
-
const
|
|
11132
|
-
|
|
11133
|
-
|
|
11134
|
-
|
|
11135
|
-
}
|
|
11136
|
-
if (archivedPaths.length === 0) {
|
|
11137
|
-
skipped += 1;
|
|
11138
|
-
refused += 1;
|
|
11139
|
-
yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every member was already gone from the tree`);
|
|
11140
|
-
continue;
|
|
11141
|
-
}
|
|
11142
|
-
yield* writeFileBytes(env, canonicalPath, renderTemplate({
|
|
11143
|
-
title: synthesis.title.trim(),
|
|
11144
|
-
claim: synthesis.claim,
|
|
11145
|
-
body: synthesis.paragraphs,
|
|
11146
|
-
memoryType: "semantic",
|
|
11147
|
-
at: env.at,
|
|
11148
|
-
author: "agent:sleep"
|
|
11149
|
-
}));
|
|
11150
|
-
for (const archivedPath of archivedPaths) yield* stampFile(env, canonicalPath, [link("supersedes", hrefFor(archivedPath))]);
|
|
11151
|
-
yield* env.deps.git.add([canonicalPath]);
|
|
11152
|
-
archived += archivedPaths.length;
|
|
11153
|
-
canonicals += 1;
|
|
11154
|
-
const commitSha = yield* commitPhase(env, "compress", `fold ${members.length} memories into ${synthesis.title}`, {
|
|
11155
|
-
...counts,
|
|
11156
|
-
canonicals,
|
|
11157
|
-
archived,
|
|
11158
|
-
skipped,
|
|
11159
|
-
failed,
|
|
11160
|
-
refused
|
|
11433
|
+
const groups = [...byCommunity.entries()].sort(([left], [right]) => left < right ? -1 : 1).map(([, members]) => [...members].sort((left, right) => left.row.path < right.row.path ? -1 : left.row.path > right.row.path ? 1 : 0));
|
|
11434
|
+
const batches = assembleBatches(groups, {
|
|
11435
|
+
maxMembers: 8,
|
|
11436
|
+
minMembers: 2
|
|
11161
11437
|
});
|
|
11162
|
-
|
|
11438
|
+
const tally = {
|
|
11439
|
+
candidates: candidates.length,
|
|
11440
|
+
communities: byCommunity.size,
|
|
11441
|
+
entityGroups: new Set(entities.labels.values()).size,
|
|
11442
|
+
batches: batches.length,
|
|
11443
|
+
canonicals: 0,
|
|
11444
|
+
archived: 0,
|
|
11445
|
+
skipped: 0,
|
|
11446
|
+
failed: 0,
|
|
11447
|
+
refused: 0,
|
|
11448
|
+
budget: 0,
|
|
11449
|
+
llmCalls: 0
|
|
11450
|
+
};
|
|
11451
|
+
passes.push(tally);
|
|
11452
|
+
if (batches.length === 0 || env.dryRun) break;
|
|
11453
|
+
for (const batch of batches) {
|
|
11454
|
+
/** Opaque keys again, so `absorbedKeys` cannot name a path. */
|
|
11455
|
+
const keyed = keyMembers(batch, (entry) => `${entry.row.title}\n${entry.row.gist}\n${entry.row.body_text}`, { charBudget: COMPRESS_MEMBER_CHARS });
|
|
11456
|
+
/**
|
|
11457
|
+
* The deep budget is charged BEFORE the call, and an exhausted budget is its own skip
|
|
11458
|
+
* reason: a night that stopped because the operator's cap ran out and a night whose model
|
|
11459
|
+
* fell over need different mornings-after, and `skipped` alone cannot say which this was.
|
|
11460
|
+
*/
|
|
11461
|
+
if (!takeLlmCall(env.deep)) {
|
|
11462
|
+
tally.skipped += 1;
|
|
11463
|
+
tally.budget += 1;
|
|
11464
|
+
continue;
|
|
11465
|
+
}
|
|
11466
|
+
tally.llmCalls += 1;
|
|
11467
|
+
const synthesis = yield* batchCall(model, `compress batch of ${batch.length}`, {
|
|
11468
|
+
schema: CompressSynthesis,
|
|
11469
|
+
system: COMPRESS_SYSTEM,
|
|
11470
|
+
prompt: compressPrompt(keyed.keyed),
|
|
11471
|
+
modelKey,
|
|
11472
|
+
effort: "high",
|
|
11473
|
+
toolDescription: "Emit the canonical memory and the members whose content it absorbs."
|
|
11474
|
+
});
|
|
11475
|
+
if (synthesis === void 0) {
|
|
11476
|
+
tally.skipped += 1;
|
|
11477
|
+
tally.failed += 1;
|
|
11478
|
+
continue;
|
|
11479
|
+
}
|
|
11480
|
+
/** A key the batch never offered resolves to nothing, so a fold reaches only offered files. */
|
|
11481
|
+
const absorbed = resolveKeys(keyed, synthesis.absorbedKeys).map((entry) => entry.row.path);
|
|
11482
|
+
if (absorbed.length < 2 || synthesis.title.trim() === "" || synthesis.claim.trim() === "") {
|
|
11483
|
+
tally.skipped += 1;
|
|
11484
|
+
tally.refused += 1;
|
|
11485
|
+
yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: the model absorbed ${absorbed.length} of ${synthesis.absorbedKeys.length} named keys` + (synthesis.absorbedKeys.length > 0 && absorbed.length === 0 ? ` (none of the named keys resolved: ${synthesis.absorbedKeys.slice(0, 3).join(", ")}${synthesis.absorbedKeys.length > 3 ? ", …" : ""})` : ""));
|
|
11486
|
+
continue;
|
|
11487
|
+
}
|
|
11488
|
+
/**
|
|
11489
|
+
* The canonical is placed in the batch's own directory when the members agree on one, and in the
|
|
11490
|
+
* inbox otherwise. Placing it under a member's directory keeps a compressed group where a reader
|
|
11491
|
+
* would look for it, and `memhtml doctor` reports inbox depth so a disagreeing batch is visible.
|
|
11492
|
+
*/
|
|
11493
|
+
const directories = new Set(absorbed.map((path) => path.slice(0, path.lastIndexOf("/"))));
|
|
11494
|
+
const canonicalPath = `${(directories.size === 1 ? [...directories][0] : "areas/inbox") ?? "areas/inbox"}/${slugify(synthesis.title)}.html`;
|
|
11495
|
+
const members = excludeSelfSupersede(canonicalPath, absorbed);
|
|
11496
|
+
if (members.length === 0) {
|
|
11497
|
+
tally.skipped += 1;
|
|
11498
|
+
tally.refused += 1;
|
|
11499
|
+
yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every absorbed member was the canonical itself`);
|
|
11500
|
+
continue;
|
|
11501
|
+
}
|
|
11502
|
+
/**
|
|
11503
|
+
* The members are archived FIRST, and the canonical is written only if at least one member was
|
|
11504
|
+
* actually moved. A batch whose members an earlier phase already evicted would otherwise leave a
|
|
11505
|
+
* canonical behind claiming to supersede files it never absorbed.
|
|
11506
|
+
*/
|
|
11507
|
+
const archivedPaths = [];
|
|
11508
|
+
for (const member of members) {
|
|
11509
|
+
const archivedPath = yield* archiveFile(env, member, [meta("memhtml-superseded-by", hrefFor(canonicalPath))]);
|
|
11510
|
+
if (archivedPath !== null) archivedPaths.push(archivedPath);
|
|
11511
|
+
}
|
|
11512
|
+
if (archivedPaths.length === 0) {
|
|
11513
|
+
tally.skipped += 1;
|
|
11514
|
+
tally.refused += 1;
|
|
11515
|
+
yield* Effect.logWarning(`sleep.llm compress batch of ${batch.length} refused: every member was already gone from the tree`);
|
|
11516
|
+
continue;
|
|
11517
|
+
}
|
|
11518
|
+
yield* writeFileBytes(env, canonicalPath, renderTemplate({
|
|
11519
|
+
title: synthesis.title.trim(),
|
|
11520
|
+
claim: synthesis.claim,
|
|
11521
|
+
body: synthesis.paragraphs,
|
|
11522
|
+
memoryType: "semantic",
|
|
11523
|
+
at: env.at,
|
|
11524
|
+
author: "agent:sleep"
|
|
11525
|
+
}));
|
|
11526
|
+
for (const archivedPath of archivedPaths) yield* stampFile(env, canonicalPath, [link("supersedes", hrefFor(archivedPath))]);
|
|
11527
|
+
yield* env.deps.git.add([canonicalPath]);
|
|
11528
|
+
tally.archived += archivedPaths.length;
|
|
11529
|
+
tally.canonicals += 1;
|
|
11530
|
+
const commitSha = yield* commitPhase(env, "compress", `fold ${members.length} memories into ${synthesis.title}`, totalsOf(passes));
|
|
11531
|
+
if (commitSha !== null) lastCommit = commitSha;
|
|
11532
|
+
}
|
|
11533
|
+
/**
|
|
11534
|
+
* Iterate-until-quiet, deep only (issue #63). A pass that folded nothing has reached the
|
|
11535
|
+
* fixed point; one that folded something changed the neighbor structure, so the branch is
|
|
11536
|
+
* re-indexed (renames tracked, new canonicals embedded), both bands are re-mined over the new
|
|
11537
|
+
* vectors, and the next pass sees the canonicals as members. The nightly cycle exits here
|
|
11538
|
+
* unconditionally, which is what keeps its behavior single-pass and byte-identical.
|
|
11539
|
+
*/
|
|
11540
|
+
if (env.deep === void 0 || tally.canonicals === 0) break;
|
|
11541
|
+
if (passAt >= 3) break;
|
|
11542
|
+
yield* env.deps.indexer.update({ embed: true });
|
|
11543
|
+
yield* mineAllBands(env);
|
|
11163
11544
|
}
|
|
11164
11545
|
return {
|
|
11165
|
-
counts:
|
|
11166
|
-
...counts,
|
|
11167
|
-
canonicals,
|
|
11168
|
-
archived,
|
|
11169
|
-
skipped,
|
|
11170
|
-
failed,
|
|
11171
|
-
refused
|
|
11172
|
-
},
|
|
11546
|
+
counts: totalsOf(passes),
|
|
11173
11547
|
commitSha: lastCommit,
|
|
11174
|
-
llmCalls
|
|
11548
|
+
llmCalls: passes.reduce((total, tally) => total + tally.llmCalls, 0)
|
|
11175
11549
|
};
|
|
11176
11550
|
});
|
|
11551
|
+
/**
|
|
11552
|
+
* The pass tallies as one counts record. Work counters SUM across passes; `candidates` and
|
|
11553
|
+
* `communities` describe the corpus the run STARTED from (the first pass), because a sum of
|
|
11554
|
+
* re-scans of one shrinking corpus counts nothing a reviewer can reconcile. On a nightly
|
|
11555
|
+
* single-pass run every key and every number is byte-identical to what this phase has always
|
|
11556
|
+
* reported — the deep-only keys appear only when a deep quantity is nonzero or a second pass ran,
|
|
11557
|
+
* which cannot happen without the flag. Response counts are append-only, so the deep keys are
|
|
11558
|
+
* additions and no shipped key changes meaning.
|
|
11559
|
+
*/
|
|
11560
|
+
const totalsOf = (passes) => {
|
|
11561
|
+
const first = passes[0];
|
|
11562
|
+
const sum = (of) => passes.reduce((total, tally) => total + of(tally), 0);
|
|
11563
|
+
const deepish = passes.length > 1 || (first?.entityGroups ?? 0) > 0 || sum((tally) => tally.budget) > 0;
|
|
11564
|
+
return {
|
|
11565
|
+
candidates: first?.candidates ?? 0,
|
|
11566
|
+
communities: first?.communities ?? 0,
|
|
11567
|
+
batches: sum((tally) => tally.batches),
|
|
11568
|
+
canonicals: sum((tally) => tally.canonicals),
|
|
11569
|
+
archived: sum((tally) => tally.archived),
|
|
11570
|
+
skipped: sum((tally) => tally.skipped),
|
|
11571
|
+
failed: sum((tally) => tally.failed),
|
|
11572
|
+
refused: sum((tally) => tally.refused),
|
|
11573
|
+
...deepish ? {
|
|
11574
|
+
passes: passes.length,
|
|
11575
|
+
entityGroups: first?.entityGroups ?? 0,
|
|
11576
|
+
budgetSkipped: sum((tally) => tally.budget)
|
|
11577
|
+
} : {}
|
|
11578
|
+
};
|
|
11579
|
+
};
|
|
11177
11580
|
|
|
11178
11581
|
//#endregion
|
|
11179
11582
|
//#region packages/sleep/dist/phases/confidence-decay.js
|
|
@@ -14129,6 +14532,228 @@ const personLinks = (env) => Effect.gen(function* () {
|
|
|
14129
14532
|
};
|
|
14130
14533
|
});
|
|
14131
14534
|
|
|
14535
|
+
//#endregion
|
|
14536
|
+
//#region packages/sleep/dist/phases/placement-triage.js
|
|
14537
|
+
/**
|
|
14538
|
+
* Phase 14, placement triage. DEEP-ONLY (issue #63): propose a topic directory for each inbox
|
|
14539
|
+
* memory that even deep grouping left communityless, and `git mv` the ones the model places with
|
|
14540
|
+
* confidence. ONE COMMIT.
|
|
14541
|
+
*
|
|
14542
|
+
* The mechanism this phase exists for: a bulk-imported inbox is mostly NOT compressible — each file
|
|
14543
|
+
* is a distinct fact — and what keeps the inbox crowded is that nothing ever re-files them. `doctor`
|
|
14544
|
+
* reports `inboxCrowded` and no phase acts on it. This one does, under the same discipline as every
|
|
14545
|
+
* other mutation: on the review branch, in its own commit, reversible by discarding the branch.
|
|
14546
|
+
*
|
|
14547
|
+
* **On a nightly run the phase returns immediately**, before any read, with a reason — the same
|
|
14548
|
+
* shape `compress` has for a missing model. That is the whole no-flag contract: a run without
|
|
14549
|
+
* `--deep` cannot reach a single line of this phase's work.
|
|
14550
|
+
*
|
|
14551
|
+
* The deterministic guardrails, each enforced in code after the model answers:
|
|
14552
|
+
*
|
|
14553
|
+
* - a destination must be an EXISTING directory (as the index knows them) or one of at most
|
|
14554
|
+
* {@link PLACEMENT_NEW_DIR_CAP} new `areas/<topic>` or `resources/<topic>` directories per run —
|
|
14555
|
+
* first proposed, first minted, lexicographic order within one batch;
|
|
14556
|
+
* - the managed surfaces refuse: the inbox itself, `areas/arcs`, `resources/people`, anything under
|
|
14557
|
+
* `archive/`, and any path outside the PARA buckets;
|
|
14558
|
+
* - tasks never move (excluded from the scan AND re-checked per row, matching the double guard
|
|
14559
|
+
* task-detection carries);
|
|
14560
|
+
* - a file another phase already touched this run never moves, read from `git diff --name-only
|
|
14561
|
+
* base..HEAD` — a `mv` of a path compress just archived would fail, and one of a path an earlier
|
|
14562
|
+
* phase stamped would tear that phase's edit out of its own commit's diff;
|
|
14563
|
+
* - `keep-inbox`, an unknown key, a below-floor confidence, and an omitted member all leave the
|
|
14564
|
+
* file where it is.
|
|
14565
|
+
*
|
|
14566
|
+
* **Inbound hrefs are rewritten in the same commit.** Integrity's dangling-href repair chases a
|
|
14567
|
+
* target into the ARCHIVE by deriving `archivePathFor`; a placement move is not an archive, so this
|
|
14568
|
+
* phase rewrites `<link>` elements in the files that point at each moved path itself, the same
|
|
14569
|
+
* remove-then-add splice integrity uses. The indexer tracks the rename (`diff -M` + `movePath`), so
|
|
14570
|
+
* chunks and embeddings carry over and nothing re-embeds.
|
|
14571
|
+
*/
|
|
14572
|
+
/** Inbox memories offered per model call. Placement is a lighter question than a fold, so wider. */
|
|
14573
|
+
const PLACEMENT_BATCH_SIZE = 16;
|
|
14574
|
+
/** Inbox memories considered per run. The model-cost guard, in the spirit of compress's 2000. */
|
|
14575
|
+
const PLACEMENT_CANDIDATE_LIMIT = 2e3;
|
|
14576
|
+
/** Characters of each member shown. Placement needs the topic, not every fact. */
|
|
14577
|
+
const PLACEMENT_MEMBER_CHARS = 600;
|
|
14578
|
+
/**
|
|
14579
|
+
* New topic directories one run may mint. Small on purpose: a new directory is a new place every
|
|
14580
|
+
* future reader and `placementFor` caller has to know about, and a bulk import that genuinely needs
|
|
14581
|
+
* thirty new topics should earn them over several reviewed runs, not one.
|
|
14582
|
+
*/
|
|
14583
|
+
const PLACEMENT_NEW_DIR_CAP = 5;
|
|
14584
|
+
const placementTriage = (env) => Effect.gen(function* () {
|
|
14585
|
+
if (env.deep === void 0) return {
|
|
14586
|
+
...emptyOutcome({ candidates: 0 }),
|
|
14587
|
+
detail: "deep-only phase; run with --deep"
|
|
14588
|
+
};
|
|
14589
|
+
const model = env.deps.model;
|
|
14590
|
+
if (model === void 0) return {
|
|
14591
|
+
...emptyOutcome({ candidates: 0 }),
|
|
14592
|
+
detail: "no model bound"
|
|
14593
|
+
};
|
|
14594
|
+
/**
|
|
14595
|
+
* Candidates: active inbox memories with no community even under the WIDENED graph — nightly
|
|
14596
|
+
* edges plus the deep grouping band, the same partition deep compress groups by. Issue #63's
|
|
14597
|
+
* "true singleton" is exactly this: a file even the 0.72 band could not attach to anything.
|
|
14598
|
+
* A file the widened graph DID reach belongs to compress's pipeline (this run or a later one),
|
|
14599
|
+
* not to placement; an ENTITY-grouped file is deliberately still placeable, because entity
|
|
14600
|
+
* grouping is a compress-band mechanism and a KEEP-band inbox file it cannot fold still needs
|
|
14601
|
+
* a home.
|
|
14602
|
+
*/
|
|
14603
|
+
const widened = yield* deepCommunityLabels(env);
|
|
14604
|
+
const corpus = yield* activeCorpus(env.deps.db);
|
|
14605
|
+
const inInbox = (path) => path.startsWith(`${"areas/inbox"}/`) && !path.startsWith(`${"areas/inbox"}/tasks/`);
|
|
14606
|
+
const candidates = corpus.filter((row) => inInbox(row.path) && !isSleepExcluded(row.memory_type) && row.memory_type !== "arc" && widened.get(row.path) === void 0).sort((left, right) => left.path < right.path ? -1 : 1).slice(0, PLACEMENT_CANDIDATE_LIMIT);
|
|
14607
|
+
/**
|
|
14608
|
+
* The directories the corpus already has, from the index's own path set: every distinct
|
|
14609
|
+
* directory holding an active file under `areas/` or `resources/`, minus the managed surfaces.
|
|
14610
|
+
* Offered to the model verbatim and used as the validation set, so the question and the gate
|
|
14611
|
+
* cannot disagree.
|
|
14612
|
+
*/
|
|
14613
|
+
const managed = /* @__PURE__ */ new Set([
|
|
14614
|
+
INBOX_DIR,
|
|
14615
|
+
`${INBOX_DIR}/tasks`,
|
|
14616
|
+
ARCS_DIR,
|
|
14617
|
+
PEOPLE_DIR
|
|
14618
|
+
]);
|
|
14619
|
+
const existingDirs = [...new Set(corpus.map((row) => row.path.slice(0, row.path.lastIndexOf("/"))).filter((dir) => (dir.startsWith("areas/") || dir.startsWith("resources/")) && !managed.has(dir)))].sort();
|
|
14620
|
+
/** Paths this run already touched. A move of one would cross-contaminate another phase's diff. */
|
|
14621
|
+
const touched = new Set(env.baseSha === "" ? [] : yield* env.deps.git.run([
|
|
14622
|
+
"diff",
|
|
14623
|
+
"--name-only",
|
|
14624
|
+
`${env.baseSha}..HEAD`
|
|
14625
|
+
]).pipe(Effect.map((out) => out.split("\n").map((line) => line.trim()).filter(Boolean)), Effect.orElseSucceed(() => [])));
|
|
14626
|
+
const batches = assembleBatches([candidates], { maxMembers: 16 });
|
|
14627
|
+
const counts = {
|
|
14628
|
+
candidates: candidates.length,
|
|
14629
|
+
batches: batches.length,
|
|
14630
|
+
proposed: 0,
|
|
14631
|
+
applied: 0,
|
|
14632
|
+
refused: 0,
|
|
14633
|
+
keptInbox: 0,
|
|
14634
|
+
newDirs: 0,
|
|
14635
|
+
budgetSkipped: 0,
|
|
14636
|
+
failed: 0
|
|
14637
|
+
};
|
|
14638
|
+
if (batches.length === 0 || env.dryRun) return emptyOutcome(counts);
|
|
14639
|
+
const modelKey = modelFor(env.deps, "placement-triage");
|
|
14640
|
+
const mintedDirs = /* @__PURE__ */ new Set();
|
|
14641
|
+
const movedFrom = /* @__PURE__ */ new Map();
|
|
14642
|
+
let llmCalls = 0;
|
|
14643
|
+
for (const batch of batches) {
|
|
14644
|
+
const keyed = keyMembers(batch, (row) => `${row.title}\n${row.gist}\n${row.body_text}`, { charBudget: 600 });
|
|
14645
|
+
if (!takeLlmCall(env.deep)) {
|
|
14646
|
+
counts["budgetSkipped"] = (counts["budgetSkipped"] ?? 0) + 1;
|
|
14647
|
+
continue;
|
|
14648
|
+
}
|
|
14649
|
+
llmCalls += 1;
|
|
14650
|
+
const answer = yield* batchCall(model, `placement batch of ${batch.length}`, {
|
|
14651
|
+
schema: PlacementTriage,
|
|
14652
|
+
system: PLACEMENT_SYSTEM,
|
|
14653
|
+
prompt: placementPrompt(existingDirs, keyed.keyed),
|
|
14654
|
+
modelKey,
|
|
14655
|
+
effort: "high",
|
|
14656
|
+
toolDescription: "Propose a destination directory (or keep-inbox) per offered memory."
|
|
14657
|
+
});
|
|
14658
|
+
if (answer === void 0) {
|
|
14659
|
+
counts["failed"] = (counts["failed"] ?? 0) + 1;
|
|
14660
|
+
continue;
|
|
14661
|
+
}
|
|
14662
|
+
/**
|
|
14663
|
+
* Placements resolve through the kernel, so an invented key drops. Sorted by (destination,
|
|
14664
|
+
* key) so within one answer the minting order of new directories is a function of the answer's
|
|
14665
|
+
* CONTENT rather than of its field order.
|
|
14666
|
+
*/
|
|
14667
|
+
const resolvedRows = answer.placements.filter((placement) => placement.destination.trim() !== "").sort((left, right) => left.destination < right.destination ? -1 : left.destination > right.destination ? 1 : left.memberKey < right.memberKey ? -1 : 1);
|
|
14668
|
+
for (const placement of resolvedRows) {
|
|
14669
|
+
const row = resolveKeys(keyed, [placement.memberKey])[0];
|
|
14670
|
+
if (row === void 0) continue;
|
|
14671
|
+
counts["proposed"] = (counts["proposed"] ?? 0) + 1;
|
|
14672
|
+
const destination = normalizePath(placement.destination.trim());
|
|
14673
|
+
if (destination === "keep-inbox") {
|
|
14674
|
+
counts["keptInbox"] = (counts["keptInbox"] ?? 0) + 1;
|
|
14675
|
+
continue;
|
|
14676
|
+
}
|
|
14677
|
+
const refuse = (reason) => {
|
|
14678
|
+
counts["refused"] = (counts["refused"] ?? 0) + 1;
|
|
14679
|
+
return Effect.logWarning(`sleep.placement refused ${row.path} -> ${destination}: ${reason}`);
|
|
14680
|
+
};
|
|
14681
|
+
if (placement.confidence < .7) {
|
|
14682
|
+
yield* refuse("below the confidence floor");
|
|
14683
|
+
continue;
|
|
14684
|
+
}
|
|
14685
|
+
const bucket = paraBucketOf(destination);
|
|
14686
|
+
if (bucket !== "areas" && bucket !== "resources" || managed.has(destination) || destination === row.path.slice(0, row.path.lastIndexOf("/")) || destination.split("/").some((segment) => segment === "" || segment === "." || segment === "..")) {
|
|
14687
|
+
yield* refuse("not a placeable directory");
|
|
14688
|
+
continue;
|
|
14689
|
+
}
|
|
14690
|
+
if (isSleepExcluded(row.memory_type)) {
|
|
14691
|
+
yield* refuse("tasks never move");
|
|
14692
|
+
continue;
|
|
14693
|
+
}
|
|
14694
|
+
if (touched.has(row.path) || movedFrom.has(row.path)) {
|
|
14695
|
+
yield* refuse("another phase touched this file this run");
|
|
14696
|
+
continue;
|
|
14697
|
+
}
|
|
14698
|
+
const isNew = !existingDirs.includes(destination) && !mintedDirs.has(destination);
|
|
14699
|
+
if (isNew && mintedDirs.size >= 5) {
|
|
14700
|
+
yield* refuse("the new-directory cap for this run is spent");
|
|
14701
|
+
continue;
|
|
14702
|
+
}
|
|
14703
|
+
const target = `${destination}/${row.path.slice(row.path.lastIndexOf("/") + 1)}`;
|
|
14704
|
+
if (!isValidMemoryPath(target)) {
|
|
14705
|
+
yield* refuse("the destination path is not a valid memory path");
|
|
14706
|
+
continue;
|
|
14707
|
+
}
|
|
14708
|
+
if ((yield* readFileBytes(env, target)) !== void 0) {
|
|
14709
|
+
yield* refuse("the destination already holds a file by that name");
|
|
14710
|
+
continue;
|
|
14711
|
+
}
|
|
14712
|
+
if ((yield* readFileBytes(env, row.path)) === void 0) {
|
|
14713
|
+
yield* refuse("the source file is already gone from the tree");
|
|
14714
|
+
continue;
|
|
14715
|
+
}
|
|
14716
|
+
yield* attemptIo(`sleep.placement.mkdir:${target}`, async () => {
|
|
14717
|
+
const { mkdir } = await import("node:fs/promises");
|
|
14718
|
+
const { dirname } = await import("node:path");
|
|
14719
|
+
await mkdir(dirname(absoluteIn(env, target)), { recursive: true });
|
|
14720
|
+
});
|
|
14721
|
+
yield* env.deps.git.mv(row.path, target);
|
|
14722
|
+
yield* stampFile(env, target, [meta("memhtml-updated", env.at)]);
|
|
14723
|
+
if (isNew) {
|
|
14724
|
+
mintedDirs.add(destination);
|
|
14725
|
+
counts["newDirs"] = (counts["newDirs"] ?? 0) + 1;
|
|
14726
|
+
}
|
|
14727
|
+
movedFrom.set(row.path, target);
|
|
14728
|
+
counts["applied"] = (counts["applied"] ?? 0) + 1;
|
|
14729
|
+
}
|
|
14730
|
+
}
|
|
14731
|
+
/**
|
|
14732
|
+
* Inbound href repair, inside the same commit as the moves. Every authored `<link>` whose
|
|
14733
|
+
* target moved is respliced in ITS OWN file to the new path — remove-then-add, integrity's
|
|
14734
|
+
* idempotent shape — so the branch never carries a commit whose links dangle by construction.
|
|
14735
|
+
*/
|
|
14736
|
+
let rewritten = 0;
|
|
14737
|
+
for (const [from, to] of [...movedFrom.entries()].sort(([left], [right]) => left < right ? -1 : 1)) {
|
|
14738
|
+
const inbound = yield* inboundAuthoredEdges(env.deps.db, from);
|
|
14739
|
+
for (const edge of inbound) {
|
|
14740
|
+
if (!isEdgeRel(edge.rel)) continue;
|
|
14741
|
+
const holder = movedFrom.get(edge.src_path) ?? edge.src_path;
|
|
14742
|
+
if (yield* stampFile(env, holder, [
|
|
14743
|
+
unlink(edge.rel, hrefFor(from)),
|
|
14744
|
+
link(edge.rel, hrefFor(to)),
|
|
14745
|
+
meta("memhtml-updated", env.at)
|
|
14746
|
+
])) rewritten += 1;
|
|
14747
|
+
}
|
|
14748
|
+
}
|
|
14749
|
+
counts["hrefsRewritten"] = rewritten;
|
|
14750
|
+
return {
|
|
14751
|
+
counts,
|
|
14752
|
+
commitSha: yield* commitPhase(env, "placement-triage", `re-file ${counts["applied"] ?? 0} inbox memories into topic directories`, counts),
|
|
14753
|
+
llmCalls
|
|
14754
|
+
};
|
|
14755
|
+
});
|
|
14756
|
+
|
|
14132
14757
|
//#endregion
|
|
14133
14758
|
//#region packages/sleep/dist/phases/preflight.js
|
|
14134
14759
|
/**
|
|
@@ -14165,60 +14790,6 @@ const preflight = (env) => Effect.gen(function* () {
|
|
|
14165
14790
|
}) };
|
|
14166
14791
|
});
|
|
14167
14792
|
|
|
14168
|
-
//#endregion
|
|
14169
|
-
//#region packages/sleep/dist/phases/relationship-mining.js
|
|
14170
|
-
/**
|
|
14171
|
-
* Phase 5, relationship mining. Derived `relates_to` edges in the index only. NO COMMIT.
|
|
14172
|
-
*
|
|
14173
|
-
* A mined edge is a re-derivable function of the corpus and the embedder. `index rebuild` plus the
|
|
14174
|
-
* next night's mining regenerates the identical set, so committing thousands of them would bury every
|
|
14175
|
-
* real diff in machine noise for zero recoverable information. The `derived` column is the firewall
|
|
14176
|
-
* that makes losing them cheap. The retention penalty counts only `derived = 0`, so an
|
|
14177
|
-
* uncorroborated machine suspicion cannot evict a memory.
|
|
14178
|
-
*
|
|
14179
|
-
* The insert is scoped to `provenance = 'sleep'` and `derived = 1` and the whole replace is one
|
|
14180
|
-
* atomic batch, so an authored edge is unreachable from here and the corpus is never left with the
|
|
14181
|
-
* old mined set deleted and the new one not yet written. In that window the lateral arm would
|
|
14182
|
-
* silently return nothing.
|
|
14183
|
-
*/
|
|
14184
|
-
/** The similarity floor a pair must clear to become a mined `relates_to`. */
|
|
14185
|
-
const MINING_COSINE_FLOOR = .85;
|
|
14186
|
-
/** Nearest neighbors considered per source file. */
|
|
14187
|
-
const MINING_PER_SOURCE_K = 5;
|
|
14188
|
-
/**
|
|
14189
|
-
* Pairs mined per cycle: a cap on what {@link replaceMinedEdges} writes, not on the scan — the
|
|
14190
|
-
* kernel's arithmetic is O(n²·d) whatever this says, and it bounds the edge table so one dense
|
|
14191
|
-
* neighborhood cannot flood the graph the lateral arm and PageRank read.
|
|
14192
|
-
*/
|
|
14193
|
-
const MINING_SAMPLE_LIMIT = 2e3;
|
|
14194
|
-
const relationshipMining = (env) => Effect.gen(function* () {
|
|
14195
|
-
/**
|
|
14196
|
-
* Tasks are excluded, and here the exclusion is the graph firewall, not a cost guard.
|
|
14197
|
-
* Every mined edge is written with `edge_class = 'memory'`, so a pair with a task endpoint
|
|
14198
|
-
* would put a task INTO the memory graph, reaching PageRank, MMR, and the retention bridge
|
|
14199
|
-
* count. The `edges` CHECK cannot refuse it, because `relates_to` under `memory` is a
|
|
14200
|
-
* well-formed edge whatever files sit at its ends.
|
|
14201
|
-
*/
|
|
14202
|
-
const pairs = yield* neighborPairs(env.deps.db, {
|
|
14203
|
-
floor: MINING_COSINE_FLOOR,
|
|
14204
|
-
perSourceK: 5,
|
|
14205
|
-
limit: MINING_SAMPLE_LIMIT,
|
|
14206
|
-
excludeTypes: SLEEP_EXCLUDED_TYPES
|
|
14207
|
-
});
|
|
14208
|
-
const counts = {
|
|
14209
|
-
candidates: pairs.length,
|
|
14210
|
-
mined: pairs.length
|
|
14211
|
-
};
|
|
14212
|
-
if (env.dryRun) return emptyOutcome(counts);
|
|
14213
|
-
yield* replaceMinedEdges(env.deps.db, {
|
|
14214
|
-
runId: env.runId,
|
|
14215
|
-
at: env.at,
|
|
14216
|
-
rel: "relates_to",
|
|
14217
|
-
pairs
|
|
14218
|
-
});
|
|
14219
|
-
return emptyOutcome(counts);
|
|
14220
|
-
});
|
|
14221
|
-
|
|
14222
14793
|
//#endregion
|
|
14223
14794
|
//#region packages/sleep/dist/report.js
|
|
14224
14795
|
/**
|
|
@@ -15812,6 +16383,7 @@ const PHASE_BODIES = {
|
|
|
15812
16383
|
reprieve,
|
|
15813
16384
|
"trace-consolidation": traceConsolidation,
|
|
15814
16385
|
"task-detection": taskDetection,
|
|
16386
|
+
"placement-triage": placementTriage,
|
|
15815
16387
|
integrity,
|
|
15816
16388
|
"state-export": stateExport,
|
|
15817
16389
|
report: reportPhase([])
|
|
@@ -15872,7 +16444,14 @@ const run = (deps, options) => Effect.gen(function* () {
|
|
|
15872
16444
|
* Created per run rather than held in a module, which is what keeps two runs in one process (and
|
|
15873
16445
|
* two tests in one file) from sharing a counter.
|
|
15874
16446
|
*/
|
|
15875
|
-
detectionBudget: makeDetectionBudget()
|
|
16447
|
+
detectionBudget: makeDetectionBudget(),
|
|
16448
|
+
/**
|
|
16449
|
+
* The deep switches, shaped once here so every phase reads one field. The LLM budget is the
|
|
16450
|
+
* same created-per-run discipline as the detection budget above, and it exists only when the
|
|
16451
|
+
* caller stated a cap: an uncapped deep run is a valid ask, and a phantom cap of zero (or of
|
|
16452
|
+
* some default nobody chose) would silently skip work the operator paid for the flag to reach.
|
|
16453
|
+
*/
|
|
16454
|
+
...options.deep === true ? { deep: { ...options.maxLlmCalls === void 0 ? {} : { budget: makeLlmBudget(options.maxLlmCalls) } } } : {}
|
|
15876
16455
|
};
|
|
15877
16456
|
if (!dryRun) yield* deps.git.checkoutBranch(runId, { create: true }).pipe(Effect.orElseSucceed(() => {}));
|
|
15878
16457
|
yield* ignoreFailure(recordRun(deps.db, {
|
|
@@ -15939,7 +16518,9 @@ const resume = (deps, runId, options = {}) => Effect.gen(function* () {
|
|
|
15939
16518
|
* finishing. The cost of a fresh one is bounded by the cap, and the mints a resume repeats are
|
|
15940
16519
|
* refreshes rather than duplicates, which cost no budget at all.
|
|
15941
16520
|
*/
|
|
15942
|
-
detectionBudget: makeDetectionBudget()
|
|
16521
|
+
detectionBudget: makeDetectionBudget(),
|
|
16522
|
+
/** A fresh LLM budget too, for the same reason the detection budget above is fresh. */
|
|
16523
|
+
...options.deep === true ? { deep: { ...options.maxLlmCalls === void 0 ? {} : { budget: makeLlmBudget(options.maxLlmCalls) } } } : {}
|
|
15943
16524
|
};
|
|
15944
16525
|
const remaining = SLEEP_PHASES.filter((phase) => !completed.has(phase));
|
|
15945
16526
|
const executed = yield* executePhases(env, remaining, /* @__PURE__ */ new Set());
|
|
@@ -15947,7 +16528,7 @@ const resume = (deps, runId, options = {}) => Effect.gen(function* () {
|
|
|
15947
16528
|
const ended = yield* nowIso;
|
|
15948
16529
|
/**
|
|
15949
16530
|
* Skipped-because-already-done rows are reported explicitly, so a resume's report accounts for all
|
|
15950
|
-
*
|
|
16531
|
+
* seventeen phases. A report that showed only the eight it ran would read as a partial run.
|
|
15951
16532
|
*/
|
|
15952
16533
|
const priorRows = yield* ignoreFailureWith(readPhases(deps.db, runId), []);
|
|
15953
16534
|
const already = [...completed].map((phase) => {
|
|
@@ -16988,4 +17569,4 @@ const latest = (left, right) => left === null ? right : right === null ? left :
|
|
|
16988
17569
|
|
|
16989
17570
|
//#endregion
|
|
16990
17571
|
export { STATE_DB_PATH as $, IndexRecorder as A, IndexGit as B, ModelClient as C, EmbeddingsLive as D, Embeddings as E, makeRetrieval as F, MIGRATIONS_DIR as G, sanitizeFtsQuery as H, reinforce as I, Store as J, STATE_MIGRATIONS_DIR as K, Indexer as L, persistScanned as M, readWatermark as N, EMBED_DIM as O, Retrieval as P, SLEEP_REPORTS_DIR as Q, makeIndexer as R, runDiscrimination as S, wrapAsData as T, DatabaseService as U, makeGitPort as V, makeDatabase as W, makeStore as X, expandRoot as Y, INDEX_DB_PATH as Z, meta as _, parseSidecar as a, makeGit as at, isSleepPhase as b, generateArtifacts as c, setMeta as ct, allPaths as d, fenceOpeningOf as dt, STATE_SIDECAR_PATH as et, danglingEdges as f, REINFORCE_SIGNALS as ft, link as g, hrefFor as h, makeSleep as i, Git as it, makeIndexRecorder as j, EMBED_WATERMARK as k, DETECTION_PREFIX as l, isValidDatetime as lt, applyHeadEdits as m, scanTraceRoot as n, initRepo as nt, renderSidecar as o, commitSubject as ot, publishRows as p, frameKeyOf as pt, STATE_SCHEMA as q, Sleep as r, readFileOrNull as rt, archivedFormOf as s, checkMemory as st, mergeTailExtract as t, attemptIo as tt, accessRows as u, closesFence as ut, unlink as v, ModelClientLive as w, discriminationGate as x, SLEEP_PHASES as y, readIndexState as z };
|
|
16991
|
-
//# sourceMappingURL=dist-
|
|
17572
|
+
//# sourceMappingURL=dist-CHoz5uHd.mjs.map
|