@elvisw/dsh-magic-context 0.0.0-bootstrap.0

package/lib/dsh/historian.js

@@ -0,0 +1,907 @@
+/**
+ * The historian — Magic Context's summarizer, translated to a durable
+ * compartment instead of an in-band checkpoint.
+ *
+ * Magic Context's discipline, kept intact:
+ *  - the historian is a MODEL-FREE transform from the pass's point of view:
+ *    its LLM call runs out of band and only writes a store row; nothing on the
+ *    wire changes because of it (publishing is a separate, busting pass);
+ *  - it digests a contiguous span from the FRONT of the context, never the
+ *    protected tail, never a span that would break tool-call pairing;
+ *  - it fires only when the eligible span is worth a call (Magic Context's
+ *    trigger budget: 5% of the window, clamped to 5k–50k tokens);
+ *  - a failed or invalid response changes nothing (fail closed, one repair
+ *    attempt only).
+ *
+ * Divergence from Magic Context, deliberate and documented: the four detail
+ * tiers are produced in one call (MC's ladder is accumulated across calls),
+ * and the token estimate is chars/4 rather than a real tokenizer.
+ */
+import { estimateTokens, HistorianOutputError, parseHistorianOutput, } from "../core/compartments.js";
+import { REFERENCE_ANCHORS } from "../core/reference-seeds.js";
+import { errText } from "../core/types.js";
+import { spanRequestMessages } from "./seams.js";
+/**
+ * The historian's default wall-clock budget, used only as a runtime guard for
+ * a policy object that predates `policy.historianTimeoutMs` (the core policy
+ * table now always carries the field; see `DEFAULT_POLICY`).
+ */
+export const DEFAULT_HISTORIAN_TIMEOUT_MS = 120_000;
+/**
+ * The run exceeded its wall-clock budget. Distinct from a provider failure so
+ * the journal can say *why* nothing was stored.
+ */
+export class HistorianTimeoutError extends Error {
+    timeoutMs;
+    constructor(timeoutMs) {
+        super(`historian call timed out after ${String(timeoutMs)}ms`);
+        this.timeoutMs = timeoutMs;
+        this.name = 'HistorianTimeoutError';
+    }
+}
+/**
+ * The provider stopped at the output-token cap before the JSON contract was
+ * complete. Observed in production with `historianMaxTokens: 2048`: the model
+ * spends part of the budget on reasoning and the digest never closes, so every
+ * run failed and no compartment (and therefore no fact) was ever stored.
+ *
+ * Retryable ONCE with an escalated cap — unlike a transport error, the request
+ * itself was valid and the answer was merely too long.
+ */
+export class HistorianTruncatedError extends Error {
+    maxTokens;
+    partialChars;
+    providerDetail;
+    constructor(maxTokens, 
+    /** Characters already streamed when the cap was hit — 0 means the whole
+     * budget went to reasoning, large means the JSON contract was too long. */
+    partialChars = 0, 
+    /** The provider's own terminal message, kept in the failed row. */
+    providerDetail) {
+        super(`historian call truncated at the ${String(maxTokens)}-token output cap `
+            + `(incomplete digest, ${String(partialChars)} chars streamed)`
+            + (providerDetail === undefined ? '' : `; provider: ${providerDetail}`));
+        this.maxTokens = maxTokens;
+        this.partialChars = partialChars;
+        this.providerDetail = providerDetail;
+        this.name = 'HistorianTruncatedError';
+    }
+}
+/**
+ * The output cap below which the four-tier JSON contract cannot reliably close.
+ * A configured `historianMaxTokens` under this floor is raised TO it (and the
+ * raise is journaled), because a smaller cap is a guaranteed truncation rather
+ * than a preference. The ceiling matches the largest useful answer and keeps a
+ * single attempt inside the default wall-clock budget.
+ */
+export const HISTORIAN_MIN_OUTPUT_TOKENS = 8_192;
+/** The hard upper bound for one historian answer. */
+export const HISTORIAN_MAX_OUTPUT_TOKENS = 32_768;
+/**
+ * Magic Context's trigger budget: the eligible span must exceed 5% of the
+ * context window, clamped to [5k, 50k] tokens, before a call is worth making.
+ */
+export function historianTriggerBudget(contextWindow, assumedWindow, minBudgetTokens = 5_000) {
+    const window = contextWindow === undefined || contextWindow <= 0 ? assumedWindow : contextWindow;
+    const floor = Math.max(1, minBudgetTokens);
+    return Math.max(floor, Math.min(50_000, Math.round(window * 0.05)));
+}
+/** A node whose text OPENS with a reclaim marker (a placeholder, or a manual
+ * checkpoint that carries its digest inline) holds no content worth digesting. */
+const SKIPPABLE_PLACEHOLDER = /^\[(?:dropped|compacted|checkpoint) §/;
+/**
+ * Select the oldest contiguous eligible span.
+ *
+ * Skipped: the system head, the composed baseline node (our own synthetic
+ * node), anything already covered by a compartment, and empty reclaim
+ * placeholders. The span never reaches into the newest `protectedNodes`
+ * surface nodes, and both of its edges are pulled to a tool-pairing boundary:
+ * it never OPENS on a tool result (whose assistant tool-call precedes the
+ * span) and never CLOSES on a tool result or on an assistant that requested
+ * tools (whose results fall outside the span). Either violation makes the
+ * auxiliary request structurally invalid for tool-using providers, which
+ * reject it before the historian can produce anything.
+ */
+export function selectHistorianRange(input) {
+    const { nodes, covered } = input;
+    let start = 0;
+    const skippable = (node) => node.role === 'system'
+        || node.seq === input.baselineNodeSeq
+        || covered.has(node.seq)
+        || node.isPlaceholder
+        || SKIPPABLE_PLACEHOLDER.test(node.text.trim())
+        || !node.hasText;
+    while (start < nodes.length && skippable(nodes[start]))
+        start += 1;
+    // TOOL-PAIRING, head: a `tool` node can never OPEN the replay, because its
+    // assistant tool-call always precedes it in the surface — opening on one
+    // sends an orphan `tool_result`, which tool-using providers reject outright
+    // (HTTP 400) before the historian ever sees a token. The skipped node that
+    // exposed it (a tool-call-only assistant, an already-digested or placeholder
+    // producer) is not eligible either, so skip the whole unpaired run.
+    while (start < nodes.length && nodes[start].role === 'tool')
+        start += 1;
+    if (start >= nodes.length)
+        return undefined;
+    let cut = Math.max(start, nodes.length - Math.max(0, input.protectedNodes));
+    // TOOL-PAIRING, both edges: never end the span on a `tool` result (its
+    // producer would be inside while the result is out) and never end it on an
+    // assistant that REQUESTED tools (the request would carry a dangling
+    // `tool_use`). Dropping the call — and the results it would have dragged
+    // along — costs a little digest scope and keeps the auxiliary request valid.
+    for (;;) {
+        while (cut > start && nodes[cut]?.role === 'tool')
+            cut -= 1;
+        if (cut > start && nodes[cut - 1]?.hasToolCalls === true) {
+            cut -= 1;
+            continue;
+        }
+        break;
+    }
+    const slice = nodes.slice(start, cut);
+    if (slice.length < Math.max(1, input.minNodes))
+        return undefined;
+    const tokens = slice.reduce((total, node) => total + estimateTokens(node.text), 0);
+    if (tokens <= 0)
+        return undefined;
+    return { nodes: slice, tokens };
+}
+/** The JSON contract the historian must return (validated by parseHistorianOutput). */
+export const HISTORIAN_INSTRUCTION = [
+    'You are now acting as the HISTORIAN for this coding assistant.',
+    'Condense the conversation ABOVE into durable context for a model that will continue this work after the span is removed from its context window.',
+    '',
+    'Return ONLY one JSON object, no prose, no code fences, with exactly this shape:',
+    '{',
+    '  "summary": { "p1": "...", "p2": "...", "p3": "...", "p4": "..." },',
+    '  "importance": 0,',
+    '  "facts": [{ "category": "CONSTRAINTS", "content": "..." }],',
+    '  "events": ["..."],',
+    '  "userObservations": ["..."]',
+    '}',
+    '',
+    'The four summary tiers are the SAME digest at four levels of detail, so it can be degraded as it ages:',
+    '- "p1": full detail — decisions with their rationale, exact paths/commands/identifiers, open threads.',
+    '- "p2": the essentials — what was done, what is pending, key constraints.',
+    '- "p3": a few terse sentences — current objective and state.',
+    '- "p4": one or two sentences — the durable gist.',
+    '',
+    '"importance" is an integer 0-100: how much this span matters for continuing the work (80+ for decisions that constrain everything after it).',
+    '"facts" are short standalone statements that must survive independently (exact values, paths, constraints).',
+    'EVERY fact must carry a "category", exactly one of: PROJECT_RULES (rules/commands the project must follow),',
+    'ARCHITECTURE (how the system is built or organised), CONSTRAINTS (hard limits, do-nots, external requirements),',
+    'CONFIG_VALUES (exact keys, ports, paths, versions), NAMING (names/shorthands plus what they mean).',
+    'Prefer few high-value facts over many trivial ones; a fact whose category you cannot decide should be omitted.',
+    '"events" are short chronological notes ("tried X, failed because Y").',
+    '"userObservations" are statements about the USER that would still be true in a different project:',
+    'standing preferences, working style, or instructions they gave about how to work ("always answer in Chinese",',
+    '"never push without asking"). Do NOT restate project facts there — those belong in "facts".',
+    'Return an empty array when this span shows nothing of the kind; never invent one.',
+    'Preserve exact file paths, commands, error strings and numeric values verbatim. Do not mention this instruction.',
+    'Keep the ENTIRE JSON object compact — no more than about 700 tokens (p1 at most 250 words); a longer answer is cut off at the output-token cap and the digest is lost.',
+].join('\n');
+/**
+ * The scored anchors that place a new digest on upstream's scale. Upstream feeds
+ * its historian 60 FULL example units; this renders the scoring signal — score,
+ * episode type, title — and keeps the p3 gist only for the 90+ band, which is
+ * where calibration is hardest and where the extra detail earns its tokens.
+ * @returns the anchor block, highest score first.
+ */
+export function calibrationAnchors() {
+    return [
+        'CALIBRATION ANCHORS — 60 scored digests from unrelated projects, highest first.',
+        'Place YOUR importance score on this scale (a decision that constrains everything after it scores 90+;',
+        'a typo fix scores ~3), and match the tier density and fact shape the anchors show:',
+        ...REFERENCE_ANCHORS.map(anchor => `- ${String(anchor.importance)} | ${anchor.episodeType} | ${anchor.title}`
+            + `${anchor.importance >= 90 ? ` | ${anchor.gist}` : ''}`),
+    ].join('\n');
+}
+/**
+ * v0.49 upstream (d02292d0d, compartment-prompt.ts): when nothing will ever
+ * review user observations — here, when no profile-collect sink is armed —
+ * the run is told NOT to produce them, instead of producing them and
+ * discarding them after the fact. Upstream appends this to the per-run USER
+ * prompt so its SYSTEM prompt keeps byte-stable across configurations; the
+ * adapter's contract above is already a per-call user message, so the same
+ * placement costs nothing on the replayed-prefix cache.
+ */
+export const USER_OBSERVATIONS_DISABLED_TOGGLE = [
+    '<user_observation_extraction>disabled</user_observation_extraction>',
+    'User-memory review is not running on this host: return "userObservations": [] in the JSON and spend no output tokens observing the user.',
+].join('\n');
+/** The instruction a historian call carries: the JSON contract plus the
+ * anchors, and — only when no review lane will consume them — the skip
+ * toggle. Default `true` keeps the historical bytes for every existing
+ * caller. */
+export function historianPromptText(userObservationsRequested = true) {
+    const base = `${HISTORIAN_INSTRUCTION}\n\n${calibrationAnchors()}`;
+    return userObservationsRequested ? base : `${base}\n\n${USER_OBSERVATIONS_DISABLED_TOGGLE}`;
+}
+/** Consume a chunk stream into text, honoring the terminal finish reason.
+ *
+ * @param maxTokens - the cap the request carried, for the truncation error.
+ */
+async function collectText(chunks, maxTokens) {
+    const byIndex = new Map();
+    let usage;
+    // The stream's terminal reason, when one arrived at all. A provider that cuts
+    // the stream without a `finish` chunk leaves this undefined — the shape that
+    // produces invalid JSON with no max-tokens signal (measured live 13:06: JSON
+    // cut at 3.2KB, no truncation classified, four attempts burned).
+    let finishReason;
+    for await (const raw of chunks) {
+        const chunk = raw;
+        switch (chunk.type) {
+            case 'text-delta': {
+                const delta = chunk;
+                byIndex.set(delta.index, (byIndex.get(delta.index) ?? '') + delta.text);
+                break;
+            }
+            case 'usage': {
+                const reported = chunk.usage;
+                const numeric = {};
+                for (const [key, value] of Object.entries(reported)) {
+                    if (typeof value === 'number')
+                        numeric[key] = value;
+                }
+                usage = numeric;
+                break;
+            }
+            case 'finish': {
+                const reason = chunk.reason;
+                finishReason = reason.kind;
+                if (reason.kind === 'error' || reason.kind === 'aborted') {
+                    // The host turns a provider throw into a terminal error chunk; the
+                    // code survives on `reason.failure.code`, not on the thrown Error.
+                    if (isOutputLimitCode(reason.failure?.code)) {
+                        throw new HistorianTruncatedError(maxTokens, 0, reason.failure?.message);
+                    }
+                    throw new Error(`historian call failed: ${reason.failure?.message ?? reason.kind}`);
+                }
+                if (reason.kind === 'max-tokens') {
+                    const partial = [...byIndex.entries()]
+                        .sort(([a], [b]) => a - b)
+                        .map(([, value]) => value)
+                        .join('');
+                    throw new HistorianTruncatedError(maxTokens, partial.length);
+                }
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    const text = [...byIndex.entries()].sort(([a], [b]) => a - b).map(([, value]) => value).join('');
+    return { text, ...usage === undefined ? {} : { usage }, ...finishReason === undefined ? {} : { finishReason } };
+}
+/**
+ * In-flight historian runs, one per session: Magic Context never runs two
+ * historians for one session. The value (not just a flag) is the point — the
+ * force band and the provider-overflow path can AWAIT the running digest
+ * (`historianInFlight`) instead of dropping tool results while a compartment
+ * is seconds from landing (the ready-bank pattern from mon3stera's
+ * dsh-magic-context `engine.inFlight`).
+ */
+const inFlight = new Map();
+/** The historian run in flight for one session, when any. */
+export function historianInFlight(sessionId) {
+    return inFlight.get(sessionId);
+}
+/**
+ * A provider can reject an empty, token-limited completion before the host
+ * emits a `finish` chunk (measured on commandcode: `OUTPUT_TOKEN_LIMIT`). When
+ * the throw reaches us directly the code is on the Error; when the host catches
+ * the adapter throw first, the code is on `finish.reason.failure.code`. Both
+ * paths must normalize to truncation, otherwise the designed cap-escalation
+ * retry never fires.
+ */
+function isOutputLimitCode(code) {
+    return code === 'OUTPUT_TOKEN_LIMIT';
+}
+function isOutputLimitFailure(error) {
+    return error instanceof Error && isOutputLimitCode(error.code);
+}
+/**
+ * One attempt's stream, bounded by the run's remaining wall-clock budget.
+ *
+ * The budget is enforced twice on purpose: the abort signal tells a compliant
+ * provider stream to stop (so the socket is released), and `Promise.race`
+ * rejects even when the seam ignores the signal entirely. Without the race a
+ * host whose `llm.stream()` never settles would leave the run — and the
+ * session's `inFlight` slot — hanging forever, which is exactly the "start
+ * with no terminal event" failure this guards.
+ */
+async function collectAttempt(deps, options, budgetMs, outerSignal) {
+    const controller = new AbortController();
+    const relayAbort = () => controller.abort(outerSignal?.reason);
+    if (outerSignal !== undefined) {
+        if (outerSignal.aborted)
+            relayAbort();
+        else
+            outerSignal.addEventListener('abort', relayAbort, { once: true });
+    }
+    let timer;
+    const expiry = new Promise((_resolve, reject) => {
+        timer = setTimeout(() => {
+            controller.abort();
+            reject(new HistorianTimeoutError(budgetMs));
+        }, budgetMs);
+        timer.unref?.();
+    });
+    try {
+        try {
+            return await Promise.race([
+                collectText(deps.llm.stream({ ...options, signal: controller.signal }), options.maxTokens),
+                expiry,
+            ]);
+        }
+        catch (error) {
+            // Normalise the provider's pre-finish token-limit rejection into the
+            // truncation path so the next attempt doubles the cap and asks for a
+            // shorter JSON object.
+            if (isOutputLimitFailure(error)) {
+                throw new HistorianTruncatedError(options.maxTokens, 0, error instanceof Error ? error.message : String(error));
+            }
+            throw error;
+        }
+    }
+    finally {
+        if (timer !== undefined)
+            clearTimeout(timer);
+        outerSignal?.removeEventListener('abort', relayAbort);
+    }
+}
+/**
+ * Numeric-key accumulation across one run's calls. Providers report usage under
+ * their own key names (`inputTokens`/`outputTokens` is the common host shape),
+ * and a chunked run makes several calls — so the done row carries the
+ * per-key SUM, keyed exactly as the provider reported it (no guessing about
+ * semantics). Attempts that never produced a usage chunk (timeouts, transport
+ * errors) contribute nothing: their cost is already visible in `attempts`.
+ */
+function addUsage(total, reported) {
+    if (reported === undefined)
+        return total;
+    const merged = { ...total };
+    for (const [key, value] of Object.entries(reported)) {
+        if (typeof value === 'number' && Number.isFinite(value))
+            merged[key] = (merged[key] ?? 0) + value;
+    }
+    return merged;
+}
+/** Why a historian run produced no digest, in the journal's vocabulary. */
+function classifyFailure(error) {
+    if (error instanceof HistorianTimeoutError)
+        return 'timeout';
+    if (error instanceof HistorianTruncatedError)
+        return 'max-tokens';
+    if (error instanceof HistorianOutputError)
+        return 'invalid-output';
+    if (error instanceof Error && error.name === 'AbortError')
+        return 'aborted';
+    return 'stream-error';
+}
+/**
+ * The cap one attempt carries: the floor-respecting cap, doubled on retry and
+ * clamped to the ceiling (`HISTORIAN_MAX_OUTPUT_TOKENS` unless the policy
+ * carries an explicit `historianMaxTokensCeiling`).
+ */
+function attemptOutputCap(effective, attempt, ceiling = HISTORIAN_MAX_OUTPUT_TOKENS) {
+    return Math.min(ceiling, effective * 2 ** attempt);
+}
+/**
+ * The retry ceiling, read defensively: `historianMaxTokensCeiling` is an
+ * optional policy key, so this adapter works both with and without the Config
+ * entry. The ceiling is never allowed below the first attempt's cap.
+ */
+function retryCeiling(policy, effective) {
+    const configured = policy.historianMaxTokensCeiling;
+    const ceiling = Math.min(HISTORIAN_MAX_OUTPUT_TOKENS, typeof configured === 'number' && Number.isFinite(configured) && configured > 0
+        ? configured
+        : HISTORIAN_MAX_OUTPUT_TOKENS);
+    return Math.max(effective, ceiling);
+}
+/**
+ * Split an oversized span into contiguous chunks of about `chunkTokens`
+ * tokens (never splitting a node, at most 4 chunks, only when the span is at
+ * least 4 nodes — smaller spans digest in one call exactly as before).
+ */
+export function planChunkSpans(nodes, spanTokens, chunkTokens) {
+    if (chunkTokens === undefined || chunkTokens <= 0 || spanTokens <= chunkTokens || nodes.length < 4) {
+        return [nodes];
+    }
+    const count = Math.min(4, Math.max(2, Math.ceil(spanTokens / chunkTokens)));
+    const target = Math.ceil(spanTokens / count);
+    const groups = [];
+    let current = [];
+    let acc = 0;
+    for (let index = 0; index < nodes.length; index += 1) {
+        const node = nodes[index];
+        current.push(node);
+        acc += Math.max(1, Math.ceil(node.text.length / 4));
+        if (acc < target || groups.length >= count - 1)
+            continue;
+        // TOOL-PAIRING at the chunk edges, not only at the span edges the scan above
+        // guards: a chunk must not OPEN on a tool result (its assistant tool-call
+        // would sit in the previous chunk) and must not CLOSE on an assistant that
+        // requested tools (whose results would land in the next one). Providers
+        // reject both (`host: packages/core/session/src/repair.ts:164`: "providers
+        // reject dangling assistant calls"), and the live failure was this shape — a
+        // chunked run whose second call opened on a tool result answered 400
+        // "tool_result ... must have a corresponding tool_use block in the previous
+        // message". Waiting for the next safe boundary keeps the target approximate:
+        // the chunk count is a cap, not a promise.
+        const next = nodes[index + 1];
+        if (next === undefined || next.role === 'tool' || node.hasToolCalls)
+            continue;
+        groups.push(current);
+        current = [];
+        acc = 0;
+    }
+    if (current.length > 0)
+        groups.push(current);
+    return groups.length > 0 ? groups : [nodes];
+}
+/** The merge step's contract: one final JSON object from the partial digests. */
+export const HISTORIAN_MERGE_INSTRUCTION = [
+    'You are the HISTORIAN merging PARTIAL digests of ONE long span into the final single JSON object.',
+    'Return ONLY one JSON object with exactly the same shape: summary {p1,p2,p3,p4}, importance, facts [{category,content}], events.',
+    'Merge and deduplicate the partials; keep the most durable facts; preserve exact paths, commands and error strings verbatim.',
+    'p1 at most 250 words; the other tiers one or two sentences each; at most about 700 tokens overall.',
+].join('\n');
+/**
+ * Run the historian's auxiliary call over a span and return the validated
+ * digest WITHOUT storing it (the manual/checkpoint path owns the result).
+ *
+ * Every exit after the `start` event writes exactly one terminal event:
+ * `done` on a validated digest, `failed` (with `reason`, `durationMs`,
+ * `attempts`) on every other outcome. The two repair attempts share ONE
+ * wall-clock budget so the run cannot outlive `timeoutMs`.
+ */
+export async function callHistorian(deps, input) {
+    const route = deps.route ?? input.route;
+    if (route === undefined || route.provider.length === 0 || route.model.length === 0) {
+        throw new Error('no routed provider/model available for the historian call');
+    }
+    const reasoningEffort = route.reasoningEffort?.trim() === '' ? undefined : route.reasoningEffort?.trim();
+    const trigger = input.trigger ?? 'automatic';
+    const spanTokens = input.nodes.reduce((total, node) => total + Math.max(1, Math.ceil(node.text.length / 4)), 0);
+    const clock = deps.now ?? Date.now;
+    const startedAt = clock();
+    const timeoutMs = deps.timeoutMs ?? deps.policy.historianTimeoutMs ?? DEFAULT_HISTORIAN_TIMEOUT_MS;
+    // A cap below the contract floor is a guaranteed truncation, not a
+    // preference: the production failure was `historianMaxTokens: 2048`, which
+    // the model spent on reasoning before the JSON ever closed.
+    const configuredMaxTokens = deps.policy.historianMaxTokens;
+    const effectiveMaxTokens = Math.min(HISTORIAN_MAX_OUTPUT_TOKENS, Math.max(HISTORIAN_MIN_OUTPUT_TOKENS, configuredMaxTokens));
+    const ceiling = retryCeiling(deps.policy, effectiveMaxTokens);
+    // Everything that can throw is prepared BEFORE the `start` row. The invariant
+    // is "no row, or `start` + exactly one terminal sibling", and a preparation
+    // failure (e.g. a session store that throws inside `spanRequestMessages`)
+    // must not be able to leave a `start` without its terminal event.
+    const coveredSeqs = input.nodes.map(node => node.seq);
+    const chunks = planChunkSpans(input.nodes, spanTokens, input.chunkTokens);
+    const merging = chunks.length > 1;
+    const plannedCalls = chunks.length + (merging ? 1 : 0);
+    // D3: the run budget scales with the call count (per-call attempts share
+    // it), capped at MC's 600s historian wall clock. The extra slot reserves
+    // budget for the merge call — measured live (the 17:50 timeout): four
+    // chunks with retries spent ~5.9min of the 4×180s budget, leaving the
+    // merge 129s, which it overran.
+    const deadline = startedAt + Math.min(600_000, timeoutMs * plannedCalls + (merging ? timeoutMs : 0));
+    const spanSeqsOf = (call) => chunks[Math.min(call, chunks.length - 1)].map(node => node.seq);
+    const replayOf = (call) => {
+        const seqs = spanSeqsOf(call);
+        return spanRequestMessages(input.session, input.systemSeq === undefined ? seqs : [input.systemSeq, ...seqs]);
+    };
+    // d02292d0d: observations are requested only when a sink is armed to
+    // consume them; otherwise the per-call instruction says so (see the toggle).
+    const userObservationsRequested = deps.profileCollect !== undefined;
+    const baseMessages = [
+        ...replayOf(0),
+        { role: 'user', content: [{ type: 'text', text: historianPromptText(userObservationsRequested) }] },
+    ];
+    const baseData = {
+        trigger,
+        nodes: input.nodes.length,
+        tokens: spanTokens,
+        route: `${route.provider}/${route.model}`,
+        ...reasoningEffort === undefined ? {} : { reasoningEffort },
+        maxTokens: effectiveMaxTokens,
+        ...effectiveMaxTokens === configuredMaxTokens ? {} : { configuredMaxTokens },
+        timeoutMs,
+        ...merging ? { chunks: chunks.length } : {},
+    };
+    deps.journal?.info('historian', `historian ${trigger} run started over ${String(input.nodes.length)} nodes${merging ? ` in ${String(chunks.length)} chunks` : ''}`, {
+        phase: 'start',
+        trigger,
+        nodes: input.nodes.length,
+        tokens: spanTokens,
+        route: `${route.provider}/${route.model}`,
+        ...reasoningEffort === undefined ? {} : { reasoningEffort },
+        maxTokens: effectiveMaxTokens,
+        ...effectiveMaxTokens === configuredMaxTokens ? {} : { configuredMaxTokens },
+        timeoutMs,
+        ...merging ? { chunks: chunks.length } : {},
+    }, input.sessionId);
+    let terminal = false;
+    let attempts = 0;
+    let failure;
+    // What the LAST attempt's stream actually did: its terminal reason (undefined
+    // when the provider never sent a finish chunk) and how many characters made
+    // it out. Both land on an invalid-output row so a cut stream is
+    // distinguishable from a model that simply wrote bad JSON.
+    let lastFinish;
+    let lastText = '';
+    /** Per-key usage sum over the run's successful calls (journal-observed cost). */
+    let usageTotal;
+    /** The output cap each attempt carried, in order (did escalation happen?). */
+    const attemptCaps = [];
+    // The stage a failure should be attributed to: updated as the run advances
+    // ('start' → 'chunk 2/4' → 'merge'), so the failed row says WHERE it died.
+    let stage = 'start';
+    try {
+        const partials = [];
+        let finalDraft;
+        let lastCap = effectiveMaxTokens;
+        for (let call = 0; call < plannedCalls; call += 1) {
+            const isMerge = merging && call === plannedCalls - 1;
+            stage = isMerge ? 'merge' : `chunk ${String(call + 1)}/${String(chunks.length)}`;
+            if (isMerge) {
+                // The merge was invisible until it timed out (the 17:50 failure) —
+                // give it the same per-call beat the chunks have.
+                deps.journal?.info('historian', `historian merging ${String(partials.length)} partial digest(s)`, {
+                    phase: 'merge',
+                    partials: partials.length,
+                    ...baseData,
+                }, input.sessionId);
+            }
+            // Annotated: without it the nested ternary widens the message literals
+            // (`type: 'text'` becomes `string`) and the whole array collapses to
+            // `unknown[]`, which the LLM seam then rejects.
+            const baseMessagesForCall = isMerge
+                ? [
+                    {
+                        role: 'user',
+                        content: [{
+                                type: 'text',
+                                text: `${HISTORIAN_MERGE_INSTRUCTION}\n\n${partials
+                                    .map((partial, index) => `Partial digest ${String(index + 1)}:\n${JSON.stringify(partial)}`)
+                                    .join('\n\n')}`,
+                            }],
+                    },
+                ]
+                : call === 0
+                    ? baseMessages
+                    : [
+                        ...replayOf(call),
+                        { role: 'user', content: [{ type: 'text', text: historianPromptText(userObservationsRequested) }] },
+                    ];
+            let callDraft;
+            for (let attempt = 0; attempt < 2; attempt += 1) {
+                attempts += 1;
+                const maxTokens = attemptOutputCap(effectiveMaxTokens, attempt, ceiling);
+                lastCap = maxTokens;
+                attemptCaps.push(maxTokens);
+                const truncated = failure instanceof HistorianTruncatedError;
+                const messages = attempt === 0
+                    ? baseMessagesForCall
+                    : [
+                        ...baseMessagesForCall,
+                        {
+                            role: 'user',
+                            content: [{
+                                    type: 'text',
+                                    text: truncated
+                                        ? `Your previous response was cut off at the ${String(attemptOutputCap(effectiveMaxTokens, attempt - 1, ceiling))}-token output cap before the JSON closed. `
+                                            + 'Return ONLY the JSON object, and keep it far shorter: p1 at most 150 words, the other tiers one or two sentences.'
+                                        : `Your previous response was rejected: ${failure?.message ?? 'invalid output'}. `
+                                            + 'Return ONLY the JSON object described above.',
+                                }],
+                        },
+                    ];
+                const budgetMs = deadline - clock();
+                if (budgetMs <= 0)
+                    throw new HistorianTimeoutError(timeoutMs);
+                try {
+                    const collected = await collectAttempt(deps, {
+                        provider: route.provider,
+                        model: route.model,
+                        ...reasoningEffort === undefined ? {} : { reasoningEffort },
+                        messages,
+                        maxTokens,
+                        sessionId: input.sessionId,
+                        purpose: 'compaction',
+                    }, budgetMs, input.signal);
+                    lastFinish = collected.finishReason;
+                    lastText = collected.text;
+                    if (collected.usage !== undefined)
+                        usageTotal = addUsage(usageTotal, collected.usage);
+                    const parsed = parseHistorianOutput(collected.text);
+                    if (isMerge) {
+                        finalDraft = parsed;
+                    }
+                    else {
+                        partials.push(parsed);
+                        if (merging) {
+                            deps.journal?.info('historian', `historian chunk ${String(call + 1)}/${String(chunks.length)} digested (importance ${String(parsed.importance)})`, {
+                                phase: 'chunk',
+                                index: call + 1,
+                                of: chunks.length,
+                                nodes: chunks[call].length,
+                                importance: parsed.importance,
+                                facts: parsed.facts.length,
+                                events: parsed.events.length,
+                                attempts,
+                            }, input.sessionId);
+                        }
+                    }
+                    callDraft = parsed;
+                    break;
+                }
+                catch (error) {
+                    // Truncation and a rejected digest are retryable (once, with a longer
+                    // cap or a repair nudge). A provider or transport failure is not: the
+                    // request itself failed, so retrying it would only double the cost.
+                    if (!(error instanceof HistorianTruncatedError) && !(error instanceof HistorianOutputError))
+                        throw error;
+                    failure = error;
+                    deps.logger.warn(`magic-context historian: ${merging ? `call ${String(call + 1)}/${String(plannedCalls)} ` : ''}attempt ${String(attempt + 1)} rejected at ${String(maxTokens)} tokens (${failure.message})`);
+                }
+            }
+            if (callDraft === undefined)
+                throw failure ?? new Error('historian produced no digest');
+        }
+        const draft = finalDraft ?? partials[0]
+            ?? (() => { throw new Error('historian produced no digest'); })();
+        deps.journal?.info('historian', `historian digest ready (importance ${String(draft.importance)}, ${String(input.nodes.length)} nodes)`, {
+            phase: 'done',
+            ...baseData,
+            maxTokens: lastCap,
+            importance: draft.importance,
+            facts: draft.facts.length,
+            events: draft.events.length,
+            attempts,
+            durationMs: clock() - startedAt,
+            // The run's observed LLM cost, keyed as the provider reported it
+            // (absent when the provider never sent a usage chunk).
+            ...(usageTotal !== undefined ? { usage: usageTotal } : {}),
+            // 'stop' on a healthy run; 'none' would mean the stream ended without a
+            // terminal reason even though the JSON parsed (worth knowing).
+            finishReason: lastFinish ?? 'none',
+        }, input.sessionId);
+        terminal = true;
+        return draft;
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        // The invariant this repairs: a `start` row with no terminal sibling is
+        // indistinguishable from a wedged run. Anything that escapes the attempt
+        // loop — provider errors, aborted streams, the timeout, an invalid digest
+        // twice — lands here exactly once.
+        if (!terminal) {
+            terminal = true;
+            deps.journal?.error('historian', `historian run failed (${classifyFailure(error)}): ${reason}`, {
+                phase: 'failed',
+                ...baseData,
+                // WHERE it died: the merge after a long chunk run is the observed
+                // failure point (the 17:50 timeout), so name the stage explicitly.
+                stage,
+                kind: classifyFailure(error),
+                reason,
+                errorName: error instanceof Error ? error.name : typeof error,
+                attempts,
+                // Every cap tried, in order: an escalated retry is visible at a glance
+                // (and a run that never escalated is the tell for a model-side cut).
+                attemptCaps,
+                durationMs: clock() - startedAt,
+                // The measured shape of a truncation: how much of the answer made it
+                // out before the cap. 0 chars ⇒ the budget was spent on reasoning;
+                // a long partial ⇒ the requested digest itself is too big.
+                ...error instanceof HistorianTruncatedError
+                    ? { maxTokens: error.maxTokens, partialChars: error.partialChars }
+                    : {},
+                // A rejected digest says nothing about WHY by itself: the same
+                // "invalid JSON" is produced by a cut stream and by a chatty model.
+                // The last attempt's terminal reason and its output length (plus a
+                // short tail) decide between them — `finishReason: 'none'` means the
+                // provider closed without a `finish` chunk.
+                ...error instanceof HistorianOutputError
+                    ? {
+                        finishReason: lastFinish ?? 'none',
+                        outputChars: lastText.length,
+                        // 400, not the original 120: every live invalid-JSON failure ended at
+                        // the LAST character while its recorded tail was well-formed, which
+                        // means the defect was in the middle and 120 chars could not show it.
+                        outputTail: lastText.slice(-400),
+                    }
+                    : {},
+            }, input.sessionId);
+        }
+        deps.logger.warn(`magic-context historian: run failed (${classifyFailure(error)}): ${reason}`);
+        throw error;
+    }
+}
+/**
+ * #802 (upstream e9075bfec): a run that published nothing must not be
+ * re-attempted over the same messages — every later pass would pay for the
+ * same failure again. The watermark lives in the store's meta kv (no schema
+ * change): `historian_no_progress:<sessionId>` → `{endSeq, at, kind}`.
+ * An automatic run waits until the surface has a node NEWER than the failed
+ * span's end; manual paths (`forcedRange`, the checkpoint/recomp lane) bypass
+ * the gate entirely, mirroring upstream's "near-full and emergency paths
+ * still get one attempt".
+ */
+const NO_PROGRESS_KEY_PREFIX = 'historian_no_progress:';
+/** Fail-open: an unreadable record never blocks a run, and a store double
+ * without meta access (spec fakes) degrades to "no watermark". */
+function readHistorianNoProgress(stores, sessionId) {
+    try {
+        const raw = stores.getMeta(`${NO_PROGRESS_KEY_PREFIX}${sessionId}`);
+        if (raw === undefined || raw === '')
+            return undefined;
+        const parsed = JSON.parse(raw);
+        if (typeof parsed.endSeq !== 'number' || typeof parsed.at !== 'number' || typeof parsed.kind !== 'string') {
+            return undefined;
+        }
+        return { endSeq: parsed.endSeq, at: parsed.at, kind: parsed.kind };
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Fail-open: failing to persist the watermark must not mask the run's error. */
+function writeHistorianNoProgress(stores, sessionId, value) {
+    try {
+        stores.setMeta(`${NO_PROGRESS_KEY_PREFIX}${sessionId}`, JSON.stringify(value));
+    }
+    catch {
+        /* the gate is advisory; a store hiccup only costs one retry */
+    }
+}
+function clearHistorianNoProgress(stores, sessionId) {
+    try {
+        stores.setMeta(`${NO_PROGRESS_KEY_PREFIX}${sessionId}`, '');
+    }
+    catch {
+        /* same fail-open */
+    }
+}
+/**
+ * Run one historian pass for a session. Returns the stored compartment, or
+ * undefined when nothing was eligible. A run already in flight is JOINED —
+ * the same eventual compartment, never a second concurrent call. Throws only
+ * for call or contract failures (the caller logs and continues).
+ */
+export function runHistorian(deps, input) {
+    const existing = inFlight.get(input.sessionId);
+    if (existing !== undefined)
+        return existing;
+    const run = runHistorianExclusive(deps, input).finally(() => {
+        if (inFlight.get(input.sessionId) === run)
+            inFlight.delete(input.sessionId);
+    });
+    inFlight.set(input.sessionId, run);
+    return run;
+}
+async function runHistorianExclusive(deps, input) {
+    // #802: after a run that published nothing, automatic runs wait for a node
+    // newer than the failed span's end before paying for the span again.
+    // Manual paths (`forcedRange`) bypass the gate.
+    if (input.forcedRange === undefined) {
+        const noProgress = readHistorianNoProgress(deps.stores, input.sessionId);
+        const lastNode = input.nodes.length > 0 ? input.nodes[input.nodes.length - 1] : undefined;
+        if (noProgress !== undefined && lastNode !== undefined && noProgress.endSeq >= lastNode.seq) {
+            deps.journal?.info('historian', 'historian auto-run skipped: the previous run published nothing and no newer message has arrived (#802)', {
+                phase: 'no-progress-skip',
+                failedEndSeq: noProgress.endSeq,
+                lastSeq: lastNode.seq,
+                failedAt: noProgress.at,
+                kind: noProgress.kind,
+            }, input.sessionId);
+            return undefined;
+        }
+    }
+    const range = input.forcedRange !== undefined
+        ? { nodes: input.forcedRange, tokens: input.forcedRange.reduce((total, node) => total + estimateTokens(node.text), 0) }
+        : selectHistorianRange({
+            nodes: input.nodes,
+            covered: deps.stores.coveredSeqs(input.sessionId),
+            ...input.baselineNodeSeq === undefined ? {} : { baselineNodeSeq: input.baselineNodeSeq },
+            protectedNodes: deps.policy.historianProtectedNodes,
+            minNodes: deps.policy.historianMinNodes,
+        });
+    if (range === undefined)
+        return undefined;
+    const budget = historianTriggerBudget(input.contextWindow, deps.policy.historianAssumedWindow, deps.policy.historianMinBudgetTokens);
+    if (input.forcedRange === undefined && range.tokens < budget) {
+        deps.logger.info(`magic-context historian: eligible span ${String(range.tokens)} tokens is below the ${String(budget)}-token trigger`);
+        return undefined;
+    }
+    const now = deps.now?.() ?? Date.now();
+    const first = range.nodes[0];
+    const last = range.nodes[range.nodes.length - 1];
+    try {
+        const draft = await callHistorian(deps, {
+            sessionId: input.sessionId,
+            session: input.session,
+            nodes: range.nodes,
+            trigger: 'automatic',
+            // D1/D3: an oversized span digests in ~budget-token chunks with a merge
+            // call; the run budget scales with the chunk count (capped 600s).
+            chunkTokens: budget,
+            ...input.route === undefined ? {} : { route: input.route },
+            ...input.signal === undefined ? {} : { signal: input.signal },
+            ...input.nodes.find(node => node.role === 'system')?.seq === undefined
+                ? {}
+                : { systemSeq: input.nodes.find(node => node.role === 'system')?.seq },
+        });
+        const baselineEpoch = deps.stores.baselineState(input.sessionId)?.epoch ?? 0;
+        const compartment = deps.stores.allocateCompartment(input.sessionId, draft, {
+            startSeq: first.seq,
+            endSeq: last.seq,
+            seqs: range.nodes.map(node => node.seq),
+            tokens: range.tokens,
+        }, baselineEpoch, now);
+        // #802: a stored compartment IS progress — release the watermark so the
+        // next automatic run is judged on its own merits.
+        clearHistorianNoProgress(deps.stores, input.sessionId);
+        deps.logger.info(`magic-context historian: compartment §C${String(compartment.compartmentId)}§ stored `
+            + `(${String(range.nodes.length)} nodes, ~${String(range.tokens)} tokens, importance ${String(draft.importance)})`);
+        // WP5 — the `user` domain's collection half. This runs AFTER the compartment
+        // is durable, so a candidate that cannot be staged never costs the digest.
+        // Unlike the fact promotion next door (which may retry from the stored
+        // facts), the observations exist only in this draft: if the write fails they
+        // are gone, so the failure is journaled rather than swallowed.
+        const observations = draft.userObservations ?? [];
+        if (deps.profileCollect !== undefined && observations.length > 0) {
+            try {
+                const staged = deps.profileCollect({
+                    sessionId: input.sessionId,
+                    observations,
+                    compartmentStart: first.seq,
+                    compartmentEnd: last.seq,
+                });
+                deps.journal?.info('historian', `staged ${String(staged?.inserted ?? observations.length)} user observation(s) for review`, {
+                    phase: 'profile-collect',
+                    compartmentId: compartment.compartmentId,
+                    observations: observations.length,
+                    ...staged === undefined ? {} : { inserted: staged.inserted, skipped: staged.skipped },
+                }, input.sessionId);
+            }
+            catch (error) {
+                deps.journal?.error('error', `user observation staging failed: ${errText(error)}`, {
+                    scope: 'profile-collect',
+                    compartmentId: compartment.compartmentId,
+                    observations: observations.length,
+                }, input.sessionId);
+            }
+        }
+        deps.journal?.info('historian', `compartment §C${String(compartment.compartmentId)}§ stored (~${String(range.tokens)} tokens, importance ${String(draft.importance)})`, {
+            phase: 'stored',
+            compartmentId: compartment.compartmentId,
+            nodes: range.nodes.length,
+            tokens: range.tokens,
+            importance: draft.importance,
+            // WP5 observability: without these two fields a missing profile-collect
+            // row is ambiguous — "empty observations" and "lane not armed" look
+            // identical from the journal. The count is the draft's raw output; the
+            // armed/off flag is whether the pre-step pass supplied the sink.
+            userObservations: observations.length,
+            profileCollect: deps.profileCollect === undefined ? 'off' : 'armed',
+            coveredStartSeq: first.seq,
+            coveredEndSeq: last.seq,
+        }, input.sessionId);
+        return compartment;
+    }
+    catch (error) {
+        // #802: nothing was published — the surface must grow past `last.seq`
+        // before the automatic lane pays for this span again.
+        writeHistorianNoProgress(deps.stores, input.sessionId, {
+            endSeq: last.seq,
+            at: deps.now?.() ?? Date.now(),
+            kind: classifyFailure(error),
+        });
+        throw error;
+    }
+}