@seanhogg/builderforce-memory 2026.10.24 → 2026.10.25

package/dist/working/transcript.js

@@ -0,0 +1,243 @@
+/**
+ * The working transcript — what of a run the model is SHOWN each turn.
+ *
+ * A run's full transcript grows without bound; the request sent to the model must not.
+ * This module owns the two ways it is kept in bounds: the message-count + token WINDOW
+ * (drop-oldest, anchored on a user turn) and AUTO-COMPACTION (fold the older part into a
+ * memory note, keep every later message verbatim). A host supplies only what is
+ * run-specific: the summarizer it calls (see `summary.ts`) and what a fold records.
+ *
+ * Pure except for {@link buildWorkingTranscript}, which awaits the injected summarizer and
+ * updates the caller's `compactMemo`. Generic over the host's message shape: anything with
+ * the OpenAI-style `role` / `content` / `tool_calls` fields (`HandoffMessage`).
+ */
+import { defaultEstimateTokens, messageTokens as handoffMessageTokens, tokensOf as handoffTokensOf } from '../handoff/budget.js';
+/** How much history we send to the model (message-count ceiling). */
+export const HISTORY_WINDOW = 80;
+/**
+ * Token budget for the working transcript sent to the model each turn — how much of this
+ * run the model can SEE: the file windows it read, the searches it ran, what it decided.
+ *
+ * It bounds context, which message-count windowing (HISTORY_WINDOW) alone does not: one
+ * `tasks.list` result can be tens of thousands of tokens. It is NOT what keeps a request
+ * inside the serving model's window — the gateway fits every request to a model that can
+ * hold it.
+ *
+ * It was 24k, "sized under the smallest pool model's window". That protected nothing — the
+ * ~16k-token system prompt + tool catalog in front of it already put a full turn at ~40k,
+ * past any 32k window — and it cost every coding run its memory. Six 4k-token file windows
+ * filled it, compaction folded them into a 1.2k-token note, and the model went back for the
+ * files it had just read (chat #105: 55 turns, 63% of calls revisiting, zero edits, prompt
+ * peak 40,351). 64k holds a coding task's working set — a dozen-plus file windows —
+ * verbatim, while a full turn stays well inside the 128k window every coding-capable pool
+ * model has. See {@link windowed} and {@link buildWorkingTranscript}.
+ */
+export const HISTORY_TOKEN_BUDGET = 64_000;
+/**
+ * How much of {@link HISTORY_TOKEN_BUDGET} the verbatim TAIL may keep when the older part
+ * of the transcript is compacted. The rest is headroom new work fills before the next
+ * fold, so the memo is re-folded once per ~20k tokens of progress, not on every turn.
+ */
+export const COMPACT_TAIL_TOKEN_BUDGET = 40_000;
+/** Estimated tokens for one message (content + tool-call payload + framing) — THE
+ *  hand-off packer's estimate, so the working transcript and a hand-off to a smaller
+ *  model count a message the same way. */
+function messageTokens(m) {
+    return handoffMessageTokens(m, defaultEstimateTokens);
+}
+/**
+ * Trim the in-memory transcript to the history window before sending it to the
+ * model. Slicing the last N can leave the window starting on an `assistant`
+ * tool-call turn or an orphaned `tool` result (whose owning call fell off the
+ * front). That payload is invalid for strict vendors: Gemini rejects a request
+ * whose conversation does not begin with a user turn — surfaced as the cascade
+ * `[googleai] 400 INVALID_ARGUMENT` after a long tool-loop crossed the window
+ * boundary. So anchor the window at a user turn.
+ *
+ * If the last-N slice contains no user turn (a tool loop longer than the
+ * window), fall back to the most recent user turn in the FULL transcript and
+ * keep everything after it — correctness over the size cap.
+ */
+export function windowed(convo) {
+    let w = convo.slice(-HISTORY_WINDOW);
+    while (w.length > 0 && w[0].role !== 'user')
+        w = w.slice(1);
+    if (w.length === 0) {
+        const lastUser = convo.map((m) => m.role).lastIndexOf('user');
+        w = lastUser >= 0 ? convo.slice(lastUser) : convo.slice();
+    }
+    return tokenBounded(w);
+}
+/**
+ * Enforce the token budget on an already message-count-windowed slice. Drops the
+ * OLDEST turns first, then re-anchors on a user turn (a strict vendor rejects a window
+ * that doesn't start on `user`, and dropping a turn can orphan a `tool` result whose
+ * `assistant` tool-call fell off the front). The most recent user turn is never
+ * dropped: correctness over the budget when a single turn is itself oversized.
+ */
+function tokenBounded(w) {
+    let total = transcriptTokens(w);
+    if (total <= HISTORY_TOKEN_BUDGET)
+        return w;
+    const lastUser = w.map((m) => m.role).lastIndexOf('user');
+    let start = 0;
+    while (total > HISTORY_TOKEN_BUDGET && start < lastUser) {
+        total -= messageTokens(w[start]);
+        start += 1;
+    }
+    let trimmed = w.slice(start);
+    while (trimmed.length > 1 && trimmed[0].role !== 'user')
+        trimmed = trimmed.slice(1);
+    return trimmed;
+}
+/**
+ * Is a transcript message still part of what the model sees? False once auto-compaction
+ * has folded it into the memory note, or the drop-oldest window has let it fall off the
+ * front. Deliberately conservative: it judges against the CURRENT transcript (which has
+ * grown since the last working set was built), so it can only ever call a message gone a
+ * turn early — a needless replay costs tokens, a stub for a vanished result costs the run.
+ */
+export function stillInWorkingContext(state, anchor) {
+    const convo = state.transcript;
+    const idx = convo.indexOf(anchor);
+    if (idx < 0)
+        return false;
+    if (state.compactMemo)
+        return idx >= verbatimStart(convo, state.compactMemo.coveredEnd);
+    return windowed(convo).includes(convo[idx]);
+}
+// ---------------------------------------------------------------------------
+// Auto-compaction — summarize the bulky MIDDLE instead of dropping it.
+//
+// `tokenBounded` keeps the request inside the window by DROPPING the oldest turns. That
+// never 413s, but it silently LOSES context — which made a weak model re-read files and
+// thrash ("LOOP EXHAUSTED", chat #50). When a summarizer is available the older turns are
+// compressed into ONE memory note instead, so the model keeps working from a distilled
+// memory. Falls back to `tokenBounded` when no summarizer is reachable.
+//
+// Why this fold is NOT `packHandoff` (`../handoff`): the packer is stateless and gives the
+// verbatim tail priority, handing the note whatever room the tail leaves. The working
+// transcript needs the opposite on both counts — an INCREMENTAL memo reused across turns
+// (one summarizer call per ~20k tokens of progress), a tail capped at
+// COMPACT_TAIL_TOKEN_BUDGET so the note and the next turns have headroom, and a
+// message-count window. What the two share — the token estimate — is the packer's.
+// ---------------------------------------------------------------------------
+/** The fewest recent messages the verbatim tail keeps, however large they are. */
+export const COMPACT_TAIL_TURNS = 8;
+/** The most recent messages the verbatim tail may hold — half the message window, so a
+ *  compacted transcript stays inside {@link HISTORY_WINDOW} with room to grow. */
+const COMPACT_TAIL_MAX_MESSAGES = HISTORY_WINDOW / 2;
+/** The first index at or after `from` that is not a `tool` result: a tool row whose
+ *  assistant call was folded into the memo would be orphaned (strict vendors 400 on it). */
+export function verbatimStart(convo, from) {
+    let start = Math.max(0, Math.min(from, convo.length));
+    while (start < convo.length && convo[start].role === 'tool')
+        start += 1;
+    return start;
+}
+/** Start index of the last `tailTurns` messages, walked off a leading `tool` result. */
+export function compactTailStart(convo, tailTurns) {
+    return verbatimStart(convo, convo.length - tailTurns);
+}
+/**
+ * Where the verbatim tail begins when the transcript is compacted: as many of the most
+ * recent messages as fit in `budgetTokens` — never fewer than {@link COMPACT_TAIL_TURNS},
+ * never more than {@link COMPACT_TAIL_MAX_MESSAGES} — walked off a leading tool result.
+ *
+ * Sized by TOKENS, not a message count: a fixed eight messages is four tool results, so
+ * after a fold the fifth file the model needed was always the one it had to read again.
+ */
+export function compactTailStartForBudget(convo, budgetTokens) {
+    let start = convo.length;
+    let tokens = 0;
+    while (start > 0) {
+        const kept = convo.length - start;
+        if (kept >= COMPACT_TAIL_MAX_MESSAGES)
+            break;
+        const next = messageTokens(convo[start - 1]);
+        if (kept >= COMPACT_TAIL_TURNS && tokens + next > budgetTokens)
+            break;
+        tokens += next;
+        start -= 1;
+    }
+    return verbatimStart(convo, start);
+}
+/**
+ * Index of the MOST RECENT user turn to re-inject verbatim ahead of the tail — the
+ * ACTIVE directive — or -1 when the latest user turn is already inside the tail. Pinning
+ * the FIRST user turn (the old behaviour) made a multi-instruction chat revert to its
+ * opening request (chat #55); the current instruction is always the LATEST user turn.
+ */
+export function pinnedDirectiveIndex(convo, tailStart) {
+    const lastUser = convo.map((m) => m.role).lastIndexOf('user');
+    return lastUser >= 0 && lastUser < tailStart ? lastUser : -1;
+}
+/**
+ * Assemble the working transcript from a compressed-memory `note` that covers
+ * `convo[0, coveredEnd)`: system + the memory note + the ACTIVE user directive re-injected
+ * verbatim (when the note covers it) + EVERY message from `coveredEnd` on (tool-pairing
+ * safe via {@link verbatimStart}). Nothing is in neither the note nor the tail. The
+ * directive sits immediately before the tail so the model reads it as the CURRENT
+ * instruction.
+ */
+export function assembleCompacted(systemPrompt, convo, note, coveredEnd) {
+    const tailStart = verbatimStart(convo, coveredEnd);
+    const out = [{ role: 'system', content: systemPrompt }];
+    out.push({ role: 'assistant', content: note });
+    const directiveIdx = pinnedDirectiveIndex(convo, tailStart);
+    if (directiveIdx >= 0)
+        out.push(convo[directiveIdx]);
+    out.push(...convo.slice(tailStart));
+    return out;
+}
+/** Estimated tokens of a message list — the ONE count compaction budgets by, exported so
+ *  a run measures what compaction kept out of a call with the same rule. */
+export function transcriptTokens(msgs) {
+    return handoffTokensOf(msgs, defaultEstimateTokens);
+}
+/** Can these messages (plus `extraTokens` in front of them) go to the model verbatim —
+ *  inside both the token budget and the message window? */
+function fitsVerbatim(msgs, extraTokens = 0) {
+    return msgs.length <= HISTORY_WINDOW && extraTokens + transcriptTokens(msgs) <= HISTORY_TOKEN_BUDGET;
+}
+/**
+ * Build the working transcript for a turn. When the whole transcript fits → all of it,
+ * verbatim. Otherwise → a memory note of the older part plus EVERY message after it,
+ * verbatim; `onFolded` is told about each NEW fold, so the caller can record it (a
+ * visible step, a durable consolidation marker).
+ *
+ * The note is folded INCREMENTALLY, and only when it has to be. A note whose verbatim
+ * remainder still fits is reused as-is; once it no longer fits, the previous note plus
+ * exactly the messages since it are folded into a new one, up to a token-sized tail
+ * ({@link compactTailStartForBudget}). Falls back to drop-oldest when the summarizer is
+ * unavailable (`summarize` resolves null). A message-count overflow folds too.
+ */
+export async function buildWorkingTranscript(state, systemPrompt, summarize, onFolded) {
+    const convo = state.transcript;
+    if (fitsVerbatim(convo)) {
+        state.compactMemo = null; // back under budget — a later overflow summarizes afresh
+        return [{ role: 'system', content: systemPrompt }, ...windowed(convo)];
+    }
+    // A note written against a transcript that has since been replaced covers nothing here.
+    const memo = state.compactMemo && state.compactMemo.coveredEnd <= convo.length ? state.compactMemo : null;
+    if (memo && fitsVerbatim(convo.slice(verbatimStart(convo, memo.coveredEnd)), defaultEstimateTokens(memo.note))) {
+        return assembleCompacted(systemPrompt, convo, memo.note, memo.coveredEnd);
+    }
+    const from = memo?.coveredEnd ?? 0;
+    const to = Math.max(from, compactTailStartForBudget(convo, COMPACT_TAIL_TOKEN_BUDGET));
+    const fold = memo
+        ? [{ role: 'assistant', content: memo.note }, ...convo.slice(from, to)]
+        : convo.slice(from, to);
+    const summary = to > from ? await summarize(fold) : null;
+    if (summary == null) {
+        // Nothing new to fold, or no summarizer: keep the note already held, else drop-oldest.
+        return memo
+            ? assembleCompacted(systemPrompt, convo, memo.note, memo.coveredEnd)
+            : [{ role: 'system', content: systemPrompt }, ...windowed(convo)];
+    }
+    const note = `Compressed memory of the first ${to} message(s) of this conversation:\n${summary}`;
+    state.compactMemo = { note, coveredEnd: to };
+    onFolded({ droppedMessages: to - from, note, coveredEnd: to });
+    return assembleCompacted(systemPrompt, convo, note, to);
+}
+//# sourceMappingURL=transcript.js.map

package/dist/working/transcript.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transcript.js","sourceRoot":"","sources":["../../src/working/transcript.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,qBAAqB,EAAE,aAAa,IAAI,oBAAoB,EAAE,QAAQ,IAAI,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAMjI,qEAAqE;AACrE,MAAM,CAAC,MAAM,cAAc,GAAG,EAAE,CAAC;AAEjC;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAC3C;;;;GAIG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AAEhD;;0CAE0C;AAC1C,SAAS,aAAa,CAAC,CAAoB;IACzC,OAAO,oBAAoB,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;AACxD,CAAC;AAoBD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,QAAQ,CAA8B,KAAU;IAC9D,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,cAAc,CAAC,CAAC;IACrC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,MAAM;QAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAC9D,CAAC,GAAG,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC5D,CAAC;IACD,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC;AACzB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,YAAY,CAA8B,CAAM;IACvD,IAAI,KAAK,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAChC,IAAI,KAAK,IAAI,oBAAoB;QAAE,OAAO,CAAC,CAAC;IAC5C,MAAM,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAC1D,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,oBAAoB,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;QACxD,KAAK,IAAI,aAAa,CAAC,CAAC,CAAC,KAAK,CAAE,CAAC,CAAC;QAClC,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CAA8B,KAAgC,EAAE,MAAe;IAClH,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC;IAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,MAAW,CAAC,CAAC;IACvC,IAAI,GAAG,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1B,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,GAAG,IAAI,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;IACxF,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAE,CAAC,CAAC;AAC/C,CAAC;AAED,8EAA8E;AAC9E,uEAAuE;AACvE,EAAE;AACF,wFAAwF;AACxF,wFAAwF;AACxF,0FAA0F;AAC1F,uFAAuF;AACvF,wEAAwE;AACxE,EAAE;AACF,2FAA2F;AAC3F,sFAAsF;AACtF,yFAAyF;AACzF,sEAAsE;AACtE,gFAAgF;AAChF,mFAAmF;AACnF,8EAA8E;AAE9E,kFAAkF;AAClF,MAAM,CAAC,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAEpC;kFACkF;AAClF,MAAM,yBAAyB,GAAG,cAAc,GAAG,CAAC,CAAC;AAErD;4FAC4F;AAC5F,MAAM,UAAU,aAAa,CAAC,KAAmC,EAAE,IAAY;IAC7E,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACtD,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,KAAK,CAAE,CAAC,IAAI,KAAK,MAAM;QAAE,KAAK,IAAI,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,gBAAgB,CAAC,KAAmC,EAAE,SAAiB;IACrF,OAAO,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;AACxD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CAAC,KAAmC,EAAE,YAAoB;IACjG,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC;QAClC,IAAI,IAAI,IAAI,yBAAyB;YAAE,MAAM;QAC7C,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,CAAC;QAC9C,IAAI,IAAI,IAAI,kBAAkB,IAAI,MAAM,GAAG,IAAI,GAAG,YAAY;YAAE,MAAM;QACtE,MAAM,IAAI,IAAI,CAAC;QACf,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IACD,OAAO,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AACrC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAmC,EAAE,SAAiB;IACzF,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;IAC9D,OAAO,QAAQ,IAAI,CAAC,IAAI,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,YAAoB,EACpB,KAAU,EACV,IAAY,EACZ,UAAkB;IAElB,MAAM,SAAS,GAAG,aAAa,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IACnD,MAAM,GAAG,GAAQ,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAO,CAAC,CAAC;IAClE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAO,CAAC,CAAC;IACpD,MAAM,YAAY,GAAG,oBAAoB,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5D,IAAI,YAAY,IAAI,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAE,CAAC,CAAC;IACtD,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;IACpC,OAAO,GAAG,CAAC;AACb,CAAC;AAED;4EAC4E;AAC5E,MAAM,UAAU,gBAAgB,CAAC,IAAkC;IACjE,OAAO,eAAe,CAAC,IAAI,EAAE,qBAAqB,CAAC,CAAC;AACtD,CAAC;AAED;2DAC2D;AAC3D,SAAS,YAAY,CAAC,IAAkC,EAAE,WAAW,GAAG,CAAC;IACvE,OAAO,IAAI,CAAC,MAAM,IAAI,cAAc,IAAI,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,oBAAoB,CAAC;AACvG,CAAC;AAUD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAgC,EAChC,YAAoB,EACpB,SAAgD,EAChD,QAAwC;IAExC,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC;IAC/B,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,yDAAyD;QACnF,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAO,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9E,CAAC;IACD,wFAAwF;IACxF,MAAM,IAAI,GAAG,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,WAAW,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1G,IAAI,IAAI,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,EAAE,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/G,OAAO,iBAAiB,CAAC,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC5E,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,EAAE,UAAU,IAAI,CAAC,CAAC;IACnC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,yBAAyB,CAAC,KAAK,EAAE,yBAAyB,CAAC,CAAC,CAAC;IACvF,MAAM,IAAI,GAAQ,IAAI;QACpB,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,EAAO,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC5E,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC1B,MAAM,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;QACpB,uFAAuF;QACvF,OAAO,IAAI;YACT,CAAC,CAAC,iBAAiB,CAAC,YAAY,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC;YACpE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAO,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,IAAI,GAAG,kCAAkC,EAAE,sCAAsC,OAAO,EAAE,CAAC;IACjG,KAAK,CAAC,WAAW,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC;IAC7C,QAAQ,CAAC,EAAE,eAAe,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;IAC/D,OAAO,iBAAiB,CAAC,YAAY,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;AAC1D,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@seanhogg/builderforce-memory",
-  "version": "2026.10.24",
+  "version": "2026.10.25",
   "description": "BuilderForce Agent Memory — runtime layer. SSM execution, Transformer orchestration, online distillation, enterprise RAG (ingestion pipelines, vector-store adapters, tenant/ACL-scoped hybrid retrieval), multi-agent graph orchestration, LLM telemetry and cost accounting, and persistent agent memory for BuilderForce.ai agents.",
   "main": "dist/index.js",
   "module": "dist/index.js",
@@ -103,7 +103,7 @@
   },
   "homepage": "https://github.com/SeanHogg/builderforce-memory/tree/main/packages/memory#readme",
   "peerDependencies": {
-    "@seanhogg/builderforce-memory-engine": "^2026.10.24"
+    "@seanhogg/builderforce-memory-engine": "^2026.10.25"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",
@@ -117,7 +117,7 @@
     "ts-jest": "^29.2.0",
     "typescript": "^6.0.3",
     "@typescript/native-preview": "7.0.0-dev.20260627.2",
-    "@seanhogg/builderforce-memory-engine": "2026.10.24"
+    "@seanhogg/builderforce-memory-engine": "2026.10.25"
   },
   "jest": {
     "preset": "ts-jest/presets/default-esm",

package/src/working/consolidationMarker.test.ts

@@ -0,0 +1,95 @@
+/**
+ * src/working/consolidationMarker.test.ts
+ * Consolidation markers (moved unchanged from brain-embedded) + the auto-fold coverage rule.
+ */
+import {
+  consolidationMetadata,
+  consolidationMarkerContent,
+  isConsolidationMarker,
+  lastConsolidationIndex,
+  scopeToConsolidation,
+  CONSOLIDATION_MARKER_PREFIX,
+  consolidationMarkerOf,
+  foldKeepFromId,
+} from './consolidationMarker.js';
+
+const msg = (content: string, metadata: string | null = null) => ({ content, metadata });
+
+describe('consolidation markers', () => {
+  it('flags a message tagged in metadata', () => {
+    expect(isConsolidationMarker(msg('x', consolidationMetadata()))).toBe(true);
+    expect(isConsolidationMarker(msg('x', null))).toBe(false);
+    expect(isConsolidationMarker(msg('x', '{"feedback":"up"}'))).toBe(false);
+    expect(isConsolidationMarker(msg('x', 'not json'))).toBe(false);
+  });
+
+  it('finds the LAST marker index', () => {
+    const list = [
+      msg('a'),
+      msg('sum1', consolidationMetadata()),
+      msg('b'),
+      msg('sum2', consolidationMetadata()),
+      msg('c'),
+    ];
+    expect(lastConsolidationIndex(list)).toBe(3);
+    expect(lastConsolidationIndex([msg('a'), msg('b')])).toBe(-1);
+  });
+
+  it('scopes the seed from the last marker (inclusive)', () => {
+    const list = [msg('a'), msg('b'), msg('sum', consolidationMetadata()), msg('c')];
+    expect(scopeToConsolidation(list).map((m) => m.content)).toEqual(['sum', 'c']);
+  });
+
+  it('returns the full list unchanged when there is no marker', () => {
+    const list = [msg('a'), msg('b')];
+    expect(scopeToConsolidation(list)).toBe(list);
+  });
+
+  it('wraps a summary with the visible marker prefix', () => {
+    const content = consolidationMarkerContent('  hello  ');
+    expect(content).toBe(`${CONSOLIDATION_MARKER_PREFIX}hello`);
+  });
+});
+
+describe('auto consolidation markers (a persisted fold)', () => {
+  const row = (id: number, content: string, metadata: string | null = null) => ({ id, content, metadata });
+
+  it('keeps every message from keepFromId on, after the marker, and drops older notes', () => {
+    const list = [
+      row(1, 'old user'), row(2, 'old reply'),
+      row(3, 'superseded note', consolidationMetadata({ keepFromId: 2 })),
+      row(4, 'user turn'), row(5, 'narration'),
+      row(6, 'note', consolidationMetadata({ keepFromId: 4, folded: 12 })),
+      row(7, 'final reply'),
+    ];
+    expect(scopeToConsolidation(list).map((m) => m.content)).toEqual(['note', 'user turn', 'narration', 'final reply']);
+  });
+
+  it('reads as manual when keepFromId is gone or after the marker', () => {
+    const gone = [row(1, 'a'), row(2, 'note', consolidationMetadata({ keepFromId: 99 })), row(3, 'b')];
+    expect(scopeToConsolidation(gone).map((m) => m.content)).toEqual(['note', 'b']);
+  });
+
+  it('round-trips the auto metadata', () => {
+    expect(consolidationMarkerOf(row(1, 'x', consolidationMetadata({ keepFromId: 4, folded: 3 })))).toEqual({ consolidation: true, auto: true, keepFromId: 4, folded: 3 });
+    expect(consolidationMarkerOf(row(1, 'x', consolidationMetadata()))).toEqual({ consolidation: true });
+    expect(consolidationMarkerOf(row(1, 'x', null))).toBeNull();
+  });
+});
+
+describe('foldKeepFromId', () => {
+  const anchors = [{ index: 0, id: 10 }, { index: 1, id: 11 }, { index: 2, id: 12 }, { index: 7, id: 20 }];
+
+  it('is exact when a persisted message sits at the cut', () => {
+    expect(foldKeepFromId(anchors, 2)).toBe(12);
+  });
+
+  it('falls back to the nearest persisted message before the cut (covered twice, never lost)', () => {
+    expect(foldKeepFromId(anchors, 5)).toBe(12);
+    expect(foldKeepFromId(anchors, 30)).toBe(20);
+  });
+
+  it('is undefined with nothing persisted before the cut', () => {
+    expect(foldKeepFromId([{ index: 4, id: 1 }], 2)).toBeUndefined();
+  });
+});

package/src/working/consolidationMarker.ts

@@ -0,0 +1,115 @@
+/**
+ * Chat consolidation markers — the durable form of a compressed conversation.
+ *
+ * A long conversation is compressed into ONE summary that becomes the new base context:
+ * everything before the marker stays visible in the transcript, but the next run's model
+ * seed starts from the marker instead of the full history. The marker is a normal
+ * assistant message (so a reader SEES the summary) tagged in its metadata.
+ *
+ * Two writers:
+ *   - MANUAL — "Consolidate" / "Fork": the summary covers everything before the marker.
+ *   - AUTO — a run's working-transcript fold (`transcript.ts`), persisted so the next run
+ *     (after a reload, on another surface) starts from the note instead of summarizing the
+ *     whole history again. A fold covers only the OLDER part of the transcript; its
+ *     `keepFromId` names the first persisted message it did NOT cover, and every message
+ *     from there on stays verbatim in the seed. Nothing is in neither.
+ */
+
+/** The marker's metadata. `auto` / `keepFromId` / `folded` are set only by a run's fold. */
+export interface ConsolidationMarkerMeta {
+  consolidation: true;
+  /** Written by auto-compaction, not by a person. */
+  auto?: true;
+  /** The first persisted message the note does NOT cover (kept verbatim after the marker). */
+  keepFromId?: number;
+  /** How many transcript messages the fold newly compressed. */
+  folded?: number;
+}
+
+/** The metadata of a manual marker. */
+export const CONSOLIDATION_META: ConsolidationMarkerMeta = { consolidation: true };
+
+/** Serialized metadata for a marker message (ready to persist). Manual when no `auto`. */
+export function consolidationMetadata(auto?: { keepFromId?: number; folded?: number }): string {
+  if (!auto) return JSON.stringify(CONSOLIDATION_META);
+  const meta: ConsolidationMarkerMeta = {
+    consolidation: true,
+    auto: true,
+    ...(auto.keepFromId !== undefined ? { keepFromId: auto.keepFromId } : {}),
+    ...(auto.folded !== undefined ? { folded: auto.folded } : {}),
+  };
+  return JSON.stringify(meta);
+}
+
+/** The marker metadata a persisted message carries, or null when it is not a marker. */
+export function consolidationMarkerOf(msg: { metadata?: string | null }): ConsolidationMarkerMeta | null {
+  if (!msg.metadata) return null;
+  try {
+    const m = JSON.parse(msg.metadata) as Partial<ConsolidationMarkerMeta> | null;
+    if (m?.consolidation !== true) return null;
+    return {
+      consolidation: true,
+      ...(m.auto === true ? { auto: true as const } : {}),
+      ...(typeof m.keepFromId === 'number' && Number.isFinite(m.keepFromId) ? { keepFromId: m.keepFromId } : {}),
+      ...(typeof m.folded === 'number' && Number.isFinite(m.folded) && m.folded > 0 ? { folded: m.folded } : {}),
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** True when a persisted message is a consolidation marker (by its metadata flag). */
+export function isConsolidationMarker(msg: { metadata?: string | null }): boolean {
+  return consolidationMarkerOf(msg) !== null;
+}
+
+/** The index of the LAST consolidation marker in a message list, or -1 if none. */
+export function lastConsolidationIndex(messages: ReadonlyArray<{ metadata?: string | null }>): number {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    if (isConsolidationMarker(messages[i]!)) return i;
+  }
+  return -1;
+}
+
+/**
+ * Trim a message list to the compressed context: the last marker, then what it does not
+ * cover. A manual marker covers everything before it (the list from the marker on). An
+ * auto marker covers up to its `keepFromId`: the marker, then every message from that id
+ * on (other markers dropped — an older note is superseded by the newer one). Unchanged
+ * when there is no marker; a `keepFromId` no longer in the list reads as manual.
+ */
+export function scopeToConsolidation<T extends { id?: number; metadata?: string | null }>(messages: T[]): T[] {
+  const idx = lastConsolidationIndex(messages);
+  if (idx < 0) return messages;
+  const marker = messages[idx]!;
+  const keepFromId = consolidationMarkerOf(marker)?.keepFromId;
+  const from = keepFromId === undefined ? -1 : messages.findIndex((m) => m.id === keepFromId);
+  if (from < 0 || from > idx) return messages.slice(idx);
+  return [marker, ...messages.slice(from).filter((m) => m !== marker && !isConsolidationMarker(m))];
+}
+
+/**
+ * Where an auto marker's verbatim part begins: given the persisted messages the run's
+ * transcript holds (`anchors`: transcript index → message id) and the index the fold now
+ * covers up to (exclusive), the id of the persisted message the next seed must keep from.
+ *
+ * Exact when a persisted message sits AT `coveredEnd` (the fold ended inside seeded
+ * history). Otherwise the nearest persisted message BEFORE it — conservative: the span
+ * between it and `coveredEnd` is then both in the note and verbatim, never in neither.
+ * Undefined when no persisted message precedes the cut (nothing to keep from).
+ */
+export function foldKeepFromId(anchors: ReadonlyArray<{ index: number; id: number }>, coveredEnd: number): number | undefined {
+  let best: { index: number; id: number } | undefined;
+  for (const a of anchors) {
+    if (a.index <= coveredEnd && (!best || a.index > best.index)) best = a;
+  }
+  return best?.id;
+}
+
+/** The visible header prefixed onto a consolidation summary so a reader recognizes it. */
+export const CONSOLIDATION_MARKER_PREFIX = '📌 **Consolidated summary** — context continues from here.\n\n';
+
+/** Wrap a raw summary as the marker's visible content (prefix + summary). */
+export function consolidationMarkerContent(summary: string): string {
+  return `${CONSOLIDATION_MARKER_PREFIX}${summary.trim()}`;
+}

package/src/working/index.ts

@@ -15,6 +15,11 @@
  * The memory knows observations ("read this file", "changed that one"), never tool
  * names — a host maps its own tool vocabulary onto {@link Observation}.
  *
+ * And what the MODEL sees of a run each turn — the working TRANSCRIPT (`transcript.ts`):
+ * the drop-oldest window and the incremental auto-compaction fold, the ONE summarizer
+ * prompt it folds with (`summary.ts`), and the consolidation markers a fold or a
+ * "Consolidate" persists so the next run starts from the note (`consolidationMarker.ts`).
+ *
  * Pure: no `node:*`, no I/O — runs in a Cloudflare Worker, a browser page and Node alike.
  */
 
@@ -22,3 +27,33 @@ export { WorkingMemory } from './workingMemory.js';
 export { consolidate, mergeConsolidated, consolidatedKey, CONSOLIDATED_KEY_PREFIX, MIN_CONSOLIDATED_FILES } from './consolidate.js';
 export { rankByCount } from './rank.js';
 export type { Observation, WorkingMemorySnapshot, ConsolidatedMemory, ConsolidateOptions } from './types.js';
+export {
+  HISTORY_WINDOW,
+  HISTORY_TOKEN_BUDGET,
+  COMPACT_TAIL_TOKEN_BUDGET,
+  COMPACT_TAIL_TURNS,
+  windowed,
+  stillInWorkingContext,
+  verbatimStart,
+  compactTailStart,
+  compactTailStartForBudget,
+  pinnedDirectiveIndex,
+  assembleCompacted,
+  transcriptTokens,
+  buildWorkingTranscript,
+} from './transcript.js';
+export type { TranscriptMessage, CompactMemo, WorkingTranscriptState, TranscriptFold } from './transcript.js';
+export { TRANSCRIPT_SUMMARY_PROMPT, TRANSCRIPT_SUMMARY_MAX_TOKENS, renderForSummary, summarizeTranscript } from './summary.js';
+export type { SummaryCompletion } from './summary.js';
+export {
+  CONSOLIDATION_META,
+  CONSOLIDATION_MARKER_PREFIX,
+  consolidationMetadata,
+  consolidationMarkerOf,
+  consolidationMarkerContent,
+  isConsolidationMarker,
+  lastConsolidationIndex,
+  scopeToConsolidation,
+  foldKeepFromId,
+} from './consolidationMarker.js';
+export type { ConsolidationMarkerMeta } from './consolidationMarker.js';

package/src/working/summary.test.ts

@@ -0,0 +1,51 @@
+/**
+ * src/working/summary.test.ts
+ * The ONE transcript summarizer: what it asks, how it renders tool calls, and that a
+ * failing or empty completion reads as "no summary" (the caller falls back).
+ */
+
+import { renderForSummary, summarizeTranscript, TRANSCRIPT_SUMMARY_MAX_TOKENS, TRANSCRIPT_SUMMARY_PROMPT } from './summary.js';
+import { buildWorkingTranscript, HISTORY_TOKEN_BUDGET, type TranscriptFold, type TranscriptMessage, type WorkingTranscriptState } from './transcript.js';
+
+describe('renderForSummary', () => {
+  it('names the tools an assistant turn called', () => {
+    const text = renderForSummary([
+      { role: 'user', content: 'fix it' },
+      { role: 'assistant', content: '', tool_calls: [{ function: { name: 'read_file' } }, { function: { name: 'edit' } }] },
+      { role: 'tool', content: { ok: true } },
+    ]);
+    expect(text).toBe('user: fix it\n\nassistant [called: read_file, edit]: \n\ntool: {"ok":true}');
+  });
+});
+
+describe('summarizeTranscript', () => {
+  it('asks the shared prompt with the bounded budget and trims the answer', async () => {
+    const seen: Array<{ system: string; maxTokens: number }> = [];
+    const out = await summarizeTranscript(async (req) => { seen.push(req); return '  the note  '; }, [{ role: 'user', content: 'x' }]);
+    expect(out).toBe('the note');
+    expect(seen[0]).toMatchObject({ system: TRANSCRIPT_SUMMARY_PROMPT, maxTokens: TRANSCRIPT_SUMMARY_MAX_TOKENS });
+  });
+
+  it('is null for nothing to summarize, an empty answer, or a failure', async () => {
+    expect(await summarizeTranscript(async () => 'x', [])).toBeNull();
+    expect(await summarizeTranscript(async () => '   ', [{ role: 'user', content: 'x' }])).toBeNull();
+    expect(await summarizeTranscript(async () => { throw new Error('down'); }, [{ role: 'user', content: 'x' }])).toBeNull();
+  });
+});
+
+describe('buildWorkingTranscript fold report', () => {
+  it('reports the note and the index it covers up to', async () => {
+    const big = 'x'.repeat(HISTORY_TOKEN_BUDGET); // ~16k tokens each
+    const state: WorkingTranscriptState = {
+      transcript: Array.from({ length: 12 }, (_, i): TranscriptMessage => ({ role: i % 2 ? 'assistant' : 'user', content: big })),
+      compactMemo: null,
+    };
+    const folds: TranscriptFold[] = [];
+    const out = await buildWorkingTranscript(state, 'sys', async () => 'summary', (f) => folds.push(f));
+    expect(folds).toHaveLength(1);
+    expect(folds[0]!.droppedMessages).toBe(folds[0]!.coveredEnd);
+    expect(folds[0]!.note).toContain('summary');
+    expect(state.compactMemo).toEqual({ note: folds[0]!.note, coveredEnd: folds[0]!.coveredEnd });
+    expect(out[1]).toEqual({ role: 'assistant', content: folds[0]!.note });
+  });
+});

package/src/working/summary.ts

@@ -0,0 +1,68 @@
+/**
+ * The transcript summarizer — the ONE prompt every fold, hand-off note and chat
+ * consolidation compresses a conversation with. The model call is the host's
+ * ({@link SummaryCompletion}); this module owns what is asked and how the transcript is
+ * rendered for it, so a compaction note, a hand-off note and a "Consolidate" summary read
+ * the same everywhere.
+ */
+
+import type { TranscriptMessage } from './transcript.js';
+
+/** The summarizer's instructions. */
+export const TRANSCRIPT_SUMMARY_PROMPT =
+  'You compress an in-progress AI agent transcript into a concise MEMORY the agent keeps working from. Capture: the CURRENT outstanding instruction from the user (the most recent user message is authoritative — earlier requests it supersedes are history, not the active task), concrete facts/answers discovered, tool results that matter (ids, paths, values), decisions made, and what still remains to do. Keep every file path, symbol name and line number the remaining work depends on, with the specific facts learned from each file — the agent no longer sees those results, so what you leave out it must read again. Be information-dense; drop pleasantries. No preamble.';
+
+/**
+ * The note's output ceiling. A compaction note is a UTILITY completion: a bounded answer
+ * with no thinking. ~2.5k tokens: the note is the agent's only record of every file it
+ * folds (paths, symbols, line numbers), and 1.2k was too little to carry them.
+ */
+export const TRANSCRIPT_SUMMARY_MAX_TOKENS = 2_500;
+
+/** One no-tools, no-reasoning completion — the host's model call. Resolves the text. */
+export type SummaryCompletion = (request: {
+  system: string;
+  user: string;
+  maxTokens: number;
+  signal?: AbortSignal;
+}) => Promise<string | null | undefined>;
+
+interface NamedToolCall {
+  function?: { name?: unknown };
+}
+
+/** Render a slice of the transcript to the compact text the summarizer compresses. */
+export function renderForSummary(msgs: readonly TranscriptMessage[]): string {
+  return msgs
+    .map((m) => {
+      const content = typeof m.content === 'string' ? m.content : JSON.stringify(m.content ?? '');
+      const calls = Array.isArray(m.tool_calls) && m.tool_calls.length > 0
+        ? ` [called: ${(m.tool_calls as NamedToolCall[]).map((t) => t.function?.name).filter((n): n is string => typeof n === 'string' && n.length > 0).join(', ')}]`
+        : '';
+      return `${m.role}${calls}: ${content}`;
+    })
+    .join('\n\n');
+}
+
+/**
+ * Compress `msgs` into a memory note. Null on any failure or an empty answer, so the
+ * caller falls back (drop-oldest, elision, "nothing to consolidate").
+ */
+export async function summarizeTranscript(
+  complete: SummaryCompletion,
+  msgs: readonly TranscriptMessage[],
+  signal?: AbortSignal,
+): Promise<string | null> {
+  if (msgs.length === 0) return null;
+  try {
+    const out = ((await complete({
+      system: TRANSCRIPT_SUMMARY_PROMPT,
+      user: renderForSummary(msgs),
+      maxTokens: TRANSCRIPT_SUMMARY_MAX_TOKENS,
+      ...(signal ? { signal } : {}),
+    })) ?? '').trim();
+    return out.length > 0 ? out : null;
+  } catch {
+    return null;
+  }
+}