@fungi.computer/shiitake 0.2.0 → 0.3.0

package/dist/internal/compaction/compaction-slot.d.ts

@@ -0,0 +1,36 @@
+import * as Effect from "effect/Effect";
+import type * as Fiber from "effect/Fiber";
+import * as Scope from "effect/Scope";
+import type { SqliteOwner } from "../contracts.js";
+/**
+ * An owner's one compaction slot (background compaction design §5). From
+ * claim until release it holds the only extra decoded copy of a
+ * conversation the owner may have: the summary fiber while it runs, then the
+ * summary it hands back. Every other trigger joins it instead of starting a
+ * second one. Process memory only: a crash loses it, unwritten.
+ *
+ * Deliberate Shiitake difference, not Pi parity: Pi Durable's background
+ * compaction is a durable task that survives a Conversation abort and
+ * resumes after a crash, and Pi does not join triggers into one. Ours is
+ * scoped to the run and single-flight; that is what bounds an owner to one
+ * extra copy of one conversation.
+ */
+export type CompactionSlot<A, E> = {
+    fiber: Fiber.Fiber<void, E> | undefined;
+    summary: A | undefined;
+};
+/** Whether a compaction already holds this owner's slot. */
+export declare function slotTaken(owner: SqliteOwner): boolean;
+/**
+ * Claim the owner's slot and fork `work` into `scope`, which interrupts it
+ * when the run ends. The release finalizer is registered before the fiber
+ * or the slot entry exist, and the entry is written in the same synchronous
+ * step that receives the fiber, so an interrupt anywhere leaves no entry.
+ * `work` hands its summary back through the slot, never through the fiber's
+ * exit, so releasing the slot makes the summary unreachable.
+ */
+export declare function claimSlot<A, E>(owner: SqliteOwner, scope: Scope.Scope, work: (handBack: (summary: A) => void) => Effect.Effect<void, E>): Effect.Effect<CompactionSlot<A, E>>;
+/** Take the finished summary out of the slot, which no longer reaches it. */
+export declare function takeSummary<A, E>(slot: CompactionSlot<A, E>): A | undefined;
+/** Drop every reference the slot holds and free it, however it settled. */
+export declare function releaseSlot<A, E>(owner: SqliteOwner, slot: CompactionSlot<A, E>): void;

package/dist/internal/compaction/compaction-slot.js

@@ -0,0 +1,41 @@
+import * as Effect from "effect/Effect";
+import * as Scope from "effect/Scope";
+const slots = new WeakMap();
+/** Whether a compaction already holds this owner's slot. */
+export function slotTaken(owner) {
+    return slots.has(owner);
+}
+/**
+ * Claim the owner's slot and fork `work` into `scope`, which interrupts it
+ * when the run ends. The release finalizer is registered before the fiber
+ * or the slot entry exist, and the entry is written in the same synchronous
+ * step that receives the fiber, so an interrupt anywhere leaves no entry.
+ * `work` hands its summary back through the slot, never through the fiber's
+ * exit, so releasing the slot makes the summary unreachable.
+ */
+export function claimSlot(owner, scope, work) {
+    const slot = { fiber: undefined, summary: undefined };
+    const task = work((summary) => {
+        slot.summary = summary;
+    });
+    return Scope.addFinalizer(scope, Effect.sync(() => {
+        releaseSlot(owner, slot);
+    })).pipe(Effect.andThen(Effect.forkIn(task, scope)), Effect.map((fiber) => {
+        slot.fiber = fiber;
+        slots.set(owner, slot);
+        return slot;
+    }));
+}
+/** Take the finished summary out of the slot, which no longer reaches it. */
+export function takeSummary(slot) {
+    const summary = slot.summary;
+    slot.summary = undefined;
+    return summary;
+}
+/** Drop every reference the slot holds and free it, however it settled. */
+export function releaseSlot(owner, slot) {
+    slot.fiber = undefined;
+    slot.summary = undefined;
+    if (slots.get(owner) === slot)
+        slots.delete(owner);
+}

package/dist/internal/compaction/compaction.d.ts

@@ -2,6 +2,8 @@
  * Pi v0.84.4 semantic compaction, adapted only at Woodstock's entry boundary.
  * The retained preparation/result vocabulary is intentionally visible here;
  * Shiitake supplies native Message values and its selected model stream.
+ * Range selection follows Pi Durable spec §8.7 (`earendil-works/pi` at
+ * 8efd314f) and token estimates are pi-ai's own.
  */
 import type { Api, Model, Usage } from "@earendil-works/pi-ai";
 import type { MuleStreamFunction } from "@fungi.computer/mule/effect";
@@ -16,9 +18,10 @@ export type CompactionSourceEntry = Readonly<{
 }> | Readonly<{
     id: string;
     kind: "compaction";
-    timestamp: number;
-    firstKeptEntryId: string;
-    summary: string;
+    /** Pi's entry head: the first entry the summary keeps. */
+    head: string;
+    /** Pi's entry model message: the summary the model reads. */
+    model: import("@earendil-works/pi-ai").UserMessage;
     usage: Usage;
     details?: Readonly<{
         readFiles: readonly string[];
@@ -51,4 +54,4 @@ export declare function prepareCompaction(entries: readonly CompactionSourceEntr
 export declare function compact(preparation: CompactionPreparation, model: Model<Api>, streamFn?: MuleStreamFunction, 
 /** A manual compaction's focus, appended to the history summary prompt. */
 instructions?: string): Effect.Effect<CompactionResult, ShiitakeError>;
-export declare function borrowedView(preparation: CompactionPreparation, result: CompactionResult, contextRevision: string): BorrowedCompactionViewV0;
+export declare function borrowedView(preparation: CompactionPreparation, result: CompactionResult, sessionId: string, contextRevision: string): BorrowedCompactionViewV0;

package/dist/internal/compaction/compaction.js

@@ -1,11 +1,12 @@
 // Leaf helpers, not Pi's root entry, which also evaluates its validators.
+import { estimateContextTokens, estimateMessageTokens, } from "@earendil-works/pi-ai/utils/estimate";
 import { retryAssistantCall } from "@earendil-works/pi-ai/utils/retry";
 import { contentText } from "@earendil-works/pi-ai/utils/text";
 import { normalizeContext } from "@earendil-works/pi-ai/utils/transcript";
 import { uuidv7 } from "@earendil-works/pi-ai/utils/uuid";
 import * as Effect from "effect/Effect";
 import { ShiitakeError } from "../../errors.js";
-import { computeFileLists, createFileOps, extractFileOpsFromMessage, formatFileOperations, serializeConversation, SUMMARIZATION_PROMPT, SUMMARIZATION_SYSTEM_PROMPT, TURN_PREFIX_SUMMARIZATION_PROMPT, UPDATE_SUMMARIZATION_PROMPT, compactionSummaryMessage, estimateContextTokens, estimateTokens, summaryFailure, } from "./utils.js";
+import { computeFileLists, createFileOps, extractFileOpsFromMessage, formatFileOperations, serializeConversation, SUMMARIZATION_PROMPT, SUMMARIZATION_SYSTEM_PROMPT, TURN_PREFIX_SUMMARIZATION_PROMPT, UPDATE_SUMMARIZATION_PROMPT, compactionSummaryText, summaryFailure, } from "./utils.js";
 /** A summary is a model call: its bounded provider message is shown. */
 function failure(message) {
     return new ShiitakeError("model", "rejected", message);
@@ -41,14 +42,11 @@ function contextMessages(entries) {
     const latestCompaction = latest;
     const keptIndex = latestCompaction === undefined
         ? 0
-        : Math.max(0, entries.findIndex(({ id }) => id === latestCompaction.firstKeptEntryId));
+        : Math.max(0, entries.findIndex(({ id }) => id === latestCompaction.head));
     const messages = entries.flatMap((entry, index) => entry.kind === "message" && index >= keptIndex ? [entry.message] : []);
     if (latestCompaction === undefined)
         return messages;
-    return [
-        compactionSummaryMessage(latestCompaction.summary, latestCompaction.timestamp),
-        ...messages,
-    ];
+    return [latestCompaction.model, ...messages];
 }
 function extractFileOperations(messages, entries, previousIndex) {
     const fileOps = createFileOps();
@@ -63,35 +61,65 @@ function extractFileOperations(messages, entries, previousIndex) {
         extractFileOpsFromMessage(message, fileOps);
     return fileOps;
 }
-function isCutPoint(message) {
-    return message.role !== "toolResult";
+/**
+ * Whether a result for one of `calls` follows `from` before the next assistant
+ * message, so a user message there sits between a tool call and its result.
+ */
+function resultFollows(entries, from, endIndex, calls) {
+    for (let index = from; index < endIndex; index += 1) {
+        const entry = entries[index];
+        if (entry?.kind !== "message")
+            continue;
+        if (entry.message.role === "assistant")
+            return false;
+        if (entry.message.role === "toolResult" &&
+            calls.has(entry.message.toolCallId))
+            return true;
+    }
+    return false;
+}
+/**
+ * Pi Durable §8.7 rule 1: cut candidates are user and assistant messages.
+ * Tool results and system messages never are, so a kept assistant message
+ * keeps its tool results; a user message is not one while a result for a call
+ * of the assistant before it still follows it.
+ */
+function cutPoints(entries, startIndex, endIndex) {
+    const points = [];
+    let openCalls = new Set();
+    for (let index = startIndex; index < endIndex; index += 1) {
+        const entry = entries[index];
+        if (entry?.kind !== "message")
+            continue;
+        const message = entry.message;
+        if (message.role === "assistant") {
+            points.push(index);
+            openCalls = new Set(message.content.flatMap((block) => block.type === "toolCall" ? [block.id] : []));
+        }
+        else if (message.role === "user" &&
+            !resultFollows(entries, index + 1, endIndex, openCalls))
+            points.push(index);
+    }
+    return points;
 }
+/**
+ * Pi Durable §8.7 rules 2-3: walk newest to oldest adding each message's
+ * estimate; where the sum reaches `keepRecentTokens`, cut at the first
+ * candidate at or after that entry, or the newest candidate when none
+ * follows. Nothing is compacted while the sum never reaches it.
+ */
 function selectCutIndex(entries, startIndex, endIndex, keepRecentTokens) {
-    const points = entries.flatMap((entry, index) => entry.kind === "message" &&
-        index >= startIndex &&
-        index < endIndex &&
-        isCutPoint(entry.message)
-        ? [index]
-        : []);
-    const firstPoint = points[0];
-    if (firstPoint === undefined)
-        return undefined;
+    const points = cutPoints(entries, startIndex, endIndex);
     let accumulated = 0;
-    let cut = firstPoint;
     for (let index = endIndex - 1; index >= startIndex; index -= 1) {
         const entry = entries[index];
         if (entry?.kind !== "message")
             continue;
-        accumulated += estimateTokens(entry.message);
-        if (accumulated >= keepRecentTokens) {
-            // Pi 0.99.2: when trailing tool results alone exceed the budget, keep
-            // their tool call (the last cut point), never fall back to the start.
-            cut =
-                points.find((candidate) => candidate >= index) ?? points.at(-1) ?? cut;
-            break;
-        }
+        accumulated += estimateMessageTokens(entry.message);
+        if (accumulated >= keepRecentTokens)
+            return points.find((candidate) => candidate >= index) ?? points.at(-1);
     }
-    return cut;
+    return undefined;
 }
 function classifyCut(entries, startIndex, cut) {
     const cutEntry = entries[cut];
@@ -132,9 +160,9 @@ export function prepareCompaction(entries, settings) {
     for (let index = entries.length - 1; index >= 0; index -= 1) {
         const entry = entries[index];
         if (entry?.kind === "compaction") {
-            previousSummary = entry.summary;
+            previousSummary = compactionSummaryText(entry.model);
             previousIndex = index;
-            const kept = entries.findIndex((candidate) => candidate.id === entry.firstKeptEntryId);
+            const kept = entries.findIndex((candidate) => candidate.id === entry.head);
             boundaryStart = kept >= 0 ? kept : index + 1;
             break;
         }
@@ -281,8 +309,9 @@ instructions) {
         catch: (error) => failure(error instanceof Error ? error.message : String(error)),
     });
 }
-export function borrowedView(preparation, result, contextRevision) {
+export function borrowedView(preparation, result, sessionId, contextRevision) {
     return {
+        sessionId,
         messagesToSummarize: preparation.messagesToSummarize,
         turnPrefixMessages: preparation.turnPrefixMessages,
         result,

package/dist/internal/compaction/session-compaction.d.ts

@@ -1,12 +1,15 @@
 import { type Message, type Model, type Api, type MuleRunOutcome, type MuleStreamFunction } from "@fungi.computer/mule/effect";
 import * as Effect from "effect/Effect";
+import type * as Scope from "effect/Scope";
 import type { CompactionHandoffV0 } from "../../extensions.js";
 import { ShiitakeError } from "../../errors.js";
+import { type CompactionResult } from "./compaction.js";
 import { type CompactionSettings } from "./utils.js";
 import { type StoredSessionRow } from "../woodstock/index.js";
 import type { SqliteOwner } from "../contracts.js";
 declare const commitFailures: {
     readonly handoff: "The compaction handoff failed; the conversation context is unchanged";
+    readonly handoffResult: "The compaction handoff failed: it returned an invalid result; the conversation context is unchanged";
     readonly conflict: "The Session changed during compaction; the conversation context is unchanged";
 };
 /**
@@ -15,7 +18,8 @@ declare const commitFailures: {
  */
 export declare class CompactionCommitError extends ShiitakeError {
     readonly phase: keyof typeof commitFailures;
-    constructor(phase: keyof typeof commitFailures);
+    readonly usage?: CompactionResult["usage"] | undefined;
+    constructor(phase: keyof typeof commitFailures, usage?: CompactionResult["usage"] | undefined);
     get diagnostic(): string;
 }
 /**
@@ -39,22 +43,10 @@ export type SessionCompactionInput = Readonly<{
     run: (messages: readonly Message[], betweenTurns: BetweenTurns) => Effect.Effect<MuleRunOutcome, ShiitakeError>;
 }>;
 /**
- * The compaction owner's context estimate: Pi's usage-plus-trailing count,
- * excluding provider usage that belongs before Woodstock's latest boundary.
- *
- * A provider's reported usage measures the context it actually read, so it
- * is authoritative only while nothing has reshaped that context since. Once
- * a compaction lands, every kept message's own usage figure is stale: it
- * priced the pre-compaction history, not the summary that replaced it. The
- * one safe fallback is a fresh chars/4 estimate of every message still in
- * view, the same method `tokensAfterCompaction` uses when it records the
- * marker's own `tokensAfter` at commit time; trailing-after-usage alone
- * would undercount (even to zero, when the kept tail's own reply is the
- * newest message and nothing followed it).
+ * Compose the retained pre-run, between-turn and post-run compaction phases.
+ * A background compaction is a child fiber in the run's own scope.
  */
-export declare function estimateThresholdContextTokens(path: readonly StoredSessionRow[], messages: readonly Message[]): number;
-/** Compose the retained pre-run, between-turn and post-run compaction phases. */
-export declare function runWithCompaction(input: SessionCompactionInput): Effect.Effect<MuleRunOutcome, ShiitakeError>;
+export declare function runWithCompaction(input: SessionCompactionInput): Effect.Effect<MuleRunOutcome, ShiitakeError, Scope.Scope>;
 /** What a manual compaction did; a failure is the Effect's error. */
 export type ManualCompactionOutcome = "compacted" | "nothing_to_compact";
 /**