@inhuman.tools/turn 0.0.0-stage → 0.2.0

package/src/context/retention/view.ts

@@ -0,0 +1,161 @@
+/**
+ * The positions a retention decision is made against. Everything here is read
+ * from recorded facts: the completion order, each turn's R1 stamp, and the
+ * tip's R3 clock. Nothing reads config, kv, the catalog or the wall clock, so
+ * a restart re-projects byte-identically.
+ */
+import { CompletionExpectation, owningTurn, SessionTurn } from "@inhuman.tools/harness-model";
+
+import { LEGACY_POLICY, parsePolicy, type ResolvedPolicy } from "./resolve.js";
+import type { Horizon } from "./schema.js";
+
+export type ClockReason = "turn" | "every" | "freed" | "cold" | "compaction" | "hold";
+
+export type RetentionClock = {
+  /** Uuid of the completion the deep horizons are evaluated against. */
+  readonly at: string;
+  readonly reason: ClockReason;
+  readonly coldByMs?: number;
+  readonly freedTokens?: number;
+};
+
+const REASONS: ReadonlySet<string> = new Set(["turn", "every", "freed", "cold", "compaction", "hold"]);
+
+export function parseClock(json: string): RetentionClock | null {
+  if (json === "") return null;
+  try {
+    const raw = JSON.parse(json) as Record<string, unknown>;
+    if (typeof raw.at !== "string" || raw.at === "" || typeof raw.reason !== "string" || !REASONS.has(raw.reason)) {
+      return null;
+    }
+    return {
+      at: raw.at,
+      reason: raw.reason as ClockReason,
+      ...(typeof raw.coldByMs === "number" ? { coldByMs: raw.coldByMs } : {}),
+      ...(typeof raw.freedTokens === "number" ? { freedTokens: raw.freedTokens } : {}),
+    };
+  } catch {
+    return null;
+  }
+}
+
+/** A turn's policy: its own R1 stamp, else the nearest tail's, else the legacy rule set. */
+export function policyOfTurn(turn: SessionTurn | null): ResolvedPolicy {
+  const seen = new Set<SessionTurn>();
+  for (let cell: unknown = turn; cell instanceof SessionTurn && !seen.has(cell); cell = cell.turnTail) {
+    seen.add(cell);
+    const parsed = parsePolicy(cell.retentionJson);
+    if (parsed !== null) return parsed;
+  }
+  return LEGACY_POLICY;
+}
+
+export type RetentionView = {
+  readonly tip: CompletionExpectation;
+  readonly order: readonly CompletionExpectation[];
+  readonly tipPos: number;
+  readonly tipTurn: number;
+  /** Null: no recorded clock, so deep horizons never expire anything. */
+  readonly clockPos: number | null;
+  readonly clockTurn: number | null;
+  readonly dialect: string | undefined;
+  posOf(c: CompletionExpectation): number;
+  turnOf(c: CompletionExpectation): number;
+  policyOf(c: CompletionExpectation): ResolvedPolicy;
+  /** The completion after `c`, which consumes `c`'s tool results. */
+  nextOf(c: CompletionExpectation): CompletionExpectation | null;
+};
+
+export type ViewOptions = {
+  /** Evaluate the deep horizons here instead of at the recorded clock. Used to estimate a step. */
+  readonly clockAt?: CompletionExpectation | null;
+  readonly dialect?: string;
+};
+
+/** `order` is the lead (kept zone) completions, then the spine, ending at the tip. */
+export function buildView(
+  tip: CompletionExpectation,
+  order: readonly CompletionExpectation[],
+  options: ViewOptions = {},
+): RetentionView {
+  const pos = new Map<CompletionExpectation, number>();
+  const turnOrd = new Map<CompletionExpectation, number>();
+  const turns = new Map<SessionTurn | null, number>();
+  for (const [index, c] of order.entries()) {
+    pos.set(c, index);
+    const turn = owningTurn(c);
+    if (!turns.has(turn)) turns.set(turn, turns.size);
+    turnOrd.set(c, turns.get(turn) ?? 0);
+  }
+  const posOf = (c: CompletionExpectation): number => pos.get(c) ?? -1;
+  const turnOf = (c: CompletionExpectation): number => turnOrd.get(c) ?? 0;
+  let clockCompletion: CompletionExpectation | null;
+  if (options.clockAt !== undefined) clockCompletion = options.clockAt;
+  else {
+    const clock = parseClock(tip.retentionClockJson);
+    clockCompletion = clock === null ? null : (order.find((c) => c.uuid === clock.at) ?? null);
+  }
+  const clockPos = clockCompletion === null ? null : posOf(clockCompletion);
+  return {
+    tip,
+    order,
+    tipPos: posOf(tip),
+    tipTurn: turnOf(tip),
+    clockPos: clockPos === null || clockPos < 0 ? null : clockPos,
+    clockTurn: clockPos === null || clockPos < 0 || clockCompletion === null ? null : turnOf(clockCompletion),
+    dialect: options.dialect,
+    posOf,
+    turnOf,
+    policyOf: (c) => policyOfTurn(owningTurn(c)),
+    nextOf: (c) => order[posOf(c) + 1] ?? null,
+  };
+}
+
+export type ItemPlace = {
+  /** The completion whose input first contains the item. */
+  readonly consumer: CompletionExpectation;
+  /** The completion that produced or claimed the item; its turn is the item's turn. */
+  readonly owner: CompletionExpectation;
+  /** Evaluate against the tip instead of the clock. Pictures live only within their own turn. */
+  readonly againstTip?: boolean;
+};
+
+/**
+ * Any condition met expires the item. `newest` is decided by the caller,
+ * which ranks items of one kind; it is not a span.
+ */
+export function horizonExpired(keep: readonly Horizon[], view: RetentionView, place: ItemPlace): boolean {
+  const consumerPos = view.posOf(place.consumer);
+  const ownerTurn = view.turnOf(place.owner);
+  if (consumerPos < 0) return false;
+  for (const h of keep) {
+    if (h === "round") {
+      if (view.tipPos - consumerPos >= 1) return true;
+      continue;
+    }
+    if (h === "turn") {
+      const ref = place.againstTip === true ? view.tipTurn : view.clockTurn;
+      if (ref !== null && ref > ownerTurn) return true;
+      continue;
+    }
+    if (h === "compaction" || h === "forever") continue;
+    if ("rounds" in h) {
+      const ref = h.rounds === 1 || place.againstTip === true ? view.tipPos : view.clockPos;
+      if (ref !== null && ref - consumerPos >= h.rounds) return true;
+      continue;
+    }
+    if ("turns" in h) {
+      const ref = place.againstTip === true ? view.tipTurn : view.clockTurn;
+      if (ref !== null && ref - ownerTurn >= h.turns) return true;
+    }
+  }
+  return false;
+}
+
+export function newestOf(keep: readonly Horizon[]): number | null {
+  let out: number | null = null;
+  for (const h of keep) {
+    if (typeof h === "object" && "newest" in h) out = out === null ? h.newest : Math.min(out, h.newest);
+  }
+  return out;
+}

package/src/context/squeeze.ts

@@ -0,0 +1,218 @@
+/**
+ * Squeeze — tool-call observation paint. Not compact. Not settlement.
+ *
+ * Stored `outcomeJson` stays the source. This module projects it for the wire.
+ * Compact.scm does not inspect payload size.
+ *
+ * A reduced observation points at `context_recall`: the sealed receipt is read
+ * back from the graph, paged, instead of from a `/tmp` copy. There is no second
+ * world on disk. Regenerateable results (`read` | `net`) get no pointer:
+ * re-invoking is the address.
+ *
+ * Top-level strings stay raw text (a file body stays a file body). Serializer
+ * quoting of a top-level string is the wrong observation for read_file.
+ * Structured `value` payloads ride `toSExprString` caps — not a second compact
+ * pass in the driver.
+ *
+ * MCP image blocks never ride as base64 text: {@link liftToolImages} takes them
+ * out of the painted observation, and the projection decides whether the model
+ * sees the picture or a stub.
+ */
+import { toSExprString, toSExprStringWithElisions } from "@here.build/arrival-serializer";
+import { ProgrammaticJob, ToolCallJob, type ToolCallRepresentation } from "@inhuman.tools/harness-model";
+
+import { humanSize, refOfNode } from "./retention/render.js";
+
+export type SqueezeTo = "compact" | "micro";
+
+/** Measured optimal observation size. Chars, not tokens. */
+export const SQUEEZE_COMPACT_CHARS = 5000;
+
+/** Below this, micro is a no-op — replacing a short answer with a path is pointless. */
+export const SQUEEZE_MICRO_MIN_CHARS = 100;
+
+export type SqueezeInput = {
+  readonly to: SqueezeTo;
+  readonly payload: unknown;
+  readonly persistable: boolean;
+  /** `job:<ref>` the model passes to `context_recall`. Without it a reduced result carries no pointer. */
+  readonly recallRef?: string;
+  readonly name: string;
+};
+
+/** Source observation as a string. Top-level strings stay raw; values ride the serializer. */
+export function paintObservation(payload: unknown): string {
+  if (payload === undefined) return "";
+  if (typeof payload === "string") return payload;
+  return toSExprString(payload);
+}
+
+export function squeezeObservation(input: SqueezeInput): string {
+  const { to, payload, persistable, name } = input;
+  const pointer = persistable && input.recallRef !== undefined ? recallComment(input.recallRef) : "";
+
+  if (typeof payload === "string") {
+    if (to === "micro") {
+      if (payload.length < SQUEEZE_MICRO_MIN_CHARS) return payload;
+      if (!persistable) return clearedStub(name);
+      return pointer === "" ? compactBody(payload).text : pointer;
+    }
+    const { text, reduced } = compactBody(payload);
+    if (reduced && pointer !== "") return `${pointer}\n${text}`;
+    return text;
+  }
+
+  if (to === "micro") {
+    const full = paintObservation(payload);
+    if (full.length < SQUEEZE_MICRO_MIN_CHARS) return full;
+    if (!persistable) return clearedStub(name);
+    return pointer === "" ? squeezeValue(payload) : pointer;
+  }
+
+  const { text, reduced } = toSExprStringWithElisions(payload, {
+    maxTotalChars: SQUEEZE_COMPACT_CHARS,
+    maxStringChars: SQUEEZE_COMPACT_CHARS,
+  });
+  if (reduced && pointer !== "") return `${pointer}\n${text}`;
+  return text;
+}
+
+function squeezeValue(payload: unknown): string {
+  return toSExprString(payload, {
+    maxTotalChars: SQUEEZE_COMPACT_CHARS,
+    maxStringChars: SQUEEZE_COMPACT_CHARS,
+  });
+}
+
+function compactBody(payload: string): { text: string; reduced: boolean } {
+  if (payload.length <= SQUEEZE_COMPACT_CHARS) return { text: payload, reduced: false };
+  const dropped = payload.length - SQUEEZE_COMPACT_CHARS;
+  return {
+    text: `${payload.slice(0, SQUEEZE_COMPACT_CHARS)}…(+${dropped} chars)`,
+    reduced: true,
+  };
+}
+
+function recallComment(ref: string): string {
+  return `;;full response: context_recall ${JSON.stringify({ ref })}`;
+}
+
+function clearedStub(name: string): string {
+  return name === "" ? "[Old tool result content cleared]" : `[Old ${name} result content cleared]`;
+}
+
+const MAX_TOOL_IMAGE_BYTES = 5 * 1024 * 1024;
+
+export type ToolImage = {
+  /** 1-based position among this result's images. */
+  readonly index: number;
+  readonly mime: string;
+  readonly size: number;
+  /** Null when the block was not valid base64 or exceeds the wire cap. */
+  readonly bytes: Uint8Array | null;
+};
+
+type ImageBlock = {
+  readonly type: "image";
+  readonly data?: unknown;
+  readonly mimeType?: unknown;
+  readonly mime_type?: unknown;
+};
+
+function isImageBlock(block: unknown): block is ImageBlock {
+  return typeof block === "object" && block !== null && (block as { type?: unknown }).type === "image";
+}
+
+function decodeBase64(data: unknown): Uint8Array | null {
+  if (typeof data !== "string") return null;
+  const compact = data.replaceAll(/\s/g, "");
+  if (compact === "" || compact.length % 4 !== 0 || !/^[\w+/-]+={0,2}$/.test(compact)) return null;
+  const decoded = Buffer.from(compact, "base64");
+  if (decoded.length === 0 || decoded.length > MAX_TOOL_IMAGE_BYTES) return null;
+  return new Uint8Array(decoded);
+}
+
+/** Dimensionless note: `[Image 1: image/png, 31.4 KB]`, or the elided form with a recall pointer. */
+export function toolImageLine(image: ToolImage, state: "shown" | "elided", ref?: string): string {
+  const head = `Image ${image.index}${state === "elided" ? " elided" : ""}: ${image.mime}, ${humanSize(image.size)}`;
+  return state === "elided" && ref !== undefined ? `[${head} — recall ${ref}]` : `[${head}]`;
+}
+
+/**
+ * Pull image content blocks out of a result so base64 never reaches the model
+ * as text. The painted payload keeps a one-line note per image.
+ */
+export function liftToolImages(
+  payload: unknown,
+  note: (image: ToolImage) => string = (image) => toolImageLine(image, "shown"),
+): { readonly payload: unknown; readonly images: readonly ToolImage[] } {
+  if (!Array.isArray(payload) || !payload.some((block) => isImageBlock(block))) return { payload, images: [] };
+  const images: ToolImage[] = [];
+  const lifted = payload.map((block: unknown) => {
+    if (!isImageBlock(block)) return block;
+    const mimeRaw = block.mimeType ?? block.mime_type;
+    const bytes = decodeBase64(block.data);
+    const size =
+      bytes === null ? (typeof block.data === "string" ? Math.floor((block.data.length * 3) / 4) : 0) : bytes.length;
+    const image: ToolImage = {
+      index: images.length + 1,
+      mime: typeof mimeRaw === "string" && mimeRaw !== "" ? mimeRaw : "image",
+      size,
+      bytes,
+    };
+    images.push(image);
+    return { type: "text", text: note(image) };
+  });
+  return { payload: lifted, images };
+}
+
+/** The images a sealed tool result carries, in order. */
+export function toolImagesOf(tool: ToolCallJob): readonly ToolImage[] {
+  return liftToolImages(tool.observationPayload()).images;
+}
+
+function refField(node: { readonly uuid: string }): { readonly recallRef?: string } {
+  const ref = refOfNode(node);
+  return ref === undefined ? {} : { recallRef: ref };
+}
+
+/** Compact line derived from the sealed receipt. */
+export function compactToolLine(job: ToolCallJob | ProgrammaticJob): string {
+  if (job instanceof ToolCallJob && job.regenerateable) return "";
+  if (job instanceof ToolCallJob) return squeezeToolCall(job, "compact");
+  const parts = job.resultParts();
+  if (parts !== undefined) return parts.map((part) => squeezeProgrammaticText(job, part.text)).join("\n");
+  const text = job.resultText();
+  return squeezeProgrammaticText(job, text !== "" ? text : (job.receiptJson ?? ""));
+}
+
+/** scheme_repl / exam results ride the same cap and pointer as tool results. */
+export function squeezeProgrammaticText(job: ProgrammaticJob, text: string): string {
+  return squeezeObservation({
+    to: "compact",
+    payload: text,
+    persistable: true,
+    ...refField(job),
+    name: "",
+  });
+}
+
+/** Server-side compact wire paint. Not a client-document method. */
+export function squeezeToolCall(tool: ToolCallJob, to: SqueezeTo, note?: (image: ToolImage) => string): string {
+  return squeezeObservation({
+    to,
+    payload: liftToolImages(tool.observationPayload(), note).payload,
+    persistable: !tool.regenerateable,
+    ...refField(tool),
+    name: tool.name,
+  });
+}
+
+export function toolCallRepresentation(tool: ToolCallJob): ToolCallRepresentation {
+  return {
+    id: tool.toolCallId,
+    name: tool.name,
+    args: tool.invocation(),
+    result: paintObservation(liftToolImages(tool.observationPayload()).payload),
+  };
+}

package/src/context/tools.ts

@@ -0,0 +1,92 @@
+/**
+ * Completion tools[] face: frozen turn.tools when present, else live catalog.
+ * Vendor OpenAI wrapping stays in the actor. Live-catalog scheme_repl
+ * description overlay is also the actor — frozen ToolSpec text is left as-is.
+ */
+import {
+  AGENT_RESPOND,
+  ApprovableJob,
+  CompletionExpectation,
+  owningTurn,
+  SessionRoot,
+} from "@inhuman.tools/harness-model";
+import invariant from "tiny-invariant";
+
+import { catalogOf, KernelToolCatalog } from "../tool-catalog/index.js";
+
+export type ContextTool = {
+  readonly name: string;
+  readonly description: string;
+  readonly parameters: Record<string, unknown>;
+};
+
+export function liveCatalogOf(
+  expectation: CompletionExpectation,
+  catalog?: KernelToolCatalog,
+): KernelToolCatalog | undefined {
+  return catalog ?? catalogOf(expectation.plexus?.root);
+}
+
+/** Frozen turn set wins so MCP churn cannot rewrite the prefix. */
+export function selectCompletionTools(
+  expectation: CompletionExpectation,
+  catalog?: KernelToolCatalog,
+): readonly ContextTool[] {
+  const root = expectation.plexus?.root;
+  invariant(root instanceof SessionRoot, "selectCompletionTools: expectation must be wired on a SessionRoot");
+  if (expectation.completionMode === "isolated") return [];
+  if (expectation.parent instanceof ApprovableJob) return [];
+  const turn = owningTurn(expectation);
+  const live = liveCatalogOf(expectation, catalog);
+  const base =
+    turn != null && turn.tools.entries.length > 0
+      ? turn.tools.entries.map((entry) => {
+          const parameters = entry.parameters ?? {};
+          return {
+            name: entry.name,
+            description: entry.description,
+            parameters,
+          };
+        })
+      : live === undefined
+        ? []
+        : live
+            .catalogEntries()
+            .filter((entry) => entry.representation === "tool" || entry.representation === "both")
+            .map((entry) => ({
+              name: entry.wireName,
+              description: entry.description,
+              parameters: entry.parameters,
+            }));
+  const rest = base.filter((tool) => tool.name !== AGENT_RESPOND);
+  return receivesCalls(root) ? [...rest, AGENT_RESPOND_TOOL] : rest;
+}
+
+/**
+ * `agent__respond` is in tools[] for every round of a session that has taken a call, with
+ * one fixed schema and text. The open request ids are not in it: tools[] is the head of the
+ * cached prefix, and a tool that appears with the first open call, lists its ids and vanishes
+ * with the last would rewrite the whole prefix each time. The ids reach the model in the
+ * round's system note instead (`callbackRequiredNote`), after the cached part.
+ */
+const AGENT_RESPOND_TOOL: ContextTool = {
+  name: AGENT_RESPOND,
+  description:
+    "Complete an open worker call. requestId must be one of the open ids listed in the system note. Not assistant text.",
+  parameters: {
+    type: "object",
+    properties: {
+      requestId: { type: "string", description: "Open call to settle" },
+      response: { type: "string", description: "Reply that settles the call" },
+    },
+    required: ["requestId", "response"],
+    additionalProperties: false,
+  },
+};
+
+/** The session was ever asked: a call is open now, or an inbound call turn is on the tail. */
+function receivesCalls(root: SessionRoot): boolean {
+  if (root.openCallbackCount > 0) return true;
+  for (const turn of root.liveTurns) if (turn.inbound || turn.agentRequestId !== "") return true;
+  return false;
+}