effect-agent 0.1.0-beta.85 → 0.1.0-beta.87

package/src/durable/ThreadProjection.ts

@@ -0,0 +1,269 @@
+import { Schema } from "effect";
+
+import { ThreadId, RunId, ToolCallId } from "../core/Identifiers.ts";
+import { EMPTY_TAIL_DIGEST } from "./Digest.ts";
+import type { CanonicalRecordEnvelope } from "./Records.ts";
+import {
+  AbortRequested,
+  CanonicalSequence,
+  Digest,
+  PersistedJson,
+  SubagentJoined,
+  SubagentLineageRecorded,
+  SubagentRequested,
+  SubagentStarted,
+  SubmissionSettledRecord,
+  ToolApprovalDecided,
+  ToolApprovalRequested,
+  ToolCallPrepared,
+  ToolCallUnknown,
+} from "./Records.ts";
+
+/** One prepared ordinary Tool Call still awaiting a canonical settled/resolved outcome. */
+export class OpenToolCallState extends Schema.Class<OpenToolCallState>(
+  "@effect-agent/thread/OpenToolCallState",
+)({
+  toolCallId: ToolCallId,
+  toolName: ToolCallPrepared.fields.toolName,
+  turn: ToolCallPrepared.fields.turn,
+  runId: RunId,
+}) {}
+
+/** The canonical approval trail: requests and decisions in canonical order. */
+export const ApprovalRecord = Schema.Union([ToolApprovalRequested, ToolApprovalDecided]);
+export type ApprovalRecord = typeof ApprovalRecord.Type;
+
+/**
+ * Parent-side view of one Subagent Invocation, keyed by the parent Run and Tool Call: the canonical
+ * `SubagentRequested`/`SubagentStarted`/`SubagentJoined` payloads as they become canonical
+ * in history. A disposable derived view — the canonical records and the child's
+ * own Settlement remain the recovery truth (DUR-015).
+ */
+export class SubagentInvocationState extends Schema.Class<SubagentInvocationState>(
+  "@effect-agent/thread/SubagentInvocationState",
+)({
+  runId: RunId,
+  toolCallId: ToolCallId,
+  requested: Schema.optionalKey(SubagentRequested),
+  started: Schema.optionalKey(SubagentStarted),
+  joined: Schema.optionalKey(SubagentJoined),
+}) {}
+
+/**
+ * Rebuildable canonical projection. It contains only canonical values and can be discarded and
+ * reconstructed from the record stream at any time.
+ *
+ * Phase 4 added `settlements` and `abortRequests`; Phase 5 added `openToolCalls` (the
+ * prepared-minus-settled/resolved fold), `unknownToolCalls`, and `approvals`; S2 adds
+ * `subagentInvocations` (the parent-side requested/started/joined fold) and `parentLink` (the
+ * child-side immutable lineage). A checkpoint whose persisted state lacks these fields fails to
+ * decode against this schema; callers must reject the checkpoint and rebuild the projection from
+ * canonical records (documented disposable-checkpoint behavior, STORE-007/STORE-008).
+ * Version 2 scopes Tool identities by Run. Earlier projections must rebuild, including empty
+ * invocation views that may have already lost an earlier Run's unresolved Tool evidence.
+ * `ModelResponseRecorded` advances `throughSequence` without dedicated projection state: Prompt
+ * reconstruction reads canonical records directly.
+ */
+export class ThreadProjection extends Schema.Class<ThreadProjection>(
+  "@effect-agent/thread/ThreadProjection",
+)({
+  schemaVersion: Schema.Literal(2),
+  threadId: ThreadId,
+  throughSequence: CanonicalSequence,
+  tailDigest: Digest,
+  inputs: Schema.Array(PersistedJson),
+  modelOutputs: Schema.Array(PersistedJson),
+  completedRuns: Schema.Array(RunId),
+  failedRuns: Schema.Array(RunId),
+  settlements: Schema.Array(SubmissionSettledRecord),
+  abortRequests: Schema.Array(AbortRequested),
+  openToolCalls: Schema.Array(OpenToolCallState),
+  unknownToolCalls: Schema.Array(ToolCallUnknown),
+  approvals: Schema.Array(ApprovalRecord),
+  subagentInvocations: Schema.Array(SubagentInvocationState),
+  parentLink: Schema.optionalKey(SubagentLineageRecorded),
+}) {}
+
+export const initialThreadProjection = (threadId: ThreadId): ThreadProjection =>
+  ThreadProjection.make({
+    schemaVersion: 2,
+    threadId,
+    throughSequence: Schema.decodeSync(CanonicalSequence)(0),
+    tailDigest: EMPTY_TAIL_DIGEST,
+    inputs: [],
+    modelOutputs: [],
+    completedRuns: [],
+    failedRuns: [],
+    settlements: [],
+    abortRequests: [],
+    openToolCalls: [],
+    unknownToolCalls: [],
+    approvals: [],
+    subagentInvocations: [],
+  });
+
+/**
+ * Idempotent per-Run-and-Tool-Call upsert of the parent-side Subagent fold. Deterministic record
+ * identities make duplicates impossible in one canonical stream; the first canonical payload of
+ * each stage wins so a replayed reduce is a no-op.
+ */
+const upsertSubagentInvocation = (
+  invocations: ReadonlyArray<SubagentInvocationState>,
+  payload: SubagentRequested | SubagentStarted | SubagentJoined,
+): ReadonlyArray<SubagentInvocationState> => {
+  const existing = invocations.find(
+    (invocation) =>
+      invocation.runId === payload.runId && invocation.toolCallId === payload.toolCallId,
+  );
+
+  const requested =
+    existing?.requested ?? (payload._tag === "SubagentRequested" ? payload : undefined);
+
+  const started = existing?.started ?? (payload._tag === "SubagentStarted" ? payload : undefined);
+  const joined = existing?.joined ?? (payload._tag === "SubagentJoined" ? payload : undefined);
+
+  const next = SubagentInvocationState.make({
+    runId: payload.runId,
+    toolCallId: payload.toolCallId,
+    ...(requested === undefined ? {} : { requested }),
+    ...(started === undefined ? {} : { started }),
+    ...(joined === undefined ? {} : { joined }),
+  });
+
+  return existing === undefined
+    ? [...invocations, next]
+    : invocations.map((invocation) =>
+        invocation.runId === payload.runId && invocation.toolCallId === payload.toolCallId
+          ? next
+          : invocation,
+      );
+};
+
+/** Pure one-record Thread transition. */
+export const reduceThreadRecord = (
+  projection: ThreadProjection,
+  envelope: CanonicalRecordEnvelope,
+  tailDigest: Digest = projection.tailDigest,
+): ThreadProjection => {
+  const payload = envelope.record.payload;
+
+  const inputs =
+    payload._tag === "UserInputRecorded"
+      ? [...projection.inputs, payload.input]
+      : projection.inputs;
+
+  const modelOutputs =
+    payload._tag === "ModelCompleted"
+      ? [...projection.modelOutputs, payload.output]
+      : projection.modelOutputs;
+
+  const completedRuns =
+    payload._tag === "RunCompleted"
+      ? [...projection.completedRuns, payload.runId]
+      : projection.completedRuns;
+
+  const failedRuns =
+    payload._tag === "RunFailed"
+      ? [...projection.failedRuns, payload.runId]
+      : projection.failedRuns;
+
+  const settlements =
+    payload._tag === "SubmissionSettled"
+      ? [...projection.settlements, payload]
+      : projection.settlements;
+
+  const abortRequests =
+    payload._tag === "AbortRequested"
+      ? [...projection.abortRequests, payload]
+      : projection.abortRequests;
+
+  // `ToolCallPrepared` opens a call; `ToolCallSettled` (results batch or late settle) and
+  // `ToolCallResolved` (DUR-017) close it. `ToolCallUnknown` does NOT close it: the call stays
+  // open until an authorized resolution or a recovered result arrives.
+  const openToolCalls =
+    payload._tag === "ToolCallPrepared"
+      ? projection.openToolCalls.some(
+          (call) => call.runId === payload.runId && call.toolCallId === payload.toolCallId,
+        )
+        ? projection.openToolCalls
+        : [
+            ...projection.openToolCalls,
+            OpenToolCallState.make({
+              toolCallId: payload.toolCallId,
+              toolName: payload.toolName,
+              turn: payload.turn,
+              runId: payload.runId,
+            }),
+          ]
+      : payload._tag === "ToolCallSettled" || payload._tag === "ToolCallResolved"
+        ? projection.openToolCalls.filter(
+            (call) => call.runId !== payload.runId || call.toolCallId !== payload.toolCallId,
+          )
+        : projection.openToolCalls;
+
+  const unknownToolCalls =
+    payload._tag === "ToolCallUnknown"
+      ? [...projection.unknownToolCalls, payload]
+      : projection.unknownToolCalls;
+
+  const approvals =
+    payload._tag === "ToolApprovalRequested" || payload._tag === "ToolApprovalDecided"
+      ? [...projection.approvals, payload]
+      : projection.approvals;
+
+  const subagentInvocations =
+    payload._tag === "SubagentRequested" ||
+    payload._tag === "SubagentStarted" ||
+    payload._tag === "SubagentJoined"
+      ? upsertSubagentInvocation(projection.subagentInvocations, payload)
+      : projection.subagentInvocations;
+
+  // The child-side lineage is immutable (SUB-004): the first canonical record wins forever.
+  const parentLink =
+    projection.parentLink ?? (payload._tag === "SubagentLineageRecorded" ? payload : undefined);
+
+  return ThreadProjection.make({
+    schemaVersion: 2,
+    threadId: projection.threadId,
+    throughSequence: envelope.sequence,
+    tailDigest,
+    inputs,
+    modelOutputs,
+    completedRuns,
+    failedRuns,
+    settlements,
+    abortRequests,
+    openToolCalls,
+    unknownToolCalls,
+    approvals,
+    subagentInvocations,
+    ...(parentLink === undefined ? {} : { parentLink }),
+  });
+};
+
+/** Pure full replay from the canonical beginning. */
+export const replayThread = (
+  threadId: ThreadId,
+  records: ReadonlyArray<CanonicalRecordEnvelope>,
+  tailDigest: Digest = EMPTY_TAIL_DIGEST,
+): ThreadProjection =>
+  replayThreadFromCheckpoint(initialThreadProjection(threadId), records, tailDigest);
+
+/**
+ * Pure checkpoint replay. A validated checkpoint projection and its canonical tail produce the
+ * same reducer path as full replay for every later record.
+ * Decode persisted state with `ThreadProjection` first; an incompatible state requires full replay.
+ */
+export const replayThreadFromCheckpoint = (
+  checkpoint: ThreadProjection,
+  records: ReadonlyArray<CanonicalRecordEnvelope>,
+  tailDigest: Digest = checkpoint.tailDigest,
+): ThreadProjection => {
+  let projection = checkpoint;
+
+  for (const record of records) {
+    projection = reduceThreadRecord(projection, record, tailDigest);
+  }
+
+  return projection;
+};

package/src/durable/ThreadProjectionMaintenance.ts

@@ -0,0 +1,67 @@
+import { Clock, Context, Effect, Layer, Option, Schema } from "effect";
+
+import type { AppendResult, FencedAppendRequest } from "./ThreadStore.ts";
+
+/** A derived projection failed; the canonical source remains authoritative. */
+export class ThreadProjectionError extends Schema.TaggedError<ThreadProjectionError>()(
+  "ThreadProjectionError",
+  {
+    operation: Schema.String,
+    message: Schema.String,
+    cause: Schema.optionalKey(Schema.Defect()),
+  },
+) {}
+
+/**
+ * Host-owned, disposable indexes over a canonical Thread. Hooks receive raw local source
+ * ports at construction and must not mutate them or acquire source producer ownership.
+ * Rows and their contiguous watermark commit atomically; an empty projection still consumes
+ * its record. Concurrent work must compare the starting watermark before committing.
+ *
+ * No method owns a scheduler. The platform owns wakeups, durable scheduling metadata and
+ * source-producer fencing. Per-call resources belong to Scope; interruption remains interruption.
+ */
+export interface ThreadProjectionMaintenanceService {
+  /**
+   * Runs after a successful source commit and before dependent Tools can execute. An index
+   * caught up through firstSequence - 1 must process the entire committed range, including
+   * empty records, before returning. Canonical batches contain at most 256 records: chunk
+   * within local byte limits and stop at result.lastSequence, never chase a changing tail.
+   * An earlier gap defers to drain. Replayed or superseded ranges are idempotent no-ops.
+   * Failure must leave the unprocessed prefix visible to pendingDeadline.
+   */
+  readonly applyCommitted: (
+    request: FencedAppendRequest,
+    result: AppendResult,
+  ) => Effect.Effect<void, ThreadProjectionError>;
+  /** One bounded, restartable backfill batch. Never drain until caught up in a loop. */
+  readonly drain: Effect.Effect<void, ThreadProjectionError>;
+  /**
+   * Bounded local inspection. None means caught up; a pending epoch-millisecond deadline
+   * survives reconstruction and repeated reads must not postpone it. Projection backlog
+   * does not gate canonical execution or host approval publication.
+   */
+  readonly pendingDeadline: Effect.Effect<Option.Option<number>, ThreadProjectionError>;
+}
+
+export class ThreadProjectionMaintenance extends Context.Service<
+  ThreadProjectionMaintenance,
+  ThreadProjectionMaintenanceService
+>()("@effect-agent/thread/ThreadProjectionMaintenance") {
+  static readonly layer = Layer.succeed(this)({
+    applyCommitted: () => Effect.void,
+    drain: Effect.void,
+    pendingDeadline: Effect.succeed(Option.none()),
+  });
+}
+
+/** Run at most one due backfill batch, preserving typed failures, defects and interruption. */
+export const drainDue = Effect.gen(function* () {
+  const projection = yield* ThreadProjectionMaintenance;
+  const deadline = yield* projection.pendingDeadline;
+
+  if (Option.isNone(deadline) || deadline.value > (yield* Clock.currentTimeMillis)) return false;
+  yield* projection.drain;
+
+  return true;
+});

package/src/durable/ThreadStore.ts

@@ -0,0 +1,242 @@
+import type { Effect, Option, Stream } from "effect";
+import { Context, Schema } from "effect";
+
+import { ThreadId } from "../core/Identifiers.ts";
+import {
+  BatchId,
+  CanonicalBatch,
+  CanonicalRecordEnvelope,
+  CanonicalSequence,
+  Digest,
+  ObservationOffset,
+  PersistedJson,
+  ProducerEpoch,
+} from "./Records.ts";
+
+export class ThreadMaterialization extends Schema.Class<ThreadMaterialization>(
+  "@effect-agent/thread/ThreadMaterialization",
+)({
+  threadId: ThreadId,
+  producerEpoch: ProducerEpoch,
+}) {}
+
+export class FencedAppendRequest extends Schema.Class<FencedAppendRequest>(
+  "@effect-agent/thread/FencedAppendRequest",
+)({
+  threadId: ThreadId,
+  batch: CanonicalBatch,
+  expectedTailSequence: CanonicalSequence,
+  expectedTailDigest: Digest,
+  producerEpoch: ProducerEpoch,
+}) {}
+
+export class AppendResult extends Schema.Class<AppendResult>("@effect-agent/thread/AppendResult")({
+  firstSequence: CanonicalSequence,
+  lastSequence: CanonicalSequence,
+  tailDigest: Digest,
+  replayed: Schema.Boolean,
+}) {}
+
+export class ThreadRead extends Schema.Class<ThreadRead>("@effect-agent/thread/ThreadRead")({
+  threadId: ThreadId,
+  afterSequence: Schema.optionalKey(CanonicalSequence),
+  limit: Schema.Natural.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
+}) {}
+
+export class ThreadObservation extends Schema.Class<ThreadObservation>(
+  "@effect-agent/thread/ThreadObservation",
+)({
+  threadId: ThreadId,
+  afterOffset: Schema.optionalKey(ObservationOffset),
+}) {}
+
+export class ThreadExportRequest extends Schema.Class<ThreadExportRequest>(
+  "@effect-agent/thread/ThreadExportRequest",
+)({
+  threadId: ThreadId,
+}) {}
+
+export class ThreadTailRequest extends Schema.Class<ThreadTailRequest>(
+  "@effect-agent/thread/ThreadTailRequest",
+)({
+  threadId: ThreadId,
+}) {}
+
+/**
+ * The committed tail of one Thread Log. A resuming producer composes its next
+ * FencedAppendRequest from this value instead of exporting the whole log.
+ */
+export class ThreadTail extends Schema.Class<ThreadTail>("@effect-agent/thread/ThreadTail")({
+  threadId: ThreadId,
+  tailSequence: CanonicalSequence,
+  tailDigest: Digest,
+  producerEpoch: ProducerEpoch,
+}) {}
+
+/** Maximum canonical records represented by one Thread export. */
+export const MAX_THREAD_EXPORT_RECORDS = 131_072;
+
+export class ThreadExport extends Schema.Class<ThreadExport>("@effect-agent/thread/ThreadExport")({
+  format: Schema.Literal("effect-agent/thread@1"),
+  threadId: ThreadId,
+  tailSequence: CanonicalSequence,
+  tailDigest: Digest,
+  records: Schema.Array(CanonicalRecordEnvelope).check(
+    Schema.isMaxLength(MAX_THREAD_EXPORT_RECORDS),
+  ),
+}) {}
+
+/**
+ * A disposable projection snapshot. Adapters bind its sequence and digest to the canonical
+ * log; consumers decode `state` and decide projection compatibility before suffix replay.
+ * Legacy metadata is optional for application projections and preserved when supplied.
+ * Runtime-owned recovery checkpoints populate and compare it before using cached state.
+ */
+export class ThreadCheckpoint extends Schema.Class<ThreadCheckpoint>(
+  "@effect-agent/thread/ThreadCheckpoint",
+)({
+  schemaVersion: Schema.Literal(1),
+  threadId: ThreadId,
+  throughSequence: CanonicalSequence,
+  tailDigest: Digest,
+  /** @deprecated For application projections; retained for existing data and runtime recovery. */
+  engineVersion: Schema.optionalKey(Schema.NonEmptyString),
+  /** @deprecated For application projections; retained for existing data and runtime recovery. */
+  agentDefinitionDigest: Schema.optionalKey(Digest),
+  /** @deprecated For application projections; retained for existing data and runtime recovery. */
+  modelDigest: Schema.optionalKey(Digest),
+  /** @deprecated For application projections; retained for existing data and runtime recovery. */
+  toolDigest: Schema.optionalKey(Digest),
+  state: PersistedJson,
+  createdAt: Schema.DateTimeUtcFromString,
+}) {}
+
+export class SaveCheckpointRequest extends Schema.Class<SaveCheckpointRequest>(
+  "@effect-agent/thread/SaveCheckpointRequest",
+)({
+  checkpoint: ThreadCheckpoint,
+}) {}
+
+/** Replace the disposable recovery view only under the current canonical producer fence. */
+export class SaveRecoveryCheckpointRequest extends Schema.Class<SaveRecoveryCheckpointRequest>(
+  "@effect-agent/thread/SaveRecoveryCheckpointRequest",
+)({
+  checkpoint: ThreadCheckpoint,
+  producerEpoch: ProducerEpoch,
+}) {}
+
+export class LoadCheckpointRequest extends Schema.Class<LoadCheckpointRequest>(
+  "@effect-agent/thread/LoadCheckpointRequest",
+)({
+  threadId: ThreadId,
+  atOrBeforeSequence: Schema.optionalKey(CanonicalSequence),
+}) {}
+
+export class ThreadStoreError extends Schema.TaggedError<ThreadStoreError>()("ThreadStoreError", {
+  operation: Schema.String,
+  message: Schema.String,
+  cause: Schema.optionalKey(Schema.Defect()),
+}) {}
+
+export class ThreadNotMaterialized extends Schema.TaggedError<ThreadNotMaterialized>()(
+  "ThreadNotMaterialized",
+  { threadId: ThreadId },
+) {}
+
+/**
+ * A canonical append that cannot commit: `batch-digest` replays a batch ID with different
+ * content, `record-identity` reuses a canonical record ID, and `tail` declares a stale expected
+ * tail. Tail conflicts carry the actual committed tail as a diagnostic resume hint.
+ */
+export class AppendConflict extends Schema.TaggedError<AppendConflict>()("AppendConflict", {
+  threadId: ThreadId,
+  batchId: BatchId,
+  reason: Schema.Literals(["batch-digest", "record-identity", "tail"]),
+  actualTailSequence: Schema.optionalKey(CanonicalSequence),
+  actualTailDigest: Schema.optionalKey(Digest),
+}) {}
+
+export class FenceRejected extends Schema.TaggedError<FenceRejected>()("FenceRejected", {
+  threadId: ThreadId,
+  actualEpoch: ProducerEpoch,
+  attemptedEpoch: ProducerEpoch,
+}) {}
+
+export class CheckpointRejected extends Schema.TaggedError<CheckpointRejected>()(
+  "CheckpointRejected",
+  {
+    threadId: ThreadId,
+    reason: Schema.Literals(["ahead-of-tail", "digest-mismatch", "unsupported-version", "corrupt"]),
+  },
+) {}
+
+export type ThreadStoreFailure =
+  | ThreadStoreError
+  | ThreadNotMaterialized
+  | AppendConflict
+  | FenceRejected;
+
+/**
+ * Optional, disposable projection storage. Neither history execution nor durable recovery
+ * requires it. Adapters that offer it must bind every checkpoint to a canonical batch tail.
+ * Application consumers own projection compatibility; this port does not interpret metadata.
+ */
+export interface ThreadCheckpoints {
+  readonly save: (
+    request: SaveCheckpointRequest,
+  ) => Effect.Effect<void, ThreadStoreError | ThreadNotMaterialized | CheckpointRejected>;
+  readonly load: (
+    request: LoadCheckpointRequest,
+  ) => Effect.Effect<
+    Option.Option<ThreadCheckpoint>,
+    ThreadStoreError | ThreadNotMaterialized | CheckpointRejected
+  >;
+}
+
+/**
+ * Optional latest-only recovery cache, independent of application projection checkpoints.
+ * Saves atomically validate the producer epoch and canonical batch tail. An older snapshot
+ * cannot replace a newer one; equal-tail replacement repairs disposable state. Invalid cached
+ * data fails with CheckpointRejected, while infrastructure failures remain ThreadStoreError.
+ * Loading at an earlier tail or outside the adapter's cache locality may return none so callers
+ * can replay canonical records. Canonical records and the ledger remain authority.
+ */
+export interface ThreadRecoveryCheckpoints {
+  readonly save: (
+    request: SaveRecoveryCheckpointRequest,
+  ) => Effect.Effect<
+    void,
+    ThreadStoreError | ThreadNotMaterialized | CheckpointRejected | FenceRejected
+  >;
+  readonly load: ThreadCheckpoints["load"];
+}
+
+export class ThreadStore extends Context.Service<
+  ThreadStore,
+  {
+    readonly materialize: (
+      request: ThreadMaterialization,
+    ) => Effect.Effect<void, ThreadStoreError | FenceRejected>;
+    readonly append: (
+      request: FencedAppendRequest,
+    ) => Effect.Effect<
+      AppendResult,
+      ThreadStoreError | ThreadNotMaterialized | AppendConflict | FenceRejected
+    >;
+    readonly read: (
+      request: ThreadRead,
+    ) => Stream.Stream<CanonicalRecordEnvelope, ThreadStoreError | ThreadNotMaterialized>;
+    readonly observe: (
+      request: ThreadObservation,
+    ) => Stream.Stream<CanonicalRecordEnvelope, ThreadStoreError | ThreadNotMaterialized>;
+    readonly export: (
+      request: ThreadExportRequest,
+    ) => Effect.Effect<ThreadExport, ThreadStoreError | ThreadNotMaterialized>;
+    readonly inspectTail: (
+      request: ThreadTailRequest,
+    ) => Effect.Effect<ThreadTail, ThreadStoreError | ThreadNotMaterialized>;
+    /** Absent when this adapter does not support disposable checkpoints. */
+    readonly checkpoints?: ThreadCheckpoints | undefined;
+    readonly recoveryCheckpoints?: ThreadRecoveryCheckpoints | undefined;
+  }
+>()("@effect-agent/thread/ThreadStore") {}