effect-agent 0.1.0-beta.144 → 0.1.0-beta.149

package/src/durable/LifecyclePublication.ts

@@ -1,4 +1,4 @@
-import { Clock, Context, Crypto, Effect, Layer, Option, Schema, type Scope } from "effect";
+import { Cause, Clock, Context, Crypto, Effect, Layer, Option, Schema, type Scope } from "effect";
 
 import { SubmissionId, ThreadId } from "../core/Identifiers.ts";
 import { Receipt } from "../core/Receipt.ts";
@@ -122,37 +122,70 @@ export class LifecyclePublicationError extends Schema.TaggedError<LifecyclePubli
   },
 ) {}
 
-/** Native recovery state, not a second application record or launch queue. */
+/** Bound both retained facts and the immutable admission evidence resolved before dispatch. */
+export const lifecyclePublicationBatchMaxFacts = 8;
+
+/** One owner's bounded pending prefix, strictly ordered by its durable ordinals. */
+export const LifecyclePublicationBatch = Schema.NonEmptyArray(LifecyclePublication).check(
+  Schema.isMaxLength(lifecyclePublicationBatchMaxFacts),
+  Schema.makeFilter(
+    (facts) =>
+      facts.every((fact, index) => {
+        const previous = facts[index - 1];
+
+        return (
+          fact.ownerThreadId === facts[0].ownerThreadId &&
+          (previous === undefined || fact.ordinal > previous.ordinal)
+        );
+      }),
+    { expected: "One owner with strictly increasing publication ordinals" },
+  ),
+);
+
+export type LifecyclePublicationBatch = typeof LifecyclePublicationBatch.Type;
+
+/** Native recovery state, independent of Attempts. Hosts serialize drains per storage owner. */
 export interface LifecyclePublicationStorage {
+  /** Select up to `limit` due owners, with a bounded pending prefix for each selected owner. */
   readonly pending: (
     nowMillis: number,
     limit: number,
-  ) => Effect.Effect<ReadonlyArray<LifecyclePublication>, LifecyclePublicationError>;
+  ) => Effect.Effect<ReadonlyArray<LifecyclePublicationBatch>, LifecyclePublicationError>;
+  /** Atomically acknowledge the exact batch, retaining its identities and fingerprints. */
   readonly acknowledge: (
-    publication: LifecyclePublication,
-  ) => Effect.Effect<void, LifecyclePublicationError>;
-  readonly defer: (
-    publication: LifecyclePublication,
-    untilMillis: number,
+    publications: LifecyclePublicationBatch,
   ) => Effect.Effect<void, LifecyclePublicationError>;
+  /**
+   * Claim a due owner batch and persist its retry before dispatch. Eight automatic attempts;
+   * 1s exponential backoff capped at 60s, after the dispatch timeout. The final attempt parks
+   * the payload before dispatch so process loss cannot renew its budget. False means not due.
+   */
+  readonly claim: (
+    publications: LifecyclePublicationBatch,
+    nowMillis: number,
+    timeoutMillis: number,
+  ) => Effect.Effect<boolean, LifecyclePublicationError>;
   readonly pendingDeadline: Effect.Effect<Option.Option<number>, LifecyclePublicationError>;
-  /** Earliest retained owner obligation, including deferred facts that still gate execution. */
-  readonly pendingDeadlineFor: (
+  /** Explicit operator retry after repairing a parked owner's destination. */
+  readonly retryParked: (
     ownerThreadId: ThreadId,
-  ) => Effect.Effect<Option.Option<number>, LifecyclePublicationError>;
+    nowMillis: number,
+  ) => Effect.Effect<void, LifecyclePublicationError>;
 }
 
 /**
- * Publish an idempotent domain command, including its authorization, exact receipt and delivery
- * intent. Return only after that command commits. Treat private input/update/result fields as
- * private; the application must select its declared public fields. Revocation/deletion is an
- * acknowledged domain decision, not an indefinitely retryable infrastructure failure.
+ * Publish one owner's ordered batch in one idempotent host transaction, including authorization,
+ * records, receipts and delivery intents. Larger backlogs continue in later batches.
+ * Return only after the entire batch commits. Retries
+ * may include already committed identities plus later facts; deduplicate each fact's `id`.
+ * Private input/update/result fields remain private; select declared public fields explicitly.
+ * Revocation/deletion is an acknowledged domain decision, not an infrastructure retry.
  */
 export class LifecyclePublicationHandler extends Context.Service<
   LifecyclePublicationHandler,
   {
     readonly publish: (
-      publication: LifecyclePublication,
+      publications: LifecyclePublicationBatch,
     ) => Effect.Effect<void, LifecyclePublicationError, Scope.Scope>;
   }
 >()("@effect-agent/thread/LifecyclePublicationHandler") {}
@@ -218,7 +251,8 @@ const withSource = Effect.fn("LifecyclePublication.withSource")(
 
 /**
  * One finite wave for the host's existing maintenance coordinator. Persist the next deadline
- * before dispatch, so interruption and process loss retain the same fact without another timer.
+ * before dispatch, so interruption and process loss retain the same facts without another timer.
+ * Finish independent owner batches before surfacing failures; interruption stops the wave.
  * Acknowledgements are exact and idempotent. No producer or external Tool is re-executed here.
  */
 export const drainLifecyclePublications = Effect.fn("LifecyclePublication.drain")(function* (
@@ -229,16 +263,36 @@ export const drainLifecyclePublications = Effect.fn("LifecyclePublication.drain"
   const handler = yield* LifecyclePublicationHandler;
   const pending = yield* storage.pending(yield* Clock.currentTimeMillis, limit);
 
-  for (const publication of pending) {
-    yield* storage.defer(publication, (yield* Clock.currentTimeMillis) + timeoutMillis);
-    yield* Effect.scoped(withSource(publication).pipe(Effect.flatMap(handler.publish))).pipe(
-      Effect.timeoutOrElse({
-        duration: timeoutMillis,
-        orElse: () => LifecyclePublicationError.make({ reason: "unavailable" }),
-      }),
+  let failures: Cause.Cause<LifecyclePublicationError> = Cause.empty;
+
+  for (const batch of pending) {
+    yield* Effect.gen(function* () {
+      if (!(yield* storage.claim(batch, yield* Clock.currentTimeMillis, timeoutMillis))) return;
+      yield* Effect.scoped(
+        Effect.gen(function* () {
+          const publications = yield* Effect.forEach(batch, withSource);
+
+          yield* handler.publish(publications);
+        }),
+      ).pipe(
+        Effect.timeoutOrElse({
+          duration: timeoutMillis,
+          orElse: () => LifecyclePublicationError.make({ reason: "unavailable" }),
+        }),
+      );
+      yield* storage.acknowledge(batch);
+    }).pipe(
+      Effect.catchCauseIf(
+        (cause) => !Cause.hasInterrupts(cause),
+        (cause) => {
+          failures = Cause.combine(failures, cause);
+
+          return Effect.void;
+        },
+      ),
     );
-    yield* storage.acknowledge(publication);
   }
+  if (failures.reasons.length > 0) return yield* Effect.failCause(failures);
 
   return pending.length;
 });

package/src/durable/WorkerHost.ts

@@ -2,10 +2,11 @@ import { Context, Effect, Option, Schema } from "effect";
 
 import type * as Agent from "../core/Agent.ts";
 import type { AgentPolicy } from "../core/AgentPolicy.ts";
-import type { SubmissionId, ThreadId } from "../core/Identifiers.ts";
+import type { AgentId, SubmissionId, ThreadId } from "../core/Identifiers.ts";
 import type { SubagentBudgetReservation } from "../core/SubagentContract.ts";
 import { WorkerError, type WorkerRef, type WorkerSource } from "../core/Worker.ts";
 import type {
+  CanonicalSequence,
   DefinitionDigests,
   Digest,
   PersistedJson,
@@ -14,6 +15,24 @@ import type {
 } from "./Records.ts";
 import type { Principal, SubmissionSnapshot } from "./SubmissionLedger.ts";
 
+/**
+ * One successful start authorization at an exact source tail. This host-created hint is
+ * available only during that start's preparation and synchronous admission; it is never
+ * persisted or transported. Hooks may reuse their admission decision, but must retain
+ * current access checks at commit boundaries. A retry or changed tail gets no old grant.
+ */
+export interface WorkerStartAdmission {
+  readonly sourceThreadId: ThreadId;
+  readonly sourceSubmissionId?: SubmissionId;
+  readonly requestedPrincipal: Principal;
+  readonly principal: Principal;
+  readonly targetAgentId: AgentId;
+  readonly continuationOf?: WorkerContinuation;
+  readonly access: "send";
+  readonly tailSequence: CanonicalSequence;
+  readonly tailDigest: Digest;
+}
+
 /** Host-verified source input; absence never means the latest input in the Thread. */
 export interface WorkerPolicySource {
   readonly threadId: ThreadId;
@@ -30,6 +49,7 @@ export type WorkerPolicyTarget = {
 } & (
   | {
       readonly _tag: "InitialInput";
+      readonly admission?: WorkerStartAdmission;
       readonly continuationOf?: WorkerContinuation;
       readonly sourceSubmission?: SubmissionSnapshot;
       readonly input: PersistedJson;
@@ -136,6 +156,7 @@ export const WorkerHostConfig = Context.Reference<WorkerHostLimits>(
  * identifiers alone never confer any of them.
  */
 export interface WorkerHostAuthorizationRequest {
+  readonly admission?: WorkerStartAdmission;
   readonly sourceThreadId: ThreadId;
   /** An explicitly selected owner input; authorize this locator as well as the Thread. */
   readonly sourceSubmissionId?: SubmissionId;
@@ -170,6 +191,7 @@ export const WorkerHostAuthorizer = Context.Reference<{
 export const WorkerBudgetAuthorizer = Context.Reference<{
   readonly authorize: (request: {
     readonly source: WorkerSource;
+    readonly admission?: WorkerStartAdmission;
     readonly principal: Principal;
     readonly worker: WorkerRef;
     readonly continuationOf?: WorkerContinuation;

package/src/durable/internal/worker-host.ts

@@ -128,6 +128,7 @@ import {
   WorkerHostConfig,
   WorkerPolicyResolver,
   type WorkerPolicyTarget,
+  type WorkerStartAdmission,
 } from "../WorkerHost.ts";
 import {
   definitionDigestsEqual,
@@ -263,6 +264,20 @@ export class WorkerInputControl extends Context.Service<
   }
 >()("@effect-agent/thread/internal/WorkerInputControl") {}
 
+const CurrentStartPreparation = Context.Reference<WorkerStartAdmission | undefined>(
+  "@effect-agent/thread/internal/CurrentStartPreparation",
+  { defaultValue: () => undefined },
+);
+
+const CurrentStartDelivery = Context.Reference<
+  | {
+      readonly authorization: WorkerStartAdmission;
+      readonly admission: WorkerAdmission;
+      readonly inputDigest: Digest;
+    }
+  | undefined
+>("@effect-agent/thread/internal/CurrentStartDelivery", { defaultValue: () => undefined });
+
 /** No in-memory ownership: every mutation is reserved in source canonical history with CAS. */
 export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
   options: WorkerRuntimeOptions,
@@ -290,9 +305,40 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
   const withCrypto = <A, E>(effect: Effect.Effect<A, E, Crypto.Crypto>) =>
     Effect.provideService(effect, Crypto.Crypto, deps.crypto);
 
+  const currentStart = Effect.fnUntraced(function* (
+    admission: WorkerStartAdmission | undefined,
+    sourceThreadId: ThreadId,
+    sourceSubmissionId: SubmissionId | undefined,
+    principal: Principal,
+    targetAgentId: AgentId,
+    continuationOf: WorkerContinuation | undefined,
+  ) {
+    if (
+      admission === undefined ||
+      admission.sourceThreadId !== sourceThreadId ||
+      admission.sourceSubmissionId !== sourceSubmissionId ||
+      (admission.principal !== principal && admission.requestedPrincipal !== principal) ||
+      admission.targetAgentId !== targetAgentId ||
+      !Schema.toEquivalence(Schema.UndefinedOr(WorkerContinuation))(
+        admission.continuationOf,
+        continuationOf,
+      )
+    )
+      return undefined;
+
+    const tail = yield* deps.store
+      .inspectTail(ThreadTailRequest.make({ threadId: sourceThreadId }))
+      .pipe(Effect.mapError(storageFailure("start")));
+
+    return tail.tailSequence === admission.tailSequence && tail.tailDigest === admission.tailDigest
+      ? admission
+      : undefined;
+  });
+
   const authorizeBudget = Effect.fn("WorkerHost.authorizeBudget")(function* (
     origin: WorkerOrigin,
     principal: Principal,
+    admission?: WorkerStartAdmission,
   ) {
     yield* verifyContinuation(origin);
     if (origin.budgetScope !== "worker-run") return;
@@ -300,6 +346,7 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
     if (origin.depth !== 1) return yield* failure("start", "denied");
 
     yield* deps.budgetAuthorizer.authorize({
+      ...(admission === undefined ? {} : { admission }),
       source: origin.source,
       principal,
       worker: origin.worker,
@@ -1049,6 +1096,7 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
     input: PersistedJson,
     principal: Principal,
     retainedFrameworkInput: boolean,
+    authorization?: WorkerStartAdmission,
   ): Effect.fn.Return<void, WorkerError> {
     const origin = admission.origin;
 
@@ -1133,6 +1181,11 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
           ? {
               ...targetRequest,
               _tag: "InitialInput",
+              ...(authorization === undefined ||
+              authorization.tailSequence !== current.tailSequence ||
+              authorization.tailDigest !== current.tailDigest
+                ? {}
+                : { admission: authorization }),
               ...(origin.continuationOf === undefined
                 ? {}
                 : { continuationOf: origin.continuationOf }),
@@ -1351,7 +1404,24 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
     }
 
     yield* verifyContinuation(admission.origin);
+    const delivery = yield* CurrentStartDelivery;
+
+    const authorization =
+      delivery !== undefined &&
+      sameAdmission(delivery.admission, admission) &&
+      delivery.inputDigest === inputDigest
+        ? yield* currentStart(
+            delivery.authorization,
+            admission.origin.source.threadId,
+            admission.sourceSubmissionId,
+            options.principal,
+            agentId,
+            admission.origin.continuationOf,
+          )
+        : undefined;
+
     yield* deps.authorizer.authorize({
+      ...(authorization === undefined ? {} : { admission: authorization }),
       sourceThreadId: admission.origin.source.threadId,
       ...(admission.sourceSubmissionId === undefined
         ? {}
@@ -1364,7 +1434,7 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
         ? {}
         : { continuationOf: admission.origin.continuationOf }),
     });
-    yield* authorizeBudget(admission.origin, options.principal);
+    yield* authorizeBudget(admission.origin, options.principal, authorization);
     if (
       admission.origin.worker.threadId !== options.threadId ||
       admission.origin.worker.targetAgentId !== agentId ||
@@ -1378,7 +1448,14 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
 
     if (retainedFrameworkInput && admission.reportKind !== "update")
       yield* validateCompletion(options.messageAdmission, options, agentId, inputDigest);
-    yield* reservation(admission, inputDigest, input, options.principal, retainedFrameworkInput);
+    yield* reservation(
+      admission,
+      inputDigest,
+      input,
+      options.principal,
+      retainedFrameworkInput,
+      authorization,
+    );
 
     return admission;
   });
@@ -1866,13 +1943,17 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
     principal: Principal,
     sourceSubmissionId?: SubmissionId,
   ): SubagentHost["Service"] => {
+    const requestedPrincipal = principal;
+
     const authorize = (
       operation: WorkerError["operation"],
       access: "context" | "read" | "send" | "control",
       worker?: WorkerRef,
       continuationOf?: WorkerContinuation,
+      admission?: WorkerStartAdmission,
     ) =>
       deps.authorizer.authorize({
+        ...(admission === undefined ? {} : { admission }),
         sourceThreadId: context.source.threadId,
         ...(sourceSubmissionId === undefined ? {} : { sourceSubmissionId }),
         principal,
@@ -1894,6 +1975,7 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
       target: Agent.AnyDefinition,
       encodedInput: unknown,
       continuationOf?: WorkerContinuation,
+      authorization?: WorkerStartAdmission,
     ) {
       const resolved = yield* binding(target, "start");
 
@@ -1908,8 +1990,18 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
 
       const source = yield* sourceAuthority(context.source.threadId, sourceSubmissionId);
 
+      const admission = yield* currentStart(
+        authorization,
+        context.source.threadId,
+        sourceSubmissionId,
+        principal,
+        target.id,
+        continuationOf,
+      );
+
       const policy = yield* resolveTargetPolicy({
         _tag: "InitialInput",
+        ...(admission === undefined ? {} : { admission }),
         ...(continuationOf === undefined ? {} : { continuationOf }),
         definition: resolved.definition,
         definitions: resolved.digests,
@@ -2064,6 +2156,7 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
       encodedParameters: unknown,
       principal: Principal,
       operation: "start" | "followUp",
+      authorization?: WorkerStartAdmission,
     ) {
       if (Option.isNone(deps.deliveries)) return yield* failure(operation, "unavailable");
       const deliveries = deps.deliveries.value;
@@ -2146,17 +2239,25 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
         Effect.provideService(MessageDeliveryStore, deliveries),
         Effect.provideService(PreparedInputAdmission, {
           submit: (prepared) =>
-            deps
-              .submit(prepared)
-              .pipe(
-                Effect.mapError((error) =>
-                  error._tag === "AdmissionConflict" || error._tag === "AgentInputError"
-                    ? ScheduledInputRefused.make({ code: error._tag, cause: error })
-                    : error._tag === "AdmissionPolicyError" && error.reason === "refused"
-                      ? ScheduledInputRefused.make({ code: error.code, cause: error })
-                      : ScheduledInputRetryable.make({ reason: "storage", cause: error }),
-                ),
+            deps.submit(prepared).pipe(
+              Effect.provideService(
+                CurrentStartDelivery,
+                authorization !== undefined &&
+                  prepared.workerAdmission !== undefined &&
+                  prepared.admissionKey === messageId &&
+                  prepared.inputDigest === inputDigest &&
+                  sameOrigin(prepared.workerAdmission.origin, origin)
+                  ? { authorization, admission: prepared.workerAdmission, inputDigest }
+                  : undefined,
               ),
+              Effect.mapError((error) =>
+                error._tag === "AdmissionConflict" || error._tag === "AgentInputError"
+                  ? ScheduledInputRefused.make({ code: error._tag, cause: error })
+                  : error._tag === "AdmissionPolicyError" && error.reason === "refused"
+                    ? ScheduledInputRefused.make({ code: error.code, cause: error })
+                    : ScheduledInputRetryable.make({ reason: "storage", cause: error }),
+              ),
+            ),
           submissionStatus: (receipt) =>
             deps
               .status(receipt)
@@ -2393,10 +2494,23 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
                 request.continuationOf.delegationId,
               );
 
-        yield* authorize("start", "send", undefined, previous?.evidence);
+        const admission = yield* currentStart(
+          yield* CurrentStartPreparation,
+          context.source.threadId,
+          sourceSubmissionId,
+          principal,
+          request.target.id,
+          previous?.evidence,
+        );
+
+        yield* authorize("start", "send", undefined, previous?.evidence, admission);
 
-        return (yield* preparedTarget(request.target, request.encodedInput, previous?.evidence))
-          .policy;
+        return (yield* preparedTarget(
+          request.target,
+          request.encodedInput,
+          previous?.evidence,
+          admission,
+        )).policy;
       }),
       start: Effect.fn("WorkerHost.start")(function* <E = never, R = never>(
         command: StartWorkerRequest | DeferredStartWorkerRequest<E, R>,
@@ -2411,8 +2525,26 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
                 command.delegationId,
               );
 
+        // Bind the decision to the tail before authorization. If it changes while the
+        // authorizer reads current state, currentStart rejects reuse of this decision.
+        const tail = yield* deps.store
+          .inspectTail(ThreadTailRequest.make({ threadId: context.source.threadId }))
+          .pipe(Effect.mapError(storageFailure("start")));
+
         const principal = yield* authorize("start", "send", undefined, previous?.evidence);
 
+        const startAdmission: WorkerStartAdmission = {
+          sourceThreadId: context.source.threadId,
+          ...(sourceSubmissionId === undefined ? {} : { sourceSubmissionId }),
+          requestedPrincipal,
+          principal,
+          targetAgentId: command.target.id,
+          ...(previous === undefined ? {} : { continuationOf: previous.evidence }),
+          access: "send",
+          tailSequence: tail.tailSequence,
+          tailDigest: tail.tailDigest,
+        };
+
         if (context.depth !== 0 && sourceSubmissionId === undefined)
           return yield* failure("start", "denied");
 
@@ -2507,7 +2639,14 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
 
         return yield* Effect.gen(function* () {
           const request =
-            "prepare" in command ? { ...(yield* command.prepare), ...command } : command;
+            "prepare" in command
+              ? {
+                  ...(yield* command.prepare.pipe(
+                    Effect.provideService(CurrentStartPreparation, startAdmission),
+                  )),
+                  ...command,
+                }
+              : command;
 
           const currentPrevious =
             command.continuationOf === undefined
@@ -2535,6 +2674,7 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
             request.target,
             request.encodedInput,
             currentPrevious?.evidence,
+            startAdmission,
           );
 
           const resolved = prepared.resolved;
@@ -2596,7 +2736,18 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
             "start",
           );
 
-          yield* authorizeBudget(origin, principal);
+          yield* authorizeBudget(
+            origin,
+            principal,
+            yield* currentStart(
+              startAdmission,
+              context.source.threadId,
+              sourceSubmissionId,
+              principal,
+              request.target.id,
+              currentPrevious?.evidence,
+            ),
+          );
 
           const targetPolicy = Option.isSome(prepared.policy)
             ? prepared.policy.value
@@ -2623,6 +2774,7 @@ export const makeWorkerRuntime = Effect.fn("WorkerHost.make")(function* (
             request.encodedParameters,
             principal,
             "start",
+            startAdmission,
           );
 
           return { worker: origin.worker, delivery };

package/src/engine/RunOptions.ts

@@ -210,6 +210,14 @@ export interface RunContextRequest {
    * identities rather than infer durable state from this Prompt alone.
    */
   readonly source: Prompt.Prompt;
+  /**
+   * Durable coordinator's exact prior-Run prefix length in `source`. A preparation hook
+   * can replace `source.content.slice(0, priorRunPrefixLength)` with application context
+   * while preserving this Run's instructions, input, updates and recovered Tool exchanges.
+   * This is a model-only transformation: canonical records and retry receipts are retained.
+   * Absent for non-durable Runs; never infer this boundary from message roles or text.
+   */
+  readonly priorRunPrefixLength?: number | undefined;
   /**
    * The exact model-visible final-output contract the engine appends to the
    * prepared prompt after this hook returns (RUN-028), or