effect-agent 0.1.0-beta.85 → 0.1.0-beta.87

package/src/durable/Certification.ts

@@ -0,0 +1,210 @@
+import type { Crypto } from "effect";
+import { Cause, Effect, Exit, Option, Schema } from "effect";
+
+import { DurableRuntimeFailpointLocation } from "./DurableFailpoint.ts";
+import { inspectForeignDiagnostic, safeUnknownString } from "./internal/foreign-diagnostic.ts";
+import type { SubmissionLedger } from "./SubmissionLedger.ts";
+import { LedgerCapabilities } from "./SubmissionLedger.ts";
+import { submissionLedgerConformanceCases } from "./SubmissionLedgerConformance.ts";
+import type { ThreadStore } from "./ThreadStore.ts";
+import { threadStoreConformanceCases } from "./ThreadStoreConformance.ts";
+
+// ---------------------------------------------------------------------------
+// Certification report vocabulary (WP2, plan §1)
+// ---------------------------------------------------------------------------
+
+const BoundedCertificationDetail = Schema.String.check(Schema.isMaxLength(4_096));
+
+/**
+ * Adapter self-description carried by a certificate. `durability` is read from the candidate
+ * ledger's `capabilities` — the adapter's honest claim (persistence.md), never inferred by the
+ * certification runner.
+ */
+export class CertifiedAdapterIdentity extends Schema.Class<CertifiedAdapterIdentity>(
+  "@effect-agent/thread/CertifiedAdapterIdentity",
+)({
+  /** Workspace/package name of the adapter pair under certification. */
+  name: Schema.NonEmptyString.check(Schema.isMaxLength(256)),
+  version: Schema.optionalKey(Schema.String.check(Schema.isMaxLength(256))),
+  durability: LedgerCapabilities.fields.durability,
+}) {}
+
+/** Which shared suite one Tier-1 (or lever-driven Tier-3) case belongs to. */
+export const CertificationSuite = Schema.Literals([
+  "submission-ledger",
+  "thread-store",
+  "real-loss",
+]);
+
+export type CertificationSuite = typeof CertificationSuite.Type;
+
+/** One executed certification case: a conformance contract case or a Tier-3 lever row. */
+export class CertificationCaseResult extends Schema.Class<CertificationCaseResult>(
+  "@effect-agent/thread/CertificationCaseResult",
+)({
+  suite: CertificationSuite,
+  name: Schema.NonEmptyString.check(Schema.isMaxLength(512)),
+  status: Schema.Literals(["passed", "failed"]),
+  detail: Schema.optionalKey(BoundedCertificationDetail),
+}) {}
+
+/**
+ * The six Tier-2 scenario shapes: each reached coordinator fault boundary is armed once
+ * per shape. Locations outside a shape's path share its verified clean run.
+ */
+export const CertificationScenario = Schema.Literals([
+  "plain",
+  "uncertain-tool",
+  "durable-steps",
+  "approval",
+  "join",
+  "delegation",
+]);
+
+export type CertificationScenario = typeof CertificationScenario.Type;
+
+/**
+ * Outcome of one Tier-2 sweep cell:
+ *
+ * - `converged` — the armed fault fired once, recovery classified the state, and the re-drive
+ *   converged to verified invariants;
+ * - `not-triggered` — the location was not reached with that fault armed, or was absent from
+ *   the scenario's verified clean path (shared across such locations). Never fault coverage;
+ * - `failed` — anything else.
+ */
+export const CertificationSweepStatus = Schema.Literals(["converged", "not-triggered", "failed"]);
+export type CertificationSweepStatus = typeof CertificationSweepStatus.Type;
+
+/** One Tier-2 scenario/location result, including locations outside the verified path. */
+export class CertificationSweepResult extends Schema.Class<CertificationSweepResult>(
+  "@effect-agent/thread/CertificationSweepResult",
+)({
+  scenario: CertificationScenario,
+  location: DurableRuntimeFailpointLocation,
+  /** The armed fault actually fired (one-shot) somewhere in the cell's drive or re-drive. */
+  failpointFired: Schema.Boolean,
+  status: CertificationSweepStatus,
+  /**
+   * Every lane's digest chain recomputed from `EMPTY_TAIL_DIGEST` and matched the exported
+   * tail — fully discharged here because the runner captures per-batch producer identity at
+   * append time (the port-only admin `verify` reports this check `skipped` instead).
+   * Unreached locations may share the scenario's clean-run verification.
+   */
+  digestChainVerified: Schema.Boolean,
+  detail: Schema.optionalKey(BoundedCertificationDetail),
+}) {}
+
+/**
+ * How the real-loss tier was discharged. Tier 3 is never silently claimed: an adapter either
+ * exercised a real loss lever in this run (`exercised`), cites committed real-loss evidence
+ * (`recorded-evidence`), honestly records that nothing exercised it (`not-exercised`), or is a
+ * non-durable reference adapter with no real loss to exercise (`not-applicable`).
+ */
+export const CertificationTierThreeStatus = Schema.Literals([
+  "exercised",
+  "recorded-evidence",
+  "not-exercised",
+  "not-applicable",
+]);
+
+export type CertificationTierThreeStatus = typeof CertificationTierThreeStatus.Type;
+
+/** The Tier-3 (real loss lever) section of a certificate. */
+export class CertificationTierThreeReport extends Schema.Class<CertificationTierThreeReport>(
+  "@effect-agent/thread/CertificationTierThreeReport",
+)({
+  status: CertificationTierThreeStatus,
+  /** Cited real-loss suites. The runner does not execute or verify these references. */
+  evidence: Schema.Array(BoundedCertificationDetail),
+  /** Rows executed by a supplied crash lever in THIS run (`suite: "real-loss"`). */
+  cases: Schema.Array(CertificationCaseResult),
+  detail: Schema.optionalKey(BoundedCertificationDetail),
+}) {}
+
+/**
+ * The Schema-encoded certification report (plan §1): Tier 1 = the two shared port contract
+ * suites verbatim; Tier 2 = the coordinator failpoint convergence sweep; Tier 3 = the real
+ * loss lever record. `ok` is true exactly when every EXECUTED check passed — honest scope
+ * statements (`not-triggered`, `recorded-evidence`, `not-exercised`, `not-applicable`) never
+ * count as failures and never count as silent coverage. `fullyCertified` additionally requires
+ * a durable adapter and passing real-loss cases executed in this run. Citations alone do not
+ * certify recovery. A non-durable adapter can pass conformance but is never fully certified.
+ * The semantic assertions live in the runner tests.
+ */
+export class CertificationReport extends Schema.Class<CertificationReport>(
+  "@effect-agent/thread/CertificationReport",
+)({
+  format: Schema.Literal("effect-agent/certification@2"),
+  adapter: CertifiedAdapterIdentity,
+  generatedAt: Schema.DateTimeUtcFromString,
+  tier1: Schema.Array(CertificationCaseResult),
+  tier2: Schema.Array(CertificationSweepResult),
+  tier3: CertificationTierThreeReport,
+  ok: Schema.Boolean,
+  fullyCertified: Schema.Boolean,
+}) {}
+
+/** Bounded operator-readable rendering of one failure Cause for a certification detail. */
+const causeDetail = (cause: Cause.Cause<unknown>): string => {
+  const failure = Cause.findErrorOption(cause);
+
+  if (Option.isSome(failure)) {
+    const diagnostic = inspectForeignDiagnostic(failure.value);
+
+    if (diagnostic.tag !== undefined || diagnostic.message !== undefined) {
+      const tag = diagnostic.tag ?? "Error";
+
+      return `${tag}${diagnostic.message === undefined ? "" : `: ${diagnostic.message}`}`.slice(
+        0,
+        4_096,
+      );
+    }
+
+    return safeUnknownString(failure.value, "Unknown foreign failure").slice(0, 4_096);
+  }
+
+  return `defect: ${safeUnknownString(cause, "Unknown defect")}`.slice(0, 4_096);
+};
+
+const caseResult = <A, E>(
+  suite: CertificationSuite,
+  name: string,
+  exit: Exit.Exit<A, E>,
+): CertificationCaseResult =>
+  Exit.isSuccess(exit)
+    ? CertificationCaseResult.make({ suite, name, status: "passed" })
+    : CertificationCaseResult.make({
+        suite,
+        name,
+        status: "failed",
+        detail: causeDetail(exit.cause),
+      });
+
+/**
+ * Tier 1 — port contract (plan §1): run BOTH shared conformance case arrays verbatim against
+ * the provided candidate adapters and fold every case into a typed result. Case failures are
+ * captured per-case (never short-circuiting the sweep), so a certificate always reports the
+ * complete contract surface. Requires the candidate `SubmissionLedger` and `ThreadStore`
+ * plus `Crypto.Crypto`, and must run under a TestClock (the ledger cases drive lease expiry
+ * through virtual time).
+ */
+export const certifyPorts = Effect.fn("Thread.certifyPorts")(function* (): Effect.fn.Return<
+  ReadonlyArray<CertificationCaseResult>,
+  never,
+  SubmissionLedger | ThreadStore | Crypto.Crypto
+> {
+  const results: Array<CertificationCaseResult> = [];
+
+  for (const contractCase of submissionLedgerConformanceCases) {
+    const exit = yield* Effect.exit(contractCase.run);
+
+    results.push(caseResult("submission-ledger", contractCase.name, exit));
+  }
+  for (const contractCase of threadStoreConformanceCases) {
+    const exit = yield* Effect.exit(contractCase.run);
+
+    results.push(caseResult("thread-store", contractCase.name, exit));
+  }
+
+  return results;
+});

package/src/durable/CommittedActivity.ts

@@ -0,0 +1,296 @@
+import { Clock, Effect, Schema, Stream, type Crypto, type Scope } from "effect";
+
+import {
+  ActivityClaim,
+  ActivityClaimRequest,
+  ActivityProcessorKey,
+  ActivityProcessorStore,
+  PreparedActivity,
+  type ActivityStoreFailure,
+} from "./ActivityStore.ts";
+import { digestJson, type DigestError } from "./Digest.ts";
+import {
+  CanonicalRecordEnvelope,
+  CanonicalSequence,
+  PersistedJson,
+  RecordEnvelope,
+} from "./Records.ts";
+import {
+  ThreadRead,
+  ThreadStore,
+  ThreadTail,
+  ThreadTailRequest,
+  type ThreadNotMaterialized,
+  type ThreadStoreError,
+} from "./ThreadStore.ts";
+
+export class ActivityPassLimits extends Schema.Class<ActivityPassLimits>(
+  "@effect-agent/thread/ActivityPassLimits",
+)({
+  maxRecords: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 4_096 })),
+  pageSize: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 128 })),
+  timeoutMillis: Schema.Int.check(Schema.isBetween({ minimum: 1, maximum: 300_000 })),
+  leaseMillis: ActivityClaimRequest.fields.leaseMillis,
+}) {}
+
+export class ActivityProcessingError extends Schema.TaggedError<ActivityProcessingError>()(
+  "ActivityProcessingError",
+  {
+    reason: Schema.Literals(["invalid-input", "noncontiguous", "timeout"]),
+    message: Schema.String.check(Schema.isMaxLength(1_024)),
+  },
+) {}
+
+export class ActivityPassResult extends Schema.Class<ActivityPassResult>(
+  "@effect-agent/thread/ActivityPassResult",
+)({
+  key: ActivityProcessorKey,
+  capturedTail: CanonicalSequence,
+  throughSequence: CanonicalSequence,
+  processed: Schema.Natural,
+  pendingRecords: Schema.Natural,
+  startedAt: Schema.Finite,
+  finishedAt: Schema.Finite,
+}) {}
+
+export interface CommittedActivityProcessor<E = never, R = never, EApply = never, RApply = never> {
+  readonly key: ActivityProcessorKey;
+  /** Unique to this worker lifetime; it is not canonical Thread ownership. */
+  readonly owner: string;
+  readonly limits: ActivityPassLimits;
+  /** Select eligibility and encode an application Schema; null can represent no output. */
+  readonly extract: (record: CanonicalRecordEnvelope) => Effect.Effect<PersistedJson, E, R>;
+  /**
+   * Must durably reconcile workId before returning, using atomic idempotency and conditional
+   * writes. This may run again after any lost acknowledgement. A stale invocation can finish
+   * only the already pinned output; it must not overwrite a later correction or withdrawal.
+   * Retain reconciliation evidence for the lifetime of pending work, including any supported
+   * backup/restore window. Returning before durable application would permit skipped output.
+   */
+  readonly apply: (work: PreparedActivity) => Effect.Effect<void, EApply, RApply>;
+}
+
+const invalid = (message: string) =>
+  ActivityProcessingError.make({ reason: "invalid-input", message });
+
+const contiguous = () =>
+  ActivityProcessingError.make({
+    reason: "noncontiguous",
+    message: "Committed activity does not match its captured Thread prefix or pinned record",
+  });
+
+/** Stable work identity independent of worker claims, retry count, extraction, and clocks. */
+export const activityWorkId = Effect.fn("activityWorkId")(function* (
+  key: ActivityProcessorKey,
+  sequence: CanonicalSequence,
+) {
+  return yield* digestJson([
+    "effect-agent/activity@1",
+    key.processorId,
+    key.processorVersion,
+    key.threadId,
+    sequence,
+  ]);
+});
+
+/**
+ * Process one bounded committed prefix of one application-selected Thread. Nothing runs until
+ * invoked, and no daemon, global Thread directory, admission ledger, or engine checkpoint is
+ * involved. PersistentHistory makes a successful Run's batch visible together; durable Runs
+ * expose incremental committed records. The application decides what each record means.
+ *
+ * A durable pending output wins over a new extraction after restart. Apply finishes before
+ * progress advances, so separate output stores require durable idempotency. This does not
+ * promise exactly-once external side effects. Each callback owns a Scope; the finite pass has
+ * a timeout and a longer lease. Claim release has a separate 500ms deadline; failure leaves
+ * the bounded lease to expire.
+ */
+export const processCommittedActivity = Effect.fn("processCommittedActivity")(function* <
+  E = never,
+  R = never,
+  EApply = never,
+  RApply = never,
+>(
+  processor: CommittedActivityProcessor<E, R, EApply, RApply>,
+): Effect.fn.Return<
+  ActivityPassResult,
+  | E
+  | EApply
+  | ActivityStoreFailure
+  | ActivityProcessingError
+  | DigestError
+  | ThreadStoreError
+  | ThreadNotMaterialized,
+  ActivityProcessorStore | ThreadStore | Crypto.Crypto | Exclude<R | RApply, Scope.Scope>
+> {
+  const key = yield* Schema.decodeEffect(ActivityProcessorKey)(processor.key).pipe(
+    Effect.mapError(() => invalid("Invalid activity processor key")),
+  );
+
+  const limits = yield* Schema.decodeEffect(ActivityPassLimits)(processor.limits).pipe(
+    Effect.mapError(() => invalid("Invalid activity pass limits")),
+  );
+
+  if (limits.leaseMillis < limits.timeoutMillis + 1_000) {
+    return yield* invalid("The activity lease must exceed the pass timeout by at least 1000ms");
+  }
+
+  const request = yield* ActivityClaimRequest.makeEffect({
+    key,
+    owner: processor.owner,
+    leaseMillis: limits.leaseMillis,
+  }).pipe(Effect.mapError(() => invalid("Invalid activity worker identity")));
+
+  const progress = yield* ActivityProcessorStore;
+  const threads = yield* ThreadStore;
+
+  return yield* Effect.scoped(
+    Effect.gen(function* () {
+      const acquired = yield* Effect.acquireRelease(
+        progress.claim(request).pipe(Effect.interruptible),
+        (claim) =>
+          progress
+            .release(claim)
+            .pipe(Effect.interruptible, Effect.timeoutOption(500), Effect.ignore),
+      );
+
+      let claim = yield* Schema.decodeEffect(ActivityClaim)(acquired).pipe(
+        Effect.mapError(() => invalid("Malformed activity claim")),
+      );
+
+      if (
+        !Schema.toEquivalence(ActivityProcessorKey)(claim.key, key) ||
+        claim.owner !== request.owner
+      )
+        return yield* invalid("Activity claim identity does not match its request");
+      const startedAt = yield* Clock.currentTimeMillis;
+
+      const tail = yield* threads
+        .inspectTail(ThreadTailRequest.make({ threadId: key.threadId }))
+        .pipe(
+          Effect.flatMap((value) =>
+            Schema.decodeEffect(ThreadTail)(value).pipe(Effect.mapError(contiguous)),
+          ),
+        );
+
+      if (
+        tail.threadId !== key.threadId ||
+        tail.tailSequence < claim.throughSequence ||
+        (claim.pending !== null && tail.tailSequence < claim.pending.sequence)
+      ) {
+        return yield* contiguous();
+      }
+      const through = Math.min(tail.tailSequence, claim.throughSequence + limits.maxRecords);
+      let processed = 0;
+
+      while (claim.throughSequence < through) {
+        const limit = Math.min(limits.pageSize, through - claim.throughSequence);
+
+        const page = yield* threads
+          .read(
+            ThreadRead.make({
+              threadId: key.threadId,
+              afterSequence: claim.throughSequence,
+              limit,
+            }),
+          )
+          .pipe(Stream.take(limit + 1), Stream.runCollect);
+
+        if (page.length !== limit) return yield* contiguous();
+        for (const rawRecord of page) {
+          const record = yield* Schema.decodeEffect(Schema.toType(CanonicalRecordEnvelope))(
+            rawRecord,
+          ).pipe(Effect.mapError(contiguous));
+
+          if (record.threadId !== key.threadId || record.sequence !== claim.throughSequence + 1) {
+            return yield* contiguous();
+          }
+          const workId = yield* activityWorkId(key, record.sequence);
+
+          const encodedRecord = yield* Schema.encodeEffect(RecordEnvelope)(record.record).pipe(
+            Effect.mapError(contiguous),
+          );
+
+          const recordDigest = yield* digestJson(encodedRecord);
+          let work = claim.pending;
+
+          if (work === null) {
+            const output = yield* Effect.scoped(processor.extract(record)).pipe(
+              Effect.flatMap((value) =>
+                Schema.decodeEffect(PersistedJson)(value).pipe(
+                  Effect.mapError(() => invalid("Malformed activity output")),
+                ),
+              ),
+            );
+
+            work = yield* progress.prepare({
+              claim,
+              work: PreparedActivity.make({
+                version: 1,
+                key,
+                sequence: record.sequence,
+                workId,
+                recordId: record.record.recordId,
+                recordDigest,
+                output,
+              }),
+            });
+          }
+
+          const prepared = yield* Schema.decodeEffect(PreparedActivity)(work).pipe(
+            Effect.mapError(() => invalid("Malformed prepared activity")),
+          );
+
+          if (
+            !Schema.toEquivalence(ActivityProcessorKey)(prepared.key, key) ||
+            prepared.sequence !== record.sequence ||
+            prepared.workId !== workId ||
+            prepared.recordId !== record.record.recordId ||
+            prepared.recordDigest !== recordDigest
+          )
+            return yield* contiguous();
+          yield* Effect.scoped(processor.apply(prepared));
+
+          const next = yield* progress
+            .advance({ claim, workId })
+            .pipe(
+              Effect.flatMap((value) =>
+                Schema.decodeEffect(ActivityClaim)(value).pipe(
+                  Effect.mapError(() => invalid("Malformed activity progress")),
+                ),
+              ),
+            );
+
+          if (
+            !Schema.toEquivalence(ActivityProcessorKey)(next.key, key) ||
+            next.owner !== claim.owner ||
+            next.epoch !== claim.epoch ||
+            next.throughSequence !== record.sequence ||
+            next.pending !== null
+          )
+            return yield* invalid("Activity progress advanced outside the claimed record");
+          claim = next;
+          processed += 1;
+        }
+      }
+
+      return ActivityPassResult.make({
+        key,
+        capturedTail: tail.tailSequence,
+        throughSequence: claim.throughSequence,
+        processed,
+        pendingRecords: tail.tailSequence - claim.throughSequence,
+        startedAt,
+        finishedAt: yield* Clock.currentTimeMillis,
+      });
+    }),
+  ).pipe(
+    Effect.timeoutOrElse({
+      duration: limits.timeoutMillis,
+      orElse: () =>
+        Effect.fail(
+          ActivityProcessingError.make({ reason: "timeout", message: "Activity pass timed out" }),
+        ),
+    }),
+  );
+});

package/src/durable/Digest.ts

@@ -0,0 +1,93 @@
+import type { PlatformError } from "effect";
+import { Array, Crypto, Effect, Encoding, Schema } from "effect";
+
+import type { DefinitionDigestInput } from "./Records.ts";
+import { CanonicalBatch, DefinitionDigests, Digest } from "./Records.ts";
+
+export class DigestError extends Schema.TaggedError<DigestError>()("DigestError", {
+  message: Schema.String,
+  cause: Schema.optionalKey(Schema.Defect()),
+}) {}
+
+const utf8 = new TextEncoder();
+
+const canonicalJson = (value: Schema.Json): string => {
+  if (
+    value === null ||
+    typeof value === "boolean" ||
+    typeof value === "number" ||
+    typeof value === "string"
+  ) {
+    return JSON.stringify(value);
+  }
+  if (Array.isArray<Schema.Json>(value)) {
+    return `[${globalThis.Array.from(value, canonicalJson).join(",")}]`;
+  }
+
+  const entries = Object.entries(value).sort(([left], [right]) =>
+    left < right ? -1 : left > right ? 1 : 0,
+  );
+
+  return `{${entries
+    .map(([key, entry]) => `${JSON.stringify(key)}:${canonicalJson(entry)}`)
+    .join(",")}}`;
+};
+
+/** Digest a JSON value using a stable, locale-independent object-key ordering (UTF-16 code units, RFC 8785 style). */
+export const digestJson = Effect.fn("Thread.digestJson")(function* (
+  value: Schema.Json,
+): Effect.fn.Return<Digest, DigestError, Crypto.Crypto> {
+  const crypto = yield* Crypto.Crypto;
+
+  const bytes = utf8.encode(canonicalJson(value));
+
+  const digest = yield* crypto
+    .digest("SHA-256", bytes)
+    .pipe(
+      Effect.mapError((error: PlatformError.PlatformError) =>
+        DigestError.make({ message: `SHA-256 failed: ${error.message}`, cause: error }),
+      ),
+    );
+
+  return yield* Schema.decodeEffect(Digest)(Encoding.encodeHex(digest)).pipe(
+    Effect.mapError(() => DigestError.make({ message: "SHA-256 returned an invalid digest" })),
+  );
+});
+
+/** Digest a canonical batch together with the prior tail to form an append-only hash chain. */
+export const digestCanonicalBatch = Effect.fn("Thread.digestCanonicalBatch")(
+  (
+    previousTailDigest: Digest,
+    batch: CanonicalBatch,
+  ): Effect.Effect<Digest, DigestError, Crypto.Crypto> =>
+    Schema.encodeEffect(CanonicalBatch)(batch).pipe(
+      Effect.mapError(() => DigestError.make({ message: "Canonical batch encoding failed" })),
+      Effect.flatMap((encoded) =>
+        digestJson({
+          previousTailDigest,
+          batch: encoded,
+        }),
+      ),
+    ),
+);
+
+/** Digest one schema-encoded Agent, Model, or Toolkit definition. */
+export const digestDefinition = Effect.fn("Thread.digestDefinition")((definition: Schema.Json) =>
+  digestJson(definition),
+);
+
+/** Digest all replay-relevant definitions without hiding which authority changed. */
+export const digestDefinitions = Effect.fn("Thread.digestDefinitions")(
+  (
+    definitions: DefinitionDigestInput,
+  ): Effect.Effect<DefinitionDigests, DigestError, Crypto.Crypto> =>
+    Effect.all({
+      agent: digestDefinition(definitions.agent),
+      model: digestDefinition(definitions.model),
+      tools: digestDefinition(definitions.tools),
+    }).pipe(Effect.map((digests) => DefinitionDigests.make(digests))),
+);
+
+export const EMPTY_TAIL_DIGEST = Schema.decodeSync(Digest)(
+  "0000000000000000000000000000000000000000000000000000000000000000",
+);