@yielded/agent-storage-cloudflare 0.1.0-beta.175 → 0.1.0-beta.177

package/src/DoSubmissionLedger.ts

@@ -110,6 +110,8 @@ import {
   UnknownResolutionKind,
   unknownResolutionKind,
   unknownResolutionMatchesResult,
+  submissionInputRecordId,
+  submissionInputBatchId,
   submissionAbortRecordId,
   submissionSettlementRecordId,
   submissionSettlementBatchId,
@@ -1146,9 +1148,7 @@ const makeServices = Effect.fnUntraced(function* () {
       Effect.gen(function* () {
         const keyRowKey = `${validated.threadId}/${validated.principal}/${validated.idempotencyKey}`;
 
-        const existingRows = yield* byKey(validated, operation);
-
-        const existing = yield* decodeSubmissionRows(operation, keyRowKey, existingRows);
+        const existing = yield* byKey(validated, operation);
 
         if (existing.length > 1) {
           return yield* corruptionFailure(
@@ -1530,13 +1530,7 @@ const makeServices = Effect.fnUntraced(function* () {
       return Option.some(yield* decodeSubmissionSnapshot(operation, row.value));
     }
 
-    const foundRows = yield* byKey(validated, operation);
-
-    const decoded = yield* decodeSubmissionRows(
-      operation,
-      `${validated.threadId}/${validated.principal}/${validated.idempotencyKey}`,
-      foundRows,
-    );
+    const decoded = yield* byKey(validated, operation);
 
     if (decoded.length > 1) {
       return yield* corruptionFailure(
@@ -1563,13 +1557,7 @@ const makeServices = Effect.fnUntraced(function* () {
         request,
       ).pipe(Effect.mapError(internalFailure(operation)));
 
-      const foundRows = yield* byKey(validated, operation);
-
-      const decoded = yield* decodeSubmissionRows(
-        operation,
-        `${validated.threadId}/${validated.principal}/${validated.idempotencyKey}`,
-        foundRows,
-      );
+      const decoded = yield* byKey(validated, operation);
 
       if (decoded.length > 1) {
         return yield* corruptionFailure(
@@ -1706,6 +1694,45 @@ const makeServices = Effect.fnUntraced(function* () {
     return claimed;
   });
 
+  const markInputAppliedInTransaction = Effect.fnUntraced(function* (
+    validated: MarkInputAppliedRequest,
+  ) {
+    const operation = "ledger mark input applied";
+
+    const submission = yield* requireSubmission(operation, validated.submissionId);
+
+    yield* requireOwner(operation, submission);
+    if (submission.input_applied_record_id !== null) {
+      if (
+        submission.input_applied_record_id === validated.recordId &&
+        submission.input_applied_sequence === validated.sequence
+      ) {
+        return;
+      }
+
+      return yield* corruptionFailure(
+        operation,
+        "effect_agent_submissions",
+        validated.submissionId,
+        "A different canonical input marker is already recorded for this Submission.",
+      );
+    }
+    yield* sql`
+    UPDATE effect_agent_submissions
+    SET
+      input_applied_record_id = ${validated.recordId},
+      input_applied_sequence = ${validated.sequence},
+      state = CASE
+        WHEN state IN ('admitted', 'ready') THEN 'input-applied'
+        ELSE state
+      END
+    WHERE submission_id = ${validated.submissionId}
+   RETURNING *, ${definitionJson(sql)} AS agent_digests_json`.pipe(
+      rows.submissions.write,
+      Effect.mapError(internalFailure(operation)),
+    );
+  });
+
   const markInputApplied: SubmissionLedger["Service"]["markInputApplied"] = Effect.fnUntraced(
     function* (request: MarkInputAppliedRequest) {
       const operation = "ledger mark input applied";
@@ -1715,44 +1742,111 @@ const makeServices = Effect.fnUntraced(function* () {
       );
 
       yield* hitFailpoint("ledger:mark-input-applied:before", operation);
-      yield* inWriteTransaction(
-        operation,
-        Effect.gen(function* () {
-          const submission = yield* requireSubmission(operation, validated.submissionId);
+      yield* inWriteTransaction(operation, markInputAppliedInTransaction(validated));
+      yield* hitFailpoint("ledger:mark-input-applied:after", operation);
+    },
+  );
 
-          yield* requireOwner(operation, submission);
-          if (submission.input_applied_record_id !== null) {
-            if (
-              submission.input_applied_record_id === validated.recordId &&
-              submission.input_applied_sequence === validated.sequence
-            ) {
-              return;
-            }
+  const appendInput: NonNullable<SubmissionLedger["Service"]["appendInput"]> = Effect.fnUntraced(
+    function* (submissionId, request) {
+      const operation = "append original input and context";
 
-            return yield* corruptionFailure(
-              operation,
-              "effect_agent_submissions",
-              validated.submissionId,
-              "A different canonical input marker is already recorded for this Submission.",
-            );
-          }
-          yield* sql`
-          UPDATE effect_agent_submissions
-          SET
-            input_applied_record_id = ${validated.recordId},
-            input_applied_sequence = ${validated.sequence},
-            state = CASE
-              WHEN state IN ('admitted', 'ready') THEN 'input-applied'
-              ELSE state
-            END
-          WHERE submission_id = ${validated.submissionId}
-         RETURNING *, ${definitionJson(sql)} AS agent_digests_json`.pipe(
-            rows.submissions.write,
-            Effect.mapError(internalFailure(operation)),
-          );
-        }),
+      const prepared = yield* prepareCanonicalAppend(request).pipe(
+        Effect.provideService(Crypto.Crypto, crypto),
+      );
+
+      const record = prepared.request.batch.records[0];
+
+      const context = prepared.request.executions
+        ?.flatMap((update) => update.facts ?? [])
+        .find(({ payload }) => payload._tag === "RunContextRecorded");
+
+      if (
+        prepared.request.executions?.length !== 1 ||
+        prepared.request.executions[0]?.previousRevision !== 0 ||
+        prepared.request.batch.records.length !== 1 ||
+        record?.payload._tag !== "UserInputRecorded" ||
+        record.payload.kind !== "user" ||
+        record.payload.submissionId !== submissionId ||
+        record.payload.runId !== runIdForSubmission(submissionId) ||
+        record.recordId !== submissionInputRecordId(submissionId) ||
+        prepared.request.batch.batchId !== submissionInputBatchId(submissionId) ||
+        (context?.payload._tag === "RunContextRecorded" &&
+          context.payload.historyThrough !== prepared.request.expectedTailSequence)
+      )
+        return yield* ThreadStoreError.make({
+          operation,
+          message: "Original input publication has a different context boundary",
+        });
+
+      const mapError = (cause: { readonly message: string }) =>
+        ThreadStoreError.make({ operation, message: cause.message, cause });
+
+      yield* journal.prepareAppend(prepared.raw).pipe(Effect.mapError(mapError));
+
+      return yield* owner.withProgress(
+        prepared.request.threadId,
+        owner.withMutation(
+          Effect.gen(function* () {
+            yield* hitFailpoint("append:before", operation);
+            yield* hitFailpoint("ledger:mark-input-applied:before", operation);
+
+            const result = yield* journal
+              .withWriteTransaction(operation)(
+                Effect.gen(function* () {
+                  const submission = yield* requireSubmission(operation, submissionId);
+
+                  yield* requireOwner(operation, submission);
+                  if (submission.thread_id !== prepared.request.threadId)
+                    return yield* LedgerError.make({
+                      operation,
+                      message: "Original input belongs to another Thread",
+                    });
+                  const result = yield* journal.appendPrepared(prepared.raw);
+
+                  yield* markInputAppliedInTransaction(
+                    MarkInputAppliedRequest.make({
+                      submissionId,
+                      recordId: record.recordId,
+                      sequence: result.firstSequence,
+                    }),
+                  );
+
+                  return result;
+                }),
+              )
+              .pipe(
+                Effect.mapError((error) =>
+                  error._tag === "DoAppendConflict"
+                    ? AppendConflict.make({
+                        threadId: prepared.request.threadId,
+                        batchId: prepared.request.batch.batchId,
+                        reason: error.reason,
+                        ...(error.actualTailSequence === undefined
+                          ? {}
+                          : { actualTailSequence: error.actualTailSequence }),
+                      })
+                    : error._tag === "OwnershipLost"
+                      ? error
+                      : mapError(error),
+                ),
+              );
+
+            yield* hitFailpoint("ledger:mark-input-applied:after", operation);
+            yield* hitFailpoint("append:after", operation);
+
+            return result;
+          }).pipe(
+            Effect.mapError((error) =>
+              error._tag === "OwnershipLost" ||
+              error._tag === "ThreadStoreError" ||
+              error._tag === "AppendConflict"
+                ? error
+                : mapError(error),
+            ),
+          ),
+        ),
       );
-      yield* hitFailpoint("ledger:mark-input-applied:after", operation);
     },
   );
 
@@ -3702,19 +3796,16 @@ const makeServices = Effect.fnUntraced(function* () {
             .filter((row) => row.joined_host_submission_id === validated.submissionId)
             .sort((a, b) => a.queue_sequence - b.queue_sequence);
 
-          const joinSubmissions = yield* decodeSubmissionRows(
-            operation,
-            validated.submissionId,
-            joinRows,
-          );
+          const joins: Array<JoinSnapshot> = [];
 
-          const joins = yield* Effect.forEach(joinSubmissions, (row) =>
-            decodeJoinSnapshot({
-              submissionId: row.submission_id,
-              state: row.state,
-              hostSubmissionId: validated.submissionId,
-            }).pipe(Effect.mapError(internalFailure(operation))),
-          );
+          for (const row of joinRows)
+            joins.push(
+              yield* decodeJoinSnapshot({
+                submissionId: row.submission_id,
+                state: row.state,
+                hostSubmissionId: validated.submissionId,
+              }).pipe(Effect.mapError(internalFailure(operation))),
+            );
 
           let hostSubmissionId: RecoverySnapshot["hostSubmissionId"];
 
@@ -3755,15 +3846,17 @@ const makeServices = Effect.fnUntraced(function* () {
 
           const decisionRows = yield* readApprovalDecisions(operation, validated.submissionId);
 
-          const approvalDecisions = yield* Effect.forEach(decisionRows, (row) =>
-            approvalIntentFromRow(operation, row),
-          );
+          const approvalDecisions: Array<ApprovalDecisionIntent> = [];
+
+          for (const row of decisionRows)
+            approvalDecisions.push(yield* approvalIntentFromRow(operation, row));
 
           const resolutionRows = yield* readUnknownResolutions(operation, validated.submissionId);
 
-          const unknownResolutions = yield* Effect.forEach(resolutionRows, (row) =>
-            unknownResolutionIntentFromRow(operation, row),
-          );
+          const unknownResolutions: Array<UnknownResolutionIntent> = [];
+
+          for (const row of resolutionRows)
+            unknownResolutions.push(yield* unknownResolutionIntentFromRow(operation, row));
 
           // Parent-side subagent view: this Submission's child budget reservations in parent
           // Tool Call order, plus each attached child's current lane state (a disposable
@@ -3778,21 +3871,16 @@ const makeServices = Effect.fnUntraced(function* () {
             .filter((row) => row.parent_submission_id === validated.submissionId)
             .sort((a, b) => a.parent_tool_call_id.localeCompare(b.parent_tool_call_id));
 
-          const decodedChildReservations = yield* decodeChildReservationRows(
-            operation,
-            validated.submissionId,
-            childReservationRows,
-          );
+          const childReservations: Array<ChildBudgetReservationSnapshot> = [];
 
-          const childReservations = yield* Effect.forEach(decodedChildReservations, (row) =>
-            childReservationSnapshotFromRow(operation, row),
-          );
+          for (const row of childReservationRows)
+            childReservations.push(yield* childReservationSnapshotFromRow(operation, row));
 
           const markers = yield* readChildSettlementMarkers(operation, validated.submissionId);
           const markersByChild = new Map(markers.map((row) => [row.child_submission_id, row]));
           const childAttachments: Array<ChildAttachmentSnapshot> = [];
 
-          for (const row of decodedChildReservations) {
+          for (const row of childReservationRows) {
             if (row.child_submission_id === null) continue;
             const child = yield* readSubmission(operation, row.child_submission_id);
 
@@ -3863,6 +3951,7 @@ const makeServices = Effect.fnUntraced(function* () {
       resolveAdmission,
       selectHead,
       markInputApplied,
+      appendInput,
       finalizeSettlement: (request) => owner.withMutation(finalizeSettlement(request)),
       requestAbort,
       stopWorker,

package/src/DoThreadStore.ts

@@ -633,6 +633,13 @@ export const threadStoreLayer: Layer.Layer<
   DoStorageConfig | DoStorageFailpoint | SqlClientService.SqlClient | Crypto.Crypto
 > = ThreadReader.layer().pipe(Layer.provideMerge(Layer.effectContext(makeServices())));
 
+/**
+ * Share one initialized journal between local storage ports in one Object build. Provide
+ * this beneath both ports and above their common SQL, owner, configuration, failpoint,
+ * progress and lifecycle providers. Standalone ports initialize their own journal.
+ */
+export { initializedDoJournalLayer as journalLayer } from "./internal/do-journal.ts";
+
 /**
  * Validated Durable Object storage configuration Layer with the documented defaults applied.
  * Shared by the ThreadStore and SubmissionLedger convenience layers so their defaults

package/src/PortRouting.ts

@@ -236,35 +236,36 @@ const LOCAL: RouteTarget = { _tag: "local" };
  * Identities beyond the ledger's 1,024-character row bound fail typed: the minting side
  * refused them at admission, so they cannot name any stored row anywhere.
  */
-const routableSubmissionTarget = (
-  ownsThread: RoutedPortOptions["ownsThread"],
-): ((operation: string, submissionId: string) => Effect.Effect<RouteTarget, LedgerError>) =>
-  Effect.fnUntraced(function* (
-    operation: string,
-    submissionId: string,
-  ): Effect.fn.Return<RouteTarget, LedgerError> {
-    if (submissionId.length > MAX_ROUTABLE_SUBMISSION_ID_LENGTH) {
-      return yield* LedgerError.make({
-        operation,
-        message:
-          `A Submission identity of ${submissionId.length} characters exceeds the ` +
-          `${MAX_ROUTABLE_SUBMISSION_ID_LENGTH}-character routable identity bound; admission ` +
-          "refuses such identities, so it cannot name any stored row.",
-      });
-    }
-    const separator = submissionId.indexOf(":");
-
-    if (separator === -1) return LOCAL;
-    if (!UUID_HEAD_PATTERN.test(submissionId.slice(0, separator))) return LOCAL;
-    const tail = submissionId.slice(separator + 1);
-
-    return yield* decodeThreadId(tail).pipe(
-      Effect.map((threadId): RouteTarget =>
-        ownsThread(threadId) ? LOCAL : { _tag: "foreign", threadId },
-      ),
-      Effect.orElseSucceed(() => LOCAL),
-    );
-  });
+const routableSubmissionTarget =
+  (
+    ownsThread: RoutedPortOptions["ownsThread"],
+  ): ((operation: string, submissionId: string) => Effect.Effect<RouteTarget, LedgerError>) =>
+  (operation, submissionId) =>
+    Effect.suspend(() => {
+      if (submissionId.length > MAX_ROUTABLE_SUBMISSION_ID_LENGTH) {
+        return Effect.fail(
+          LedgerError.make({
+            operation,
+            message:
+              `A Submission identity of ${submissionId.length} characters exceeds the ` +
+              `${MAX_ROUTABLE_SUBMISSION_ID_LENGTH}-character routable identity bound; admission ` +
+              "refuses such identities, so it cannot name any stored row.",
+          }),
+        );
+      }
+      const separator = submissionId.indexOf(":");
+
+      if (separator === -1 || !UUID_HEAD_PATTERN.test(submissionId.slice(0, separator)))
+        return Effect.succeed(LOCAL);
+
+      return decodeThreadId(submissionId.slice(separator + 1)).pipe(
+        Effect.match({
+          onFailure: () => LOCAL,
+          onSuccess: (threadId): RouteTarget =>
+            ownsThread(threadId) ? LOCAL : { _tag: "foreign", threadId },
+        }),
+      );
+    });
 
 /**
  * Read a Submission only from this physical owner. Encoded foreign identities fail before
@@ -730,6 +731,23 @@ const makeRoutedLedgerServices = Effect.fnUntraced(function* (options: RoutedPor
         ? local.claimJoining(request)
         : Effect.fail(crossThreadLedgerError("ledger claim joining", request.threadId)),
 
+    ...(local.appendInput === undefined
+      ? {}
+      : {
+          appendInput: (
+            submissionId: SubmissionSnapshot["submissionId"],
+            request: Parameters<NonNullable<SubmissionLedger["Service"]["appendInput"]>>[1],
+          ) =>
+            options.ownsThread(request.threadId)
+              ? local.appendInput!(submissionId, request)
+              : Effect.fail(
+                  ThreadStoreError.make({
+                    operation: "append original input",
+                    message: "Original input publication is lane-local",
+                  }),
+                ),
+        }),
+
     markInputApplied: (request) =>
       submissionTarget("ledger mark input applied", request.submissionId).pipe(
         Effect.flatMap((target) =>

package/src/internal/do-journal.ts

@@ -33,7 +33,9 @@ import {
 import {
   Cause,
   Clock,
+  Context,
   Effect,
+  Layer,
   Option,
   Predicate,
   References,
@@ -46,6 +48,7 @@ import * as SqlClient from "effect/sql/SqlClient";
 import { classifySqliteError, SqlError } from "effect/sql/SqlError";
 import { CurrentTransformer, type Statement } from "effect/sql/Statement";
 
+import { DoStorageConfig } from "../DoStorageConfig.ts";
 import {
   type DoStorageFailpointError,
   DoAppendConflict,
@@ -55,7 +58,7 @@ import {
   DoValueBoundExceeded,
   type DoStorageFailpointLocation,
 } from "../DoStorageError.ts";
-import { isDefaultDoStorageFailpoint } from "../DoStorageFailpoint.ts";
+import { DoStorageFailpoint, isDefaultDoStorageFailpoint } from "../DoStorageFailpoint.ts";
 import { ensureDoStorageLayout } from "./migrations.ts";
 import { ownedState, ownedRows, type OwnedState } from "./owned-state.ts";
 import {
@@ -498,35 +501,37 @@ const makeJournal = Effect.fnUntraced(function* (
    * transaction. Expected outcomes may be captured only when every such refusal precedes
    * mutation; real faults still roll back. Successful preflight reads never authorize a write.
    */
-  const withWriteTransaction =
-    (operation: string, expected: (error: { readonly _tag: string }) => boolean = () => false) =>
-    <A, E extends { readonly _tag: string }>(
+  const withWriteTransaction = (
+    operation: string,
+    expected?: (error: { readonly _tag: string }) => boolean,
+  ) => {
+    const transaction = <A, E extends { readonly _tag: string }>(
       effect: Effect.Effect<A, E>,
     ): Effect.Effect<A, E | DoStorageError> =>
       Effect.withFiber((fiber) => {
-        const body = state.owner
-          .withMutation(owner.transaction(storageResult(effect, expected)))
-          .pipe(
-            Effect.provideService(SqlClient.SqlClient, sql),
-            Effect.mapError((error) =>
-              isSqlError(error) ? storageError(operation)(error) : error,
-            ),
-          );
+        const body = state.owner.withMutation(owner.transaction(effect)).pipe(
+          Effect.provideService(SqlClient.SqlClient, sql),
+          Effect.mapError((error) => (isSqlError(error) ? storageError(operation)(error) : error)),
+        );
 
         const traced =
           fiber.getRef(References.TracerEnabled) &&
           fiber.getRef(Tracer.Tracer) !== Tracer.nativeTracer;
 
-        return (
-          traced
-            ? body.pipe(
-                Effect.tapError(annotateStorageError),
-                Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
-              )
-            : body
-        ).pipe(Effect.flatMap(Effect.fromResult));
+        return traced
+          ? body.pipe(
+              Effect.tapError(annotateStorageError),
+              Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
+            )
+          : body;
       });
 
+    return <A, E extends { readonly _tag: string }>(effect: Effect.Effect<A, E>) =>
+      expected === undefined
+        ? transaction(effect)
+        : transaction(storageResult(effect, expected)).pipe(Effect.flatMap(Effect.fromResult));
+  };
+
   const materialize = Effect.fnUntraced(function* (
     threadId: string,
     createdAt: string,
@@ -690,7 +695,7 @@ const makeJournal = Effect.fnUntraced(function* (
     )
       return false;
 
-    const workPlan = work.prepareClosedPair(
+    const workPlan = yield* work.prepareClosedPair(
       request.threadId,
       first,
       last,
@@ -729,7 +734,6 @@ const makeJournal = Effect.fnUntraced(function* (
           // A zero-row predicate miss has made no change. A successful update
           // must never fall back: every later refusal rolls back this transaction.
           if (rows.length === 0) return undefined;
-          const closed = yield* workPlan.fold();
           const executionPlans = executions.prepareCommit(request);
 
           if (!batchInserted)
@@ -760,7 +764,8 @@ const makeJournal = Effect.fnUntraced(function* (
               sql`INSERT INTO effect_agent_record_runs ${sql.insert(group)}`,
               "index canonical Run membership",
             );
-          for (const pointer of closed.pointers) yield* query(pointer, "index canonical pointers");
+          for (const pointer of workPlan.pointers)
+            yield* query(pointer, "index canonical pointers");
 
           for (const plan of executionPlans) {
             const changed = yield* query(plan.statement, "commit active execution");
@@ -1568,4 +1573,27 @@ const makeJournal = Effect.fnUntraced(function* (
 
 export type DoJournal = Effect.Success<ReturnType<typeof makeJournal>>;
 
-export const initializeDoJournal = ensureCurrentStorage;
+const InitializedDoJournal = Context.Reference<DoJournal | undefined>(
+  "@effect-agent/storage-cloudflare/InitializedDoJournal",
+  { defaultValue: () => undefined },
+);
+
+/** Build once in the common owner, SQL, configuration, failpoint and lifecycle context. */
+export const initializedDoJournalLayer = Layer.effect(InitializedDoJournal)(
+  Effect.gen(function* () {
+    const sql = yield* SqlClient.SqlClient;
+    const config = yield* DoStorageConfig;
+    const failpoint = yield* DoStorageFailpoint;
+
+    return yield* ensureCurrentStorage(sql, failpoint.hit, config.maxStoredValueBytes);
+  }),
+);
+
+export const initializeDoJournal = (
+  sql: SqlClient.SqlClient,
+  failpoint: DoJournalFailpoint,
+  maxStoredValueBytes: number,
+) =>
+  Effect.filterOrElse(InitializedDoJournal, Predicate.isNotUndefined, () =>
+    ensureCurrentStorage(sql, failpoint, maxStoredValueBytes),
+  );

package/src/internal/owned-state.ts

@@ -156,7 +156,7 @@ export const ownedRows = <A, I>(
 
     const decode = (rows: unknown) =>
       decodeRows(rows).pipe(
-        Effect.mapError(() =>
+        Effect.mapErrorEager(() =>
           DoStorageCorruptionError.make({
             table,
             rowKey: "owned-state",
@@ -200,9 +200,8 @@ export const ownedRows = <A, I>(
 
           if (view !== undefined) return Effect.succeed(view.rows);
 
-          return query.pipe(
-            Effect.flatMap(decode),
-            Effect.map((rows) => {
+          return Effect.flatMap(query, (rows) =>
+            Effect.mapEager(decode(rows), (rows) => {
               retain(id, matches, rows, maxRows);
 
               return rows;