@yielded/agent-storage-cloudflare 0.1.0-beta.174 → 0.1.0-beta.176

package/src/DoThreadStore.ts

@@ -18,7 +18,6 @@ import { ThreadImport } from "@yielded/agent/thread-import";
 import {
   type ThreadExportRequest,
   AppendConflict,
-  AppendResult,
   CheckpointRejected,
   ThreadCheckpoint,
   ThreadMaterialization,
@@ -104,6 +103,11 @@ const isDoCheckpointConflict = Schema.is(DoCheckpointConflict);
 const ThreadCheckpointJson = Schema.fromJsonString(ThreadCheckpoint);
 const decodeCanonicalRecord = decodeExportRecordJson;
 
+const decodeThreadMaterializationType = Schema.decodeEffect(Schema.toType(ThreadMaterialization));
+const decodeThreadReadRequestType = Schema.decodeEffect(Schema.toType(ThreadReadRequest));
+const decodeThreadTailRequestType = Schema.decodeEffect(Schema.toType(ThreadTailRequest));
+const decodeStorageConfig = Schema.decodeEffect(DoStorageConfigValue);
+
 const storeError = (
   operation: string,
   error: { readonly message: string; readonly diagnostic?: ThreadStoreDiagnostic },
@@ -126,19 +130,6 @@ const schemaStoreError = (operation: string, error: Schema.SchemaError) =>
     }),
   });
 
-const makeOffset = (
-  threadId: ThreadMaterialization["threadId"],
-  sequence: number,
-): Effect.Effect<ObservationOffset, ThreadStoreError> =>
-  Schema.decodeEffect(CanonicalSequence)(sequence).pipe(
-    Effect.flatMap((validatedSequence) =>
-      Schema.decodeEffect(ObservationOffset)(
-        `${DO_OFFSET_PREFIX}${encodeURIComponent(threadId)}:${validatedSequence}`,
-      ),
-    ),
-    Effect.mapError((error) => schemaStoreError("encode observation offset", error)),
-  );
-
 const parseOffset = Effect.fnUntraced(function* (
   threadId: ThreadMaterialization["threadId"],
   offset: ObservationOffset | undefined,
@@ -179,8 +170,8 @@ const encodeCheckpoint = (checkpoint: ThreadCheckpoint): Effect.Effect<string, T
 const envelopes = new WeakMap<object, CanonicalRecordEnvelope>();
 
 const decodeEnvelope = Effect.fnUntraced(function* (row: {
-  readonly batch_id: string;
-  readonly thread_id: string;
+  readonly batch_id: CanonicalRecordEnvelope["batchId"];
+  readonly thread_id: CanonicalRecordEnvelope["threadId"];
   readonly record_json: string;
   readonly sequence: CanonicalSequence;
 }) {
@@ -213,15 +204,11 @@ const decodeEnvelope = Effect.fnUntraced(function* (row: {
     ),
   );
 
-  const threadId = yield* Schema.decodeEffect(CanonicalRecordEnvelope.fields.threadId)(
-    row.thread_id,
-  ).pipe(Effect.mapError((error) => schemaStoreError("decode thread identity", error)));
+  const { thread_id: threadId, batch_id: batchId } = row;
 
-  const offset = yield* makeOffset(threadId, row.sequence);
-
-  const batchId = yield* Schema.decodeEffect(CanonicalRecordEnvelope.fields.batchId)(
-    row.batch_id,
-  ).pipe(Effect.mapError((error) => schemaStoreError("decode batch identity", error)));
+  const offset = ObservationOffset.make(
+    `${DO_OFFSET_PREFIX}${encodeURIComponent(threadId)}:${row.sequence}`,
+  );
 
   const envelope = CanonicalRecordEnvelope.make(
     { threadId, batchId, sequence: row.sequence, offset, record },
@@ -317,9 +304,9 @@ const makeServices = Effect.fnUntraced(function* () {
   const materialize: ThreadStore["Service"]["materialize"] = Effect.fnUntraced(function* (
     request: ThreadMaterialization,
   ) {
-    const validated = yield* Schema.decodeEffect(Schema.toType(ThreadMaterialization))(
-      request,
-    ).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
+    const validated = yield* decodeThreadMaterializationType(request).pipe(
+      Effect.mapError((error) => schemaStoreError("validate materialization", error)),
+    );
 
     const now = yield* Clock.currentTimeMillis;
 
@@ -362,11 +349,6 @@ const makeServices = Effect.fnUntraced(function* () {
 
             return storeError("append canonical batch", error);
           }),
-          Effect.flatMap((result) =>
-            Schema.decodeEffect(AppendResult)(result).pipe(
-              Effect.mapError((error) => schemaStoreError("decode append result", error)),
-            ),
-          ),
         );
 
       yield* hitFailpoint("append:after");
@@ -391,7 +373,7 @@ const makeServices = Effect.fnUntraced(function* () {
   });
 
   const readEffect = Effect.fnUntraced(function* (request: ThreadReadRequest) {
-    const validated = yield* Schema.decodeEffect(Schema.toType(ThreadReadRequest))(request).pipe(
+    const validated = yield* decodeThreadReadRequestType(request).pipe(
       Effect.mapError((error) => schemaStoreError("validate thread read", error)),
     );
 
@@ -503,7 +485,7 @@ const makeServices = Effect.fnUntraced(function* () {
   const inspectTail: ThreadStore["Service"]["inspectTail"] = Effect.fnUntraced(function* (
     request: ThreadTailRequest,
   ) {
-    const validated = yield* Schema.decodeEffect(Schema.toType(ThreadTailRequest))(request).pipe(
+    const validated = yield* decodeThreadTailRequestType(request).pipe(
       Effect.mapError((error) => schemaStoreError("validate tail inspection", error)),
     );
 
@@ -602,15 +584,7 @@ const makeServices = Effect.fnUntraced(function* () {
       snapshot: journal.state.read,
       tail: (threadId) =>
         journal.getThread(threadId).pipe(
-          Effect.flatMap((rows) =>
-            Schema.decodeEffect(
-              Schema.UndefinedOr(
-                Schema.Struct({
-                  tail_sequence: CanonicalSequence,
-                }),
-              ),
-            )(rows[0]),
-          ),
+          Effect.map((rows) => rows[0]),
           Effect.mapError((error) => storeError("native thread tail", error)),
         ),
     }),
@@ -659,6 +633,13 @@ export const threadStoreLayer: Layer.Layer<
   DoStorageConfig | DoStorageFailpoint | SqlClientService.SqlClient | Crypto.Crypto
 > = ThreadReader.layer().pipe(Layer.provideMerge(Layer.effectContext(makeServices())));
 
+/**
+ * Share one initialized journal between local storage ports in one Object build. Provide
+ * this beneath both ports and above their common SQL, owner, configuration, failpoint,
+ * progress and lifecycle providers. Standalone ports initialize their own journal.
+ */
+export { initializedDoJournalLayer as journalLayer } from "./internal/do-journal.ts";
+
 /**
  * Validated Durable Object storage configuration Layer with the documented defaults applied.
  * Shared by the ThreadStore and SubmissionLedger convenience layers so their defaults
@@ -668,7 +649,7 @@ export const storageConfigLayer = (
   options: DoStorageOptions,
 ): Layer.Layer<DoStorageConfig, DoStorageError> =>
   Layer.effect(DoStorageConfig)(
-    Schema.decodeEffect(DoStorageConfigValue)({
+    decodeStorageConfig({
       observationPollInterval: options.observationPollInterval ?? 25,
       maxStoredValueBytes: options.maxStoredValueBytes ?? DEFAULT_MAX_STORED_VALUE_BYTES,
       verifyOnOpen: options.verifyOnOpen ?? false,

package/src/PortRouting.ts

@@ -236,35 +236,36 @@ const LOCAL: RouteTarget = { _tag: "local" };
  * Identities beyond the ledger's 1,024-character row bound fail typed: the minting side
  * refused them at admission, so they cannot name any stored row anywhere.
  */
-const routableSubmissionTarget = (
-  ownsThread: RoutedPortOptions["ownsThread"],
-): ((operation: string, submissionId: string) => Effect.Effect<RouteTarget, LedgerError>) =>
-  Effect.fnUntraced(function* (
-    operation: string,
-    submissionId: string,
-  ): Effect.fn.Return<RouteTarget, LedgerError> {
-    if (submissionId.length > MAX_ROUTABLE_SUBMISSION_ID_LENGTH) {
-      return yield* LedgerError.make({
-        operation,
-        message:
-          `A Submission identity of ${submissionId.length} characters exceeds the ` +
-          `${MAX_ROUTABLE_SUBMISSION_ID_LENGTH}-character routable identity bound; admission ` +
-          "refuses such identities, so it cannot name any stored row.",
-      });
-    }
-    const separator = submissionId.indexOf(":");
-
-    if (separator === -1) return LOCAL;
-    if (!UUID_HEAD_PATTERN.test(submissionId.slice(0, separator))) return LOCAL;
-    const tail = submissionId.slice(separator + 1);
-
-    return yield* decodeThreadId(tail).pipe(
-      Effect.map((threadId): RouteTarget =>
-        ownsThread(threadId) ? LOCAL : { _tag: "foreign", threadId },
-      ),
-      Effect.orElseSucceed(() => LOCAL),
-    );
-  });
+const routableSubmissionTarget =
+  (
+    ownsThread: RoutedPortOptions["ownsThread"],
+  ): ((operation: string, submissionId: string) => Effect.Effect<RouteTarget, LedgerError>) =>
+  (operation, submissionId) =>
+    Effect.suspend(() => {
+      if (submissionId.length > MAX_ROUTABLE_SUBMISSION_ID_LENGTH) {
+        return Effect.fail(
+          LedgerError.make({
+            operation,
+            message:
+              `A Submission identity of ${submissionId.length} characters exceeds the ` +
+              `${MAX_ROUTABLE_SUBMISSION_ID_LENGTH}-character routable identity bound; admission ` +
+              "refuses such identities, so it cannot name any stored row.",
+          }),
+        );
+      }
+      const separator = submissionId.indexOf(":");
+
+      if (separator === -1 || !UUID_HEAD_PATTERN.test(submissionId.slice(0, separator)))
+        return Effect.succeed(LOCAL);
+
+      return decodeThreadId(submissionId.slice(separator + 1)).pipe(
+        Effect.match({
+          onFailure: () => LOCAL,
+          onSuccess: (threadId): RouteTarget =>
+            ownsThread(threadId) ? LOCAL : { _tag: "foreign", threadId },
+        }),
+      );
+    });
 
 /**
  * Read a Submission only from this physical owner. Encoded foreign identities fail before
@@ -570,9 +571,9 @@ const makeRoutedLedgerServices = Effect.fnUntraced(function* (options: RoutedPor
   const routed = SubmissionLedger.of({
     capabilities: local.capabilities,
 
-    admit: (request) =>
+    admit: (request, admissionOptions) =>
       options.ownsThread(request.threadId)
-        ? local.admit(request)
+        ? local.admit(request, admissionOptions)
         : foreignLedgerCall(
             "ledger admit",
             request.threadId,
@@ -730,6 +731,23 @@ const makeRoutedLedgerServices = Effect.fnUntraced(function* (options: RoutedPor
         ? local.claimJoining(request)
         : Effect.fail(crossThreadLedgerError("ledger claim joining", request.threadId)),
 
+    ...(local.appendInput === undefined
+      ? {}
+      : {
+          appendInput: (
+            submissionId: SubmissionSnapshot["submissionId"],
+            request: Parameters<NonNullable<SubmissionLedger["Service"]["appendInput"]>>[1],
+          ) =>
+            options.ownsThread(request.threadId)
+              ? local.appendInput!(submissionId, request)
+              : Effect.fail(
+                  ThreadStoreError.make({
+                    operation: "append original input",
+                    message: "Original input publication is lane-local",
+                  }),
+                ),
+        }),
+
     markInputApplied: (request) =>
       submissionTarget("ledger mark input applied", request.submissionId).pipe(
         Effect.flatMap((target) =>
@@ -824,6 +842,7 @@ const makeRoutedLedgerServices = Effect.fnUntraced(function* (options: RoutedPor
 
     // The local scan IS the whole worklist: one Thread per Object (durability §5).
     scanNonterminal: local.scanNonterminal,
+    ...(local.listNonterminal === undefined ? {} : { listNonterminal: local.listNonterminal }),
 
     readAbortIntent: (request) =>
       submissionTarget("ledger read abort intent", request.submissionId).pipe(
@@ -1046,6 +1065,13 @@ const makeRoutedStoreServices = Effect.fnUntraced(function* (options: RoutedPort
             ).pipe(Effect.map((reply) => reply.range)),
     },
     work: {
+      isOnlyFreshAdmission: (request) => {
+        const prove = local.work?.isOnlyFreshAdmission;
+
+        return prove !== undefined && options.ownsThread(request.threadId)
+          ? prove(request)
+          : Effect.succeed(false);
+      },
       threads: (request) =>
         Effect.flatMap(CurrentNativeSource, (nativeSource) =>
           nativeSource && local.work !== undefined