@yielded/agent-storage-cloudflare 0.1.0-beta.173 → 0.1.0-beta.175

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/dist/DoMemoryStore.mjs +1 -1
  2. package/dist/DoMessageDeliveryStore.mjs +1 -1
  3. package/dist/DoStorageFailpoint.d.mts +3 -1
  4. package/dist/DoStorageFailpoint.mjs +7 -2
  5. package/dist/DoStorageFailpoint.mjs.map +1 -1
  6. package/dist/DoStorageFailpointTesting.mjs +1 -4
  7. package/dist/DoStorageFailpointTesting.mjs.map +1 -1
  8. package/dist/{DoStorageVersion-NKnQU3eR.d.mts → DoStorageVersion-Mc7CXMdN.d.mts} +2 -2
  9. package/dist/DoStorageVersion.d.mts +1 -1
  10. package/dist/DoStorageVersion.mjs +2 -2
  11. package/dist/DoSubmissionLedger.mjs +121 -67
  12. package/dist/DoSubmissionLedger.mjs.map +1 -1
  13. package/dist/DoSubscriptionStore.mjs.map +1 -1
  14. package/dist/{DoThreadStore-CtyahRTW.mjs → DoThreadStore-CgIZQUNR.mjs} +30 -25
  15. package/dist/DoThreadStore-CgIZQUNR.mjs.map +1 -0
  16. package/dist/DoThreadStore.mjs +1 -1
  17. package/dist/PortProtocol.d.mts +12 -0
  18. package/dist/PortRouting.mjs +6 -1
  19. package/dist/PortRouting.mjs.map +1 -1
  20. package/dist/{do-journal-_MdLGTd3.mjs → do-journal-7P3DUBz0.mjs} +140 -65
  21. package/dist/do-journal-7P3DUBz0.mjs.map +1 -0
  22. package/dist/index.d.mts +1 -1
  23. package/dist/index.mjs +1 -1
  24. package/dist/{migrations-C3N9GN6A.mjs → migrations-CAO_akhD.mjs} +45 -22
  25. package/dist/migrations-CAO_akhD.mjs.map +1 -0
  26. package/dist/{owned-state-BHzUi0eb.mjs → owned-state-B4QhezMp.mjs} +8 -6
  27. package/dist/owned-state-B4QhezMp.mjs.map +1 -0
  28. package/package.json +1 -1
  29. package/src/DoStorageFailpoint.ts +4 -0
  30. package/src/DoStorageFailpointTesting.ts +9 -7
  31. package/src/DoSubmissionLedger.ts +271 -150
  32. package/src/DoSubscriptionStore.ts +1 -0
  33. package/src/DoThreadStore.ts +27 -50
  34. package/src/PortRouting.ts +10 -2
  35. package/src/internal/canonical-append.ts +1 -1
  36. package/src/internal/do-journal.ts +987 -800
  37. package/src/internal/migrations.ts +53 -10
  38. package/src/internal/owned-state.ts +18 -7
  39. package/src/internal/prompt-checkpoint.ts +19 -13
  40. package/src/internal/storage-span.ts +22 -13
  41. package/dist/DoThreadStore-CtyahRTW.mjs.map +0 -1
  42. package/dist/do-journal-_MdLGTd3.mjs.map +0 -1
  43. package/dist/migrations-C3N9GN6A.mjs.map +0 -1
  44. package/dist/owned-state-BHzUi0eb.mjs.map +0 -1
@@ -1,3 +1,4 @@
1
+ import { SqliteClient } from "@effect/sql-sqlite-do";
1
2
  import {
2
3
  makeCanonicalStorage,
3
4
  recordBatchId,
@@ -21,16 +22,29 @@ import {
21
22
  LifecyclePublicationFact,
22
23
  LifecyclePublicationError,
23
24
  } from "@yielded/agent/lifecycle-publication";
24
- import { CanonicalRecord, CanonicalSequence } from "@yielded/agent/records";
25
+ import { BatchId, CanonicalRecord, CanonicalSequence } from "@yielded/agent/records";
25
26
  import { SqlStorageOwner } from "@yielded/agent/sql-memory-store";
26
27
  import {
27
28
  type AppendConflict,
29
+ AppendResult,
28
30
  ThreadStoreDiagnostic,
29
31
  ThreadStoreError,
30
32
  } from "@yielded/agent/thread-store";
31
- import { Cause, Clock, Effect, Option, Schema, Stream } from "effect";
33
+ import {
34
+ Cause,
35
+ Clock,
36
+ Effect,
37
+ Option,
38
+ Predicate,
39
+ References,
40
+ Result,
41
+ Schema,
42
+ Stream,
43
+ Tracer,
44
+ } from "effect";
32
45
  import * as SqlClient from "effect/sql/SqlClient";
33
- import { SqlError } from "effect/sql/SqlError";
46
+ import { classifySqliteError, SqlError } from "effect/sql/SqlError";
47
+ import { CurrentTransformer, type Statement } from "effect/sql/Statement";
34
48
 
35
49
  import {
36
50
  type DoStorageFailpointError,
@@ -41,6 +55,7 @@ import {
41
55
  DoValueBoundExceeded,
42
56
  type DoStorageFailpointLocation,
43
57
  } from "../DoStorageError.ts";
58
+ import { isDefaultDoStorageFailpoint } from "../DoStorageFailpoint.ts";
44
59
  import { ensureDoStorageLayout } from "./migrations.ts";
45
60
  import { ownedState, ownedRows, type OwnedState } from "./owned-state.ts";
46
61
  import {
@@ -68,6 +83,12 @@ const MAX_RECORD_CACHE_ENTRIES = 4_096;
68
83
  const MAX_BOUND_PARAMETERS = 100;
69
84
  const isSqlError = Schema.is(SqlError);
70
85
 
86
+ const isNativeClient = Schema.is(
87
+ Schema.declare<SqliteClient.SqliteClient>((value): value is SqliteClient.SqliteClient =>
88
+ Predicate.hasProperty(value, SqliteClient.TypeId),
89
+ ),
90
+ );
91
+
71
92
  const storedTextBytes = (value: string): number => new TextEncoder().encode(value).byteLength;
72
93
 
73
94
  const chunked = <A>(values: ReadonlyArray<A>, size: number): Array<ReadonlyArray<A>> => {
@@ -94,8 +115,8 @@ const threadRows = ownedRows(
94
115
  );
95
116
 
96
117
  class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
97
- batch_id: Schema.fromJsonString(BoundedIdentifier),
98
- thread_id: BoundedIdentifier,
118
+ batch_id: Schema.fromJsonString(BatchId.check(Schema.isMaxLength(MAX_IDENTIFIER_LENGTH))),
119
+ thread_id: ThreadId.check(Schema.isMaxLength(MAX_IDENTIFIER_LENGTH)),
99
120
  record_id: Schema.fromJsonString(BoundedIdentifier),
100
121
  record_json: BoundedStoredText,
101
122
  sequence: CanonicalSequence,
@@ -200,14 +221,6 @@ const initializedLifecycleSources = new WeakSet<OwnedState>();
200
221
 
201
222
  export type { RawAppendRequest } from "@yielded/agent-storage-sql/sql-journal";
202
223
 
203
- export class RawAppendResult extends Schema.Class<RawAppendResult>(
204
- "@effect-agent/storage-cloudflare/RawAppendResult",
205
- )({
206
- firstSequence: CanonicalSequence,
207
- lastSequence: CanonicalSequence,
208
- replayed: Schema.Boolean,
209
- }) {}
210
-
211
224
  export class RawReadRequest extends Schema.Class<RawReadRequest>(
212
225
  "@effect-agent/storage-cloudflare/RawReadRequest",
213
226
  )({
@@ -317,10 +330,14 @@ const ensureCurrentStorage = Effect.fnUntraced(function* (
317
330
  failpoint: DoJournalFailpoint = noFailpoint,
318
331
  maxStoredValueBytes: number,
319
332
  ) {
320
- yield* ensureDoStorageLayout().pipe(Effect.provideService(SqlClient.SqlClient, sql));
321
-
322
333
  const state = yield* ownedState(sql);
323
334
  const owner = (yield* SqlStorageOwner) ?? state;
335
+
336
+ yield* ensureDoStorageLayout().pipe(
337
+ Effect.provideService(SqlStorageOwner, owner),
338
+ Effect.provideService(SqlClient.SqlClient, sql),
339
+ );
340
+
324
341
  let journal: DoJournal | undefined;
325
342
 
326
343
  const lifecycle = yield* makeSqlLifecyclePublication(undefined, maxStoredValueBytes).pipe(
@@ -356,173 +373,196 @@ const ensureCurrentStorage = Effect.fnUntraced(function* (
356
373
  return journal;
357
374
  });
358
375
 
359
- const makeJournal = (
376
+ const makeJournal = Effect.fnUntraced(function* (
360
377
  sql: SqlClient.SqlClient,
361
378
  failpoint: DoJournalFailpoint,
362
379
  maxStoredValueBytes: number,
363
380
  lifecycle: Effect.Success<ReturnType<typeof makeSqlLifecyclePublication>>,
364
381
  state: OwnedState,
365
- ) =>
366
- Effect.gen(function* () {
367
- const owner = (yield* SqlStorageOwner) ?? state;
368
- const progress = yield* SqlStorageProgress;
369
-
370
- const work = yield* makeSqlThreadWork({
371
- read: (body) => state.read(body),
372
- write: (body) =>
373
- owner.transaction(body.pipe(Effect.tap(() => state.invalidate))).pipe(
374
- Effect.provideService(SqlClient.SqlClient, sql),
375
- Effect.catchTag("SqlError", (cause) =>
376
- ThreadStoreError.make({
377
- operation: "rebuild Thread work",
378
- message: "Thread work transaction failed",
379
- cause,
380
- }),
381
- ),
382
+ ) {
383
+ const owner = (yield* SqlStorageOwner) ?? state;
384
+ const progress = yield* SqlStorageProgress;
385
+
386
+ const nativeDb =
387
+ isNativeClient(sql) &&
388
+ sql.config.transformQueryNames === undefined &&
389
+ sql.config.transformResultNames === undefined &&
390
+ isDefaultDoStorageFailpoint(failpoint)
391
+ ? sql.config.storage?.sql
392
+ : undefined;
393
+
394
+ const work = yield* makeSqlThreadWork({
395
+ read: (body) => state.read(body),
396
+ write: (body) =>
397
+ owner.transaction(body.pipe(Effect.tap(() => state.invalidate))).pipe(
398
+ Effect.provideService(SqlClient.SqlClient, sql),
399
+ Effect.catchTag("SqlError", (cause) =>
400
+ ThreadStoreError.make({
401
+ operation: "rebuild Thread work",
402
+ message: "Thread work transaction failed",
403
+ cause,
404
+ }),
382
405
  ),
383
- }).pipe(Effect.provideService(SqlClient.SqlClient, sql));
406
+ ),
407
+ }).pipe(Effect.provideService(SqlClient.SqlClient, sql));
384
408
 
385
- yield* Effect.acquireRelease(
386
- Effect.sync(() => state.invalidators.add(work.invalidate)),
387
- () => Effect.sync(() => state.invalidators.delete(work.invalidate)),
388
- );
409
+ yield* Effect.acquireRelease(
410
+ Effect.sync(() => state.invalidators.add(work.invalidate)),
411
+ () => Effect.sync(() => state.invalidators.delete(work.invalidate)),
412
+ );
389
413
 
390
- const workFailure = (cause: unknown) =>
391
- DoStorageError.make({
392
- operation: "publish Thread work",
393
- message: "Canonical work metadata could not be published",
394
- cause,
395
- });
414
+ const workFailure = (cause: unknown) =>
415
+ DoStorageError.make({
416
+ operation: "publish Thread work",
417
+ message: "Canonical work metadata could not be published",
418
+ cause,
419
+ });
396
420
 
397
- const executions = yield* makeSqlExecutions().pipe(
398
- Effect.provideService(SqlClient.SqlClient, sql),
399
- );
421
+ const executions = yield* makeSqlExecutions().pipe(
422
+ Effect.provideService(SqlClient.SqlClient, sql),
423
+ );
400
424
 
401
- const executionConflict = <A>(effect: Effect.Effect<A, AppendConflict | ThreadStoreError>) =>
402
- effect.pipe(
403
- Effect.catchTag("AppendConflict", (error) =>
404
- Effect.fail(
405
- DoAppendConflict.make({
406
- reason: error.reason,
407
- message: "Active execution conflicts with the captured commit",
408
- ...(error.actualTailSequence === undefined
409
- ? {}
410
- : { actualTailSequence: error.actualTailSequence }),
411
- }),
412
- ),
425
+ const executionConflict = <A>(effect: Effect.Effect<A, AppendConflict | ThreadStoreError>) =>
426
+ effect.pipe(
427
+ Effect.catchTag("AppendConflict", (error) =>
428
+ Effect.fail(
429
+ DoAppendConflict.make({
430
+ reason: error.reason,
431
+ message: "Active execution conflicts with the captured commit",
432
+ ...(error.actualTailSequence === undefined
433
+ ? {}
434
+ : { actualTailSequence: error.actualTailSequence }),
435
+ }),
413
436
  ),
414
- );
437
+ ),
438
+ );
415
439
 
416
- const threads = threadRows(state, sql);
417
- const cursors = lifecycleCursorRows(state, sql);
418
- let records = recordCaches.get(state);
440
+ const threads = threadRows(state, sql);
441
+ const cursors = lifecycleCursorRows(state, sql);
442
+ let records = recordCaches.get(state);
419
443
 
420
- if (records === undefined) {
421
- records = makeRecordCache();
422
- recordCaches.set(state, records);
423
- state.invalidators.add(records.clear);
424
- }
425
- const recordCache = records;
426
-
427
- const archives = yield* makeSqlThreadArchiveRange({
428
- read: (body) => state.read(body),
429
- write: (body) =>
430
- owner.transaction(body.pipe(Effect.tap(() => state.invalidate))).pipe(
431
- Effect.provideService(SqlClient.SqlClient, sql),
432
- Effect.catchTag("SqlError", (cause) =>
433
- ThreadStoreError.make({
434
- operation: "archive write",
435
- message: "Archive transaction failed.",
436
- cause,
437
- }),
438
- ),
444
+ if (records === undefined) {
445
+ records = makeRecordCache();
446
+ recordCaches.set(state, records);
447
+ state.invalidators.add(records.clear);
448
+ }
449
+ const recordCache = records;
450
+
451
+ const archives = yield* makeSqlThreadArchiveRange({
452
+ read: (body) => state.read(body),
453
+ write: (body) =>
454
+ owner.transaction(body.pipe(Effect.tap(() => state.invalidate))).pipe(
455
+ Effect.provideService(SqlClient.SqlClient, sql),
456
+ Effect.catchTag("SqlError", (cause) =>
457
+ ThreadStoreError.make({
458
+ operation: "archive write",
459
+ message: "Archive transaction failed.",
460
+ cause,
461
+ }),
439
462
  ),
440
- }).pipe(Effect.provideService(SqlClient.SqlClient, sql));
463
+ ),
464
+ }).pipe(Effect.provideService(SqlClient.SqlClient, sql));
441
465
 
442
- const recordJson = canonicalRecordJson(sql);
466
+ const recordJson = canonicalRecordJson(sql);
443
467
 
444
- const physical = yield* makeCanonicalStorage().pipe(
445
- Effect.provideService(SqlClient.SqlClient, sql),
446
- );
468
+ const physical = yield* makeCanonicalStorage().pipe(
469
+ Effect.provideService(SqlClient.SqlClient, sql),
470
+ );
447
471
 
448
- /** Typed pre-write refusal for any single value over the configured byte bound. */
449
- const checkValueBound = (
450
- operation: string,
451
- value: string,
452
- actualBytes = storedTextBytes(value),
453
- ): Effect.Effect<void, DoValueBoundExceeded> => {
454
- return actualBytes > maxStoredValueBytes
455
- ? Effect.fail(
456
- DoValueBoundExceeded.make({
457
- actualBytes,
458
- maxBytes: maxStoredValueBytes,
459
- operation,
460
- }),
461
- )
462
- : Effect.void;
463
- };
472
+ /** Typed pre-write refusal for any single value over the configured byte bound. */
473
+ const checkValueBound = (
474
+ operation: string,
475
+ value: string,
476
+ actualBytes = storedTextBytes(value),
477
+ ): Effect.Effect<void, DoValueBoundExceeded> => {
478
+ return actualBytes > maxStoredValueBytes
479
+ ? Effect.fail(
480
+ DoValueBoundExceeded.make({
481
+ actualBytes,
482
+ maxBytes: maxStoredValueBytes,
483
+ operation,
484
+ }),
485
+ )
486
+ : Effect.void;
487
+ };
464
488
 
465
- /**
466
- * Runs one journal write transaction on the Durable Object storage-backed
467
- * `withTransaction` (`ctx.storage.transaction()` under the hood). Within one Durable
468
- * Object there is exactly ONE writer, so the Node `BEGIN IMMEDIATE` + busy-retry +
469
- * `SqliteWriteContention` machinery has no analogue here and is deliberately absent.
470
- * The shared local owner gate covers the transaction and acknowledgement.
471
- *
472
- * Journal write transactions are always top level: the Durable Object client rejects
473
- * nested transactions, so new journal operations must not wrap this helper inside another
474
- * transaction. Expected outcomes may be captured only when every such refusal precedes
475
- * mutation; real faults still roll back. Successful preflight reads never authorize a write.
476
- */
477
- const withWriteTransaction =
478
- (operation: string, expected: (error: { readonly _tag: string }) => boolean = () => false) =>
479
- <A, E extends { readonly _tag: string }>(
480
- effect: Effect.Effect<A, E>,
481
- ): Effect.Effect<A, E | DoStorageError> =>
482
- state.owner.withMutation(owner.transaction(storageResult(effect, expected))).pipe(
483
- Effect.provideService(SqlClient.SqlClient, sql),
484
- Effect.mapError((error) => (isSqlError(error) ? storageError(operation)(error) : error)),
485
- Effect.tapError(annotateStorageError),
486
- Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
487
- Effect.flatMap(Effect.fromResult),
488
- );
489
+ /**
490
+ * Runs one journal write transaction on the Durable Object storage-backed
491
+ * `withTransaction` (`ctx.storage.transaction()` under the hood). Within one Durable
492
+ * Object there is exactly ONE writer, so the Node `BEGIN IMMEDIATE` + busy-retry +
493
+ * `SqliteWriteContention` machinery has no analogue here and is deliberately absent.
494
+ * The shared local owner gate covers the transaction and acknowledgement.
495
+ *
496
+ * Journal write transactions are always top level: the Durable Object client rejects
497
+ * nested transactions, so new journal operations must not wrap this helper inside another
498
+ * transaction. Expected outcomes may be captured only when every such refusal precedes
499
+ * mutation; real faults still roll back. Successful preflight reads never authorize a write.
500
+ */
501
+ const withWriteTransaction =
502
+ (operation: string, expected: (error: { readonly _tag: string }) => boolean = () => false) =>
503
+ <A, E extends { readonly _tag: string }>(
504
+ effect: Effect.Effect<A, E>,
505
+ ): Effect.Effect<A, E | DoStorageError> =>
506
+ Effect.withFiber((fiber) => {
507
+ const body = state.owner
508
+ .withMutation(owner.transaction(storageResult(effect, expected)))
509
+ .pipe(
510
+ Effect.provideService(SqlClient.SqlClient, sql),
511
+ Effect.mapError((error) =>
512
+ isSqlError(error) ? storageError(operation)(error) : error,
513
+ ),
514
+ );
489
515
 
490
- const materialize = Effect.fnUntraced(function* (
491
- threadId: string,
492
- createdAt: string,
493
- ): Effect.fn.Return<void, DoStorageCorruptionError | DoStorageError | DoValueBoundExceeded> {
494
- if (threadId.length > MAX_IDENTIFIER_LENGTH) {
495
- return yield* DoStorageError.make({
496
- operation: "materialize thread",
497
- message: "Thread identity exceeds the Durable Object storage bounds.",
498
- });
499
- }
500
- // Refusals and idempotent materialization need no transaction. Recheck before writes.
501
- const current = yield* getThread(threadId);
516
+ const traced =
517
+ fiber.getRef(References.TracerEnabled) &&
518
+ fiber.getRef(Tracer.Tracer) !== Tracer.nativeTracer;
519
+
520
+ return (
521
+ traced
522
+ ? body.pipe(
523
+ Effect.tapError(annotateStorageError),
524
+ Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
525
+ )
526
+ : body
527
+ ).pipe(Effect.flatMap(Effect.fromResult));
528
+ });
502
529
 
503
- if (current.length === 1) {
504
- return;
505
- }
506
- yield* withWriteTransaction("materialize transaction")(
507
- Effect.gen(function* () {
508
- const existingRows = yield* getThread(threadId);
530
+ const materialize = Effect.fnUntraced(function* (
531
+ threadId: string,
532
+ createdAt: string,
533
+ ): Effect.fn.Return<void, DoStorageCorruptionError | DoStorageError | DoValueBoundExceeded> {
534
+ if (threadId.length > MAX_IDENTIFIER_LENGTH) {
535
+ return yield* DoStorageError.make({
536
+ operation: "materialize thread",
537
+ message: "Thread identity exceeds the Durable Object storage bounds.",
538
+ });
539
+ }
540
+ // Refusals and idempotent materialization need no transaction. Recheck before writes.
541
+ const current = yield* getThread(threadId);
509
542
 
510
- const existing = yield* decodeRows(
511
- Schema.Array(ThreadRow),
512
- "effect_agent_threads",
513
- threadId,
514
- existingRows,
515
- );
543
+ if (current.length === 1) {
544
+ return;
545
+ }
546
+ yield* withWriteTransaction("materialize transaction")(
547
+ Effect.gen(function* () {
548
+ const existingRows = yield* getThread(threadId);
549
+
550
+ const existing = yield* decodeRows(
551
+ Schema.Array(ThreadRow),
552
+ "effect_agent_threads",
553
+ threadId,
554
+ existingRows,
555
+ );
516
556
 
517
- if (existing.length > 1) {
518
- return yield* DoStorageCorruptionError.make({
519
- table: "effect_agent_threads",
520
- rowKey: threadId,
521
- message: "A thread primary key returned more than one row.",
522
- });
523
- }
524
- if (existing.length === 0) {
525
- yield* sql`
557
+ if (existing.length > 1) {
558
+ return yield* DoStorageCorruptionError.make({
559
+ table: "effect_agent_threads",
560
+ rowKey: threadId,
561
+ message: "A thread primary key returned more than one row.",
562
+ });
563
+ }
564
+ if (existing.length === 0) {
565
+ yield* sql`
526
566
  INSERT INTO effect_agent_threads (
527
567
  thread_id,
528
568
  created_at,
@@ -534,113 +574,280 @@ const makeJournal = (
534
574
  ) ON CONFLICT(thread_id) DO UPDATE SET created_at=excluded.created_at
535
575
  WHERE effect_agent_threads.created_at IS NULL
536
576
  RETURNING *`.pipe(
537
- threads.write,
577
+ threads.write,
578
+ Effect.mapError((error) =>
579
+ error._tag === "SqlError" ? storageError("materialize thread")(error) : error,
580
+ ),
581
+ );
582
+
583
+ if (lifecycle !== undefined)
584
+ yield* sql`INSERT INTO effect_agent_lifecycle_cursors (thread_id, through_sequence)
585
+ VALUES (${threadId}, 0) RETURNING *`.pipe(
586
+ cursors.write,
538
587
  Effect.mapError((error) =>
539
- error._tag === "SqlError" ? storageError("materialize thread")(error) : error,
588
+ error._tag === "SqlError"
589
+ ? storageError("initialize lifecycle source")(error)
590
+ : error,
540
591
  ),
541
592
  );
542
593
 
543
- if (lifecycle !== undefined)
544
- yield* sql`INSERT INTO effect_agent_lifecycle_cursors (thread_id, through_sequence)
545
- VALUES (${threadId}, 0) RETURNING *`.pipe(
546
- cursors.write,
547
- Effect.mapError((error) =>
548
- error._tag === "SqlError"
549
- ? storageError("initialize lifecycle source")(error)
550
- : error,
551
- ),
552
- );
594
+ return;
595
+ }
596
+ }),
597
+ );
598
+ }, withStorageSpan("DoJournal.materialize"));
553
599
 
554
- return;
555
- }
556
- }),
557
- );
558
- }, withStorageSpan("DoJournal.materialize"));
559
-
560
- const getThread = Effect.fnUntraced(function* (threadId: string) {
561
- return (yield* threads
562
- .by("thread_id", threadId)
563
- .pipe(
564
- Effect.mapError((error) =>
565
- error._tag === "SqlError" ? storageError("read thread")(error) : error,
566
- ),
567
- )).filter((row) => row.thread_id === threadId && row.created_at !== null);
568
- });
600
+ const getThread = Effect.fnUntraced(function* (threadId: string) {
601
+ return (yield* threads
602
+ .by("thread_id", threadId)
603
+ .pipe(
604
+ Effect.mapError((error) =>
605
+ error._tag === "SqlError" ? storageError("read thread")(error) : error,
606
+ ),
607
+ )).filter((row) => row.thread_id === threadId && row.created_at !== null);
608
+ });
569
609
 
570
- const hasRecord = Effect.fnUntraced(function* (threadId: string, recordId: string) {
571
- const thread = (yield* getThread(threadId))[0];
572
- const prefix = recordCache.prefix(threadId, thread?.tail_sequence ?? 0);
610
+ const hasRecord = Effect.fnUntraced(function* (threadId: string, recordId: string) {
611
+ const thread = (yield* getThread(threadId))[0];
612
+ const prefix = recordCache.prefix(threadId, thread?.tail_sequence ?? 0);
573
613
 
574
- if (prefix !== undefined) return prefix.some((row) => row.record_id === recordId);
614
+ if (prefix !== undefined) return prefix.some((row) => row.record_id === recordId);
575
615
 
576
- const rows = yield* sql`SELECT record_id FROM effect_agent_canonical_records
616
+ const rows = yield* sql`SELECT record_id FROM effect_agent_canonical_records
577
617
  WHERE thread_id = ${threadId} AND record_id = ${JSON.stringify(recordId)}`.pipe(
578
- Effect.mapError(storageError("read canonical record identity")),
579
- );
618
+ Effect.mapError(storageError("read canonical record identity")),
619
+ );
580
620
 
581
- return rows.length > 0;
582
- });
621
+ return rows.length > 0;
622
+ });
583
623
 
584
- const prepareAppend = Effect.fnUntraced(function* (request: RawAppendRequest) {
585
- if (
586
- request.threadId.length > MAX_IDENTIFIER_LENGTH ||
587
- request.batchId.length > MAX_IDENTIFIER_LENGTH ||
588
- request.executionWrites.some(
589
- ({ update }) => update.checkpoint.progress.runId.length > MAX_IDENTIFIER_LENGTH,
590
- ) ||
591
- request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)
592
- ) {
593
- return yield* DoStorageError.make({
594
- operation: "append canonical batch",
595
- message: "Canonical identifiers exceed the Durable Object storage bounds.",
624
+ const prepareAppend = Effect.fnUntraced(function* (request: RawAppendRequest) {
625
+ if (
626
+ request.threadId.length > MAX_IDENTIFIER_LENGTH ||
627
+ request.batchId.length > MAX_IDENTIFIER_LENGTH ||
628
+ request.executionWrites.some(
629
+ ({ update }) => update.checkpoint.progress.runId.length > MAX_IDENTIFIER_LENGTH,
630
+ ) ||
631
+ request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)
632
+ ) {
633
+ return yield* DoStorageError.make({
634
+ operation: "append canonical batch",
635
+ message: "Canonical identifiers exceed the Durable Object storage bounds.",
636
+ });
637
+ }
638
+ // Archives store the full batch as one value, so their ceiling also bounds hot appends.
639
+ // The trusted append SPI pairs captured JSON with its exact UTF-8 width.
640
+ yield* checkValueBound("append canonical batch", request.batchJson, request.batchBytes);
641
+ for (const record of request.records) {
642
+ if (record.recordBytes > maxStoredValueBytes)
643
+ return yield* DoValueBoundExceeded.make({
644
+ actualBytes: record.recordBytes,
645
+ maxBytes: maxStoredValueBytes,
646
+ operation: "append canonical record",
596
647
  });
597
- }
598
- // Archives store the full batch as one value, so their ceiling also bounds hot appends.
599
- // The trusted append SPI pairs captured JSON with its exact UTF-8 width.
600
- yield* checkValueBound("append canonical batch", request.batchJson, request.batchBytes);
601
- for (const record of request.records) {
602
- if (record.recordBytes > maxStoredValueBytes)
603
- return yield* DoValueBoundExceeded.make({
604
- actualBytes: record.recordBytes,
605
- maxBytes: maxStoredValueBytes,
606
- operation: "append canonical record",
607
- });
608
- }
609
- for (const write of request.executionWrites)
610
- yield* checkValueBound("commit active execution", write.json, write.bytes);
648
+ }
649
+ for (const write of request.executionWrites)
650
+ yield* checkValueBound("commit active execution", write.json, write.bytes);
611
651
 
612
- const recordIds: Array<string> = request.records.map((record) => record.recordId);
652
+ const recordIds: Array<string> = request.records.map((record) => record.recordId);
613
653
 
614
- if (new Set(recordIds).size !== recordIds.length) {
615
- return yield* DoAppendConflict.make({
616
- message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
617
- reason: "record-identity",
654
+ if (new Set(recordIds).size !== recordIds.length) {
655
+ return yield* DoAppendConflict.make({
656
+ message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
657
+ reason: "record-identity",
658
+ });
659
+ }
660
+
661
+ return request;
662
+ });
663
+
664
+ // Replay and progress validation precede this body in the same owner transaction.
665
+ // A miss makes no native change; the ordinary path reuses any tentative batch.
666
+ // After advancing the frontier, failures must roll back rather than fall back.
667
+ const appendNativeBody = Effect.fnUntraced(function* (
668
+ request: RawAppendRequest,
669
+ first: CanonicalSequence,
670
+ last: CanonicalSequence,
671
+ batchInserted: boolean,
672
+ ) {
673
+ const db = nativeDb;
674
+
675
+ // Effectful statement transformers and exported SQL spans retain the ordinary path.
676
+ const ordinaryStatement = yield* Effect.withFiber((fiber) =>
677
+ Effect.succeed(
678
+ fiber.getRef(CurrentTransformer) !== undefined ||
679
+ (fiber.getRef(References.TracerEnabled) &&
680
+ fiber.getRef(Tracer.Tracer) !== Tracer.nativeTracer),
681
+ ),
682
+ );
683
+
684
+ if (ordinaryStatement) return false;
685
+ if (
686
+ db === undefined ||
687
+ request.executionWrites.length !== 1 ||
688
+ request.executionWrites[0]?.update.retire ||
689
+ request.executionWrites[0]?.update.facts.length !== 0
690
+ )
691
+ return false;
692
+
693
+ const workPlan = work.prepareClosedPair(
694
+ request.threadId,
695
+ first,
696
+ last,
697
+ request.records.map((record) => record.canonical),
698
+ );
699
+
700
+ if (workPlan === undefined) return false;
701
+
702
+ const native = yield* Effect.suspend(() => {
703
+ const query = (statement: Statement<object>, operation: string) =>
704
+ Result.try({
705
+ try: () => {
706
+ const [text, parameters] = statement.compile();
707
+
708
+ return db.exec(text, ...parameters).toArray();
709
+ },
710
+ catch: (cause) =>
711
+ storageError(operation)(
712
+ SqlError.make({
713
+ reason: classifySqliteError(cause, {
714
+ operation,
715
+ message: "Native canonical statement failed",
716
+ }),
717
+ }),
718
+ ),
618
719
  });
619
- }
620
720
 
621
- return request;
721
+ return Effect.fromResult(
722
+ Result.gen(function* () {
723
+ const captured = physical.cachedRecords(request.threadId, first, request.records);
724
+
725
+ if (captured === undefined) return undefined;
726
+ const physicalRecords = yield* captured;
727
+ const rows = yield* query(workPlan.advance, "advance native work frontier");
728
+
729
+ // A zero-row predicate miss has made no change. A successful update
730
+ // must never fall back: every later refusal rolls back this transaction.
731
+ if (rows.length === 0) return undefined;
732
+ const closed = yield* workPlan.fold();
733
+ const executionPlans = executions.prepareCommit(request);
734
+
735
+ if (!batchInserted)
736
+ yield* query(
737
+ sql`INSERT INTO effect_agent_canonical_batches
738
+ (thread_id, batch_id, first_sequence, last_sequence, producer_id_json)
739
+ VALUES (${request.threadId}, ${JSON.stringify(request.batchId)}, ${first}, ${last}, ${request.producerIdJson})`,
740
+ "insert canonical batch",
741
+ );
742
+
743
+ const inserted = yield* query(
744
+ sql`INSERT INTO effect_agent_canonical_records ${sql.insert(physicalRecords.map((row) => row.columns))}
745
+ ON CONFLICT(thread_id, record_id) DO NOTHING RETURNING 1`,
746
+ "insert canonical records",
747
+ );
748
+
749
+ if (inserted.length !== request.records.length)
750
+ return yield* Result.fail(
751
+ DoAppendConflict.make({
752
+ message: "A canonical record ID already exists.",
753
+ reason: "record-identity",
754
+ }),
755
+ );
756
+ const memberships = physicalRecords.flatMap((row) => row.memberships);
757
+
758
+ for (const group of chunked(memberships, Math.floor(MAX_BOUND_PARAMETERS / 3)))
759
+ yield* query(
760
+ sql`INSERT INTO effect_agent_record_runs ${sql.insert(group)}`,
761
+ "index canonical Run membership",
762
+ );
763
+ for (const pointer of closed.pointers) yield* query(pointer, "index canonical pointers");
764
+
765
+ for (const plan of executionPlans) {
766
+ const changed = yield* query(plan.statement, "commit active execution");
767
+ const accepted = plan.accept(changed);
768
+
769
+ if (Result.isFailure(accepted))
770
+ return yield* Result.fail(
771
+ DoAppendConflict.make({
772
+ reason: accepted.failure.reason,
773
+ message: "Active execution conflicts with the captured commit",
774
+ ...(accepted.failure.actualTailSequence === undefined
775
+ ? {}
776
+ : { actualTailSequence: accepted.failure.actualTailSequence }),
777
+ }),
778
+ );
779
+ }
780
+ for (let index = 0; index < request.records.length; index++) {
781
+ const record = request.records[index]!;
782
+
783
+ recordCache.put(
784
+ RecordRow.make({
785
+ ...physicalRecords[index]!.columns,
786
+ thread_id: request.threadId,
787
+ record_id: record.recordId,
788
+ batch_id: request.batchId,
789
+ record_json: record.recordJson,
790
+ sequence: CanonicalSequence.make(first + index),
791
+ }),
792
+ record.recordBytes,
793
+ );
794
+ }
795
+
796
+ return rows;
797
+ }),
798
+ );
622
799
  });
623
800
 
624
- const appendPrepared = Effect.fnUntraced(function* (
625
- request: RawAppendRequest,
626
- ): Effect.fn.Return<RawAppendResult, AppendError> {
627
- const recordIds: Array<string> = request.records.map((record) => record.recordId);
628
- const threadRows = yield* getThread(request.threadId);
801
+ if (native === undefined) return false;
802
+ yield* threads.write(Effect.succeed(native));
629
803
 
630
- const thread = threadRows.length === 1 ? threadRows[0] : undefined;
804
+ return true;
805
+ });
631
806
 
632
- if (thread === undefined)
633
- return yield* DoStorageCorruptionError.make({
634
- table: "effect_agent_threads",
635
- rowKey: request.threadId,
636
- message: `Expected exactly one row but found ${threadRows.length}.`,
637
- });
807
+ const appendPrepared = Effect.fnUntraced(function* (
808
+ request: RawAppendRequest,
809
+ ): Effect.fn.Return<AppendResult, AppendError> {
810
+ const threadRows = yield* getThread(request.threadId);
638
811
 
639
- const prefix = recordCache.prefix(request.threadId, thread.tail_sequence);
812
+ const thread = threadRows.length === 1 ? threadRows[0] : undefined;
640
813
 
814
+ if (thread === undefined)
815
+ return yield* DoStorageCorruptionError.make({
816
+ table: "effect_agent_threads",
817
+ rowKey: request.threadId,
818
+ message: `Expected exactly one row but found ${threadRows.length}.`,
819
+ });
820
+
821
+ // A fresh canonical batch key and absence of an active receipt prove the two
822
+ // negative replay lookups in the same statement that retains the batch header.
823
+ // Keep the read-first path for stale/overflowed coordinates so exact retries
824
+ // still precede sequence and tail refusals.
825
+ const canInsert =
826
+ request.records.length > 0 &&
827
+ request.expectedTailSequence === thread.tail_sequence &&
828
+ Number.isSafeInteger(thread.tail_sequence + request.records.length);
829
+
830
+ const inserted = canInsert
831
+ ? yield* sql`INSERT INTO effect_agent_canonical_batches
832
+ (thread_id, batch_id, first_sequence, last_sequence, producer_id_json)
833
+ SELECT ${request.threadId}, ${JSON.stringify(request.batchId)},
834
+ ${thread.tail_sequence + 1}, ${thread.tail_sequence + request.records.length},
835
+ ${request.producerIdJson}
836
+ WHERE NOT EXISTS (
837
+ SELECT 1 FROM effect_agent_executions
838
+ WHERE thread_id=${request.threadId} AND last_commit_id=${request.batchId}
839
+ )
840
+ ON CONFLICT(thread_id, batch_id) DO NOTHING RETURNING 1`.pipe(
841
+ Effect.mapError(storageError("insert canonical batch")),
842
+ )
843
+ : [];
844
+
845
+ if (inserted.length === 0) {
641
846
  const replay = yield* executionConflict(executions.replay(request));
642
847
 
643
- if (replay !== undefined) return RawAppendResult.make(replay);
848
+ if (replay !== undefined) return AppendResult.make(replay);
849
+
850
+ const prefix = recordCache.prefix(request.threadId, thread.tail_sequence);
644
851
 
645
852
  const existing =
646
853
  prefix !== undefined && !prefix.some((row) => row.batch_id === request.batchId)
@@ -655,94 +862,69 @@ const makeJournal = (
655
862
  });
656
863
  }
657
864
 
658
- return RawAppendResult.make({
865
+ return AppendResult.make({
659
866
  firstSequence: existing.first_sequence,
660
867
  lastSequence: existing.last_sequence,
661
868
  replayed: true,
662
869
  });
663
870
  }
871
+ if (canInsert)
872
+ return yield* DoStorageCorruptionError.make({
873
+ table: "effect_agent_canonical_batches",
874
+ rowKey: request.threadId,
875
+ message: "Batch insertion conflicted without a durable replay receipt.",
876
+ });
877
+ }
664
878
 
665
- yield* executionConflict(executions.rejectSuperseded(request));
879
+ yield* executionConflict(executions.rejectSuperseded(request));
666
880
 
667
- if (request.expectedTailSequence !== thread.tail_sequence) {
668
- return yield* DoAppendConflict.make({
669
- message:
670
- `Expected tail ${request.expectedTailSequence} ` + `but found ${thread.tail_sequence}.`,
671
- reason: "tail",
672
- actualTailSequence: thread.tail_sequence,
673
- });
674
- }
881
+ if (request.expectedTailSequence !== thread.tail_sequence) {
882
+ return yield* DoAppendConflict.make({
883
+ message:
884
+ `Expected tail ${request.expectedTailSequence} ` + `but found ${thread.tail_sequence}.`,
885
+ reason: "tail",
886
+ actualTailSequence: thread.tail_sequence,
887
+ });
888
+ }
675
889
 
676
- // Chunked to respect the Durable Object platform's 100-bound-parameter statement
677
- // limit: a batch may carry up to 256 records.
678
- const existingRecords: Array<RecordRow> =
679
- prefix === undefined ? [] : prefix.filter((row) => recordIds.includes(row.record_id));
890
+ const result = yield* Schema.decodeEffect(AppendResult)({
891
+ firstSequence: thread.tail_sequence + (request.records.length === 0 ? 0 : 1),
892
+ lastSequence: thread.tail_sequence + request.records.length,
893
+ replayed: false,
894
+ }).pipe(
895
+ Effect.mapError((error) =>
896
+ DoStorageError.make({
897
+ cause: error,
898
+ operation: "append canonical batch",
899
+ message: error.message,
900
+ }),
901
+ ),
902
+ );
680
903
 
681
- for (const chunk of prefix === undefined
682
- ? chunked(recordIds, MAX_BOUND_PARAMETERS - 10)
683
- : []) {
684
- const existingRecordRows = yield* sql<Record<string, unknown>>`
685
- SELECT
686
- thread_id,
687
- sequence,
688
- record_id,
689
- ${recordBatchId(sql)} AS batch_id,
690
- ${recordJson} AS record_json
691
- FROM effect_agent_canonical_records
692
- WHERE thread_id = ${request.threadId}
693
- AND record_id IN ${sql.in(chunk.map((id) => JSON.stringify(id)))}
694
- `.pipe(Effect.mapError(storageError("check canonical record identities")));
695
-
696
- existingRecords.push(
697
- ...(yield* decodeRows(
698
- Schema.Array(RecordRow),
699
- "effect_agent_canonical_records",
700
- `${request.threadId}/record_ids`,
701
- existingRecordRows,
702
- )),
703
- );
704
- }
705
- if (existingRecords.length > 0) {
706
- return yield* DoAppendConflict.make({
707
- message: `Canonical record ID ${existingRecords[0].record_id} already exists.`,
708
- reason: "record-identity",
709
- });
710
- }
904
+ const { firstSequence, lastSequence } = result;
711
905
 
712
- const firstSequence = yield* Schema.decodeEffect(CanonicalSequence)(
713
- thread.tail_sequence + (request.records.length === 0 ? 0 : 1),
714
- ).pipe(
715
- Effect.mapError((error) =>
716
- DoStorageError.make({
717
- cause: error,
718
- operation: "append canonical batch",
719
- message: error.message,
720
- }),
721
- ),
722
- );
906
+ yield* executionConflict(executions.validate(request));
723
907
 
724
- const lastSequence = yield* Schema.decodeEffect(CanonicalSequence)(
725
- thread.tail_sequence + request.records.length,
726
- ).pipe(
727
- Effect.mapError((error) =>
728
- DoStorageError.make({
729
- cause: error,
730
- operation: "append canonical batch",
731
- message: error.message,
732
- }),
908
+ if (yield* appendNativeBody(request, firstSequence, lastSequence, inserted.length > 0)) {
909
+ yield* progress.committed("canonical").pipe(
910
+ Effect.catchCause((cause) =>
911
+ Effect.failCause(
912
+ Cause.map(cause, (error) =>
913
+ DoStorageError.make({
914
+ operation: "enroll canonical progress",
915
+ message: error.message,
916
+ cause: error,
917
+ }),
918
+ ),
919
+ ),
733
920
  ),
734
921
  );
735
922
 
736
- yield* executionConflict(executions.validate(request));
737
-
738
- const result = RawAppendResult.make({
739
- firstSequence,
740
- lastSequence,
741
- replayed: false,
742
- });
923
+ return result;
924
+ }
743
925
 
744
- if (request.records.length > 0) {
745
- yield* sql`
926
+ if (request.records.length > 0 && inserted.length === 0) {
927
+ yield* sql`
746
928
  INSERT INTO effect_agent_canonical_batches (
747
929
  thread_id,
748
930
  batch_id,
@@ -757,162 +939,169 @@ const makeJournal = (
757
939
  ${request.producerIdJson}
758
940
  )
759
941
  `.pipe(Effect.mapError(storageError("insert canonical batch")));
760
- }
761
- yield* failpoint("append:after-batch-insert");
942
+ }
943
+ yield* failpoint("append:after-batch-insert");
762
944
 
763
- const physicalRecords = yield* physical.records(
764
- request.threadId,
765
- firstSequence,
766
- request.records,
767
- );
945
+ const physicalRecords = yield* physical.records(
946
+ request.threadId,
947
+ firstSequence,
948
+ request.records,
949
+ );
768
950
 
769
- const records = physicalRecords.map(({ columns, memberships }, index) => ({
770
- record: request.records[index]!,
771
- canonical: request.records[index]!.canonical,
772
- row: columns,
773
- memberships,
774
- }));
775
-
776
- // Captured scalar locators avoid parsing canonical JSON under the writer.
777
- // Eighteen bound columns per row stay within workerd's statement limit.
778
- for (const group of chunked(records, Math.floor(MAX_BOUND_PARAMETERS / 18))) {
779
- yield* sql`INSERT INTO effect_agent_canonical_records ${sql.insert(group.map(({ row }) => row))}`.pipe(
951
+ const records = physicalRecords.map(({ columns, memberships }, index) => ({
952
+ record: request.records[index]!,
953
+ canonical: request.records[index]!.canonical,
954
+ row: columns,
955
+ memberships,
956
+ }));
957
+
958
+ // Captured scalar locators avoid parsing canonical JSON under the writer.
959
+ // Eighteen bound columns per row stay within workerd's statement limit.
960
+ for (const group of chunked(records, Math.floor(MAX_BOUND_PARAMETERS / 18))) {
961
+ const inserted =
962
+ yield* sql`INSERT INTO effect_agent_canonical_records ${sql.insert(group.map(({ row }) => row))}
963
+ ON CONFLICT(thread_id, record_id) DO NOTHING RETURNING 1`.pipe(
780
964
  Effect.mapError(storageError("insert canonical records")),
781
965
  );
782
966
 
783
- const memberships = group.flatMap(({ memberships }) => memberships);
967
+ if (inserted.length !== group.length)
968
+ return yield* DoAppendConflict.make({
969
+ message: "A canonical record ID already exists.",
970
+ reason: "record-identity",
971
+ });
784
972
 
785
- for (const members of chunked(memberships, Math.floor(MAX_BOUND_PARAMETERS / 3)))
786
- yield* sql`INSERT INTO effect_agent_record_runs ${sql.insert(members)}`.pipe(
787
- Effect.mapError(storageError("index canonical Run membership")),
788
- );
789
- for (const { row, record } of group) {
790
- recordCache.put(
791
- RecordRow.make({
792
- ...row,
793
- record_id: record.recordId,
794
- batch_id: request.batchId,
795
- record_json: record.recordJson,
796
- sequence: Schema.decodeSync(CanonicalSequence)(row.sequence),
797
- }),
798
- record.recordBytes,
799
- );
800
- yield* failpoint("append:after-record-insert");
801
- }
802
- }
973
+ const memberships = group.flatMap(({ memberships }) => memberships);
803
974
 
804
- if (request.records.length > 0) {
805
- const workFields = yield* work
806
- .apply(
807
- yield* Schema.decodeEffect(ThreadId)(request.threadId).pipe(
808
- Effect.mapError(workFailure),
809
- ),
810
- firstSequence,
811
- request.records.map((record) => record.canonical),
812
- )
813
- .pipe(Effect.mapError(workFailure));
814
-
815
- yield* sql`UPDATE effect_agent_threads SET ${sql.update({ tail_sequence: lastSequence, ...workFields })}
816
- WHERE thread_id=${request.threadId} RETURNING *`.pipe(
817
- threads.write,
818
- Effect.mapError((error) =>
819
- error._tag === "SqlError" ? storageError("advance thread tail")(error) : error,
820
- ),
975
+ for (const members of chunked(memberships, Math.floor(MAX_BOUND_PARAMETERS / 3)))
976
+ yield* sql`INSERT INTO effect_agent_record_runs ${sql.insert(members)}`.pipe(
977
+ Effect.mapError(storageError("index canonical Run membership")),
821
978
  );
822
- }
823
- yield* executionConflict(executions.commit(request, result));
824
- if (request.executionWrites.length > 0) yield* failpoint("append:after-record-insert");
825
- if (
826
- lifecycle !== undefined &&
827
- request.executionWrites.some(({ update }) =>
828
- update.facts.some(({ payload }) => payload._tag === "RunStarted"),
829
- )
830
- )
831
- yield* flushCanonical(request.threadId).pipe(
832
- Effect.provideService(SqlLifecycleRetainer, { retainMany: lifecycle.retainMany }),
833
- Effect.mapError((cause) =>
834
- DoStorageError.make({
835
- operation: "retain lifecycle start prefix",
836
- message: "Lifecycle start intent could not be retained",
837
- cause,
838
- }),
839
- ),
979
+ for (const { row, record } of group) {
980
+ recordCache.put(
981
+ RecordRow.make({
982
+ ...row,
983
+ thread_id: request.threadId,
984
+ record_id: record.recordId,
985
+ batch_id: request.batchId,
986
+ record_json: record.recordJson,
987
+ sequence: Schema.decodeSync(CanonicalSequence)(row.sequence),
988
+ }),
989
+ record.recordBytes,
840
990
  );
841
- for (const { update } of request.executionWrites) {
842
- for (const fact of update.facts) {
843
- if (
844
- lifecycle !== undefined &&
845
- Schema.is(Schema.toType(LifecyclePublicationFact))(fact.payload)
846
- )
847
- yield* lifecycle
848
- .retain({
849
- id: JSON.stringify([request.threadId, "record", fact.recordId]),
850
- ownerThreadId: request.threadId,
851
- canonicalSequence: lastSequence,
852
- createdAt: fact.createdAt,
853
- fact: fact.payload,
854
- })
855
- .pipe(
856
- Effect.mapError((cause) =>
857
- DoStorageError.make({
858
- operation: "retain operational lifecycle",
859
- message: cause.message,
860
- cause,
861
- }),
862
- ),
863
- );
864
- }
991
+ yield* failpoint("append:after-record-insert");
865
992
  }
866
- yield* failpoint("append:after-tail-update");
993
+ }
867
994
 
868
- // Retain one start prefix atomically with its canonical proof, before model/tool
869
- // execution. Later facts stay journal-backed until the settlement publication wave.
870
- if (
871
- lifecycle !== undefined &&
872
- records.some(
873
- ({ canonical: { payload } }) =>
874
- payload._tag === "RunStarted" || payload._tag === "SubagentStarted",
995
+ if (request.records.length > 0) {
996
+ const workFields = yield* work
997
+ .apply(
998
+ yield* Schema.decodeEffect(ThreadId)(request.threadId).pipe(Effect.mapError(workFailure)),
999
+ firstSequence,
1000
+ request.records.map((record) => record.canonical),
1001
+ )
1002
+ .pipe(Effect.mapError(workFailure));
1003
+
1004
+ yield* sql`UPDATE effect_agent_threads SET ${sql.update({ tail_sequence: lastSequence, ...workFields })}
1005
+ WHERE thread_id=${request.threadId} RETURNING *`.pipe(
1006
+ threads.write,
1007
+ Effect.mapError((error) =>
1008
+ error._tag === "SqlError" ? storageError("advance thread tail")(error) : error,
1009
+ ),
1010
+ );
1011
+ }
1012
+ yield* executionConflict(executions.commit(request, result));
1013
+ if (request.executionWrites.length > 0) yield* failpoint("append:after-record-insert");
1014
+ if (
1015
+ lifecycle !== undefined &&
1016
+ request.executionWrites.some(({ update }) =>
1017
+ update.facts.some(({ payload }) => payload._tag === "RunStarted"),
1018
+ )
1019
+ )
1020
+ yield* flushCanonical(request.threadId).pipe(
1021
+ Effect.provideService(SqlLifecycleRetainer, { retainMany: lifecycle.retainMany }),
1022
+ Effect.mapError((cause) =>
1023
+ DoStorageError.make({
1024
+ operation: "retain lifecycle start prefix",
1025
+ message: "Lifecycle start intent could not be retained",
1026
+ cause,
1027
+ }),
1028
+ ),
1029
+ );
1030
+ for (const { update } of request.executionWrites) {
1031
+ for (const fact of update.facts) {
1032
+ if (
1033
+ lifecycle !== undefined &&
1034
+ Schema.is(Schema.toType(LifecyclePublicationFact))(fact.payload)
875
1035
  )
1036
+ yield* lifecycle
1037
+ .retain({
1038
+ id: JSON.stringify([request.threadId, "record", fact.recordId]),
1039
+ ownerThreadId: request.threadId,
1040
+ canonicalSequence: lastSequence,
1041
+ createdAt: fact.createdAt,
1042
+ fact: fact.payload,
1043
+ })
1044
+ .pipe(
1045
+ Effect.mapError((cause) =>
1046
+ DoStorageError.make({
1047
+ operation: "retain operational lifecycle",
1048
+ message: cause.message,
1049
+ cause,
1050
+ }),
1051
+ ),
1052
+ );
1053
+ }
1054
+ }
1055
+ yield* failpoint("append:after-tail-update");
1056
+
1057
+ // Retain one start prefix atomically with its canonical proof, before model/tool
1058
+ // execution. Later facts stay journal-backed until the settlement publication wave.
1059
+ if (
1060
+ lifecycle !== undefined &&
1061
+ records.some(
1062
+ ({ canonical: { payload } }) =>
1063
+ payload._tag === "RunStarted" || payload._tag === "SubagentStarted",
876
1064
  )
877
- yield* flushCanonical(request.threadId).pipe(
878
- Effect.provideService(SqlLifecycleRetainer, { retainMany: lifecycle.retainMany }),
879
- Effect.mapError((cause) =>
1065
+ )
1066
+ yield* flushCanonical(request.threadId).pipe(
1067
+ Effect.provideService(SqlLifecycleRetainer, { retainMany: lifecycle.retainMany }),
1068
+ Effect.mapError((cause) =>
1069
+ DoStorageError.make({
1070
+ operation: "retain lifecycle start prefix",
1071
+ message: "Lifecycle start intent could not be retained",
1072
+ cause,
1073
+ }),
1074
+ ),
1075
+ );
1076
+
1077
+ yield* progress.committed("canonical").pipe(
1078
+ Effect.catchCause((cause) =>
1079
+ Effect.failCause(
1080
+ Cause.map(cause, (error) =>
880
1081
  DoStorageError.make({
881
- operation: "retain lifecycle start prefix",
882
- message: "Lifecycle start intent could not be retained",
883
- cause,
1082
+ operation: "enroll canonical progress",
1083
+ message: error.message,
1084
+ cause: error,
884
1085
  }),
885
1086
  ),
886
- );
887
-
888
- yield* progress.committed("canonical").pipe(
889
- Effect.catchCause((cause) =>
890
- Effect.failCause(
891
- Cause.map(cause, (error) =>
892
- DoStorageError.make({
893
- operation: "enroll canonical progress",
894
- message: error.message,
895
- cause: error,
896
- }),
897
- ),
898
- ),
899
1087
  ),
900
- );
1088
+ ),
1089
+ );
901
1090
 
902
- return result;
903
- });
1091
+ return result;
1092
+ });
904
1093
 
905
- const append = Effect.fnUntraced(
906
- function* (
907
- request: RawAppendRequest,
908
- observed: ThreadRow,
909
- ): Effect.fn.Return<RawAppendResult, AppendError> {
910
- yield* prepareAppend(request);
911
- // The facade already read this tail. Reject a known stale writer without opening a
912
- // transaction; a matching tail is still checked atomically below. Replays precede tail
913
- // conflicts, preserving retry identity even after the log has advanced.
914
- if (request.records.length > 0 && request.expectedTailSequence !== observed.tail_sequence) {
915
- const batches = yield* sql`
1094
+ const append = Effect.fnUntraced(
1095
+ function* (
1096
+ request: RawAppendRequest,
1097
+ observed: ThreadRow,
1098
+ ): Effect.fn.Return<AppendResult, AppendError> {
1099
+ yield* prepareAppend(request);
1100
+ // The facade already read this tail. Reject a known stale writer without opening a
1101
+ // transaction; a matching tail is still checked atomically below. Replays precede tail
1102
+ // conflicts, preserving retry identity even after the log has advanced.
1103
+ if (request.records.length > 0 && request.expectedTailSequence !== observed.tail_sequence) {
1104
+ const batches = yield* sql`
916
1105
  SELECT batch_id FROM effect_agent_canonical_batches
917
1106
  WHERE thread_id = ${request.threadId} AND batch_id = ${JSON.stringify(request.batchId)}
918
1107
  UNION ALL SELECT last_commit_id FROM effect_agent_executions
@@ -920,47 +1109,46 @@ const makeJournal = (
920
1109
  LIMIT 1
921
1110
  `.pipe(Effect.mapError(storageError("preflight append replay")));
922
1111
 
923
- // A possible replay still checks exact content in the transaction.
924
- if (batches.length === 0) {
925
- return yield* DoAppendConflict.make({
926
- reason: "tail",
927
- message: "The canonical tail has advanced.",
928
- actualTailSequence: observed.tail_sequence,
929
- });
930
- }
1112
+ // A possible replay still checks exact content in the transaction.
1113
+ if (batches.length === 0) {
1114
+ return yield* DoAppendConflict.make({
1115
+ reason: "tail",
1116
+ message: "The canonical tail has advanced.",
1117
+ actualTailSequence: observed.tail_sequence,
1118
+ });
931
1119
  }
1120
+ }
932
1121
 
933
- return yield* withWriteTransaction(
934
- "append transaction",
935
- isAppendContention,
936
- )(appendPrepared(request));
937
- },
938
- withStorageSpan("DoJournal.append", isAppendContention),
939
- );
1122
+ // Receipt arbitration may have tentatively inserted a batch. Even typed
1123
+ // contention must roll back before leaving the owner mutation gate.
1124
+ return yield* withWriteTransaction("append transaction")(appendPrepared(request));
1125
+ },
1126
+ withStorageSpan("DoJournal.append", isAppendContention),
1127
+ );
940
1128
 
941
- const read = Effect.fnUntraced(function* (request: RawReadRequest) {
942
- const thread = (yield* getThread(request.threadId))[0];
1129
+ const read = Effect.fnUntraced(function* (request: RawReadRequest) {
1130
+ const thread = (yield* getThread(request.threadId))[0];
943
1131
 
944
- const through = Math.min(
945
- thread?.tail_sequence ?? 0,
946
- request.fromSequenceExclusive + request.limit,
947
- );
1132
+ const through = Math.min(
1133
+ thread?.tail_sequence ?? 0,
1134
+ request.fromSequenceExclusive + request.limit,
1135
+ );
948
1136
 
949
- const cached: Array<RecordRow> = [];
1137
+ const cached: Array<RecordRow> = [];
950
1138
 
951
- for (let sequence = request.fromSequenceExclusive + 1; sequence <= through; sequence++) {
952
- const record = recordCache.get(request.threadId, sequence);
1139
+ for (let sequence = request.fromSequenceExclusive + 1; sequence <= through; sequence++) {
1140
+ const record = recordCache.get(request.threadId, sequence);
953
1141
 
954
- if (record === undefined) break;
955
- cached.push(record);
956
- }
957
- if (cached.length === Math.max(0, through - request.fromSequenceExclusive)) {
958
- return { count: cached.length, records: Stream.fromIterable(cached) };
959
- }
1142
+ if (record === undefined) break;
1143
+ cached.push(record);
1144
+ }
1145
+ if (cached.length === Math.max(0, through - request.fromSequenceExclusive)) {
1146
+ return { count: cached.length, records: Stream.fromIterable(cached) };
1147
+ }
960
1148
 
961
- // Capture membership without retaining payloads. Append-only sequences keep each later
962
- // payload query inside this snapshot, even when new records arrive during consumption.
963
- const planRows = yield* sql<Record<string, unknown>>`
1149
+ // Capture membership without retaining payloads. Append-only sequences keep each later
1150
+ // payload query inside this snapshot, even when new records arrive during consumption.
1151
+ const planRows = yield* sql<Record<string, unknown>>`
964
1152
  SELECT
965
1153
  sequence,
966
1154
  length(CAST(${recordJson} AS BLOB)) AS record_json_bytes
@@ -971,44 +1159,43 @@ const makeJournal = (
971
1159
  LIMIT ${request.limit}
972
1160
  `.pipe(Effect.mapError(storageError("read canonical records")));
973
1161
 
974
- const plan = yield* decodeRows(
975
- Schema.Array(ReadPlanRow),
976
- "effect_agent_canonical_records",
977
- `${request.threadId}>${request.fromSequenceExclusive}`,
978
- planRows,
979
- "ReadPlanRow",
980
- );
1162
+ const plan = yield* decodeRows(
1163
+ Schema.Array(ReadPlanRow),
1164
+ "effect_agent_canonical_records",
1165
+ `${request.threadId}>${request.fromSequenceExclusive}`,
1166
+ planRows,
1167
+ "ReadPlanRow",
1168
+ );
981
1169
 
982
- const mismatch = () =>
983
- DoStorageCorruptionError.make({
984
- table: "effect_agent_canonical_records",
985
- rowKey: `${request.threadId}>${request.fromSequenceExclusive}`,
986
- message:
987
- "Canonical read membership, sequence, or payload size changed from its read plan.",
988
- });
1170
+ const mismatch = () =>
1171
+ DoStorageCorruptionError.make({
1172
+ table: "effect_agent_canonical_records",
1173
+ rowKey: `${request.threadId}>${request.fromSequenceExclusive}`,
1174
+ message: "Canonical read membership, sequence, or payload size changed from its read plan.",
1175
+ });
989
1176
 
990
- if (plan.length > request.limit) return yield* mismatch();
991
-
992
- const pages: Array<ReadPage> = [];
993
- let page: ReadPage | undefined;
994
- let pageBytes = 0;
995
- let previousSequence = request.fromSequenceExclusive;
996
-
997
- for (const row of plan) {
998
- if (row.sequence !== previousSequence + 1) return yield* mismatch();
999
- previousSequence = row.sequence;
1000
- if (page === undefined || pageBytes + row.record_json_bytes > MAX_READ_PAGE_JSON_BYTES) {
1001
- page = [row];
1002
- pages.push(page);
1003
- pageBytes = row.record_json_bytes;
1004
- } else {
1005
- page.push(row);
1006
- pageBytes += row.record_json_bytes;
1007
- }
1177
+ if (plan.length > request.limit) return yield* mismatch();
1178
+
1179
+ const pages: Array<ReadPage> = [];
1180
+ let page: ReadPage | undefined;
1181
+ let pageBytes = 0;
1182
+ let previousSequence = request.fromSequenceExclusive;
1183
+
1184
+ for (const row of plan) {
1185
+ if (row.sequence !== previousSequence + 1) return yield* mismatch();
1186
+ previousSequence = row.sequence;
1187
+ if (page === undefined || pageBytes + row.record_json_bytes > MAX_READ_PAGE_JSON_BYTES) {
1188
+ page = [row];
1189
+ pages.push(page);
1190
+ pageBytes = row.record_json_bytes;
1191
+ } else {
1192
+ page.push(row);
1193
+ pageBytes += row.record_json_bytes;
1008
1194
  }
1195
+ }
1009
1196
 
1010
- const readPage = Effect.fnUntraced(function* (page: ReadPage) {
1011
- const rows = yield* sql<Record<string, unknown>>`
1197
+ const readPage = Effect.fnUntraced(function* (page: ReadPage) {
1198
+ const rows = yield* sql<Record<string, unknown>>`
1012
1199
  SELECT thread_id, sequence, record_id, ${recordBatchId(sql)} AS batch_id, ${recordJson} AS record_json
1013
1200
  FROM effect_agent_canonical_records
1014
1201
  WHERE thread_id = ${request.threadId}
@@ -1017,77 +1204,77 @@ const makeJournal = (
1017
1204
  ORDER BY sequence
1018
1205
  `.pipe(Effect.mapError(storageError("read canonical records")));
1019
1206
 
1020
- const decoded = yield* decodeRows(
1021
- Schema.Array(RecordRow),
1022
- "effect_agent_canonical_records",
1023
- `${request.threadId}/${page[0].sequence}`,
1024
- rows,
1025
- "RecordRow",
1026
- );
1027
-
1028
- if (
1029
- decoded.length !== page.length ||
1030
- decoded.some(
1031
- (row, index) =>
1032
- row.thread_id !== request.threadId ||
1033
- row.sequence !== page[index].sequence ||
1034
- storedTextBytes(row.record_json) !== page[index].record_json_bytes,
1035
- )
1036
- ) {
1037
- return yield* mismatch();
1038
- }
1207
+ const decoded = yield* decodeRows(
1208
+ Schema.Array(RecordRow),
1209
+ "effect_agent_canonical_records",
1210
+ `${request.threadId}/${page[0].sequence}`,
1211
+ rows,
1212
+ "RecordRow",
1213
+ );
1039
1214
 
1040
- for (const row of decoded) recordCache.put(row);
1215
+ if (
1216
+ decoded.length !== page.length ||
1217
+ decoded.some(
1218
+ (row, index) =>
1219
+ row.thread_id !== request.threadId ||
1220
+ row.sequence !== page[index].sequence ||
1221
+ storedTextBytes(row.record_json) !== page[index].record_json_bytes,
1222
+ )
1223
+ ) {
1224
+ return yield* mismatch();
1225
+ }
1041
1226
 
1042
- return decoded;
1043
- });
1227
+ for (const row of decoded) recordCache.put(row);
1044
1228
 
1045
- return {
1046
- count: plan.length,
1047
- records: Stream.fromIterable(pages).pipe(
1048
- Stream.flatMap((page) => Stream.fromIterableEffect(state.read(readPage(page)))),
1049
- ),
1050
- };
1229
+ return decoded;
1051
1230
  });
1052
1231
 
1053
- const saveCheckpoint = Effect.fnUntraced(function* (
1054
- checkpoint: RawCheckpoint,
1055
- ): Effect.fn.Return<void, CheckpointError> {
1056
- if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
1057
- return yield* DoStorageError.make({
1058
- operation: "save checkpoint",
1059
- message: "Checkpoint identity exceeds the Durable Object storage bounds.",
1060
- });
1061
- }
1062
- yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
1063
- yield* withWriteTransaction("checkpoint transaction")(
1064
- Effect.gen(function* () {
1065
- const threadRows = yield* getThread(checkpoint.threadId);
1232
+ return {
1233
+ count: plan.length,
1234
+ records: Stream.fromIterable(pages).pipe(
1235
+ Stream.flatMap((page) => Stream.fromIterableEffect(state.read(readPage(page)))),
1236
+ ),
1237
+ };
1238
+ });
1066
1239
 
1067
- const thread = yield* decodeSingleRow(
1068
- Schema.Array(ThreadRow),
1069
- "effect_agent_threads",
1070
- checkpoint.threadId,
1071
- threadRows,
1072
- );
1240
+ const saveCheckpoint = Effect.fnUntraced(function* (
1241
+ checkpoint: RawCheckpoint,
1242
+ ): Effect.fn.Return<void, CheckpointError> {
1243
+ if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
1244
+ return yield* DoStorageError.make({
1245
+ operation: "save checkpoint",
1246
+ message: "Checkpoint identity exceeds the Durable Object storage bounds.",
1247
+ });
1248
+ }
1249
+ yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
1250
+ yield* withWriteTransaction("checkpoint transaction")(
1251
+ Effect.gen(function* () {
1252
+ const threadRows = yield* getThread(checkpoint.threadId);
1253
+
1254
+ const thread = yield* decodeSingleRow(
1255
+ Schema.Array(ThreadRow),
1256
+ "effect_agent_threads",
1257
+ checkpoint.threadId,
1258
+ threadRows,
1259
+ );
1073
1260
 
1074
- if (checkpoint.throughSequence > thread.tail_sequence) {
1075
- return yield* DoCheckpointConflict.make({
1076
- reason: "ahead-of-tail",
1077
- message:
1078
- `Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
1079
- `${thread.tail_sequence}.`,
1080
- });
1081
- }
1261
+ if (checkpoint.throughSequence > thread.tail_sequence) {
1262
+ return yield* DoCheckpointConflict.make({
1263
+ reason: "ahead-of-tail",
1264
+ message:
1265
+ `Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
1266
+ `${thread.tail_sequence}.`,
1267
+ });
1268
+ }
1082
1269
 
1083
- if (!(yield* isBatchBoundary(checkpoint.threadId, checkpoint.throughSequence))) {
1084
- return yield* DoCheckpointConflict.make({
1085
- reason: "not-batch-boundary",
1086
- message: "Checkpoint sequence is not the end of a complete canonical batch.",
1087
- });
1088
- }
1270
+ if (!(yield* isBatchBoundary(checkpoint.threadId, checkpoint.throughSequence))) {
1271
+ return yield* DoCheckpointConflict.make({
1272
+ reason: "not-batch-boundary",
1273
+ message: "Checkpoint sequence is not the end of a complete canonical batch.",
1274
+ });
1275
+ }
1089
1276
 
1090
- const checkpointRows = yield* sql<Record<string, unknown>>`
1277
+ const checkpointRows = yield* sql<Record<string, unknown>>`
1091
1278
  SELECT
1092
1279
  thread_id,
1093
1280
  through_sequence,
@@ -1097,32 +1284,32 @@ const makeJournal = (
1097
1284
  AND through_sequence = ${checkpoint.throughSequence}
1098
1285
  `.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
1099
1286
 
1100
- const existing = yield* decodeRows(
1101
- Schema.Array(CheckpointRow),
1102
- "effect_agent_checkpoints",
1103
- `${checkpoint.threadId}/${checkpoint.throughSequence}`,
1104
- checkpointRows,
1105
- );
1287
+ const existing = yield* decodeRows(
1288
+ Schema.Array(CheckpointRow),
1289
+ "effect_agent_checkpoints",
1290
+ `${checkpoint.threadId}/${checkpoint.throughSequence}`,
1291
+ checkpointRows,
1292
+ );
1106
1293
 
1107
- if (existing.length > 1) {
1108
- return yield* DoStorageCorruptionError.make({
1109
- table: "effect_agent_checkpoints",
1110
- rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
1111
- message: "A checkpoint primary key returned more than one row.",
1294
+ if (existing.length > 1) {
1295
+ return yield* DoStorageCorruptionError.make({
1296
+ table: "effect_agent_checkpoints",
1297
+ rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
1298
+ message: "A checkpoint primary key returned more than one row.",
1299
+ });
1300
+ }
1301
+ if (existing.length === 1) {
1302
+ if (existing[0].checkpoint_json !== checkpoint.checkpointJson) {
1303
+ return yield* DoCheckpointConflict.make({
1304
+ reason: "corrupt",
1305
+ message: "A different checkpoint already exists at this canonical sequence.",
1112
1306
  });
1113
1307
  }
1114
- if (existing.length === 1) {
1115
- if (existing[0].checkpoint_json !== checkpoint.checkpointJson) {
1116
- return yield* DoCheckpointConflict.make({
1117
- reason: "corrupt",
1118
- message: "A different checkpoint already exists at this canonical sequence.",
1119
- });
1120
- }
1121
1308
 
1122
- return;
1123
- }
1309
+ return;
1310
+ }
1124
1311
 
1125
- yield* sql`
1312
+ yield* sql`
1126
1313
  INSERT INTO effect_agent_checkpoints (
1127
1314
  thread_id,
1128
1315
  through_sequence,
@@ -1133,15 +1320,15 @@ const makeJournal = (
1133
1320
  ${checkpoint.checkpointJson}
1134
1321
  )
1135
1322
  `.pipe(Effect.mapError(storageError("insert checkpoint")));
1136
- }),
1137
- );
1138
- });
1323
+ }),
1324
+ );
1325
+ });
1139
1326
 
1140
- const loadCheckpoint = Effect.fnUntraced(function* (
1141
- threadId: string,
1142
- atOrBeforeSequence: CanonicalSequence,
1143
- ) {
1144
- const rows = yield* sql<Record<string, unknown>>`
1327
+ const loadCheckpoint = Effect.fnUntraced(function* (
1328
+ threadId: string,
1329
+ atOrBeforeSequence: CanonicalSequence,
1330
+ ) {
1331
+ const rows = yield* sql<Record<string, unknown>>`
1145
1332
  SELECT
1146
1333
  thread_id,
1147
1334
  through_sequence,
@@ -1153,21 +1340,21 @@ const makeJournal = (
1153
1340
  LIMIT 1
1154
1341
  `.pipe(Effect.mapError(storageError("load checkpoint")));
1155
1342
 
1156
- return yield* decodeRows(
1157
- Schema.Array(CheckpointRow),
1158
- "effect_agent_checkpoints",
1159
- `${threadId}<=${atOrBeforeSequence}`,
1160
- rows,
1161
- );
1162
- });
1343
+ return yield* decodeRows(
1344
+ Schema.Array(CheckpointRow),
1345
+ "effect_agent_checkpoints",
1346
+ `${threadId}<=${atOrBeforeSequence}`,
1347
+ rows,
1348
+ );
1349
+ });
1163
1350
 
1164
- const isBatchBoundary = Effect.fnUntraced(function* (
1165
- threadId: string,
1166
- sequence: CanonicalSequence,
1167
- ) {
1168
- if (sequence === 0) return true;
1351
+ const isBatchBoundary = Effect.fnUntraced(function* (
1352
+ threadId: string,
1353
+ sequence: CanonicalSequence,
1354
+ ) {
1355
+ if (sequence === 0) return true;
1169
1356
 
1170
- const rows = yield* sql<Record<string, unknown>>`
1357
+ const rows = yield* sql<Record<string, unknown>>`
1171
1358
  SELECT last_sequence FROM effect_agent_canonical_batches
1172
1359
  WHERE thread_id = ${threadId}
1173
1360
  AND batch_id = (
@@ -1177,207 +1364,207 @@ const makeJournal = (
1177
1364
  AND last_sequence = ${sequence}
1178
1365
  `.pipe(Effect.mapError(storageError("read canonical batch boundary")));
1179
1366
 
1180
- const batches = yield* decodeRows(
1181
- Schema.Array(Schema.Struct({ last_sequence: CanonicalSequence })),
1182
- "effect_agent_canonical_batches",
1183
- `${threadId}/${sequence}`,
1184
- rows,
1185
- );
1367
+ const batches = yield* decodeRows(
1368
+ Schema.Array(Schema.Struct({ last_sequence: CanonicalSequence })),
1369
+ "effect_agent_canonical_batches",
1370
+ `${threadId}/${sequence}`,
1371
+ rows,
1372
+ );
1186
1373
 
1187
- return batches.length === 1;
1188
- });
1374
+ return batches.length === 1;
1375
+ });
1189
1376
 
1190
- const sourceFailure = (cause: unknown) =>
1191
- LifecyclePublicationError.make({ reason: "unavailable", cause });
1377
+ const sourceFailure = (cause: unknown) =>
1378
+ LifecyclePublicationError.make({ reason: "unavailable", cause });
1192
1379
 
1193
- const sourceThreads = Effect.fnUntraced(function* () {
1194
- const all = yield* threads.matching(
1195
- "lifecycle-owners",
1196
- (row) => row.created_at !== null,
1197
- sql`SELECT * FROM effect_agent_threads WHERE created_at IS NOT NULL LIMIT 129`,
1198
- 128,
1199
- );
1380
+ const sourceThreads = Effect.fnUntraced(function* () {
1381
+ const all = yield* threads.matching(
1382
+ "lifecycle-owners",
1383
+ (row) => row.created_at !== null,
1384
+ sql`SELECT * FROM effect_agent_threads WHERE created_at IS NOT NULL LIMIT 129`,
1385
+ 128,
1386
+ );
1200
1387
 
1201
- if (all.length <= 128) return all;
1388
+ if (all.length <= 128) return all;
1202
1389
 
1203
- // Large custom hosts hydrate only owners whose journal has an unmaterialized suffix.
1204
- return yield* Schema.decodeUnknownEffect(Schema.Array(ThreadRow))(
1205
- yield* sql`SELECT t.* FROM effect_agent_threads t LEFT JOIN effect_agent_lifecycle_cursors c ON c.thread_id=t.thread_id
1390
+ // Large custom hosts hydrate only owners whose journal has an unmaterialized suffix.
1391
+ return yield* Schema.decodeUnknownEffect(Schema.Array(ThreadRow))(
1392
+ yield* sql`SELECT t.* FROM effect_agent_threads t LEFT JOIN effect_agent_lifecycle_cursors c ON c.thread_id=t.thread_id
1206
1393
  WHERE t.created_at IS NOT NULL AND (c.thread_id IS NULL OR t.tail_sequence > c.through_sequence) ORDER BY t.thread_id LIMIT 128`,
1207
- );
1208
- });
1394
+ );
1395
+ });
1209
1396
 
1210
- const initializeLifecycleSource = Effect.fnUntraced(function* () {
1211
- if (initializedLifecycleSources.has(state)) return;
1212
- yield* state
1213
- .transaction(
1214
- Effect.gen(function* () {
1215
- const existing =
1216
- yield* sql`SELECT name FROM sqlite_master WHERE name='effect_agent_lifecycle_cursors'`;
1397
+ const initializeLifecycleSource = Effect.fnUntraced(function* () {
1398
+ if (initializedLifecycleSources.has(state)) return;
1399
+ yield* state
1400
+ .transaction(
1401
+ Effect.gen(function* () {
1402
+ const existing =
1403
+ yield* sql`SELECT name FROM sqlite_master WHERE name='effect_agent_lifecycle_cursors'`;
1217
1404
 
1218
- if (existing.length === 0) {
1219
- yield* sql`CREATE TABLE effect_agent_lifecycle_cursors (
1405
+ if (existing.length === 0) {
1406
+ yield* sql`CREATE TABLE effect_agent_lifecycle_cursors (
1220
1407
  thread_id TEXT PRIMARY KEY, through_sequence INTEGER NOT NULL
1221
1408
  )`;
1222
- // Existing inline writers already retained their facts. Enabling publication on an
1223
- // old store starts here too; it does not publish historical facts retroactively.
1224
- }
1225
- // New Threads created with publication disabled have no cursor. Never advance
1226
- // existing cursors: they can owe intent from an interrupted Attempt.
1227
- yield* sql`INSERT INTO effect_agent_lifecycle_cursors(thread_id, through_sequence)
1409
+ // Existing inline writers already retained their facts. Enabling publication on an
1410
+ // old store starts here too; it does not publish historical facts retroactively.
1411
+ }
1412
+ // New Threads created with publication disabled have no cursor. Never advance
1413
+ // existing cursors: they can owe intent from an interrupted Attempt.
1414
+ yield* sql`INSERT INTO effect_agent_lifecycle_cursors(thread_id, through_sequence)
1228
1415
  SELECT thread_id, tail_sequence FROM effect_agent_threads WHERE created_at IS NOT NULL
1229
1416
  ON CONFLICT(thread_id) DO NOTHING`;
1230
- yield* sourceThreads();
1417
+ yield* sourceThreads();
1418
+ }),
1419
+ )
1420
+ .pipe(
1421
+ Effect.provideService(SqlClient.SqlClient, sql),
1422
+ Effect.mapError((cause) =>
1423
+ DoStorageError.make({
1424
+ operation: "initialize lifecycle source",
1425
+ message: "Lifecycle source cursor unavailable",
1426
+ cause,
1231
1427
  }),
1232
- )
1233
- .pipe(
1234
- Effect.provideService(SqlClient.SqlClient, sql),
1235
- Effect.mapError((cause) =>
1236
- DoStorageError.make({
1237
- operation: "initialize lifecycle source",
1238
- message: "Lifecycle source cursor unavailable",
1239
- cause,
1240
- }),
1241
- ),
1242
- );
1243
- initializedLifecycleSources.add(state);
1244
- });
1245
-
1246
- /** The journal is durable intent; only the bounded publication wave materializes a write set. */
1247
- const flushCanonical = Effect.fnUntraced(function* (threadId: string, maxRecords = Infinity) {
1248
- const { retainMany } = yield* SqlLifecycleRetainer;
1249
- const thread = (yield* getThread(threadId))[0];
1250
-
1251
- if (thread === undefined) return;
1252
- const cursor = (yield* cursors.by("thread_id", threadId))[0];
1428
+ ),
1429
+ );
1430
+ initializedLifecycleSources.add(state);
1431
+ });
1253
1432
 
1254
- if (cursor === undefined || cursor.through_sequence > thread.tail_sequence)
1255
- return yield* LifecyclePublicationError.make({ reason: "corrupt" });
1256
- let through = cursor.through_sequence;
1257
- const bound = Math.min(thread.tail_sequence, through + maxRecords);
1258
-
1259
- while (through < bound) {
1260
- const page = yield* read(
1261
- RawReadRequest.make({
1262
- threadId,
1263
- fromSequenceExclusive: through,
1264
- limit: Math.min(128, bound - through),
1265
- }),
1266
- );
1433
+ /** The journal is durable intent; only the bounded publication wave materializes a write set. */
1434
+ const flushCanonical = Effect.fnUntraced(function* (threadId: string, maxRecords = Infinity) {
1435
+ const { retainMany } = yield* SqlLifecycleRetainer;
1436
+ const thread = (yield* getThread(threadId))[0];
1437
+
1438
+ if (thread === undefined) return;
1439
+ const cursor = (yield* cursors.by("thread_id", threadId))[0];
1440
+
1441
+ if (cursor === undefined || cursor.through_sequence > thread.tail_sequence)
1442
+ return yield* LifecyclePublicationError.make({ reason: "corrupt" });
1443
+ let through = cursor.through_sequence;
1444
+ const bound = Math.min(thread.tail_sequence, through + maxRecords);
1445
+
1446
+ while (through < bound) {
1447
+ const page = yield* read(
1448
+ RawReadRequest.make({
1449
+ threadId,
1450
+ fromSequenceExclusive: through,
1451
+ limit: Math.min(128, bound - through),
1452
+ }),
1453
+ );
1267
1454
 
1268
- const facts: Array<Parameters<SqlLifecycleRetainMany>[0][number]> = [];
1269
- let bytes = 0;
1455
+ const facts: Array<Parameters<SqlLifecycleRetainMany>[0][number]> = [];
1456
+ let bytes = 0;
1270
1457
 
1271
- yield* Stream.runForEach(page.records, (row) =>
1272
- Effect.gen(function* () {
1273
- const record = yield* Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(
1274
- row.record_json,
1275
- );
1458
+ yield* Stream.runForEach(page.records, (row) =>
1459
+ Effect.gen(function* () {
1460
+ const record = yield* Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(
1461
+ row.record_json,
1462
+ );
1276
1463
 
1277
- if (record.recordId !== row.record_id || row.sequence !== through + 1)
1278
- return yield* LifecyclePublicationError.make({ reason: "corrupt" });
1279
- if (Schema.is(Schema.toType(LifecyclePublicationFact))(record.payload)) {
1280
- if (
1281
- bytes + storedTextBytes(row.record_json) > MAX_READ_PAGE_JSON_BYTES &&
1282
- facts.length > 0
1283
- ) {
1284
- yield* retainMany(facts);
1285
- facts.length = 0;
1286
- bytes = 0;
1287
- }
1288
- facts.push({
1289
- id: JSON.stringify([threadId, "record", record.recordId]),
1290
- ownerThreadId: yield* Schema.decodeEffect(ThreadId)(threadId),
1291
- canonicalSequence: row.sequence,
1292
- createdAt: record.createdAt,
1293
- fact: record.payload,
1294
- });
1295
- bytes += storedTextBytes(row.record_json);
1464
+ if (record.recordId !== row.record_id || row.sequence !== through + 1)
1465
+ return yield* LifecyclePublicationError.make({ reason: "corrupt" });
1466
+ if (Schema.is(Schema.toType(LifecyclePublicationFact))(record.payload)) {
1467
+ if (
1468
+ bytes + storedTextBytes(row.record_json) > MAX_READ_PAGE_JSON_BYTES &&
1469
+ facts.length > 0
1470
+ ) {
1471
+ yield* retainMany(facts);
1472
+ facts.length = 0;
1473
+ bytes = 0;
1296
1474
  }
1297
- through = row.sequence;
1298
- }),
1299
- );
1300
- if (page.count === 0) return yield* LifecyclePublicationError.make({ reason: "corrupt" });
1301
- yield* retainMany(facts);
1302
- yield* sql`UPDATE effect_agent_lifecycle_cursors SET through_sequence=${through} WHERE thread_id=${threadId} RETURNING *`.pipe(
1303
- cursors.write,
1304
- );
1305
- }
1306
- }, Effect.mapError(sourceFailure));
1475
+ facts.push({
1476
+ id: JSON.stringify([threadId, "record", record.recordId]),
1477
+ ownerThreadId: yield* Schema.decodeEffect(ThreadId)(threadId),
1478
+ canonicalSequence: row.sequence,
1479
+ createdAt: record.createdAt,
1480
+ fact: record.payload,
1481
+ });
1482
+ bytes += storedTextBytes(row.record_json);
1483
+ }
1484
+ through = row.sequence;
1485
+ }),
1486
+ );
1487
+ if (page.count === 0) return yield* LifecyclePublicationError.make({ reason: "corrupt" });
1488
+ yield* retainMany(facts);
1489
+ yield* sql`UPDATE effect_agent_lifecycle_cursors SET through_sequence=${through} WHERE thread_id=${threadId} RETURNING *`.pipe(
1490
+ cursors.write,
1491
+ );
1492
+ }
1493
+ }, Effect.mapError(sourceFailure));
1307
1494
 
1308
- const flushPublications = Effect.fnUntraced(function* () {
1309
- let owners = 0;
1495
+ const flushPublications = Effect.fnUntraced(function* () {
1496
+ let owners = 0;
1310
1497
 
1311
- for (const thread of yield* sourceThreads()) {
1312
- const cursor = (yield* cursors.by("thread_id", thread.thread_id))[0];
1498
+ for (const thread of yield* sourceThreads()) {
1499
+ const cursor = (yield* cursors.by("thread_id", thread.thread_id))[0];
1313
1500
 
1314
- if (cursor !== undefined && cursor.through_sequence === thread.tail_sequence) continue;
1315
- yield* flushCanonical(thread.thread_id, 1024);
1316
- if (++owners === 4) break;
1317
- }
1318
- }, Effect.mapError(sourceFailure));
1501
+ if (cursor !== undefined && cursor.through_sequence === thread.tail_sequence) continue;
1502
+ yield* flushCanonical(thread.thread_id, 1024);
1503
+ if (++owners === 4) break;
1504
+ }
1505
+ }, Effect.mapError(sourceFailure));
1319
1506
 
1320
- const sourcePending = Effect.gen(function* () {
1321
- for (const thread of yield* sourceThreads()) {
1322
- const cursor = (yield* cursors.by("thread_id", thread.thread_id))[0];
1507
+ const sourcePending = Effect.gen(function* () {
1508
+ for (const thread of yield* sourceThreads()) {
1509
+ const cursor = (yield* cursors.by("thread_id", thread.thread_id))[0];
1323
1510
 
1324
- if (cursor === undefined || cursor.through_sequence > thread.tail_sequence)
1325
- return yield* LifecyclePublicationError.make({ reason: "corrupt" });
1326
- if (cursor.through_sequence < thread.tail_sequence) return true;
1327
- }
1511
+ if (cursor === undefined || cursor.through_sequence > thread.tail_sequence)
1512
+ return yield* LifecyclePublicationError.make({ reason: "corrupt" });
1513
+ if (cursor.through_sequence < thread.tail_sequence) return true;
1514
+ }
1328
1515
 
1329
- return false;
1330
- }).pipe(Effect.mapError(sourceFailure));
1331
-
1332
- const ownedLifecycle =
1333
- lifecycle === undefined
1334
- ? undefined
1335
- : {
1336
- ...lifecycle,
1337
- storage: {
1338
- ...lifecycle.storage,
1339
- retainedPendingDeadline: lifecycle.storage.pendingDeadline,
1340
- pendingDeadline: state.read(
1341
- Effect.gen(function* () {
1342
- const deadline = yield* lifecycle.storage.pendingDeadline;
1343
-
1344
- if (!(yield* sourcePending)) return deadline;
1345
- const now = yield* Clock.currentTimeMillis;
1346
-
1347
- return Option.some(Option.isSome(deadline) ? Math.min(now, deadline.value) : now);
1348
- }),
1349
- ),
1350
- },
1351
- };
1516
+ return false;
1517
+ }).pipe(Effect.mapError(sourceFailure));
1518
+
1519
+ const ownedLifecycle =
1520
+ lifecycle === undefined
1521
+ ? undefined
1522
+ : {
1523
+ ...lifecycle,
1524
+ storage: {
1525
+ ...lifecycle.storage,
1526
+ retainedPendingDeadline: lifecycle.storage.pendingDeadline,
1527
+ pendingDeadline: state.read(
1528
+ Effect.gen(function* () {
1529
+ const deadline = yield* lifecycle.storage.pendingDeadline;
1530
+
1531
+ if (!(yield* sourcePending)) return deadline;
1532
+ const now = yield* Clock.currentTimeMillis;
1533
+
1534
+ return Option.some(Option.isSome(deadline) ? Math.min(now, deadline.value) : now);
1535
+ }),
1536
+ ),
1537
+ },
1538
+ };
1352
1539
 
1353
- return {
1354
- archives: archives.storage,
1355
- archiveRanges: archives,
1356
- work,
1357
- state,
1358
- owner,
1359
- threads,
1360
- lifecycle: ownedLifecycle,
1361
- initializeLifecycleSource,
1362
- flushCanonical,
1363
- flushPublications,
1364
- append,
1365
- prepareAppend,
1366
- appendPrepared,
1367
- checkCommitIntent: (request: RawAppendRequest) =>
1368
- executionConflict(executions.checkCommitIntent(request)),
1369
- readExecution: executions.read,
1370
- checkValueBound,
1371
- getThread,
1372
- hasRecord,
1373
- isBatchBoundary,
1374
- loadCheckpoint,
1375
- materialize,
1376
- read: (request: RawReadRequest) => state.read(read(request)),
1377
- saveCheckpoint,
1378
- withWriteTransaction,
1379
- } as const;
1380
- });
1540
+ return {
1541
+ archives: archives.storage,
1542
+ archiveRanges: archives,
1543
+ work,
1544
+ state,
1545
+ owner,
1546
+ threads,
1547
+ lifecycle: ownedLifecycle,
1548
+ initializeLifecycleSource,
1549
+ flushCanonical,
1550
+ flushPublications,
1551
+ append,
1552
+ prepareAppend,
1553
+ appendPrepared,
1554
+ checkCommitIntent: (request: RawAppendRequest) =>
1555
+ executionConflict(executions.checkCommitIntent(request)),
1556
+ readExecution: executions.read,
1557
+ checkValueBound,
1558
+ getThread,
1559
+ hasRecord,
1560
+ isBatchBoundary,
1561
+ loadCheckpoint,
1562
+ materialize,
1563
+ read: (request: RawReadRequest) => state.read(read(request)),
1564
+ saveCheckpoint,
1565
+ withWriteTransaction,
1566
+ } as const;
1567
+ });
1381
1568
 
1382
1569
  export type DoJournal = Effect.Success<ReturnType<typeof makeJournal>>;
1383
1570