@yielded/agent-storage-cloudflare 0.1.0-beta.173 → 0.1.0-beta.174
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DoMemoryStore.mjs +1 -1
- package/dist/DoMessageDeliveryStore.mjs +1 -1
- package/dist/DoStorageFailpointTesting.mjs +1 -4
- package/dist/DoStorageFailpointTesting.mjs.map +1 -1
- package/dist/DoSubmissionLedger.mjs +8 -5
- package/dist/DoSubmissionLedger.mjs.map +1 -1
- package/dist/DoSubscriptionStore.mjs.map +1 -1
- package/dist/{DoThreadStore-CtyahRTW.mjs → DoThreadStore-DyzOZDXK.mjs} +9 -9
- package/dist/DoThreadStore-DyzOZDXK.mjs.map +1 -0
- package/dist/DoThreadStore.mjs +1 -1
- package/dist/PortProtocol.d.mts +7553 -7541
- package/dist/{do-journal-_MdLGTd3.mjs → do-journal-ZMAzzENZ.mjs} +12 -25
- package/dist/do-journal-ZMAzzENZ.mjs.map +1 -0
- package/dist/index.mjs +1 -1
- package/dist/migrations-C3N9GN6A.mjs.map +1 -1
- package/dist/{owned-state-BHzUi0eb.mjs → owned-state-DcFh0YNo.mjs} +7 -6
- package/dist/owned-state-DcFh0YNo.mjs.map +1 -0
- package/package.json +1 -1
- package/src/DoStorageFailpointTesting.ts +9 -7
- package/src/DoSubmissionLedger.ts +9 -1
- package/src/DoSubscriptionStore.ts +1 -0
- package/src/DoThreadStore.ts +9 -6
- package/src/internal/canonical-append.ts +1 -1
- package/src/internal/do-journal.ts +774 -807
- package/src/internal/migrations.ts +1 -0
- package/src/internal/owned-state.ts +9 -6
- package/src/internal/prompt-checkpoint.ts +13 -10
- package/src/internal/storage-span.ts +9 -4
- package/dist/DoThreadStore-CtyahRTW.mjs.map +0 -1
- package/dist/do-journal-_MdLGTd3.mjs.map +0 -1
- package/dist/owned-state-BHzUi0eb.mjs.map +0 -1
|
@@ -356,173 +356,172 @@ const ensureCurrentStorage = Effect.fnUntraced(function* (
|
|
|
356
356
|
return journal;
|
|
357
357
|
});
|
|
358
358
|
|
|
359
|
-
const makeJournal = (
|
|
359
|
+
const makeJournal = Effect.fnUntraced(function* (
|
|
360
360
|
sql: SqlClient.SqlClient,
|
|
361
361
|
failpoint: DoJournalFailpoint,
|
|
362
362
|
maxStoredValueBytes: number,
|
|
363
363
|
lifecycle: Effect.Success<ReturnType<typeof makeSqlLifecyclePublication>>,
|
|
364
364
|
state: OwnedState,
|
|
365
|
-
)
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
}),
|
|
381
|
-
),
|
|
365
|
+
) {
|
|
366
|
+
const owner = (yield* SqlStorageOwner) ?? state;
|
|
367
|
+
const progress = yield* SqlStorageProgress;
|
|
368
|
+
|
|
369
|
+
const work = yield* makeSqlThreadWork({
|
|
370
|
+
read: (body) => state.read(body),
|
|
371
|
+
write: (body) =>
|
|
372
|
+
owner.transaction(body.pipe(Effect.tap(() => state.invalidate))).pipe(
|
|
373
|
+
Effect.provideService(SqlClient.SqlClient, sql),
|
|
374
|
+
Effect.catchTag("SqlError", (cause) =>
|
|
375
|
+
ThreadStoreError.make({
|
|
376
|
+
operation: "rebuild Thread work",
|
|
377
|
+
message: "Thread work transaction failed",
|
|
378
|
+
cause,
|
|
379
|
+
}),
|
|
382
380
|
),
|
|
383
|
-
|
|
381
|
+
),
|
|
382
|
+
}).pipe(Effect.provideService(SqlClient.SqlClient, sql));
|
|
384
383
|
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
384
|
+
yield* Effect.acquireRelease(
|
|
385
|
+
Effect.sync(() => state.invalidators.add(work.invalidate)),
|
|
386
|
+
() => Effect.sync(() => state.invalidators.delete(work.invalidate)),
|
|
387
|
+
);
|
|
389
388
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
389
|
+
const workFailure = (cause: unknown) =>
|
|
390
|
+
DoStorageError.make({
|
|
391
|
+
operation: "publish Thread work",
|
|
392
|
+
message: "Canonical work metadata could not be published",
|
|
393
|
+
cause,
|
|
394
|
+
});
|
|
396
395
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
396
|
+
const executions = yield* makeSqlExecutions().pipe(
|
|
397
|
+
Effect.provideService(SqlClient.SqlClient, sql),
|
|
398
|
+
);
|
|
400
399
|
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
),
|
|
400
|
+
const executionConflict = <A>(effect: Effect.Effect<A, AppendConflict | ThreadStoreError>) =>
|
|
401
|
+
effect.pipe(
|
|
402
|
+
Effect.catchTag("AppendConflict", (error) =>
|
|
403
|
+
Effect.fail(
|
|
404
|
+
DoAppendConflict.make({
|
|
405
|
+
reason: error.reason,
|
|
406
|
+
message: "Active execution conflicts with the captured commit",
|
|
407
|
+
...(error.actualTailSequence === undefined
|
|
408
|
+
? {}
|
|
409
|
+
: { actualTailSequence: error.actualTailSequence }),
|
|
410
|
+
}),
|
|
413
411
|
),
|
|
414
|
-
)
|
|
412
|
+
),
|
|
413
|
+
);
|
|
415
414
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
415
|
+
const threads = threadRows(state, sql);
|
|
416
|
+
const cursors = lifecycleCursorRows(state, sql);
|
|
417
|
+
let records = recordCaches.get(state);
|
|
419
418
|
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
),
|
|
419
|
+
if (records === undefined) {
|
|
420
|
+
records = makeRecordCache();
|
|
421
|
+
recordCaches.set(state, records);
|
|
422
|
+
state.invalidators.add(records.clear);
|
|
423
|
+
}
|
|
424
|
+
const recordCache = records;
|
|
425
|
+
|
|
426
|
+
const archives = yield* makeSqlThreadArchiveRange({
|
|
427
|
+
read: (body) => state.read(body),
|
|
428
|
+
write: (body) =>
|
|
429
|
+
owner.transaction(body.pipe(Effect.tap(() => state.invalidate))).pipe(
|
|
430
|
+
Effect.provideService(SqlClient.SqlClient, sql),
|
|
431
|
+
Effect.catchTag("SqlError", (cause) =>
|
|
432
|
+
ThreadStoreError.make({
|
|
433
|
+
operation: "archive write",
|
|
434
|
+
message: "Archive transaction failed.",
|
|
435
|
+
cause,
|
|
436
|
+
}),
|
|
439
437
|
),
|
|
440
|
-
|
|
438
|
+
),
|
|
439
|
+
}).pipe(Effect.provideService(SqlClient.SqlClient, sql));
|
|
441
440
|
|
|
442
|
-
|
|
441
|
+
const recordJson = canonicalRecordJson(sql);
|
|
443
442
|
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
/** Typed pre-write refusal for any single value over the configured byte bound. */
|
|
449
|
-
const checkValueBound = (
|
|
450
|
-
operation: string,
|
|
451
|
-
value: string,
|
|
452
|
-
actualBytes = storedTextBytes(value),
|
|
453
|
-
): Effect.Effect<void, DoValueBoundExceeded> => {
|
|
454
|
-
return actualBytes > maxStoredValueBytes
|
|
455
|
-
? Effect.fail(
|
|
456
|
-
DoValueBoundExceeded.make({
|
|
457
|
-
actualBytes,
|
|
458
|
-
maxBytes: maxStoredValueBytes,
|
|
459
|
-
operation,
|
|
460
|
-
}),
|
|
461
|
-
)
|
|
462
|
-
: Effect.void;
|
|
463
|
-
};
|
|
443
|
+
const physical = yield* makeCanonicalStorage().pipe(
|
|
444
|
+
Effect.provideService(SqlClient.SqlClient, sql),
|
|
445
|
+
);
|
|
464
446
|
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
): Effect.Effect<A, E | DoStorageError> =>
|
|
482
|
-
state.owner.withMutation(owner.transaction(storageResult(effect, expected))).pipe(
|
|
483
|
-
Effect.provideService(SqlClient.SqlClient, sql),
|
|
484
|
-
Effect.mapError((error) => (isSqlError(error) ? storageError(operation)(error) : error)),
|
|
485
|
-
Effect.tapError(annotateStorageError),
|
|
486
|
-
Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
|
|
487
|
-
Effect.flatMap(Effect.fromResult),
|
|
488
|
-
);
|
|
447
|
+
/** Typed pre-write refusal for any single value over the configured byte bound. */
|
|
448
|
+
const checkValueBound = (
|
|
449
|
+
operation: string,
|
|
450
|
+
value: string,
|
|
451
|
+
actualBytes = storedTextBytes(value),
|
|
452
|
+
): Effect.Effect<void, DoValueBoundExceeded> => {
|
|
453
|
+
return actualBytes > maxStoredValueBytes
|
|
454
|
+
? Effect.fail(
|
|
455
|
+
DoValueBoundExceeded.make({
|
|
456
|
+
actualBytes,
|
|
457
|
+
maxBytes: maxStoredValueBytes,
|
|
458
|
+
operation,
|
|
459
|
+
}),
|
|
460
|
+
)
|
|
461
|
+
: Effect.void;
|
|
462
|
+
};
|
|
489
463
|
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
464
|
+
/**
|
|
465
|
+
* Runs one journal write transaction on the Durable Object storage-backed
|
|
466
|
+
* `withTransaction` (`ctx.storage.transaction()` under the hood). Within one Durable
|
|
467
|
+
* Object there is exactly ONE writer, so the Node `BEGIN IMMEDIATE` + busy-retry +
|
|
468
|
+
* `SqliteWriteContention` machinery has no analogue here and is deliberately absent.
|
|
469
|
+
* The shared local owner gate covers the transaction and acknowledgement.
|
|
470
|
+
*
|
|
471
|
+
* Journal write transactions are always top level: the Durable Object client rejects
|
|
472
|
+
* nested transactions, so new journal operations must not wrap this helper inside another
|
|
473
|
+
* transaction. Expected outcomes may be captured only when every such refusal precedes
|
|
474
|
+
* mutation; real faults still roll back. Successful preflight reads never authorize a write.
|
|
475
|
+
*/
|
|
476
|
+
const withWriteTransaction =
|
|
477
|
+
(operation: string, expected: (error: { readonly _tag: string }) => boolean = () => false) =>
|
|
478
|
+
<A, E extends { readonly _tag: string }>(
|
|
479
|
+
effect: Effect.Effect<A, E>,
|
|
480
|
+
): Effect.Effect<A, E | DoStorageError> =>
|
|
481
|
+
state.owner.withMutation(owner.transaction(storageResult(effect, expected))).pipe(
|
|
482
|
+
Effect.provideService(SqlClient.SqlClient, sql),
|
|
483
|
+
Effect.mapError((error) => (isSqlError(error) ? storageError(operation)(error) : error)),
|
|
484
|
+
Effect.tapError(annotateStorageError),
|
|
485
|
+
Effect.withSpan("DoJournal.withWriteTransaction", { attributes: { operation } }),
|
|
486
|
+
Effect.flatMap(Effect.fromResult),
|
|
487
|
+
);
|
|
502
488
|
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
489
|
+
const materialize = Effect.fnUntraced(function* (
|
|
490
|
+
threadId: string,
|
|
491
|
+
createdAt: string,
|
|
492
|
+
): Effect.fn.Return<void, DoStorageCorruptionError | DoStorageError | DoValueBoundExceeded> {
|
|
493
|
+
if (threadId.length > MAX_IDENTIFIER_LENGTH) {
|
|
494
|
+
return yield* DoStorageError.make({
|
|
495
|
+
operation: "materialize thread",
|
|
496
|
+
message: "Thread identity exceeds the Durable Object storage bounds.",
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
// Refusals and idempotent materialization need no transaction. Recheck before writes.
|
|
500
|
+
const current = yield* getThread(threadId);
|
|
509
501
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
502
|
+
if (current.length === 1) {
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
yield* withWriteTransaction("materialize transaction")(
|
|
506
|
+
Effect.gen(function* () {
|
|
507
|
+
const existingRows = yield* getThread(threadId);
|
|
508
|
+
|
|
509
|
+
const existing = yield* decodeRows(
|
|
510
|
+
Schema.Array(ThreadRow),
|
|
511
|
+
"effect_agent_threads",
|
|
512
|
+
threadId,
|
|
513
|
+
existingRows,
|
|
514
|
+
);
|
|
516
515
|
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
516
|
+
if (existing.length > 1) {
|
|
517
|
+
return yield* DoStorageCorruptionError.make({
|
|
518
|
+
table: "effect_agent_threads",
|
|
519
|
+
rowKey: threadId,
|
|
520
|
+
message: "A thread primary key returned more than one row.",
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
if (existing.length === 0) {
|
|
524
|
+
yield* sql`
|
|
526
525
|
INSERT INTO effect_agent_threads (
|
|
527
526
|
thread_id,
|
|
528
527
|
created_at,
|
|
@@ -534,215 +533,178 @@ const makeJournal = (
|
|
|
534
533
|
) ON CONFLICT(thread_id) DO UPDATE SET created_at=excluded.created_at
|
|
535
534
|
WHERE effect_agent_threads.created_at IS NULL
|
|
536
535
|
RETURNING *`.pipe(
|
|
537
|
-
|
|
536
|
+
threads.write,
|
|
537
|
+
Effect.mapError((error) =>
|
|
538
|
+
error._tag === "SqlError" ? storageError("materialize thread")(error) : error,
|
|
539
|
+
),
|
|
540
|
+
);
|
|
541
|
+
|
|
542
|
+
if (lifecycle !== undefined)
|
|
543
|
+
yield* sql`INSERT INTO effect_agent_lifecycle_cursors (thread_id, through_sequence)
|
|
544
|
+
VALUES (${threadId}, 0) RETURNING *`.pipe(
|
|
545
|
+
cursors.write,
|
|
538
546
|
Effect.mapError((error) =>
|
|
539
|
-
error._tag === "SqlError"
|
|
547
|
+
error._tag === "SqlError"
|
|
548
|
+
? storageError("initialize lifecycle source")(error)
|
|
549
|
+
: error,
|
|
540
550
|
),
|
|
541
551
|
);
|
|
542
552
|
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
error._tag === "SqlError"
|
|
549
|
-
? storageError("initialize lifecycle source")(error)
|
|
550
|
-
: error,
|
|
551
|
-
),
|
|
552
|
-
);
|
|
553
|
-
|
|
554
|
-
return;
|
|
555
|
-
}
|
|
556
|
-
}),
|
|
557
|
-
);
|
|
558
|
-
}, withStorageSpan("DoJournal.materialize"));
|
|
559
|
-
|
|
560
|
-
const getThread = Effect.fnUntraced(function* (threadId: string) {
|
|
561
|
-
return (yield* threads
|
|
562
|
-
.by("thread_id", threadId)
|
|
563
|
-
.pipe(
|
|
564
|
-
Effect.mapError((error) =>
|
|
565
|
-
error._tag === "SqlError" ? storageError("read thread")(error) : error,
|
|
566
|
-
),
|
|
567
|
-
)).filter((row) => row.thread_id === threadId && row.created_at !== null);
|
|
568
|
-
});
|
|
569
|
-
|
|
570
|
-
const hasRecord = Effect.fnUntraced(function* (threadId: string, recordId: string) {
|
|
571
|
-
const thread = (yield* getThread(threadId))[0];
|
|
572
|
-
const prefix = recordCache.prefix(threadId, thread?.tail_sequence ?? 0);
|
|
553
|
+
return;
|
|
554
|
+
}
|
|
555
|
+
}),
|
|
556
|
+
);
|
|
557
|
+
}, withStorageSpan("DoJournal.materialize"));
|
|
573
558
|
|
|
574
|
-
|
|
559
|
+
const getThread = Effect.fnUntraced(function* (threadId: string) {
|
|
560
|
+
return (yield* threads
|
|
561
|
+
.by("thread_id", threadId)
|
|
562
|
+
.pipe(
|
|
563
|
+
Effect.mapError((error) =>
|
|
564
|
+
error._tag === "SqlError" ? storageError("read thread")(error) : error,
|
|
565
|
+
),
|
|
566
|
+
)).filter((row) => row.thread_id === threadId && row.created_at !== null);
|
|
567
|
+
});
|
|
575
568
|
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
);
|
|
569
|
+
const hasRecord = Effect.fnUntraced(function* (threadId: string, recordId: string) {
|
|
570
|
+
const thread = (yield* getThread(threadId))[0];
|
|
571
|
+
const prefix = recordCache.prefix(threadId, thread?.tail_sequence ?? 0);
|
|
580
572
|
|
|
581
|
-
|
|
582
|
-
});
|
|
573
|
+
if (prefix !== undefined) return prefix.some((row) => row.record_id === recordId);
|
|
583
574
|
|
|
584
|
-
const
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
request.executionWrites.some(
|
|
589
|
-
({ update }) => update.checkpoint.progress.runId.length > MAX_IDENTIFIER_LENGTH,
|
|
590
|
-
) ||
|
|
591
|
-
request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)
|
|
592
|
-
) {
|
|
593
|
-
return yield* DoStorageError.make({
|
|
594
|
-
operation: "append canonical batch",
|
|
595
|
-
message: "Canonical identifiers exceed the Durable Object storage bounds.",
|
|
596
|
-
});
|
|
597
|
-
}
|
|
598
|
-
// Archives store the full batch as one value, so their ceiling also bounds hot appends.
|
|
599
|
-
// The trusted append SPI pairs captured JSON with its exact UTF-8 width.
|
|
600
|
-
yield* checkValueBound("append canonical batch", request.batchJson, request.batchBytes);
|
|
601
|
-
for (const record of request.records) {
|
|
602
|
-
if (record.recordBytes > maxStoredValueBytes)
|
|
603
|
-
return yield* DoValueBoundExceeded.make({
|
|
604
|
-
actualBytes: record.recordBytes,
|
|
605
|
-
maxBytes: maxStoredValueBytes,
|
|
606
|
-
operation: "append canonical record",
|
|
607
|
-
});
|
|
608
|
-
}
|
|
609
|
-
for (const write of request.executionWrites)
|
|
610
|
-
yield* checkValueBound("commit active execution", write.json, write.bytes);
|
|
575
|
+
const rows = yield* sql`SELECT record_id FROM effect_agent_canonical_records
|
|
576
|
+
WHERE thread_id = ${threadId} AND record_id = ${JSON.stringify(recordId)}`.pipe(
|
|
577
|
+
Effect.mapError(storageError("read canonical record identity")),
|
|
578
|
+
);
|
|
611
579
|
|
|
612
|
-
|
|
580
|
+
return rows.length > 0;
|
|
581
|
+
});
|
|
613
582
|
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
583
|
+
const prepareAppend = Effect.fnUntraced(function* (request: RawAppendRequest) {
|
|
584
|
+
if (
|
|
585
|
+
request.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
586
|
+
request.batchId.length > MAX_IDENTIFIER_LENGTH ||
|
|
587
|
+
request.executionWrites.some(
|
|
588
|
+
({ update }) => update.checkpoint.progress.runId.length > MAX_IDENTIFIER_LENGTH,
|
|
589
|
+
) ||
|
|
590
|
+
request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH)
|
|
591
|
+
) {
|
|
592
|
+
return yield* DoStorageError.make({
|
|
593
|
+
operation: "append canonical batch",
|
|
594
|
+
message: "Canonical identifiers exceed the Durable Object storage bounds.",
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
// Archives store the full batch as one value, so their ceiling also bounds hot appends.
|
|
598
|
+
// The trusted append SPI pairs captured JSON with its exact UTF-8 width.
|
|
599
|
+
yield* checkValueBound("append canonical batch", request.batchJson, request.batchBytes);
|
|
600
|
+
for (const record of request.records) {
|
|
601
|
+
if (record.recordBytes > maxStoredValueBytes)
|
|
602
|
+
return yield* DoValueBoundExceeded.make({
|
|
603
|
+
actualBytes: record.recordBytes,
|
|
604
|
+
maxBytes: maxStoredValueBytes,
|
|
605
|
+
operation: "append canonical record",
|
|
618
606
|
});
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
});
|
|
607
|
+
}
|
|
608
|
+
for (const write of request.executionWrites)
|
|
609
|
+
yield* checkValueBound("commit active execution", write.json, write.bytes);
|
|
623
610
|
|
|
624
|
-
const
|
|
625
|
-
request: RawAppendRequest,
|
|
626
|
-
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
627
|
-
const recordIds: Array<string> = request.records.map((record) => record.recordId);
|
|
628
|
-
const threadRows = yield* getThread(request.threadId);
|
|
611
|
+
const recordIds: Array<string> = request.records.map((record) => record.recordId);
|
|
629
612
|
|
|
630
|
-
|
|
613
|
+
if (new Set(recordIds).size !== recordIds.length) {
|
|
614
|
+
return yield* DoAppendConflict.make({
|
|
615
|
+
message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
|
|
616
|
+
reason: "record-identity",
|
|
617
|
+
});
|
|
618
|
+
}
|
|
631
619
|
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
table: "effect_agent_threads",
|
|
635
|
-
rowKey: request.threadId,
|
|
636
|
-
message: `Expected exactly one row but found ${threadRows.length}.`,
|
|
637
|
-
});
|
|
620
|
+
return request;
|
|
621
|
+
});
|
|
638
622
|
|
|
639
|
-
|
|
623
|
+
const appendPrepared = Effect.fnUntraced(function* (
|
|
624
|
+
request: RawAppendRequest,
|
|
625
|
+
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
626
|
+
const threadRows = yield* getThread(request.threadId);
|
|
640
627
|
|
|
641
|
-
|
|
628
|
+
const thread = threadRows.length === 1 ? threadRows[0] : undefined;
|
|
642
629
|
|
|
643
|
-
|
|
630
|
+
if (thread === undefined)
|
|
631
|
+
return yield* DoStorageCorruptionError.make({
|
|
632
|
+
table: "effect_agent_threads",
|
|
633
|
+
rowKey: request.threadId,
|
|
634
|
+
message: `Expected exactly one row but found ${threadRows.length}.`,
|
|
635
|
+
});
|
|
644
636
|
|
|
645
|
-
|
|
646
|
-
prefix !== undefined && !prefix.some((row) => row.batch_id === request.batchId)
|
|
647
|
-
? undefined
|
|
648
|
-
: yield* archives.replay(request.threadId, request.batchId);
|
|
637
|
+
const prefix = recordCache.prefix(request.threadId, thread.tail_sequence);
|
|
649
638
|
|
|
650
|
-
|
|
651
|
-
if (existing.batch_json !== request.batchJson) {
|
|
652
|
-
return yield* DoAppendConflict.make({
|
|
653
|
-
message: `Batch ${request.batchId} already exists with different canonical content.`,
|
|
654
|
-
reason: "batch-content",
|
|
655
|
-
});
|
|
656
|
-
}
|
|
639
|
+
const replay = yield* executionConflict(executions.replay(request));
|
|
657
640
|
|
|
658
|
-
|
|
659
|
-
firstSequence: existing.first_sequence,
|
|
660
|
-
lastSequence: existing.last_sequence,
|
|
661
|
-
replayed: true,
|
|
662
|
-
});
|
|
663
|
-
}
|
|
641
|
+
if (replay !== undefined) return RawAppendResult.make(replay);
|
|
664
642
|
|
|
665
|
-
|
|
643
|
+
const existing =
|
|
644
|
+
prefix !== undefined && !prefix.some((row) => row.batch_id === request.batchId)
|
|
645
|
+
? undefined
|
|
646
|
+
: yield* archives.replay(request.threadId, request.batchId);
|
|
666
647
|
|
|
667
|
-
|
|
648
|
+
if (existing !== undefined) {
|
|
649
|
+
if (existing.batch_json !== request.batchJson) {
|
|
668
650
|
return yield* DoAppendConflict.make({
|
|
669
|
-
message:
|
|
670
|
-
|
|
671
|
-
reason: "tail",
|
|
672
|
-
actualTailSequence: thread.tail_sequence,
|
|
651
|
+
message: `Batch ${request.batchId} already exists with different canonical content.`,
|
|
652
|
+
reason: "batch-content",
|
|
673
653
|
});
|
|
674
654
|
}
|
|
675
655
|
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
656
|
+
return RawAppendResult.make({
|
|
657
|
+
firstSequence: existing.first_sequence,
|
|
658
|
+
lastSequence: existing.last_sequence,
|
|
659
|
+
replayed: true,
|
|
660
|
+
});
|
|
661
|
+
}
|
|
680
662
|
|
|
681
|
-
|
|
682
|
-
? chunked(recordIds, MAX_BOUND_PARAMETERS - 10)
|
|
683
|
-
: []) {
|
|
684
|
-
const existingRecordRows = yield* sql<Record<string, unknown>>`
|
|
685
|
-
SELECT
|
|
686
|
-
thread_id,
|
|
687
|
-
sequence,
|
|
688
|
-
record_id,
|
|
689
|
-
${recordBatchId(sql)} AS batch_id,
|
|
690
|
-
${recordJson} AS record_json
|
|
691
|
-
FROM effect_agent_canonical_records
|
|
692
|
-
WHERE thread_id = ${request.threadId}
|
|
693
|
-
AND record_id IN ${sql.in(chunk.map((id) => JSON.stringify(id)))}
|
|
694
|
-
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
695
|
-
|
|
696
|
-
existingRecords.push(
|
|
697
|
-
...(yield* decodeRows(
|
|
698
|
-
Schema.Array(RecordRow),
|
|
699
|
-
"effect_agent_canonical_records",
|
|
700
|
-
`${request.threadId}/record_ids`,
|
|
701
|
-
existingRecordRows,
|
|
702
|
-
)),
|
|
703
|
-
);
|
|
704
|
-
}
|
|
705
|
-
if (existingRecords.length > 0) {
|
|
706
|
-
return yield* DoAppendConflict.make({
|
|
707
|
-
message: `Canonical record ID ${existingRecords[0].record_id} already exists.`,
|
|
708
|
-
reason: "record-identity",
|
|
709
|
-
});
|
|
710
|
-
}
|
|
663
|
+
yield* executionConflict(executions.rejectSuperseded(request));
|
|
711
664
|
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
}),
|
|
721
|
-
),
|
|
722
|
-
);
|
|
665
|
+
if (request.expectedTailSequence !== thread.tail_sequence) {
|
|
666
|
+
return yield* DoAppendConflict.make({
|
|
667
|
+
message:
|
|
668
|
+
`Expected tail ${request.expectedTailSequence} ` + `but found ${thread.tail_sequence}.`,
|
|
669
|
+
reason: "tail",
|
|
670
|
+
actualTailSequence: thread.tail_sequence,
|
|
671
|
+
});
|
|
672
|
+
}
|
|
723
673
|
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
674
|
+
const firstSequence = yield* Schema.decodeEffect(CanonicalSequence)(
|
|
675
|
+
thread.tail_sequence + (request.records.length === 0 ? 0 : 1),
|
|
676
|
+
).pipe(
|
|
677
|
+
Effect.mapError((error) =>
|
|
678
|
+
DoStorageError.make({
|
|
679
|
+
cause: error,
|
|
680
|
+
operation: "append canonical batch",
|
|
681
|
+
message: error.message,
|
|
682
|
+
}),
|
|
683
|
+
),
|
|
684
|
+
);
|
|
735
685
|
|
|
736
|
-
|
|
686
|
+
const lastSequence = yield* Schema.decodeEffect(CanonicalSequence)(
|
|
687
|
+
thread.tail_sequence + request.records.length,
|
|
688
|
+
).pipe(
|
|
689
|
+
Effect.mapError((error) =>
|
|
690
|
+
DoStorageError.make({
|
|
691
|
+
cause: error,
|
|
692
|
+
operation: "append canonical batch",
|
|
693
|
+
message: error.message,
|
|
694
|
+
}),
|
|
695
|
+
),
|
|
696
|
+
);
|
|
737
697
|
|
|
738
|
-
|
|
739
|
-
firstSequence,
|
|
740
|
-
lastSequence,
|
|
741
|
-
replayed: false,
|
|
742
|
-
});
|
|
698
|
+
yield* executionConflict(executions.validate(request));
|
|
743
699
|
|
|
744
|
-
|
|
745
|
-
|
|
700
|
+
const result = RawAppendResult.make({
|
|
701
|
+
firstSequence,
|
|
702
|
+
lastSequence,
|
|
703
|
+
replayed: false,
|
|
704
|
+
});
|
|
705
|
+
|
|
706
|
+
if (request.records.length > 0) {
|
|
707
|
+
yield* sql`
|
|
746
708
|
INSERT INTO effect_agent_canonical_batches (
|
|
747
709
|
thread_id,
|
|
748
710
|
batch_id,
|
|
@@ -757,162 +719,168 @@ const makeJournal = (
|
|
|
757
719
|
${request.producerIdJson}
|
|
758
720
|
)
|
|
759
721
|
`.pipe(Effect.mapError(storageError("insert canonical batch")));
|
|
760
|
-
|
|
761
|
-
|
|
722
|
+
}
|
|
723
|
+
yield* failpoint("append:after-batch-insert");
|
|
762
724
|
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
725
|
+
const physicalRecords = yield* physical.records(
|
|
726
|
+
request.threadId,
|
|
727
|
+
firstSequence,
|
|
728
|
+
request.records,
|
|
729
|
+
);
|
|
768
730
|
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
731
|
+
const records = physicalRecords.map(({ columns, memberships }, index) => ({
|
|
732
|
+
record: request.records[index]!,
|
|
733
|
+
canonical: request.records[index]!.canonical,
|
|
734
|
+
row: columns,
|
|
735
|
+
memberships,
|
|
736
|
+
}));
|
|
737
|
+
|
|
738
|
+
// Captured scalar locators avoid parsing canonical JSON under the writer.
|
|
739
|
+
// Eighteen bound columns per row stay within workerd's statement limit.
|
|
740
|
+
for (const group of chunked(records, Math.floor(MAX_BOUND_PARAMETERS / 18))) {
|
|
741
|
+
const inserted =
|
|
742
|
+
yield* sql`INSERT INTO effect_agent_canonical_records ${sql.insert(group.map(({ row }) => row))}
|
|
743
|
+
ON CONFLICT(thread_id, record_id) DO NOTHING RETURNING 1`.pipe(
|
|
780
744
|
Effect.mapError(storageError("insert canonical records")),
|
|
781
745
|
);
|
|
782
746
|
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
);
|
|
789
|
-
for (const { row, record } of group) {
|
|
790
|
-
recordCache.put(
|
|
791
|
-
RecordRow.make({
|
|
792
|
-
...row,
|
|
793
|
-
record_id: record.recordId,
|
|
794
|
-
batch_id: request.batchId,
|
|
795
|
-
record_json: record.recordJson,
|
|
796
|
-
sequence: Schema.decodeSync(CanonicalSequence)(row.sequence),
|
|
797
|
-
}),
|
|
798
|
-
record.recordBytes,
|
|
799
|
-
);
|
|
800
|
-
yield* failpoint("append:after-record-insert");
|
|
801
|
-
}
|
|
802
|
-
}
|
|
747
|
+
if (inserted.length !== group.length)
|
|
748
|
+
return yield* DoAppendConflict.make({
|
|
749
|
+
message: "A canonical record ID already exists.",
|
|
750
|
+
reason: "record-identity",
|
|
751
|
+
});
|
|
803
752
|
|
|
804
|
-
|
|
805
|
-
const workFields = yield* work
|
|
806
|
-
.apply(
|
|
807
|
-
yield* Schema.decodeEffect(ThreadId)(request.threadId).pipe(
|
|
808
|
-
Effect.mapError(workFailure),
|
|
809
|
-
),
|
|
810
|
-
firstSequence,
|
|
811
|
-
request.records.map((record) => record.canonical),
|
|
812
|
-
)
|
|
813
|
-
.pipe(Effect.mapError(workFailure));
|
|
753
|
+
const memberships = group.flatMap(({ memberships }) => memberships);
|
|
814
754
|
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
Effect.mapError((error) =>
|
|
819
|
-
error._tag === "SqlError" ? storageError("advance thread tail")(error) : error,
|
|
820
|
-
),
|
|
755
|
+
for (const members of chunked(memberships, Math.floor(MAX_BOUND_PARAMETERS / 3)))
|
|
756
|
+
yield* sql`INSERT INTO effect_agent_record_runs ${sql.insert(members)}`.pipe(
|
|
757
|
+
Effect.mapError(storageError("index canonical Run membership")),
|
|
821
758
|
);
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
Effect.provideService(SqlLifecycleRetainer, { retainMany: lifecycle.retainMany }),
|
|
833
|
-
Effect.mapError((cause) =>
|
|
834
|
-
DoStorageError.make({
|
|
835
|
-
operation: "retain lifecycle start prefix",
|
|
836
|
-
message: "Lifecycle start intent could not be retained",
|
|
837
|
-
cause,
|
|
838
|
-
}),
|
|
839
|
-
),
|
|
759
|
+
for (const { row, record } of group) {
|
|
760
|
+
recordCache.put(
|
|
761
|
+
RecordRow.make({
|
|
762
|
+
...row,
|
|
763
|
+
record_id: record.recordId,
|
|
764
|
+
batch_id: request.batchId,
|
|
765
|
+
record_json: record.recordJson,
|
|
766
|
+
sequence: Schema.decodeSync(CanonicalSequence)(row.sequence),
|
|
767
|
+
}),
|
|
768
|
+
record.recordBytes,
|
|
840
769
|
);
|
|
841
|
-
|
|
842
|
-
for (const fact of update.facts) {
|
|
843
|
-
if (
|
|
844
|
-
lifecycle !== undefined &&
|
|
845
|
-
Schema.is(Schema.toType(LifecyclePublicationFact))(fact.payload)
|
|
846
|
-
)
|
|
847
|
-
yield* lifecycle
|
|
848
|
-
.retain({
|
|
849
|
-
id: JSON.stringify([request.threadId, "record", fact.recordId]),
|
|
850
|
-
ownerThreadId: request.threadId,
|
|
851
|
-
canonicalSequence: lastSequence,
|
|
852
|
-
createdAt: fact.createdAt,
|
|
853
|
-
fact: fact.payload,
|
|
854
|
-
})
|
|
855
|
-
.pipe(
|
|
856
|
-
Effect.mapError((cause) =>
|
|
857
|
-
DoStorageError.make({
|
|
858
|
-
operation: "retain operational lifecycle",
|
|
859
|
-
message: cause.message,
|
|
860
|
-
cause,
|
|
861
|
-
}),
|
|
862
|
-
),
|
|
863
|
-
);
|
|
864
|
-
}
|
|
770
|
+
yield* failpoint("append:after-record-insert");
|
|
865
771
|
}
|
|
866
|
-
|
|
772
|
+
}
|
|
867
773
|
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
(
|
|
874
|
-
|
|
774
|
+
if (request.records.length > 0) {
|
|
775
|
+
const workFields = yield* work
|
|
776
|
+
.apply(
|
|
777
|
+
yield* Schema.decodeEffect(ThreadId)(request.threadId).pipe(Effect.mapError(workFailure)),
|
|
778
|
+
firstSequence,
|
|
779
|
+
request.records.map((record) => record.canonical),
|
|
780
|
+
)
|
|
781
|
+
.pipe(Effect.mapError(workFailure));
|
|
782
|
+
|
|
783
|
+
yield* sql`UPDATE effect_agent_threads SET ${sql.update({ tail_sequence: lastSequence, ...workFields })}
|
|
784
|
+
WHERE thread_id=${request.threadId} RETURNING *`.pipe(
|
|
785
|
+
threads.write,
|
|
786
|
+
Effect.mapError((error) =>
|
|
787
|
+
error._tag === "SqlError" ? storageError("advance thread tail")(error) : error,
|
|
788
|
+
),
|
|
789
|
+
);
|
|
790
|
+
}
|
|
791
|
+
yield* executionConflict(executions.commit(request, result));
|
|
792
|
+
if (request.executionWrites.length > 0) yield* failpoint("append:after-record-insert");
|
|
793
|
+
if (
|
|
794
|
+
lifecycle !== undefined &&
|
|
795
|
+
request.executionWrites.some(({ update }) =>
|
|
796
|
+
update.facts.some(({ payload }) => payload._tag === "RunStarted"),
|
|
797
|
+
)
|
|
798
|
+
)
|
|
799
|
+
yield* flushCanonical(request.threadId).pipe(
|
|
800
|
+
Effect.provideService(SqlLifecycleRetainer, { retainMany: lifecycle.retainMany }),
|
|
801
|
+
Effect.mapError((cause) =>
|
|
802
|
+
DoStorageError.make({
|
|
803
|
+
operation: "retain lifecycle start prefix",
|
|
804
|
+
message: "Lifecycle start intent could not be retained",
|
|
805
|
+
cause,
|
|
806
|
+
}),
|
|
807
|
+
),
|
|
808
|
+
);
|
|
809
|
+
for (const { update } of request.executionWrites) {
|
|
810
|
+
for (const fact of update.facts) {
|
|
811
|
+
if (
|
|
812
|
+
lifecycle !== undefined &&
|
|
813
|
+
Schema.is(Schema.toType(LifecyclePublicationFact))(fact.payload)
|
|
875
814
|
)
|
|
815
|
+
yield* lifecycle
|
|
816
|
+
.retain({
|
|
817
|
+
id: JSON.stringify([request.threadId, "record", fact.recordId]),
|
|
818
|
+
ownerThreadId: request.threadId,
|
|
819
|
+
canonicalSequence: lastSequence,
|
|
820
|
+
createdAt: fact.createdAt,
|
|
821
|
+
fact: fact.payload,
|
|
822
|
+
})
|
|
823
|
+
.pipe(
|
|
824
|
+
Effect.mapError((cause) =>
|
|
825
|
+
DoStorageError.make({
|
|
826
|
+
operation: "retain operational lifecycle",
|
|
827
|
+
message: cause.message,
|
|
828
|
+
cause,
|
|
829
|
+
}),
|
|
830
|
+
),
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
yield* failpoint("append:after-tail-update");
|
|
835
|
+
|
|
836
|
+
// Retain one start prefix atomically with its canonical proof, before model/tool
|
|
837
|
+
// execution. Later facts stay journal-backed until the settlement publication wave.
|
|
838
|
+
if (
|
|
839
|
+
lifecycle !== undefined &&
|
|
840
|
+
records.some(
|
|
841
|
+
({ canonical: { payload } }) =>
|
|
842
|
+
payload._tag === "RunStarted" || payload._tag === "SubagentStarted",
|
|
876
843
|
)
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
844
|
+
)
|
|
845
|
+
yield* flushCanonical(request.threadId).pipe(
|
|
846
|
+
Effect.provideService(SqlLifecycleRetainer, { retainMany: lifecycle.retainMany }),
|
|
847
|
+
Effect.mapError((cause) =>
|
|
848
|
+
DoStorageError.make({
|
|
849
|
+
operation: "retain lifecycle start prefix",
|
|
850
|
+
message: "Lifecycle start intent could not be retained",
|
|
851
|
+
cause,
|
|
852
|
+
}),
|
|
853
|
+
),
|
|
854
|
+
);
|
|
855
|
+
|
|
856
|
+
yield* progress.committed("canonical").pipe(
|
|
857
|
+
Effect.catchCause((cause) =>
|
|
858
|
+
Effect.failCause(
|
|
859
|
+
Cause.map(cause, (error) =>
|
|
880
860
|
DoStorageError.make({
|
|
881
|
-
operation: "
|
|
882
|
-
message:
|
|
883
|
-
cause,
|
|
861
|
+
operation: "enroll canonical progress",
|
|
862
|
+
message: error.message,
|
|
863
|
+
cause: error,
|
|
884
864
|
}),
|
|
885
865
|
),
|
|
886
|
-
);
|
|
887
|
-
|
|
888
|
-
yield* progress.committed("canonical").pipe(
|
|
889
|
-
Effect.catchCause((cause) =>
|
|
890
|
-
Effect.failCause(
|
|
891
|
-
Cause.map(cause, (error) =>
|
|
892
|
-
DoStorageError.make({
|
|
893
|
-
operation: "enroll canonical progress",
|
|
894
|
-
message: error.message,
|
|
895
|
-
cause: error,
|
|
896
|
-
}),
|
|
897
|
-
),
|
|
898
|
-
),
|
|
899
866
|
),
|
|
900
|
-
)
|
|
867
|
+
),
|
|
868
|
+
);
|
|
901
869
|
|
|
902
|
-
|
|
903
|
-
|
|
870
|
+
return result;
|
|
871
|
+
});
|
|
904
872
|
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
873
|
+
const append = Effect.fnUntraced(
|
|
874
|
+
function* (
|
|
875
|
+
request: RawAppendRequest,
|
|
876
|
+
observed: ThreadRow,
|
|
877
|
+
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
878
|
+
yield* prepareAppend(request);
|
|
879
|
+
// The facade already read this tail. Reject a known stale writer without opening a
|
|
880
|
+
// transaction; a matching tail is still checked atomically below. Replays precede tail
|
|
881
|
+
// conflicts, preserving retry identity even after the log has advanced.
|
|
882
|
+
if (request.records.length > 0 && request.expectedTailSequence !== observed.tail_sequence) {
|
|
883
|
+
const batches = yield* sql`
|
|
916
884
|
SELECT batch_id FROM effect_agent_canonical_batches
|
|
917
885
|
WHERE thread_id = ${request.threadId} AND batch_id = ${JSON.stringify(request.batchId)}
|
|
918
886
|
UNION ALL SELECT last_commit_id FROM effect_agent_executions
|
|
@@ -920,47 +888,47 @@ const makeJournal = (
|
|
|
920
888
|
LIMIT 1
|
|
921
889
|
`.pipe(Effect.mapError(storageError("preflight append replay")));
|
|
922
890
|
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
}
|
|
891
|
+
// A possible replay still checks exact content in the transaction.
|
|
892
|
+
if (batches.length === 0) {
|
|
893
|
+
return yield* DoAppendConflict.make({
|
|
894
|
+
reason: "tail",
|
|
895
|
+
message: "The canonical tail has advanced.",
|
|
896
|
+
actualTailSequence: observed.tail_sequence,
|
|
897
|
+
});
|
|
931
898
|
}
|
|
899
|
+
}
|
|
932
900
|
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
901
|
+
return yield* withWriteTransaction(
|
|
902
|
+
"append transaction",
|
|
903
|
+
isAppendContention,
|
|
904
|
+
)(appendPrepared(request));
|
|
905
|
+
},
|
|
906
|
+
withStorageSpan("DoJournal.append", isAppendContention),
|
|
907
|
+
);
|
|
940
908
|
|
|
941
|
-
|
|
942
|
-
|
|
909
|
+
const read = Effect.fnUntraced(function* (request: RawReadRequest) {
|
|
910
|
+
const thread = (yield* getThread(request.threadId))[0];
|
|
943
911
|
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
912
|
+
const through = Math.min(
|
|
913
|
+
thread?.tail_sequence ?? 0,
|
|
914
|
+
request.fromSequenceExclusive + request.limit,
|
|
915
|
+
);
|
|
948
916
|
|
|
949
|
-
|
|
917
|
+
const cached: Array<RecordRow> = [];
|
|
950
918
|
|
|
951
|
-
|
|
952
|
-
|
|
919
|
+
for (let sequence = request.fromSequenceExclusive + 1; sequence <= through; sequence++) {
|
|
920
|
+
const record = recordCache.get(request.threadId, sequence);
|
|
953
921
|
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
922
|
+
if (record === undefined) break;
|
|
923
|
+
cached.push(record);
|
|
924
|
+
}
|
|
925
|
+
if (cached.length === Math.max(0, through - request.fromSequenceExclusive)) {
|
|
926
|
+
return { count: cached.length, records: Stream.fromIterable(cached) };
|
|
927
|
+
}
|
|
960
928
|
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
929
|
+
// Capture membership without retaining payloads. Append-only sequences keep each later
|
|
930
|
+
// payload query inside this snapshot, even when new records arrive during consumption.
|
|
931
|
+
const planRows = yield* sql<Record<string, unknown>>`
|
|
964
932
|
SELECT
|
|
965
933
|
sequence,
|
|
966
934
|
length(CAST(${recordJson} AS BLOB)) AS record_json_bytes
|
|
@@ -971,44 +939,43 @@ const makeJournal = (
|
|
|
971
939
|
LIMIT ${request.limit}
|
|
972
940
|
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
973
941
|
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
942
|
+
const plan = yield* decodeRows(
|
|
943
|
+
Schema.Array(ReadPlanRow),
|
|
944
|
+
"effect_agent_canonical_records",
|
|
945
|
+
`${request.threadId}>${request.fromSequenceExclusive}`,
|
|
946
|
+
planRows,
|
|
947
|
+
"ReadPlanRow",
|
|
948
|
+
);
|
|
981
949
|
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
});
|
|
950
|
+
const mismatch = () =>
|
|
951
|
+
DoStorageCorruptionError.make({
|
|
952
|
+
table: "effect_agent_canonical_records",
|
|
953
|
+
rowKey: `${request.threadId}>${request.fromSequenceExclusive}`,
|
|
954
|
+
message: "Canonical read membership, sequence, or payload size changed from its read plan.",
|
|
955
|
+
});
|
|
989
956
|
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
}
|
|
957
|
+
if (plan.length > request.limit) return yield* mismatch();
|
|
958
|
+
|
|
959
|
+
const pages: Array<ReadPage> = [];
|
|
960
|
+
let page: ReadPage | undefined;
|
|
961
|
+
let pageBytes = 0;
|
|
962
|
+
let previousSequence = request.fromSequenceExclusive;
|
|
963
|
+
|
|
964
|
+
for (const row of plan) {
|
|
965
|
+
if (row.sequence !== previousSequence + 1) return yield* mismatch();
|
|
966
|
+
previousSequence = row.sequence;
|
|
967
|
+
if (page === undefined || pageBytes + row.record_json_bytes > MAX_READ_PAGE_JSON_BYTES) {
|
|
968
|
+
page = [row];
|
|
969
|
+
pages.push(page);
|
|
970
|
+
pageBytes = row.record_json_bytes;
|
|
971
|
+
} else {
|
|
972
|
+
page.push(row);
|
|
973
|
+
pageBytes += row.record_json_bytes;
|
|
1008
974
|
}
|
|
975
|
+
}
|
|
1009
976
|
|
|
1010
|
-
|
|
1011
|
-
|
|
977
|
+
const readPage = Effect.fnUntraced(function* (page: ReadPage) {
|
|
978
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
1012
979
|
SELECT thread_id, sequence, record_id, ${recordBatchId(sql)} AS batch_id, ${recordJson} AS record_json
|
|
1013
980
|
FROM effect_agent_canonical_records
|
|
1014
981
|
WHERE thread_id = ${request.threadId}
|
|
@@ -1017,77 +984,77 @@ const makeJournal = (
|
|
|
1017
984
|
ORDER BY sequence
|
|
1018
985
|
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
1019
986
|
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
if (
|
|
1029
|
-
decoded.length !== page.length ||
|
|
1030
|
-
decoded.some(
|
|
1031
|
-
(row, index) =>
|
|
1032
|
-
row.thread_id !== request.threadId ||
|
|
1033
|
-
row.sequence !== page[index].sequence ||
|
|
1034
|
-
storedTextBytes(row.record_json) !== page[index].record_json_bytes,
|
|
1035
|
-
)
|
|
1036
|
-
) {
|
|
1037
|
-
return yield* mismatch();
|
|
1038
|
-
}
|
|
987
|
+
const decoded = yield* decodeRows(
|
|
988
|
+
Schema.Array(RecordRow),
|
|
989
|
+
"effect_agent_canonical_records",
|
|
990
|
+
`${request.threadId}/${page[0].sequence}`,
|
|
991
|
+
rows,
|
|
992
|
+
"RecordRow",
|
|
993
|
+
);
|
|
1039
994
|
|
|
1040
|
-
|
|
995
|
+
if (
|
|
996
|
+
decoded.length !== page.length ||
|
|
997
|
+
decoded.some(
|
|
998
|
+
(row, index) =>
|
|
999
|
+
row.thread_id !== request.threadId ||
|
|
1000
|
+
row.sequence !== page[index].sequence ||
|
|
1001
|
+
storedTextBytes(row.record_json) !== page[index].record_json_bytes,
|
|
1002
|
+
)
|
|
1003
|
+
) {
|
|
1004
|
+
return yield* mismatch();
|
|
1005
|
+
}
|
|
1041
1006
|
|
|
1042
|
-
|
|
1043
|
-
});
|
|
1007
|
+
for (const row of decoded) recordCache.put(row);
|
|
1044
1008
|
|
|
1045
|
-
return
|
|
1046
|
-
count: plan.length,
|
|
1047
|
-
records: Stream.fromIterable(pages).pipe(
|
|
1048
|
-
Stream.flatMap((page) => Stream.fromIterableEffect(state.read(readPage(page)))),
|
|
1049
|
-
),
|
|
1050
|
-
};
|
|
1009
|
+
return decoded;
|
|
1051
1010
|
});
|
|
1052
1011
|
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
});
|
|
1061
|
-
}
|
|
1062
|
-
yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
|
|
1063
|
-
yield* withWriteTransaction("checkpoint transaction")(
|
|
1064
|
-
Effect.gen(function* () {
|
|
1065
|
-
const threadRows = yield* getThread(checkpoint.threadId);
|
|
1012
|
+
return {
|
|
1013
|
+
count: plan.length,
|
|
1014
|
+
records: Stream.fromIterable(pages).pipe(
|
|
1015
|
+
Stream.flatMap((page) => Stream.fromIterableEffect(state.read(readPage(page)))),
|
|
1016
|
+
),
|
|
1017
|
+
};
|
|
1018
|
+
});
|
|
1066
1019
|
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1020
|
+
const saveCheckpoint = Effect.fnUntraced(function* (
|
|
1021
|
+
checkpoint: RawCheckpoint,
|
|
1022
|
+
): Effect.fn.Return<void, CheckpointError> {
|
|
1023
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH) {
|
|
1024
|
+
return yield* DoStorageError.make({
|
|
1025
|
+
operation: "save checkpoint",
|
|
1026
|
+
message: "Checkpoint identity exceeds the Durable Object storage bounds.",
|
|
1027
|
+
});
|
|
1028
|
+
}
|
|
1029
|
+
yield* checkValueBound("save checkpoint", checkpoint.checkpointJson);
|
|
1030
|
+
yield* withWriteTransaction("checkpoint transaction")(
|
|
1031
|
+
Effect.gen(function* () {
|
|
1032
|
+
const threadRows = yield* getThread(checkpoint.threadId);
|
|
1033
|
+
|
|
1034
|
+
const thread = yield* decodeSingleRow(
|
|
1035
|
+
Schema.Array(ThreadRow),
|
|
1036
|
+
"effect_agent_threads",
|
|
1037
|
+
checkpoint.threadId,
|
|
1038
|
+
threadRows,
|
|
1039
|
+
);
|
|
1073
1040
|
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1041
|
+
if (checkpoint.throughSequence > thread.tail_sequence) {
|
|
1042
|
+
return yield* DoCheckpointConflict.make({
|
|
1043
|
+
reason: "ahead-of-tail",
|
|
1044
|
+
message:
|
|
1045
|
+
`Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
|
|
1046
|
+
`${thread.tail_sequence}.`,
|
|
1047
|
+
});
|
|
1048
|
+
}
|
|
1082
1049
|
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1050
|
+
if (!(yield* isBatchBoundary(checkpoint.threadId, checkpoint.throughSequence))) {
|
|
1051
|
+
return yield* DoCheckpointConflict.make({
|
|
1052
|
+
reason: "not-batch-boundary",
|
|
1053
|
+
message: "Checkpoint sequence is not the end of a complete canonical batch.",
|
|
1054
|
+
});
|
|
1055
|
+
}
|
|
1089
1056
|
|
|
1090
|
-
|
|
1057
|
+
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
1091
1058
|
SELECT
|
|
1092
1059
|
thread_id,
|
|
1093
1060
|
through_sequence,
|
|
@@ -1097,32 +1064,32 @@ const makeJournal = (
|
|
|
1097
1064
|
AND through_sequence = ${checkpoint.throughSequence}
|
|
1098
1065
|
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
1099
1066
|
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1067
|
+
const existing = yield* decodeRows(
|
|
1068
|
+
Schema.Array(CheckpointRow),
|
|
1069
|
+
"effect_agent_checkpoints",
|
|
1070
|
+
`${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
1071
|
+
checkpointRows,
|
|
1072
|
+
);
|
|
1106
1073
|
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1074
|
+
if (existing.length > 1) {
|
|
1075
|
+
return yield* DoStorageCorruptionError.make({
|
|
1076
|
+
table: "effect_agent_checkpoints",
|
|
1077
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
1078
|
+
message: "A checkpoint primary key returned more than one row.",
|
|
1079
|
+
});
|
|
1080
|
+
}
|
|
1081
|
+
if (existing.length === 1) {
|
|
1082
|
+
if (existing[0].checkpoint_json !== checkpoint.checkpointJson) {
|
|
1083
|
+
return yield* DoCheckpointConflict.make({
|
|
1084
|
+
reason: "corrupt",
|
|
1085
|
+
message: "A different checkpoint already exists at this canonical sequence.",
|
|
1112
1086
|
});
|
|
1113
1087
|
}
|
|
1114
|
-
if (existing.length === 1) {
|
|
1115
|
-
if (existing[0].checkpoint_json !== checkpoint.checkpointJson) {
|
|
1116
|
-
return yield* DoCheckpointConflict.make({
|
|
1117
|
-
reason: "corrupt",
|
|
1118
|
-
message: "A different checkpoint already exists at this canonical sequence.",
|
|
1119
|
-
});
|
|
1120
|
-
}
|
|
1121
1088
|
|
|
1122
|
-
|
|
1123
|
-
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1124
1091
|
|
|
1125
|
-
|
|
1092
|
+
yield* sql`
|
|
1126
1093
|
INSERT INTO effect_agent_checkpoints (
|
|
1127
1094
|
thread_id,
|
|
1128
1095
|
through_sequence,
|
|
@@ -1133,15 +1100,15 @@ const makeJournal = (
|
|
|
1133
1100
|
${checkpoint.checkpointJson}
|
|
1134
1101
|
)
|
|
1135
1102
|
`.pipe(Effect.mapError(storageError("insert checkpoint")));
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1103
|
+
}),
|
|
1104
|
+
);
|
|
1105
|
+
});
|
|
1139
1106
|
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1107
|
+
const loadCheckpoint = Effect.fnUntraced(function* (
|
|
1108
|
+
threadId: string,
|
|
1109
|
+
atOrBeforeSequence: CanonicalSequence,
|
|
1110
|
+
) {
|
|
1111
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
1145
1112
|
SELECT
|
|
1146
1113
|
thread_id,
|
|
1147
1114
|
through_sequence,
|
|
@@ -1153,21 +1120,21 @@ const makeJournal = (
|
|
|
1153
1120
|
LIMIT 1
|
|
1154
1121
|
`.pipe(Effect.mapError(storageError("load checkpoint")));
|
|
1155
1122
|
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1123
|
+
return yield* decodeRows(
|
|
1124
|
+
Schema.Array(CheckpointRow),
|
|
1125
|
+
"effect_agent_checkpoints",
|
|
1126
|
+
`${threadId}<=${atOrBeforeSequence}`,
|
|
1127
|
+
rows,
|
|
1128
|
+
);
|
|
1129
|
+
});
|
|
1163
1130
|
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1131
|
+
const isBatchBoundary = Effect.fnUntraced(function* (
|
|
1132
|
+
threadId: string,
|
|
1133
|
+
sequence: CanonicalSequence,
|
|
1134
|
+
) {
|
|
1135
|
+
if (sequence === 0) return true;
|
|
1169
1136
|
|
|
1170
|
-
|
|
1137
|
+
const rows = yield* sql<Record<string, unknown>>`
|
|
1171
1138
|
SELECT last_sequence FROM effect_agent_canonical_batches
|
|
1172
1139
|
WHERE thread_id = ${threadId}
|
|
1173
1140
|
AND batch_id = (
|
|
@@ -1177,207 +1144,207 @@ const makeJournal = (
|
|
|
1177
1144
|
AND last_sequence = ${sequence}
|
|
1178
1145
|
`.pipe(Effect.mapError(storageError("read canonical batch boundary")));
|
|
1179
1146
|
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1147
|
+
const batches = yield* decodeRows(
|
|
1148
|
+
Schema.Array(Schema.Struct({ last_sequence: CanonicalSequence })),
|
|
1149
|
+
"effect_agent_canonical_batches",
|
|
1150
|
+
`${threadId}/${sequence}`,
|
|
1151
|
+
rows,
|
|
1152
|
+
);
|
|
1186
1153
|
|
|
1187
|
-
|
|
1188
|
-
|
|
1154
|
+
return batches.length === 1;
|
|
1155
|
+
});
|
|
1189
1156
|
|
|
1190
|
-
|
|
1191
|
-
|
|
1157
|
+
const sourceFailure = (cause: unknown) =>
|
|
1158
|
+
LifecyclePublicationError.make({ reason: "unavailable", cause });
|
|
1192
1159
|
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1160
|
+
const sourceThreads = Effect.fnUntraced(function* () {
|
|
1161
|
+
const all = yield* threads.matching(
|
|
1162
|
+
"lifecycle-owners",
|
|
1163
|
+
(row) => row.created_at !== null,
|
|
1164
|
+
sql`SELECT * FROM effect_agent_threads WHERE created_at IS NOT NULL LIMIT 129`,
|
|
1165
|
+
128,
|
|
1166
|
+
);
|
|
1200
1167
|
|
|
1201
|
-
|
|
1168
|
+
if (all.length <= 128) return all;
|
|
1202
1169
|
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1170
|
+
// Large custom hosts hydrate only owners whose journal has an unmaterialized suffix.
|
|
1171
|
+
return yield* Schema.decodeUnknownEffect(Schema.Array(ThreadRow))(
|
|
1172
|
+
yield* sql`SELECT t.* FROM effect_agent_threads t LEFT JOIN effect_agent_lifecycle_cursors c ON c.thread_id=t.thread_id
|
|
1206
1173
|
WHERE t.created_at IS NOT NULL AND (c.thread_id IS NULL OR t.tail_sequence > c.through_sequence) ORDER BY t.thread_id LIMIT 128`,
|
|
1207
|
-
|
|
1208
|
-
|
|
1174
|
+
);
|
|
1175
|
+
});
|
|
1209
1176
|
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1177
|
+
const initializeLifecycleSource = Effect.fnUntraced(function* () {
|
|
1178
|
+
if (initializedLifecycleSources.has(state)) return;
|
|
1179
|
+
yield* state
|
|
1180
|
+
.transaction(
|
|
1181
|
+
Effect.gen(function* () {
|
|
1182
|
+
const existing =
|
|
1183
|
+
yield* sql`SELECT name FROM sqlite_master WHERE name='effect_agent_lifecycle_cursors'`;
|
|
1217
1184
|
|
|
1218
|
-
|
|
1219
|
-
|
|
1185
|
+
if (existing.length === 0) {
|
|
1186
|
+
yield* sql`CREATE TABLE effect_agent_lifecycle_cursors (
|
|
1220
1187
|
thread_id TEXT PRIMARY KEY, through_sequence INTEGER NOT NULL
|
|
1221
1188
|
)`;
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1189
|
+
// Existing inline writers already retained their facts. Enabling publication on an
|
|
1190
|
+
// old store starts here too; it does not publish historical facts retroactively.
|
|
1191
|
+
}
|
|
1192
|
+
// New Threads created with publication disabled have no cursor. Never advance
|
|
1193
|
+
// existing cursors: they can owe intent from an interrupted Attempt.
|
|
1194
|
+
yield* sql`INSERT INTO effect_agent_lifecycle_cursors(thread_id, through_sequence)
|
|
1228
1195
|
SELECT thread_id, tail_sequence FROM effect_agent_threads WHERE created_at IS NOT NULL
|
|
1229
1196
|
ON CONFLICT(thread_id) DO NOTHING`;
|
|
1230
|
-
|
|
1197
|
+
yield* sourceThreads();
|
|
1198
|
+
}),
|
|
1199
|
+
)
|
|
1200
|
+
.pipe(
|
|
1201
|
+
Effect.provideService(SqlClient.SqlClient, sql),
|
|
1202
|
+
Effect.mapError((cause) =>
|
|
1203
|
+
DoStorageError.make({
|
|
1204
|
+
operation: "initialize lifecycle source",
|
|
1205
|
+
message: "Lifecycle source cursor unavailable",
|
|
1206
|
+
cause,
|
|
1231
1207
|
}),
|
|
1232
|
-
)
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
DoStorageError.make({
|
|
1237
|
-
operation: "initialize lifecycle source",
|
|
1238
|
-
message: "Lifecycle source cursor unavailable",
|
|
1239
|
-
cause,
|
|
1240
|
-
}),
|
|
1241
|
-
),
|
|
1242
|
-
);
|
|
1243
|
-
initializedLifecycleSources.add(state);
|
|
1244
|
-
});
|
|
1245
|
-
|
|
1246
|
-
/** The journal is durable intent; only the bounded publication wave materializes a write set. */
|
|
1247
|
-
const flushCanonical = Effect.fnUntraced(function* (threadId: string, maxRecords = Infinity) {
|
|
1248
|
-
const { retainMany } = yield* SqlLifecycleRetainer;
|
|
1249
|
-
const thread = (yield* getThread(threadId))[0];
|
|
1250
|
-
|
|
1251
|
-
if (thread === undefined) return;
|
|
1252
|
-
const cursor = (yield* cursors.by("thread_id", threadId))[0];
|
|
1208
|
+
),
|
|
1209
|
+
);
|
|
1210
|
+
initializedLifecycleSources.add(state);
|
|
1211
|
+
});
|
|
1253
1212
|
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1213
|
+
/** The journal is durable intent; only the bounded publication wave materializes a write set. */
|
|
1214
|
+
const flushCanonical = Effect.fnUntraced(function* (threadId: string, maxRecords = Infinity) {
|
|
1215
|
+
const { retainMany } = yield* SqlLifecycleRetainer;
|
|
1216
|
+
const thread = (yield* getThread(threadId))[0];
|
|
1217
|
+
|
|
1218
|
+
if (thread === undefined) return;
|
|
1219
|
+
const cursor = (yield* cursors.by("thread_id", threadId))[0];
|
|
1220
|
+
|
|
1221
|
+
if (cursor === undefined || cursor.through_sequence > thread.tail_sequence)
|
|
1222
|
+
return yield* LifecyclePublicationError.make({ reason: "corrupt" });
|
|
1223
|
+
let through = cursor.through_sequence;
|
|
1224
|
+
const bound = Math.min(thread.tail_sequence, through + maxRecords);
|
|
1225
|
+
|
|
1226
|
+
while (through < bound) {
|
|
1227
|
+
const page = yield* read(
|
|
1228
|
+
RawReadRequest.make({
|
|
1229
|
+
threadId,
|
|
1230
|
+
fromSequenceExclusive: through,
|
|
1231
|
+
limit: Math.min(128, bound - through),
|
|
1232
|
+
}),
|
|
1233
|
+
);
|
|
1267
1234
|
|
|
1268
|
-
|
|
1269
|
-
|
|
1235
|
+
const facts: Array<Parameters<SqlLifecycleRetainMany>[0][number]> = [];
|
|
1236
|
+
let bytes = 0;
|
|
1270
1237
|
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1238
|
+
yield* Stream.runForEach(page.records, (row) =>
|
|
1239
|
+
Effect.gen(function* () {
|
|
1240
|
+
const record = yield* Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(
|
|
1241
|
+
row.record_json,
|
|
1242
|
+
);
|
|
1276
1243
|
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
}
|
|
1288
|
-
facts.push({
|
|
1289
|
-
id: JSON.stringify([threadId, "record", record.recordId]),
|
|
1290
|
-
ownerThreadId: yield* Schema.decodeEffect(ThreadId)(threadId),
|
|
1291
|
-
canonicalSequence: row.sequence,
|
|
1292
|
-
createdAt: record.createdAt,
|
|
1293
|
-
fact: record.payload,
|
|
1294
|
-
});
|
|
1295
|
-
bytes += storedTextBytes(row.record_json);
|
|
1244
|
+
if (record.recordId !== row.record_id || row.sequence !== through + 1)
|
|
1245
|
+
return yield* LifecyclePublicationError.make({ reason: "corrupt" });
|
|
1246
|
+
if (Schema.is(Schema.toType(LifecyclePublicationFact))(record.payload)) {
|
|
1247
|
+
if (
|
|
1248
|
+
bytes + storedTextBytes(row.record_json) > MAX_READ_PAGE_JSON_BYTES &&
|
|
1249
|
+
facts.length > 0
|
|
1250
|
+
) {
|
|
1251
|
+
yield* retainMany(facts);
|
|
1252
|
+
facts.length = 0;
|
|
1253
|
+
bytes = 0;
|
|
1296
1254
|
}
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1255
|
+
facts.push({
|
|
1256
|
+
id: JSON.stringify([threadId, "record", record.recordId]),
|
|
1257
|
+
ownerThreadId: yield* Schema.decodeEffect(ThreadId)(threadId),
|
|
1258
|
+
canonicalSequence: row.sequence,
|
|
1259
|
+
createdAt: record.createdAt,
|
|
1260
|
+
fact: record.payload,
|
|
1261
|
+
});
|
|
1262
|
+
bytes += storedTextBytes(row.record_json);
|
|
1263
|
+
}
|
|
1264
|
+
through = row.sequence;
|
|
1265
|
+
}),
|
|
1266
|
+
);
|
|
1267
|
+
if (page.count === 0) return yield* LifecyclePublicationError.make({ reason: "corrupt" });
|
|
1268
|
+
yield* retainMany(facts);
|
|
1269
|
+
yield* sql`UPDATE effect_agent_lifecycle_cursors SET through_sequence=${through} WHERE thread_id=${threadId} RETURNING *`.pipe(
|
|
1270
|
+
cursors.write,
|
|
1271
|
+
);
|
|
1272
|
+
}
|
|
1273
|
+
}, Effect.mapError(sourceFailure));
|
|
1310
1274
|
|
|
1311
|
-
|
|
1312
|
-
|
|
1275
|
+
const flushPublications = Effect.fnUntraced(function* () {
|
|
1276
|
+
let owners = 0;
|
|
1313
1277
|
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
if (++owners === 4) break;
|
|
1317
|
-
}
|
|
1318
|
-
}, Effect.mapError(sourceFailure));
|
|
1278
|
+
for (const thread of yield* sourceThreads()) {
|
|
1279
|
+
const cursor = (yield* cursors.by("thread_id", thread.thread_id))[0];
|
|
1319
1280
|
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1281
|
+
if (cursor !== undefined && cursor.through_sequence === thread.tail_sequence) continue;
|
|
1282
|
+
yield* flushCanonical(thread.thread_id, 1024);
|
|
1283
|
+
if (++owners === 4) break;
|
|
1284
|
+
}
|
|
1285
|
+
}, Effect.mapError(sourceFailure));
|
|
1323
1286
|
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
}
|
|
1287
|
+
const sourcePending = Effect.gen(function* () {
|
|
1288
|
+
for (const thread of yield* sourceThreads()) {
|
|
1289
|
+
const cursor = (yield* cursors.by("thread_id", thread.thread_id))[0];
|
|
1328
1290
|
|
|
1329
|
-
|
|
1330
|
-
|
|
1291
|
+
if (cursor === undefined || cursor.through_sequence > thread.tail_sequence)
|
|
1292
|
+
return yield* LifecyclePublicationError.make({ reason: "corrupt" });
|
|
1293
|
+
if (cursor.through_sequence < thread.tail_sequence) return true;
|
|
1294
|
+
}
|
|
1331
1295
|
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1296
|
+
return false;
|
|
1297
|
+
}).pipe(Effect.mapError(sourceFailure));
|
|
1298
|
+
|
|
1299
|
+
const ownedLifecycle =
|
|
1300
|
+
lifecycle === undefined
|
|
1301
|
+
? undefined
|
|
1302
|
+
: {
|
|
1303
|
+
...lifecycle,
|
|
1304
|
+
storage: {
|
|
1305
|
+
...lifecycle.storage,
|
|
1306
|
+
retainedPendingDeadline: lifecycle.storage.pendingDeadline,
|
|
1307
|
+
pendingDeadline: state.read(
|
|
1308
|
+
Effect.gen(function* () {
|
|
1309
|
+
const deadline = yield* lifecycle.storage.pendingDeadline;
|
|
1310
|
+
|
|
1311
|
+
if (!(yield* sourcePending)) return deadline;
|
|
1312
|
+
const now = yield* Clock.currentTimeMillis;
|
|
1313
|
+
|
|
1314
|
+
return Option.some(Option.isSome(deadline) ? Math.min(now, deadline.value) : now);
|
|
1315
|
+
}),
|
|
1316
|
+
),
|
|
1317
|
+
},
|
|
1318
|
+
};
|
|
1352
1319
|
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1320
|
+
return {
|
|
1321
|
+
archives: archives.storage,
|
|
1322
|
+
archiveRanges: archives,
|
|
1323
|
+
work,
|
|
1324
|
+
state,
|
|
1325
|
+
owner,
|
|
1326
|
+
threads,
|
|
1327
|
+
lifecycle: ownedLifecycle,
|
|
1328
|
+
initializeLifecycleSource,
|
|
1329
|
+
flushCanonical,
|
|
1330
|
+
flushPublications,
|
|
1331
|
+
append,
|
|
1332
|
+
prepareAppend,
|
|
1333
|
+
appendPrepared,
|
|
1334
|
+
checkCommitIntent: (request: RawAppendRequest) =>
|
|
1335
|
+
executionConflict(executions.checkCommitIntent(request)),
|
|
1336
|
+
readExecution: executions.read,
|
|
1337
|
+
checkValueBound,
|
|
1338
|
+
getThread,
|
|
1339
|
+
hasRecord,
|
|
1340
|
+
isBatchBoundary,
|
|
1341
|
+
loadCheckpoint,
|
|
1342
|
+
materialize,
|
|
1343
|
+
read: (request: RawReadRequest) => state.read(read(request)),
|
|
1344
|
+
saveCheckpoint,
|
|
1345
|
+
withWriteTransaction,
|
|
1346
|
+
} as const;
|
|
1347
|
+
});
|
|
1381
1348
|
|
|
1382
1349
|
export type DoJournal = Effect.Success<ReturnType<typeof makeJournal>>;
|
|
1383
1350
|
|