@effect-agent/storage-sqlite 0.1.0-beta.38 → 0.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +34 -114
- package/dist/index.mjs +1693 -494
- package/dist/index.mjs.map +1 -1
- package/dist/sqlite-storage-failpoint-Cr0SXflP.d.mts +95 -0
- package/dist/sqlite-storage-failpoint-DIwWk5dw.mjs +12 -0
- package/dist/sqlite-storage-failpoint-DIwWk5dw.mjs.map +1 -0
- package/dist/testing.d.mts +15 -0
- package/dist/testing.mjs +19 -0
- package/dist/testing.mjs.map +1 -0
- package/package.json +14 -6
- package/src/errors.ts +3 -3
- package/src/index.ts +5 -2
- package/src/migrations.ts +106 -84
- package/src/sqlite-activity-store.ts +503 -0
- package/src/sqlite-journal.ts +147 -156
- package/src/sqlite-ledger.ts +60 -60
- package/src/sqlite-memory-store.ts +532 -0
- package/src/sqlite-schedule-store.ts +2 -2
- package/src/sqlite-storage-config.ts +1 -1
- package/src/sqlite-storage-failpoint-testing.ts +38 -0
- package/src/sqlite-storage-failpoint.ts +1 -31
- package/src/sqlite-subscription-store.ts +935 -0
- package/src/{sqlite-conversation-store.ts → sqlite-thread-store.ts} +248 -267
- package/src/testing.ts +2 -0
package/dist/index.mjs
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as SqliteStorageFailpoint } from "./sqlite-storage-failpoint-DIwWk5dw.mjs";
|
|
2
|
+
import { AbortCommand, AbortIntent, AcceptedEvent, ActivityBusy, ActivityClaim, ActivityClaimRequest, ActivityMutationFailpoint, ActivityOwnershipLost, ActivityProcessorKey, ActivityProcessorStore, ActivityProgress, ActivityStoreError, ActivityWorkConflict, AdmissionAdmitted, AdmissionConflict, AdmissionNotAdmitted, AdmissionRequest, AdmissionResult, AppendConflict, AppendResult, ApprovalConflict, ApprovalDecision, ApprovalDecisionCommand, ApprovalDecisionIntent, AttachChildToReservationRequest, BeginChildBudgetReleaseRequest, CanonicalBatch, CanonicalRecord, CanonicalRecordEnvelope, CanonicalSequence, CheckpointRejected, ChildAttachmentSnapshot, ChildBudgetReservationRequest, ChildBudgetReservationSnapshot, ChildReservationConflict, ChildReservationStatus, ChildSettledNotification, Claim, ClaimJoiningRequest, ClaimRequest, DEFAULT_OWNERSHIP_LEASE_DURATION, DefinitionDigests, DeliveryChange, Digest, EMPTY_TAIL_DIGEST, FenceRejected, FencedAppendRequest, InputAppliedMarker, JoinSnapshot, JoinedToHost, JoiningClaim, LedgerCapabilities, LedgerError, LoadCheckpointRequest, MarkInputAppliedRequest, MarkJoinedRequest, MarkReadyRequest, MarkUnknownRequest, ObservationOffset, OwnershipLost, OwnershipRenewal, OwnershipSnapshot, ParentLinkage, PersistedJson, PreparedActivity, ProducerEpoch, QueueSequence, RecordEnvelope, RecoverySnapshot, RecoverySnapshotRequest, ReleaseChildBudgetRequest, ReleaseOwnershipRequest, RenewOwnershipRequest, ReservedChildBudget, ReservedSettlement, RevertJoiningRequest, SaveCheckpointRequest, ScheduleCapacityError, ScheduleChange, ScheduleConflict, ScheduleDueCursor, ScheduleFailpoint, ScheduleId, ScheduleInstant, ScheduleKey, ScheduleNotFound, ScheduleOwner, SchedulePageRequest, ScheduleRecord, ScheduleStorageError, ScheduleStore, Settlement, SettlementConflict, SettlementFinalization, SettlementOutcome, SettlementReservation, SettlementReservationSnapshot, SourcePartition, SubmissionLedger, SubmissionLookup, SubmissionLookupByKey, SubmissionSnapshot, SubmissionState, SubscriptionDelivery, SubscriptionDeliveryKey, SubscriptionError, SubscriptionFailpoint, SubscriptionKey, SubscriptionLimits, SubscriptionName, SubscriptionRecord, SubscriptionScanCursors, SubscriptionStore, SuspendRequest, SuspensionReason, SuspensionSnapshot, ThreadCheckpoint, ThreadExport, ThreadExportRequest, ThreadMaterialization, ThreadNotMaterialized, ThreadObservation, ThreadRead, ThreadStore, ThreadStoreError, ThreadTail, ThreadTailRequest, UnknownResolution, UnknownResolutionCommand, UnknownResolutionConflict, UnknownResolutionIntent, applyScheduleChange, applySubscriptionDeliveryChange, defaultSchedulingLimits, digestCanonicalBatch, sameAcceptedEventIdentity, sameSourcePartition, scheduleDeadline, scheduleUsesCapacity, settlementFailureFromRecord, submissionAbortRecordId, subscriptionCanSelect, subscriptionDeliveryCanSelect, subscriptionDeliveryKeyString } from "@effect-agent/thread";
|
|
2
3
|
import { Clock, Context, Crypto, DateTime, Duration, Effect, Exit, Layer, Option, Ref, Result, Schema, Stream } from "effect";
|
|
3
4
|
import { SqliteClient, SqliteMigrator } from "@effect/sql-sqlite-node";
|
|
4
5
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
5
6
|
import { NodeCrypto } from "@effect/platform-node";
|
|
7
|
+
import { MemoryDocument, MemoryKey, MemoryMutationFailpoint, MemoryNamespaceAddress, MemoryOperationConflict, MemoryReader, MemoryStorageError, MemoryWrite, MemoryWriter, applyMemoryWrite } from "@effect-agent/core";
|
|
6
8
|
//#region src/errors.ts
|
|
7
9
|
/** The SQLite file uses a private-development format this adapter cannot read. */
|
|
8
10
|
var SqliteStorageCompatibilityError = class extends Schema.TaggedError()("SqliteStorageCompatibilityError", {
|
|
@@ -47,7 +49,7 @@ var SqliteAppendConflict = class extends Schema.TaggedError()("SqliteAppendConfl
|
|
|
47
49
|
actualTailDigest: Schema.optionalKey(Schema.String)
|
|
48
50
|
}) {};
|
|
49
51
|
/**
|
|
50
|
-
* A producer epoch does not match the
|
|
52
|
+
* A producer epoch does not match the Thread's current writer registration. Appends
|
|
51
53
|
* require the exact registered epoch, so both older and newer unregistered epochs are fenced;
|
|
52
54
|
* a newer epoch takes over by materializing first.
|
|
53
55
|
*/
|
|
@@ -76,7 +78,7 @@ const SqliteStorageFailpointLocation = Schema.Literals([
|
|
|
76
78
|
"append:after-record-insert",
|
|
77
79
|
"append:after-tail-update",
|
|
78
80
|
"append:after",
|
|
79
|
-
"export:after-
|
|
81
|
+
"export:after-thread-read",
|
|
80
82
|
"save-checkpoint:before",
|
|
81
83
|
"save-checkpoint:after",
|
|
82
84
|
"ledger:admit:before",
|
|
@@ -130,72 +132,68 @@ var SqliteStorageFailpointError = class extends Schema.TaggedError()("SqliteStor
|
|
|
130
132
|
};
|
|
131
133
|
//#endregion
|
|
132
134
|
//#region src/migrations.ts
|
|
133
|
-
const CurrentSqliteStorageVersion =
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
CREATE TABLE
|
|
139
|
-
|
|
135
|
+
const CurrentSqliteStorageVersion = 7;
|
|
136
|
+
/** Initialize empty storage with the complete current schema. */
|
|
137
|
+
const sqliteMigrations = SqliteMigrator.fromRecord({ "1_current_thread_storage": Effect.gen(function* () {
|
|
138
|
+
const sql = yield* SqlClient.SqlClient;
|
|
139
|
+
yield* sql`
|
|
140
|
+
CREATE TABLE effect_agent_threads (
|
|
141
|
+
thread_id TEXT PRIMARY KEY NOT NULL,
|
|
140
142
|
created_at TEXT NOT NULL,
|
|
141
143
|
tail_sequence INTEGER NOT NULL,
|
|
142
144
|
tail_digest TEXT NOT NULL,
|
|
143
145
|
producer_epoch INTEGER NOT NULL
|
|
144
146
|
)
|
|
145
147
|
`.withoutTransform;
|
|
146
|
-
|
|
148
|
+
yield* sql`
|
|
147
149
|
CREATE TABLE effect_agent_canonical_batches (
|
|
148
|
-
|
|
150
|
+
thread_id TEXT NOT NULL,
|
|
149
151
|
batch_id TEXT NOT NULL,
|
|
150
152
|
first_sequence INTEGER NOT NULL,
|
|
151
153
|
last_sequence INTEGER NOT NULL,
|
|
152
154
|
batch_digest TEXT NOT NULL,
|
|
153
155
|
tail_digest TEXT NOT NULL,
|
|
154
156
|
batch_json TEXT NOT NULL,
|
|
155
|
-
PRIMARY KEY (
|
|
156
|
-
FOREIGN KEY (
|
|
157
|
-
REFERENCES
|
|
157
|
+
PRIMARY KEY (thread_id, batch_id),
|
|
158
|
+
FOREIGN KEY (thread_id)
|
|
159
|
+
REFERENCES effect_agent_threads(thread_id)
|
|
158
160
|
ON DELETE RESTRICT
|
|
159
161
|
)
|
|
160
162
|
`.withoutTransform;
|
|
161
|
-
|
|
163
|
+
yield* sql`
|
|
162
164
|
CREATE TABLE effect_agent_canonical_records (
|
|
163
|
-
|
|
165
|
+
thread_id TEXT NOT NULL,
|
|
164
166
|
sequence INTEGER NOT NULL,
|
|
165
167
|
record_id TEXT NOT NULL,
|
|
166
168
|
batch_id TEXT NOT NULL,
|
|
167
169
|
record_json TEXT NOT NULL,
|
|
168
|
-
PRIMARY KEY (
|
|
169
|
-
UNIQUE (
|
|
170
|
-
FOREIGN KEY (
|
|
171
|
-
REFERENCES effect_agent_canonical_batches(
|
|
170
|
+
PRIMARY KEY (thread_id, sequence),
|
|
171
|
+
UNIQUE (thread_id, record_id),
|
|
172
|
+
FOREIGN KEY (thread_id, batch_id)
|
|
173
|
+
REFERENCES effect_agent_canonical_batches(thread_id, batch_id)
|
|
172
174
|
ON DELETE RESTRICT
|
|
173
175
|
)
|
|
174
176
|
`.withoutTransform;
|
|
175
|
-
|
|
177
|
+
yield* sql`
|
|
176
178
|
CREATE INDEX effect_agent_canonical_records_batch
|
|
177
|
-
ON effect_agent_canonical_records (
|
|
179
|
+
ON effect_agent_canonical_records (thread_id, batch_id, sequence)
|
|
178
180
|
`.withoutTransform;
|
|
179
|
-
|
|
181
|
+
yield* sql`
|
|
180
182
|
CREATE TABLE effect_agent_checkpoints (
|
|
181
|
-
|
|
183
|
+
thread_id TEXT NOT NULL,
|
|
182
184
|
through_sequence INTEGER NOT NULL,
|
|
183
185
|
tail_digest TEXT NOT NULL,
|
|
184
186
|
checkpoint_json TEXT NOT NULL,
|
|
185
|
-
PRIMARY KEY (
|
|
186
|
-
FOREIGN KEY (
|
|
187
|
-
REFERENCES
|
|
187
|
+
PRIMARY KEY (thread_id, through_sequence),
|
|
188
|
+
FOREIGN KEY (thread_id)
|
|
189
|
+
REFERENCES effect_agent_threads(thread_id)
|
|
188
190
|
ON DELETE RESTRICT
|
|
189
191
|
)
|
|
190
192
|
`.withoutTransform;
|
|
191
|
-
|
|
192
|
-
}),
|
|
193
|
-
"2_durable_submission_ledger": Effect.gen(function* () {
|
|
194
|
-
const sql = yield* SqlClient.SqlClient;
|
|
195
|
-
yield* sql`
|
|
193
|
+
yield* sql`
|
|
196
194
|
CREATE TABLE effect_agent_submissions (
|
|
197
195
|
submission_id TEXT PRIMARY KEY NOT NULL,
|
|
198
|
-
|
|
196
|
+
thread_id TEXT NOT NULL,
|
|
199
197
|
queue_sequence INTEGER NOT NULL,
|
|
200
198
|
principal TEXT NOT NULL,
|
|
201
199
|
idempotency_key TEXT NOT NULL,
|
|
@@ -211,11 +209,18 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
211
209
|
ready_at TEXT,
|
|
212
210
|
input_applied_record_id TEXT,
|
|
213
211
|
input_applied_sequence INTEGER,
|
|
214
|
-
|
|
215
|
-
|
|
212
|
+
joined_host_submission_id TEXT,
|
|
213
|
+
suspended_reason_json TEXT,
|
|
214
|
+
suspended_at TEXT,
|
|
215
|
+
unknown_reason TEXT,
|
|
216
|
+
unknown_tool_call_ids_json TEXT,
|
|
217
|
+
parent_submission_id TEXT,
|
|
218
|
+
parent_tool_call_id TEXT,
|
|
219
|
+
UNIQUE (thread_id, principal, idempotency_key),
|
|
220
|
+
UNIQUE (thread_id, queue_sequence)
|
|
216
221
|
)
|
|
217
222
|
`.withoutTransform;
|
|
218
|
-
|
|
223
|
+
yield* sql`
|
|
219
224
|
CREATE TABLE effect_agent_submission_ownership (
|
|
220
225
|
submission_id TEXT PRIMARY KEY NOT NULL,
|
|
221
226
|
attempt_id TEXT NOT NULL,
|
|
@@ -228,11 +233,11 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
228
233
|
ON DELETE RESTRICT
|
|
229
234
|
)
|
|
230
235
|
`.withoutTransform;
|
|
231
|
-
|
|
236
|
+
yield* sql`
|
|
232
237
|
CREATE TABLE effect_agent_attempts (
|
|
233
238
|
attempt_id TEXT PRIMARY KEY NOT NULL,
|
|
234
239
|
submission_id TEXT NOT NULL,
|
|
235
|
-
|
|
240
|
+
thread_id TEXT NOT NULL,
|
|
236
241
|
owner_producer_id TEXT NOT NULL,
|
|
237
242
|
producer_epoch INTEGER NOT NULL,
|
|
238
243
|
claimed_at TEXT NOT NULL,
|
|
@@ -241,7 +246,7 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
241
246
|
ON DELETE RESTRICT
|
|
242
247
|
)
|
|
243
248
|
`.withoutTransform;
|
|
244
|
-
|
|
249
|
+
yield* sql`
|
|
245
250
|
CREATE TABLE effect_agent_settlement_reservations (
|
|
246
251
|
submission_id TEXT PRIMARY KEY NOT NULL,
|
|
247
252
|
settlement_id TEXT NOT NULL,
|
|
@@ -256,7 +261,7 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
256
261
|
ON DELETE RESTRICT
|
|
257
262
|
)
|
|
258
263
|
`.withoutTransform;
|
|
259
|
-
|
|
264
|
+
yield* sql`
|
|
260
265
|
CREATE TABLE effect_agent_abort_intents (
|
|
261
266
|
submission_id TEXT PRIMARY KEY NOT NULL,
|
|
262
267
|
author TEXT NOT NULL,
|
|
@@ -268,35 +273,11 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
268
273
|
ON DELETE RESTRICT
|
|
269
274
|
)
|
|
270
275
|
`.withoutTransform;
|
|
271
|
-
|
|
272
|
-
}),
|
|
273
|
-
"3_durable_tools_and_joined_input": Effect.gen(function* () {
|
|
274
|
-
const sql = yield* SqlClient.SqlClient;
|
|
275
|
-
yield* sql`
|
|
276
|
-
ALTER TABLE effect_agent_submissions
|
|
277
|
-
ADD COLUMN joined_host_submission_id TEXT
|
|
278
|
-
`.withoutTransform;
|
|
279
|
-
yield* sql`
|
|
280
|
-
ALTER TABLE effect_agent_submissions
|
|
281
|
-
ADD COLUMN suspended_reason_json TEXT
|
|
282
|
-
`.withoutTransform;
|
|
283
|
-
yield* sql`
|
|
284
|
-
ALTER TABLE effect_agent_submissions
|
|
285
|
-
ADD COLUMN suspended_at TEXT
|
|
286
|
-
`.withoutTransform;
|
|
287
|
-
yield* sql`
|
|
288
|
-
ALTER TABLE effect_agent_submissions
|
|
289
|
-
ADD COLUMN unknown_reason TEXT
|
|
290
|
-
`.withoutTransform;
|
|
291
|
-
yield* sql`
|
|
292
|
-
ALTER TABLE effect_agent_submissions
|
|
293
|
-
ADD COLUMN unknown_tool_call_ids_json TEXT
|
|
294
|
-
`.withoutTransform;
|
|
295
|
-
yield* sql`
|
|
276
|
+
yield* sql`
|
|
296
277
|
CREATE INDEX effect_agent_submissions_joined_host
|
|
297
278
|
ON effect_agent_submissions (joined_host_submission_id)
|
|
298
279
|
`.withoutTransform;
|
|
299
|
-
|
|
280
|
+
yield* sql`
|
|
300
281
|
CREATE TABLE effect_agent_approval_decisions (
|
|
301
282
|
submission_id TEXT NOT NULL,
|
|
302
283
|
tool_call_id TEXT NOT NULL,
|
|
@@ -310,7 +291,7 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
310
291
|
ON DELETE RESTRICT
|
|
311
292
|
)
|
|
312
293
|
`.withoutTransform;
|
|
313
|
-
|
|
294
|
+
yield* sql`
|
|
314
295
|
CREATE TABLE effect_agent_unknown_resolutions (
|
|
315
296
|
submission_id TEXT NOT NULL,
|
|
316
297
|
tool_call_id TEXT NOT NULL,
|
|
@@ -324,23 +305,11 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
324
305
|
ON DELETE RESTRICT
|
|
325
306
|
)
|
|
326
307
|
`.withoutTransform;
|
|
327
|
-
|
|
328
|
-
}),
|
|
329
|
-
"4_durable_subagents": Effect.gen(function* () {
|
|
330
|
-
const sql = yield* SqlClient.SqlClient;
|
|
331
|
-
yield* sql`
|
|
332
|
-
ALTER TABLE effect_agent_submissions
|
|
333
|
-
ADD COLUMN parent_submission_id TEXT
|
|
334
|
-
`.withoutTransform;
|
|
335
|
-
yield* sql`
|
|
336
|
-
ALTER TABLE effect_agent_submissions
|
|
337
|
-
ADD COLUMN parent_tool_call_id TEXT
|
|
338
|
-
`.withoutTransform;
|
|
339
|
-
yield* sql`
|
|
308
|
+
yield* sql`
|
|
340
309
|
CREATE INDEX effect_agent_submissions_parent
|
|
341
310
|
ON effect_agent_submissions (parent_submission_id)
|
|
342
311
|
`.withoutTransform;
|
|
343
|
-
|
|
312
|
+
yield* sql`
|
|
344
313
|
CREATE TABLE effect_agent_child_reservations (
|
|
345
314
|
reservation_id TEXT PRIMARY KEY NOT NULL,
|
|
346
315
|
parent_submission_id TEXT NOT NULL,
|
|
@@ -359,11 +328,7 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
359
328
|
ON DELETE RESTRICT
|
|
360
329
|
)
|
|
361
330
|
`.withoutTransform;
|
|
362
|
-
|
|
363
|
-
}),
|
|
364
|
-
"5_durable_schedules": Effect.gen(function* () {
|
|
365
|
-
const sql = yield* SqlClient.SqlClient;
|
|
366
|
-
yield* sql`
|
|
331
|
+
yield* sql`
|
|
367
332
|
CREATE TABLE effect_agent_schedules (
|
|
368
333
|
tenant_id TEXT NOT NULL,
|
|
369
334
|
owner_id TEXT NOT NULL,
|
|
@@ -373,19 +338,383 @@ const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
373
338
|
PRIMARY KEY (tenant_id, owner_id, schedule_id)
|
|
374
339
|
)
|
|
375
340
|
`.withoutTransform;
|
|
376
|
-
|
|
341
|
+
yield* sql`
|
|
377
342
|
CREATE INDEX effect_agent_schedules_deadline
|
|
378
343
|
ON effect_agent_schedules (deadline_at_millis, tenant_id, owner_id, schedule_id)
|
|
379
344
|
WHERE deadline_at_millis IS NOT NULL
|
|
380
345
|
`.withoutTransform;
|
|
381
|
-
|
|
346
|
+
yield* sql`
|
|
382
347
|
CREATE INDEX effect_agent_schedules_owner_deadline
|
|
383
348
|
ON effect_agent_schedules (tenant_id, owner_id, deadline_at_millis, schedule_id)
|
|
384
349
|
WHERE deadline_at_millis IS NOT NULL
|
|
385
350
|
`.withoutTransform;
|
|
386
|
-
|
|
387
|
-
|
|
351
|
+
yield* sql`
|
|
352
|
+
CREATE TABLE effect_agent_subscription_sequences (
|
|
353
|
+
tenant_id TEXT NOT NULL,
|
|
354
|
+
source_address TEXT NOT NULL,
|
|
355
|
+
sequence INTEGER NOT NULL,
|
|
356
|
+
event_scan_cursor TEXT NOT NULL,
|
|
357
|
+
delivery_scan_cursor TEXT NOT NULL,
|
|
358
|
+
recovery_scan_cursor INTEGER NOT NULL,
|
|
359
|
+
PRIMARY KEY (tenant_id, source_address)
|
|
360
|
+
)
|
|
361
|
+
`.withoutTransform;
|
|
362
|
+
yield* sql`
|
|
363
|
+
CREATE TABLE effect_agent_subscriptions (
|
|
364
|
+
tenant_id TEXT NOT NULL,
|
|
365
|
+
source_address TEXT NOT NULL,
|
|
366
|
+
owner_id TEXT NOT NULL,
|
|
367
|
+
subscription_id TEXT NOT NULL,
|
|
368
|
+
ordinal INTEGER NOT NULL,
|
|
369
|
+
source_name TEXT NOT NULL,
|
|
370
|
+
source_version TEXT NOT NULL,
|
|
371
|
+
matching_key TEXT NOT NULL,
|
|
372
|
+
state TEXT NOT NULL,
|
|
373
|
+
expires_at_millis INTEGER NOT NULL,
|
|
374
|
+
recovery_at_millis INTEGER,
|
|
375
|
+
record_json TEXT NOT NULL,
|
|
376
|
+
PRIMARY KEY (tenant_id, source_address, owner_id, subscription_id),
|
|
377
|
+
UNIQUE (tenant_id, source_address, ordinal)
|
|
378
|
+
)
|
|
379
|
+
`.withoutTransform;
|
|
380
|
+
yield* sql`CREATE INDEX effect_agent_subscriptions_owner ON effect_agent_subscriptions (tenant_id, source_address, owner_id, ordinal)`.withoutTransform;
|
|
381
|
+
yield* sql`CREATE INDEX effect_agent_subscriptions_candidates ON effect_agent_subscriptions (tenant_id, source_address, source_name, source_version, matching_key, ordinal)`.withoutTransform;
|
|
382
|
+
yield* sql`CREATE INDEX effect_agent_subscriptions_recovery ON effect_agent_subscriptions (tenant_id, source_address, recovery_at_millis, ordinal) WHERE recovery_at_millis IS NOT NULL`.withoutTransform;
|
|
383
|
+
yield* sql`
|
|
384
|
+
CREATE TABLE effect_agent_subscription_events (
|
|
385
|
+
tenant_id TEXT NOT NULL,
|
|
386
|
+
source_address TEXT NOT NULL,
|
|
387
|
+
event_id TEXT NOT NULL,
|
|
388
|
+
source_name TEXT NOT NULL,
|
|
389
|
+
source_version TEXT NOT NULL,
|
|
390
|
+
matching_key TEXT NOT NULL,
|
|
391
|
+
payload_digest TEXT NOT NULL,
|
|
392
|
+
cutoff INTEGER NOT NULL,
|
|
393
|
+
cursor INTEGER NOT NULL,
|
|
394
|
+
routing_complete INTEGER NOT NULL,
|
|
395
|
+
next_attempt_at_millis INTEGER NOT NULL,
|
|
396
|
+
record_json TEXT NOT NULL,
|
|
397
|
+
PRIMARY KEY (tenant_id, source_address, event_id)
|
|
398
|
+
)
|
|
399
|
+
`.withoutTransform;
|
|
400
|
+
yield* sql`CREATE INDEX effect_agent_subscription_events_pending ON effect_agent_subscription_events (tenant_id, source_address, routing_complete, next_attempt_at_millis, event_id)`.withoutTransform;
|
|
401
|
+
yield* sql`
|
|
402
|
+
CREATE TABLE effect_agent_subscription_deliveries (
|
|
403
|
+
tenant_id TEXT NOT NULL,
|
|
404
|
+
source_address TEXT NOT NULL,
|
|
405
|
+
owner_id TEXT NOT NULL,
|
|
406
|
+
subscription_id TEXT NOT NULL,
|
|
407
|
+
event_id TEXT NOT NULL,
|
|
408
|
+
delivery_key TEXT NOT NULL,
|
|
409
|
+
state TEXT NOT NULL,
|
|
410
|
+
next_attempt_at_millis INTEGER NOT NULL,
|
|
411
|
+
record_json TEXT NOT NULL,
|
|
412
|
+
PRIMARY KEY (tenant_id, source_address, owner_id, subscription_id, event_id),
|
|
413
|
+
UNIQUE (tenant_id, source_address, delivery_key)
|
|
414
|
+
)
|
|
415
|
+
`.withoutTransform;
|
|
416
|
+
yield* sql`CREATE INDEX effect_agent_subscription_deliveries_pending ON effect_agent_subscription_deliveries (tenant_id, source_address, state, next_attempt_at_millis, delivery_key)`.withoutTransform;
|
|
417
|
+
yield* sql`CREATE INDEX effect_agent_subscription_deliveries_registration ON effect_agent_subscription_deliveries (tenant_id, source_address, owner_id, subscription_id, delivery_key)`.withoutTransform;
|
|
418
|
+
yield* sql`PRAGMA user_version = 7`.withoutTransform;
|
|
419
|
+
}) });
|
|
420
|
+
//#endregion
|
|
421
|
+
//#region src/sqlite-activity-store.ts
|
|
422
|
+
const STORAGE_VERSION$1 = 1;
|
|
423
|
+
const METADATA_COMPONENT$1 = "activity";
|
|
424
|
+
const STATE_TABLE = "effect_agent_activity_processor_state_v1";
|
|
425
|
+
const StoredJson$2 = Schema.String.check(Schema.isMaxLength(16777216));
|
|
426
|
+
const sameKey = Schema.toEquivalence(ActivityProcessorKey);
|
|
427
|
+
const sameWork = Schema.toEquivalence(PreparedActivity);
|
|
428
|
+
var ActivityMetadataRow = class extends Schema.Class("@effect-agent/storage-sqlite/ActivityMetadataRow")({ version: Schema.Int }) {};
|
|
429
|
+
var ActivityTableRow = class extends Schema.Class("@effect-agent/storage-sqlite/ActivityTableRow")({ name: Schema.NonEmptyString }) {};
|
|
430
|
+
var ActivityStateRow = class extends Schema.Class("@effect-agent/storage-sqlite/ActivityStateRow")({
|
|
431
|
+
processor_id: ActivityProcessorKey.fields.processorId,
|
|
432
|
+
processor_version: ActivityProcessorKey.fields.processorVersion,
|
|
433
|
+
thread_id: ActivityProcessorKey.fields.threadId,
|
|
434
|
+
format_version: Schema.Int,
|
|
435
|
+
through_sequence: ActivityProgress.fields.throughSequence,
|
|
436
|
+
epoch: ActivityProgress.fields.epoch,
|
|
437
|
+
owner: ActivityProgress.fields.owner,
|
|
438
|
+
lease_expires_at: ActivityProgress.fields.leaseExpiresAt,
|
|
439
|
+
progress_json: StoredJson$2
|
|
440
|
+
}) {};
|
|
441
|
+
var ActivityChangeCountRow = class extends Schema.Class("@effect-agent/storage-sqlite/ActivityChangeCountRow")({ changed: Schema.Int }) {};
|
|
442
|
+
const StoredVersionHeader$1 = Schema.Struct({ version: Schema.Int });
|
|
443
|
+
const storeError$1 = (operation, reason = "unavailable") => ActivityStoreError.make({
|
|
444
|
+
operation,
|
|
445
|
+
reason
|
|
446
|
+
});
|
|
447
|
+
const query$1 = (effect, operation) => effect.pipe(Effect.mapError(() => storeError$1(operation)));
|
|
448
|
+
const decodeRows$4 = Effect.fn("SqliteActivityStore.decodeRows")(function* (schema, rows, operation) {
|
|
449
|
+
return yield* Schema.decodeUnknownEffect(Schema.Array(schema))(rows).pipe(Effect.mapError(() => storeError$1(operation, "corrupt")));
|
|
450
|
+
});
|
|
451
|
+
const decodeInput$2 = Effect.fn("SqliteActivityStore.decodeInput")(function* (schema, value, operation) {
|
|
452
|
+
return yield* Schema.decodeUnknownEffect(schema)(value).pipe(Effect.mapError(() => storeError$1(operation, "invalid-input")));
|
|
388
453
|
});
|
|
454
|
+
const encodeProgress = Effect.fn("SqliteActivityStore.encodeProgress")(function* (progress, operation) {
|
|
455
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(ActivityProgress))(progress).pipe(Effect.mapError(() => storeError$1(operation, "corrupt")), Effect.flatMap((encoded) => Schema.decodeEffect(StoredJson$2)(encoded).pipe(Effect.mapError(() => storeError$1(operation, "invalid-input")))));
|
|
456
|
+
});
|
|
457
|
+
const decodeProgress = Effect.fn("SqliteActivityStore.decodeProgress")(function* (value, operation) {
|
|
458
|
+
if ((yield* Schema.decodeEffect(Schema.fromJsonString(StoredVersionHeader$1))(value).pipe(Effect.mapError(() => storeError$1(operation, "corrupt")))).version !== STORAGE_VERSION$1) return yield* storeError$1(operation, "incompatible");
|
|
459
|
+
const progress = yield* Schema.decodeEffect(Schema.fromJsonString(ActivityProgress))(value).pipe(Effect.mapError(() => storeError$1(operation, "corrupt")));
|
|
460
|
+
if ((yield* encodeProgress(progress, operation)) !== value) return yield* storeError$1(operation, "corrupt");
|
|
461
|
+
return progress;
|
|
462
|
+
});
|
|
463
|
+
const validateProgress = Effect.fn("SqliteActivityStore.validateProgress")(function* (progress, operation) {
|
|
464
|
+
if (progress.epoch < 1 || progress.owner === null && progress.leaseExpiresAt !== 0 || progress.pending !== null && (!sameKey(progress.pending.key, progress.key) || progress.pending.sequence !== progress.throughSequence + 1)) return yield* storeError$1(operation, "corrupt");
|
|
465
|
+
return progress;
|
|
466
|
+
});
|
|
467
|
+
const makeClaim = (progress) => progress.owner === null ? null : ActivityClaim.make({
|
|
468
|
+
key: progress.key,
|
|
469
|
+
owner: progress.owner,
|
|
470
|
+
epoch: progress.epoch,
|
|
471
|
+
throughSequence: progress.throughSequence,
|
|
472
|
+
leaseExpiresAt: progress.leaseExpiresAt,
|
|
473
|
+
pending: progress.pending
|
|
474
|
+
});
|
|
475
|
+
const ownershipLost = (claim) => ActivityOwnershipLost.make({
|
|
476
|
+
key: claim.key,
|
|
477
|
+
owner: claim.owner,
|
|
478
|
+
epoch: claim.epoch
|
|
479
|
+
});
|
|
480
|
+
const makeActivityStore = Effect.fn("SqliteActivityStore.make")(function* () {
|
|
481
|
+
const sql = yield* SqlClient.SqlClient;
|
|
482
|
+
const failpoint = yield* ActivityMutationFailpoint;
|
|
483
|
+
yield* failpoint.hit("activity:initialize:before");
|
|
484
|
+
yield* sql.withTransaction(Effect.gen(function* () {
|
|
485
|
+
yield* sql`
|
|
486
|
+
CREATE TABLE IF NOT EXISTS effect_agent_activity_metadata (
|
|
487
|
+
component TEXT PRIMARY KEY NOT NULL,
|
|
488
|
+
version INTEGER NOT NULL
|
|
489
|
+
)
|
|
490
|
+
`;
|
|
491
|
+
const metadataRows = yield* sql`
|
|
492
|
+
SELECT version FROM effect_agent_activity_metadata
|
|
493
|
+
WHERE component = ${METADATA_COMPONENT$1}
|
|
494
|
+
`;
|
|
495
|
+
const metadata = yield* decodeRows$4(ActivityMetadataRow, metadataRows, "decode activity schema version");
|
|
496
|
+
if (metadata.length > 1) return yield* storeError$1("decode activity schema version", "corrupt");
|
|
497
|
+
const currentVersion = metadata[0]?.version;
|
|
498
|
+
if (currentVersion !== void 0 && currentVersion !== STORAGE_VERSION$1) return yield* storeError$1("initialize activity schema", "incompatible");
|
|
499
|
+
if (currentVersion === void 0) {
|
|
500
|
+
const tableRows = yield* sql`
|
|
501
|
+
SELECT name FROM sqlite_master
|
|
502
|
+
WHERE type = 'table' AND name = ${STATE_TABLE}
|
|
503
|
+
`;
|
|
504
|
+
if ((yield* decodeRows$4(ActivityTableRow, tableRows, "inspect activity schema")).length > 0) return yield* storeError$1("initialize activity schema", "incompatible");
|
|
505
|
+
yield* sql`
|
|
506
|
+
CREATE TABLE effect_agent_activity_processor_state_v1 (
|
|
507
|
+
processor_id TEXT NOT NULL,
|
|
508
|
+
processor_version TEXT NOT NULL,
|
|
509
|
+
thread_id TEXT NOT NULL,
|
|
510
|
+
format_version INTEGER NOT NULL,
|
|
511
|
+
through_sequence INTEGER NOT NULL,
|
|
512
|
+
epoch INTEGER NOT NULL,
|
|
513
|
+
owner TEXT,
|
|
514
|
+
lease_expires_at REAL NOT NULL,
|
|
515
|
+
progress_json TEXT NOT NULL,
|
|
516
|
+
PRIMARY KEY (processor_id, processor_version, thread_id)
|
|
517
|
+
)
|
|
518
|
+
`;
|
|
519
|
+
yield* sql`
|
|
520
|
+
INSERT INTO effect_agent_activity_metadata (component, version)
|
|
521
|
+
VALUES (${METADATA_COMPONENT$1}, ${STORAGE_VERSION$1})
|
|
522
|
+
`;
|
|
523
|
+
}
|
|
524
|
+
yield* sql`
|
|
525
|
+
SELECT processor_id, processor_version, thread_id, format_version,
|
|
526
|
+
through_sequence, epoch, owner, lease_expires_at, progress_json
|
|
527
|
+
FROM effect_agent_activity_processor_state_v1
|
|
528
|
+
LIMIT 0
|
|
529
|
+
`;
|
|
530
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError$1("initialize activity schema"))));
|
|
531
|
+
yield* failpoint.hit("activity:initialize:after");
|
|
532
|
+
const readProgress = Effect.fn("SqliteActivityStore.readProgress")(function* (key, operation) {
|
|
533
|
+
const rawRows = yield* query$1(sql`
|
|
534
|
+
SELECT processor_id, processor_version, thread_id, format_version,
|
|
535
|
+
through_sequence, epoch, owner, lease_expires_at, progress_json
|
|
536
|
+
FROM effect_agent_activity_processor_state_v1
|
|
537
|
+
WHERE processor_id = ${key.processorId}
|
|
538
|
+
AND processor_version = ${key.processorVersion}
|
|
539
|
+
AND thread_id = ${key.threadId}
|
|
540
|
+
`, operation);
|
|
541
|
+
const rows = yield* decodeRows$4(ActivityStateRow, rawRows, operation);
|
|
542
|
+
if (rows.length === 0) return null;
|
|
543
|
+
if (rows.length !== 1) return yield* storeError$1(operation, "corrupt");
|
|
544
|
+
const row = rows[0];
|
|
545
|
+
if (row.format_version !== STORAGE_VERSION$1) return yield* storeError$1(operation, "incompatible");
|
|
546
|
+
const progress = yield* decodeProgress(row.progress_json, operation);
|
|
547
|
+
yield* validateProgress(progress, operation);
|
|
548
|
+
if (!sameKey(progress.key, key) || row.processor_id !== key.processorId || row.processor_version !== key.processorVersion || row.thread_id !== key.threadId || row.through_sequence !== progress.throughSequence || row.epoch !== progress.epoch || row.owner !== progress.owner || row.lease_expires_at !== progress.leaseExpiresAt) return yield* storeError$1(operation, "corrupt");
|
|
549
|
+
return progress;
|
|
550
|
+
});
|
|
551
|
+
const checkChanged = Effect.fn("SqliteActivityStore.checkChanged")(function* (operation) {
|
|
552
|
+
const rawRows = yield* query$1(sql`SELECT changes() AS changed`, operation);
|
|
553
|
+
const rows = yield* decodeRows$4(ActivityChangeCountRow, rawRows, operation);
|
|
554
|
+
if (rows.length !== 1 || rows[0].changed !== 1) return yield* storeError$1(operation, "corrupt");
|
|
555
|
+
});
|
|
556
|
+
const insertProgress = Effect.fn("SqliteActivityStore.insertProgress")(function* (progress, operation) {
|
|
557
|
+
const progressJson = yield* encodeProgress(progress, operation);
|
|
558
|
+
yield* sql`
|
|
559
|
+
INSERT INTO effect_agent_activity_processor_state_v1 (
|
|
560
|
+
processor_id, processor_version, thread_id, format_version, through_sequence,
|
|
561
|
+
epoch, owner, lease_expires_at, progress_json
|
|
562
|
+
) VALUES (
|
|
563
|
+
${progress.key.processorId}, ${progress.key.processorVersion}, ${progress.key.threadId},
|
|
564
|
+
${STORAGE_VERSION$1}, ${progress.throughSequence}, ${progress.epoch}, ${progress.owner},
|
|
565
|
+
${progress.leaseExpiresAt}, ${progressJson}
|
|
566
|
+
)
|
|
567
|
+
`;
|
|
568
|
+
yield* checkChanged(operation);
|
|
569
|
+
});
|
|
570
|
+
const updateProgress = Effect.fn("SqliteActivityStore.updateProgress")(function* (current, next, operation) {
|
|
571
|
+
const progressJson = yield* encodeProgress(next, operation);
|
|
572
|
+
yield* sql`
|
|
573
|
+
UPDATE effect_agent_activity_processor_state_v1
|
|
574
|
+
SET format_version = ${STORAGE_VERSION$1},
|
|
575
|
+
through_sequence = ${next.throughSequence},
|
|
576
|
+
epoch = ${next.epoch},
|
|
577
|
+
owner = ${next.owner},
|
|
578
|
+
lease_expires_at = ${next.leaseExpiresAt},
|
|
579
|
+
progress_json = ${progressJson}
|
|
580
|
+
WHERE processor_id = ${current.key.processorId}
|
|
581
|
+
AND processor_version = ${current.key.processorVersion}
|
|
582
|
+
AND thread_id = ${current.key.threadId}
|
|
583
|
+
AND through_sequence = ${current.throughSequence}
|
|
584
|
+
AND epoch = ${current.epoch}
|
|
585
|
+
`;
|
|
586
|
+
yield* checkChanged(operation);
|
|
587
|
+
});
|
|
588
|
+
const requireLive = Effect.fn("SqliteActivityStore.requireLive")(function* (progress, claim, requireSequence) {
|
|
589
|
+
const now = yield* Clock.currentTimeMillis;
|
|
590
|
+
if (progress === null || !sameKey(progress.key, claim.key) || progress.owner !== claim.owner || progress.epoch !== claim.epoch || progress.leaseExpiresAt <= now || requireSequence && progress.throughSequence !== claim.throughSequence) return yield* ownershipLost(claim);
|
|
591
|
+
return progress;
|
|
592
|
+
});
|
|
593
|
+
const inspect = Effect.fn("SqliteActivityStore.inspect")(function* (key) {
|
|
594
|
+
const decodedKey = yield* decodeInput$2(ActivityProcessorKey, key, "inspect activity progress");
|
|
595
|
+
return yield* readProgress(decodedKey, "inspect activity progress");
|
|
596
|
+
});
|
|
597
|
+
const claim = Effect.fn("SqliteActivityStore.claim")(function* (request) {
|
|
598
|
+
const operation = "claim activity progress";
|
|
599
|
+
const decoded = yield* decodeInput$2(ActivityClaimRequest, request, operation);
|
|
600
|
+
yield* failpoint.hit("activity:claim:before");
|
|
601
|
+
const claimed = yield* sql.withTransaction(Effect.gen(function* () {
|
|
602
|
+
const current = yield* readProgress(decoded.key, operation);
|
|
603
|
+
const now = yield* Clock.currentTimeMillis;
|
|
604
|
+
if (current !== null && current.owner !== null && current.leaseExpiresAt > now) return yield* ActivityBusy.make({
|
|
605
|
+
key: decoded.key,
|
|
606
|
+
leaseExpiresAt: current.leaseExpiresAt
|
|
607
|
+
});
|
|
608
|
+
const next = yield* Schema.decodeUnknownEffect(ActivityProgress)({
|
|
609
|
+
version: STORAGE_VERSION$1,
|
|
610
|
+
key: decoded.key,
|
|
611
|
+
throughSequence: current?.throughSequence ?? 0,
|
|
612
|
+
epoch: (current?.epoch ?? 0) + 1,
|
|
613
|
+
owner: decoded.owner,
|
|
614
|
+
leaseExpiresAt: now + decoded.leaseMillis,
|
|
615
|
+
pending: current?.pending ?? null,
|
|
616
|
+
advancedAt: current?.advancedAt ?? null
|
|
617
|
+
}).pipe(Effect.mapError(() => storeError$1(operation, "corrupt")));
|
|
618
|
+
if (current === null) yield* insertProgress(next, operation);
|
|
619
|
+
else yield* updateProgress(current, next, operation);
|
|
620
|
+
yield* failpoint.hit("activity:claim:after-state");
|
|
621
|
+
const result = makeClaim(next);
|
|
622
|
+
if (result === null) return yield* storeError$1(operation, "corrupt");
|
|
623
|
+
return result;
|
|
624
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError$1(operation))));
|
|
625
|
+
yield* failpoint.hit("activity:claim:after");
|
|
626
|
+
return claimed;
|
|
627
|
+
});
|
|
628
|
+
const prepare = Effect.fn("SqliteActivityStore.prepare")(function* (request) {
|
|
629
|
+
const operation = "prepare activity output";
|
|
630
|
+
const claim = yield* decodeInput$2(ActivityClaim, request.claim, operation);
|
|
631
|
+
const work = yield* decodeInput$2(PreparedActivity, request.work, operation);
|
|
632
|
+
yield* failpoint.hit("activity:prepare:before");
|
|
633
|
+
const result = yield* sql.withTransaction(Effect.gen(function* () {
|
|
634
|
+
const current = yield* requireLive(yield* readProgress(claim.key, operation), claim, true);
|
|
635
|
+
if (current.pending !== null) {
|
|
636
|
+
if (sameWork(current.pending, work)) return {
|
|
637
|
+
work: current.pending,
|
|
638
|
+
changed: false
|
|
639
|
+
};
|
|
640
|
+
return yield* ActivityWorkConflict.make({
|
|
641
|
+
key: claim.key,
|
|
642
|
+
workId: work.workId
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
if (!sameKey(work.key, claim.key) || work.sequence !== current.throughSequence + 1) return yield* ActivityWorkConflict.make({
|
|
646
|
+
key: claim.key,
|
|
647
|
+
workId: work.workId
|
|
648
|
+
});
|
|
649
|
+
const next = ActivityProgress.make({
|
|
650
|
+
...current,
|
|
651
|
+
pending: work
|
|
652
|
+
});
|
|
653
|
+
yield* updateProgress(current, next, operation);
|
|
654
|
+
yield* failpoint.hit("activity:prepare:after-state");
|
|
655
|
+
return {
|
|
656
|
+
work,
|
|
657
|
+
changed: true
|
|
658
|
+
};
|
|
659
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError$1(operation))));
|
|
660
|
+
if (result.changed) yield* failpoint.hit("activity:prepare:after");
|
|
661
|
+
return result.work;
|
|
662
|
+
});
|
|
663
|
+
const advance = Effect.fn("SqliteActivityStore.advance")(function* (request) {
|
|
664
|
+
const operation = "advance activity progress";
|
|
665
|
+
const claim = yield* decodeInput$2(ActivityClaim, request.claim, operation);
|
|
666
|
+
const workId = yield* decodeInput$2(Digest, request.workId, operation);
|
|
667
|
+
yield* failpoint.hit("activity:advance:before");
|
|
668
|
+
const nextClaim = yield* sql.withTransaction(Effect.gen(function* () {
|
|
669
|
+
const current = yield* requireLive(yield* readProgress(claim.key, operation), claim, true);
|
|
670
|
+
if (current.pending === null || current.pending.workId !== workId) return yield* ActivityWorkConflict.make({
|
|
671
|
+
key: claim.key,
|
|
672
|
+
workId
|
|
673
|
+
});
|
|
674
|
+
const next = ActivityProgress.make({
|
|
675
|
+
...current,
|
|
676
|
+
throughSequence: current.pending.sequence,
|
|
677
|
+
pending: null,
|
|
678
|
+
advancedAt: yield* Clock.currentTimeMillis
|
|
679
|
+
});
|
|
680
|
+
yield* updateProgress(current, next, operation);
|
|
681
|
+
yield* failpoint.hit("activity:advance:after-state");
|
|
682
|
+
const result = makeClaim(next);
|
|
683
|
+
if (result === null) return yield* storeError$1(operation, "corrupt");
|
|
684
|
+
return result;
|
|
685
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError$1(operation))));
|
|
686
|
+
yield* failpoint.hit("activity:advance:after");
|
|
687
|
+
return nextClaim;
|
|
688
|
+
});
|
|
689
|
+
const release = Effect.fn("SqliteActivityStore.release")(function* (claim) {
|
|
690
|
+
const operation = "release activity claim";
|
|
691
|
+
const decoded = yield* decodeInput$2(ActivityClaim, claim, operation);
|
|
692
|
+
yield* failpoint.hit("activity:release:before");
|
|
693
|
+
yield* sql.withTransaction(Effect.gen(function* () {
|
|
694
|
+
const current = yield* readProgress(decoded.key, operation);
|
|
695
|
+
if (current === null || current.owner !== decoded.owner || current.epoch !== decoded.epoch) return yield* ownershipLost(decoded);
|
|
696
|
+
const next = ActivityProgress.make({
|
|
697
|
+
...current,
|
|
698
|
+
owner: null,
|
|
699
|
+
leaseExpiresAt: 0
|
|
700
|
+
});
|
|
701
|
+
yield* updateProgress(current, next, operation);
|
|
702
|
+
yield* failpoint.hit("activity:release:after-state");
|
|
703
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError$1(operation))));
|
|
704
|
+
yield* failpoint.hit("activity:release:after");
|
|
705
|
+
});
|
|
706
|
+
return ActivityProcessorStore.of({
|
|
707
|
+
inspect,
|
|
708
|
+
claim,
|
|
709
|
+
prepare,
|
|
710
|
+
advance,
|
|
711
|
+
release
|
|
712
|
+
});
|
|
713
|
+
});
|
|
714
|
+
/** SQLite activity progress with mutation failpoints kept injectable for recovery tests. */
|
|
715
|
+
const activityProcessorStoreLayerWithFailpoints = Layer.effect(ActivityProcessorStore, makeActivityStore());
|
|
716
|
+
/** SQLite activity progress with the production no-op mutation failpoint. */
|
|
717
|
+
const activityProcessorStoreLayer = activityProcessorStoreLayerWithFailpoints.pipe(Layer.provide(ActivityMutationFailpoint.layer));
|
|
389
718
|
//#endregion
|
|
390
719
|
//#region src/sqlite-storage-config.ts
|
|
391
720
|
const ObservationPollInterval = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0));
|
|
@@ -404,7 +733,7 @@ var SqliteStorageConfigValue = class extends Schema.Class("@effect-agent/storage
|
|
|
404
733
|
* Submission ownership lease duration in milliseconds (D5). The lease is a liveness hint
|
|
405
734
|
* that makes an abandoned claim reclaimable; correctness never depends on it because every
|
|
406
735
|
* canonical append is fenced by producer epoch. Convenience layers default this to
|
|
407
|
-
* `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@effect-agent/
|
|
736
|
+
* `DEFAULT_OWNERSHIP_LEASE_DURATION` from `@effect-agent/thread`.
|
|
408
737
|
*/
|
|
409
738
|
ownershipLeaseDuration: OwnershipLeaseMillis,
|
|
410
739
|
/**
|
|
@@ -417,37 +746,19 @@ var SqliteStorageConfigValue = class extends Schema.Class("@effect-agent/storage
|
|
|
417
746
|
/** Explicit SQLite storage configuration authority. */
|
|
418
747
|
var SqliteStorageConfig = class extends Context.Service()("@effect-agent/storage-sqlite/SqliteStorageConfig") {};
|
|
419
748
|
//#endregion
|
|
420
|
-
//#region src/sqlite-storage-failpoint.ts
|
|
421
|
-
const noFailpoint = () => Effect.void;
|
|
422
|
-
/** Test-only control for replacing the active SQLite failpoint handler. */
|
|
423
|
-
var SqliteStorageFailpointTestControl = class extends Context.Service()("@effect-agent/storage-sqlite/SqliteStorageFailpointTestControl") {};
|
|
424
|
-
/** Explicit fault-injection authority used at SQLite operation boundaries. */
|
|
425
|
-
var SqliteStorageFailpoint = class SqliteStorageFailpoint extends Context.Service()("@effect-agent/storage-sqlite/SqliteStorageFailpoint") {
|
|
426
|
-
/** Production default: no fault injection. */
|
|
427
|
-
static layer = Layer.succeed(this)({ hit: noFailpoint });
|
|
428
|
-
/** Reusable test Layer with a control service backed by the same handler Ref. */
|
|
429
|
-
static layerTest = Layer.effectContext(Effect.gen(function* () {
|
|
430
|
-
const handler = yield* Ref.make(noFailpoint);
|
|
431
|
-
return Context.make(SqliteStorageFailpoint, SqliteStorageFailpoint.of({ hit: (location) => Ref.get(handler).pipe(Effect.flatMap((current) => current(location))) })).pipe(Context.add(SqliteStorageFailpointTestControl, SqliteStorageFailpointTestControl.of({
|
|
432
|
-
clear: Ref.set(handler, noFailpoint),
|
|
433
|
-
setHandler: (next) => Ref.set(handler, next)
|
|
434
|
-
})));
|
|
435
|
-
}));
|
|
436
|
-
};
|
|
437
|
-
//#endregion
|
|
438
749
|
//#region src/sqlite-journal.ts
|
|
439
|
-
const BoundedStoredText$1 = Schema.String.check(Schema.isMaxLength(
|
|
750
|
+
const BoundedStoredText$1 = Schema.String.check(Schema.isMaxLength(16777216));
|
|
440
751
|
const BoundedIdentifier$1 = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
441
752
|
const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0));
|
|
442
|
-
const
|
|
443
|
-
const MAX_STORED_TEXT_BYTES =
|
|
753
|
+
const MAX_RECORDS_PER_THREAD = 65536;
|
|
754
|
+
const MAX_STORED_TEXT_BYTES = 16777216;
|
|
444
755
|
const MAX_IDENTIFIER_LENGTH = 1024;
|
|
445
756
|
const storedTextBytes = (value) => new TextEncoder().encode(value).byteLength;
|
|
446
757
|
var SqliteVersionRow = class extends Schema.Class("SqliteVersionRow")({ user_version: NonNegativeInt }) {};
|
|
447
758
|
var SqliteJournalModeRow = class extends Schema.Class("SqliteJournalModeRow")({ journal_mode: Schema.NonEmptyString.check(Schema.isMaxLength(32)) }) {};
|
|
448
759
|
var SqliteNameRow = class extends Schema.Class("SqliteNameRow")({ name: BoundedIdentifier$1 }) {};
|
|
449
|
-
var
|
|
450
|
-
|
|
760
|
+
var ThreadRow = class extends Schema.Class("ThreadRow")({
|
|
761
|
+
thread_id: BoundedIdentifier$1,
|
|
451
762
|
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
452
763
|
producer_epoch: ProducerEpoch,
|
|
453
764
|
tail_digest: BoundedStoredText$1,
|
|
@@ -457,21 +768,21 @@ var BatchRow = class extends Schema.Class("BatchRow")({
|
|
|
457
768
|
batch_digest: BoundedStoredText$1,
|
|
458
769
|
batch_id: BoundedIdentifier$1,
|
|
459
770
|
batch_json: BoundedStoredText$1,
|
|
460
|
-
|
|
771
|
+
thread_id: BoundedIdentifier$1,
|
|
461
772
|
first_sequence: CanonicalSequence,
|
|
462
773
|
last_sequence: CanonicalSequence,
|
|
463
774
|
tail_digest: BoundedStoredText$1
|
|
464
775
|
}) {};
|
|
465
776
|
var RecordRow = class extends Schema.Class("RecordRow")({
|
|
466
777
|
batch_id: BoundedIdentifier$1,
|
|
467
|
-
|
|
778
|
+
thread_id: BoundedIdentifier$1,
|
|
468
779
|
record_id: BoundedIdentifier$1,
|
|
469
780
|
record_json: BoundedStoredText$1,
|
|
470
781
|
sequence: CanonicalSequence
|
|
471
782
|
}) {};
|
|
472
783
|
var CheckpointRow = class extends Schema.Class("CheckpointRow")({
|
|
473
784
|
checkpoint_json: BoundedStoredText$1,
|
|
474
|
-
|
|
785
|
+
thread_id: BoundedIdentifier$1,
|
|
475
786
|
tail_digest: BoundedStoredText$1,
|
|
476
787
|
through_sequence: CanonicalSequence
|
|
477
788
|
}) {};
|
|
@@ -483,7 +794,7 @@ var RawAppendRequest = class extends Schema.Class("@effect-agent/storage-sqlite/
|
|
|
483
794
|
batchDigest: BoundedStoredText$1,
|
|
484
795
|
batchId: BoundedIdentifier$1,
|
|
485
796
|
batchJson: BoundedStoredText$1,
|
|
486
|
-
|
|
797
|
+
threadId: BoundedIdentifier$1,
|
|
487
798
|
expectedTailDigest: BoundedStoredText$1,
|
|
488
799
|
expectedTailSequence: CanonicalSequence,
|
|
489
800
|
producerEpoch: ProducerEpoch,
|
|
@@ -497,35 +808,35 @@ var RawAppendResult = class extends Schema.Class("@effect-agent/storage-sqlite/R
|
|
|
497
808
|
tailDigest: BoundedStoredText$1
|
|
498
809
|
}) {};
|
|
499
810
|
var RawReadRequest = class extends Schema.Class("@effect-agent/storage-sqlite/RawReadRequest")({
|
|
500
|
-
|
|
811
|
+
threadId: BoundedIdentifier$1,
|
|
501
812
|
fromSequenceExclusive: CanonicalSequence,
|
|
502
813
|
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1024))
|
|
503
814
|
}) {};
|
|
504
815
|
var RawCheckpoint = class extends Schema.Class("@effect-agent/storage-sqlite/RawCheckpoint")({
|
|
505
816
|
checkpointJson: BoundedStoredText$1,
|
|
506
|
-
|
|
817
|
+
threadId: BoundedIdentifier$1,
|
|
507
818
|
tailDigest: BoundedStoredText$1,
|
|
508
819
|
throughSequence: CanonicalSequence
|
|
509
820
|
}) {};
|
|
510
|
-
var
|
|
821
|
+
var RawThreadExport = class extends Schema.Class("@effect-agent/storage-sqlite/RawThreadExport")({
|
|
511
822
|
batches: Schema.Array(BatchRow),
|
|
512
823
|
checkpoints: Schema.Array(CheckpointRow),
|
|
513
|
-
|
|
824
|
+
thread: ThreadRow,
|
|
514
825
|
records: Schema.Array(RecordRow)
|
|
515
826
|
}) {};
|
|
516
|
-
const storageError = (operation) => (error) => SqliteStorageError.make({
|
|
827
|
+
const storageError$1 = (operation) => (error) => SqliteStorageError.make({
|
|
517
828
|
cause: error,
|
|
518
829
|
operation,
|
|
519
830
|
message: error.message
|
|
520
831
|
});
|
|
521
832
|
/** Decode raw SQLite rows against a Schema, reporting failures as typed corruption. */
|
|
522
|
-
const decodeRows$
|
|
833
|
+
const decodeRows$3 = Effect.fn("SqliteJournal.decodeRows")((schema, table, rowKey, rows) => Schema.decodeUnknownEffect(schema)(rows).pipe(Effect.mapError((error) => SqliteStorageCorruptionError.make({
|
|
523
834
|
table,
|
|
524
835
|
rowKey,
|
|
525
836
|
message: String(error)
|
|
526
837
|
}))));
|
|
527
838
|
/** Decode exactly one raw SQLite row against a Schema, reporting failures as typed corruption. */
|
|
528
|
-
const decodeSingleRow = Effect.fn("SqliteJournal.decodeSingleRow")((schema, table, rowKey, rows) => decodeRows$
|
|
839
|
+
const decodeSingleRow = Effect.fn("SqliteJournal.decodeSingleRow")((schema, table, rowKey, rows) => decodeRows$3(schema, table, rowKey, rows).pipe(Effect.flatMap((decoded) => decoded.length === 1 ? Effect.succeed(decoded[0]) : Effect.fail(SqliteStorageCorruptionError.make({
|
|
529
840
|
table,
|
|
530
841
|
rowKey,
|
|
531
842
|
message: `Expected exactly one row but found ${decoded.length}.`
|
|
@@ -534,21 +845,21 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
534
845
|
const sql = yield* SqlClient.SqlClient;
|
|
535
846
|
const { hit: failpoint } = yield* SqliteStorageFailpoint;
|
|
536
847
|
const { busyTimeout } = yield* SqliteStorageConfig;
|
|
537
|
-
yield* sql`PRAGMA foreign_keys = ON`.pipe(Effect.mapError(storageError("enable foreign keys")));
|
|
538
|
-
yield* sql.unsafe(`PRAGMA busy_timeout = ${busyTimeout}`).pipe(Effect.mapError(storageError("configure busy timeout")));
|
|
539
|
-
const journalModeRows = yield* sql`PRAGMA journal_mode`.pipe(Effect.mapError(storageError("read journal mode")));
|
|
848
|
+
yield* sql`PRAGMA foreign_keys = ON`.pipe(Effect.mapError(storageError$1("enable foreign keys")));
|
|
849
|
+
yield* sql.unsafe(`PRAGMA busy_timeout = ${busyTimeout}`).pipe(Effect.mapError(storageError$1("configure busy timeout")));
|
|
850
|
+
const journalModeRows = yield* sql`PRAGMA journal_mode`.pipe(Effect.mapError(storageError$1("read journal mode")));
|
|
540
851
|
const journalMode = yield* decodeSingleRow(Schema.Array(SqliteJournalModeRow), "pragma_journal_mode", "singleton", journalModeRows);
|
|
541
852
|
if (journalMode.journal_mode.toLowerCase() !== "wal") return yield* SqliteStorageCompatibilityError.make({
|
|
542
853
|
actualVersion: 0,
|
|
543
|
-
supportedVersion:
|
|
854
|
+
supportedVersion: 7,
|
|
544
855
|
message: `SQLite WAL mode is required; the database reported ${journalMode.journal_mode}.`
|
|
545
856
|
});
|
|
546
|
-
const versionRows = yield* sql`PRAGMA user_version`.pipe(Effect.mapError(storageError("read storage version")));
|
|
857
|
+
const versionRows = yield* sql`PRAGMA user_version`.pipe(Effect.mapError(storageError$1("read storage version")));
|
|
547
858
|
const version = yield* decodeSingleRow(Schema.Array(SqliteVersionRow), "pragma_user_version", "singleton", versionRows);
|
|
548
|
-
if (version.user_version !== 0 && version.user_version !==
|
|
859
|
+
if (version.user_version !== 0 && version.user_version !== 7) return yield* SqliteStorageCompatibilityError.make({
|
|
549
860
|
actualVersion: version.user_version,
|
|
550
|
-
supportedVersion:
|
|
551
|
-
message: `The SQLite file uses private-development storage version ${version.user_version}; this build supports exactly version
|
|
861
|
+
supportedVersion: 7,
|
|
862
|
+
message: `The SQLite file uses private-development storage version ${version.user_version}; this build supports exactly version 7. Reset the database file explicitly; automatic stored-data migrations are not provided during private development.`
|
|
552
863
|
});
|
|
553
864
|
if (version.user_version === 0) {
|
|
554
865
|
const existingRows = yield* sql`
|
|
@@ -557,10 +868,10 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
557
868
|
WHERE type = 'table'
|
|
558
869
|
AND name LIKE 'effect_agent_%'
|
|
559
870
|
ORDER BY name
|
|
560
|
-
`.pipe(Effect.mapError(storageError("inspect unversioned storage")));
|
|
561
|
-
if ((yield* decodeRows$
|
|
871
|
+
`.pipe(Effect.mapError(storageError$1("inspect unversioned storage")));
|
|
872
|
+
if ((yield* decodeRows$3(Schema.Array(SqliteNameRow), "sqlite_master", "effect_agent_%", existingRows)).length > 0) return yield* SqliteStorageCompatibilityError.make({
|
|
562
873
|
actualVersion: 0,
|
|
563
|
-
supportedVersion:
|
|
874
|
+
supportedVersion: 7,
|
|
564
875
|
message: "The SQLite file contains unversioned Effect Agent tables. Reset it explicitly; refusing to mutate ambiguous stored data."
|
|
565
876
|
});
|
|
566
877
|
yield* SqliteMigrator.run({ loader: sqliteMigrations }).pipe(Effect.mapError((error) => SqliteStorageError.make({
|
|
@@ -574,7 +885,7 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
574
885
|
FROM sqlite_master
|
|
575
886
|
WHERE type = 'table'
|
|
576
887
|
AND name IN (
|
|
577
|
-
'
|
|
888
|
+
'effect_agent_threads',
|
|
578
889
|
'effect_agent_canonical_batches',
|
|
579
890
|
'effect_agent_canonical_records',
|
|
580
891
|
'effect_agent_checkpoints',
|
|
@@ -588,17 +899,17 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
588
899
|
'effect_agent_schedules'
|
|
589
900
|
)
|
|
590
901
|
ORDER BY name
|
|
591
|
-
`.pipe(Effect.mapError(storageError("verify storage tables")));
|
|
592
|
-
if ((yield* decodeRows$
|
|
593
|
-
actualVersion:
|
|
594
|
-
supportedVersion:
|
|
902
|
+
`.pipe(Effect.mapError(storageError$1("verify storage tables")));
|
|
903
|
+
if ((yield* decodeRows$3(Schema.Array(SqliteNameRow), "sqlite_master", "required_tables", requiredRows)).length !== 12) return yield* SqliteStorageCompatibilityError.make({
|
|
904
|
+
actualVersion: 7,
|
|
905
|
+
supportedVersion: 7,
|
|
595
906
|
message: "The SQLite file claims the current format but is missing required tables. Reset the corrupt private-development data."
|
|
596
907
|
});
|
|
597
908
|
const classifyWriteFailure = (operation) => (error) => error.reason._tag === "LockTimeoutError" ? SqliteWriteContention.make({
|
|
598
909
|
cause: error,
|
|
599
910
|
operation,
|
|
600
911
|
message: `Another producer holds the SQLite write lock; ${operation} is safe to retry.`
|
|
601
|
-
}) : storageError(operation)(error);
|
|
912
|
+
}) : storageError$1(operation)(error);
|
|
602
913
|
/**
|
|
603
914
|
* Runs one journal write transaction under `BEGIN IMMEDIATE`. SQLite's deferred `BEGIN`
|
|
604
915
|
* would let a read-then-write transaction start as a reader and fail with
|
|
@@ -631,54 +942,54 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
631
942
|
* `BEGIN` is reported directly because there is no transaction to roll back.
|
|
632
943
|
*/
|
|
633
944
|
const withReadTransaction = (operation) => (effect) => Effect.uninterruptibleMask((restore) => Effect.scoped(Effect.gen(function* () {
|
|
634
|
-
const connection = yield* sql.reserve.pipe(Effect.mapError(storageError(operation)));
|
|
635
|
-
yield* connection.executeUnprepared("BEGIN", [], void 0).pipe(Effect.mapError(storageError(operation)));
|
|
945
|
+
const connection = yield* sql.reserve.pipe(Effect.mapError(storageError$1(operation)));
|
|
946
|
+
yield* connection.executeUnprepared("BEGIN", [], void 0).pipe(Effect.mapError(storageError$1(operation)));
|
|
636
947
|
const exit = yield* restore(Effect.provideService(effect, sql.transactionService, [connection, 0])).pipe(Effect.exit);
|
|
637
948
|
if (Exit.isSuccess(exit)) {
|
|
638
|
-
yield* connection.executeUnprepared("COMMIT", [], void 0).pipe(Effect.mapError(storageError(operation)));
|
|
949
|
+
yield* connection.executeUnprepared("COMMIT", [], void 0).pipe(Effect.mapError(storageError$1(operation)));
|
|
639
950
|
return exit.value;
|
|
640
951
|
}
|
|
641
952
|
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], void 0));
|
|
642
953
|
return yield* exit;
|
|
643
954
|
})).pipe(Effect.withSpan("SqliteJournal.withReadTransaction", { attributes: { operation } })));
|
|
644
|
-
const materialize = Effect.fn("SqliteJournal.materialize")(function* (
|
|
645
|
-
if (
|
|
646
|
-
operation: "materialize
|
|
647
|
-
message: "
|
|
955
|
+
const materialize = Effect.fn("SqliteJournal.materialize")(function* (threadId, createdAt, emptyTailDigest, producerEpoch) {
|
|
956
|
+
if (threadId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(emptyTailDigest) > MAX_STORED_TEXT_BYTES) return yield* SqliteStorageError.make({
|
|
957
|
+
operation: "materialize thread",
|
|
958
|
+
message: "Thread identity or initial digest exceeds the SQLite storage bounds."
|
|
648
959
|
});
|
|
649
960
|
yield* withWriteTransaction("materialize transaction")(Effect.gen(function* () {
|
|
650
961
|
const existingRows = yield* sql`
|
|
651
962
|
SELECT
|
|
652
|
-
|
|
963
|
+
thread_id,
|
|
653
964
|
created_at,
|
|
654
965
|
tail_sequence,
|
|
655
966
|
tail_digest,
|
|
656
967
|
producer_epoch
|
|
657
|
-
FROM
|
|
658
|
-
WHERE
|
|
659
|
-
`.pipe(Effect.mapError(storageError("read materialized
|
|
660
|
-
const existing = yield* decodeRows$
|
|
968
|
+
FROM effect_agent_threads
|
|
969
|
+
WHERE thread_id = ${threadId}
|
|
970
|
+
`.pipe(Effect.mapError(storageError$1("read materialized thread")));
|
|
971
|
+
const existing = yield* decodeRows$3(Schema.Array(ThreadRow), "effect_agent_threads", threadId, existingRows);
|
|
661
972
|
if (existing.length > 1) return yield* SqliteStorageCorruptionError.make({
|
|
662
|
-
table: "
|
|
663
|
-
rowKey:
|
|
664
|
-
message: "A
|
|
973
|
+
table: "effect_agent_threads",
|
|
974
|
+
rowKey: threadId,
|
|
975
|
+
message: "A thread primary key returned more than one row."
|
|
665
976
|
});
|
|
666
977
|
if (existing.length === 0) {
|
|
667
978
|
yield* sql`
|
|
668
|
-
INSERT INTO
|
|
669
|
-
|
|
979
|
+
INSERT INTO effect_agent_threads (
|
|
980
|
+
thread_id,
|
|
670
981
|
created_at,
|
|
671
982
|
tail_sequence,
|
|
672
983
|
tail_digest,
|
|
673
984
|
producer_epoch
|
|
674
985
|
) VALUES (
|
|
675
|
-
${
|
|
986
|
+
${threadId},
|
|
676
987
|
${createdAt},
|
|
677
988
|
0,
|
|
678
989
|
${emptyTailDigest},
|
|
679
990
|
${producerEpoch}
|
|
680
991
|
)
|
|
681
|
-
`.pipe(Effect.mapError(storageError("materialize
|
|
992
|
+
`.pipe(Effect.mapError(storageError$1("materialize thread")));
|
|
682
993
|
return;
|
|
683
994
|
}
|
|
684
995
|
if (producerEpoch < existing[0].producer_epoch) return yield* SqliteFenceRejected.make({
|
|
@@ -687,27 +998,27 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
687
998
|
message: `Producer epoch ${producerEpoch} is stale; current epoch is ${existing[0].producer_epoch}.`
|
|
688
999
|
});
|
|
689
1000
|
if (producerEpoch > existing[0].producer_epoch) yield* sql`
|
|
690
|
-
UPDATE
|
|
1001
|
+
UPDATE effect_agent_threads
|
|
691
1002
|
SET producer_epoch = ${producerEpoch}
|
|
692
|
-
WHERE
|
|
693
|
-
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
1003
|
+
WHERE thread_id = ${threadId}
|
|
1004
|
+
`.pipe(Effect.mapError(storageError$1("advance materialization epoch")));
|
|
694
1005
|
}));
|
|
695
1006
|
});
|
|
696
|
-
const
|
|
1007
|
+
const getThread = Effect.fn("SqliteJournal.getThread")(function* (threadId) {
|
|
697
1008
|
const rows = yield* sql`
|
|
698
1009
|
SELECT
|
|
699
|
-
|
|
1010
|
+
thread_id,
|
|
700
1011
|
created_at,
|
|
701
1012
|
tail_sequence,
|
|
702
1013
|
tail_digest,
|
|
703
1014
|
producer_epoch
|
|
704
|
-
FROM
|
|
705
|
-
WHERE
|
|
706
|
-
`.pipe(Effect.mapError(storageError("read
|
|
707
|
-
return yield* decodeRows$
|
|
1015
|
+
FROM effect_agent_threads
|
|
1016
|
+
WHERE thread_id = ${threadId}
|
|
1017
|
+
`.pipe(Effect.mapError(storageError$1("read thread")));
|
|
1018
|
+
return yield* decodeRows$3(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
708
1019
|
});
|
|
709
1020
|
const append = Effect.fn("SqliteJournal.append")(function* (request) {
|
|
710
|
-
if (request.
|
|
1021
|
+
if (request.threadId.length > MAX_IDENTIFIER_LENGTH || request.batchId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(request.batchJson) > MAX_STORED_TEXT_BYTES || storedTextBytes(request.batchDigest) > MAX_STORED_TEXT_BYTES || storedTextBytes(request.tailDigest) > MAX_STORED_TEXT_BYTES || request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(record.recordJson) > MAX_STORED_TEXT_BYTES)) return yield* SqliteStorageError.make({
|
|
711
1022
|
operation: "append canonical batch",
|
|
712
1023
|
message: "Canonical identifiers or encoded JSON exceed the SQLite storage bounds."
|
|
713
1024
|
});
|
|
@@ -717,25 +1028,25 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
717
1028
|
message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
|
|
718
1029
|
reason: "record-identity"
|
|
719
1030
|
});
|
|
720
|
-
const
|
|
1031
|
+
const threadRows = yield* sql`
|
|
721
1032
|
SELECT
|
|
722
|
-
|
|
1033
|
+
thread_id,
|
|
723
1034
|
created_at,
|
|
724
1035
|
tail_sequence,
|
|
725
1036
|
tail_digest,
|
|
726
1037
|
producer_epoch
|
|
727
|
-
FROM
|
|
728
|
-
WHERE
|
|
729
|
-
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
730
|
-
const
|
|
731
|
-
if (request.producerEpoch !==
|
|
1038
|
+
FROM effect_agent_threads
|
|
1039
|
+
WHERE thread_id = ${request.threadId}
|
|
1040
|
+
`.pipe(Effect.mapError(storageError$1("read append tail")));
|
|
1041
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", request.threadId, threadRows);
|
|
1042
|
+
if (request.producerEpoch !== thread.producer_epoch) return yield* SqliteFenceRejected.make({
|
|
732
1043
|
producerEpoch: request.producerEpoch,
|
|
733
|
-
actualEpoch:
|
|
734
|
-
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${
|
|
1044
|
+
actualEpoch: thread.producer_epoch,
|
|
1045
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`
|
|
735
1046
|
});
|
|
736
1047
|
const batchRows = yield* sql`
|
|
737
1048
|
SELECT
|
|
738
|
-
|
|
1049
|
+
thread_id,
|
|
739
1050
|
batch_id,
|
|
740
1051
|
first_sequence,
|
|
741
1052
|
last_sequence,
|
|
@@ -743,13 +1054,13 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
743
1054
|
tail_digest,
|
|
744
1055
|
batch_json
|
|
745
1056
|
FROM effect_agent_canonical_batches
|
|
746
|
-
WHERE
|
|
1057
|
+
WHERE thread_id = ${request.threadId}
|
|
747
1058
|
AND batch_id = ${request.batchId}
|
|
748
|
-
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
749
|
-
const batches = yield* decodeRows$
|
|
1059
|
+
`.pipe(Effect.mapError(storageError$1("read idempotent batch")));
|
|
1060
|
+
const batches = yield* decodeRows$3(Schema.Array(BatchRow), "effect_agent_canonical_batches", `${request.threadId}/${request.batchId}`, batchRows);
|
|
750
1061
|
if (batches.length > 1) return yield* SqliteStorageCorruptionError.make({
|
|
751
1062
|
table: "effect_agent_canonical_batches",
|
|
752
|
-
rowKey: `${request.
|
|
1063
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
753
1064
|
message: "A canonical batch primary key returned more than one row."
|
|
754
1065
|
});
|
|
755
1066
|
if (batches.length === 1) {
|
|
@@ -765,34 +1076,34 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
765
1076
|
tailDigest: existing.tail_digest
|
|
766
1077
|
});
|
|
767
1078
|
}
|
|
768
|
-
if (request.expectedTailSequence !==
|
|
769
|
-
message: `Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} but found ${
|
|
1079
|
+
if (request.expectedTailSequence !== thread.tail_sequence || request.expectedTailDigest !== thread.tail_digest) return yield* SqliteAppendConflict.make({
|
|
1080
|
+
message: `Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
770
1081
|
reason: "tail",
|
|
771
|
-
actualTailSequence:
|
|
772
|
-
actualTailDigest:
|
|
1082
|
+
actualTailSequence: thread.tail_sequence,
|
|
1083
|
+
actualTailDigest: thread.tail_digest
|
|
773
1084
|
});
|
|
774
|
-
if (
|
|
1085
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) return yield* SqliteStorageError.make({
|
|
775
1086
|
operation: "append canonical batch",
|
|
776
|
-
message: `
|
|
1087
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`
|
|
777
1088
|
});
|
|
778
1089
|
const existingRecordRows = yield* sql`
|
|
779
1090
|
SELECT
|
|
780
|
-
|
|
1091
|
+
thread_id,
|
|
781
1092
|
sequence,
|
|
782
1093
|
record_id,
|
|
783
1094
|
batch_id,
|
|
784
1095
|
record_json
|
|
785
1096
|
FROM effect_agent_canonical_records
|
|
786
|
-
WHERE
|
|
1097
|
+
WHERE thread_id = ${request.threadId}
|
|
787
1098
|
AND record_id IN ${sql.in(recordIds)}
|
|
788
1099
|
ORDER BY sequence
|
|
789
|
-
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
790
|
-
const existingRecords = yield* decodeRows$
|
|
1100
|
+
`.pipe(Effect.mapError(storageError$1("check canonical record identities")));
|
|
1101
|
+
const existingRecords = yield* decodeRows$3(Schema.Array(RecordRow), "effect_agent_canonical_records", `${request.threadId}/record_ids`, existingRecordRows);
|
|
791
1102
|
if (existingRecords.length > 0) return yield* SqliteAppendConflict.make({
|
|
792
1103
|
message: `Canonical record ID ${existingRecords[0].record_id} already exists.`,
|
|
793
1104
|
reason: "record-identity"
|
|
794
1105
|
});
|
|
795
|
-
const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
1106
|
+
const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(thread.tail_sequence + 1).pipe(Effect.mapError((error) => SqliteStorageError.make({
|
|
796
1107
|
cause: error,
|
|
797
1108
|
operation: "append canonical batch",
|
|
798
1109
|
message: error.message
|
|
@@ -804,7 +1115,7 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
804
1115
|
})));
|
|
805
1116
|
yield* sql`
|
|
806
1117
|
INSERT INTO effect_agent_canonical_batches (
|
|
807
|
-
|
|
1118
|
+
thread_id,
|
|
808
1119
|
batch_id,
|
|
809
1120
|
first_sequence,
|
|
810
1121
|
last_sequence,
|
|
@@ -812,7 +1123,7 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
812
1123
|
tail_digest,
|
|
813
1124
|
batch_json
|
|
814
1125
|
) VALUES (
|
|
815
|
-
${request.
|
|
1126
|
+
${request.threadId},
|
|
816
1127
|
${request.batchId},
|
|
817
1128
|
${firstSequence},
|
|
818
1129
|
${lastSequence},
|
|
@@ -820,34 +1131,34 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
820
1131
|
${request.tailDigest},
|
|
821
1132
|
${request.batchJson}
|
|
822
1133
|
)
|
|
823
|
-
`.pipe(Effect.mapError(storageError("insert canonical batch")));
|
|
1134
|
+
`.pipe(Effect.mapError(storageError$1("insert canonical batch")));
|
|
824
1135
|
yield* failpoint("append:after-batch-insert");
|
|
825
1136
|
yield* Effect.forEach(request.records, (record, index) => Effect.gen(function* () {
|
|
826
1137
|
yield* sql`
|
|
827
1138
|
INSERT INTO effect_agent_canonical_records (
|
|
828
|
-
|
|
1139
|
+
thread_id,
|
|
829
1140
|
sequence,
|
|
830
1141
|
record_id,
|
|
831
1142
|
batch_id,
|
|
832
1143
|
record_json
|
|
833
1144
|
) VALUES (
|
|
834
|
-
${request.
|
|
1145
|
+
${request.threadId},
|
|
835
1146
|
${firstSequence + index},
|
|
836
1147
|
${record.recordId},
|
|
837
1148
|
${request.batchId},
|
|
838
1149
|
${record.recordJson}
|
|
839
1150
|
)
|
|
840
|
-
`.pipe(Effect.mapError(storageError("insert canonical record")));
|
|
1151
|
+
`.pipe(Effect.mapError(storageError$1("insert canonical record")));
|
|
841
1152
|
yield* failpoint("append:after-record-insert");
|
|
842
1153
|
}), { discard: true });
|
|
843
1154
|
yield* sql`
|
|
844
|
-
UPDATE
|
|
1155
|
+
UPDATE effect_agent_threads
|
|
845
1156
|
SET
|
|
846
1157
|
tail_sequence = ${lastSequence},
|
|
847
1158
|
tail_digest = ${request.tailDigest},
|
|
848
1159
|
producer_epoch = ${request.producerEpoch}
|
|
849
|
-
WHERE
|
|
850
|
-
`.pipe(Effect.mapError(storageError("advance
|
|
1160
|
+
WHERE thread_id = ${request.threadId}
|
|
1161
|
+
`.pipe(Effect.mapError(storageError$1("advance thread tail")));
|
|
851
1162
|
yield* failpoint("append:after-tail-update");
|
|
852
1163
|
return RawAppendResult.make({
|
|
853
1164
|
firstSequence,
|
|
@@ -860,36 +1171,36 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
860
1171
|
const read = Effect.fn("SqliteJournal.read")(function* (request) {
|
|
861
1172
|
const rows = yield* sql`
|
|
862
1173
|
SELECT
|
|
863
|
-
|
|
1174
|
+
thread_id,
|
|
864
1175
|
sequence,
|
|
865
1176
|
record_id,
|
|
866
1177
|
batch_id,
|
|
867
1178
|
record_json
|
|
868
1179
|
FROM effect_agent_canonical_records
|
|
869
|
-
WHERE
|
|
1180
|
+
WHERE thread_id = ${request.threadId}
|
|
870
1181
|
AND sequence > ${request.fromSequenceExclusive}
|
|
871
1182
|
ORDER BY sequence
|
|
872
1183
|
LIMIT ${request.limit}
|
|
873
|
-
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
874
|
-
return yield* decodeRows$
|
|
1184
|
+
`.pipe(Effect.mapError(storageError$1("read canonical records")));
|
|
1185
|
+
return yield* decodeRows$3(Schema.Array(RecordRow), "effect_agent_canonical_records", `${request.threadId}>${request.fromSequenceExclusive}`, rows);
|
|
875
1186
|
});
|
|
876
|
-
const
|
|
1187
|
+
const exportThread = Effect.fn("SqliteJournal.exportThread")(function* (threadId) {
|
|
877
1188
|
return yield* withReadTransaction("export transaction")(Effect.gen(function* () {
|
|
878
|
-
const
|
|
1189
|
+
const threadRows = yield* sql`
|
|
879
1190
|
SELECT
|
|
880
|
-
|
|
1191
|
+
thread_id,
|
|
881
1192
|
created_at,
|
|
882
1193
|
tail_sequence,
|
|
883
1194
|
tail_digest,
|
|
884
1195
|
producer_epoch
|
|
885
|
-
FROM
|
|
886
|
-
WHERE
|
|
887
|
-
`.pipe(Effect.mapError(storageError("export
|
|
888
|
-
const
|
|
889
|
-
yield* failpoint("export:after-
|
|
1196
|
+
FROM effect_agent_threads
|
|
1197
|
+
WHERE thread_id = ${threadId}
|
|
1198
|
+
`.pipe(Effect.mapError(storageError$1("export thread")));
|
|
1199
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", threadId, threadRows);
|
|
1200
|
+
yield* failpoint("export:after-thread-read");
|
|
890
1201
|
const batchRows = yield* sql`
|
|
891
1202
|
SELECT
|
|
892
|
-
|
|
1203
|
+
thread_id,
|
|
893
1204
|
batch_id,
|
|
894
1205
|
first_sequence,
|
|
895
1206
|
last_sequence,
|
|
@@ -897,70 +1208,70 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
897
1208
|
tail_digest,
|
|
898
1209
|
batch_json
|
|
899
1210
|
FROM effect_agent_canonical_batches
|
|
900
|
-
WHERE
|
|
1211
|
+
WHERE thread_id = ${threadId}
|
|
901
1212
|
ORDER BY first_sequence
|
|
902
|
-
`.pipe(Effect.mapError(storageError("export canonical batches")));
|
|
1213
|
+
`.pipe(Effect.mapError(storageError$1("export canonical batches")));
|
|
903
1214
|
const recordRows = yield* sql`
|
|
904
1215
|
SELECT
|
|
905
|
-
|
|
1216
|
+
thread_id,
|
|
906
1217
|
sequence,
|
|
907
1218
|
record_id,
|
|
908
1219
|
batch_id,
|
|
909
1220
|
record_json
|
|
910
1221
|
FROM effect_agent_canonical_records
|
|
911
|
-
WHERE
|
|
1222
|
+
WHERE thread_id = ${threadId}
|
|
912
1223
|
ORDER BY sequence
|
|
913
|
-
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
1224
|
+
`.pipe(Effect.mapError(storageError$1("export canonical records")));
|
|
914
1225
|
const checkpointRows = yield* sql`
|
|
915
1226
|
SELECT
|
|
916
|
-
|
|
1227
|
+
thread_id,
|
|
917
1228
|
through_sequence,
|
|
918
1229
|
tail_digest,
|
|
919
1230
|
checkpoint_json
|
|
920
1231
|
FROM effect_agent_checkpoints
|
|
921
|
-
WHERE
|
|
1232
|
+
WHERE thread_id = ${threadId}
|
|
922
1233
|
ORDER BY through_sequence
|
|
923
|
-
`.pipe(Effect.mapError(storageError("export checkpoints")));
|
|
924
|
-
return
|
|
925
|
-
|
|
926
|
-
batches: yield* decodeRows$
|
|
927
|
-
records: yield* decodeRows$
|
|
928
|
-
checkpoints: yield* decodeRows$
|
|
1234
|
+
`.pipe(Effect.mapError(storageError$1("export checkpoints")));
|
|
1235
|
+
return RawThreadExport.make({
|
|
1236
|
+
thread,
|
|
1237
|
+
batches: yield* decodeRows$3(Schema.Array(BatchRow), "effect_agent_canonical_batches", threadId, batchRows),
|
|
1238
|
+
records: yield* decodeRows$3(Schema.Array(RecordRow), "effect_agent_canonical_records", threadId, recordRows),
|
|
1239
|
+
checkpoints: yield* decodeRows$3(Schema.Array(CheckpointRow), "effect_agent_checkpoints", threadId, checkpointRows)
|
|
929
1240
|
});
|
|
930
1241
|
}));
|
|
931
1242
|
});
|
|
932
1243
|
const saveCheckpoint = Effect.fn("SqliteJournal.saveCheckpoint")(function* (checkpoint) {
|
|
933
|
-
if (checkpoint.
|
|
1244
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(checkpoint.checkpointJson) > MAX_STORED_TEXT_BYTES) return yield* SqliteStorageError.make({
|
|
934
1245
|
operation: "save checkpoint",
|
|
935
1246
|
message: "Checkpoint identity or encoded JSON exceeds the SQLite storage bounds."
|
|
936
1247
|
});
|
|
937
1248
|
yield* withWriteTransaction("checkpoint transaction")(Effect.gen(function* () {
|
|
938
|
-
const
|
|
1249
|
+
const threadRows = yield* sql`
|
|
939
1250
|
SELECT
|
|
940
|
-
|
|
1251
|
+
thread_id,
|
|
941
1252
|
created_at,
|
|
942
1253
|
tail_sequence,
|
|
943
1254
|
tail_digest,
|
|
944
1255
|
producer_epoch
|
|
945
|
-
FROM
|
|
946
|
-
WHERE
|
|
947
|
-
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
948
|
-
const
|
|
949
|
-
if (checkpoint.throughSequence >
|
|
1256
|
+
FROM effect_agent_threads
|
|
1257
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
1258
|
+
`.pipe(Effect.mapError(storageError$1("read checkpoint tail")));
|
|
1259
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", checkpoint.threadId, threadRows);
|
|
1260
|
+
if (checkpoint.throughSequence > thread.tail_sequence) return yield* SqliteCheckpointConflict.make({ message: `Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ${thread.tail_sequence}.` });
|
|
950
1261
|
const checkpointRows = yield* sql`
|
|
951
1262
|
SELECT
|
|
952
|
-
|
|
1263
|
+
thread_id,
|
|
953
1264
|
through_sequence,
|
|
954
1265
|
tail_digest,
|
|
955
1266
|
checkpoint_json
|
|
956
1267
|
FROM effect_agent_checkpoints
|
|
957
|
-
WHERE
|
|
1268
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
958
1269
|
AND through_sequence = ${checkpoint.throughSequence}
|
|
959
|
-
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
960
|
-
const existing = yield* decodeRows$
|
|
1270
|
+
`.pipe(Effect.mapError(storageError$1("read idempotent checkpoint")));
|
|
1271
|
+
const existing = yield* decodeRows$3(Schema.Array(CheckpointRow), "effect_agent_checkpoints", `${checkpoint.threadId}/${checkpoint.throughSequence}`, checkpointRows);
|
|
961
1272
|
if (existing.length > 1) return yield* SqliteStorageCorruptionError.make({
|
|
962
1273
|
table: "effect_agent_checkpoints",
|
|
963
|
-
rowKey: `${checkpoint.
|
|
1274
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
964
1275
|
message: "A checkpoint primary key returned more than one row."
|
|
965
1276
|
});
|
|
966
1277
|
if (existing.length === 1) {
|
|
@@ -969,46 +1280,46 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
969
1280
|
}
|
|
970
1281
|
yield* sql`
|
|
971
1282
|
INSERT INTO effect_agent_checkpoints (
|
|
972
|
-
|
|
1283
|
+
thread_id,
|
|
973
1284
|
through_sequence,
|
|
974
1285
|
tail_digest,
|
|
975
1286
|
checkpoint_json
|
|
976
1287
|
) VALUES (
|
|
977
|
-
${checkpoint.
|
|
1288
|
+
${checkpoint.threadId},
|
|
978
1289
|
${checkpoint.throughSequence},
|
|
979
1290
|
${checkpoint.tailDigest},
|
|
980
1291
|
${checkpoint.checkpointJson}
|
|
981
1292
|
)
|
|
982
|
-
`.pipe(Effect.mapError(storageError("insert checkpoint")));
|
|
1293
|
+
`.pipe(Effect.mapError(storageError$1("insert checkpoint")));
|
|
983
1294
|
}));
|
|
984
1295
|
});
|
|
985
|
-
const loadCheckpoint = Effect.fn("SqliteJournal.loadCheckpoint")(function* (
|
|
1296
|
+
const loadCheckpoint = Effect.fn("SqliteJournal.loadCheckpoint")(function* (threadId, atOrBeforeSequence) {
|
|
986
1297
|
const rows = yield* sql`
|
|
987
1298
|
SELECT
|
|
988
|
-
|
|
1299
|
+
thread_id,
|
|
989
1300
|
through_sequence,
|
|
990
1301
|
tail_digest,
|
|
991
1302
|
checkpoint_json
|
|
992
1303
|
FROM effect_agent_checkpoints
|
|
993
|
-
WHERE
|
|
1304
|
+
WHERE thread_id = ${threadId}
|
|
994
1305
|
AND through_sequence <= ${atOrBeforeSequence}
|
|
995
1306
|
ORDER BY through_sequence DESC
|
|
996
1307
|
LIMIT 1
|
|
997
|
-
`.pipe(Effect.mapError(storageError("load checkpoint")));
|
|
998
|
-
return yield* decodeRows$
|
|
1308
|
+
`.pipe(Effect.mapError(storageError$1("load checkpoint")));
|
|
1309
|
+
return yield* decodeRows$3(Schema.Array(CheckpointRow), "effect_agent_checkpoints", `${threadId}<=${atOrBeforeSequence}`, rows);
|
|
999
1310
|
});
|
|
1000
1311
|
return {
|
|
1001
1312
|
append,
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
getTailDigestAt: Effect.fn("SqliteJournal.getTailDigestAt")(function* (
|
|
1313
|
+
exportThread,
|
|
1314
|
+
getThread,
|
|
1315
|
+
getTailDigestAt: Effect.fn("SqliteJournal.getTailDigestAt")(function* (threadId, sequence) {
|
|
1005
1316
|
if (sequence === 0) {
|
|
1006
|
-
const
|
|
1007
|
-
return
|
|
1317
|
+
const threads = yield* getThread(threadId);
|
|
1318
|
+
return threads.length === 0 ? [] : [threads[0].tail_sequence === 0 ? threads[0].tail_digest : void 0].filter((value) => value !== void 0);
|
|
1008
1319
|
}
|
|
1009
1320
|
const rows = yield* sql`
|
|
1010
1321
|
SELECT
|
|
1011
|
-
|
|
1322
|
+
thread_id,
|
|
1012
1323
|
batch_id,
|
|
1013
1324
|
first_sequence,
|
|
1014
1325
|
last_sequence,
|
|
@@ -1016,10 +1327,10 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
1016
1327
|
tail_digest,
|
|
1017
1328
|
batch_json
|
|
1018
1329
|
FROM effect_agent_canonical_batches
|
|
1019
|
-
WHERE
|
|
1330
|
+
WHERE thread_id = ${threadId}
|
|
1020
1331
|
AND last_sequence = ${sequence}
|
|
1021
|
-
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
1022
|
-
return (yield* decodeRows$
|
|
1332
|
+
`.pipe(Effect.mapError(storageError$1("read canonical digest at sequence")));
|
|
1333
|
+
return (yield* decodeRows$3(Schema.Array(BatchRow), "effect_agent_canonical_batches", `${threadId}/${sequence}`, rows)).map((batch) => batch.tail_digest);
|
|
1023
1334
|
}),
|
|
1024
1335
|
loadCheckpoint,
|
|
1025
1336
|
materialize,
|
|
@@ -1027,19 +1338,19 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
1027
1338
|
saveCheckpoint,
|
|
1028
1339
|
scanStoredPayloads: Effect.fn("SqliteJournal.scanStoredPayloads")(function* () {
|
|
1029
1340
|
return yield* withReadTransaction("startup scan transaction")(Effect.gen(function* () {
|
|
1030
|
-
const
|
|
1341
|
+
const threads = yield* sql`
|
|
1031
1342
|
SELECT
|
|
1032
|
-
|
|
1343
|
+
thread_id,
|
|
1033
1344
|
created_at,
|
|
1034
1345
|
tail_sequence,
|
|
1035
1346
|
tail_digest,
|
|
1036
1347
|
producer_epoch
|
|
1037
|
-
FROM
|
|
1038
|
-
ORDER BY
|
|
1039
|
-
`.pipe(Effect.mapError(storageError("scan
|
|
1348
|
+
FROM effect_agent_threads
|
|
1349
|
+
ORDER BY thread_id
|
|
1350
|
+
`.pipe(Effect.mapError(storageError$1("scan threads")));
|
|
1040
1351
|
const batches = yield* sql`
|
|
1041
1352
|
SELECT
|
|
1042
|
-
|
|
1353
|
+
thread_id,
|
|
1043
1354
|
batch_id,
|
|
1044
1355
|
first_sequence,
|
|
1045
1356
|
last_sequence,
|
|
@@ -1047,32 +1358,32 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
1047
1358
|
tail_digest,
|
|
1048
1359
|
batch_json
|
|
1049
1360
|
FROM effect_agent_canonical_batches
|
|
1050
|
-
ORDER BY
|
|
1051
|
-
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
1361
|
+
ORDER BY thread_id, first_sequence
|
|
1362
|
+
`.pipe(Effect.mapError(storageError$1("scan canonical batches")));
|
|
1052
1363
|
const records = yield* sql`
|
|
1053
1364
|
SELECT
|
|
1054
|
-
|
|
1365
|
+
thread_id,
|
|
1055
1366
|
sequence,
|
|
1056
1367
|
record_id,
|
|
1057
1368
|
batch_id,
|
|
1058
1369
|
record_json
|
|
1059
1370
|
FROM effect_agent_canonical_records
|
|
1060
|
-
ORDER BY
|
|
1061
|
-
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
1371
|
+
ORDER BY thread_id, sequence
|
|
1372
|
+
`.pipe(Effect.mapError(storageError$1("scan canonical records")));
|
|
1062
1373
|
const checkpoints = yield* sql`
|
|
1063
1374
|
SELECT
|
|
1064
|
-
|
|
1375
|
+
thread_id,
|
|
1065
1376
|
through_sequence,
|
|
1066
1377
|
tail_digest,
|
|
1067
1378
|
checkpoint_json
|
|
1068
1379
|
FROM effect_agent_checkpoints
|
|
1069
|
-
ORDER BY
|
|
1070
|
-
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
1380
|
+
ORDER BY thread_id, through_sequence
|
|
1381
|
+
`.pipe(Effect.mapError(storageError$1("scan checkpoints")));
|
|
1071
1382
|
return {
|
|
1072
|
-
|
|
1073
|
-
batches: yield* decodeRows$
|
|
1074
|
-
records: yield* decodeRows$
|
|
1075
|
-
checkpoints: yield* decodeRows$
|
|
1383
|
+
threads: yield* decodeRows$3(Schema.Array(ThreadRow), "effect_agent_threads", "startup_scan", threads),
|
|
1384
|
+
batches: yield* decodeRows$3(Schema.Array(BatchRow), "effect_agent_canonical_batches", "startup_scan", batches),
|
|
1385
|
+
records: yield* decodeRows$3(Schema.Array(RecordRow), "effect_agent_canonical_records", "startup_scan", records),
|
|
1386
|
+
checkpoints: yield* decodeRows$3(Schema.Array(CheckpointRow), "effect_agent_checkpoints", "startup_scan", checkpoints)
|
|
1076
1387
|
};
|
|
1077
1388
|
}));
|
|
1078
1389
|
}),
|
|
@@ -1080,85 +1391,85 @@ const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function*
|
|
|
1080
1391
|
};
|
|
1081
1392
|
});
|
|
1082
1393
|
//#endregion
|
|
1083
|
-
//#region src/sqlite-
|
|
1084
|
-
const OffsetText = Schema.String.check(Schema.isMaxLength(
|
|
1394
|
+
//#region src/sqlite-thread-store.ts
|
|
1395
|
+
const OffsetText = Schema.String.check(Schema.isMaxLength(4096));
|
|
1085
1396
|
const SQLITE_OFFSET_PREFIX = "effect-agent-sqlite@1:";
|
|
1086
1397
|
const ZERO_CANONICAL_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
1087
1398
|
const isDigest = Schema.is(Digest);
|
|
1088
1399
|
const isSqliteFenceRejected = Schema.is(SqliteFenceRejected);
|
|
1089
1400
|
const isSqliteAppendConflict = Schema.is(SqliteAppendConflict);
|
|
1090
1401
|
const isSqliteCheckpointConflict = Schema.is(SqliteCheckpointConflict);
|
|
1091
|
-
const storeError = (operation, error) =>
|
|
1402
|
+
const storeError = (operation, error) => ThreadStoreError.make({
|
|
1092
1403
|
cause: error,
|
|
1093
1404
|
operation,
|
|
1094
1405
|
message: error.message
|
|
1095
1406
|
});
|
|
1096
|
-
const schemaStoreError = (operation, error) =>
|
|
1407
|
+
const schemaStoreError = (operation, error) => ThreadStoreError.make({
|
|
1097
1408
|
cause: error,
|
|
1098
1409
|
operation,
|
|
1099
1410
|
message: error.message
|
|
1100
1411
|
});
|
|
1101
|
-
const makeOffset = Effect.fn("
|
|
1102
|
-
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(sequence).pipe(Effect.flatMap((validatedSequence) => Schema.decodeUnknownEffect(ObservationOffset)(`${SQLITE_OFFSET_PREFIX}${encodeURIComponent(
|
|
1412
|
+
const makeOffset = Effect.fn("SqliteThreadStore.makeOffset")(function* (threadId, sequence) {
|
|
1413
|
+
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(sequence).pipe(Effect.flatMap((validatedSequence) => Schema.decodeUnknownEffect(ObservationOffset)(`${SQLITE_OFFSET_PREFIX}${encodeURIComponent(threadId)}:${validatedSequence}`)), Effect.mapError((error) => schemaStoreError("encode observation offset", error)));
|
|
1103
1414
|
});
|
|
1104
|
-
const parseOffset = Effect.fn("
|
|
1415
|
+
const parseOffset = Effect.fn("SqliteThreadStore.parseOffset")(function* (threadId, offset) {
|
|
1105
1416
|
if (offset === void 0) return ZERO_CANONICAL_SEQUENCE;
|
|
1106
1417
|
const text = yield* Schema.decodeUnknownEffect(OffsetText)(offset).pipe(Effect.mapError((error) => schemaStoreError("decode observation offset", error)));
|
|
1107
|
-
const
|
|
1108
|
-
if (!text.startsWith(
|
|
1418
|
+
const threadPrefix = `${SQLITE_OFFSET_PREFIX}${encodeURIComponent(threadId)}:`;
|
|
1419
|
+
if (!text.startsWith(threadPrefix)) return yield* ThreadStoreError.make({
|
|
1109
1420
|
operation: "decode observation offset",
|
|
1110
|
-
message: "The observation offset belongs to a different adapter, storage version, or
|
|
1421
|
+
message: "The observation offset belongs to a different adapter, storage version, or Thread."
|
|
1111
1422
|
});
|
|
1112
|
-
const sequenceText = text.slice(
|
|
1113
|
-
if (!/^(0|[1-9][0-9]*)$/.test(sequenceText)) return yield*
|
|
1423
|
+
const sequenceText = text.slice(threadPrefix.length);
|
|
1424
|
+
if (!/^(0|[1-9][0-9]*)$/.test(sequenceText)) return yield* ThreadStoreError.make({
|
|
1114
1425
|
operation: "decode observation offset",
|
|
1115
1426
|
message: "The observation offset is malformed."
|
|
1116
1427
|
});
|
|
1117
1428
|
return yield* Schema.decodeUnknownEffect(CanonicalSequence)(Number(sequenceText)).pipe(Effect.mapError((error) => schemaStoreError("decode observation offset", error)));
|
|
1118
1429
|
});
|
|
1119
|
-
const mapFence = (
|
|
1120
|
-
|
|
1430
|
+
const mapFence = (threadId, error) => FenceRejected.make({
|
|
1431
|
+
threadId,
|
|
1121
1432
|
actualEpoch: error.actualEpoch,
|
|
1122
1433
|
attemptedEpoch: error.producerEpoch
|
|
1123
1434
|
});
|
|
1124
|
-
const encodeCanonicalRecord = Effect.fn("
|
|
1435
|
+
const encodeCanonicalRecord = Effect.fn("SqliteThreadStore.encodeCanonicalRecord")(function* (record) {
|
|
1125
1436
|
return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalRecord))(record).pipe(Effect.mapError((error) => schemaStoreError("encode canonical record", error)));
|
|
1126
1437
|
});
|
|
1127
|
-
const encodeCanonicalBatch = Effect.fn("
|
|
1438
|
+
const encodeCanonicalBatch = Effect.fn("SqliteThreadStore.encodeCanonicalBatch")(function* (batch) {
|
|
1128
1439
|
return yield* Schema.encodeEffect(Schema.fromJsonString(CanonicalBatch))(batch).pipe(Effect.mapError((error) => schemaStoreError("encode canonical batch", error)));
|
|
1129
1440
|
});
|
|
1130
|
-
const encodeCheckpoint = Effect.fn("
|
|
1131
|
-
return yield* Schema.encodeEffect(Schema.fromJsonString(
|
|
1441
|
+
const encodeCheckpoint = Effect.fn("SqliteThreadStore.encodeCheckpoint")(function* (checkpoint) {
|
|
1442
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint).pipe(Effect.mapError((error) => schemaStoreError("encode checkpoint", error)));
|
|
1132
1443
|
});
|
|
1133
|
-
const decodeEnvelope = Effect.fn("
|
|
1134
|
-
const record = yield* Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(row.record_json).pipe(Effect.mapError((error) =>
|
|
1444
|
+
const decodeEnvelope = Effect.fn("SqliteThreadStore.decodeEnvelope")(function* (row) {
|
|
1445
|
+
const record = yield* Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(row.record_json).pipe(Effect.mapError((error) => ThreadStoreError.make({
|
|
1135
1446
|
operation: "decode canonical record",
|
|
1136
1447
|
message: error.message
|
|
1137
1448
|
})));
|
|
1138
|
-
const
|
|
1139
|
-
const offset = yield* makeOffset(
|
|
1449
|
+
const threadId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.threadId)(row.thread_id).pipe(Effect.mapError((error) => schemaStoreError("decode thread identity", error)));
|
|
1450
|
+
const offset = yield* makeOffset(threadId, row.sequence);
|
|
1140
1451
|
const batchId = yield* Schema.decodeUnknownEffect(CanonicalRecordEnvelope.fields.batchId)(row.batch_id).pipe(Effect.mapError((error) => schemaStoreError("decode batch identity", error)));
|
|
1141
1452
|
return CanonicalRecordEnvelope.make({
|
|
1142
|
-
|
|
1453
|
+
threadId,
|
|
1143
1454
|
batchId,
|
|
1144
1455
|
sequence: row.sequence,
|
|
1145
1456
|
offset,
|
|
1146
1457
|
record
|
|
1147
1458
|
});
|
|
1148
1459
|
});
|
|
1149
|
-
const decodeCheckpoint = Effect.fn("
|
|
1150
|
-
return yield* Schema.decodeEffect(Schema.fromJsonString(
|
|
1460
|
+
const decodeCheckpoint = Effect.fn("SqliteThreadStore.decodeCheckpoint")(function* (checkpointJson) {
|
|
1461
|
+
return yield* Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpointJson).pipe(Effect.mapError((error) => schemaStoreError("decode checkpoint", error)));
|
|
1151
1462
|
});
|
|
1152
|
-
const
|
|
1153
|
-
const rows = yield* journal.
|
|
1154
|
-
if (rows.length === 0) return yield*
|
|
1463
|
+
const requireThread = Effect.fn("SqliteThreadStore.requireThread")(function* (journal, threadId) {
|
|
1464
|
+
const rows = yield* journal.getThread(threadId).pipe(Effect.mapError((error) => storeError("read thread", error)));
|
|
1465
|
+
if (rows.length === 0) return yield* ThreadNotMaterialized.make({ threadId });
|
|
1155
1466
|
return rows[0];
|
|
1156
1467
|
});
|
|
1157
|
-
const tailDigestAt = Effect.fn("
|
|
1468
|
+
const tailDigestAt = Effect.fn("SqliteThreadStore.tailDigestAt")(function* (journal, threadId, sequence) {
|
|
1158
1469
|
if (sequence === 0) return EMPTY_TAIL_DIGEST;
|
|
1159
|
-
const digests = yield* journal.getTailDigestAt(
|
|
1470
|
+
const digests = yield* journal.getTailDigestAt(threadId, sequence).pipe(Effect.mapError((error) => storeError("read checkpoint digest", error)));
|
|
1160
1471
|
if (digests.length !== 1) return yield* CheckpointRejected.make({
|
|
1161
|
-
|
|
1472
|
+
threadId,
|
|
1162
1473
|
reason: "digest-mismatch"
|
|
1163
1474
|
});
|
|
1164
1475
|
return yield* Schema.decodeUnknownEffect(Digest)(digests[0]).pipe(Effect.mapError((error) => schemaStoreError("decode checkpoint digest", error)));
|
|
@@ -1177,14 +1488,14 @@ const groupByKey = (rows, key) => {
|
|
|
1177
1488
|
* and re-digested against the canonical chain. Routine opens skip this scan: per-operation
|
|
1178
1489
|
* Schema decoding plus the digest chain already fail clearly on corrupt rows.
|
|
1179
1490
|
*/
|
|
1180
|
-
const decodeStartupPayloads = Effect.fn("
|
|
1491
|
+
const decodeStartupPayloads = Effect.fn("SqliteThreadStore.decodeStartupPayloads")(function* (journal, crypto) {
|
|
1181
1492
|
const stored = yield* journal.scanStoredPayloads();
|
|
1182
1493
|
const batches = yield* Effect.forEach(stored.batches, (batch) => Schema.decodeEffect(Schema.fromJsonString(CanonicalBatch))(batch.batch_json).pipe(Effect.map((decoded) => ({
|
|
1183
1494
|
decoded,
|
|
1184
1495
|
row: batch
|
|
1185
1496
|
})), Effect.mapError((error) => SqliteStorageCorruptionError.make({
|
|
1186
1497
|
table: "effect_agent_canonical_batches",
|
|
1187
|
-
rowKey: `${batch.
|
|
1498
|
+
rowKey: `${batch.thread_id}/${batch.batch_id}`,
|
|
1188
1499
|
message: error.message
|
|
1189
1500
|
}))));
|
|
1190
1501
|
const records = yield* Effect.forEach(stored.records, (record) => Schema.decodeEffect(Schema.fromJsonString(CanonicalRecord))(record.record_json).pipe(Effect.map((decoded) => ({
|
|
@@ -1192,31 +1503,31 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
1192
1503
|
row: record
|
|
1193
1504
|
})), Effect.mapError((error) => SqliteStorageCorruptionError.make({
|
|
1194
1505
|
table: "effect_agent_canonical_records",
|
|
1195
|
-
rowKey: `${record.
|
|
1506
|
+
rowKey: `${record.thread_id}/${record.sequence}`,
|
|
1196
1507
|
message: error.message
|
|
1197
1508
|
}))));
|
|
1198
|
-
const checkpoints = yield* Effect.forEach(stored.checkpoints, (checkpoint) => Schema.decodeEffect(Schema.fromJsonString(
|
|
1509
|
+
const checkpoints = yield* Effect.forEach(stored.checkpoints, (checkpoint) => Schema.decodeEffect(Schema.fromJsonString(ThreadCheckpoint))(checkpoint.checkpoint_json).pipe(Effect.map((decoded) => ({
|
|
1199
1510
|
decoded,
|
|
1200
1511
|
row: checkpoint
|
|
1201
1512
|
})), Effect.mapError((error) => SqliteStorageCorruptionError.make({
|
|
1202
1513
|
table: "effect_agent_checkpoints",
|
|
1203
|
-
rowKey: `${checkpoint.
|
|
1514
|
+
rowKey: `${checkpoint.thread_id}/${checkpoint.through_sequence}`,
|
|
1204
1515
|
message: error.message
|
|
1205
1516
|
}))));
|
|
1206
|
-
const
|
|
1207
|
-
const
|
|
1208
|
-
const
|
|
1209
|
-
const materializedIds = new Set(stored.
|
|
1210
|
-
for (const
|
|
1211
|
-
const
|
|
1212
|
-
const
|
|
1213
|
-
const
|
|
1214
|
-
const recordsByBatch = groupByKey(
|
|
1517
|
+
const batchesByThread = groupByKey(batches, ({ row }) => row.thread_id);
|
|
1518
|
+
const recordsByThread = groupByKey(records, ({ row }) => row.thread_id);
|
|
1519
|
+
const checkpointsByThread = groupByKey(checkpoints, ({ row }) => row.thread_id);
|
|
1520
|
+
const materializedIds = new Set(stored.threads.map((thread) => thread.thread_id));
|
|
1521
|
+
for (const thread of stored.threads) {
|
|
1522
|
+
const threadBatches = batchesByThread.get(thread.thread_id) ?? [];
|
|
1523
|
+
const threadRecords = recordsByThread.get(thread.thread_id) ?? [];
|
|
1524
|
+
const threadCheckpoints = checkpointsByThread.get(thread.thread_id) ?? [];
|
|
1525
|
+
const recordsByBatch = groupByKey(threadRecords, ({ row }) => row.batch_id);
|
|
1215
1526
|
let previousDigest = EMPTY_TAIL_DIGEST;
|
|
1216
1527
|
let expectedSequence = 1;
|
|
1217
1528
|
const tailDigests = /* @__PURE__ */ new Map([[0, EMPTY_TAIL_DIGEST]]);
|
|
1218
|
-
for (const { decoded: canonicalBatch, row: batchRow } of
|
|
1219
|
-
const key = `${batchRow.
|
|
1529
|
+
for (const { decoded: canonicalBatch, row: batchRow } of threadBatches) {
|
|
1530
|
+
const key = `${batchRow.thread_id}/${batchRow.batch_id}`;
|
|
1220
1531
|
if (canonicalBatch.batchId !== batchRow.batch_id || batchRow.first_sequence !== expectedSequence || batchRow.last_sequence !== batchRow.first_sequence + canonicalBatch.records.length - 1) return yield* SqliteStorageCorruptionError.make({
|
|
1221
1532
|
table: "effect_agent_canonical_batches",
|
|
1222
1533
|
rowKey: key,
|
|
@@ -1261,41 +1572,41 @@ const decodeStartupPayloads = Effect.fn("SqliteConversationStore.decodeStartupPa
|
|
|
1261
1572
|
expectedSequence = batchRow.last_sequence + 1;
|
|
1262
1573
|
tailDigests.set(batchRow.last_sequence, digest);
|
|
1263
1574
|
}
|
|
1264
|
-
if (
|
|
1265
|
-
table: "
|
|
1266
|
-
rowKey:
|
|
1267
|
-
message: "
|
|
1575
|
+
if (threadRecords.length !== thread.tail_sequence || thread.tail_sequence !== expectedSequence - 1 || thread.tail_digest !== previousDigest) return yield* SqliteStorageCorruptionError.make({
|
|
1576
|
+
table: "effect_agent_threads",
|
|
1577
|
+
rowKey: thread.thread_id,
|
|
1578
|
+
message: "Thread tail does not match its canonical batch chain."
|
|
1268
1579
|
});
|
|
1269
|
-
for (const checkpoint of
|
|
1580
|
+
for (const checkpoint of threadCheckpoints) if (checkpoint.decoded.threadId !== thread.thread_id || checkpoint.decoded.throughSequence !== checkpoint.row.through_sequence || checkpoint.decoded.tailDigest !== checkpoint.row.tail_digest || tailDigests.get(checkpoint.row.through_sequence) !== checkpoint.row.tail_digest) return yield* SqliteStorageCorruptionError.make({
|
|
1270
1581
|
table: "effect_agent_checkpoints",
|
|
1271
|
-
rowKey: `${
|
|
1582
|
+
rowKey: `${thread.thread_id}/${checkpoint.row.through_sequence}`,
|
|
1272
1583
|
message: "Checkpoint identity or digest is not bound to a canonical batch tail."
|
|
1273
1584
|
});
|
|
1274
1585
|
}
|
|
1275
|
-
if (batches.some(({ row }) => !materializedIds.has(row.
|
|
1276
|
-
table: "
|
|
1586
|
+
if (batches.some(({ row }) => !materializedIds.has(row.thread_id)) || records.some(({ row }) => !materializedIds.has(row.thread_id)) || checkpoints.some(({ row }) => !materializedIds.has(row.thread_id))) return yield* SqliteStorageCorruptionError.make({
|
|
1587
|
+
table: "effect_agent_threads",
|
|
1277
1588
|
rowKey: "startup_scan",
|
|
1278
|
-
message: "Canonical rows exist without a materialized
|
|
1589
|
+
message: "Canonical rows exist without a materialized Thread."
|
|
1279
1590
|
});
|
|
1280
1591
|
});
|
|
1281
|
-
const makeServices$1 = Effect.fn("
|
|
1592
|
+
const makeServices$1 = Effect.fn("SqliteThreadStore.makeServices")(function* () {
|
|
1282
1593
|
const config = yield* SqliteStorageConfig;
|
|
1283
1594
|
const failpoint = yield* SqliteStorageFailpoint;
|
|
1284
1595
|
const crypto = yield* Crypto.Crypto;
|
|
1285
1596
|
const journal = yield* initializeSqliteJournal();
|
|
1286
1597
|
if (config.verifyOnOpen) yield* decodeStartupPayloads(journal, crypto);
|
|
1287
1598
|
const provideCrypto = (effect) => Effect.provideService(effect, Crypto.Crypto, crypto);
|
|
1288
|
-
const hitFailpoint = Effect.fn("
|
|
1289
|
-
const materialize = Effect.fn("
|
|
1290
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
1599
|
+
const hitFailpoint = Effect.fn("SqliteThreadStore.hitFailpoint")((location) => failpoint.hit(location).pipe(Effect.mapError((error) => storeError(`storage failpoint ${location}`, error))));
|
|
1600
|
+
const materialize = Effect.fn("SqliteThreadStore.materialize")(function* (request) {
|
|
1601
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadMaterialization))(request).pipe(Effect.mapError((error) => schemaStoreError("validate materialization", error)));
|
|
1291
1602
|
const now = yield* Clock.currentTimeMillis;
|
|
1292
1603
|
yield* hitFailpoint("materialize:before");
|
|
1293
|
-
yield* journal.materialize(validated.
|
|
1604
|
+
yield* journal.materialize(validated.threadId, new Date(now).toISOString(), EMPTY_TAIL_DIGEST, validated.producerEpoch).pipe(Effect.mapError((error) => error._tag === "SqliteFenceRejected" ? mapFence(validated.threadId, error) : storeError("materialize thread", error)));
|
|
1294
1605
|
yield* hitFailpoint("materialize:after");
|
|
1295
1606
|
});
|
|
1296
|
-
const append = Effect.fn("
|
|
1607
|
+
const append = Effect.fn("SqliteThreadStore.append")(function* (request) {
|
|
1297
1608
|
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(FencedAppendRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate canonical append", error)));
|
|
1298
|
-
yield*
|
|
1609
|
+
yield* requireThread(journal, validated.threadId);
|
|
1299
1610
|
const tailDigest = yield* provideCrypto(digestCanonicalBatch(validated.expectedTailDigest, validated.batch)).pipe(Effect.mapError((error) => storeError("digest canonical append", error)));
|
|
1300
1611
|
const batchJson = yield* encodeCanonicalBatch(validated.batch);
|
|
1301
1612
|
const rawRecords = yield* Effect.forEach(validated.batch.records, (record) => encodeCanonicalRecord(record).pipe(Effect.map((recordJson) => ({
|
|
@@ -1303,7 +1614,7 @@ const makeServices$1 = Effect.fn("SqliteConversationStore.makeServices")(functio
|
|
|
1303
1614
|
recordJson
|
|
1304
1615
|
}))));
|
|
1305
1616
|
const rawRequest = yield* Schema.decodeUnknownEffect(RawAppendRequest)({
|
|
1306
|
-
|
|
1617
|
+
threadId: validated.threadId,
|
|
1307
1618
|
batchId: validated.batch.batchId,
|
|
1308
1619
|
batchDigest: tailDigest,
|
|
1309
1620
|
batchJson,
|
|
@@ -1315,15 +1626,15 @@ const makeServices$1 = Effect.fn("SqliteConversationStore.makeServices")(functio
|
|
|
1315
1626
|
}).pipe(Effect.mapError((error) => schemaStoreError("encode canonical append", error)));
|
|
1316
1627
|
yield* hitFailpoint("append:before");
|
|
1317
1628
|
const result = yield* journal.append(rawRequest).pipe(Effect.mapError((error) => {
|
|
1318
|
-
if (isSqliteFenceRejected(error)) return mapFence(validated.
|
|
1629
|
+
if (isSqliteFenceRejected(error)) return mapFence(validated.threadId, error);
|
|
1319
1630
|
if (isSqliteAppendConflict(error)) return error.actualTailSequence !== void 0 && isDigest(error.actualTailDigest) ? AppendConflict.make({
|
|
1320
|
-
|
|
1631
|
+
threadId: validated.threadId,
|
|
1321
1632
|
batchId: validated.batch.batchId,
|
|
1322
1633
|
reason: error.reason,
|
|
1323
1634
|
actualTailSequence: error.actualTailSequence,
|
|
1324
1635
|
actualTailDigest: error.actualTailDigest
|
|
1325
1636
|
}) : AppendConflict.make({
|
|
1326
|
-
|
|
1637
|
+
threadId: validated.threadId,
|
|
1327
1638
|
batchId: validated.batch.batchId,
|
|
1328
1639
|
reason: error.reason
|
|
1329
1640
|
});
|
|
@@ -1332,30 +1643,30 @@ const makeServices$1 = Effect.fn("SqliteConversationStore.makeServices")(functio
|
|
|
1332
1643
|
yield* hitFailpoint("append:after");
|
|
1333
1644
|
return result;
|
|
1334
1645
|
});
|
|
1335
|
-
const loadRecords = Effect.fn("
|
|
1646
|
+
const loadRecords = Effect.fn("SqliteThreadStore.loadRecords")(function* (request) {
|
|
1336
1647
|
const rows = yield* journal.read(request).pipe(Effect.mapError((error) => storeError("read canonical records", error)));
|
|
1337
1648
|
return yield* Effect.forEach(rows, decodeEnvelope);
|
|
1338
1649
|
});
|
|
1339
|
-
const readEffect = Effect.fn("
|
|
1340
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
1341
|
-
yield*
|
|
1650
|
+
const readEffect = Effect.fn("SqliteThreadStore.read")(function* (request) {
|
|
1651
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadRead))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread read", error)));
|
|
1652
|
+
yield* requireThread(journal, validated.threadId);
|
|
1342
1653
|
const records = yield* loadRecords(RawReadRequest.make({
|
|
1343
|
-
|
|
1654
|
+
threadId: validated.threadId,
|
|
1344
1655
|
fromSequenceExclusive: validated.afterSequence ?? ZERO_CANONICAL_SEQUENCE,
|
|
1345
1656
|
limit: validated.limit
|
|
1346
1657
|
}));
|
|
1347
1658
|
return Stream.fromIterable(records);
|
|
1348
1659
|
});
|
|
1349
1660
|
const read = (request) => Stream.unwrap(readEffect(request));
|
|
1350
|
-
const observeEffect = Effect.fn("
|
|
1351
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
1352
|
-
yield*
|
|
1353
|
-
const initialSequence = yield* parseOffset(validated.
|
|
1661
|
+
const observeEffect = Effect.fn("SqliteThreadStore.observe")(function* (request) {
|
|
1662
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadObservation))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread observation", error)));
|
|
1663
|
+
yield* requireThread(journal, validated.threadId);
|
|
1664
|
+
const initialSequence = yield* parseOffset(validated.threadId, validated.afterOffset);
|
|
1354
1665
|
const cursor = yield* Ref.make(initialSequence);
|
|
1355
|
-
const poll = Effect.fn("
|
|
1666
|
+
const poll = Effect.fn("SqliteThreadStore.observePoll")(function* () {
|
|
1356
1667
|
const fromSequenceExclusive = yield* Ref.get(cursor);
|
|
1357
1668
|
const records = yield* loadRecords(RawReadRequest.make({
|
|
1358
|
-
|
|
1669
|
+
threadId: validated.threadId,
|
|
1359
1670
|
fromSequenceExclusive,
|
|
1360
1671
|
limit: 1024
|
|
1361
1672
|
}));
|
|
@@ -1369,96 +1680,98 @@ const makeServices$1 = Effect.fn("SqliteConversationStore.makeServices")(functio
|
|
|
1369
1680
|
return Stream.fromIterableEffectRepeat(poll());
|
|
1370
1681
|
});
|
|
1371
1682
|
const observe = (request) => Stream.unwrap(observeEffect(request));
|
|
1372
|
-
const
|
|
1373
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
1374
|
-
yield*
|
|
1375
|
-
const exported = yield* journal.
|
|
1683
|
+
const exportThread = Effect.fn("SqliteThreadStore.export")(function* (request) {
|
|
1684
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadExportRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate thread export", error)));
|
|
1685
|
+
yield* requireThread(journal, validated.threadId);
|
|
1686
|
+
const exported = yield* journal.exportThread(validated.threadId).pipe(Effect.mapError((error) => storeError("export thread", error)));
|
|
1376
1687
|
const records = yield* Effect.forEach(exported.records, decodeEnvelope);
|
|
1377
|
-
if (records.length > 65536) return yield*
|
|
1378
|
-
operation: "decode
|
|
1379
|
-
message: "The
|
|
1688
|
+
if (records.length > 65536) return yield* ThreadStoreError.make({
|
|
1689
|
+
operation: "decode thread export",
|
|
1690
|
+
message: "The thread exceeds the current export record limit."
|
|
1380
1691
|
});
|
|
1381
|
-
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(exported.
|
|
1382
|
-
return
|
|
1383
|
-
format: "effect-agent/
|
|
1384
|
-
|
|
1385
|
-
tailSequence: exported.
|
|
1692
|
+
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(exported.thread.tail_digest).pipe(Effect.mapError((error) => schemaStoreError("decode export tail digest", error)));
|
|
1693
|
+
return ThreadExport.make({
|
|
1694
|
+
format: "effect-agent/thread@1",
|
|
1695
|
+
threadId: validated.threadId,
|
|
1696
|
+
tailSequence: exported.thread.tail_sequence,
|
|
1386
1697
|
tailDigest,
|
|
1387
1698
|
records
|
|
1388
1699
|
});
|
|
1389
1700
|
});
|
|
1390
|
-
const inspectTail = Effect.fn("
|
|
1391
|
-
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(
|
|
1392
|
-
const
|
|
1393
|
-
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(
|
|
1394
|
-
return
|
|
1395
|
-
|
|
1396
|
-
tailSequence:
|
|
1701
|
+
const inspectTail = Effect.fn("SqliteThreadStore.inspectTail")(function* (request) {
|
|
1702
|
+
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(ThreadTailRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate tail inspection", error)));
|
|
1703
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
1704
|
+
const tailDigest = yield* Schema.decodeUnknownEffect(Digest)(thread.tail_digest).pipe(Effect.mapError((error) => schemaStoreError("decode tail digest", error)));
|
|
1705
|
+
return ThreadTail.make({
|
|
1706
|
+
threadId: validated.threadId,
|
|
1707
|
+
tailSequence: thread.tail_sequence,
|
|
1397
1708
|
tailDigest,
|
|
1398
|
-
producerEpoch:
|
|
1709
|
+
producerEpoch: thread.producer_epoch
|
|
1399
1710
|
});
|
|
1400
1711
|
});
|
|
1401
|
-
const saveCheckpoint = Effect.fn("
|
|
1712
|
+
const saveCheckpoint = Effect.fn("SqliteThreadStore.saveCheckpoint")(function* (request) {
|
|
1402
1713
|
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(SaveCheckpointRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint", error)));
|
|
1403
|
-
const
|
|
1404
|
-
if (validated.checkpoint.throughSequence >
|
|
1405
|
-
|
|
1714
|
+
const thread = yield* requireThread(journal, validated.checkpoint.threadId);
|
|
1715
|
+
if (validated.checkpoint.throughSequence > thread.tail_sequence) return yield* CheckpointRejected.make({
|
|
1716
|
+
threadId: validated.checkpoint.threadId,
|
|
1406
1717
|
reason: "ahead-of-tail"
|
|
1407
1718
|
});
|
|
1408
|
-
if ((yield* tailDigestAt(journal, validated.checkpoint.
|
|
1409
|
-
|
|
1719
|
+
if ((yield* tailDigestAt(journal, validated.checkpoint.threadId, validated.checkpoint.throughSequence)) !== validated.checkpoint.tailDigest) return yield* CheckpointRejected.make({
|
|
1720
|
+
threadId: validated.checkpoint.threadId,
|
|
1410
1721
|
reason: "digest-mismatch"
|
|
1411
1722
|
});
|
|
1412
1723
|
const checkpointJson = yield* encodeCheckpoint(validated.checkpoint);
|
|
1413
1724
|
const raw = RawCheckpoint.make({
|
|
1414
|
-
|
|
1725
|
+
threadId: validated.checkpoint.threadId,
|
|
1415
1726
|
throughSequence: validated.checkpoint.throughSequence,
|
|
1416
1727
|
tailDigest: validated.checkpoint.tailDigest,
|
|
1417
1728
|
checkpointJson
|
|
1418
1729
|
});
|
|
1419
1730
|
yield* hitFailpoint("save-checkpoint:before");
|
|
1420
1731
|
yield* journal.saveCheckpoint(raw).pipe(Effect.mapError((error) => isSqliteCheckpointConflict(error) ? CheckpointRejected.make({
|
|
1421
|
-
|
|
1732
|
+
threadId: validated.checkpoint.threadId,
|
|
1422
1733
|
reason: "digest-mismatch"
|
|
1423
1734
|
}) : storeError("save checkpoint", error)));
|
|
1424
1735
|
yield* hitFailpoint("save-checkpoint:after");
|
|
1425
1736
|
});
|
|
1426
|
-
const loadCheckpoint = Effect.fn("
|
|
1737
|
+
const loadCheckpoint = Effect.fn("SqliteThreadStore.loadCheckpoint")(function* (request) {
|
|
1427
1738
|
const validated = yield* Schema.decodeUnknownEffect(Schema.toType(LoadCheckpointRequest))(request).pipe(Effect.mapError((error) => schemaStoreError("validate checkpoint lookup", error)));
|
|
1428
|
-
const
|
|
1429
|
-
const rows = yield* journal.loadCheckpoint(validated.
|
|
1739
|
+
const thread = yield* requireThread(journal, validated.threadId);
|
|
1740
|
+
const rows = yield* journal.loadCheckpoint(validated.threadId, validated.atOrBeforeSequence ?? thread.tail_sequence).pipe(Effect.mapError((error) => storeError("load checkpoint", error)));
|
|
1430
1741
|
if (rows.length === 0) return Option.none();
|
|
1431
|
-
if (rows.length !== 1) return yield*
|
|
1742
|
+
if (rows.length !== 1) return yield* ThreadStoreError.make({
|
|
1432
1743
|
operation: "load checkpoint",
|
|
1433
1744
|
message: `Expected at most one checkpoint row but found ${rows.length}.`
|
|
1434
1745
|
});
|
|
1435
1746
|
const checkpoint = yield* decodeCheckpoint(rows[0].checkpoint_json);
|
|
1436
|
-
if ((yield* tailDigestAt(journal, checkpoint.
|
|
1437
|
-
|
|
1747
|
+
if ((yield* tailDigestAt(journal, checkpoint.threadId, checkpoint.throughSequence)) !== checkpoint.tailDigest) return yield* CheckpointRejected.make({
|
|
1748
|
+
threadId: checkpoint.threadId,
|
|
1438
1749
|
reason: "digest-mismatch"
|
|
1439
1750
|
});
|
|
1440
1751
|
return Option.some(checkpoint);
|
|
1441
1752
|
});
|
|
1442
|
-
const
|
|
1753
|
+
const threadStore = ThreadStore.of({
|
|
1443
1754
|
append,
|
|
1444
|
-
export:
|
|
1755
|
+
export: exportThread,
|
|
1445
1756
|
inspectTail,
|
|
1446
|
-
loadCheckpoint,
|
|
1447
1757
|
materialize,
|
|
1448
1758
|
observe,
|
|
1449
1759
|
read,
|
|
1450
|
-
|
|
1760
|
+
checkpoints: {
|
|
1761
|
+
save: saveCheckpoint,
|
|
1762
|
+
load: loadCheckpoint
|
|
1763
|
+
}
|
|
1451
1764
|
});
|
|
1452
|
-
return Context.make(
|
|
1765
|
+
return Context.make(ThreadStore, threadStore);
|
|
1453
1766
|
});
|
|
1454
1767
|
/**
|
|
1455
|
-
* SQLite
|
|
1768
|
+
* SQLite Thread Store implementation with configuration, failpoint, SQL, and Crypto
|
|
1456
1769
|
* authority kept visible in its input channel.
|
|
1457
1770
|
*/
|
|
1458
|
-
const
|
|
1771
|
+
const threadStoreLayer = Layer.effectContext(makeServices$1());
|
|
1459
1772
|
/**
|
|
1460
1773
|
* Validated SQLite storage configuration Layer with the documented defaults applied. Shared
|
|
1461
|
-
* by the
|
|
1774
|
+
* by the ThreadStore and SubmissionLedger convenience layers so their defaults cannot
|
|
1462
1775
|
* drift.
|
|
1463
1776
|
*/
|
|
1464
1777
|
const storageConfigLayer = (options) => Layer.effect(SqliteStorageConfig)(Schema.decodeUnknownEffect(SqliteStorageConfigValue)({
|
|
@@ -1474,15 +1787,15 @@ const storageConfigLayer = (options) => Layer.effect(SqliteStorageConfig)(Schema
|
|
|
1474
1787
|
/** The failpoint Layer selected by convenience options: explicit handler or the no-op default. */
|
|
1475
1788
|
const storageFailpointLayer = (options) => options.failpoint === void 0 ? SqliteStorageFailpoint.layer : Layer.succeed(SqliteStorageFailpoint)({ hit: options.failpoint });
|
|
1476
1789
|
/**
|
|
1477
|
-
* A composition-root convenience Layer for canonical
|
|
1790
|
+
* A composition-root convenience Layer for canonical Threads. Durable accepted work is
|
|
1478
1791
|
* served by the separate SubmissionLedger port.
|
|
1479
1792
|
*/
|
|
1480
|
-
const layer = (options) => Layer.unwrap(Effect.map(SqliteStorageConfig, (config) =>
|
|
1793
|
+
const layer = (options) => Layer.unwrap(Effect.map(SqliteStorageConfig, (config) => threadStoreLayer.pipe(Layer.provide(Layer.mergeAll(Layer.succeed(SqliteStorageConfig)(config), storageFailpointLayer(options), SqliteClient.layer({ filename: options.filename }), NodeCrypto.layer))))).pipe(Layer.provide(storageConfigLayer(options)));
|
|
1481
1794
|
/** Create an adapter-owned resumable observation offset for a known canonical sequence. */
|
|
1482
1795
|
const observationOffsetAt = makeOffset;
|
|
1483
1796
|
//#endregion
|
|
1484
1797
|
//#region src/sqlite-ledger.ts
|
|
1485
|
-
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(
|
|
1798
|
+
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(16777216));
|
|
1486
1799
|
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
1487
1800
|
const BoundedTimestamp = Schema.NonEmptyString.check(Schema.isMaxLength(128));
|
|
1488
1801
|
const SCAN_PAGE_SIZE = 256;
|
|
@@ -1494,7 +1807,7 @@ const STILL_WAITING = "still-waiting";
|
|
|
1494
1807
|
const WOKEN = "woken";
|
|
1495
1808
|
var SubmissionRow = class extends Schema.Class("SubmissionRow")({
|
|
1496
1809
|
submission_id: BoundedIdentifier,
|
|
1497
|
-
|
|
1810
|
+
thread_id: BoundedIdentifier,
|
|
1498
1811
|
queue_sequence: QueueSequence,
|
|
1499
1812
|
principal: BoundedIdentifier,
|
|
1500
1813
|
idempotency_key: BoundedIdentifier,
|
|
@@ -1576,7 +1889,7 @@ var MaxQueueSequenceRow = class extends Schema.Class("MaxQueueSequenceRow")({ ma
|
|
|
1576
1889
|
var CanonicalRecordIdRow = class extends Schema.Class("CanonicalRecordIdRow")({ record_id: BoundedIdentifier }) {};
|
|
1577
1890
|
const SUBMISSION_COLUMNS = `
|
|
1578
1891
|
submission_id,
|
|
1579
|
-
|
|
1892
|
+
thread_id,
|
|
1580
1893
|
queue_sequence,
|
|
1581
1894
|
principal,
|
|
1582
1895
|
idempotency_key,
|
|
@@ -1613,7 +1926,7 @@ const CHILD_RESERVATION_COLUMNS = `
|
|
|
1613
1926
|
release_began_at,
|
|
1614
1927
|
released_at
|
|
1615
1928
|
`;
|
|
1616
|
-
/** The branded ToolCallId schema, reached through the
|
|
1929
|
+
/** The branded ToolCallId schema, reached through the thread port so no core import is needed. */
|
|
1617
1930
|
const ToolCallIdSchema = ApprovalDecisionCommand.fields.toolCallId;
|
|
1618
1931
|
const ToolCallIdList = Schema.Array(ToolCallIdSchema);
|
|
1619
1932
|
const encodePersistedJsonText = Schema.encodeEffect(Schema.fromJsonString(PersistedJson));
|
|
@@ -1691,7 +2004,7 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1691
2004
|
iso: new Date(millis).toISOString()
|
|
1692
2005
|
}));
|
|
1693
2006
|
const timestampMillis = (operation, rowKey) => (timestamp) => decodeUtcInstant(timestamp).pipe(Effect.map(DateTime.toEpochMillis), Effect.mapError((error) => corruptionFailure(operation, "effect_agent_submission_ownership", rowKey, error.message)));
|
|
1694
|
-
const decodeSubmissionRows = (operation, rowKey, rows) => decodeRows$
|
|
2007
|
+
const decodeSubmissionRows = (operation, rowKey, rows) => decodeRows$3(Schema.Array(SubmissionRow), "effect_agent_submissions", rowKey, rows).pipe(Effect.mapError(internalFailure(operation)));
|
|
1695
2008
|
const readSubmission = Effect.fn("SqliteSubmissionLedger.readSubmission")(function* (operation, submissionId) {
|
|
1696
2009
|
const rows = yield* sql`
|
|
1697
2010
|
SELECT ${sql.literal(SUBMISSION_COLUMNS)}
|
|
@@ -1722,23 +2035,23 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1722
2035
|
FROM effect_agent_submission_ownership
|
|
1723
2036
|
WHERE submission_id = ${submissionId}
|
|
1724
2037
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
1725
|
-
const decoded = yield* decodeRows$
|
|
2038
|
+
const decoded = yield* decodeRows$3(Schema.Array(OwnershipRow), "effect_agent_submission_ownership", submissionId, rows).pipe(Effect.mapError(internalFailure(operation)));
|
|
1726
2039
|
if (decoded.length > 1) return yield* corruptionFailure(operation, "effect_agent_submission_ownership", submissionId, "An ownership primary key returned more than one row.");
|
|
1727
2040
|
return decoded.length === 0 ? Option.none() : Option.some(decoded[0]);
|
|
1728
2041
|
});
|
|
1729
|
-
const
|
|
1730
|
-
const
|
|
1731
|
-
return
|
|
2042
|
+
const threadEpoch = Effect.fn("SqliteSubmissionLedger.threadEpoch")(function* (operation, threadId) {
|
|
2043
|
+
const threads = yield* journal.getThread(threadId).pipe(Effect.mapError(internalFailure(operation)));
|
|
2044
|
+
return threads.length === 0 ? EPOCH_ZERO : threads[0].producer_epoch;
|
|
1732
2045
|
});
|
|
1733
2046
|
/**
|
|
1734
2047
|
* Verify inside the surrounding write transaction that the presented token still owns the
|
|
1735
2048
|
* Submission's lane; a superseded or missing token fails with OwnershipLost carrying the
|
|
1736
|
-
*
|
|
2049
|
+
* Thread's current producer epoch (DUR-006).
|
|
1737
2050
|
*/
|
|
1738
2051
|
const requireOwnership = Effect.fn("SqliteSubmissionLedger.requireOwnership")(function* (operation, submission, ownershipToken) {
|
|
1739
2052
|
const ownership = yield* readOwnership(operation, submission.submission_id);
|
|
1740
2053
|
if (Option.isNone(ownership) || ownership.value.ownership_token !== ownershipToken) {
|
|
1741
|
-
const actualEpoch = yield*
|
|
2054
|
+
const actualEpoch = yield* threadEpoch(operation, submission.thread_id);
|
|
1742
2055
|
const submissionId = yield* Schema.decodeUnknownEffect(SubmissionSnapshot.fields.submissionId)(submission.submission_id).pipe(Effect.mapError(internalFailure(operation)));
|
|
1743
2056
|
return yield* OwnershipLost.make({
|
|
1744
2057
|
submissionId,
|
|
@@ -1753,7 +2066,7 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1753
2066
|
if (row.parent_submission_id === null !== (row.parent_tool_call_id === null)) return yield* corruptionFailure(operation, "effect_agent_submissions", row.submission_id, "A parent linkage must record both the parent Submission and the parent Tool Call.");
|
|
1754
2067
|
return yield* decodeSubmissionSnapshotUnknown({
|
|
1755
2068
|
submissionId: row.submission_id,
|
|
1756
|
-
|
|
2069
|
+
threadId: row.thread_id,
|
|
1757
2070
|
queueSequence: row.queue_sequence,
|
|
1758
2071
|
principal: row.principal,
|
|
1759
2072
|
idempotencyKey: row.idempotency_key,
|
|
@@ -1787,7 +2100,7 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1787
2100
|
FROM effect_agent_settlement_reservations
|
|
1788
2101
|
WHERE submission_id = ${submissionId}
|
|
1789
2102
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
1790
|
-
const decoded = yield* decodeRows$
|
|
2103
|
+
const decoded = yield* decodeRows$3(Schema.Array(ReservationRow), "effect_agent_settlement_reservations", submissionId, rows).pipe(Effect.mapError(internalFailure(operation)));
|
|
1791
2104
|
if (decoded.length > 1) return yield* corruptionFailure(operation, "effect_agent_settlement_reservations", submissionId, "A settlement reservation primary key returned more than one row.");
|
|
1792
2105
|
return decoded.length === 0 ? Option.none() : Option.some(decoded[0]);
|
|
1793
2106
|
});
|
|
@@ -1802,11 +2115,11 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1802
2115
|
FROM effect_agent_abort_intents
|
|
1803
2116
|
WHERE submission_id = ${submissionId}
|
|
1804
2117
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
1805
|
-
const decoded = yield* decodeRows$
|
|
2118
|
+
const decoded = yield* decodeRows$3(Schema.Array(AbortIntentRow), "effect_agent_abort_intents", submissionId, rows).pipe(Effect.mapError(internalFailure(operation)));
|
|
1806
2119
|
if (decoded.length > 1) return yield* corruptionFailure(operation, "effect_agent_abort_intents", submissionId, "An abort intent primary key returned more than one row.");
|
|
1807
2120
|
return decoded.length === 0 ? Option.none() : Option.some(decoded[0]);
|
|
1808
2121
|
});
|
|
1809
|
-
const decodeChildReservationRows = (operation, rowKey, rows) => decodeRows$
|
|
2122
|
+
const decodeChildReservationRows = (operation, rowKey, rows) => decodeRows$3(Schema.Array(ChildReservationRow), "effect_agent_child_reservations", rowKey, rows).pipe(Effect.mapError(internalFailure(operation)));
|
|
1810
2123
|
const readChildReservation = Effect.fn("SqliteSubmissionLedger.readChildReservation")(function* (operation, reservationId) {
|
|
1811
2124
|
const rows = yield* sql`
|
|
1812
2125
|
SELECT ${sql.literal(CHILD_RESERVATION_COLUMNS)}
|
|
@@ -1859,7 +2172,7 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1859
2172
|
WHERE submission_id = ${submissionId}
|
|
1860
2173
|
ORDER BY tool_call_id ASC
|
|
1861
2174
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
1862
|
-
return yield* decodeRows$
|
|
2175
|
+
return yield* decodeRows$3(Schema.Array(ApprovalDecisionRow), "effect_agent_approval_decisions", submissionId, rows).pipe(Effect.mapError(internalFailure(operation)));
|
|
1863
2176
|
});
|
|
1864
2177
|
const approvalIntentFromRow = Effect.fn("SqliteSubmissionLedger.approvalIntentFromRow")(function* (operation, row) {
|
|
1865
2178
|
return yield* decodeApprovalDecisionIntent({
|
|
@@ -1884,7 +2197,7 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1884
2197
|
WHERE submission_id = ${submissionId}
|
|
1885
2198
|
ORDER BY tool_call_id ASC
|
|
1886
2199
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
1887
|
-
return yield* decodeRows$
|
|
2200
|
+
return yield* decodeRows$3(Schema.Array(UnknownResolutionRow), "effect_agent_unknown_resolutions", submissionId, rows).pipe(Effect.mapError(internalFailure(operation)));
|
|
1888
2201
|
});
|
|
1889
2202
|
const unknownResolutionIntentFromRow = Effect.fn("SqliteSubmissionLedger.unknownResolutionIntentFromRow")(function* (operation, row) {
|
|
1890
2203
|
const resolution = yield* parseStoredJsonText(row.resolution_json).pipe(Effect.mapError((error) => corruptionFailure(operation, "effect_agent_unknown_resolutions", `${row.submission_id}/${row.tool_call_id}`, error.message)));
|
|
@@ -1907,18 +2220,18 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1907
2220
|
* derived from the shared canonical-records table using the deterministic abort record
|
|
1908
2221
|
* identity, never from a cached ledger marker.
|
|
1909
2222
|
*/
|
|
1910
|
-
const canonicalAbortRecordId = Effect.fn("SqliteSubmissionLedger.canonicalAbortRecordId")(function* (operation,
|
|
2223
|
+
const canonicalAbortRecordId = Effect.fn("SqliteSubmissionLedger.canonicalAbortRecordId")(function* (operation, threadId, submissionId) {
|
|
1911
2224
|
const recordId = submissionAbortRecordId(submissionId);
|
|
1912
2225
|
const rows = yield* sql`
|
|
1913
2226
|
SELECT record_id
|
|
1914
2227
|
FROM effect_agent_canonical_records
|
|
1915
|
-
WHERE
|
|
2228
|
+
WHERE thread_id = ${threadId}
|
|
1916
2229
|
AND record_id = ${recordId}
|
|
1917
2230
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
1918
|
-
return (yield* decodeRows$
|
|
2231
|
+
return (yield* decodeRows$3(Schema.Array(CanonicalRecordIdRow), "effect_agent_canonical_records", `${threadId}/${recordId}`, rows).pipe(Effect.mapError(internalFailure(operation)))).length === 0 ? void 0 : recordId;
|
|
1919
2232
|
});
|
|
1920
2233
|
const abortIntentFromRow = Effect.fn("SqliteSubmissionLedger.abortIntentFromRow")(function* (operation, submission, submissionId, row) {
|
|
1921
|
-
const canonicalRecordId = yield* canonicalAbortRecordId(operation, submission.
|
|
2234
|
+
const canonicalRecordId = yield* canonicalAbortRecordId(operation, submission.thread_id, submissionId);
|
|
1922
2235
|
return yield* decodeAbortIntent({
|
|
1923
2236
|
submissionId: row.submission_id,
|
|
1924
2237
|
author: row.author,
|
|
@@ -1937,11 +2250,11 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1937
2250
|
const mintedReceiptId = yield* mintIdentifier("receipt", operation);
|
|
1938
2251
|
yield* hitFailpoint("ledger:admit:before", operation);
|
|
1939
2252
|
const result = yield* inWriteTransaction(operation, Effect.gen(function* () {
|
|
1940
|
-
const keyRowKey = `${validated.
|
|
2253
|
+
const keyRowKey = `${validated.threadId}/${validated.principal}/${validated.idempotencyKey}`;
|
|
1941
2254
|
const existingRows = yield* sql`
|
|
1942
2255
|
SELECT ${sql.literal(SUBMISSION_COLUMNS)}
|
|
1943
2256
|
FROM effect_agent_submissions
|
|
1944
|
-
WHERE
|
|
2257
|
+
WHERE thread_id = ${validated.threadId}
|
|
1945
2258
|
AND principal = ${validated.principal}
|
|
1946
2259
|
AND idempotency_key = ${validated.idempotencyKey}
|
|
1947
2260
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
@@ -1950,7 +2263,7 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1950
2263
|
if (existing.length === 1) {
|
|
1951
2264
|
const sameLinkage = validated.parentLinkage === void 0 ? existing[0].parent_submission_id === null && existing[0].parent_tool_call_id === null : existing[0].parent_submission_id === validated.parentLinkage.parentSubmissionId && existing[0].parent_tool_call_id === validated.parentLinkage.parentToolCallId;
|
|
1952
2265
|
if (existing[0].input_digest !== validated.inputDigest || !sameLinkage) return yield* AdmissionConflict.make({
|
|
1953
|
-
|
|
2266
|
+
threadId: validated.threadId,
|
|
1954
2267
|
principal: validated.principal,
|
|
1955
2268
|
idempotencyKey: validated.idempotencyKey,
|
|
1956
2269
|
existingInputDigest: existing[0].input_digest,
|
|
@@ -1967,15 +2280,15 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1967
2280
|
const maxRows = yield* sql`
|
|
1968
2281
|
SELECT COALESCE(MAX(queue_sequence), 0) AS max_queue_sequence
|
|
1969
2282
|
FROM effect_agent_submissions
|
|
1970
|
-
WHERE
|
|
2283
|
+
WHERE thread_id = ${validated.threadId}
|
|
1971
2284
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
1972
|
-
const decodedMax = yield* decodeRows$
|
|
2285
|
+
const decodedMax = yield* decodeRows$3(Schema.Array(MaxQueueSequenceRow), "effect_agent_submissions", validated.threadId, maxRows).pipe(Effect.mapError(internalFailure(operation)));
|
|
1973
2286
|
const queueSequence = yield* decodeQueueSequence((decodedMax[0]?.max_queue_sequence ?? 0) + 1).pipe(Effect.mapError(internalFailure(operation)));
|
|
1974
2287
|
const now = yield* currentInstant;
|
|
1975
2288
|
yield* sql`
|
|
1976
2289
|
INSERT INTO effect_agent_submissions (
|
|
1977
2290
|
submission_id,
|
|
1978
|
-
|
|
2291
|
+
thread_id,
|
|
1979
2292
|
queue_sequence,
|
|
1980
2293
|
principal,
|
|
1981
2294
|
idempotency_key,
|
|
@@ -1991,7 +2304,7 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
1991
2304
|
parent_tool_call_id
|
|
1992
2305
|
) VALUES (
|
|
1993
2306
|
${mintedSubmissionId},
|
|
1994
|
-
${validated.
|
|
2307
|
+
${validated.threadId},
|
|
1995
2308
|
${queueSequence},
|
|
1996
2309
|
${validated.principal},
|
|
1997
2310
|
${validated.idempotencyKey},
|
|
@@ -2044,12 +2357,12 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2044
2357
|
const rows = yield* sql`
|
|
2045
2358
|
SELECT ${sql.literal(SUBMISSION_COLUMNS)}
|
|
2046
2359
|
FROM effect_agent_submissions
|
|
2047
|
-
WHERE
|
|
2360
|
+
WHERE thread_id = ${validated.threadId}
|
|
2048
2361
|
AND principal = ${validated.principal}
|
|
2049
2362
|
AND idempotency_key = ${validated.idempotencyKey}
|
|
2050
2363
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
2051
|
-
const decoded = yield* decodeSubmissionRows(operation, `${validated.
|
|
2052
|
-
if (decoded.length > 1) return yield* corruptionFailure(operation, "effect_agent_submissions", `${validated.
|
|
2364
|
+
const decoded = yield* decodeSubmissionRows(operation, `${validated.threadId}/${validated.principal}/${validated.idempotencyKey}`, rows);
|
|
2365
|
+
if (decoded.length > 1) return yield* corruptionFailure(operation, "effect_agent_submissions", `${validated.threadId}/${validated.principal}/${validated.idempotencyKey}`, "An admission idempotency key returned more than one row.");
|
|
2053
2366
|
if (decoded.length === 0) return Option.none();
|
|
2054
2367
|
return Option.some(yield* decodeSubmissionSnapshot(operation, decoded[0]));
|
|
2055
2368
|
});
|
|
@@ -2059,12 +2372,12 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2059
2372
|
const rows = yield* sql`
|
|
2060
2373
|
SELECT ${sql.literal(SUBMISSION_COLUMNS)}
|
|
2061
2374
|
FROM effect_agent_submissions
|
|
2062
|
-
WHERE
|
|
2375
|
+
WHERE thread_id = ${validated.threadId}
|
|
2063
2376
|
AND principal = ${validated.principal}
|
|
2064
2377
|
AND idempotency_key = ${validated.idempotencyKey}
|
|
2065
2378
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
2066
|
-
const decoded = yield* decodeSubmissionRows(operation, `${validated.
|
|
2067
|
-
if (decoded.length > 1) return yield* corruptionFailure(operation, "effect_agent_submissions", `${validated.
|
|
2379
|
+
const decoded = yield* decodeSubmissionRows(operation, `${validated.threadId}/${validated.principal}/${validated.idempotencyKey}`, rows);
|
|
2380
|
+
if (decoded.length > 1) return yield* corruptionFailure(operation, "effect_agent_submissions", `${validated.threadId}/${validated.principal}/${validated.idempotencyKey}`, "An admission idempotency key returned more than one row.");
|
|
2068
2381
|
if (decoded.length === 0) return AdmissionNotAdmitted.make();
|
|
2069
2382
|
return AdmissionAdmitted.make({ submission: yield* decodeSubmissionSnapshot(operation, decoded[0]) });
|
|
2070
2383
|
});
|
|
@@ -2078,12 +2391,12 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2078
2391
|
const headRows = yield* sql`
|
|
2079
2392
|
SELECT ${sql.literal(SUBMISSION_COLUMNS)}
|
|
2080
2393
|
FROM effect_agent_submissions
|
|
2081
|
-
WHERE
|
|
2394
|
+
WHERE thread_id = ${validated.threadId}
|
|
2082
2395
|
AND state <> 'settled'
|
|
2083
2396
|
ORDER BY queue_sequence ASC
|
|
2084
2397
|
LIMIT 1
|
|
2085
2398
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
2086
|
-
const heads = yield* decodeSubmissionRows(operation, validated.
|
|
2399
|
+
const heads = yield* decodeSubmissionRows(operation, validated.threadId, headRows);
|
|
2087
2400
|
if (heads.length === 0) return Option.none();
|
|
2088
2401
|
const head = heads[0];
|
|
2089
2402
|
if (head.state === "joining" || head.state === "joined" || head.state === "suspended" || head.state === "unknown" && Option.isNone(yield* readAbortIntent(operation, head.submission_id))) return Option.none();
|
|
@@ -2092,19 +2405,19 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2092
2405
|
if (Option.isSome(ownership)) {
|
|
2093
2406
|
if ((yield* timestampMillis(operation, head.submission_id)(ownership.value.lease_expires_at)) > now.millis) return Option.none();
|
|
2094
2407
|
}
|
|
2095
|
-
const
|
|
2408
|
+
const threads = yield* journal.getThread(head.thread_id).pipe(Effect.mapError(internalFailure(operation)));
|
|
2096
2409
|
let producerEpoch;
|
|
2097
|
-
if (
|
|
2410
|
+
if (threads.length === 0) {
|
|
2098
2411
|
producerEpoch = 1;
|
|
2099
2412
|
yield* sql`
|
|
2100
|
-
INSERT INTO
|
|
2101
|
-
|
|
2413
|
+
INSERT INTO effect_agent_threads (
|
|
2414
|
+
thread_id,
|
|
2102
2415
|
created_at,
|
|
2103
2416
|
tail_sequence,
|
|
2104
2417
|
tail_digest,
|
|
2105
2418
|
producer_epoch
|
|
2106
2419
|
) VALUES (
|
|
2107
|
-
${head.
|
|
2420
|
+
${head.thread_id},
|
|
2108
2421
|
${now.iso},
|
|
2109
2422
|
0,
|
|
2110
2423
|
${EMPTY_TAIL_DIGEST},
|
|
@@ -2112,11 +2425,11 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2112
2425
|
)
|
|
2113
2426
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
2114
2427
|
} else {
|
|
2115
|
-
producerEpoch =
|
|
2428
|
+
producerEpoch = threads[0].producer_epoch + 1;
|
|
2116
2429
|
yield* sql`
|
|
2117
|
-
UPDATE
|
|
2430
|
+
UPDATE effect_agent_threads
|
|
2118
2431
|
SET producer_epoch = ${producerEpoch}
|
|
2119
|
-
WHERE
|
|
2432
|
+
WHERE thread_id = ${head.thread_id}
|
|
2120
2433
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
2121
2434
|
}
|
|
2122
2435
|
const leaseExpiresAt = new Date(now.millis + config.ownershipLeaseDuration).toISOString();
|
|
@@ -2147,14 +2460,14 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2147
2460
|
INSERT INTO effect_agent_attempts (
|
|
2148
2461
|
attempt_id,
|
|
2149
2462
|
submission_id,
|
|
2150
|
-
|
|
2463
|
+
thread_id,
|
|
2151
2464
|
owner_producer_id,
|
|
2152
2465
|
producer_epoch,
|
|
2153
2466
|
claimed_at
|
|
2154
2467
|
) VALUES (
|
|
2155
2468
|
${attemptId},
|
|
2156
2469
|
${head.submission_id},
|
|
2157
|
-
${head.
|
|
2470
|
+
${head.thread_id},
|
|
2158
2471
|
${validated.producerId},
|
|
2159
2472
|
${producerEpoch},
|
|
2160
2473
|
${now.iso}
|
|
@@ -2336,7 +2649,8 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2336
2649
|
submissionId: validated.submissionId,
|
|
2337
2650
|
existingOutcome: reservation.value.outcome
|
|
2338
2651
|
});
|
|
2339
|
-
const
|
|
2652
|
+
const reservationRecord = yield* decodeRecordEnvelopeText(reservation.value.record_json).pipe(Effect.mapError((error) => corruptionFailure(operation, "effect_agent_settlement_reservations", validated.submissionId, error.message)));
|
|
2653
|
+
const settlementFailure = settlementFailureFromRecord(reservationRecord);
|
|
2340
2654
|
if (reservation.value.outcome === "failed" !== (settlementFailure !== void 0)) return yield* corruptionFailure(operation, "effect_agent_settlement_reservations", validated.submissionId, "The reserved outcome and canonical failure diagnostic disagree.");
|
|
2341
2655
|
const submission = yield* requireSubmission(operation, validated.submissionId);
|
|
2342
2656
|
if (submission.state === "settled") {
|
|
@@ -2414,7 +2728,7 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2414
2728
|
${now.iso}
|
|
2415
2729
|
)
|
|
2416
2730
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
2417
|
-
const canonicalRecordId = yield* canonicalAbortRecordId(operation, submission.
|
|
2731
|
+
const canonicalRecordId = yield* canonicalAbortRecordId(operation, submission.thread_id, validated.submissionId);
|
|
2418
2732
|
return yield* decodeAbortIntent({
|
|
2419
2733
|
submissionId: validated.submissionId,
|
|
2420
2734
|
author: validated.author,
|
|
@@ -2432,19 +2746,19 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2432
2746
|
yield* hitFailpoint("ledger:claim-joining:before", operation);
|
|
2433
2747
|
const claims = yield* inWriteTransaction(operation, Effect.gen(function* () {
|
|
2434
2748
|
const host = yield* requireSubmission(operation, validated.hostSubmissionId);
|
|
2435
|
-
if (host.
|
|
2749
|
+
if (host.thread_id !== validated.threadId) return yield* LedgerError.make({
|
|
2436
2750
|
operation,
|
|
2437
|
-
message: `Host submission ${validated.hostSubmissionId} does not belong to
|
|
2751
|
+
message: `Host submission ${validated.hostSubmissionId} does not belong to thread ${validated.threadId}.`
|
|
2438
2752
|
});
|
|
2439
2753
|
yield* requireOwnership(operation, host, validated.ownershipToken);
|
|
2440
2754
|
const laterRows = yield* sql`
|
|
2441
2755
|
SELECT ${sql.literal(SUBMISSION_COLUMNS)}
|
|
2442
2756
|
FROM effect_agent_submissions
|
|
2443
|
-
WHERE
|
|
2757
|
+
WHERE thread_id = ${validated.threadId}
|
|
2444
2758
|
AND queue_sequence > ${host.queue_sequence}
|
|
2445
2759
|
ORDER BY queue_sequence ASC
|
|
2446
2760
|
`.pipe(Effect.mapError(sqlFailure(operation)));
|
|
2447
|
-
const later = yield* decodeSubmissionRows(operation, validated.
|
|
2761
|
+
const later = yield* decodeSubmissionRows(operation, validated.threadId, laterRows);
|
|
2448
2762
|
const claimed = [];
|
|
2449
2763
|
for (const row of later) {
|
|
2450
2764
|
if (claimed.length >= validated.maxCount) break;
|
|
@@ -2947,27 +3261,27 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
2947
3261
|
SELECT ${sql.literal(SUBMISSION_COLUMNS)}
|
|
2948
3262
|
FROM effect_agent_submissions
|
|
2949
3263
|
WHERE state <> 'settled'
|
|
2950
|
-
ORDER BY
|
|
3264
|
+
ORDER BY thread_id ASC, queue_sequence ASC
|
|
2951
3265
|
LIMIT ${SCAN_PAGE_SIZE}
|
|
2952
3266
|
` : sql`
|
|
2953
3267
|
SELECT ${sql.literal(SUBMISSION_COLUMNS)}
|
|
2954
3268
|
FROM effect_agent_submissions
|
|
2955
3269
|
WHERE state <> 'settled'
|
|
2956
3270
|
AND (
|
|
2957
|
-
|
|
3271
|
+
thread_id > ${cursor.threadId}
|
|
2958
3272
|
OR (
|
|
2959
|
-
|
|
3273
|
+
thread_id = ${cursor.threadId}
|
|
2960
3274
|
AND queue_sequence > ${cursor.queueSequence}
|
|
2961
3275
|
)
|
|
2962
3276
|
)
|
|
2963
|
-
ORDER BY
|
|
3277
|
+
ORDER BY thread_id ASC, queue_sequence ASC
|
|
2964
3278
|
LIMIT ${SCAN_PAGE_SIZE}
|
|
2965
3279
|
`).pipe(Effect.mapError(sqlFailure(operation)));
|
|
2966
3280
|
const decoded = yield* decodeSubmissionRows(operation, "nonterminal_scan", rows);
|
|
2967
3281
|
const snapshots = yield* Effect.forEach(decoded, (row) => decodeSubmissionSnapshot(operation, row));
|
|
2968
3282
|
const last = decoded[decoded.length - 1];
|
|
2969
3283
|
return [snapshots, last === void 0 || decoded.length < SCAN_PAGE_SIZE ? Option.none() : Option.some({
|
|
2970
|
-
|
|
3284
|
+
threadId: last.thread_id,
|
|
2971
3285
|
queueSequence: last.queue_sequence
|
|
2972
3286
|
})];
|
|
2973
3287
|
});
|
|
@@ -3112,12 +3426,312 @@ const makeServices = Effect.fn("SqliteSubmissionLedger.makeServices")(function*
|
|
|
3112
3426
|
const submissionLedgerLayer = Layer.effectContext(makeServices());
|
|
3113
3427
|
/**
|
|
3114
3428
|
* A composition-root convenience Layer for the durable Submission Ledger. Point it at the
|
|
3115
|
-
* same database file as the
|
|
3429
|
+
* same database file as the ThreadStore so claims fence the same producer epochs.
|
|
3116
3430
|
*/
|
|
3117
3431
|
const ledgerLayer = (options) => Layer.unwrap(Effect.map(SqliteStorageConfig, (config) => submissionLedgerLayer.pipe(Layer.provide(Layer.mergeAll(Layer.succeed(SqliteStorageConfig)(config), storageFailpointLayer(options), SqliteClient.layer({ filename: options.filename }), NodeCrypto.layer))))).pipe(Layer.provide(storageConfigLayer(options)));
|
|
3118
3432
|
//#endregion
|
|
3433
|
+
//#region src/sqlite-memory-store.ts
|
|
3434
|
+
const STORAGE_VERSION = 2;
|
|
3435
|
+
const METADATA_COMPONENT = "memory";
|
|
3436
|
+
const DOCUMENT_TABLE = "effect_agent_memory_documents_v1";
|
|
3437
|
+
const RECEIPT_TABLE = "effect_agent_memory_receipts_v1";
|
|
3438
|
+
const StoredJson$1 = Schema.String.check(Schema.isMaxLength(16777216));
|
|
3439
|
+
const EncodedMemoryChange = Schema.Struct({
|
|
3440
|
+
commandJson: StoredJson$1,
|
|
3441
|
+
documentJson: StoredJson$1,
|
|
3442
|
+
resultJson: StoredJson$1
|
|
3443
|
+
});
|
|
3444
|
+
const equivalentContent = Schema.toEquivalence(MemoryWrite.Wire.members[0].fields.content);
|
|
3445
|
+
const equivalentScopes = Schema.toEquivalence(MemoryWrite.Wire.members[0].fields.scopes);
|
|
3446
|
+
var MemoryMetadataRow = class extends Schema.Class("@effect-agent/storage-sqlite/MemoryMetadataRow")({ version: Schema.Int }) {};
|
|
3447
|
+
var MemoryTableRow = class extends Schema.Class("@effect-agent/storage-sqlite/MemoryTableRow")({ name: Schema.NonEmptyString }) {};
|
|
3448
|
+
var MemoryDocumentRow = class extends Schema.Class("@effect-agent/storage-sqlite/MemoryDocumentRow")({
|
|
3449
|
+
namespace: MemoryNamespaceAddress,
|
|
3450
|
+
source_id: MemoryKey.Wire.fields.id,
|
|
3451
|
+
format_version: Schema.Int,
|
|
3452
|
+
generation: Schema.Int,
|
|
3453
|
+
revision: Schema.NonEmptyString,
|
|
3454
|
+
document_json: StoredJson$1
|
|
3455
|
+
}) {};
|
|
3456
|
+
var MemoryReceiptRow = class extends Schema.Class("@effect-agent/storage-sqlite/MemoryReceiptRow")({
|
|
3457
|
+
namespace: MemoryNamespaceAddress,
|
|
3458
|
+
operation_id: MemoryWrite.Wire.members[0].fields.operationId,
|
|
3459
|
+
source_id: MemoryKey.Wire.fields.id,
|
|
3460
|
+
format_version: Schema.Int,
|
|
3461
|
+
command_json: StoredJson$1,
|
|
3462
|
+
result_json: StoredJson$1
|
|
3463
|
+
}) {};
|
|
3464
|
+
var MemoryChangeCountRow = class extends Schema.Class("@effect-agent/storage-sqlite/MemoryChangeCountRow")({ changed: Schema.Int }) {};
|
|
3465
|
+
var StoredMemoryCommand = class extends Schema.Class("@effect-agent/storage-sqlite/StoredMemoryCommand")({
|
|
3466
|
+
version: Schema.Literal(STORAGE_VERSION),
|
|
3467
|
+
value: MemoryWrite.Wire
|
|
3468
|
+
}) {};
|
|
3469
|
+
var StoredMemoryResult = class extends Schema.Class("@effect-agent/storage-sqlite/StoredMemoryResult")({
|
|
3470
|
+
version: Schema.Literal(STORAGE_VERSION),
|
|
3471
|
+
value: MemoryDocument.Wire
|
|
3472
|
+
}) {};
|
|
3473
|
+
const StoredVersionHeader = Schema.Struct({ version: Schema.Int });
|
|
3474
|
+
const storageError = (operation, reason = "unavailable") => MemoryStorageError.make({
|
|
3475
|
+
operation,
|
|
3476
|
+
reason
|
|
3477
|
+
});
|
|
3478
|
+
const query = (effect, operation) => effect.pipe(Effect.mapError(() => storageError(operation)));
|
|
3479
|
+
const decodeRows$2 = Effect.fn("SqliteMemoryStore.decodeRows")(function* (schema, rows, operation) {
|
|
3480
|
+
return yield* Schema.decodeUnknownEffect(Schema.Array(schema))(rows).pipe(Effect.mapError(() => storageError(operation, "corrupt")));
|
|
3481
|
+
});
|
|
3482
|
+
const decodeInput$1 = Effect.fn("SqliteMemoryStore.decodeInput")(function* (schema, value, operation) {
|
|
3483
|
+
return yield* Schema.decodeUnknownEffect(schema)(value).pipe(Effect.mapError(() => storageError(operation, "invalid-input")));
|
|
3484
|
+
});
|
|
3485
|
+
const encodeJson = Effect.fn("SqliteMemoryStore.encodeJson")(function* (schema, value, operation) {
|
|
3486
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(schema))(value).pipe(Effect.mapError(() => storageError(operation, "corrupt")));
|
|
3487
|
+
});
|
|
3488
|
+
const validateEncodedChange = Effect.fn("SqliteMemoryStore.validateEncodedChange")(function* (encoded, operation) {
|
|
3489
|
+
yield* Schema.decodeEffect(EncodedMemoryChange)(encoded).pipe(Effect.mapError(() => storageError(operation, "invalid-input")));
|
|
3490
|
+
});
|
|
3491
|
+
const decodeVersionedJson = Effect.fn("SqliteMemoryStore.decodeVersionedJson")(function* (schema, value, operation) {
|
|
3492
|
+
if ((yield* Schema.decodeEffect(Schema.fromJsonString(StoredVersionHeader))(value).pipe(Effect.mapError(() => storageError(operation, "corrupt")))).version !== STORAGE_VERSION) return yield* storageError(operation, "incompatible");
|
|
3493
|
+
const decoded = yield* Schema.decodeEffect(Schema.fromJsonString(schema))(value).pipe(Effect.mapError(() => storageError(operation, "corrupt")));
|
|
3494
|
+
if ((yield* encodeJson(schema, decoded, operation)) !== value) return yield* storageError(operation, "corrupt");
|
|
3495
|
+
return decoded;
|
|
3496
|
+
});
|
|
3497
|
+
const validateDocument = Effect.fn("SqliteMemoryStore.validateDocument")(function* (document, key, operation) {
|
|
3498
|
+
if (document.key.namespace.address !== key.namespace.address || document.key.id !== key.id || document.source.id !== key.id || document.source.revision !== String(document.generation) || document.generation === 1 !== (document.predecessor === null) || document.predecessor !== null && (document.predecessor.id !== key.id || document.predecessor.revision !== String(document.generation - 1))) return yield* storageError(operation, "corrupt");
|
|
3499
|
+
return document;
|
|
3500
|
+
});
|
|
3501
|
+
const validateReceiptResult = Effect.fn("SqliteMemoryStore.validateReceiptResult")(function* (command, result, operation) {
|
|
3502
|
+
if ((result.predecessor?.revision ?? null) !== command.expectedRevision) return yield* storageError(operation, "corrupt");
|
|
3503
|
+
if (command._tag === "Put") {
|
|
3504
|
+
if (result._tag !== "ActiveMemoryDocument" || result.source.locator !== command.locator) return yield* storageError(operation, "corrupt");
|
|
3505
|
+
if (!equivalentContent(command.content, result.content) || !equivalentScopes(command.scopes, result.scopes)) return yield* storageError(operation, "corrupt");
|
|
3506
|
+
return;
|
|
3507
|
+
}
|
|
3508
|
+
if (result._tag !== "WithdrawnMemoryDocument" || result.reason !== command.reason || result.predecessor === null || result.source.locator !== result.predecessor.locator) return yield* storageError(operation, "corrupt");
|
|
3509
|
+
});
|
|
3510
|
+
const initializeMemorySchema = Effect.fn("SqliteMemoryStore.initialize")(function* () {
|
|
3511
|
+
const sql = yield* SqlClient.SqlClient;
|
|
3512
|
+
const failpoint = yield* MemoryMutationFailpoint;
|
|
3513
|
+
yield* failpoint.hit("memory:initialize:before");
|
|
3514
|
+
yield* sql.withTransaction(Effect.gen(function* () {
|
|
3515
|
+
yield* sql`
|
|
3516
|
+
CREATE TABLE IF NOT EXISTS effect_agent_memory_metadata (
|
|
3517
|
+
component TEXT PRIMARY KEY NOT NULL,
|
|
3518
|
+
version INTEGER NOT NULL
|
|
3519
|
+
)
|
|
3520
|
+
`;
|
|
3521
|
+
const metadataRows = yield* sql`
|
|
3522
|
+
SELECT version
|
|
3523
|
+
FROM effect_agent_memory_metadata
|
|
3524
|
+
WHERE component = ${METADATA_COMPONENT}
|
|
3525
|
+
`;
|
|
3526
|
+
const metadata = yield* decodeRows$2(MemoryMetadataRow, metadataRows, "decode memory schema version");
|
|
3527
|
+
if (metadata.length > 1) return yield* storageError("decode memory schema version", "corrupt");
|
|
3528
|
+
const currentVersion = metadata[0]?.version;
|
|
3529
|
+
if (currentVersion !== void 0 && currentVersion !== STORAGE_VERSION) return yield* storageError("initialize memory schema", "incompatible");
|
|
3530
|
+
if (currentVersion === void 0) {
|
|
3531
|
+
const tableRows = yield* sql`
|
|
3532
|
+
SELECT name
|
|
3533
|
+
FROM sqlite_master
|
|
3534
|
+
WHERE type = 'table' AND name IN (${DOCUMENT_TABLE}, ${RECEIPT_TABLE})
|
|
3535
|
+
`;
|
|
3536
|
+
if ((yield* decodeRows$2(MemoryTableRow, tableRows, "inspect memory schema")).length > 0) return yield* storageError("initialize memory schema", "incompatible");
|
|
3537
|
+
yield* sql`
|
|
3538
|
+
CREATE TABLE effect_agent_memory_documents_v1 (
|
|
3539
|
+
namespace TEXT NOT NULL,
|
|
3540
|
+
source_id TEXT NOT NULL,
|
|
3541
|
+
format_version INTEGER NOT NULL,
|
|
3542
|
+
generation INTEGER NOT NULL,
|
|
3543
|
+
revision TEXT NOT NULL,
|
|
3544
|
+
document_json TEXT NOT NULL,
|
|
3545
|
+
PRIMARY KEY (namespace, source_id)
|
|
3546
|
+
)
|
|
3547
|
+
`;
|
|
3548
|
+
yield* sql`
|
|
3549
|
+
CREATE TABLE effect_agent_memory_receipts_v1 (
|
|
3550
|
+
namespace TEXT NOT NULL,
|
|
3551
|
+
operation_id TEXT NOT NULL,
|
|
3552
|
+
source_id TEXT NOT NULL,
|
|
3553
|
+
format_version INTEGER NOT NULL,
|
|
3554
|
+
command_json TEXT NOT NULL,
|
|
3555
|
+
result_json TEXT NOT NULL,
|
|
3556
|
+
PRIMARY KEY (namespace, operation_id)
|
|
3557
|
+
)
|
|
3558
|
+
`;
|
|
3559
|
+
yield* sql`
|
|
3560
|
+
INSERT INTO effect_agent_memory_metadata (component, version)
|
|
3561
|
+
VALUES (${METADATA_COMPONENT}, ${STORAGE_VERSION})
|
|
3562
|
+
`;
|
|
3563
|
+
}
|
|
3564
|
+
yield* sql`
|
|
3565
|
+
SELECT namespace, source_id, format_version, generation, revision, document_json
|
|
3566
|
+
FROM effect_agent_memory_documents_v1
|
|
3567
|
+
LIMIT 0
|
|
3568
|
+
`;
|
|
3569
|
+
yield* sql`
|
|
3570
|
+
SELECT namespace, operation_id, source_id, format_version, command_json, result_json
|
|
3571
|
+
FROM effect_agent_memory_receipts_v1
|
|
3572
|
+
LIMIT 0
|
|
3573
|
+
`;
|
|
3574
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(storageError("initialize memory schema"))));
|
|
3575
|
+
yield* failpoint.hit("memory:initialize:after");
|
|
3576
|
+
});
|
|
3577
|
+
const makeMemoryReader = Effect.fn("SqliteMemoryStore.makeReader")(function* () {
|
|
3578
|
+
const sql = yield* SqlClient.SqlClient;
|
|
3579
|
+
const readDocument = Effect.fn("SqliteMemoryStore.readDocument")(function* (key, operation) {
|
|
3580
|
+
const rawRows = yield* query(sql`
|
|
3581
|
+
SELECT namespace, source_id, format_version, generation, revision, document_json
|
|
3582
|
+
FROM effect_agent_memory_documents_v1
|
|
3583
|
+
WHERE namespace = ${key.namespace.address} AND source_id = ${key.id}
|
|
3584
|
+
`, operation);
|
|
3585
|
+
const rows = yield* decodeRows$2(MemoryDocumentRow, rawRows, operation);
|
|
3586
|
+
if (rows.length === 0) return null;
|
|
3587
|
+
if (rows.length !== 1) return yield* storageError(operation, "corrupt");
|
|
3588
|
+
const row = rows[0];
|
|
3589
|
+
if (row.format_version !== STORAGE_VERSION) return yield* storageError(operation, "incompatible");
|
|
3590
|
+
const document = (yield* decodeVersionedJson(StoredMemoryResult, row.document_json, operation)).value;
|
|
3591
|
+
yield* validateDocument(document, key, operation);
|
|
3592
|
+
if (row.namespace !== key.namespace.address || row.source_id !== key.id || row.generation !== document.generation || row.revision !== document.source.revision) return yield* storageError(operation, "corrupt");
|
|
3593
|
+
return document;
|
|
3594
|
+
});
|
|
3595
|
+
return {
|
|
3596
|
+
get: Effect.fn("SqliteMemoryStore.get")(function* (key) {
|
|
3597
|
+
const decodedKey = yield* decodeInput$1(MemoryKey.Wire, key, "get memory document");
|
|
3598
|
+
return yield* readDocument(decodedKey, "get memory document");
|
|
3599
|
+
}),
|
|
3600
|
+
readDocument
|
|
3601
|
+
};
|
|
3602
|
+
});
|
|
3603
|
+
const makeMemoryServices = Effect.fn("SqliteMemoryStore.make")(function* () {
|
|
3604
|
+
const sql = yield* SqlClient.SqlClient;
|
|
3605
|
+
const failpoint = yield* MemoryMutationFailpoint;
|
|
3606
|
+
yield* initializeMemorySchema();
|
|
3607
|
+
const { get, readDocument } = yield* makeMemoryReader();
|
|
3608
|
+
const readReceipt = Effect.fn("SqliteMemoryStore.readReceipt")(function* (write, operation) {
|
|
3609
|
+
const rawRows = yield* query(sql`
|
|
3610
|
+
SELECT namespace, operation_id, source_id, format_version, command_json, result_json
|
|
3611
|
+
FROM effect_agent_memory_receipts_v1
|
|
3612
|
+
WHERE namespace = ${write.key.namespace.address} AND operation_id = ${write.operationId}
|
|
3613
|
+
`, operation);
|
|
3614
|
+
const rows = yield* decodeRows$2(MemoryReceiptRow, rawRows, operation);
|
|
3615
|
+
if (rows.length === 0) return null;
|
|
3616
|
+
if (rows.length !== 1) return yield* storageError(operation, "corrupt");
|
|
3617
|
+
const row = rows[0];
|
|
3618
|
+
if (row.format_version !== STORAGE_VERSION) return yield* storageError(operation, "incompatible");
|
|
3619
|
+
const command = yield* decodeVersionedJson(StoredMemoryCommand, row.command_json, `${operation} command`);
|
|
3620
|
+
const result = yield* decodeVersionedJson(StoredMemoryResult, row.result_json, `${operation} result`);
|
|
3621
|
+
if (row.namespace !== command.value.key.namespace.address || row.operation_id !== command.value.operationId || row.source_id !== command.value.key.id || result.value.key.namespace.address !== command.value.key.namespace.address || result.value.key.id !== command.value.key.id) return yield* storageError(operation, "corrupt");
|
|
3622
|
+
yield* validateDocument(result.value, command.value.key, operation);
|
|
3623
|
+
yield* validateReceiptResult(command.value, result.value, operation);
|
|
3624
|
+
return {
|
|
3625
|
+
commandJson: row.command_json,
|
|
3626
|
+
result: result.value
|
|
3627
|
+
};
|
|
3628
|
+
});
|
|
3629
|
+
const change = Effect.fn("SqliteMemoryStore.change")(function* (write) {
|
|
3630
|
+
const operation = "change memory document";
|
|
3631
|
+
const decodedWrite = yield* decodeInput$1(MemoryWrite.Wire, write, operation);
|
|
3632
|
+
const commandJson = yield* encodeJson(StoredMemoryCommand, StoredMemoryCommand.make({
|
|
3633
|
+
version: STORAGE_VERSION,
|
|
3634
|
+
value: decodedWrite
|
|
3635
|
+
}), "encode memory command");
|
|
3636
|
+
yield* failpoint.hit("memory:change:before");
|
|
3637
|
+
const transactionResult = yield* sql.withTransaction(Effect.gen(function* () {
|
|
3638
|
+
const receipt = yield* readReceipt(decodedWrite, operation);
|
|
3639
|
+
if (receipt !== null) {
|
|
3640
|
+
if (receipt.commandJson !== commandJson) return yield* MemoryOperationConflict.make({
|
|
3641
|
+
key: decodedWrite.key,
|
|
3642
|
+
operationId: decodedWrite.operationId
|
|
3643
|
+
});
|
|
3644
|
+
return {
|
|
3645
|
+
document: receipt.result,
|
|
3646
|
+
changed: false
|
|
3647
|
+
};
|
|
3648
|
+
}
|
|
3649
|
+
const current = yield* readDocument(decodedWrite.key, operation);
|
|
3650
|
+
const modifiedAt = yield* Clock.currentTimeMillis;
|
|
3651
|
+
const next = yield* applyMemoryWrite(current, decodedWrite, modifiedAt);
|
|
3652
|
+
const resultJson = yield* encodeJson(StoredMemoryResult, StoredMemoryResult.make({
|
|
3653
|
+
version: STORAGE_VERSION,
|
|
3654
|
+
value: next
|
|
3655
|
+
}), "encode memory result");
|
|
3656
|
+
const documentJson = resultJson;
|
|
3657
|
+
yield* validateEncodedChange({
|
|
3658
|
+
commandJson,
|
|
3659
|
+
documentJson,
|
|
3660
|
+
resultJson
|
|
3661
|
+
}, operation);
|
|
3662
|
+
if (current === null) yield* sql`
|
|
3663
|
+
INSERT INTO effect_agent_memory_documents_v1 (
|
|
3664
|
+
namespace, source_id, format_version, generation, revision, document_json
|
|
3665
|
+
) VALUES (
|
|
3666
|
+
${next.key.namespace.address}, ${next.key.id}, ${STORAGE_VERSION},
|
|
3667
|
+
${next.generation}, ${next.source.revision}, ${documentJson}
|
|
3668
|
+
)
|
|
3669
|
+
`;
|
|
3670
|
+
else yield* sql`
|
|
3671
|
+
UPDATE effect_agent_memory_documents_v1
|
|
3672
|
+
SET format_version = ${STORAGE_VERSION},
|
|
3673
|
+
generation = ${next.generation},
|
|
3674
|
+
revision = ${next.source.revision},
|
|
3675
|
+
document_json = ${documentJson}
|
|
3676
|
+
WHERE namespace = ${next.key.namespace.address}
|
|
3677
|
+
AND source_id = ${next.key.id}
|
|
3678
|
+
AND generation = ${current.generation}
|
|
3679
|
+
AND revision = ${current.source.revision}
|
|
3680
|
+
`;
|
|
3681
|
+
const changedRows = yield* sql`
|
|
3682
|
+
SELECT changes() AS changed
|
|
3683
|
+
`;
|
|
3684
|
+
const changed = yield* decodeRows$2(MemoryChangeCountRow, changedRows, operation);
|
|
3685
|
+
if (changed.length !== 1 || changed[0].changed !== 1) return yield* storageError(operation, "corrupt");
|
|
3686
|
+
yield* failpoint.hit("memory:change:after-state");
|
|
3687
|
+
yield* sql`
|
|
3688
|
+
INSERT INTO effect_agent_memory_receipts_v1 (
|
|
3689
|
+
namespace, operation_id, source_id, format_version, command_json, result_json
|
|
3690
|
+
) VALUES (
|
|
3691
|
+
${decodedWrite.key.namespace.address}, ${decodedWrite.operationId}, ${decodedWrite.key.id},
|
|
3692
|
+
${STORAGE_VERSION}, ${commandJson}, ${resultJson}
|
|
3693
|
+
)
|
|
3694
|
+
`;
|
|
3695
|
+
yield* failpoint.hit("memory:change:after-receipt");
|
|
3696
|
+
return {
|
|
3697
|
+
document: next,
|
|
3698
|
+
changed: true
|
|
3699
|
+
};
|
|
3700
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(storageError(operation))));
|
|
3701
|
+
if (transactionResult.changed) yield* failpoint.hit("memory:change:after");
|
|
3702
|
+
return transactionResult.document;
|
|
3703
|
+
});
|
|
3704
|
+
return Context.make(MemoryReader, MemoryReader.fromAdapter({ get })).pipe(Context.add(MemoryWriter, MemoryWriter.fromAdapter({ change })));
|
|
3705
|
+
});
|
|
3706
|
+
/** SQLite memory ports with the mutation failpoint kept injectable for recovery tests. */
|
|
3707
|
+
const memoryStoreLayerWithFailpoints = Layer.effectContext(makeMemoryServices());
|
|
3708
|
+
/** SQLite memory reader and writer with the production no-op mutation failpoint. */
|
|
3709
|
+
const memoryStoreLayer = memoryStoreLayerWithFailpoints.pipe(Layer.provide(MemoryMutationFailpoint.layer));
|
|
3710
|
+
/** Reads an existing memory schema without writes, transactions, or mutation failpoints. */
|
|
3711
|
+
const memoryReaderLayer = Layer.effect(MemoryReader, Effect.gen(function* () {
|
|
3712
|
+
const sql = yield* SqlClient.SqlClient;
|
|
3713
|
+
const operation = "open memory reader";
|
|
3714
|
+
const tables = yield* query(sql`
|
|
3715
|
+
SELECT name FROM sqlite_master
|
|
3716
|
+
WHERE type = 'table' AND name IN (
|
|
3717
|
+
'effect_agent_memory_metadata', ${DOCUMENT_TABLE}, ${RECEIPT_TABLE}
|
|
3718
|
+
)
|
|
3719
|
+
`, operation).pipe(Effect.flatMap((rows) => decodeRows$2(MemoryTableRow, rows, operation)));
|
|
3720
|
+
if (tables.length !== 3) return yield* storageError(operation, tables.length === 0 ? "unavailable" : "incompatible");
|
|
3721
|
+
const metadata = yield* query(sql`
|
|
3722
|
+
SELECT version FROM effect_agent_memory_metadata WHERE component = ${METADATA_COMPONENT}
|
|
3723
|
+
`, operation).pipe(Effect.flatMap((rows) => decodeRows$2(MemoryMetadataRow, rows, operation)));
|
|
3724
|
+
if (metadata.length !== 1 || metadata[0].version !== STORAGE_VERSION) return yield* storageError(operation, "incompatible");
|
|
3725
|
+
yield* query(sql`
|
|
3726
|
+
SELECT namespace, source_id, format_version, generation, revision, document_json
|
|
3727
|
+
FROM effect_agent_memory_documents_v1 LIMIT 0
|
|
3728
|
+
`, operation);
|
|
3729
|
+
const { get } = yield* makeMemoryReader();
|
|
3730
|
+
return MemoryReader.fromAdapter({ get });
|
|
3731
|
+
}));
|
|
3732
|
+
//#endregion
|
|
3119
3733
|
//#region src/sqlite-schedule-store.ts
|
|
3120
|
-
const StoredScheduleJson = Schema.String.check(Schema.isMaxLength(
|
|
3734
|
+
const StoredScheduleJson = Schema.String.check(Schema.isMaxLength(16777216));
|
|
3121
3735
|
const StoredDeadline = Schema.NullOr(ScheduleInstant);
|
|
3122
3736
|
var ScheduleRow = class extends Schema.Class("@effect-agent/storage-sqlite/ScheduleRow")({
|
|
3123
3737
|
tenant_id: ScheduleOwner.fields.tenantId,
|
|
@@ -3134,27 +3748,27 @@ const ScheduleDueRow = Schema.Struct({
|
|
|
3134
3748
|
});
|
|
3135
3749
|
var ScheduleCountRow = class extends Schema.Class("@effect-agent/storage-sqlite/ScheduleCountRow")({ schedule_count: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) }) {};
|
|
3136
3750
|
var ScheduleDeadlineRow = class extends Schema.Class("@effect-agent/storage-sqlite/ScheduleDeadlineRow")({ deadline_at_millis: StoredDeadline }) {};
|
|
3137
|
-
const unavailable = (operation) => ScheduleStorageError.make({
|
|
3751
|
+
const unavailable$1 = (operation) => ScheduleStorageError.make({
|
|
3138
3752
|
operation,
|
|
3139
3753
|
reason: "unavailable"
|
|
3140
3754
|
});
|
|
3141
|
-
const corrupt = (operation) => ScheduleStorageError.make({
|
|
3755
|
+
const corrupt$1 = (operation) => ScheduleStorageError.make({
|
|
3142
3756
|
operation,
|
|
3143
3757
|
reason: "corrupt"
|
|
3144
3758
|
});
|
|
3145
|
-
const decodeRows = Effect.fn("SqliteScheduleStore.decodeRows")(function* (schema, rows, operation) {
|
|
3146
|
-
return yield* Schema.decodeUnknownEffect(schema)(rows).pipe(Effect.mapError(() => corrupt(operation)));
|
|
3759
|
+
const decodeRows$1 = Effect.fn("SqliteScheduleStore.decodeRows")(function* (schema, rows, operation) {
|
|
3760
|
+
return yield* Schema.decodeUnknownEffect(schema)(rows).pipe(Effect.mapError(() => corrupt$1(operation)));
|
|
3147
3761
|
});
|
|
3148
3762
|
const decodeRecord = Effect.fn("SqliteScheduleStore.decodeRecord")(function* (row) {
|
|
3149
|
-
const record = yield* Schema.decodeEffect(Schema.fromJsonString(ScheduleRecord))(row.record_json).pipe(Effect.mapError(() => corrupt("decode schedule")));
|
|
3150
|
-
if (record.owner.tenantId !== row.tenant_id || record.owner.ownerId !== row.owner_id || record.scheduleId !== row.schedule_id || scheduleDeadline(record) !== row.deadline_at_millis) return yield* corrupt("decode schedule identity");
|
|
3763
|
+
const record = yield* Schema.decodeEffect(Schema.fromJsonString(ScheduleRecord))(row.record_json).pipe(Effect.mapError(() => corrupt$1("decode schedule")));
|
|
3764
|
+
if (record.owner.tenantId !== row.tenant_id || record.owner.ownerId !== row.owner_id || record.scheduleId !== row.schedule_id || scheduleDeadline(record) !== row.deadline_at_millis) return yield* corrupt$1("decode schedule identity");
|
|
3151
3765
|
return record;
|
|
3152
3766
|
});
|
|
3153
3767
|
const encodeRecord = Effect.fn("SqliteScheduleStore.encodeRecord")(function* (record) {
|
|
3154
|
-
return yield* Schema.encodeEffect(Schema.fromJsonString(ScheduleRecord))(record).pipe(Effect.mapError(() => corrupt("encode schedule")));
|
|
3768
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(ScheduleRecord))(record).pipe(Effect.mapError(() => corrupt$1("encode schedule")));
|
|
3155
3769
|
});
|
|
3156
3770
|
const decodeInput = Effect.fn("SqliteScheduleStore.decodeInput")(function* (operation, schema, value) {
|
|
3157
|
-
return yield* Schema.decodeUnknownEffect(schema)(value).pipe(Effect.mapError(() => corrupt(operation)));
|
|
3771
|
+
return yield* Schema.decodeUnknownEffect(schema)(value).pipe(Effect.mapError(() => corrupt$1(operation)));
|
|
3158
3772
|
});
|
|
3159
3773
|
const makeScheduleStore = Effect.gen(function* () {
|
|
3160
3774
|
const sql = yield* SqlClient.SqlClient;
|
|
@@ -3167,13 +3781,13 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3167
3781
|
WHERE tenant_id = ${key.owner.tenantId}
|
|
3168
3782
|
AND owner_id = ${key.owner.ownerId}
|
|
3169
3783
|
AND schedule_id = ${key.scheduleId}
|
|
3170
|
-
`.pipe(Effect.mapError(() => unavailable(operation)));
|
|
3171
|
-
return yield* decodeRows(Schema.Array(ScheduleRow), rows, operation);
|
|
3784
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation)));
|
|
3785
|
+
return yield* decodeRows$1(Schema.Array(ScheduleRow), rows, operation);
|
|
3172
3786
|
});
|
|
3173
3787
|
const readOne = Effect.fn("SqliteScheduleStore.readOne")(function* (key, operation) {
|
|
3174
3788
|
const rows = yield* readRows(key, operation);
|
|
3175
3789
|
if (rows.length === 0) return null;
|
|
3176
|
-
if (rows.length !== 1) return yield* corrupt(operation);
|
|
3790
|
+
if (rows.length !== 1) return yield* corrupt$1(operation);
|
|
3177
3791
|
return yield* decodeRecord(rows[0]);
|
|
3178
3792
|
});
|
|
3179
3793
|
const insert = Effect.fn("SqliteScheduleStore.insert")(function* (record, ownerLimit) {
|
|
@@ -3202,9 +3816,9 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3202
3816
|
AND owner_id = ${canonical.owner.ownerId}
|
|
3203
3817
|
AND (json_extract(record_json, '$.pending') IS NOT NULL OR
|
|
3204
3818
|
(json_extract(record_json, '$.state') != 'cancelled' AND json_extract(record_json, '$.nextAtMillis') IS NOT NULL))
|
|
3205
|
-
`.pipe(Effect.mapError(() => unavailable(operation)));
|
|
3206
|
-
const counts = yield* decodeRows(Schema.Array(ScheduleCountRow), rawCounts, operation);
|
|
3207
|
-
if (counts.length !== 1) return yield* corrupt(operation);
|
|
3819
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation)));
|
|
3820
|
+
const counts = yield* decodeRows$1(Schema.Array(ScheduleCountRow), rawCounts, operation);
|
|
3821
|
+
if (counts.length !== 1) return yield* corrupt$1(operation);
|
|
3208
3822
|
if (counts[0].schedule_count >= ownerLimit) return yield* ScheduleCapacityError.make({ limit: ownerLimit });
|
|
3209
3823
|
yield* scheduleFailpoint.hit("schedule:insert:before");
|
|
3210
3824
|
yield* sql`
|
|
@@ -3217,12 +3831,12 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3217
3831
|
${scheduleDeadline(canonical)},
|
|
3218
3832
|
${recordJson}
|
|
3219
3833
|
)
|
|
3220
|
-
`.pipe(Effect.mapError(() => unavailable(operation)));
|
|
3834
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation)));
|
|
3221
3835
|
return {
|
|
3222
3836
|
record: canonical,
|
|
3223
3837
|
inserted: true
|
|
3224
3838
|
};
|
|
3225
|
-
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(unavailable(operation))));
|
|
3839
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(unavailable$1(operation))));
|
|
3226
3840
|
if (result.inserted) yield* scheduleFailpoint.hit("schedule:insert:after");
|
|
3227
3841
|
return result.record;
|
|
3228
3842
|
});
|
|
@@ -3240,7 +3854,7 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3240
3854
|
AND owner_id = ${decodedRequest.owner.ownerId}
|
|
3241
3855
|
ORDER BY schedule_id
|
|
3242
3856
|
LIMIT ${decodedRequest.limit + 1}
|
|
3243
|
-
`.pipe(Effect.mapError(() => unavailable(operation))) : yield* sql`
|
|
3857
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation))) : yield* sql`
|
|
3244
3858
|
SELECT tenant_id, owner_id, schedule_id, deadline_at_millis, record_json
|
|
3245
3859
|
FROM effect_agent_schedules
|
|
3246
3860
|
WHERE tenant_id = ${decodedRequest.owner.tenantId}
|
|
@@ -3248,8 +3862,8 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3248
3862
|
AND schedule_id > ${decodedRequest.after}
|
|
3249
3863
|
ORDER BY schedule_id
|
|
3250
3864
|
LIMIT ${decodedRequest.limit + 1}
|
|
3251
|
-
`.pipe(Effect.mapError(() => unavailable(operation)));
|
|
3252
|
-
const decoded = yield* decodeRows(Schema.Array(ScheduleRow), rows, operation);
|
|
3865
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation)));
|
|
3866
|
+
const decoded = yield* decodeRows$1(Schema.Array(ScheduleRow), rows, operation);
|
|
3253
3867
|
const records = yield* Effect.forEach(decoded, decodeRecord);
|
|
3254
3868
|
const hasNext = records.length > decodedRequest.limit;
|
|
3255
3869
|
const items = hasNext ? records.slice(0, decodedRequest.limit) : records;
|
|
@@ -3274,9 +3888,9 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3274
3888
|
WHERE tenant_id = ${key.owner.tenantId} AND owner_id = ${key.owner.ownerId}
|
|
3275
3889
|
AND (json_extract(record_json, '$.pending') IS NOT NULL OR
|
|
3276
3890
|
(json_extract(record_json, '$.state') != 'cancelled' AND json_extract(record_json, '$.nextAtMillis') IS NOT NULL))
|
|
3277
|
-
`.pipe(Effect.mapError(() => unavailable(operation)));
|
|
3278
|
-
const counts = yield* decodeRows(Schema.Array(ScheduleCountRow), rawCounts, operation);
|
|
3279
|
-
if (counts.length !== 1) return yield* corrupt(operation);
|
|
3891
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation)));
|
|
3892
|
+
const counts = yield* decodeRows$1(Schema.Array(ScheduleCountRow), rawCounts, operation);
|
|
3893
|
+
if (counts.length !== 1) return yield* corrupt$1(operation);
|
|
3280
3894
|
if (counts[0].schedule_count >= ownerLimit) return yield* ScheduleCapacityError.make({ limit: ownerLimit });
|
|
3281
3895
|
}
|
|
3282
3896
|
if (next === current) return {
|
|
@@ -3291,19 +3905,19 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3291
3905
|
WHERE tenant_id = ${decodedKey.owner.tenantId}
|
|
3292
3906
|
AND owner_id = ${decodedKey.owner.ownerId}
|
|
3293
3907
|
AND schedule_id = ${decodedKey.scheduleId}
|
|
3294
|
-
`.pipe(Effect.mapError(() => unavailable(operation)));
|
|
3908
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation)));
|
|
3295
3909
|
return {
|
|
3296
3910
|
record: next,
|
|
3297
3911
|
changed: true
|
|
3298
3912
|
};
|
|
3299
|
-
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(unavailable(operation))));
|
|
3913
|
+
})).pipe(Effect.catchTag("SqlError", () => Effect.fail(unavailable$1(operation))));
|
|
3300
3914
|
if (result.changed) yield* scheduleFailpoint.hit(`schedule:${decodedChange._tag.toLowerCase()}:after`);
|
|
3301
3915
|
return result.record;
|
|
3302
3916
|
});
|
|
3303
3917
|
const due = Effect.fn("SqliteScheduleStore.due")(function* (nowMillis, limit, owner, after) {
|
|
3304
3918
|
const operation = "query due schedules";
|
|
3305
3919
|
const decodedOwner = owner === void 0 ? void 0 : yield* decodeInput(operation, ScheduleOwner, owner);
|
|
3306
|
-
const cursor = after === void 0 ? void 0 : yield* Schema.decodeUnknownEffect(ScheduleDueCursor)(after).pipe(Effect.mapError(() => corrupt(operation)));
|
|
3920
|
+
const cursor = after === void 0 ? void 0 : yield* Schema.decodeUnknownEffect(ScheduleDueCursor)(after).pipe(Effect.mapError(() => corrupt$1(operation)));
|
|
3307
3921
|
const continuation = cursor === void 0 ? sql`1 = 1` : sql`
|
|
3308
3922
|
(deadline_at_millis, tenant_id, owner_id, schedule_id) >
|
|
3309
3923
|
(${cursor.deadlineAtMillis}, ${cursor.owner.tenantId}, ${cursor.owner.ownerId}, ${cursor.scheduleId})`;
|
|
@@ -3313,7 +3927,7 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3313
3927
|
WHERE deadline_at_millis <= ${nowMillis} AND ${continuation}
|
|
3314
3928
|
ORDER BY deadline_at_millis, tenant_id, owner_id, schedule_id
|
|
3315
3929
|
LIMIT ${limit}
|
|
3316
|
-
`.pipe(Effect.mapError(() => unavailable(operation))) : yield* sql`
|
|
3930
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation))) : yield* sql`
|
|
3317
3931
|
SELECT tenant_id, owner_id, schedule_id, deadline_at_millis
|
|
3318
3932
|
FROM effect_agent_schedules
|
|
3319
3933
|
WHERE tenant_id = ${decodedOwner.tenantId}
|
|
@@ -3321,8 +3935,8 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3321
3935
|
AND deadline_at_millis <= ${nowMillis} AND ${continuation}
|
|
3322
3936
|
ORDER BY deadline_at_millis, schedule_id
|
|
3323
3937
|
LIMIT ${limit}
|
|
3324
|
-
`.pipe(Effect.mapError(() => unavailable(operation)));
|
|
3325
|
-
return (yield* decodeRows(Schema.Array(ScheduleDueRow), rows, operation)).map((row) => ({
|
|
3938
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation)));
|
|
3939
|
+
return (yield* decodeRows$1(Schema.Array(ScheduleDueRow), rows, operation)).map((row) => ({
|
|
3326
3940
|
owner: {
|
|
3327
3941
|
tenantId: row.tenant_id,
|
|
3328
3942
|
ownerId: row.owner_id
|
|
@@ -3338,15 +3952,15 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3338
3952
|
SELECT MIN(deadline_at_millis) AS deadline_at_millis
|
|
3339
3953
|
FROM effect_agent_schedules
|
|
3340
3954
|
WHERE deadline_at_millis IS NOT NULL
|
|
3341
|
-
`.pipe(Effect.mapError(() => unavailable(operation))) : yield* sql`
|
|
3955
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation))) : yield* sql`
|
|
3342
3956
|
SELECT MIN(deadline_at_millis) AS deadline_at_millis
|
|
3343
3957
|
FROM effect_agent_schedules
|
|
3344
3958
|
WHERE tenant_id = ${decodedOwner.tenantId}
|
|
3345
3959
|
AND owner_id = ${decodedOwner.ownerId}
|
|
3346
3960
|
AND deadline_at_millis IS NOT NULL
|
|
3347
|
-
`.pipe(Effect.mapError(() => unavailable(operation)));
|
|
3348
|
-
const decoded = yield* decodeRows(Schema.Array(ScheduleDeadlineRow), rows, operation);
|
|
3349
|
-
if (decoded.length !== 1) return yield* corrupt(operation);
|
|
3961
|
+
`.pipe(Effect.mapError(() => unavailable$1(operation)));
|
|
3962
|
+
const decoded = yield* decodeRows$1(Schema.Array(ScheduleDeadlineRow), rows, operation);
|
|
3963
|
+
if (decoded.length !== 1) return yield* corrupt$1(operation);
|
|
3350
3964
|
return decoded[0].deadline_at_millis;
|
|
3351
3965
|
});
|
|
3352
3966
|
return ScheduleStore.of({
|
|
@@ -3361,6 +3975,591 @@ const makeScheduleStore = Effect.gen(function* () {
|
|
|
3361
3975
|
/** SQLite implementation of the atomic ScheduleStore port. */
|
|
3362
3976
|
const scheduleStoreLayer = Layer.effect(ScheduleStore)(makeScheduleStore);
|
|
3363
3977
|
//#endregion
|
|
3364
|
-
|
|
3978
|
+
//#region src/sqlite-subscription-store.ts
|
|
3979
|
+
const StoredJson = Schema.String.check(Schema.isMaxLength(16777216));
|
|
3980
|
+
const CountRow = Schema.Struct({ count: Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)) });
|
|
3981
|
+
const SequenceRow = Schema.Struct({ sequence: Schema.Natural });
|
|
3982
|
+
const ScanRow = Schema.Struct({
|
|
3983
|
+
event_scan_cursor: Schema.String,
|
|
3984
|
+
delivery_scan_cursor: Schema.String,
|
|
3985
|
+
recovery_scan_cursor: Schema.Natural
|
|
3986
|
+
});
|
|
3987
|
+
const JsonRow = Schema.Struct({ record_json: StoredJson });
|
|
3988
|
+
const RegistrationRow = Schema.Struct({
|
|
3989
|
+
owner_id: Schema.String,
|
|
3990
|
+
subscription_id: Schema.String,
|
|
3991
|
+
ordinal: Schema.Natural,
|
|
3992
|
+
source_name: Schema.String,
|
|
3993
|
+
source_version: Schema.String,
|
|
3994
|
+
matching_key: Schema.String,
|
|
3995
|
+
state: SubscriptionRecord.fields.state,
|
|
3996
|
+
expires_at_millis: Schema.Number,
|
|
3997
|
+
recovery_at_millis: Schema.NullOr(Schema.Number),
|
|
3998
|
+
record_json: StoredJson
|
|
3999
|
+
});
|
|
4000
|
+
const EventRow = Schema.Struct({
|
|
4001
|
+
event_id: Schema.String,
|
|
4002
|
+
source_name: Schema.String,
|
|
4003
|
+
source_version: Schema.String,
|
|
4004
|
+
matching_key: Schema.String,
|
|
4005
|
+
payload_digest: Digest,
|
|
4006
|
+
cutoff: Schema.Natural,
|
|
4007
|
+
cursor: Schema.Natural,
|
|
4008
|
+
routing_complete: Schema.Number,
|
|
4009
|
+
next_attempt_at_millis: Schema.Number,
|
|
4010
|
+
record_json: StoredJson
|
|
4011
|
+
});
|
|
4012
|
+
const DeliveryRow = Schema.Struct({
|
|
4013
|
+
owner_id: Schema.String,
|
|
4014
|
+
subscription_id: Schema.String,
|
|
4015
|
+
event_id: Schema.String,
|
|
4016
|
+
delivery_key: Schema.String,
|
|
4017
|
+
state: SubscriptionDelivery.fields.state,
|
|
4018
|
+
next_attempt_at_millis: Schema.Number,
|
|
4019
|
+
record_json: StoredJson
|
|
4020
|
+
});
|
|
4021
|
+
const error = (reason, code) => SubscriptionError.make({
|
|
4022
|
+
reason,
|
|
4023
|
+
code
|
|
4024
|
+
});
|
|
4025
|
+
const unavailable = (operation) => error("storage", operation);
|
|
4026
|
+
const corrupt = (operation) => error("corrupt", operation);
|
|
4027
|
+
const bytes = (value) => new TextEncoder().encode(JSON.stringify(value)).byteLength;
|
|
4028
|
+
const validate = (schema, value, code) => Schema.decodeUnknownEffect(schema)(value).pipe(Effect.mapError(() => error("validation", code)));
|
|
4029
|
+
const encode = (schema, value, code) => Schema.encodeEffect(Schema.fromJsonString(schema))(value).pipe(Effect.mapError(() => corrupt(code)));
|
|
4030
|
+
const decode = (schema, value, code) => Schema.decodeEffect(Schema.fromJsonString(schema))(value).pipe(Effect.mapError(() => corrupt(code)));
|
|
4031
|
+
const decodeRows = (schema, rows, code) => Schema.decodeUnknownEffect(Schema.Array(schema))(rows).pipe(Effect.mapError(() => corrupt(code)));
|
|
4032
|
+
const sameDeliveryIdentity = (left, right) => subscriptionDeliveryKeyString(left.key) === subscriptionDeliveryKeyString(right.key) && left.deliveryId === right.deliveryId && left.source.name === right.source.name && left.source.version === right.source.version && left.threadId === right.threadId && left.admissionKey === right.admissionKey && left.subscriptionFingerprint === right.subscriptionFingerprint && left.eventDigest === right.eventDigest;
|
|
4033
|
+
const makeSubscriptionStore = Effect.fn("SqliteSubscriptionStore.make")(function* (owned) {
|
|
4034
|
+
const partition = yield* validate(SourcePartition, owned, "partition");
|
|
4035
|
+
const sql = yield* SqlClient.SqlClient;
|
|
4036
|
+
const failpoint = yield* SubscriptionFailpoint;
|
|
4037
|
+
yield* initializeSqliteJournal();
|
|
4038
|
+
yield* sql`
|
|
4039
|
+
INSERT INTO effect_agent_subscription_sequences (
|
|
4040
|
+
tenant_id, source_address, sequence, event_scan_cursor, delivery_scan_cursor, recovery_scan_cursor
|
|
4041
|
+
) VALUES (${partition.tenantId}, ${partition.address}, 0, '', '', 0) ON CONFLICT DO NOTHING
|
|
4042
|
+
`.pipe(Effect.mapError(() => unavailable("initialize subscription partition")));
|
|
4043
|
+
const transact = (effect) => sql.withTransaction(effect).pipe(Effect.catchTag("SqlError", () => Effect.fail(unavailable("transaction"))));
|
|
4044
|
+
const query = (effect, code) => effect.pipe(Effect.mapError(() => unavailable(code)));
|
|
4045
|
+
const requirePartition = (candidate, code) => sameSourcePartition(candidate, partition) ? Effect.void : Effect.fail(error("validation", code));
|
|
4046
|
+
const requireKey = Effect.fn("SqliteSubscriptionStore.requireKey")(function* (input, code) {
|
|
4047
|
+
const key = yield* validate(SubscriptionKey, input, code);
|
|
4048
|
+
yield* requirePartition(key.partition, code);
|
|
4049
|
+
return key;
|
|
4050
|
+
});
|
|
4051
|
+
const readRegistration = Effect.fn("SqliteSubscriptionStore.readRegistration")(function* (key, code) {
|
|
4052
|
+
const rows = yield* query(sql`
|
|
4053
|
+
SELECT owner_id, subscription_id, ordinal, source_name, source_version, matching_key, state,
|
|
4054
|
+
expires_at_millis, recovery_at_millis, record_json FROM effect_agent_subscriptions
|
|
4055
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4056
|
+
AND owner_id=${key.ownerId} AND subscription_id=${key.subscriptionId}
|
|
4057
|
+
`, code);
|
|
4058
|
+
const decodedRows = yield* decodeRows(RegistrationRow, rows, code);
|
|
4059
|
+
if (decodedRows.length > 1) return yield* corrupt(code);
|
|
4060
|
+
const row = decodedRows[0];
|
|
4061
|
+
if (row === void 0) return null;
|
|
4062
|
+
const record = yield* decode(SubscriptionRecord, row.record_json, code);
|
|
4063
|
+
if (!sameSourcePartition(record.key.partition, partition) || record.key.ownerId !== row.owner_id || record.key.subscriptionId !== row.subscription_id || record.ordinal !== row.ordinal || record.configuration.source.name !== row.source_name || record.configuration.source.version !== row.source_version || record.configuration.matchingKey !== row.matching_key || record.state !== row.state || record.configuration.expiresAtMillis !== row.expires_at_millis || (record.recovery?.nextAttemptAtMillis ?? null) !== row.recovery_at_millis) return yield* corrupt(`${code}-projection`);
|
|
4064
|
+
return record;
|
|
4065
|
+
});
|
|
4066
|
+
const readEvent = Effect.fn("SqliteSubscriptionStore.readEvent")(function* (eventId, code) {
|
|
4067
|
+
const rows = yield* query(sql`
|
|
4068
|
+
SELECT event_id, source_name, source_version, matching_key, payload_digest, cutoff, cursor,
|
|
4069
|
+
routing_complete, next_attempt_at_millis, record_json FROM effect_agent_subscription_events
|
|
4070
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND event_id=${eventId}
|
|
4071
|
+
`, code);
|
|
4072
|
+
const decodedRows = yield* decodeRows(EventRow, rows, code);
|
|
4073
|
+
if (decodedRows.length > 1) return yield* corrupt(code);
|
|
4074
|
+
const row = decodedRows[0];
|
|
4075
|
+
if (row === void 0) return null;
|
|
4076
|
+
const event = yield* decode(AcceptedEvent, row.record_json, code);
|
|
4077
|
+
if (!sameSourcePartition(event.partition, partition) || event.eventId !== row.event_id || event.source.name !== row.source_name || event.source.version !== row.source_version || event.matchingKey !== row.matching_key || event.payloadDigest !== row.payload_digest || event.cutoff !== row.cutoff || event.cursor !== row.cursor || (event.routingComplete ? 1 : 0) !== row.routing_complete || event.nextAttemptAtMillis !== row.next_attempt_at_millis) return yield* corrupt(`${code}-projection`);
|
|
4078
|
+
return event;
|
|
4079
|
+
});
|
|
4080
|
+
const readDelivery = Effect.fn("SqliteSubscriptionStore.readDelivery")(function* (key, code) {
|
|
4081
|
+
const rows = yield* query(sql`
|
|
4082
|
+
SELECT owner_id, subscription_id, event_id, delivery_key, state, next_attempt_at_millis, record_json
|
|
4083
|
+
FROM effect_agent_subscription_deliveries
|
|
4084
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4085
|
+
AND owner_id=${key.subscription.ownerId} AND subscription_id=${key.subscription.subscriptionId}
|
|
4086
|
+
AND event_id=${key.eventId}
|
|
4087
|
+
`, code);
|
|
4088
|
+
const decodedRows = yield* decodeRows(DeliveryRow, rows, code);
|
|
4089
|
+
if (decodedRows.length > 1) return yield* corrupt(code);
|
|
4090
|
+
const row = decodedRows[0];
|
|
4091
|
+
if (row === void 0) return null;
|
|
4092
|
+
const delivery = yield* decode(SubscriptionDelivery, row.record_json, code);
|
|
4093
|
+
if (!sameSourcePartition(delivery.key.subscription.partition, partition) || delivery.key.subscription.ownerId !== row.owner_id || delivery.key.subscription.subscriptionId !== row.subscription_id || delivery.key.eventId !== row.event_id || subscriptionDeliveryKeyString(delivery.key) !== row.delivery_key || delivery.state !== row.state || delivery.retry.nextAttemptAtMillis !== row.next_attempt_at_millis) return yield* corrupt(`${code}-projection`);
|
|
4094
|
+
return delivery;
|
|
4095
|
+
});
|
|
4096
|
+
const count = Effect.fn("SqliteSubscriptionStore.count")(function* (statement, code) {
|
|
4097
|
+
const rows = yield* query(statement, code);
|
|
4098
|
+
const decoded = yield* decodeRows(CountRow, rows, code);
|
|
4099
|
+
if (decoded.length !== 1) return yield* corrupt(code);
|
|
4100
|
+
return decoded[0].count;
|
|
4101
|
+
});
|
|
4102
|
+
const nextSequence = Effect.fn("SqliteSubscriptionStore.nextSequence")(function* () {
|
|
4103
|
+
const rows = yield* query(sql`
|
|
4104
|
+
UPDATE effect_agent_subscription_sequences SET sequence=sequence+1
|
|
4105
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4106
|
+
RETURNING sequence
|
|
4107
|
+
`, "advance subscription sequence");
|
|
4108
|
+
const decoded = yield* decodeRows(SequenceRow, rows, "advance subscription sequence");
|
|
4109
|
+
if (decoded.length !== 1) return yield* corrupt("subscription sequence");
|
|
4110
|
+
return decoded[0].sequence;
|
|
4111
|
+
});
|
|
4112
|
+
const writeRegistration = Effect.fn("SqliteSubscriptionStore.writeRegistration")(function* (record) {
|
|
4113
|
+
const json = yield* encode(SubscriptionRecord, record, "encode subscription");
|
|
4114
|
+
yield* query(sql`
|
|
4115
|
+
UPDATE effect_agent_subscriptions SET state=${record.state}, expires_at_millis=${record.configuration.expiresAtMillis},
|
|
4116
|
+
recovery_at_millis=${record.recovery?.nextAttemptAtMillis ?? null}, record_json=${json}
|
|
4117
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4118
|
+
AND owner_id=${record.key.ownerId} AND subscription_id=${record.key.subscriptionId}
|
|
4119
|
+
`, "write subscription");
|
|
4120
|
+
});
|
|
4121
|
+
const writeEvent = Effect.fn("SqliteSubscriptionStore.writeEvent")(function* (event) {
|
|
4122
|
+
const json = yield* encode(AcceptedEvent, event, "encode event");
|
|
4123
|
+
yield* query(sql`
|
|
4124
|
+
UPDATE effect_agent_subscription_events SET cursor=${event.cursor}, routing_complete=${event.routingComplete ? 1 : 0},
|
|
4125
|
+
next_attempt_at_millis=${event.nextAttemptAtMillis}, record_json=${json}
|
|
4126
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND event_id=${event.eventId}
|
|
4127
|
+
`, "write event");
|
|
4128
|
+
});
|
|
4129
|
+
const writeDelivery = Effect.fn("SqliteSubscriptionStore.writeDelivery")(function* (delivery) {
|
|
4130
|
+
const json = yield* encode(SubscriptionDelivery, delivery, "encode delivery");
|
|
4131
|
+
yield* query(sql`
|
|
4132
|
+
UPDATE effect_agent_subscription_deliveries SET state=${delivery.state},
|
|
4133
|
+
next_attempt_at_millis=${delivery.retry.nextAttemptAtMillis}, record_json=${json}
|
|
4134
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4135
|
+
AND owner_id=${delivery.key.subscription.ownerId} AND subscription_id=${delivery.key.subscription.subscriptionId}
|
|
4136
|
+
AND event_id=${delivery.key.eventId}
|
|
4137
|
+
`, "write delivery");
|
|
4138
|
+
});
|
|
4139
|
+
const register = Effect.fn("SqliteSubscriptionStore.register")(function* (input, inputLimits) {
|
|
4140
|
+
const record = yield* validate(SubscriptionRecord, input, "register-record");
|
|
4141
|
+
const limits = yield* validate(SubscriptionLimits, inputLimits, "register-limits");
|
|
4142
|
+
yield* requirePartition(record.key.partition, "register-partition");
|
|
4143
|
+
const result = yield* transact(Effect.gen(function* () {
|
|
4144
|
+
const existing = yield* readRegistration(record.key, "register-existing");
|
|
4145
|
+
if (existing !== null) {
|
|
4146
|
+
if (existing.creationFingerprint !== record.creationFingerprint) return yield* error("conflict", "registration-identity");
|
|
4147
|
+
return {
|
|
4148
|
+
value: existing,
|
|
4149
|
+
changed: false
|
|
4150
|
+
};
|
|
4151
|
+
}
|
|
4152
|
+
if (bytes(record.configuration.context) > limits.maxContextBytes) return yield* error("capacity", "context-bytes");
|
|
4153
|
+
if (bytes(record.configuration.parameters) > limits.maxPayloadBytes) return yield* error("capacity", "parameters-bytes");
|
|
4154
|
+
if (record.configuration.expiresAtMillis - record.createdAtMillis > limits.maxLifetimeMillis) return yield* error("capacity", "lifetime");
|
|
4155
|
+
if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`, "count registrations")) >= limits.maxRegistrations) return yield* error("capacity", "registrations");
|
|
4156
|
+
if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND owner_id=${record.key.ownerId}`, "count owner registrations")) >= limits.maxRegistrationsPerOwner) return yield* error("capacity", "owner-registrations");
|
|
4157
|
+
const assigned = {
|
|
4158
|
+
...record,
|
|
4159
|
+
ordinal: yield* nextSequence()
|
|
4160
|
+
};
|
|
4161
|
+
const json = yield* encode(SubscriptionRecord, assigned, "encode registration");
|
|
4162
|
+
yield* failpoint.hit("subscription:register:before");
|
|
4163
|
+
yield* query(sql`
|
|
4164
|
+
INSERT INTO effect_agent_subscriptions (tenant_id, source_address, owner_id, subscription_id, ordinal,
|
|
4165
|
+
source_name, source_version, matching_key, state, expires_at_millis, recovery_at_millis, record_json)
|
|
4166
|
+
VALUES (${partition.tenantId}, ${partition.address}, ${assigned.key.ownerId}, ${assigned.key.subscriptionId}, ${assigned.ordinal},
|
|
4167
|
+
${assigned.configuration.source.name}, ${assigned.configuration.source.version}, ${assigned.configuration.matchingKey}, ${assigned.state},
|
|
4168
|
+
${assigned.configuration.expiresAtMillis}, ${assigned.recovery?.nextAttemptAtMillis ?? null}, ${json})
|
|
4169
|
+
`, "insert registration");
|
|
4170
|
+
return {
|
|
4171
|
+
value: assigned,
|
|
4172
|
+
changed: true
|
|
4173
|
+
};
|
|
4174
|
+
}));
|
|
4175
|
+
if (result.changed) yield* failpoint.hit("subscription:register:after");
|
|
4176
|
+
return result.value;
|
|
4177
|
+
});
|
|
4178
|
+
const get = Effect.fn("SqliteSubscriptionStore.get")(function* (input) {
|
|
4179
|
+
return yield* readRegistration(yield* requireKey(input, "get-key"), "get subscription");
|
|
4180
|
+
});
|
|
4181
|
+
const list = Effect.fn("SqliteSubscriptionStore.list")(function* (ownerId, after, limit) {
|
|
4182
|
+
const rows = yield* query(sql`
|
|
4183
|
+
SELECT owner_id, subscription_id, ordinal FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4184
|
+
AND owner_id=${ownerId} AND ordinal>${after} ORDER BY ordinal LIMIT ${limit}
|
|
4185
|
+
`, "list subscriptions");
|
|
4186
|
+
const decoded = yield* decodeRows(Schema.Struct({
|
|
4187
|
+
owner_id: Schema.String,
|
|
4188
|
+
subscription_id: Schema.String,
|
|
4189
|
+
ordinal: Schema.Natural
|
|
4190
|
+
}), rows, "list subscriptions");
|
|
4191
|
+
return yield* Effect.forEach(decoded, Effect.fn("SqliteSubscriptionStore.listRecord")(function* (row) {
|
|
4192
|
+
const record = yield* readRegistration({
|
|
4193
|
+
partition,
|
|
4194
|
+
ownerId: row.owner_id,
|
|
4195
|
+
subscriptionId: row.subscription_id
|
|
4196
|
+
}, "list subscription");
|
|
4197
|
+
if (record === null || record.ordinal !== row.ordinal) return yield* corrupt("list subscription projection");
|
|
4198
|
+
return record;
|
|
4199
|
+
}));
|
|
4200
|
+
});
|
|
4201
|
+
const cancel = Effect.fn("SqliteSubscriptionStore.cancel")(function* (input) {
|
|
4202
|
+
const key = yield* requireKey(input, "cancel-key");
|
|
4203
|
+
const result = yield* transact(Effect.gen(function* () {
|
|
4204
|
+
const current = yield* readRegistration(key, "cancel subscription");
|
|
4205
|
+
if (current === null) return yield* error("not-found", "subscription");
|
|
4206
|
+
if (current.state === "cancelled") return {
|
|
4207
|
+
value: current,
|
|
4208
|
+
changed: false
|
|
4209
|
+
};
|
|
4210
|
+
const updated = {
|
|
4211
|
+
...current,
|
|
4212
|
+
state: "cancelled",
|
|
4213
|
+
recovery: null
|
|
4214
|
+
};
|
|
4215
|
+
yield* failpoint.hit("subscription:cancel:before");
|
|
4216
|
+
yield* writeRegistration(updated);
|
|
4217
|
+
return {
|
|
4218
|
+
value: updated,
|
|
4219
|
+
changed: true
|
|
4220
|
+
};
|
|
4221
|
+
}));
|
|
4222
|
+
if (result.changed) yield* failpoint.hit("subscription:cancel:after");
|
|
4223
|
+
return result.value;
|
|
4224
|
+
});
|
|
4225
|
+
const accept = Effect.fn("SqliteSubscriptionStore.accept")(function* (input, inputLimits) {
|
|
4226
|
+
const event = yield* validate(AcceptedEvent, input, "accept-event");
|
|
4227
|
+
const limits = yield* validate(SubscriptionLimits, inputLimits, "accept-limits");
|
|
4228
|
+
yield* requirePartition(event.partition, "accept-partition");
|
|
4229
|
+
const result = yield* transact(Effect.gen(function* () {
|
|
4230
|
+
const existing = yield* readEvent(event.eventId, "accept event");
|
|
4231
|
+
if (existing !== null) {
|
|
4232
|
+
if (!sameAcceptedEventIdentity(existing, event)) return yield* error("conflict", "event-identity");
|
|
4233
|
+
return {
|
|
4234
|
+
value: existing,
|
|
4235
|
+
changed: false
|
|
4236
|
+
};
|
|
4237
|
+
}
|
|
4238
|
+
if (bytes(event.payload) > limits.maxPayloadBytes) return yield* error("capacity", "payload-bytes");
|
|
4239
|
+
if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_events WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`, "count events")) >= limits.maxEvents) return yield* error("capacity", "events");
|
|
4240
|
+
const accepted = {
|
|
4241
|
+
...event,
|
|
4242
|
+
cutoff: yield* nextSequence(),
|
|
4243
|
+
cursor: 0,
|
|
4244
|
+
routingComplete: false,
|
|
4245
|
+
routingFailure: null
|
|
4246
|
+
};
|
|
4247
|
+
const json = yield* encode(AcceptedEvent, accepted, "encode accepted event");
|
|
4248
|
+
yield* failpoint.hit("subscription:accept:before");
|
|
4249
|
+
yield* query(sql`
|
|
4250
|
+
INSERT INTO effect_agent_subscription_events (tenant_id, source_address, event_id, source_name, source_version,
|
|
4251
|
+
matching_key, payload_digest, cutoff, cursor, routing_complete, next_attempt_at_millis, record_json)
|
|
4252
|
+
VALUES (${partition.tenantId}, ${partition.address}, ${accepted.eventId}, ${accepted.source.name}, ${accepted.source.version},
|
|
4253
|
+
${accepted.matchingKey}, ${accepted.payloadDigest}, ${accepted.cutoff}, ${accepted.cursor}, 0, ${accepted.nextAttemptAtMillis}, ${json})
|
|
4254
|
+
`, "insert event");
|
|
4255
|
+
return {
|
|
4256
|
+
value: accepted,
|
|
4257
|
+
changed: true
|
|
4258
|
+
};
|
|
4259
|
+
}));
|
|
4260
|
+
if (result.changed) yield* failpoint.hit("subscription:accept:after");
|
|
4261
|
+
return result.value;
|
|
4262
|
+
});
|
|
4263
|
+
const event = Effect.fn("SqliteSubscriptionStore.event")((eventId) => readEvent(eventId, "get event"));
|
|
4264
|
+
const pendingEvents = Effect.fn("SqliteSubscriptionStore.pendingEvents")(function* (nowMillis, after, limit) {
|
|
4265
|
+
const rows = yield* query(sql`
|
|
4266
|
+
SELECT event_id FROM effect_agent_subscription_events WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4267
|
+
AND routing_complete=0 AND next_attempt_at_millis<=${nowMillis} AND event_id>${after} ORDER BY event_id LIMIT ${limit}
|
|
4268
|
+
`, "pending events");
|
|
4269
|
+
return yield* decodeRows(Schema.Struct({ event_id: Schema.String }), rows, "pending event keys").pipe(Effect.map((items) => items.map((item) => item.event_id)));
|
|
4270
|
+
});
|
|
4271
|
+
const candidates = Effect.fn("SqliteSubscriptionStore.candidates")(function* (input, limit) {
|
|
4272
|
+
const supplied = yield* validate(AcceptedEvent, input, "candidates-event");
|
|
4273
|
+
yield* requirePartition(supplied.partition, "candidates-partition");
|
|
4274
|
+
const stored = yield* readEvent(supplied.eventId, "candidates event");
|
|
4275
|
+
if (stored === null) return yield* error("not-found", "event");
|
|
4276
|
+
if (!sameAcceptedEventIdentity(stored, supplied)) return yield* error("conflict", "event");
|
|
4277
|
+
const rows = yield* query(sql`
|
|
4278
|
+
SELECT owner_id, subscription_id, ordinal FROM effect_agent_subscriptions WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4279
|
+
AND source_name=${stored.source.name} AND source_version=${stored.source.version} AND matching_key=${stored.matchingKey}
|
|
4280
|
+
AND ordinal>${stored.cursor} AND ordinal<=${stored.cutoff} ORDER BY ordinal LIMIT ${limit}
|
|
4281
|
+
`, "subscription candidates");
|
|
4282
|
+
const decoded = yield* decodeRows(Schema.Struct({
|
|
4283
|
+
owner_id: Schema.String,
|
|
4284
|
+
subscription_id: Schema.String,
|
|
4285
|
+
ordinal: Schema.Natural
|
|
4286
|
+
}), rows, "subscription candidates");
|
|
4287
|
+
return yield* Effect.forEach(decoded, Effect.fn("SqliteSubscriptionStore.candidateRecord")(function* (row) {
|
|
4288
|
+
const record = yield* readRegistration({
|
|
4289
|
+
partition,
|
|
4290
|
+
ownerId: row.owner_id,
|
|
4291
|
+
subscriptionId: row.subscription_id
|
|
4292
|
+
}, "subscription candidate");
|
|
4293
|
+
if (record === null || record.ordinal !== row.ordinal) return yield* corrupt("subscription candidate projection");
|
|
4294
|
+
return record;
|
|
4295
|
+
}));
|
|
4296
|
+
});
|
|
4297
|
+
const insertDelivery = Effect.fn("SqliteSubscriptionStore.insertDelivery")(function* (delivery) {
|
|
4298
|
+
const json = yield* encode(SubscriptionDelivery, delivery, "encode selected delivery");
|
|
4299
|
+
yield* query(sql`
|
|
4300
|
+
INSERT INTO effect_agent_subscription_deliveries (tenant_id, source_address, owner_id, subscription_id, event_id,
|
|
4301
|
+
delivery_key, state, next_attempt_at_millis, record_json)
|
|
4302
|
+
VALUES (${partition.tenantId}, ${partition.address}, ${delivery.key.subscription.ownerId}, ${delivery.key.subscription.subscriptionId},
|
|
4303
|
+
${delivery.key.eventId}, ${subscriptionDeliveryKeyString(delivery.key)}, ${delivery.state}, ${delivery.retry.nextAttemptAtMillis}, ${json})
|
|
4304
|
+
`, "insert delivery");
|
|
4305
|
+
});
|
|
4306
|
+
const select = Effect.fn("SqliteSubscriptionStore.select")(function* (inputEvent, inputDeliveries, cursor, complete, nowMillis, inputLimits) {
|
|
4307
|
+
const supplied = yield* validate(AcceptedEvent, inputEvent, "select-event");
|
|
4308
|
+
const deliveries = yield* validate(Schema.Array(SubscriptionDelivery), inputDeliveries, "select-deliveries");
|
|
4309
|
+
const limits = yield* validate(SubscriptionLimits, inputLimits, "select-limits");
|
|
4310
|
+
yield* requirePartition(supplied.partition, "select-partition");
|
|
4311
|
+
for (const candidate of deliveries) yield* requirePartition(candidate.key.subscription.partition, "select-delivery-partition");
|
|
4312
|
+
if (yield* transact(Effect.gen(function* () {
|
|
4313
|
+
const accepted = yield* readEvent(supplied.eventId, "select event");
|
|
4314
|
+
if (accepted === null) return yield* error("not-found", "event");
|
|
4315
|
+
if (!sameAcceptedEventIdentity(accepted, supplied) || accepted.cursor !== supplied.cursor) return yield* error("conflict", "event-cursor");
|
|
4316
|
+
if (accepted.routingComplete) return false;
|
|
4317
|
+
if (!Number.isSafeInteger(cursor) || cursor < accepted.cursor || cursor > accepted.cutoff) return yield* error("validation", "cursor");
|
|
4318
|
+
yield* failpoint.hit("subscription:select:before");
|
|
4319
|
+
const effectiveNowMillis = Math.max(nowMillis, yield* Clock.currentTimeMillis);
|
|
4320
|
+
const additions = [];
|
|
4321
|
+
for (const delivery of deliveries) {
|
|
4322
|
+
const record = yield* readRegistration(delivery.key.subscription, "select registration");
|
|
4323
|
+
if (record === null) return yield* error("not-found", "subscription");
|
|
4324
|
+
if (!subscriptionDeliveryCanSelect(delivery, record, accepted) || delivery.key.eventId !== accepted.eventId || delivery.source.name !== accepted.source.name || delivery.source.version !== accepted.source.version || record.ordinal <= accepted.cursor || record.ordinal > cursor) return yield* error("conflict", "selection");
|
|
4325
|
+
const existing = yield* readDelivery(delivery.key, "select existing delivery");
|
|
4326
|
+
if (existing !== null) {
|
|
4327
|
+
if (!sameDeliveryIdentity(existing, delivery)) return yield* error("conflict", "delivery-identity");
|
|
4328
|
+
continue;
|
|
4329
|
+
}
|
|
4330
|
+
if (!subscriptionCanSelect(record, accepted, effectiveNowMillis, false)) continue;
|
|
4331
|
+
additions.push({
|
|
4332
|
+
delivery,
|
|
4333
|
+
record
|
|
4334
|
+
});
|
|
4335
|
+
}
|
|
4336
|
+
if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`, "count deliveries")) + additions.length > limits.maxDeliveries) return yield* error("capacity", "deliveries");
|
|
4337
|
+
for (const ownerId of new Set(additions.map(({ record }) => record.key.ownerId))) if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND owner_id=${ownerId}`, "count owner deliveries")) + additions.filter(({ record }) => record.key.ownerId === ownerId).length > limits.maxDeliveriesPerOwner) return yield* error("capacity", "owner-deliveries");
|
|
4338
|
+
for (const addition of additions) {
|
|
4339
|
+
yield* insertDelivery(addition.delivery);
|
|
4340
|
+
if (addition.record.configuration.mode === "once") yield* writeRegistration({
|
|
4341
|
+
...addition.record,
|
|
4342
|
+
state: "consumed",
|
|
4343
|
+
recovery: null
|
|
4344
|
+
});
|
|
4345
|
+
}
|
|
4346
|
+
yield* writeEvent({
|
|
4347
|
+
...accepted,
|
|
4348
|
+
cursor,
|
|
4349
|
+
routingComplete: complete,
|
|
4350
|
+
routingFailure: null
|
|
4351
|
+
});
|
|
4352
|
+
return true;
|
|
4353
|
+
}))) yield* failpoint.hit("subscription:select:after");
|
|
4354
|
+
});
|
|
4355
|
+
const catchUp = Effect.fn("SqliteSubscriptionStore.catchUp")(function* (inputEvent, inputDelivery, nowMillis, inputLimits) {
|
|
4356
|
+
const supplied = yield* validate(AcceptedEvent, inputEvent, "catch-up-event");
|
|
4357
|
+
const delivery = yield* validate(SubscriptionDelivery, inputDelivery, "catch-up-delivery");
|
|
4358
|
+
const limits = yield* validate(SubscriptionLimits, inputLimits, "catch-up-limits");
|
|
4359
|
+
yield* requirePartition(supplied.partition, "catch-up-partition");
|
|
4360
|
+
yield* requirePartition(delivery.key.subscription.partition, "catch-up-delivery-partition");
|
|
4361
|
+
if (yield* transact(Effect.gen(function* () {
|
|
4362
|
+
const accepted = yield* readEvent(supplied.eventId, "catch-up event");
|
|
4363
|
+
const record = yield* readRegistration(delivery.key.subscription, "catch-up subscription");
|
|
4364
|
+
if (accepted === null || record === null) return yield* error("not-found", accepted === null ? "event" : "subscription");
|
|
4365
|
+
if (!sameAcceptedEventIdentity(accepted, supplied) || !subscriptionDeliveryCanSelect(delivery, record, accepted) || delivery.key.eventId !== accepted.eventId || delivery.source.name !== accepted.source.name || delivery.source.version !== accepted.source.version || record.configuration.mode !== "once") return yield* error("conflict", "catch-up-identity");
|
|
4366
|
+
const existing = yield* readDelivery(delivery.key, "catch-up existing delivery");
|
|
4367
|
+
if (existing !== null) {
|
|
4368
|
+
if (!sameDeliveryIdentity(existing, delivery)) return yield* error("conflict", "delivery-identity");
|
|
4369
|
+
return false;
|
|
4370
|
+
}
|
|
4371
|
+
yield* failpoint.hit("subscription:catch-up:before");
|
|
4372
|
+
const effectiveNowMillis = Math.max(nowMillis, yield* Clock.currentTimeMillis);
|
|
4373
|
+
if (!subscriptionCanSelect(record, accepted, effectiveNowMillis, true)) return yield* error("conflict", "catch-up-eligibility");
|
|
4374
|
+
if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}`, "count deliveries")) >= limits.maxDeliveries) return yield* error("capacity", "deliveries");
|
|
4375
|
+
if ((yield* count(sql`SELECT COUNT(*) AS count FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND owner_id=${record.key.ownerId}`, "count owner deliveries")) >= limits.maxDeliveriesPerOwner) return yield* error("capacity", "owner-deliveries");
|
|
4376
|
+
yield* insertDelivery(delivery);
|
|
4377
|
+
yield* writeRegistration({
|
|
4378
|
+
...record,
|
|
4379
|
+
state: "consumed",
|
|
4380
|
+
recovery: null
|
|
4381
|
+
});
|
|
4382
|
+
return true;
|
|
4383
|
+
}))) yield* failpoint.hit("subscription:catch-up:after");
|
|
4384
|
+
});
|
|
4385
|
+
const deferEvent = Effect.fn("SqliteSubscriptionStore.deferEvent")(function* (eventId, nextAttemptAtMillis, code) {
|
|
4386
|
+
const routingFailure = code === void 0 ? "routing-failed" : yield* validate(SubscriptionName, code, "routing-failure");
|
|
4387
|
+
yield* transact(Effect.gen(function* () {
|
|
4388
|
+
const accepted = yield* readEvent(eventId, "defer event");
|
|
4389
|
+
if (accepted === null) return yield* error("not-found", "event");
|
|
4390
|
+
yield* failpoint.hit("subscription:defer-event:before");
|
|
4391
|
+
yield* writeEvent({
|
|
4392
|
+
...accepted,
|
|
4393
|
+
nextAttemptAtMillis,
|
|
4394
|
+
routingFailure
|
|
4395
|
+
});
|
|
4396
|
+
}));
|
|
4397
|
+
yield* failpoint.hit("subscription:defer-event:after");
|
|
4398
|
+
});
|
|
4399
|
+
const delivery = Effect.fn("SqliteSubscriptionStore.delivery")(function* (input) {
|
|
4400
|
+
const key = yield* validate(SubscriptionDeliveryKey, input, "delivery-key");
|
|
4401
|
+
yield* requirePartition(key.subscription.partition, "delivery-partition");
|
|
4402
|
+
return yield* readDelivery(key, "get delivery");
|
|
4403
|
+
});
|
|
4404
|
+
const pendingDeliveries = Effect.fn("SqliteSubscriptionStore.pendingDeliveries")(function* (nowMillis, after, limit) {
|
|
4405
|
+
const rows = yield* query(sql`
|
|
4406
|
+
SELECT owner_id, subscription_id, event_id FROM effect_agent_subscription_deliveries
|
|
4407
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state NOT IN ('delivered','refused')
|
|
4408
|
+
AND next_attempt_at_millis<=${nowMillis} AND delivery_key>${after} ORDER BY delivery_key LIMIT ${limit}
|
|
4409
|
+
`, "pending deliveries");
|
|
4410
|
+
const rowSchema = Schema.Struct({
|
|
4411
|
+
owner_id: Schema.String,
|
|
4412
|
+
subscription_id: Schema.String,
|
|
4413
|
+
event_id: Schema.String
|
|
4414
|
+
});
|
|
4415
|
+
return yield* decodeRows(rowSchema, rows, "pending delivery keys").pipe(Effect.map((items) => items.map((item) => ({
|
|
4416
|
+
subscription: {
|
|
4417
|
+
partition,
|
|
4418
|
+
ownerId: item.owner_id,
|
|
4419
|
+
subscriptionId: item.subscription_id
|
|
4420
|
+
},
|
|
4421
|
+
eventId: item.event_id
|
|
4422
|
+
}))));
|
|
4423
|
+
});
|
|
4424
|
+
const listDeliveries = Effect.fn("SqliteSubscriptionStore.listDeliveries")(function* (input, after, limit) {
|
|
4425
|
+
const key = yield* requireKey(input, "list-deliveries-key");
|
|
4426
|
+
const rows = yield* query(sql`
|
|
4427
|
+
SELECT record_json FROM effect_agent_subscription_deliveries WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4428
|
+
AND owner_id=${key.ownerId} AND subscription_id=${key.subscriptionId} AND delivery_key>${after} ORDER BY delivery_key LIMIT ${limit}
|
|
4429
|
+
`, "list deliveries");
|
|
4430
|
+
const decoded = yield* decodeRows(JsonRow, rows, "list deliveries");
|
|
4431
|
+
return yield* Effect.forEach(decoded, (row) => decode(SubscriptionDelivery, row.record_json, "list delivery"));
|
|
4432
|
+
});
|
|
4433
|
+
const changeDelivery = Effect.fn("SqliteSubscriptionStore.changeDelivery")(function* (inputKey, inputDeliveryId, inputChange) {
|
|
4434
|
+
const key = yield* validate(SubscriptionDeliveryKey, inputKey, "change-delivery-key");
|
|
4435
|
+
const deliveryId = yield* validate(Digest, inputDeliveryId, "change-delivery-id");
|
|
4436
|
+
const change = yield* validate(DeliveryChange, inputChange, "change-delivery-change");
|
|
4437
|
+
yield* requirePartition(key.subscription.partition, "change-delivery-partition");
|
|
4438
|
+
const result = yield* transact(Effect.gen(function* () {
|
|
4439
|
+
const existing = yield* readDelivery(key, "change delivery");
|
|
4440
|
+
const record = yield* readRegistration(key.subscription, "change delivery subscription");
|
|
4441
|
+
if (existing === null || record === null) return yield* error("not-found", existing === null ? "delivery" : "subscription");
|
|
4442
|
+
yield* failpoint.hit(`subscription:delivery-${change._tag.toLowerCase()}:before`);
|
|
4443
|
+
const effectiveChange = change._tag === "Prepare" ? {
|
|
4444
|
+
...change,
|
|
4445
|
+
nowMillis: Math.max(change.nowMillis, yield* Clock.currentTimeMillis)
|
|
4446
|
+
} : change;
|
|
4447
|
+
const transition = applySubscriptionDeliveryChange(existing, record, deliveryId, effectiveChange);
|
|
4448
|
+
if (Result.isFailure(transition)) return yield* transition.failure;
|
|
4449
|
+
if (transition.success === existing) return {
|
|
4450
|
+
value: existing,
|
|
4451
|
+
changed: false
|
|
4452
|
+
};
|
|
4453
|
+
yield* writeDelivery(transition.success);
|
|
4454
|
+
return {
|
|
4455
|
+
value: transition.success,
|
|
4456
|
+
changed: true
|
|
4457
|
+
};
|
|
4458
|
+
}));
|
|
4459
|
+
if (result.changed) yield* failpoint.hit(`subscription:delivery-${change._tag.toLowerCase()}:after`);
|
|
4460
|
+
return result.value;
|
|
4461
|
+
});
|
|
4462
|
+
const recovering = Effect.fn("SqliteSubscriptionStore.recovering")(function* (nowMillis, after, limit) {
|
|
4463
|
+
const rows = yield* query(sql`
|
|
4464
|
+
SELECT owner_id, subscription_id, ordinal FROM effect_agent_subscriptions
|
|
4465
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state='active'
|
|
4466
|
+
AND recovery_at_millis IS NOT NULL AND recovery_at_millis<=${nowMillis} AND ordinal>${after}
|
|
4467
|
+
ORDER BY ordinal LIMIT ${limit}
|
|
4468
|
+
`, "recovering subscriptions");
|
|
4469
|
+
const rowSchema = Schema.Struct({
|
|
4470
|
+
owner_id: Schema.String,
|
|
4471
|
+
subscription_id: Schema.String,
|
|
4472
|
+
ordinal: Schema.Natural
|
|
4473
|
+
});
|
|
4474
|
+
return yield* decodeRows(rowSchema, rows, "recovering subscription keys").pipe(Effect.map((items) => items.map((item) => ({
|
|
4475
|
+
key: {
|
|
4476
|
+
partition,
|
|
4477
|
+
ownerId: item.owner_id,
|
|
4478
|
+
subscriptionId: item.subscription_id
|
|
4479
|
+
},
|
|
4480
|
+
ordinal: item.ordinal
|
|
4481
|
+
}))));
|
|
4482
|
+
});
|
|
4483
|
+
const deferRecovery = Effect.fn("SqliteSubscriptionStore.deferRecovery")(function* (input, recovery) {
|
|
4484
|
+
const key = yield* requireKey(input, "defer-recovery-key");
|
|
4485
|
+
yield* transact(Effect.gen(function* () {
|
|
4486
|
+
const record = yield* readRegistration(key, "defer recovery");
|
|
4487
|
+
if (record === null) return yield* error("not-found", "subscription");
|
|
4488
|
+
yield* failpoint.hit("subscription:defer-recovery:before");
|
|
4489
|
+
yield* writeRegistration({
|
|
4490
|
+
...record,
|
|
4491
|
+
recovery: record.state === "active" ? recovery : null
|
|
4492
|
+
});
|
|
4493
|
+
}));
|
|
4494
|
+
yield* failpoint.hit("subscription:defer-recovery:after");
|
|
4495
|
+
});
|
|
4496
|
+
const readScanCursors = Effect.gen(function* () {
|
|
4497
|
+
const rows = yield* query(sql`
|
|
4498
|
+
SELECT event_scan_cursor, delivery_scan_cursor, recovery_scan_cursor
|
|
4499
|
+
FROM effect_agent_subscription_sequences
|
|
4500
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4501
|
+
`, "read subscription scan cursors");
|
|
4502
|
+
const decoded = yield* decodeRows(ScanRow, rows, "read subscription scan cursors");
|
|
4503
|
+
if (decoded.length !== 1) return yield* corrupt("subscription scan cursors");
|
|
4504
|
+
return {
|
|
4505
|
+
events: decoded[0].event_scan_cursor,
|
|
4506
|
+
deliveries: decoded[0].delivery_scan_cursor,
|
|
4507
|
+
recovery: decoded[0].recovery_scan_cursor
|
|
4508
|
+
};
|
|
4509
|
+
});
|
|
4510
|
+
const advanceScanCursors = Effect.fn("SqliteSubscriptionStore.advanceScanCursors")(function* (input) {
|
|
4511
|
+
const cursors = yield* validate(SubscriptionScanCursors, input, "scan-cursors");
|
|
4512
|
+
yield* transact(Effect.gen(function* () {
|
|
4513
|
+
yield* failpoint.hit("subscription:advance-scan-cursors:before");
|
|
4514
|
+
yield* query(sql`
|
|
4515
|
+
UPDATE effect_agent_subscription_sequences
|
|
4516
|
+
SET event_scan_cursor=${cursors.events}, delivery_scan_cursor=${cursors.deliveries}, recovery_scan_cursor=${cursors.recovery}
|
|
4517
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address}
|
|
4518
|
+
`, "advance subscription scan cursors");
|
|
4519
|
+
}));
|
|
4520
|
+
yield* failpoint.hit("subscription:advance-scan-cursors:after");
|
|
4521
|
+
});
|
|
4522
|
+
const indexedDeadline = query(sql`
|
|
4523
|
+
SELECT MIN(deadline) AS deadline FROM (
|
|
4524
|
+
SELECT next_attempt_at_millis AS deadline FROM effect_agent_subscription_events
|
|
4525
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND routing_complete=0
|
|
4526
|
+
UNION ALL SELECT next_attempt_at_millis FROM effect_agent_subscription_deliveries
|
|
4527
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state NOT IN ('delivered','refused')
|
|
4528
|
+
UNION ALL SELECT recovery_at_millis FROM effect_agent_subscriptions
|
|
4529
|
+
WHERE tenant_id=${partition.tenantId} AND source_address=${partition.address} AND state='active' AND recovery_at_millis IS NOT NULL
|
|
4530
|
+
)
|
|
4531
|
+
`, "next subscription deadline").pipe(Effect.flatMap((rows) => decodeRows(Schema.Struct({ deadline: Schema.NullOr(Schema.Number) }), rows, "next subscription deadline")), Effect.flatMap((rows) => rows.length === 1 ? Effect.succeed(rows[0].deadline) : Effect.fail(corrupt("next subscription deadline"))));
|
|
4532
|
+
const nextDeadline = Effect.gen(function* () {
|
|
4533
|
+
const cursors = yield* readScanCursors;
|
|
4534
|
+
if (cursors.events !== "" || cursors.deliveries !== "" || cursors.recovery !== 0) return 0;
|
|
4535
|
+
return yield* indexedDeadline;
|
|
4536
|
+
});
|
|
4537
|
+
return SubscriptionStore.of({
|
|
4538
|
+
partition,
|
|
4539
|
+
register,
|
|
4540
|
+
get,
|
|
4541
|
+
list,
|
|
4542
|
+
cancel,
|
|
4543
|
+
accept,
|
|
4544
|
+
event,
|
|
4545
|
+
pendingEvents,
|
|
4546
|
+
candidates,
|
|
4547
|
+
select,
|
|
4548
|
+
catchUp,
|
|
4549
|
+
deferEvent,
|
|
4550
|
+
delivery,
|
|
4551
|
+
pendingDeliveries,
|
|
4552
|
+
listDeliveries,
|
|
4553
|
+
changeDelivery,
|
|
4554
|
+
recovering,
|
|
4555
|
+
deferRecovery,
|
|
4556
|
+
readScanCursors,
|
|
4557
|
+
advanceScanCursors,
|
|
4558
|
+
nextDeadline
|
|
4559
|
+
});
|
|
4560
|
+
});
|
|
4561
|
+
const subscriptionStoreLayer = (partition) => Layer.effect(SubscriptionStore, makeSubscriptionStore(partition));
|
|
4562
|
+
//#endregion
|
|
4563
|
+
export { CurrentSqliteStorageVersion, SqliteAppendConflict, SqliteCheckpointConflict, SqliteFenceRejected, SqliteLedgerError, SqliteStorageCompatibilityError, SqliteStorageConfig, SqliteStorageConfigValue, SqliteStorageCorruptionError, SqliteStorageError, SqliteStorageFailpoint, SqliteStorageFailpointError, SqliteStorageFailpointLocation, SqliteWriteContention, activityProcessorStoreLayer, activityProcessorStoreLayerWithFailpoints, layer, ledgerLayer, memoryReaderLayer, memoryStoreLayer, memoryStoreLayerWithFailpoints, observationOffsetAt, scheduleStoreLayer, storageConfigLayer, storageFailpointLayer, submissionLedgerLayer, subscriptionStoreLayer, threadStoreLayer };
|
|
3365
4564
|
|
|
3366
4565
|
//# sourceMappingURL=index.mjs.map
|