@effect-agent/storage-sqlite 0.1.0-beta.6 → 0.1.0-beta.60
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/SqliteActivityStore.d.mts +14 -0
- package/dist/SqliteActivityStore.mjs +310 -0
- package/dist/SqliteActivityStore.mjs.map +1 -0
- package/dist/SqliteMessageDeliveryStore.d.mts +14 -0
- package/dist/SqliteMessageDeliveryStore.mjs +24 -0
- package/dist/SqliteMessageDeliveryStore.mjs.map +1 -0
- package/dist/SqliteScheduleStore.d.mts +14 -0
- package/dist/SqliteScheduleStore.mjs +255 -0
- package/dist/SqliteScheduleStore.mjs.map +1 -0
- package/dist/SqliteStorageConfig.d.mts +34 -0
- package/dist/SqliteStorageConfig.mjs +39 -0
- package/dist/SqliteStorageConfig.mjs.map +1 -0
- package/dist/SqliteStorageError-CzKmwCLl.d.mts +86 -0
- package/dist/SqliteStorageError.d.mts +2 -0
- package/dist/SqliteStorageError.mjs +148 -0
- package/dist/SqliteStorageError.mjs.map +1 -0
- package/dist/SqliteStorageFailpoint.d.mts +17 -0
- package/dist/SqliteStorageFailpoint.mjs +14 -0
- package/dist/SqliteStorageFailpoint.mjs.map +1 -0
- package/dist/SqliteStorageFailpointTesting.d.mts +15 -0
- package/dist/SqliteStorageFailpointTesting.mjs +19 -0
- package/dist/SqliteStorageFailpointTesting.mjs.map +1 -0
- package/dist/SqliteStorageVersion-gA7_96xM.d.mts +9 -0
- package/dist/SqliteStorageVersion.d.mts +2 -0
- package/dist/SqliteStorageVersion.mjs +8 -0
- package/dist/SqliteStorageVersion.mjs.map +1 -0
- package/dist/SqliteSubmissionLedger.d.mts +23 -0
- package/dist/SqliteSubmissionLedger.mjs +1765 -0
- package/dist/SqliteSubmissionLedger.mjs.map +1 -0
- package/dist/SqliteSubscriptionStore.d.mts +13 -0
- package/dist/SqliteSubscriptionStore.mjs +28 -0
- package/dist/SqliteSubscriptionStore.mjs.map +1 -0
- package/dist/SqliteThreadStore.d.mts +49 -0
- package/dist/SqliteThreadStore.mjs +427 -0
- package/dist/SqliteThreadStore.mjs.map +1 -0
- package/dist/index.d.mts +11 -193
- package/dist/index.mjs +11 -3082
- package/dist/migrations-dnTR0RKq.mjs +326 -0
- package/dist/migrations-dnTR0RKq.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/dist/sqlite-journal-lDduEakl.mjs +887 -0
- package/dist/sqlite-journal-lDduEakl.mjs.map +1 -0
- package/package.json +1 -41
- package/src/SqliteActivityStore.ts +556 -0
- package/src/SqliteMessageDeliveryStore.ts +42 -0
- package/src/SqliteScheduleStore.ts +404 -0
- package/src/{sqlite-storage-config.ts → SqliteStorageConfig.ts} +1 -1
- package/src/{errors.ts → SqliteStorageError.ts} +8 -3
- package/src/SqliteStorageFailpoint.ts +23 -0
- package/src/{sqlite-storage-failpoint.ts → SqliteStorageFailpointTesting.ts} +7 -18
- package/src/SqliteStorageVersion.ts +2 -0
- package/src/{sqlite-ledger.ts → SqliteSubmissionLedger.ts} +753 -118
- package/src/SqliteSubscriptionStore.ts +59 -0
- package/src/{sqlite-conversation-store.ts → SqliteThreadStore.ts} +372 -302
- package/src/index.ts +10 -6
- package/src/internal/message-delivery-schema.ts +24 -0
- package/src/{migrations.ts → internal/migrations.ts} +145 -79
- package/src/{sqlite-journal.ts → internal/sqlite-journal.ts} +534 -222
- package/dist/index.mjs.map +0 -1
|
@@ -1,26 +1,40 @@
|
|
|
1
|
-
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/
|
|
1
|
+
import { CanonicalSequence, ProducerEpoch } from "@effect-agent/thread/Records";
|
|
2
|
+
import { ScheduleFailpoint, ScheduleFailpointError } from "@effect-agent/thread/Schedule";
|
|
3
|
+
import {
|
|
4
|
+
checkV2ThreadLayout,
|
|
5
|
+
upgradeV2Schedules,
|
|
6
|
+
upgradeV2Subscriptions,
|
|
7
|
+
} from "@effect-agent/thread/SqlStorageV2Upgrade";
|
|
8
|
+
import {
|
|
9
|
+
SubscriptionFailpoint,
|
|
10
|
+
SubscriptionFailpointError,
|
|
11
|
+
} from "@effect-agent/thread/Subscription";
|
|
12
|
+
import { NodeCrypto } from "@effect/platform-node";
|
|
2
13
|
import { SqliteMigrator } from "@effect/sql-sqlite-node";
|
|
3
14
|
import { Effect, Exit, Schema } from "effect";
|
|
4
15
|
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
5
16
|
import type { SqlError } from "effect/unstable/sql/SqlError";
|
|
6
17
|
|
|
18
|
+
import { SqliteStorageConfig } from "../SqliteStorageConfig.ts";
|
|
19
|
+
import type { SqliteStorageFailpointError } from "../SqliteStorageError.ts";
|
|
7
20
|
import {
|
|
8
21
|
SqliteAppendConflict,
|
|
9
22
|
SqliteCheckpointConflict,
|
|
10
23
|
SqliteFenceRejected,
|
|
11
24
|
SqliteStorageCompatibilityError,
|
|
25
|
+
SqliteStorageFailpointLocation,
|
|
12
26
|
SqliteStorageCorruptionError,
|
|
13
27
|
SqliteStorageError,
|
|
14
|
-
SqliteStorageFailpointError,
|
|
15
28
|
SqliteWriteContention,
|
|
16
|
-
|
|
17
|
-
} from "
|
|
29
|
+
} from "../SqliteStorageError.ts";
|
|
30
|
+
import { SqliteStorageFailpoint } from "../SqliteStorageFailpoint.ts";
|
|
31
|
+
import { createMessageDeliveryTables } from "./message-delivery-schema.ts";
|
|
18
32
|
import { CurrentSqliteStorageVersion, sqliteMigrations } from "./migrations.ts";
|
|
19
33
|
|
|
20
34
|
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(16 * 1024 * 1024));
|
|
21
35
|
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
22
36
|
const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0));
|
|
23
|
-
const
|
|
37
|
+
const MAX_RECORDS_PER_THREAD = 65_536;
|
|
24
38
|
const MAX_STORED_TEXT_BYTES = 16 * 1024 * 1024;
|
|
25
39
|
const MAX_IDENTIFIER_LENGTH = 1_024;
|
|
26
40
|
|
|
@@ -38,8 +52,8 @@ class SqliteNameRow extends Schema.Class<SqliteNameRow>("SqliteNameRow")({
|
|
|
38
52
|
name: BoundedIdentifier,
|
|
39
53
|
}) {}
|
|
40
54
|
|
|
41
|
-
class
|
|
42
|
-
|
|
55
|
+
class ThreadRow extends Schema.Class<ThreadRow>("ThreadRow")({
|
|
56
|
+
thread_id: BoundedIdentifier,
|
|
43
57
|
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
44
58
|
producer_epoch: ProducerEpoch,
|
|
45
59
|
tail_digest: BoundedStoredText,
|
|
@@ -50,7 +64,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
50
64
|
batch_digest: BoundedStoredText,
|
|
51
65
|
batch_id: BoundedIdentifier,
|
|
52
66
|
batch_json: BoundedStoredText,
|
|
53
|
-
|
|
67
|
+
thread_id: BoundedIdentifier,
|
|
54
68
|
first_sequence: CanonicalSequence,
|
|
55
69
|
last_sequence: CanonicalSequence,
|
|
56
70
|
tail_digest: BoundedStoredText,
|
|
@@ -58,7 +72,7 @@ class BatchRow extends Schema.Class<BatchRow>("BatchRow")({
|
|
|
58
72
|
|
|
59
73
|
class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
60
74
|
batch_id: BoundedIdentifier,
|
|
61
|
-
|
|
75
|
+
thread_id: BoundedIdentifier,
|
|
62
76
|
record_id: BoundedIdentifier,
|
|
63
77
|
record_json: BoundedStoredText,
|
|
64
78
|
sequence: CanonicalSequence,
|
|
@@ -66,7 +80,7 @@ class RecordRow extends Schema.Class<RecordRow>("RecordRow")({
|
|
|
66
80
|
|
|
67
81
|
class CheckpointRow extends Schema.Class<CheckpointRow>("CheckpointRow")({
|
|
68
82
|
checkpoint_json: BoundedStoredText,
|
|
69
|
-
|
|
83
|
+
thread_id: BoundedIdentifier,
|
|
70
84
|
tail_digest: BoundedStoredText,
|
|
71
85
|
through_sequence: CanonicalSequence,
|
|
72
86
|
}) {}
|
|
@@ -82,7 +96,7 @@ export class RawAppendRequest extends Schema.Class<RawAppendRequest>(
|
|
|
82
96
|
batchDigest: BoundedStoredText,
|
|
83
97
|
batchId: BoundedIdentifier,
|
|
84
98
|
batchJson: BoundedStoredText,
|
|
85
|
-
|
|
99
|
+
threadId: BoundedIdentifier,
|
|
86
100
|
expectedTailDigest: BoundedStoredText,
|
|
87
101
|
expectedTailSequence: CanonicalSequence,
|
|
88
102
|
producerEpoch: ProducerEpoch,
|
|
@@ -102,7 +116,7 @@ export class RawAppendResult extends Schema.Class<RawAppendResult>(
|
|
|
102
116
|
export class RawReadRequest extends Schema.Class<RawReadRequest>(
|
|
103
117
|
"@effect-agent/storage-sqlite/RawReadRequest",
|
|
104
118
|
)({
|
|
105
|
-
|
|
119
|
+
threadId: BoundedIdentifier,
|
|
106
120
|
fromSequenceExclusive: CanonicalSequence,
|
|
107
121
|
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1_024)),
|
|
108
122
|
}) {}
|
|
@@ -111,17 +125,15 @@ export class RawCheckpoint extends Schema.Class<RawCheckpoint>(
|
|
|
111
125
|
"@effect-agent/storage-sqlite/RawCheckpoint",
|
|
112
126
|
)({
|
|
113
127
|
checkpointJson: BoundedStoredText,
|
|
114
|
-
|
|
128
|
+
threadId: BoundedIdentifier,
|
|
115
129
|
tailDigest: BoundedStoredText,
|
|
116
130
|
throughSequence: CanonicalSequence,
|
|
117
131
|
}) {}
|
|
118
132
|
|
|
119
|
-
export class
|
|
120
|
-
"@effect-agent/storage-sqlite/
|
|
133
|
+
export class RawThreadExport extends Schema.Class<RawThreadExport>(
|
|
134
|
+
"@effect-agent/storage-sqlite/RawThreadExport",
|
|
121
135
|
)({
|
|
122
|
-
|
|
123
|
-
checkpoints: Schema.Array(CheckpointRow),
|
|
124
|
-
conversation: ConversationRow,
|
|
136
|
+
thread: ThreadRow,
|
|
125
137
|
records: Schema.Array(RecordRow),
|
|
126
138
|
}) {}
|
|
127
139
|
|
|
@@ -138,11 +150,6 @@ type CheckpointError =
|
|
|
138
150
|
| SqliteStorageCorruptionError
|
|
139
151
|
| SqliteStorageError
|
|
140
152
|
| SqliteWriteContention;
|
|
141
|
-
type SqliteJournalFailpoint = (
|
|
142
|
-
location: SqliteStorageFailpointLocation,
|
|
143
|
-
) => Effect.Effect<void, SqliteStorageFailpointError>;
|
|
144
|
-
|
|
145
|
-
const noFailpoint: SqliteJournalFailpoint = () => Effect.void;
|
|
146
153
|
|
|
147
154
|
const storageError =
|
|
148
155
|
(operation: string) =>
|
|
@@ -195,26 +202,219 @@ export const decodeSingleRow = Effect.fn("SqliteJournal.decodeSingleRow")(
|
|
|
195
202
|
),
|
|
196
203
|
);
|
|
197
204
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
205
|
+
/** Column inventory of the supported v8 predecessor, independent of physical column order. */
|
|
206
|
+
const predecessorColumns = {
|
|
207
|
+
effect_agent_threads: [
|
|
208
|
+
"thread_id",
|
|
209
|
+
"created_at",
|
|
210
|
+
"tail_sequence",
|
|
211
|
+
"tail_digest",
|
|
212
|
+
"producer_epoch",
|
|
213
|
+
],
|
|
214
|
+
effect_agent_canonical_batches: [
|
|
215
|
+
"thread_id",
|
|
216
|
+
"batch_id",
|
|
217
|
+
"first_sequence",
|
|
218
|
+
"last_sequence",
|
|
219
|
+
"batch_digest",
|
|
220
|
+
"tail_digest",
|
|
221
|
+
"batch_json",
|
|
222
|
+
],
|
|
223
|
+
effect_agent_canonical_records: ["thread_id", "sequence", "record_id", "batch_id", "record_json"],
|
|
224
|
+
effect_agent_checkpoints: ["thread_id", "through_sequence", "tail_digest", "checkpoint_json"],
|
|
225
|
+
effect_agent_submissions: [
|
|
226
|
+
"submission_id",
|
|
227
|
+
"thread_id",
|
|
228
|
+
"queue_sequence",
|
|
229
|
+
"principal",
|
|
230
|
+
"idempotency_key",
|
|
231
|
+
"agent_id",
|
|
232
|
+
"agent_digests_json",
|
|
233
|
+
"deployment_id",
|
|
234
|
+
"input_json",
|
|
235
|
+
"input_digest",
|
|
236
|
+
"receipt_id",
|
|
237
|
+
"state",
|
|
238
|
+
"settled_outcome",
|
|
239
|
+
"created_at",
|
|
240
|
+
"ready_at",
|
|
241
|
+
"input_applied_record_id",
|
|
242
|
+
"input_applied_sequence",
|
|
243
|
+
"joined_host_submission_id",
|
|
244
|
+
"suspended_reason_json",
|
|
245
|
+
"suspended_at",
|
|
246
|
+
"unknown_reason",
|
|
247
|
+
"unknown_tool_call_ids_json",
|
|
248
|
+
"parent_submission_id",
|
|
249
|
+
"parent_tool_call_id",
|
|
250
|
+
"admission_group",
|
|
251
|
+
"admission_fence_json",
|
|
252
|
+
],
|
|
253
|
+
effect_agent_submission_ownership: [
|
|
254
|
+
"submission_id",
|
|
255
|
+
"attempt_id",
|
|
256
|
+
"ownership_token",
|
|
257
|
+
"producer_epoch",
|
|
258
|
+
"owner_producer_id",
|
|
259
|
+
"lease_expires_at",
|
|
260
|
+
],
|
|
261
|
+
effect_agent_attempts: [
|
|
262
|
+
"attempt_id",
|
|
263
|
+
"submission_id",
|
|
264
|
+
"thread_id",
|
|
265
|
+
"owner_producer_id",
|
|
266
|
+
"producer_epoch",
|
|
267
|
+
"claimed_at",
|
|
268
|
+
],
|
|
269
|
+
effect_agent_settlement_reservations: [
|
|
270
|
+
"submission_id",
|
|
271
|
+
"settlement_id",
|
|
272
|
+
"outcome",
|
|
273
|
+
"record_id",
|
|
274
|
+
"record_json",
|
|
275
|
+
"record_digest",
|
|
276
|
+
"reserved_at",
|
|
277
|
+
"finalized_at",
|
|
278
|
+
],
|
|
279
|
+
effect_agent_abort_intents: [
|
|
280
|
+
"submission_id",
|
|
281
|
+
"author",
|
|
282
|
+
"reason",
|
|
283
|
+
"requested_at",
|
|
284
|
+
"canonical_record_id",
|
|
285
|
+
],
|
|
286
|
+
effect_agent_approval_decisions: [
|
|
287
|
+
"submission_id",
|
|
288
|
+
"tool_call_id",
|
|
289
|
+
"decision",
|
|
290
|
+
"resolver",
|
|
291
|
+
"reason",
|
|
292
|
+
"decided_at",
|
|
293
|
+
],
|
|
294
|
+
effect_agent_unknown_resolutions: [
|
|
295
|
+
"submission_id",
|
|
296
|
+
"tool_call_id",
|
|
297
|
+
"author",
|
|
298
|
+
"reason",
|
|
299
|
+
"resolution_json",
|
|
300
|
+
"resolved_at",
|
|
301
|
+
],
|
|
302
|
+
effect_agent_child_reservations: [
|
|
303
|
+
"reservation_id",
|
|
304
|
+
"parent_submission_id",
|
|
305
|
+
"parent_tool_call_id",
|
|
306
|
+
"child_submission_id",
|
|
307
|
+
"status",
|
|
308
|
+
"allocation_json",
|
|
309
|
+
"allocation_digest",
|
|
310
|
+
"accounting_json",
|
|
311
|
+
"reserved_at",
|
|
312
|
+
"release_began_at",
|
|
313
|
+
"released_at",
|
|
314
|
+
],
|
|
315
|
+
effect_agent_schedules: [
|
|
316
|
+
"tenant_id",
|
|
317
|
+
"owner_id",
|
|
318
|
+
"schedule_id",
|
|
319
|
+
"deadline_at_millis",
|
|
320
|
+
"record_json",
|
|
321
|
+
],
|
|
322
|
+
effect_agent_subscription_sequences: [
|
|
323
|
+
"tenant_id",
|
|
324
|
+
"source_address",
|
|
325
|
+
"sequence",
|
|
326
|
+
"event_scan_cursor",
|
|
327
|
+
"delivery_scan_cursor",
|
|
328
|
+
"recovery_scan_cursor",
|
|
329
|
+
],
|
|
330
|
+
effect_agent_subscriptions: [
|
|
331
|
+
"tenant_id",
|
|
332
|
+
"source_address",
|
|
333
|
+
"owner_id",
|
|
334
|
+
"subscription_id",
|
|
335
|
+
"ordinal",
|
|
336
|
+
"source_name",
|
|
337
|
+
"source_version",
|
|
338
|
+
"matching_key",
|
|
339
|
+
"state",
|
|
340
|
+
"expires_at_millis",
|
|
341
|
+
"recovery_at_millis",
|
|
342
|
+
"recovery_present",
|
|
343
|
+
"record_json",
|
|
344
|
+
],
|
|
345
|
+
effect_agent_subscription_events: [
|
|
346
|
+
"tenant_id",
|
|
347
|
+
"source_address",
|
|
348
|
+
"event_id",
|
|
349
|
+
"source_name",
|
|
350
|
+
"source_version",
|
|
351
|
+
"matching_key",
|
|
352
|
+
"payload_digest",
|
|
353
|
+
"cutoff",
|
|
354
|
+
"cursor",
|
|
355
|
+
"routing_complete",
|
|
356
|
+
"next_attempt_at_millis",
|
|
357
|
+
"record_json",
|
|
358
|
+
"tombstone",
|
|
359
|
+
],
|
|
360
|
+
effect_agent_subscription_deliveries: [
|
|
361
|
+
"tenant_id",
|
|
362
|
+
"source_address",
|
|
363
|
+
"owner_id",
|
|
364
|
+
"subscription_id",
|
|
365
|
+
"event_id",
|
|
366
|
+
"delivery_key",
|
|
367
|
+
"state",
|
|
368
|
+
"next_attempt_at_millis",
|
|
369
|
+
"record_json",
|
|
370
|
+
],
|
|
371
|
+
} as const;
|
|
372
|
+
|
|
373
|
+
const checkPredecessorLayout = Effect.fn("SqliteJournal.checkPredecessorLayout")(function* () {
|
|
374
|
+
const sql = yield* SqlClient.SqlClient;
|
|
375
|
+
|
|
376
|
+
for (const [table, expected] of Object.entries(predecessorColumns)) {
|
|
377
|
+
const columns = yield* decodeRows(
|
|
378
|
+
Schema.Array(Schema.Struct({ name: BoundedIdentifier })),
|
|
379
|
+
table,
|
|
380
|
+
"schema",
|
|
381
|
+
yield* sql.unsafe(`PRAGMA table_info(${table})`),
|
|
382
|
+
);
|
|
383
|
+
|
|
384
|
+
const names = new Set<string>(expected);
|
|
385
|
+
|
|
386
|
+
if (columns.length !== names.size || columns.some((column) => !names.has(column.name)))
|
|
387
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
388
|
+
actualVersion: 8,
|
|
389
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
390
|
+
message: `The v8 ${table} columns do not match the supported predecessor; no upgrade was committed.`,
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
export const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function* () {
|
|
396
|
+
const sql = yield* SqlClient.SqlClient;
|
|
397
|
+
const { hit: failpoint } = yield* SqliteStorageFailpoint;
|
|
398
|
+
const { busyTimeout } = yield* SqliteStorageConfig;
|
|
399
|
+
|
|
203
400
|
yield* sql`PRAGMA foreign_keys = ON`.pipe(Effect.mapError(storageError("enable foreign keys")));
|
|
204
401
|
// PRAGMA statements do not accept bound parameters; the value is a schema-validated
|
|
205
402
|
// non-negative integer, never caller-controlled text.
|
|
206
403
|
yield* sql
|
|
207
|
-
.unsafe(`PRAGMA busy_timeout = ${
|
|
404
|
+
.unsafe(`PRAGMA busy_timeout = ${busyTimeout}`)
|
|
208
405
|
.pipe(Effect.mapError(storageError("configure busy timeout")));
|
|
406
|
+
|
|
209
407
|
const journalModeRows = yield* sql<Record<string, unknown>>`PRAGMA journal_mode`.pipe(
|
|
210
408
|
Effect.mapError(storageError("read journal mode")),
|
|
211
409
|
);
|
|
410
|
+
|
|
212
411
|
const journalMode = yield* decodeSingleRow(
|
|
213
412
|
Schema.Array(SqliteJournalModeRow),
|
|
214
413
|
"pragma_journal_mode",
|
|
215
414
|
"singleton",
|
|
216
415
|
journalModeRows,
|
|
217
416
|
);
|
|
417
|
+
|
|
218
418
|
if (journalMode.journal_mode.toLowerCase() !== "wal") {
|
|
219
419
|
return yield* SqliteStorageCompatibilityError.make({
|
|
220
420
|
actualVersion: 0,
|
|
@@ -226,6 +426,7 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
226
426
|
const versionRows = yield* sql<Record<string, unknown>>`PRAGMA user_version`.pipe(
|
|
227
427
|
Effect.mapError(storageError("read storage version")),
|
|
228
428
|
);
|
|
429
|
+
|
|
229
430
|
const version = yield* decodeSingleRow(
|
|
230
431
|
Schema.Array(SqliteVersionRow),
|
|
231
432
|
"pragma_user_version",
|
|
@@ -233,20 +434,137 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
233
434
|
versionRows,
|
|
234
435
|
);
|
|
235
436
|
|
|
236
|
-
//
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
437
|
+
// Support the known beta49/beta50 and immediate predecessor formats atomically.
|
|
438
|
+
if (
|
|
439
|
+
version.user_version !== 0 &&
|
|
440
|
+
version.user_version !== 7 &&
|
|
441
|
+
version.user_version !== 8 &&
|
|
442
|
+
version.user_version !== CurrentSqliteStorageVersion
|
|
443
|
+
) {
|
|
240
444
|
return yield* SqliteStorageCompatibilityError.make({
|
|
241
445
|
actualVersion: version.user_version,
|
|
242
446
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
243
447
|
message:
|
|
244
|
-
`The SQLite file uses
|
|
448
|
+
`The SQLite file uses unsupported storage version ${version.user_version}; ` +
|
|
245
449
|
`this build supports exactly version ${CurrentSqliteStorageVersion}. ` +
|
|
246
|
-
"
|
|
450
|
+
"Only supported v7 and v8 can be upgraded automatically. Keep the original file and use a compatible library version.",
|
|
247
451
|
});
|
|
248
452
|
}
|
|
249
453
|
|
|
454
|
+
if (version.user_version === 7 || version.user_version === 8) {
|
|
455
|
+
yield* sql
|
|
456
|
+
.withTransaction(
|
|
457
|
+
Effect.gen(function* () {
|
|
458
|
+
const current = yield* sql<{ user_version: number }>`PRAGMA user_version`;
|
|
459
|
+
|
|
460
|
+
if (current.length === 1 && current[0].user_version === CurrentSqliteStorageVersion)
|
|
461
|
+
return;
|
|
462
|
+
if (
|
|
463
|
+
current.length !== 1 ||
|
|
464
|
+
(current[0].user_version !== 7 && current[0].user_version !== 8)
|
|
465
|
+
)
|
|
466
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
467
|
+
actualVersion: -1,
|
|
468
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
469
|
+
message: "Storage version changed while acquiring the upgrade transaction.",
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
const required = yield* sql<{
|
|
473
|
+
name: string;
|
|
474
|
+
}>`SELECT name FROM sqlite_master WHERE type='table' AND name IN (
|
|
475
|
+
'effect_agent_threads', 'effect_agent_canonical_batches', 'effect_agent_canonical_records',
|
|
476
|
+
'effect_agent_checkpoints', 'effect_agent_submissions', 'effect_agent_submission_ownership',
|
|
477
|
+
'effect_agent_attempts', 'effect_agent_settlement_reservations', 'effect_agent_abort_intents',
|
|
478
|
+
'effect_agent_approval_decisions', 'effect_agent_unknown_resolutions', 'effect_agent_schedules'
|
|
479
|
+
)`;
|
|
480
|
+
|
|
481
|
+
if (required.length !== 12)
|
|
482
|
+
return yield* SqliteStorageCompatibilityError.make({
|
|
483
|
+
actualVersion: current[0].user_version,
|
|
484
|
+
supportedVersion: CurrentSqliteStorageVersion,
|
|
485
|
+
message:
|
|
486
|
+
"The predecessor store is missing required tables; no upgrade was committed.",
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
if (current[0].user_version === 7) {
|
|
490
|
+
yield* checkV2ThreadLayout();
|
|
491
|
+
for (const statement of [
|
|
492
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
|
|
493
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
|
|
494
|
+
sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`,
|
|
495
|
+
]) {
|
|
496
|
+
yield* failpoint("upgrade:before-mutation");
|
|
497
|
+
yield* statement;
|
|
498
|
+
yield* failpoint("upgrade:after-mutation");
|
|
499
|
+
}
|
|
500
|
+
yield* upgradeV2Schedules(16 * 1024 * 1024).pipe(
|
|
501
|
+
Effect.provideService(ScheduleFailpoint, {
|
|
502
|
+
hit: (point) =>
|
|
503
|
+
Schema.decodeUnknownEffect(SqliteStorageFailpointLocation)(point).pipe(
|
|
504
|
+
Effect.flatMap(failpoint),
|
|
505
|
+
Effect.mapError(() => ScheduleFailpointError.make({ point })),
|
|
506
|
+
),
|
|
507
|
+
}),
|
|
508
|
+
);
|
|
509
|
+
yield* upgradeV2Subscriptions(16 * 1024 * 1024).pipe(
|
|
510
|
+
Effect.provideService(SubscriptionFailpoint, {
|
|
511
|
+
hit: (point) =>
|
|
512
|
+
Schema.decodeUnknownEffect(SqliteStorageFailpointLocation)(point).pipe(
|
|
513
|
+
Effect.flatMap(failpoint),
|
|
514
|
+
Effect.mapError(() => SubscriptionFailpointError.make({ point })),
|
|
515
|
+
),
|
|
516
|
+
}),
|
|
517
|
+
);
|
|
518
|
+
}
|
|
519
|
+
if (current[0].user_version === 8) yield* checkPredecessorLayout();
|
|
520
|
+
yield* failpoint("upgrade:before-mutation");
|
|
521
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
|
|
522
|
+
yield* failpoint("upgrade:after-mutation");
|
|
523
|
+
yield* failpoint("upgrade:before-mutation");
|
|
524
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
|
|
525
|
+
yield* failpoint("upgrade:after-mutation");
|
|
526
|
+
yield* failpoint("upgrade:before-mutation");
|
|
527
|
+
yield* createMessageDeliveryTables;
|
|
528
|
+
yield* failpoint("upgrade:after-mutation");
|
|
529
|
+
yield* failpoint("upgrade:before-version");
|
|
530
|
+
yield* sql`PRAGMA user_version = 9`;
|
|
531
|
+
yield* failpoint("upgrade:after-version");
|
|
532
|
+
}),
|
|
533
|
+
)
|
|
534
|
+
.pipe(
|
|
535
|
+
Effect.provide(NodeCrypto.layer),
|
|
536
|
+
Effect.catchTag("SqliteStorageFailpointError", (error) =>
|
|
537
|
+
SqliteStorageError.make({
|
|
538
|
+
cause: error,
|
|
539
|
+
operation: "upgrade storage",
|
|
540
|
+
message: error.message,
|
|
541
|
+
}),
|
|
542
|
+
),
|
|
543
|
+
Effect.catchTag(["ScheduleFailpointError", "SubscriptionFailpointError"], (error) =>
|
|
544
|
+
SqliteStorageError.make({
|
|
545
|
+
cause: error,
|
|
546
|
+
operation: "upgrade storage",
|
|
547
|
+
message: "Injected storage upgrade failure",
|
|
548
|
+
}),
|
|
549
|
+
),
|
|
550
|
+
Effect.catchTag("StorageUpgradeError", (error) =>
|
|
551
|
+
SqliteStorageCorruptionError.make({
|
|
552
|
+
table: error.table,
|
|
553
|
+
rowKey: error.rowKey,
|
|
554
|
+
message: error.message,
|
|
555
|
+
}),
|
|
556
|
+
),
|
|
557
|
+
Effect.catchTag("SqlError", storageError("upgrade supported storage")),
|
|
558
|
+
Effect.catchTag("SchemaError", (error) =>
|
|
559
|
+
SqliteStorageCorruptionError.make({
|
|
560
|
+
table: "upgrade",
|
|
561
|
+
rowKey: "v7",
|
|
562
|
+
message: error.message,
|
|
563
|
+
}),
|
|
564
|
+
),
|
|
565
|
+
);
|
|
566
|
+
}
|
|
567
|
+
|
|
250
568
|
if (version.user_version === 0) {
|
|
251
569
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
252
570
|
SELECT name
|
|
@@ -255,6 +573,7 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
255
573
|
AND name LIKE 'effect_agent_%'
|
|
256
574
|
ORDER BY name
|
|
257
575
|
`.pipe(Effect.mapError(storageError("inspect unversioned storage")));
|
|
576
|
+
|
|
258
577
|
const existing = yield* decodeRows(
|
|
259
578
|
Schema.Array(SqliteNameRow),
|
|
260
579
|
"sqlite_master",
|
|
@@ -267,14 +586,11 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
267
586
|
actualVersion: 0,
|
|
268
587
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
269
588
|
message:
|
|
270
|
-
"The SQLite file contains unversioned Effect Agent tables.
|
|
589
|
+
"The SQLite file contains unversioned Effect Agent tables. Refusing to mutate ambiguous stored data; retain it for inspection with its original writer.",
|
|
271
590
|
});
|
|
272
591
|
}
|
|
273
592
|
|
|
274
593
|
yield* SqliteMigrator.run({ loader: sqliteMigrations }).pipe(
|
|
275
|
-
// SqliteMigrator depends on the generic client supplied by this adapter.
|
|
276
|
-
// The concrete Node client is kept at the outer Layer boundary.
|
|
277
|
-
Effect.provideService(SqlClient.SqlClient, sql),
|
|
278
594
|
Effect.mapError((error) =>
|
|
279
595
|
SqliteStorageError.make({
|
|
280
596
|
cause: error,
|
|
@@ -290,7 +606,7 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
290
606
|
FROM sqlite_master
|
|
291
607
|
WHERE type = 'table'
|
|
292
608
|
AND name IN (
|
|
293
|
-
'
|
|
609
|
+
'effect_agent_threads',
|
|
294
610
|
'effect_agent_canonical_batches',
|
|
295
611
|
'effect_agent_canonical_records',
|
|
296
612
|
'effect_agent_checkpoints',
|
|
@@ -300,29 +616,29 @@ const ensureCurrentStorage = Effect.fn("SqliteJournal.ensureCurrentStorage")(fun
|
|
|
300
616
|
'effect_agent_settlement_reservations',
|
|
301
617
|
'effect_agent_abort_intents',
|
|
302
618
|
'effect_agent_approval_decisions',
|
|
303
|
-
'effect_agent_unknown_resolutions'
|
|
619
|
+
'effect_agent_unknown_resolutions',
|
|
620
|
+
'effect_agent_schedules',
|
|
621
|
+
'effect_agent_message_deliveries'
|
|
304
622
|
)
|
|
305
623
|
ORDER BY name
|
|
306
624
|
`.pipe(Effect.mapError(storageError("verify storage tables")));
|
|
625
|
+
|
|
307
626
|
const required = yield* decodeRows(
|
|
308
627
|
Schema.Array(SqliteNameRow),
|
|
309
628
|
"sqlite_master",
|
|
310
629
|
"required_tables",
|
|
311
630
|
requiredRows,
|
|
312
631
|
);
|
|
313
|
-
|
|
632
|
+
|
|
633
|
+
if (required.length !== 13) {
|
|
314
634
|
return yield* SqliteStorageCompatibilityError.make({
|
|
315
635
|
actualVersion: CurrentSqliteStorageVersion,
|
|
316
636
|
supportedVersion: CurrentSqliteStorageVersion,
|
|
317
637
|
message:
|
|
318
|
-
"The SQLite file claims the current format but is missing required tables.
|
|
638
|
+
"The SQLite file claims the current format but is missing required tables. Retain the original store for inspection.",
|
|
319
639
|
});
|
|
320
640
|
}
|
|
321
641
|
|
|
322
|
-
return makeJournal(sql, failpoint);
|
|
323
|
-
});
|
|
324
|
-
|
|
325
|
-
const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint) => {
|
|
326
642
|
const classifyWriteFailure =
|
|
327
643
|
(operation: string) =>
|
|
328
644
|
(error: SqlError): SqliteStorageError | SqliteWriteContention =>
|
|
@@ -357,19 +673,24 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
357
673
|
const connection = yield* sql.reserve.pipe(
|
|
358
674
|
Effect.mapError(classifyWriteFailure(operation)),
|
|
359
675
|
);
|
|
676
|
+
|
|
360
677
|
yield* connection
|
|
361
678
|
.executeUnprepared("BEGIN IMMEDIATE", [], undefined)
|
|
362
679
|
.pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
680
|
+
|
|
363
681
|
const exit = yield* restore(
|
|
364
682
|
Effect.provideService(effect, sql.transactionService, [connection, 0] as const),
|
|
365
683
|
).pipe(Effect.exit);
|
|
684
|
+
|
|
366
685
|
if (Exit.isSuccess(exit)) {
|
|
367
686
|
yield* connection
|
|
368
687
|
.executeUnprepared("COMMIT", [], undefined)
|
|
369
688
|
.pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
689
|
+
|
|
370
690
|
return exit.value;
|
|
371
691
|
}
|
|
372
692
|
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], undefined));
|
|
693
|
+
|
|
373
694
|
return yield* exit;
|
|
374
695
|
}),
|
|
375
696
|
).pipe(
|
|
@@ -392,26 +713,31 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
392
713
|
Effect.scoped(
|
|
393
714
|
Effect.gen(function* () {
|
|
394
715
|
const connection = yield* sql.reserve.pipe(Effect.mapError(storageError(operation)));
|
|
716
|
+
|
|
395
717
|
yield* connection
|
|
396
718
|
.executeUnprepared("BEGIN", [], undefined)
|
|
397
719
|
.pipe(Effect.mapError(storageError(operation)));
|
|
720
|
+
|
|
398
721
|
const exit = yield* restore(
|
|
399
722
|
Effect.provideService(effect, sql.transactionService, [connection, 0] as const),
|
|
400
723
|
).pipe(Effect.exit);
|
|
724
|
+
|
|
401
725
|
if (Exit.isSuccess(exit)) {
|
|
402
726
|
yield* connection
|
|
403
727
|
.executeUnprepared("COMMIT", [], undefined)
|
|
404
728
|
.pipe(Effect.mapError(storageError(operation)));
|
|
729
|
+
|
|
405
730
|
return exit.value;
|
|
406
731
|
}
|
|
407
732
|
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], undefined));
|
|
733
|
+
|
|
408
734
|
return yield* exit;
|
|
409
735
|
}),
|
|
410
736
|
).pipe(Effect.withSpan("SqliteJournal.withReadTransaction", { attributes: { operation } })),
|
|
411
737
|
);
|
|
412
738
|
|
|
413
739
|
const materialize = Effect.fn("SqliteJournal.materialize")(function* (
|
|
414
|
-
|
|
740
|
+
threadId: string,
|
|
415
741
|
createdAt: string,
|
|
416
742
|
emptyTailDigest: string,
|
|
417
743
|
producerEpoch: ProducerEpoch,
|
|
@@ -420,55 +746,58 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
420
746
|
SqliteFenceRejected | SqliteStorageCorruptionError | SqliteStorageError | SqliteWriteContention
|
|
421
747
|
> {
|
|
422
748
|
if (
|
|
423
|
-
|
|
749
|
+
threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
424
750
|
storedTextBytes(emptyTailDigest) > MAX_STORED_TEXT_BYTES
|
|
425
751
|
) {
|
|
426
752
|
return yield* SqliteStorageError.make({
|
|
427
|
-
operation: "materialize
|
|
428
|
-
message: "
|
|
753
|
+
operation: "materialize thread",
|
|
754
|
+
message: "Thread identity or initial digest exceeds the SQLite storage bounds.",
|
|
429
755
|
});
|
|
430
756
|
}
|
|
431
757
|
yield* withWriteTransaction("materialize transaction")(
|
|
432
758
|
Effect.gen(function* () {
|
|
433
759
|
const existingRows = yield* sql<Record<string, unknown>>`
|
|
434
760
|
SELECT
|
|
435
|
-
|
|
761
|
+
thread_id,
|
|
436
762
|
created_at,
|
|
437
763
|
tail_sequence,
|
|
438
764
|
tail_digest,
|
|
439
765
|
producer_epoch
|
|
440
|
-
FROM
|
|
441
|
-
WHERE
|
|
442
|
-
`.pipe(Effect.mapError(storageError("read materialized
|
|
766
|
+
FROM effect_agent_threads
|
|
767
|
+
WHERE thread_id = ${threadId}
|
|
768
|
+
`.pipe(Effect.mapError(storageError("read materialized thread")));
|
|
769
|
+
|
|
443
770
|
const existing = yield* decodeRows(
|
|
444
|
-
Schema.Array(
|
|
445
|
-
"
|
|
446
|
-
|
|
771
|
+
Schema.Array(ThreadRow),
|
|
772
|
+
"effect_agent_threads",
|
|
773
|
+
threadId,
|
|
447
774
|
existingRows,
|
|
448
775
|
);
|
|
776
|
+
|
|
449
777
|
if (existing.length > 1) {
|
|
450
778
|
return yield* SqliteStorageCorruptionError.make({
|
|
451
|
-
table: "
|
|
452
|
-
rowKey:
|
|
453
|
-
message: "A
|
|
779
|
+
table: "effect_agent_threads",
|
|
780
|
+
rowKey: threadId,
|
|
781
|
+
message: "A thread primary key returned more than one row.",
|
|
454
782
|
});
|
|
455
783
|
}
|
|
456
784
|
if (existing.length === 0) {
|
|
457
785
|
yield* sql`
|
|
458
|
-
INSERT INTO
|
|
459
|
-
|
|
786
|
+
INSERT INTO effect_agent_threads (
|
|
787
|
+
thread_id,
|
|
460
788
|
created_at,
|
|
461
789
|
tail_sequence,
|
|
462
790
|
tail_digest,
|
|
463
791
|
producer_epoch
|
|
464
792
|
) VALUES (
|
|
465
|
-
${
|
|
793
|
+
${threadId},
|
|
466
794
|
${createdAt},
|
|
467
795
|
0,
|
|
468
796
|
${emptyTailDigest},
|
|
469
797
|
${producerEpoch}
|
|
470
798
|
)
|
|
471
|
-
`.pipe(Effect.mapError(storageError("materialize
|
|
799
|
+
`.pipe(Effect.mapError(storageError("materialize thread")));
|
|
800
|
+
|
|
472
801
|
return;
|
|
473
802
|
}
|
|
474
803
|
if (producerEpoch < existing[0].producer_epoch) {
|
|
@@ -480,41 +809,35 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
480
809
|
}
|
|
481
810
|
if (producerEpoch > existing[0].producer_epoch) {
|
|
482
811
|
yield* sql`
|
|
483
|
-
UPDATE
|
|
812
|
+
UPDATE effect_agent_threads
|
|
484
813
|
SET producer_epoch = ${producerEpoch}
|
|
485
|
-
WHERE
|
|
814
|
+
WHERE thread_id = ${threadId}
|
|
486
815
|
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
487
816
|
}
|
|
488
817
|
}),
|
|
489
818
|
);
|
|
490
819
|
});
|
|
491
820
|
|
|
492
|
-
const
|
|
493
|
-
conversationId: string,
|
|
494
|
-
) {
|
|
821
|
+
const getThread = Effect.fn("SqliteJournal.getThread")(function* (threadId: string) {
|
|
495
822
|
const rows = yield* sql<Record<string, unknown>>`
|
|
496
823
|
SELECT
|
|
497
|
-
|
|
824
|
+
thread_id,
|
|
498
825
|
created_at,
|
|
499
826
|
tail_sequence,
|
|
500
827
|
tail_digest,
|
|
501
828
|
producer_epoch
|
|
502
|
-
FROM
|
|
503
|
-
WHERE
|
|
504
|
-
`.pipe(Effect.mapError(storageError("read
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
"effect_agent_conversations",
|
|
508
|
-
conversationId,
|
|
509
|
-
rows,
|
|
510
|
-
);
|
|
829
|
+
FROM effect_agent_threads
|
|
830
|
+
WHERE thread_id = ${threadId}
|
|
831
|
+
`.pipe(Effect.mapError(storageError("read thread")));
|
|
832
|
+
|
|
833
|
+
return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
511
834
|
});
|
|
512
835
|
|
|
513
836
|
const append = Effect.fn("SqliteJournal.append")(function* (
|
|
514
837
|
request: RawAppendRequest,
|
|
515
838
|
): Effect.fn.Return<RawAppendResult, AppendError> {
|
|
516
839
|
if (
|
|
517
|
-
request.
|
|
840
|
+
request.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
518
841
|
request.batchId.length > MAX_IDENTIFIER_LENGTH ||
|
|
519
842
|
storedTextBytes(request.batchJson) > MAX_STORED_TEXT_BYTES ||
|
|
520
843
|
storedTextBytes(request.batchDigest) > MAX_STORED_TEXT_BYTES ||
|
|
@@ -530,9 +853,11 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
530
853
|
message: "Canonical identifiers or encoded JSON exceed the SQLite storage bounds.",
|
|
531
854
|
});
|
|
532
855
|
}
|
|
856
|
+
|
|
533
857
|
return yield* withWriteTransaction("append transaction")(
|
|
534
858
|
Effect.gen(function* () {
|
|
535
859
|
const recordIds = request.records.map((record) => record.recordId);
|
|
860
|
+
|
|
536
861
|
if (new Set(recordIds).size !== recordIds.length) {
|
|
537
862
|
return yield* SqliteAppendConflict.make({
|
|
538
863
|
message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
|
|
@@ -540,34 +865,35 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
540
865
|
});
|
|
541
866
|
}
|
|
542
867
|
|
|
543
|
-
const
|
|
868
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
544
869
|
SELECT
|
|
545
|
-
|
|
870
|
+
thread_id,
|
|
546
871
|
created_at,
|
|
547
872
|
tail_sequence,
|
|
548
873
|
tail_digest,
|
|
549
874
|
producer_epoch
|
|
550
|
-
FROM
|
|
551
|
-
WHERE
|
|
875
|
+
FROM effect_agent_threads
|
|
876
|
+
WHERE thread_id = ${request.threadId}
|
|
552
877
|
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
878
|
+
|
|
879
|
+
const thread = yield* decodeSingleRow(
|
|
880
|
+
Schema.Array(ThreadRow),
|
|
881
|
+
"effect_agent_threads",
|
|
882
|
+
request.threadId,
|
|
883
|
+
threadRows,
|
|
558
884
|
);
|
|
559
885
|
|
|
560
|
-
if (request.producerEpoch !==
|
|
886
|
+
if (request.producerEpoch !== thread.producer_epoch) {
|
|
561
887
|
return yield* SqliteFenceRejected.make({
|
|
562
888
|
producerEpoch: request.producerEpoch,
|
|
563
|
-
actualEpoch:
|
|
564
|
-
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${
|
|
889
|
+
actualEpoch: thread.producer_epoch,
|
|
890
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`,
|
|
565
891
|
});
|
|
566
892
|
}
|
|
567
893
|
|
|
568
894
|
const batchRows = yield* sql<Record<string, unknown>>`
|
|
569
895
|
SELECT
|
|
570
|
-
|
|
896
|
+
thread_id,
|
|
571
897
|
batch_id,
|
|
572
898
|
first_sequence,
|
|
573
899
|
last_sequence,
|
|
@@ -575,31 +901,34 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
575
901
|
tail_digest,
|
|
576
902
|
batch_json
|
|
577
903
|
FROM effect_agent_canonical_batches
|
|
578
|
-
WHERE
|
|
904
|
+
WHERE thread_id = ${request.threadId}
|
|
579
905
|
AND batch_id = ${request.batchId}
|
|
580
906
|
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
907
|
+
|
|
581
908
|
const batches = yield* decodeRows(
|
|
582
909
|
Schema.Array(BatchRow),
|
|
583
910
|
"effect_agent_canonical_batches",
|
|
584
|
-
`${request.
|
|
911
|
+
`${request.threadId}/${request.batchId}`,
|
|
585
912
|
batchRows,
|
|
586
913
|
);
|
|
587
914
|
|
|
588
915
|
if (batches.length > 1) {
|
|
589
916
|
return yield* SqliteStorageCorruptionError.make({
|
|
590
917
|
table: "effect_agent_canonical_batches",
|
|
591
|
-
rowKey: `${request.
|
|
918
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
592
919
|
message: "A canonical batch primary key returned more than one row.",
|
|
593
920
|
});
|
|
594
921
|
}
|
|
595
922
|
if (batches.length === 1) {
|
|
596
923
|
const existing = batches[0];
|
|
924
|
+
|
|
597
925
|
if (existing.batch_digest !== request.batchDigest) {
|
|
598
926
|
return yield* SqliteAppendConflict.make({
|
|
599
927
|
message: `Batch ${request.batchId} already exists with different canonical content.`,
|
|
600
928
|
reason: "batch-digest",
|
|
601
929
|
});
|
|
602
930
|
}
|
|
931
|
+
|
|
603
932
|
return RawAppendResult.make({
|
|
604
933
|
firstSequence: existing.first_sequence,
|
|
605
934
|
lastSequence: existing.last_sequence,
|
|
@@ -609,43 +938,45 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
609
938
|
}
|
|
610
939
|
|
|
611
940
|
if (
|
|
612
|
-
request.expectedTailSequence !==
|
|
613
|
-
request.expectedTailDigest !==
|
|
941
|
+
request.expectedTailSequence !== thread.tail_sequence ||
|
|
942
|
+
request.expectedTailDigest !== thread.tail_digest
|
|
614
943
|
) {
|
|
615
944
|
return yield* SqliteAppendConflict.make({
|
|
616
945
|
message:
|
|
617
946
|
`Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} ` +
|
|
618
|
-
`but found ${
|
|
947
|
+
`but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
619
948
|
reason: "tail",
|
|
620
|
-
actualTailSequence:
|
|
621
|
-
actualTailDigest:
|
|
949
|
+
actualTailSequence: thread.tail_sequence,
|
|
950
|
+
actualTailDigest: thread.tail_digest,
|
|
622
951
|
});
|
|
623
952
|
}
|
|
624
|
-
if (
|
|
953
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) {
|
|
625
954
|
return yield* SqliteStorageError.make({
|
|
626
955
|
operation: "append canonical batch",
|
|
627
|
-
message: `
|
|
956
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`,
|
|
628
957
|
});
|
|
629
958
|
}
|
|
630
959
|
|
|
631
960
|
const existingRecordRows = yield* sql<Record<string, unknown>>`
|
|
632
961
|
SELECT
|
|
633
|
-
|
|
962
|
+
thread_id,
|
|
634
963
|
sequence,
|
|
635
964
|
record_id,
|
|
636
965
|
batch_id,
|
|
637
966
|
record_json
|
|
638
967
|
FROM effect_agent_canonical_records
|
|
639
|
-
WHERE
|
|
968
|
+
WHERE thread_id = ${request.threadId}
|
|
640
969
|
AND record_id IN ${sql.in(recordIds)}
|
|
641
970
|
ORDER BY sequence
|
|
642
971
|
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
972
|
+
|
|
643
973
|
const existingRecords = yield* decodeRows(
|
|
644
974
|
Schema.Array(RecordRow),
|
|
645
975
|
"effect_agent_canonical_records",
|
|
646
|
-
`${request.
|
|
976
|
+
`${request.threadId}/record_ids`,
|
|
647
977
|
existingRecordRows,
|
|
648
978
|
);
|
|
979
|
+
|
|
649
980
|
if (existingRecords.length > 0) {
|
|
650
981
|
return yield* SqliteAppendConflict.make({
|
|
651
982
|
message: `Canonical record ID ${existingRecords[0].record_id} already exists.`,
|
|
@@ -654,7 +985,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
654
985
|
}
|
|
655
986
|
|
|
656
987
|
const firstSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
657
|
-
|
|
988
|
+
thread.tail_sequence + 1,
|
|
658
989
|
).pipe(
|
|
659
990
|
Effect.mapError((error) =>
|
|
660
991
|
SqliteStorageError.make({
|
|
@@ -664,6 +995,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
664
995
|
}),
|
|
665
996
|
),
|
|
666
997
|
);
|
|
998
|
+
|
|
667
999
|
const lastSequence = yield* Schema.decodeUnknownEffect(CanonicalSequence)(
|
|
668
1000
|
firstSequence + request.records.length - 1,
|
|
669
1001
|
).pipe(
|
|
@@ -678,7 +1010,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
678
1010
|
|
|
679
1011
|
yield* sql`
|
|
680
1012
|
INSERT INTO effect_agent_canonical_batches (
|
|
681
|
-
|
|
1013
|
+
thread_id,
|
|
682
1014
|
batch_id,
|
|
683
1015
|
first_sequence,
|
|
684
1016
|
last_sequence,
|
|
@@ -686,7 +1018,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
686
1018
|
tail_digest,
|
|
687
1019
|
batch_json
|
|
688
1020
|
) VALUES (
|
|
689
|
-
${request.
|
|
1021
|
+
${request.threadId},
|
|
690
1022
|
${request.batchId},
|
|
691
1023
|
${firstSequence},
|
|
692
1024
|
${lastSequence},
|
|
@@ -703,13 +1035,13 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
703
1035
|
Effect.gen(function* () {
|
|
704
1036
|
yield* sql`
|
|
705
1037
|
INSERT INTO effect_agent_canonical_records (
|
|
706
|
-
|
|
1038
|
+
thread_id,
|
|
707
1039
|
sequence,
|
|
708
1040
|
record_id,
|
|
709
1041
|
batch_id,
|
|
710
1042
|
record_json
|
|
711
1043
|
) VALUES (
|
|
712
|
-
${request.
|
|
1044
|
+
${request.threadId},
|
|
713
1045
|
${firstSequence + index},
|
|
714
1046
|
${record.recordId},
|
|
715
1047
|
${request.batchId},
|
|
@@ -722,13 +1054,13 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
722
1054
|
);
|
|
723
1055
|
|
|
724
1056
|
yield* sql`
|
|
725
|
-
UPDATE
|
|
1057
|
+
UPDATE effect_agent_threads
|
|
726
1058
|
SET
|
|
727
1059
|
tail_sequence = ${lastSequence},
|
|
728
1060
|
tail_digest = ${request.tailDigest},
|
|
729
1061
|
producer_epoch = ${request.producerEpoch}
|
|
730
|
-
WHERE
|
|
731
|
-
`.pipe(Effect.mapError(storageError("advance
|
|
1062
|
+
WHERE thread_id = ${request.threadId}
|
|
1063
|
+
`.pipe(Effect.mapError(storageError("advance thread tail")));
|
|
732
1064
|
yield* failpoint("append:after-tail-update");
|
|
733
1065
|
|
|
734
1066
|
return RawAppendResult.make({
|
|
@@ -744,102 +1076,69 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
744
1076
|
const read = Effect.fn("SqliteJournal.read")(function* (request: RawReadRequest) {
|
|
745
1077
|
const rows = yield* sql<Record<string, unknown>>`
|
|
746
1078
|
SELECT
|
|
747
|
-
|
|
1079
|
+
thread_id,
|
|
748
1080
|
sequence,
|
|
749
1081
|
record_id,
|
|
750
1082
|
batch_id,
|
|
751
1083
|
record_json
|
|
752
1084
|
FROM effect_agent_canonical_records
|
|
753
|
-
WHERE
|
|
1085
|
+
WHERE thread_id = ${request.threadId}
|
|
754
1086
|
AND sequence > ${request.fromSequenceExclusive}
|
|
755
1087
|
ORDER BY sequence
|
|
756
1088
|
LIMIT ${request.limit}
|
|
757
1089
|
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
1090
|
+
|
|
758
1091
|
return yield* decodeRows(
|
|
759
1092
|
Schema.Array(RecordRow),
|
|
760
1093
|
"effect_agent_canonical_records",
|
|
761
|
-
`${request.
|
|
1094
|
+
`${request.threadId}>${request.fromSequenceExclusive}`,
|
|
762
1095
|
rows,
|
|
763
1096
|
);
|
|
764
1097
|
});
|
|
765
1098
|
|
|
766
|
-
const
|
|
767
|
-
conversationId: string,
|
|
768
|
-
) {
|
|
1099
|
+
const exportThread = Effect.fn("SqliteJournal.exportThread")(function* (threadId: string) {
|
|
769
1100
|
return yield* withReadTransaction("export transaction")(
|
|
770
1101
|
Effect.gen(function* () {
|
|
771
|
-
const
|
|
1102
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
772
1103
|
SELECT
|
|
773
|
-
|
|
1104
|
+
thread_id,
|
|
774
1105
|
created_at,
|
|
775
1106
|
tail_sequence,
|
|
776
1107
|
tail_digest,
|
|
777
1108
|
producer_epoch
|
|
778
|
-
FROM
|
|
779
|
-
WHERE
|
|
780
|
-
`.pipe(Effect.mapError(storageError("export
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
1109
|
+
FROM effect_agent_threads
|
|
1110
|
+
WHERE thread_id = ${threadId}
|
|
1111
|
+
`.pipe(Effect.mapError(storageError("export thread")));
|
|
1112
|
+
|
|
1113
|
+
const thread = yield* decodeSingleRow(
|
|
1114
|
+
Schema.Array(ThreadRow),
|
|
1115
|
+
"effect_agent_threads",
|
|
1116
|
+
threadId,
|
|
1117
|
+
threadRows,
|
|
786
1118
|
);
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
conversation_id,
|
|
791
|
-
batch_id,
|
|
792
|
-
first_sequence,
|
|
793
|
-
last_sequence,
|
|
794
|
-
batch_digest,
|
|
795
|
-
tail_digest,
|
|
796
|
-
batch_json
|
|
797
|
-
FROM effect_agent_canonical_batches
|
|
798
|
-
WHERE conversation_id = ${conversationId}
|
|
799
|
-
ORDER BY first_sequence
|
|
800
|
-
`.pipe(Effect.mapError(storageError("export canonical batches")));
|
|
1119
|
+
|
|
1120
|
+
yield* failpoint("export:after-thread-read");
|
|
1121
|
+
|
|
801
1122
|
const recordRows = yield* sql<Record<string, unknown>>`
|
|
802
1123
|
SELECT
|
|
803
|
-
|
|
1124
|
+
thread_id,
|
|
804
1125
|
sequence,
|
|
805
1126
|
record_id,
|
|
806
1127
|
batch_id,
|
|
807
1128
|
record_json
|
|
808
1129
|
FROM effect_agent_canonical_records
|
|
809
|
-
WHERE
|
|
1130
|
+
WHERE thread_id = ${threadId}
|
|
810
1131
|
ORDER BY sequence
|
|
811
1132
|
`.pipe(Effect.mapError(storageError("export canonical records")));
|
|
812
|
-
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
813
|
-
SELECT
|
|
814
|
-
conversation_id,
|
|
815
|
-
through_sequence,
|
|
816
|
-
tail_digest,
|
|
817
|
-
checkpoint_json
|
|
818
|
-
FROM effect_agent_checkpoints
|
|
819
|
-
WHERE conversation_id = ${conversationId}
|
|
820
|
-
ORDER BY through_sequence
|
|
821
|
-
`.pipe(Effect.mapError(storageError("export checkpoints")));
|
|
822
1133
|
|
|
823
|
-
return
|
|
824
|
-
|
|
825
|
-
batches: yield* decodeRows(
|
|
826
|
-
Schema.Array(BatchRow),
|
|
827
|
-
"effect_agent_canonical_batches",
|
|
828
|
-
conversationId,
|
|
829
|
-
batchRows,
|
|
830
|
-
),
|
|
1134
|
+
return RawThreadExport.make({
|
|
1135
|
+
thread,
|
|
831
1136
|
records: yield* decodeRows(
|
|
832
1137
|
Schema.Array(RecordRow),
|
|
833
1138
|
"effect_agent_canonical_records",
|
|
834
|
-
|
|
1139
|
+
threadId,
|
|
835
1140
|
recordRows,
|
|
836
1141
|
),
|
|
837
|
-
checkpoints: yield* decodeRows(
|
|
838
|
-
Schema.Array(CheckpointRow),
|
|
839
|
-
"effect_agent_checkpoints",
|
|
840
|
-
conversationId,
|
|
841
|
-
checkpointRows,
|
|
842
|
-
),
|
|
843
1142
|
});
|
|
844
1143
|
}),
|
|
845
1144
|
);
|
|
@@ -849,7 +1148,7 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
849
1148
|
checkpoint: RawCheckpoint,
|
|
850
1149
|
): Effect.fn.Return<void, CheckpointError> {
|
|
851
1150
|
if (
|
|
852
|
-
checkpoint.
|
|
1151
|
+
checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH ||
|
|
853
1152
|
storedTextBytes(checkpoint.checkpointJson) > MAX_STORED_TEXT_BYTES
|
|
854
1153
|
) {
|
|
855
1154
|
return yield* SqliteStorageError.make({
|
|
@@ -859,50 +1158,54 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
859
1158
|
}
|
|
860
1159
|
yield* withWriteTransaction("checkpoint transaction")(
|
|
861
1160
|
Effect.gen(function* () {
|
|
862
|
-
const
|
|
1161
|
+
const threadRows = yield* sql<Record<string, unknown>>`
|
|
863
1162
|
SELECT
|
|
864
|
-
|
|
1163
|
+
thread_id,
|
|
865
1164
|
created_at,
|
|
866
1165
|
tail_sequence,
|
|
867
1166
|
tail_digest,
|
|
868
1167
|
producer_epoch
|
|
869
|
-
FROM
|
|
870
|
-
WHERE
|
|
1168
|
+
FROM effect_agent_threads
|
|
1169
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
871
1170
|
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
1171
|
+
|
|
1172
|
+
const thread = yield* decodeSingleRow(
|
|
1173
|
+
Schema.Array(ThreadRow),
|
|
1174
|
+
"effect_agent_threads",
|
|
1175
|
+
checkpoint.threadId,
|
|
1176
|
+
threadRows,
|
|
877
1177
|
);
|
|
878
|
-
|
|
1178
|
+
|
|
1179
|
+
if (checkpoint.throughSequence > thread.tail_sequence) {
|
|
879
1180
|
return yield* SqliteCheckpointConflict.make({
|
|
880
1181
|
message:
|
|
881
1182
|
`Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ` +
|
|
882
|
-
`${
|
|
1183
|
+
`${thread.tail_sequence}.`,
|
|
883
1184
|
});
|
|
884
1185
|
}
|
|
885
1186
|
|
|
886
1187
|
const checkpointRows = yield* sql<Record<string, unknown>>`
|
|
887
1188
|
SELECT
|
|
888
|
-
|
|
1189
|
+
thread_id,
|
|
889
1190
|
through_sequence,
|
|
890
1191
|
tail_digest,
|
|
891
1192
|
checkpoint_json
|
|
892
1193
|
FROM effect_agent_checkpoints
|
|
893
|
-
WHERE
|
|
1194
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
894
1195
|
AND through_sequence = ${checkpoint.throughSequence}
|
|
895
1196
|
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
1197
|
+
|
|
896
1198
|
const existing = yield* decodeRows(
|
|
897
1199
|
Schema.Array(CheckpointRow),
|
|
898
1200
|
"effect_agent_checkpoints",
|
|
899
|
-
`${checkpoint.
|
|
1201
|
+
`${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
900
1202
|
checkpointRows,
|
|
901
1203
|
);
|
|
1204
|
+
|
|
902
1205
|
if (existing.length > 1) {
|
|
903
1206
|
return yield* SqliteStorageCorruptionError.make({
|
|
904
1207
|
table: "effect_agent_checkpoints",
|
|
905
|
-
rowKey: `${checkpoint.
|
|
1208
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
906
1209
|
message: "A checkpoint primary key returned more than one row.",
|
|
907
1210
|
});
|
|
908
1211
|
}
|
|
@@ -915,17 +1218,18 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
915
1218
|
message: "A different checkpoint already exists at this canonical sequence.",
|
|
916
1219
|
});
|
|
917
1220
|
}
|
|
1221
|
+
|
|
918
1222
|
return;
|
|
919
1223
|
}
|
|
920
1224
|
|
|
921
1225
|
yield* sql`
|
|
922
1226
|
INSERT INTO effect_agent_checkpoints (
|
|
923
|
-
|
|
1227
|
+
thread_id,
|
|
924
1228
|
through_sequence,
|
|
925
1229
|
tail_digest,
|
|
926
1230
|
checkpoint_json
|
|
927
1231
|
) VALUES (
|
|
928
|
-
${checkpoint.
|
|
1232
|
+
${checkpoint.threadId},
|
|
929
1233
|
${checkpoint.throughSequence},
|
|
930
1234
|
${checkpoint.tailDigest},
|
|
931
1235
|
${checkpoint.checkpointJson}
|
|
@@ -936,44 +1240,47 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
936
1240
|
});
|
|
937
1241
|
|
|
938
1242
|
const loadCheckpoint = Effect.fn("SqliteJournal.loadCheckpoint")(function* (
|
|
939
|
-
|
|
1243
|
+
threadId: string,
|
|
940
1244
|
atOrBeforeSequence: CanonicalSequence,
|
|
941
1245
|
) {
|
|
942
1246
|
const rows = yield* sql<Record<string, unknown>>`
|
|
943
1247
|
SELECT
|
|
944
|
-
|
|
1248
|
+
thread_id,
|
|
945
1249
|
through_sequence,
|
|
946
1250
|
tail_digest,
|
|
947
1251
|
checkpoint_json
|
|
948
1252
|
FROM effect_agent_checkpoints
|
|
949
|
-
WHERE
|
|
1253
|
+
WHERE thread_id = ${threadId}
|
|
950
1254
|
AND through_sequence <= ${atOrBeforeSequence}
|
|
951
1255
|
ORDER BY through_sequence DESC
|
|
952
1256
|
LIMIT 1
|
|
953
1257
|
`.pipe(Effect.mapError(storageError("load checkpoint")));
|
|
1258
|
+
|
|
954
1259
|
return yield* decodeRows(
|
|
955
1260
|
Schema.Array(CheckpointRow),
|
|
956
1261
|
"effect_agent_checkpoints",
|
|
957
|
-
`${
|
|
1262
|
+
`${threadId}<=${atOrBeforeSequence}`,
|
|
958
1263
|
rows,
|
|
959
1264
|
);
|
|
960
1265
|
});
|
|
961
1266
|
|
|
962
1267
|
const getTailDigestAt = Effect.fn("SqliteJournal.getTailDigestAt")(function* (
|
|
963
|
-
|
|
1268
|
+
threadId: string,
|
|
964
1269
|
sequence: CanonicalSequence,
|
|
965
1270
|
) {
|
|
966
1271
|
if (sequence === 0) {
|
|
967
|
-
const
|
|
968
|
-
|
|
1272
|
+
const threads = yield* getThread(threadId);
|
|
1273
|
+
|
|
1274
|
+
return threads.length === 0
|
|
969
1275
|
? []
|
|
970
|
-
: [
|
|
1276
|
+
: [threads[0].tail_sequence === 0 ? threads[0].tail_digest : undefined].filter(
|
|
971
1277
|
(value): value is string => value !== undefined,
|
|
972
1278
|
);
|
|
973
1279
|
}
|
|
1280
|
+
|
|
974
1281
|
const rows = yield* sql<Record<string, unknown>>`
|
|
975
1282
|
SELECT
|
|
976
|
-
|
|
1283
|
+
thread_id,
|
|
977
1284
|
batch_id,
|
|
978
1285
|
first_sequence,
|
|
979
1286
|
last_sequence,
|
|
@@ -981,34 +1288,37 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
981
1288
|
tail_digest,
|
|
982
1289
|
batch_json
|
|
983
1290
|
FROM effect_agent_canonical_batches
|
|
984
|
-
WHERE
|
|
1291
|
+
WHERE thread_id = ${threadId}
|
|
985
1292
|
AND last_sequence = ${sequence}
|
|
986
1293
|
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
1294
|
+
|
|
987
1295
|
const batches = yield* decodeRows(
|
|
988
1296
|
Schema.Array(BatchRow),
|
|
989
1297
|
"effect_agent_canonical_batches",
|
|
990
|
-
`${
|
|
1298
|
+
`${threadId}/${sequence}`,
|
|
991
1299
|
rows,
|
|
992
1300
|
);
|
|
1301
|
+
|
|
993
1302
|
return batches.map((batch) => batch.tail_digest);
|
|
994
1303
|
});
|
|
995
1304
|
|
|
996
1305
|
const scanStoredPayloads = Effect.fn("SqliteJournal.scanStoredPayloads")(function* () {
|
|
997
1306
|
return yield* withReadTransaction("startup scan transaction")(
|
|
998
1307
|
Effect.gen(function* () {
|
|
999
|
-
const
|
|
1308
|
+
const threads = yield* sql<Record<string, unknown>>`
|
|
1000
1309
|
SELECT
|
|
1001
|
-
|
|
1310
|
+
thread_id,
|
|
1002
1311
|
created_at,
|
|
1003
1312
|
tail_sequence,
|
|
1004
1313
|
tail_digest,
|
|
1005
1314
|
producer_epoch
|
|
1006
|
-
FROM
|
|
1007
|
-
ORDER BY
|
|
1008
|
-
`.pipe(Effect.mapError(storageError("scan
|
|
1315
|
+
FROM effect_agent_threads
|
|
1316
|
+
ORDER BY thread_id
|
|
1317
|
+
`.pipe(Effect.mapError(storageError("scan threads")));
|
|
1318
|
+
|
|
1009
1319
|
const batches = yield* sql<Record<string, unknown>>`
|
|
1010
1320
|
SELECT
|
|
1011
|
-
|
|
1321
|
+
thread_id,
|
|
1012
1322
|
batch_id,
|
|
1013
1323
|
first_sequence,
|
|
1014
1324
|
last_sequence,
|
|
@@ -1016,33 +1326,36 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
1016
1326
|
tail_digest,
|
|
1017
1327
|
batch_json
|
|
1018
1328
|
FROM effect_agent_canonical_batches
|
|
1019
|
-
ORDER BY
|
|
1329
|
+
ORDER BY thread_id, first_sequence
|
|
1020
1330
|
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
1331
|
+
|
|
1021
1332
|
const records = yield* sql<Record<string, unknown>>`
|
|
1022
1333
|
SELECT
|
|
1023
|
-
|
|
1334
|
+
thread_id,
|
|
1024
1335
|
sequence,
|
|
1025
1336
|
record_id,
|
|
1026
1337
|
batch_id,
|
|
1027
1338
|
record_json
|
|
1028
1339
|
FROM effect_agent_canonical_records
|
|
1029
|
-
ORDER BY
|
|
1340
|
+
ORDER BY thread_id, sequence
|
|
1030
1341
|
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
1342
|
+
|
|
1031
1343
|
const checkpoints = yield* sql<Record<string, unknown>>`
|
|
1032
1344
|
SELECT
|
|
1033
|
-
|
|
1345
|
+
thread_id,
|
|
1034
1346
|
through_sequence,
|
|
1035
1347
|
tail_digest,
|
|
1036
1348
|
checkpoint_json
|
|
1037
1349
|
FROM effect_agent_checkpoints
|
|
1038
|
-
ORDER BY
|
|
1350
|
+
ORDER BY thread_id, through_sequence
|
|
1039
1351
|
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
1352
|
+
|
|
1040
1353
|
return {
|
|
1041
|
-
|
|
1042
|
-
Schema.Array(
|
|
1043
|
-
"
|
|
1354
|
+
threads: yield* decodeRows(
|
|
1355
|
+
Schema.Array(ThreadRow),
|
|
1356
|
+
"effect_agent_threads",
|
|
1044
1357
|
"startup_scan",
|
|
1045
|
-
|
|
1358
|
+
threads,
|
|
1046
1359
|
),
|
|
1047
1360
|
batches: yield* decodeRows(
|
|
1048
1361
|
Schema.Array(BatchRow),
|
|
@@ -1069,8 +1382,8 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
1069
1382
|
|
|
1070
1383
|
return {
|
|
1071
1384
|
append,
|
|
1072
|
-
|
|
1073
|
-
|
|
1385
|
+
exportThread,
|
|
1386
|
+
getThread,
|
|
1074
1387
|
getTailDigestAt,
|
|
1075
1388
|
loadCheckpoint,
|
|
1076
1389
|
materialize,
|
|
@@ -1078,9 +1391,8 @@ const makeJournal = (sql: SqlClient.SqlClient, failpoint: SqliteJournalFailpoint
|
|
|
1078
1391
|
saveCheckpoint,
|
|
1079
1392
|
scanStoredPayloads,
|
|
1080
1393
|
withWriteTransaction,
|
|
1394
|
+
withReadTransaction,
|
|
1081
1395
|
} as const;
|
|
1082
|
-
};
|
|
1083
|
-
|
|
1084
|
-
export type SqliteJournal = ReturnType<typeof makeJournal>;
|
|
1396
|
+
});
|
|
1085
1397
|
|
|
1086
|
-
export
|
|
1398
|
+
export type SqliteJournal = Effect.Success<ReturnType<typeof initializeSqliteJournal>>;
|