@effect-agent/storage-sqlite 0.1.0-beta.9 → 0.1.0-beta.91
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/SqliteActivityStore.d.mts +14 -0
- package/dist/SqliteActivityStore.mjs +310 -0
- package/dist/SqliteActivityStore.mjs.map +1 -0
- package/dist/SqliteMessageDeliveryStore.d.mts +14 -0
- package/dist/SqliteMessageDeliveryStore.mjs +24 -0
- package/dist/SqliteMessageDeliveryStore.mjs.map +1 -0
- package/dist/SqliteScheduleStore.d.mts +14 -0
- package/dist/SqliteScheduleStore.mjs +255 -0
- package/dist/SqliteScheduleStore.mjs.map +1 -0
- package/dist/SqliteStorageConfig.d.mts +34 -0
- package/dist/SqliteStorageConfig.mjs +39 -0
- package/dist/SqliteStorageConfig.mjs.map +1 -0
- package/dist/SqliteStorageError-DVWeaWLm.d.mts +86 -0
- package/dist/SqliteStorageError.d.mts +2 -0
- package/dist/SqliteStorageError.mjs +150 -0
- package/dist/SqliteStorageError.mjs.map +1 -0
- package/dist/SqliteStorageFailpoint.d.mts +17 -0
- package/dist/SqliteStorageFailpoint.mjs +14 -0
- package/dist/SqliteStorageFailpoint.mjs.map +1 -0
- package/dist/SqliteStorageFailpointTesting.d.mts +15 -0
- package/dist/SqliteStorageFailpointTesting.mjs +19 -0
- package/dist/SqliteStorageFailpointTesting.mjs.map +1 -0
- package/dist/SqliteStorageVersion-BqmqX0CI.d.mts +11 -0
- package/dist/SqliteStorageVersion.d.mts +2 -0
- package/dist/SqliteStorageVersion.mjs +8 -0
- package/dist/SqliteStorageVersion.mjs.map +1 -0
- package/dist/SqliteSubmissionLedger.d.mts +23 -0
- package/dist/SqliteSubmissionLedger.mjs +1788 -0
- package/dist/SqliteSubmissionLedger.mjs.map +1 -0
- package/dist/SqliteSubscriptionStore.d.mts +13 -0
- package/dist/SqliteSubscriptionStore.mjs +28 -0
- package/dist/SqliteSubscriptionStore.mjs.map +1 -0
- package/dist/SqliteThreadStore.d.mts +49 -0
- package/dist/SqliteThreadStore.mjs +460 -0
- package/dist/SqliteThreadStore.mjs.map +1 -0
- package/dist/index.d.mts +11 -193
- package/dist/index.mjs +11 -3082
- package/dist/migrations-Dy5v0HGe.mjs +346 -0
- package/dist/migrations-Dy5v0HGe.mjs.map +1 -0
- package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
- package/dist/sqlite-journal-CBOQDySe.mjs +998 -0
- package/dist/sqlite-journal-CBOQDySe.mjs.map +1 -0
- package/package.json +1 -41
- package/src/SqliteActivityStore.ts +556 -0
- package/src/SqliteMessageDeliveryStore.ts +42 -0
- package/src/SqliteScheduleStore.ts +404 -0
- package/src/{sqlite-storage-config.ts → SqliteStorageConfig.ts} +1 -1
- package/src/{errors.ts → SqliteStorageError.ts} +10 -3
- package/src/SqliteStorageFailpoint.ts +23 -0
- package/src/{sqlite-storage-failpoint.ts → SqliteStorageFailpointTesting.ts} +7 -18
- package/src/SqliteStorageVersion.ts +2 -0
- package/src/{sqlite-ledger.ts → SqliteSubmissionLedger.ts} +912 -212
- package/src/SqliteSubscriptionStore.ts +55 -0
- package/src/SqliteThreadStore.ts +999 -0
- package/src/index.ts +10 -6
- package/src/internal/message-delivery-schema.ts +24 -0
- package/src/{migrations.ts → internal/migrations.ts} +156 -79
- package/src/internal/recovery-checkpoint-schema.ts +17 -0
- package/src/{sqlite-journal.ts → internal/sqlite-journal.ts} +751 -241
- package/dist/index.mjs.map +0 -1
- package/src/sqlite-conversation-store.ts +0 -841
|
@@ -0,0 +1,998 @@
|
|
|
1
|
+
import { SqliteStorageConfig } from "./SqliteStorageConfig.mjs";
|
|
2
|
+
import { SqliteAppendConflict, SqliteCheckpointConflict, SqliteFenceRejected, SqliteStorageCompatibilityError, SqliteStorageCorruptionError, SqliteStorageError, SqliteStorageFailpointLocation, SqliteWriteContention } from "./SqliteStorageError.mjs";
|
|
3
|
+
import { SqliteStorageFailpoint } from "./SqliteStorageFailpoint.mjs";
|
|
4
|
+
import { a as createMessageDeliveryTables, i as createRecoveryCheckpointTable, n as createNonterminalIndex, r as sqliteMigrations } from "./migrations-Dy5v0HGe.mjs";
|
|
5
|
+
import { Effect, Exit, Schema } from "effect";
|
|
6
|
+
import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
7
|
+
import { NodeCrypto } from "@effect/platform-node";
|
|
8
|
+
import { SqliteMigrator } from "@effect/sql-sqlite-node";
|
|
9
|
+
import { EMPTY_TAIL_DIGEST } from "effect-agent/digest";
|
|
10
|
+
import { CanonicalSequence, ProducerEpoch } from "effect-agent/records";
|
|
11
|
+
import { ScheduleFailpoint, ScheduleFailpointError } from "effect-agent/schedule";
|
|
12
|
+
import { checkV2ThreadLayout, upgradeV2Schedules, upgradeV2Subscriptions } from "effect-agent/sql-storage-v2-upgrade";
|
|
13
|
+
import { SubscriptionFailpoint, SubscriptionFailpointError } from "effect-agent/subscription";
|
|
14
|
+
import { CheckpointRejected, FenceRejected, MAX_THREAD_EXPORT_RECORDS, ThreadNotMaterialized } from "effect-agent/thread-store";
|
|
15
|
+
//#region src/internal/sqlite-journal.ts
|
|
16
|
+
const BoundedStoredText = Schema.String.check(Schema.isMaxLength(16777216));
|
|
17
|
+
const BoundedIdentifier = Schema.NonEmptyString.check(Schema.isMaxLength(1024));
|
|
18
|
+
const NonNegativeInt = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0));
|
|
19
|
+
const MAX_RECORDS_PER_THREAD = MAX_THREAD_EXPORT_RECORDS;
|
|
20
|
+
const ZERO_SEQUENCE = Schema.decodeSync(CanonicalSequence)(0);
|
|
21
|
+
const MAX_STORED_TEXT_BYTES = 16777216;
|
|
22
|
+
const MAX_IDENTIFIER_LENGTH = 1024;
|
|
23
|
+
const storedTextBytes = (value) => new TextEncoder().encode(value).byteLength;
|
|
24
|
+
var SqliteVersionRow = class extends Schema.Class("SqliteVersionRow")({ user_version: NonNegativeInt }) {};
|
|
25
|
+
var SqliteJournalModeRow = class extends Schema.Class("SqliteJournalModeRow")({ journal_mode: Schema.NonEmptyString.check(Schema.isMaxLength(32)) }) {};
|
|
26
|
+
var SqliteNameRow = class extends Schema.Class("SqliteNameRow")({ name: BoundedIdentifier }) {};
|
|
27
|
+
var ThreadRow = class extends Schema.Class("ThreadRow")({
|
|
28
|
+
thread_id: BoundedIdentifier,
|
|
29
|
+
created_at: Schema.NonEmptyString.check(Schema.isMaxLength(128)),
|
|
30
|
+
producer_epoch: ProducerEpoch,
|
|
31
|
+
tail_digest: BoundedStoredText,
|
|
32
|
+
tail_sequence: CanonicalSequence
|
|
33
|
+
}) {};
|
|
34
|
+
var BatchRow = class extends Schema.Class("BatchRow")({
|
|
35
|
+
batch_digest: BoundedStoredText,
|
|
36
|
+
batch_id: BoundedIdentifier,
|
|
37
|
+
batch_json: BoundedStoredText,
|
|
38
|
+
thread_id: BoundedIdentifier,
|
|
39
|
+
first_sequence: CanonicalSequence,
|
|
40
|
+
last_sequence: CanonicalSequence,
|
|
41
|
+
tail_digest: BoundedStoredText
|
|
42
|
+
}) {};
|
|
43
|
+
var RecordRow = class extends Schema.Class("RecordRow")({
|
|
44
|
+
batch_id: BoundedIdentifier,
|
|
45
|
+
thread_id: BoundedIdentifier,
|
|
46
|
+
record_id: BoundedIdentifier,
|
|
47
|
+
record_json: BoundedStoredText,
|
|
48
|
+
sequence: CanonicalSequence
|
|
49
|
+
}) {};
|
|
50
|
+
var CheckpointRow = class extends Schema.Class("CheckpointRow")({
|
|
51
|
+
checkpoint_json: BoundedStoredText,
|
|
52
|
+
thread_id: BoundedIdentifier,
|
|
53
|
+
tail_digest: BoundedStoredText,
|
|
54
|
+
through_sequence: CanonicalSequence
|
|
55
|
+
}) {};
|
|
56
|
+
var RawRecord = class extends Schema.Class("@effect-agent/storage-sqlite/RawRecord")({
|
|
57
|
+
recordId: BoundedIdentifier,
|
|
58
|
+
recordJson: BoundedStoredText
|
|
59
|
+
}) {};
|
|
60
|
+
var RawAppendRequest = class extends Schema.Class("@effect-agent/storage-sqlite/RawAppendRequest")({
|
|
61
|
+
batchDigest: BoundedStoredText,
|
|
62
|
+
batchId: BoundedIdentifier,
|
|
63
|
+
batchJson: BoundedStoredText,
|
|
64
|
+
threadId: BoundedIdentifier,
|
|
65
|
+
expectedTailDigest: BoundedStoredText,
|
|
66
|
+
expectedTailSequence: CanonicalSequence,
|
|
67
|
+
producerEpoch: ProducerEpoch,
|
|
68
|
+
records: Schema.NonEmptyArray(RawRecord).check(Schema.isMaxLength(256)),
|
|
69
|
+
tailDigest: BoundedStoredText
|
|
70
|
+
}) {};
|
|
71
|
+
var RawAppendResult = class extends Schema.Class("@effect-agent/storage-sqlite/RawAppendResult")({
|
|
72
|
+
firstSequence: CanonicalSequence,
|
|
73
|
+
lastSequence: CanonicalSequence,
|
|
74
|
+
replayed: Schema.Boolean,
|
|
75
|
+
tailDigest: BoundedStoredText
|
|
76
|
+
}) {};
|
|
77
|
+
var RawReadRequest = class extends Schema.Class("@effect-agent/storage-sqlite/RawReadRequest")({
|
|
78
|
+
threadId: BoundedIdentifier,
|
|
79
|
+
fromSequenceExclusive: CanonicalSequence,
|
|
80
|
+
limit: Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(1024))
|
|
81
|
+
}) {};
|
|
82
|
+
var RawCheckpoint = class extends Schema.Class("@effect-agent/storage-sqlite/RawCheckpoint")({
|
|
83
|
+
checkpointJson: BoundedStoredText,
|
|
84
|
+
threadId: BoundedIdentifier,
|
|
85
|
+
tailDigest: BoundedStoredText,
|
|
86
|
+
throughSequence: CanonicalSequence
|
|
87
|
+
}) {};
|
|
88
|
+
var RawThreadExport = class extends Schema.Class("@effect-agent/storage-sqlite/RawThreadExport")({
|
|
89
|
+
thread: ThreadRow,
|
|
90
|
+
records: Schema.Array(RecordRow)
|
|
91
|
+
}) {};
|
|
92
|
+
const storageError = (operation) => (error) => SqliteStorageError.make({
|
|
93
|
+
cause: error,
|
|
94
|
+
operation,
|
|
95
|
+
message: error.message
|
|
96
|
+
});
|
|
97
|
+
/** Decode raw SQLite rows against a Schema, reporting failures as typed corruption. */
|
|
98
|
+
const decodeRows = Effect.fn("SqliteJournal.decodeRows")((schema, table, rowKey, rows) => Schema.decodeUnknownEffect(schema)(rows).pipe(Effect.mapError((error) => SqliteStorageCorruptionError.make({
|
|
99
|
+
table,
|
|
100
|
+
rowKey,
|
|
101
|
+
message: String(error)
|
|
102
|
+
}))));
|
|
103
|
+
/** Decode exactly one raw SQLite row against a Schema, reporting failures as typed corruption. */
|
|
104
|
+
const decodeSingleRow = Effect.fn("SqliteJournal.decodeSingleRow")((schema, table, rowKey, rows) => decodeRows(schema, table, rowKey, rows).pipe(Effect.flatMap((decoded) => decoded.length === 1 ? Effect.succeed(decoded[0]) : Effect.fail(SqliteStorageCorruptionError.make({
|
|
105
|
+
table,
|
|
106
|
+
rowKey,
|
|
107
|
+
message: `Expected exactly one row but found ${decoded.length}.`
|
|
108
|
+
})))));
|
|
109
|
+
/** Column inventory of the supported v8 predecessor, independent of physical column order. */
|
|
110
|
+
const predecessorColumns = {
|
|
111
|
+
effect_agent_threads: [
|
|
112
|
+
"thread_id",
|
|
113
|
+
"created_at",
|
|
114
|
+
"tail_sequence",
|
|
115
|
+
"tail_digest",
|
|
116
|
+
"producer_epoch"
|
|
117
|
+
],
|
|
118
|
+
effect_agent_canonical_batches: [
|
|
119
|
+
"thread_id",
|
|
120
|
+
"batch_id",
|
|
121
|
+
"first_sequence",
|
|
122
|
+
"last_sequence",
|
|
123
|
+
"batch_digest",
|
|
124
|
+
"tail_digest",
|
|
125
|
+
"batch_json"
|
|
126
|
+
],
|
|
127
|
+
effect_agent_canonical_records: [
|
|
128
|
+
"thread_id",
|
|
129
|
+
"sequence",
|
|
130
|
+
"record_id",
|
|
131
|
+
"batch_id",
|
|
132
|
+
"record_json"
|
|
133
|
+
],
|
|
134
|
+
effect_agent_checkpoints: [
|
|
135
|
+
"thread_id",
|
|
136
|
+
"through_sequence",
|
|
137
|
+
"tail_digest",
|
|
138
|
+
"checkpoint_json"
|
|
139
|
+
],
|
|
140
|
+
effect_agent_submissions: [
|
|
141
|
+
"submission_id",
|
|
142
|
+
"thread_id",
|
|
143
|
+
"queue_sequence",
|
|
144
|
+
"principal",
|
|
145
|
+
"idempotency_key",
|
|
146
|
+
"agent_id",
|
|
147
|
+
"agent_digests_json",
|
|
148
|
+
"deployment_id",
|
|
149
|
+
"input_json",
|
|
150
|
+
"input_digest",
|
|
151
|
+
"receipt_id",
|
|
152
|
+
"state",
|
|
153
|
+
"settled_outcome",
|
|
154
|
+
"created_at",
|
|
155
|
+
"ready_at",
|
|
156
|
+
"input_applied_record_id",
|
|
157
|
+
"input_applied_sequence",
|
|
158
|
+
"joined_host_submission_id",
|
|
159
|
+
"suspended_reason_json",
|
|
160
|
+
"suspended_at",
|
|
161
|
+
"unknown_reason",
|
|
162
|
+
"unknown_tool_call_ids_json",
|
|
163
|
+
"parent_submission_id",
|
|
164
|
+
"parent_tool_call_id",
|
|
165
|
+
"admission_group",
|
|
166
|
+
"admission_fence_json"
|
|
167
|
+
],
|
|
168
|
+
effect_agent_submission_ownership: [
|
|
169
|
+
"submission_id",
|
|
170
|
+
"attempt_id",
|
|
171
|
+
"ownership_token",
|
|
172
|
+
"producer_epoch",
|
|
173
|
+
"owner_producer_id",
|
|
174
|
+
"lease_expires_at"
|
|
175
|
+
],
|
|
176
|
+
effect_agent_attempts: [
|
|
177
|
+
"attempt_id",
|
|
178
|
+
"submission_id",
|
|
179
|
+
"thread_id",
|
|
180
|
+
"owner_producer_id",
|
|
181
|
+
"producer_epoch",
|
|
182
|
+
"claimed_at"
|
|
183
|
+
],
|
|
184
|
+
effect_agent_settlement_reservations: [
|
|
185
|
+
"submission_id",
|
|
186
|
+
"settlement_id",
|
|
187
|
+
"outcome",
|
|
188
|
+
"record_id",
|
|
189
|
+
"record_json",
|
|
190
|
+
"record_digest",
|
|
191
|
+
"reserved_at",
|
|
192
|
+
"finalized_at"
|
|
193
|
+
],
|
|
194
|
+
effect_agent_abort_intents: [
|
|
195
|
+
"submission_id",
|
|
196
|
+
"author",
|
|
197
|
+
"reason",
|
|
198
|
+
"requested_at",
|
|
199
|
+
"canonical_record_id"
|
|
200
|
+
],
|
|
201
|
+
effect_agent_approval_decisions: [
|
|
202
|
+
"submission_id",
|
|
203
|
+
"tool_call_id",
|
|
204
|
+
"decision",
|
|
205
|
+
"resolver",
|
|
206
|
+
"reason",
|
|
207
|
+
"decided_at"
|
|
208
|
+
],
|
|
209
|
+
effect_agent_unknown_resolutions: [
|
|
210
|
+
"submission_id",
|
|
211
|
+
"tool_call_id",
|
|
212
|
+
"author",
|
|
213
|
+
"reason",
|
|
214
|
+
"resolution_json",
|
|
215
|
+
"resolved_at"
|
|
216
|
+
],
|
|
217
|
+
effect_agent_child_reservations: [
|
|
218
|
+
"reservation_id",
|
|
219
|
+
"parent_submission_id",
|
|
220
|
+
"parent_tool_call_id",
|
|
221
|
+
"child_submission_id",
|
|
222
|
+
"status",
|
|
223
|
+
"allocation_json",
|
|
224
|
+
"allocation_digest",
|
|
225
|
+
"accounting_json",
|
|
226
|
+
"reserved_at",
|
|
227
|
+
"release_began_at",
|
|
228
|
+
"released_at"
|
|
229
|
+
],
|
|
230
|
+
effect_agent_schedules: [
|
|
231
|
+
"tenant_id",
|
|
232
|
+
"owner_id",
|
|
233
|
+
"schedule_id",
|
|
234
|
+
"deadline_at_millis",
|
|
235
|
+
"record_json"
|
|
236
|
+
],
|
|
237
|
+
effect_agent_subscription_sequences: [
|
|
238
|
+
"tenant_id",
|
|
239
|
+
"source_address",
|
|
240
|
+
"sequence",
|
|
241
|
+
"event_scan_cursor",
|
|
242
|
+
"delivery_scan_cursor",
|
|
243
|
+
"recovery_scan_cursor"
|
|
244
|
+
],
|
|
245
|
+
effect_agent_subscriptions: [
|
|
246
|
+
"tenant_id",
|
|
247
|
+
"source_address",
|
|
248
|
+
"owner_id",
|
|
249
|
+
"subscription_id",
|
|
250
|
+
"ordinal",
|
|
251
|
+
"source_name",
|
|
252
|
+
"source_version",
|
|
253
|
+
"matching_key",
|
|
254
|
+
"state",
|
|
255
|
+
"expires_at_millis",
|
|
256
|
+
"recovery_at_millis",
|
|
257
|
+
"recovery_present",
|
|
258
|
+
"record_json"
|
|
259
|
+
],
|
|
260
|
+
effect_agent_subscription_events: [
|
|
261
|
+
"tenant_id",
|
|
262
|
+
"source_address",
|
|
263
|
+
"event_id",
|
|
264
|
+
"source_name",
|
|
265
|
+
"source_version",
|
|
266
|
+
"matching_key",
|
|
267
|
+
"payload_digest",
|
|
268
|
+
"cutoff",
|
|
269
|
+
"cursor",
|
|
270
|
+
"routing_complete",
|
|
271
|
+
"next_attempt_at_millis",
|
|
272
|
+
"record_json",
|
|
273
|
+
"tombstone"
|
|
274
|
+
],
|
|
275
|
+
effect_agent_subscription_deliveries: [
|
|
276
|
+
"tenant_id",
|
|
277
|
+
"source_address",
|
|
278
|
+
"owner_id",
|
|
279
|
+
"subscription_id",
|
|
280
|
+
"event_id",
|
|
281
|
+
"delivery_key",
|
|
282
|
+
"state",
|
|
283
|
+
"next_attempt_at_millis",
|
|
284
|
+
"record_json"
|
|
285
|
+
]
|
|
286
|
+
};
|
|
287
|
+
const checkPredecessorLayout = Effect.fn("SqliteJournal.checkPredecessorLayout")(function* (version) {
|
|
288
|
+
const sql = yield* SqlClient.SqlClient;
|
|
289
|
+
const expectedColumns = {
|
|
290
|
+
...version === 8 ? predecessorColumns : {
|
|
291
|
+
...predecessorColumns,
|
|
292
|
+
effect_agent_submissions: [
|
|
293
|
+
...predecessorColumns.effect_agent_submissions,
|
|
294
|
+
"worker_admission_json",
|
|
295
|
+
"message_admission_json"
|
|
296
|
+
],
|
|
297
|
+
effect_agent_message_deliveries: [
|
|
298
|
+
"owner_thread_id",
|
|
299
|
+
"message_id",
|
|
300
|
+
"version",
|
|
301
|
+
"state",
|
|
302
|
+
"deadline_at_millis",
|
|
303
|
+
"record_json"
|
|
304
|
+
]
|
|
305
|
+
},
|
|
306
|
+
...version === 10 ? { effect_agent_recovery_checkpoints: [
|
|
307
|
+
"thread_id",
|
|
308
|
+
"through_sequence",
|
|
309
|
+
"tail_digest",
|
|
310
|
+
"checkpoint_json"
|
|
311
|
+
] } : {}
|
|
312
|
+
};
|
|
313
|
+
for (const [table, expected] of Object.entries(expectedColumns)) {
|
|
314
|
+
const columns = yield* decodeRows(Schema.Array(Schema.Struct({ name: BoundedIdentifier })), table, "schema", yield* sql.unsafe(`PRAGMA table_info(${table})`));
|
|
315
|
+
const names = new Set(expected);
|
|
316
|
+
if (columns.length !== names.size || columns.some((column) => !names.has(column.name))) return yield* SqliteStorageCompatibilityError.make({
|
|
317
|
+
actualVersion: version,
|
|
318
|
+
supportedVersion: 11,
|
|
319
|
+
message: `The v${version} ${table} columns do not match the supported predecessor; no upgrade was committed.`
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
});
|
|
323
|
+
const initializeSqliteJournal = Effect.fn("SqliteJournal.initialize")(function* () {
|
|
324
|
+
const sql = yield* SqlClient.SqlClient;
|
|
325
|
+
const { hit: failpoint } = yield* SqliteStorageFailpoint;
|
|
326
|
+
const { busyTimeout } = yield* SqliteStorageConfig;
|
|
327
|
+
yield* sql`PRAGMA foreign_keys = ON`.pipe(Effect.mapError(storageError("enable foreign keys")));
|
|
328
|
+
yield* sql.unsafe(`PRAGMA busy_timeout = ${busyTimeout}`).pipe(Effect.mapError(storageError("configure busy timeout")));
|
|
329
|
+
const journalModeRows = yield* sql`PRAGMA journal_mode`.pipe(Effect.mapError(storageError("read journal mode")));
|
|
330
|
+
const journalMode = yield* decodeSingleRow(Schema.Array(SqliteJournalModeRow), "pragma_journal_mode", "singleton", journalModeRows);
|
|
331
|
+
if (journalMode.journal_mode.toLowerCase() !== "wal") return yield* SqliteStorageCompatibilityError.make({
|
|
332
|
+
actualVersion: 0,
|
|
333
|
+
supportedVersion: 11,
|
|
334
|
+
message: `SQLite WAL mode is required; the database reported ${journalMode.journal_mode}.`
|
|
335
|
+
});
|
|
336
|
+
const versionRows = yield* sql`PRAGMA user_version`.pipe(Effect.mapError(storageError("read storage version")));
|
|
337
|
+
const version = yield* decodeSingleRow(Schema.Array(SqliteVersionRow), "pragma_user_version", "singleton", versionRows);
|
|
338
|
+
if (version.user_version !== 0 && version.user_version !== 7 && version.user_version !== 8 && version.user_version !== 9 && version.user_version !== 10 && version.user_version !== 11) return yield* SqliteStorageCompatibilityError.make({
|
|
339
|
+
actualVersion: version.user_version,
|
|
340
|
+
supportedVersion: 11,
|
|
341
|
+
message: `The SQLite file uses unsupported storage version ${version.user_version}; this build supports exactly version 11. Only supported v7, v8, v9 and v10 can be upgraded automatically. Keep the original file and use a compatible library version.`
|
|
342
|
+
});
|
|
343
|
+
if (version.user_version === 7 || version.user_version === 8 || version.user_version === 9 || version.user_version === 10) yield* sql.withTransaction(Effect.gen(function* () {
|
|
344
|
+
const current = yield* sql`PRAGMA user_version`;
|
|
345
|
+
if (current.length === 1 && current[0].user_version === 11) return;
|
|
346
|
+
if (current.length !== 1 || current[0].user_version !== 7 && current[0].user_version !== 8 && current[0].user_version !== 9 && current[0].user_version !== 10) return yield* SqliteStorageCompatibilityError.make({
|
|
347
|
+
actualVersion: -1,
|
|
348
|
+
supportedVersion: 11,
|
|
349
|
+
message: "Storage version changed while acquiring the upgrade transaction."
|
|
350
|
+
});
|
|
351
|
+
if ((yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name IN (
|
|
352
|
+
'effect_agent_threads', 'effect_agent_canonical_batches', 'effect_agent_canonical_records',
|
|
353
|
+
'effect_agent_checkpoints', 'effect_agent_submissions', 'effect_agent_submission_ownership',
|
|
354
|
+
'effect_agent_attempts', 'effect_agent_settlement_reservations', 'effect_agent_abort_intents',
|
|
355
|
+
'effect_agent_approval_decisions', 'effect_agent_unknown_resolutions', 'effect_agent_schedules'
|
|
356
|
+
)`).length !== 12) return yield* SqliteStorageCompatibilityError.make({
|
|
357
|
+
actualVersion: current[0].user_version,
|
|
358
|
+
supportedVersion: 11,
|
|
359
|
+
message: "The predecessor store is missing required tables; no upgrade was committed."
|
|
360
|
+
});
|
|
361
|
+
if ((yield* sql`SELECT name FROM sqlite_master WHERE type='table' AND name='effect_agent_recovery_checkpoints'`).length !== (current[0].user_version === 10 ? 1 : 0)) return yield* SqliteStorageCompatibilityError.make({
|
|
362
|
+
actualVersion: current[0].user_version,
|
|
363
|
+
supportedVersion: 11,
|
|
364
|
+
message: "The predecessor recovery checkpoint storage does not match its version; refusing ambiguous data without mutation."
|
|
365
|
+
});
|
|
366
|
+
if ((yield* sql`SELECT name FROM sqlite_master WHERE name='effect_agent_submissions_nonterminal'`).length !== 0) return yield* SqliteStorageCompatibilityError.make({
|
|
367
|
+
actualVersion: current[0].user_version,
|
|
368
|
+
supportedVersion: 11,
|
|
369
|
+
message: "The predecessor already contains the nonterminal index; refusing ambiguous storage without mutation."
|
|
370
|
+
});
|
|
371
|
+
if (current[0].user_version === 7) {
|
|
372
|
+
yield* checkV2ThreadLayout();
|
|
373
|
+
for (const statement of [
|
|
374
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_group TEXT`,
|
|
375
|
+
sql`ALTER TABLE effect_agent_submissions ADD COLUMN admission_fence_json TEXT`,
|
|
376
|
+
sql`CREATE INDEX effect_agent_submissions_group ON effect_agent_submissions (thread_id, admission_group, state)`
|
|
377
|
+
]) {
|
|
378
|
+
yield* failpoint("upgrade:before-mutation");
|
|
379
|
+
yield* statement;
|
|
380
|
+
yield* failpoint("upgrade:after-mutation");
|
|
381
|
+
}
|
|
382
|
+
yield* upgradeV2Schedules(16777216).pipe(Effect.provideService(ScheduleFailpoint, { hit: (point) => Schema.decodeUnknownEffect(SqliteStorageFailpointLocation)(point).pipe(Effect.flatMap(failpoint), Effect.mapError(() => ScheduleFailpointError.make({ point }))) }));
|
|
383
|
+
yield* upgradeV2Subscriptions(16777216).pipe(Effect.provideService(SubscriptionFailpoint, { hit: (point) => Schema.decodeUnknownEffect(SqliteStorageFailpointLocation)(point).pipe(Effect.flatMap(failpoint), Effect.mapError(() => SubscriptionFailpointError.make({ point }))) }));
|
|
384
|
+
}
|
|
385
|
+
if (current[0].user_version === 8 || current[0].user_version === 9 || current[0].user_version === 10) yield* checkPredecessorLayout(current[0].user_version);
|
|
386
|
+
if (current[0].user_version === 7 || current[0].user_version === 8) {
|
|
387
|
+
yield* failpoint("upgrade:before-mutation");
|
|
388
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN worker_admission_json TEXT`;
|
|
389
|
+
yield* failpoint("upgrade:after-mutation");
|
|
390
|
+
yield* failpoint("upgrade:before-mutation");
|
|
391
|
+
yield* sql`ALTER TABLE effect_agent_submissions ADD COLUMN message_admission_json TEXT`;
|
|
392
|
+
yield* failpoint("upgrade:after-mutation");
|
|
393
|
+
yield* failpoint("upgrade:before-mutation");
|
|
394
|
+
yield* createMessageDeliveryTables;
|
|
395
|
+
yield* failpoint("upgrade:after-mutation");
|
|
396
|
+
}
|
|
397
|
+
if (current[0].user_version !== 10) {
|
|
398
|
+
yield* failpoint("upgrade:before-mutation");
|
|
399
|
+
yield* createRecoveryCheckpointTable;
|
|
400
|
+
yield* failpoint("upgrade:after-mutation");
|
|
401
|
+
}
|
|
402
|
+
yield* failpoint("upgrade:before-mutation");
|
|
403
|
+
yield* createNonterminalIndex;
|
|
404
|
+
yield* failpoint("upgrade:after-mutation");
|
|
405
|
+
yield* failpoint("upgrade:before-version");
|
|
406
|
+
yield* sql`PRAGMA user_version = 11`;
|
|
407
|
+
yield* failpoint("upgrade:after-version");
|
|
408
|
+
})).pipe(Effect.provide(NodeCrypto.layer), Effect.catchTag("SqliteStorageFailpointError", (error) => SqliteStorageError.make({
|
|
409
|
+
cause: error,
|
|
410
|
+
operation: "upgrade storage",
|
|
411
|
+
message: error.message
|
|
412
|
+
})), Effect.catchTag(["ScheduleFailpointError", "SubscriptionFailpointError"], (error) => SqliteStorageError.make({
|
|
413
|
+
cause: error,
|
|
414
|
+
operation: "upgrade storage",
|
|
415
|
+
message: "Injected storage upgrade failure"
|
|
416
|
+
})), Effect.catchTag("StorageUpgradeError", (error) => SqliteStorageCorruptionError.make({
|
|
417
|
+
table: error.table,
|
|
418
|
+
rowKey: error.rowKey,
|
|
419
|
+
message: error.message
|
|
420
|
+
})), Effect.catchTag("SqlError", storageError("upgrade supported storage")), Effect.catchTag("SchemaError", (error) => SqliteStorageCorruptionError.make({
|
|
421
|
+
table: "upgrade",
|
|
422
|
+
rowKey: "v7",
|
|
423
|
+
message: error.message
|
|
424
|
+
})));
|
|
425
|
+
if (version.user_version === 0) {
|
|
426
|
+
const existingRows = yield* sql`
|
|
427
|
+
SELECT name
|
|
428
|
+
FROM sqlite_master
|
|
429
|
+
WHERE type = 'table'
|
|
430
|
+
AND name LIKE 'effect_agent_%'
|
|
431
|
+
ORDER BY name
|
|
432
|
+
`.pipe(Effect.mapError(storageError("inspect unversioned storage")));
|
|
433
|
+
if ((yield* decodeRows(Schema.Array(SqliteNameRow), "sqlite_master", "effect_agent_%", existingRows)).length > 0) return yield* SqliteStorageCompatibilityError.make({
|
|
434
|
+
actualVersion: 0,
|
|
435
|
+
supportedVersion: 11,
|
|
436
|
+
message: "The SQLite file contains unversioned Effect Agent tables. Refusing to mutate ambiguous stored data; retain it for inspection with its original writer."
|
|
437
|
+
});
|
|
438
|
+
yield* SqliteMigrator.run({ loader: sqliteMigrations }).pipe(Effect.mapError((error) => SqliteStorageError.make({
|
|
439
|
+
cause: error,
|
|
440
|
+
operation: "initialize current storage",
|
|
441
|
+
message: error.message
|
|
442
|
+
})));
|
|
443
|
+
}
|
|
444
|
+
const requiredRows = yield* sql`
|
|
445
|
+
SELECT name
|
|
446
|
+
FROM sqlite_master
|
|
447
|
+
WHERE (type = 'table'
|
|
448
|
+
AND name IN (
|
|
449
|
+
'effect_agent_threads',
|
|
450
|
+
'effect_agent_canonical_batches',
|
|
451
|
+
'effect_agent_canonical_records',
|
|
452
|
+
'effect_agent_checkpoints',
|
|
453
|
+
'effect_agent_submissions',
|
|
454
|
+
'effect_agent_submission_ownership',
|
|
455
|
+
'effect_agent_attempts',
|
|
456
|
+
'effect_agent_settlement_reservations',
|
|
457
|
+
'effect_agent_abort_intents',
|
|
458
|
+
'effect_agent_approval_decisions',
|
|
459
|
+
'effect_agent_unknown_resolutions',
|
|
460
|
+
'effect_agent_schedules',
|
|
461
|
+
'effect_agent_message_deliveries',
|
|
462
|
+
'effect_agent_recovery_checkpoints'
|
|
463
|
+
)) OR (type = 'index' AND name = 'effect_agent_submissions_nonterminal')
|
|
464
|
+
ORDER BY name
|
|
465
|
+
`.pipe(Effect.mapError(storageError("verify storage tables")));
|
|
466
|
+
if ((yield* decodeRows(Schema.Array(SqliteNameRow), "sqlite_master", "required_tables", requiredRows)).length !== 15) return yield* SqliteStorageCompatibilityError.make({
|
|
467
|
+
actualVersion: 11,
|
|
468
|
+
supportedVersion: 11,
|
|
469
|
+
message: "The SQLite file claims the current format but is missing required tables or its nonterminal index. Retain the original store for inspection."
|
|
470
|
+
});
|
|
471
|
+
const classifyWriteFailure = (operation) => (error) => error.reason._tag === "LockTimeoutError" ? SqliteWriteContention.make({
|
|
472
|
+
cause: error,
|
|
473
|
+
operation,
|
|
474
|
+
message: `Another producer holds the SQLite write lock; ${operation} is safe to retry.`
|
|
475
|
+
}) : storageError(operation)(error);
|
|
476
|
+
/**
|
|
477
|
+
* Runs one journal write transaction under `BEGIN IMMEDIATE`. SQLite's deferred `BEGIN`
|
|
478
|
+
* would let a read-then-write transaction start as a reader and fail with
|
|
479
|
+
* SQLITE_BUSY_SNAPSHOT on upgrade, which `busy_timeout` never retries. Taking the write
|
|
480
|
+
* lock up front keeps cross-owner contention inside the bounded busy retry; a lock
|
|
481
|
+
* timeout is classified as the retryable SqliteWriteContention. A failed `BEGIN` leaves
|
|
482
|
+
* no transaction, so no rollback is attempted for it.
|
|
483
|
+
*
|
|
484
|
+
* Journal write transactions are always top level. Nesting one inside another would
|
|
485
|
+
* deadlock the single-connection client, so new journal operations must not wrap this
|
|
486
|
+
* helper inside another transaction.
|
|
487
|
+
*/
|
|
488
|
+
const withWriteTransaction = (operation) => (effect) => Effect.uninterruptibleMask((restore) => Effect.scoped(Effect.gen(function* () {
|
|
489
|
+
const connection = yield* sql.reserve.pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
490
|
+
yield* connection.executeUnprepared("BEGIN IMMEDIATE", [], void 0).pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
491
|
+
const exit = yield* restore(Effect.provideService(effect, sql.transactionService, [connection, 0])).pipe(Effect.exit);
|
|
492
|
+
if (Exit.isSuccess(exit)) {
|
|
493
|
+
yield* connection.executeUnprepared("COMMIT", [], void 0).pipe(Effect.mapError(classifyWriteFailure(operation)));
|
|
494
|
+
return exit.value;
|
|
495
|
+
}
|
|
496
|
+
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], void 0));
|
|
497
|
+
return yield* exit;
|
|
498
|
+
})).pipe(Effect.withSpan("SqliteJournal.withWriteTransaction", { attributes: { operation } })));
|
|
499
|
+
/**
|
|
500
|
+
* Runs a read-only snapshot under a deferred transaction. Effect's SQLite client now
|
|
501
|
+
* starts every writable-client `withTransaction` with `BEGIN IMMEDIATE`, which is the
|
|
502
|
+
* right default for mutations but would make exports take the write lock and block a
|
|
503
|
+
* concurrent append. Reserving the connection and beginning explicitly preserves the
|
|
504
|
+
* adapter's snapshot-with-concurrent-writer contract. As with the write helper, a failed
|
|
505
|
+
* `BEGIN` is reported directly because there is no transaction to roll back.
|
|
506
|
+
*/
|
|
507
|
+
const withReadTransaction = (operation) => (effect) => Effect.uninterruptibleMask((restore) => Effect.scoped(Effect.gen(function* () {
|
|
508
|
+
const connection = yield* sql.reserve.pipe(Effect.mapError(storageError(operation)));
|
|
509
|
+
yield* connection.executeUnprepared("BEGIN", [], void 0).pipe(Effect.mapError(storageError(operation)));
|
|
510
|
+
const exit = yield* restore(Effect.provideService(effect, sql.transactionService, [connection, 0])).pipe(Effect.exit);
|
|
511
|
+
if (Exit.isSuccess(exit)) {
|
|
512
|
+
yield* connection.executeUnprepared("COMMIT", [], void 0).pipe(Effect.mapError(storageError(operation)));
|
|
513
|
+
return exit.value;
|
|
514
|
+
}
|
|
515
|
+
yield* Effect.orDie(connection.executeUnprepared("ROLLBACK", [], void 0));
|
|
516
|
+
return yield* exit;
|
|
517
|
+
})).pipe(Effect.withSpan("SqliteJournal.withReadTransaction", { attributes: { operation } })));
|
|
518
|
+
const materialize = Effect.fn("SqliteJournal.materialize")(function* (threadId, createdAt, emptyTailDigest, producerEpoch) {
|
|
519
|
+
if (threadId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(emptyTailDigest) > MAX_STORED_TEXT_BYTES) return yield* SqliteStorageError.make({
|
|
520
|
+
operation: "materialize thread",
|
|
521
|
+
message: "Thread identity or initial digest exceeds the SQLite storage bounds."
|
|
522
|
+
});
|
|
523
|
+
yield* withWriteTransaction("materialize transaction")(Effect.gen(function* () {
|
|
524
|
+
const existingRows = yield* sql`
|
|
525
|
+
SELECT
|
|
526
|
+
thread_id,
|
|
527
|
+
created_at,
|
|
528
|
+
tail_sequence,
|
|
529
|
+
tail_digest,
|
|
530
|
+
producer_epoch
|
|
531
|
+
FROM effect_agent_threads
|
|
532
|
+
WHERE thread_id = ${threadId}
|
|
533
|
+
`.pipe(Effect.mapError(storageError("read materialized thread")));
|
|
534
|
+
const existing = yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, existingRows);
|
|
535
|
+
if (existing.length > 1) return yield* SqliteStorageCorruptionError.make({
|
|
536
|
+
table: "effect_agent_threads",
|
|
537
|
+
rowKey: threadId,
|
|
538
|
+
message: "A thread primary key returned more than one row."
|
|
539
|
+
});
|
|
540
|
+
if (existing.length === 0) {
|
|
541
|
+
yield* sql`
|
|
542
|
+
INSERT INTO effect_agent_threads (
|
|
543
|
+
thread_id,
|
|
544
|
+
created_at,
|
|
545
|
+
tail_sequence,
|
|
546
|
+
tail_digest,
|
|
547
|
+
producer_epoch
|
|
548
|
+
) VALUES (
|
|
549
|
+
${threadId},
|
|
550
|
+
${createdAt},
|
|
551
|
+
0,
|
|
552
|
+
${emptyTailDigest},
|
|
553
|
+
${producerEpoch}
|
|
554
|
+
)
|
|
555
|
+
`.pipe(Effect.mapError(storageError("materialize thread")));
|
|
556
|
+
return;
|
|
557
|
+
}
|
|
558
|
+
if (producerEpoch < existing[0].producer_epoch) return yield* SqliteFenceRejected.make({
|
|
559
|
+
producerEpoch,
|
|
560
|
+
actualEpoch: existing[0].producer_epoch,
|
|
561
|
+
message: `Producer epoch ${producerEpoch} is stale; current epoch is ${existing[0].producer_epoch}.`
|
|
562
|
+
});
|
|
563
|
+
if (producerEpoch > existing[0].producer_epoch) yield* sql`
|
|
564
|
+
UPDATE effect_agent_threads
|
|
565
|
+
SET producer_epoch = ${producerEpoch}
|
|
566
|
+
WHERE thread_id = ${threadId}
|
|
567
|
+
`.pipe(Effect.mapError(storageError("advance materialization epoch")));
|
|
568
|
+
}));
|
|
569
|
+
});
|
|
570
|
+
const getThread = Effect.fn("SqliteJournal.getThread")(function* (threadId) {
|
|
571
|
+
const rows = yield* sql`
|
|
572
|
+
SELECT
|
|
573
|
+
thread_id,
|
|
574
|
+
created_at,
|
|
575
|
+
tail_sequence,
|
|
576
|
+
tail_digest,
|
|
577
|
+
producer_epoch
|
|
578
|
+
FROM effect_agent_threads
|
|
579
|
+
WHERE thread_id = ${threadId}
|
|
580
|
+
`.pipe(Effect.mapError(storageError("read thread")));
|
|
581
|
+
return yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", threadId, rows);
|
|
582
|
+
});
|
|
583
|
+
const append = Effect.fn("SqliteJournal.append")(function* (request) {
|
|
584
|
+
if (request.threadId.length > MAX_IDENTIFIER_LENGTH || request.batchId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(request.batchJson) > MAX_STORED_TEXT_BYTES || storedTextBytes(request.batchDigest) > MAX_STORED_TEXT_BYTES || storedTextBytes(request.tailDigest) > MAX_STORED_TEXT_BYTES || request.records.some((record) => record.recordId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(record.recordJson) > MAX_STORED_TEXT_BYTES)) return yield* SqliteStorageError.make({
|
|
585
|
+
operation: "append canonical batch",
|
|
586
|
+
message: "Canonical identifiers or encoded JSON exceed the SQLite storage bounds."
|
|
587
|
+
});
|
|
588
|
+
return yield* withWriteTransaction("append transaction")(Effect.gen(function* () {
|
|
589
|
+
const recordIds = request.records.map((record) => record.recordId);
|
|
590
|
+
if (new Set(recordIds).size !== recordIds.length) return yield* SqliteAppendConflict.make({
|
|
591
|
+
message: `Batch ${request.batchId} contains duplicate canonical record IDs.`,
|
|
592
|
+
reason: "record-identity"
|
|
593
|
+
});
|
|
594
|
+
const threadRows = yield* sql`
|
|
595
|
+
SELECT
|
|
596
|
+
thread_id,
|
|
597
|
+
created_at,
|
|
598
|
+
tail_sequence,
|
|
599
|
+
tail_digest,
|
|
600
|
+
producer_epoch
|
|
601
|
+
FROM effect_agent_threads
|
|
602
|
+
WHERE thread_id = ${request.threadId}
|
|
603
|
+
`.pipe(Effect.mapError(storageError("read append tail")));
|
|
604
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", request.threadId, threadRows);
|
|
605
|
+
if (request.producerEpoch !== thread.producer_epoch) return yield* SqliteFenceRejected.make({
|
|
606
|
+
producerEpoch: request.producerEpoch,
|
|
607
|
+
actualEpoch: thread.producer_epoch,
|
|
608
|
+
message: `Producer epoch ${request.producerEpoch} is not the current epoch ${thread.producer_epoch}.`
|
|
609
|
+
});
|
|
610
|
+
const batchRows = yield* sql`
|
|
611
|
+
SELECT
|
|
612
|
+
thread_id,
|
|
613
|
+
batch_id,
|
|
614
|
+
first_sequence,
|
|
615
|
+
last_sequence,
|
|
616
|
+
batch_digest,
|
|
617
|
+
tail_digest,
|
|
618
|
+
batch_json
|
|
619
|
+
FROM effect_agent_canonical_batches
|
|
620
|
+
WHERE thread_id = ${request.threadId}
|
|
621
|
+
AND batch_id = ${request.batchId}
|
|
622
|
+
`.pipe(Effect.mapError(storageError("read idempotent batch")));
|
|
623
|
+
const batches = yield* decodeRows(Schema.Array(BatchRow), "effect_agent_canonical_batches", `${request.threadId}/${request.batchId}`, batchRows);
|
|
624
|
+
if (batches.length > 1) return yield* SqliteStorageCorruptionError.make({
|
|
625
|
+
table: "effect_agent_canonical_batches",
|
|
626
|
+
rowKey: `${request.threadId}/${request.batchId}`,
|
|
627
|
+
message: "A canonical batch primary key returned more than one row."
|
|
628
|
+
});
|
|
629
|
+
if (batches.length === 1) {
|
|
630
|
+
const existing = batches[0];
|
|
631
|
+
if (existing.batch_digest !== request.batchDigest) return yield* SqliteAppendConflict.make({
|
|
632
|
+
message: `Batch ${request.batchId} already exists with different canonical content.`,
|
|
633
|
+
reason: "batch-digest"
|
|
634
|
+
});
|
|
635
|
+
return RawAppendResult.make({
|
|
636
|
+
firstSequence: existing.first_sequence,
|
|
637
|
+
lastSequence: existing.last_sequence,
|
|
638
|
+
replayed: true,
|
|
639
|
+
tailDigest: existing.tail_digest
|
|
640
|
+
});
|
|
641
|
+
}
|
|
642
|
+
if (request.expectedTailSequence !== thread.tail_sequence || request.expectedTailDigest !== thread.tail_digest) return yield* SqliteAppendConflict.make({
|
|
643
|
+
message: `Expected tail ${request.expectedTailSequence}/${request.expectedTailDigest} but found ${thread.tail_sequence}/${thread.tail_digest}.`,
|
|
644
|
+
reason: "tail",
|
|
645
|
+
actualTailSequence: thread.tail_sequence,
|
|
646
|
+
actualTailDigest: thread.tail_digest
|
|
647
|
+
});
|
|
648
|
+
if (thread.tail_sequence + request.records.length > MAX_RECORDS_PER_THREAD) return yield* SqliteStorageError.make({
|
|
649
|
+
operation: "append canonical batch",
|
|
650
|
+
message: `Thread record limit ${MAX_RECORDS_PER_THREAD} would be exceeded.`
|
|
651
|
+
});
|
|
652
|
+
const existingRecordRows = yield* sql`
|
|
653
|
+
SELECT
|
|
654
|
+
thread_id,
|
|
655
|
+
sequence,
|
|
656
|
+
record_id,
|
|
657
|
+
batch_id,
|
|
658
|
+
record_json
|
|
659
|
+
FROM effect_agent_canonical_records
|
|
660
|
+
WHERE thread_id = ${request.threadId}
|
|
661
|
+
AND record_id IN ${sql.in(recordIds)}
|
|
662
|
+
ORDER BY sequence
|
|
663
|
+
`.pipe(Effect.mapError(storageError("check canonical record identities")));
|
|
664
|
+
const existingRecords = yield* decodeRows(Schema.Array(RecordRow), "effect_agent_canonical_records", `${request.threadId}/record_ids`, existingRecordRows);
|
|
665
|
+
if (existingRecords.length > 0) return yield* SqliteAppendConflict.make({
|
|
666
|
+
message: `Canonical record ID ${existingRecords[0].record_id} already exists.`,
|
|
667
|
+
reason: "record-identity"
|
|
668
|
+
});
|
|
669
|
+
const firstSequence = yield* Schema.decodeEffect(CanonicalSequence)(thread.tail_sequence + 1).pipe(Effect.mapError((error) => SqliteStorageError.make({
|
|
670
|
+
cause: error,
|
|
671
|
+
operation: "append canonical batch",
|
|
672
|
+
message: error.message
|
|
673
|
+
})));
|
|
674
|
+
const lastSequence = yield* Schema.decodeEffect(CanonicalSequence)(firstSequence + request.records.length - 1).pipe(Effect.mapError((error) => SqliteStorageError.make({
|
|
675
|
+
cause: error,
|
|
676
|
+
operation: "append canonical batch",
|
|
677
|
+
message: error.message
|
|
678
|
+
})));
|
|
679
|
+
yield* sql`
|
|
680
|
+
INSERT INTO effect_agent_canonical_batches (
|
|
681
|
+
thread_id,
|
|
682
|
+
batch_id,
|
|
683
|
+
first_sequence,
|
|
684
|
+
last_sequence,
|
|
685
|
+
batch_digest,
|
|
686
|
+
tail_digest,
|
|
687
|
+
batch_json
|
|
688
|
+
) VALUES (
|
|
689
|
+
${request.threadId},
|
|
690
|
+
${request.batchId},
|
|
691
|
+
${firstSequence},
|
|
692
|
+
${lastSequence},
|
|
693
|
+
${request.batchDigest},
|
|
694
|
+
${request.tailDigest},
|
|
695
|
+
${request.batchJson}
|
|
696
|
+
)
|
|
697
|
+
`.pipe(Effect.mapError(storageError("insert canonical batch")));
|
|
698
|
+
yield* failpoint("append:after-batch-insert");
|
|
699
|
+
yield* Effect.forEach(request.records, (record, index) => Effect.gen(function* () {
|
|
700
|
+
yield* sql`
|
|
701
|
+
INSERT INTO effect_agent_canonical_records (
|
|
702
|
+
thread_id,
|
|
703
|
+
sequence,
|
|
704
|
+
record_id,
|
|
705
|
+
batch_id,
|
|
706
|
+
record_json
|
|
707
|
+
) VALUES (
|
|
708
|
+
${request.threadId},
|
|
709
|
+
${firstSequence + index},
|
|
710
|
+
${record.recordId},
|
|
711
|
+
${request.batchId},
|
|
712
|
+
${record.recordJson}
|
|
713
|
+
)
|
|
714
|
+
`.pipe(Effect.mapError(storageError("insert canonical record")));
|
|
715
|
+
yield* failpoint("append:after-record-insert");
|
|
716
|
+
}), { discard: true });
|
|
717
|
+
yield* sql`
|
|
718
|
+
UPDATE effect_agent_threads
|
|
719
|
+
SET
|
|
720
|
+
tail_sequence = ${lastSequence},
|
|
721
|
+
tail_digest = ${request.tailDigest},
|
|
722
|
+
producer_epoch = ${request.producerEpoch}
|
|
723
|
+
WHERE thread_id = ${request.threadId}
|
|
724
|
+
`.pipe(Effect.mapError(storageError("advance thread tail")));
|
|
725
|
+
yield* failpoint("append:after-tail-update");
|
|
726
|
+
return RawAppendResult.make({
|
|
727
|
+
firstSequence,
|
|
728
|
+
lastSequence,
|
|
729
|
+
replayed: false,
|
|
730
|
+
tailDigest: request.tailDigest
|
|
731
|
+
});
|
|
732
|
+
}));
|
|
733
|
+
});
|
|
734
|
+
const read = Effect.fn("SqliteJournal.read")(function* (request) {
|
|
735
|
+
const rows = yield* sql`
|
|
736
|
+
SELECT
|
|
737
|
+
thread_id,
|
|
738
|
+
sequence,
|
|
739
|
+
record_id,
|
|
740
|
+
batch_id,
|
|
741
|
+
record_json
|
|
742
|
+
FROM effect_agent_canonical_records
|
|
743
|
+
WHERE thread_id = ${request.threadId}
|
|
744
|
+
AND sequence > ${request.fromSequenceExclusive}
|
|
745
|
+
ORDER BY sequence
|
|
746
|
+
LIMIT ${request.limit}
|
|
747
|
+
`.pipe(Effect.mapError(storageError("read canonical records")));
|
|
748
|
+
return yield* decodeRows(Schema.Array(RecordRow), "effect_agent_canonical_records", `${request.threadId}>${request.fromSequenceExclusive}`, rows);
|
|
749
|
+
});
|
|
750
|
+
const exportThread = Effect.fn("SqliteJournal.exportThread")(function* (threadId) {
|
|
751
|
+
return yield* withReadTransaction("export transaction")(Effect.gen(function* () {
|
|
752
|
+
const threadRows = yield* sql`
|
|
753
|
+
SELECT
|
|
754
|
+
thread_id,
|
|
755
|
+
created_at,
|
|
756
|
+
tail_sequence,
|
|
757
|
+
tail_digest,
|
|
758
|
+
producer_epoch
|
|
759
|
+
FROM effect_agent_threads
|
|
760
|
+
WHERE thread_id = ${threadId}
|
|
761
|
+
`.pipe(Effect.mapError(storageError("export thread")));
|
|
762
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", threadId, threadRows);
|
|
763
|
+
yield* failpoint("export:after-thread-read");
|
|
764
|
+
if (thread.tail_sequence > MAX_RECORDS_PER_THREAD) return yield* SqliteStorageError.make({
|
|
765
|
+
operation: "export thread",
|
|
766
|
+
message: "The thread exceeds the current export record limit."
|
|
767
|
+
});
|
|
768
|
+
const records = [];
|
|
769
|
+
let afterSequence = ZERO_SEQUENCE;
|
|
770
|
+
while (afterSequence < thread.tail_sequence) {
|
|
771
|
+
const limit = Math.min(1024, thread.tail_sequence - afterSequence);
|
|
772
|
+
const request = RawReadRequest.make({
|
|
773
|
+
threadId,
|
|
774
|
+
fromSequenceExclusive: afterSequence,
|
|
775
|
+
limit
|
|
776
|
+
});
|
|
777
|
+
const page = yield* read(request);
|
|
778
|
+
if (page.length !== limit || page.some((record, index) => record.sequence !== afterSequence + index + 1)) return yield* SqliteStorageCorruptionError.make({
|
|
779
|
+
table: "effect_agent_canonical_records",
|
|
780
|
+
rowKey: threadId,
|
|
781
|
+
message: "The exported canonical prefix is not contiguous through its captured tail."
|
|
782
|
+
});
|
|
783
|
+
records.push(...page);
|
|
784
|
+
afterSequence = page[page.length - 1].sequence;
|
|
785
|
+
}
|
|
786
|
+
if ((yield* sql`SELECT sequence FROM effect_agent_canonical_records WHERE thread_id=${threadId} AND sequence > ${thread.tail_sequence} LIMIT 1`.pipe(Effect.mapError(storageError("verify export tail")))).length !== 0) return yield* SqliteStorageCorruptionError.make({
|
|
787
|
+
table: "effect_agent_canonical_records",
|
|
788
|
+
rowKey: threadId,
|
|
789
|
+
message: "Canonical records exist beyond the captured thread tail."
|
|
790
|
+
});
|
|
791
|
+
return RawThreadExport.make({
|
|
792
|
+
thread,
|
|
793
|
+
records
|
|
794
|
+
});
|
|
795
|
+
}));
|
|
796
|
+
});
|
|
797
|
+
const saveCheckpoint = Effect.fn("SqliteJournal.saveCheckpoint")(function* (checkpoint) {
|
|
798
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(checkpoint.checkpointJson) > MAX_STORED_TEXT_BYTES) return yield* SqliteStorageError.make({
|
|
799
|
+
operation: "save checkpoint",
|
|
800
|
+
message: "Checkpoint identity or encoded JSON exceeds the SQLite storage bounds."
|
|
801
|
+
});
|
|
802
|
+
yield* withWriteTransaction("checkpoint transaction")(Effect.gen(function* () {
|
|
803
|
+
const threadRows = yield* sql`
|
|
804
|
+
SELECT
|
|
805
|
+
thread_id,
|
|
806
|
+
created_at,
|
|
807
|
+
tail_sequence,
|
|
808
|
+
tail_digest,
|
|
809
|
+
producer_epoch
|
|
810
|
+
FROM effect_agent_threads
|
|
811
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
812
|
+
`.pipe(Effect.mapError(storageError("read checkpoint tail")));
|
|
813
|
+
const thread = yield* decodeSingleRow(Schema.Array(ThreadRow), "effect_agent_threads", checkpoint.threadId, threadRows);
|
|
814
|
+
if (checkpoint.throughSequence > thread.tail_sequence) return yield* SqliteCheckpointConflict.make({ message: `Checkpoint sequence ${checkpoint.throughSequence} is after canonical tail ${thread.tail_sequence}.` });
|
|
815
|
+
const checkpointRows = yield* sql`
|
|
816
|
+
SELECT
|
|
817
|
+
thread_id,
|
|
818
|
+
through_sequence,
|
|
819
|
+
tail_digest,
|
|
820
|
+
checkpoint_json
|
|
821
|
+
FROM effect_agent_checkpoints
|
|
822
|
+
WHERE thread_id = ${checkpoint.threadId}
|
|
823
|
+
AND through_sequence = ${checkpoint.throughSequence}
|
|
824
|
+
`.pipe(Effect.mapError(storageError("read idempotent checkpoint")));
|
|
825
|
+
const existing = yield* decodeRows(Schema.Array(CheckpointRow), "effect_agent_checkpoints", `${checkpoint.threadId}/${checkpoint.throughSequence}`, checkpointRows);
|
|
826
|
+
if (existing.length > 1) return yield* SqliteStorageCorruptionError.make({
|
|
827
|
+
table: "effect_agent_checkpoints",
|
|
828
|
+
rowKey: `${checkpoint.threadId}/${checkpoint.throughSequence}`,
|
|
829
|
+
message: "A checkpoint primary key returned more than one row."
|
|
830
|
+
});
|
|
831
|
+
if (existing.length === 1) {
|
|
832
|
+
if (existing[0].tail_digest !== checkpoint.tailDigest || existing[0].checkpoint_json !== checkpoint.checkpointJson) return yield* SqliteCheckpointConflict.make({ message: "A different checkpoint already exists at this canonical sequence." });
|
|
833
|
+
return;
|
|
834
|
+
}
|
|
835
|
+
yield* sql`
|
|
836
|
+
INSERT INTO effect_agent_checkpoints (
|
|
837
|
+
thread_id,
|
|
838
|
+
through_sequence,
|
|
839
|
+
tail_digest,
|
|
840
|
+
checkpoint_json
|
|
841
|
+
) VALUES (
|
|
842
|
+
${checkpoint.threadId},
|
|
843
|
+
${checkpoint.throughSequence},
|
|
844
|
+
${checkpoint.tailDigest},
|
|
845
|
+
${checkpoint.checkpointJson}
|
|
846
|
+
)
|
|
847
|
+
`.pipe(Effect.mapError(storageError("insert checkpoint")));
|
|
848
|
+
}));
|
|
849
|
+
});
|
|
850
|
+
const saveRecoveryCheckpoint = Effect.fn("SqliteJournal.saveRecoveryCheckpoint")(function* (request, checkpointJson) {
|
|
851
|
+
const { checkpoint } = request;
|
|
852
|
+
if (checkpoint.threadId.length > MAX_IDENTIFIER_LENGTH || storedTextBytes(checkpointJson) > MAX_STORED_TEXT_BYTES) return yield* SqliteStorageError.make({
|
|
853
|
+
operation: "save recovery checkpoint",
|
|
854
|
+
message: "Checkpoint identity or encoded JSON exceeds the SQLite storage bounds."
|
|
855
|
+
});
|
|
856
|
+
yield* withWriteTransaction("recovery checkpoint transaction")(Effect.gen(function* () {
|
|
857
|
+
const thread = (yield* getThread(checkpoint.threadId))[0];
|
|
858
|
+
if (thread === void 0) return yield* ThreadNotMaterialized.make({ threadId: checkpoint.threadId });
|
|
859
|
+
if (request.producerEpoch !== thread.producer_epoch) return yield* FenceRejected.make({
|
|
860
|
+
threadId: checkpoint.threadId,
|
|
861
|
+
actualEpoch: thread.producer_epoch,
|
|
862
|
+
attemptedEpoch: request.producerEpoch
|
|
863
|
+
});
|
|
864
|
+
if (checkpoint.throughSequence > thread.tail_sequence) return yield* CheckpointRejected.make({
|
|
865
|
+
threadId: checkpoint.threadId,
|
|
866
|
+
reason: "ahead-of-tail"
|
|
867
|
+
});
|
|
868
|
+
const digests = checkpoint.throughSequence === 0 ? [EMPTY_TAIL_DIGEST] : yield* getTailDigestAt(checkpoint.threadId, checkpoint.throughSequence);
|
|
869
|
+
if (digests.length !== 1 || digests[0] !== checkpoint.tailDigest) return yield* CheckpointRejected.make({
|
|
870
|
+
threadId: checkpoint.threadId,
|
|
871
|
+
reason: "digest-mismatch"
|
|
872
|
+
});
|
|
873
|
+
yield* failpoint("save-recovery-checkpoint:before");
|
|
874
|
+
yield* sql`
|
|
875
|
+
INSERT INTO effect_agent_recovery_checkpoints (thread_id, through_sequence, tail_digest, checkpoint_json)
|
|
876
|
+
VALUES (${checkpoint.threadId}, ${checkpoint.throughSequence}, ${checkpoint.tailDigest}, ${checkpointJson})
|
|
877
|
+
ON CONFLICT (thread_id) DO UPDATE SET
|
|
878
|
+
through_sequence = excluded.through_sequence,
|
|
879
|
+
tail_digest = excluded.tail_digest,
|
|
880
|
+
checkpoint_json = excluded.checkpoint_json
|
|
881
|
+
WHERE excluded.through_sequence >= effect_agent_recovery_checkpoints.through_sequence
|
|
882
|
+
`.pipe(Effect.mapError(storageError("save recovery checkpoint")));
|
|
883
|
+
}));
|
|
884
|
+
yield* failpoint("save-recovery-checkpoint:after");
|
|
885
|
+
});
|
|
886
|
+
const loadRecoveryCheckpoint = Effect.fn("SqliteJournal.loadRecoveryCheckpoint")(function* (threadId) {
|
|
887
|
+
const rows = yield* sql`
|
|
888
|
+
SELECT thread_id, through_sequence, tail_digest, checkpoint_json
|
|
889
|
+
FROM effect_agent_recovery_checkpoints
|
|
890
|
+
WHERE thread_id = ${threadId}
|
|
891
|
+
`.pipe(Effect.mapError(storageError("load recovery checkpoint")));
|
|
892
|
+
return yield* decodeRows(Schema.Array(CheckpointRow), "effect_agent_recovery_checkpoints", threadId, rows);
|
|
893
|
+
});
|
|
894
|
+
const loadCheckpoint = Effect.fn("SqliteJournal.loadCheckpoint")(function* (threadId, atOrBeforeSequence) {
|
|
895
|
+
const rows = yield* sql`
|
|
896
|
+
SELECT
|
|
897
|
+
thread_id,
|
|
898
|
+
through_sequence,
|
|
899
|
+
tail_digest,
|
|
900
|
+
checkpoint_json
|
|
901
|
+
FROM effect_agent_checkpoints
|
|
902
|
+
WHERE thread_id = ${threadId}
|
|
903
|
+
AND through_sequence <= ${atOrBeforeSequence}
|
|
904
|
+
ORDER BY through_sequence DESC
|
|
905
|
+
LIMIT 1
|
|
906
|
+
`.pipe(Effect.mapError(storageError("load checkpoint")));
|
|
907
|
+
return yield* decodeRows(Schema.Array(CheckpointRow), "effect_agent_checkpoints", `${threadId}<=${atOrBeforeSequence}`, rows);
|
|
908
|
+
});
|
|
909
|
+
const getTailDigestAt = Effect.fn("SqliteJournal.getTailDigestAt")(function* (threadId, sequence) {
|
|
910
|
+
if (sequence === 0) {
|
|
911
|
+
const threads = yield* getThread(threadId);
|
|
912
|
+
return threads.length === 0 ? [] : [threads[0].tail_sequence === 0 ? threads[0].tail_digest : void 0].filter((value) => value !== void 0);
|
|
913
|
+
}
|
|
914
|
+
const rows = yield* sql`
|
|
915
|
+
SELECT
|
|
916
|
+
thread_id,
|
|
917
|
+
batch_id,
|
|
918
|
+
first_sequence,
|
|
919
|
+
last_sequence,
|
|
920
|
+
batch_digest,
|
|
921
|
+
tail_digest,
|
|
922
|
+
batch_json
|
|
923
|
+
FROM effect_agent_canonical_batches
|
|
924
|
+
WHERE thread_id = ${threadId}
|
|
925
|
+
AND last_sequence = ${sequence}
|
|
926
|
+
`.pipe(Effect.mapError(storageError("read canonical digest at sequence")));
|
|
927
|
+
return (yield* decodeRows(Schema.Array(BatchRow), "effect_agent_canonical_batches", `${threadId}/${sequence}`, rows)).map((batch) => batch.tail_digest);
|
|
928
|
+
});
|
|
929
|
+
return {
|
|
930
|
+
append,
|
|
931
|
+
exportThread,
|
|
932
|
+
getThread,
|
|
933
|
+
getTailDigestAt,
|
|
934
|
+
loadCheckpoint,
|
|
935
|
+
loadRecoveryCheckpoint,
|
|
936
|
+
saveRecoveryCheckpoint,
|
|
937
|
+
materialize,
|
|
938
|
+
read,
|
|
939
|
+
saveCheckpoint,
|
|
940
|
+
scanStoredPayloads: Effect.fn("SqliteJournal.scanStoredPayloads")(function* () {
|
|
941
|
+
return yield* withReadTransaction("startup scan transaction")(Effect.gen(function* () {
|
|
942
|
+
const threads = yield* sql`
|
|
943
|
+
SELECT
|
|
944
|
+
thread_id,
|
|
945
|
+
created_at,
|
|
946
|
+
tail_sequence,
|
|
947
|
+
tail_digest,
|
|
948
|
+
producer_epoch
|
|
949
|
+
FROM effect_agent_threads
|
|
950
|
+
ORDER BY thread_id
|
|
951
|
+
`.pipe(Effect.mapError(storageError("scan threads")));
|
|
952
|
+
const batches = yield* sql`
|
|
953
|
+
SELECT
|
|
954
|
+
thread_id,
|
|
955
|
+
batch_id,
|
|
956
|
+
first_sequence,
|
|
957
|
+
last_sequence,
|
|
958
|
+
batch_digest,
|
|
959
|
+
tail_digest,
|
|
960
|
+
batch_json
|
|
961
|
+
FROM effect_agent_canonical_batches
|
|
962
|
+
ORDER BY thread_id, first_sequence
|
|
963
|
+
`.pipe(Effect.mapError(storageError("scan canonical batches")));
|
|
964
|
+
const records = yield* sql`
|
|
965
|
+
SELECT
|
|
966
|
+
thread_id,
|
|
967
|
+
sequence,
|
|
968
|
+
record_id,
|
|
969
|
+
batch_id,
|
|
970
|
+
record_json
|
|
971
|
+
FROM effect_agent_canonical_records
|
|
972
|
+
ORDER BY thread_id, sequence
|
|
973
|
+
`.pipe(Effect.mapError(storageError("scan canonical records")));
|
|
974
|
+
const checkpoints = yield* sql`
|
|
975
|
+
SELECT
|
|
976
|
+
thread_id,
|
|
977
|
+
through_sequence,
|
|
978
|
+
tail_digest,
|
|
979
|
+
checkpoint_json
|
|
980
|
+
FROM effect_agent_checkpoints
|
|
981
|
+
ORDER BY thread_id, through_sequence
|
|
982
|
+
`.pipe(Effect.mapError(storageError("scan checkpoints")));
|
|
983
|
+
return {
|
|
984
|
+
threads: yield* decodeRows(Schema.Array(ThreadRow), "effect_agent_threads", "startup_scan", threads),
|
|
985
|
+
batches: yield* decodeRows(Schema.Array(BatchRow), "effect_agent_canonical_batches", "startup_scan", batches),
|
|
986
|
+
records: yield* decodeRows(Schema.Array(RecordRow), "effect_agent_canonical_records", "startup_scan", records),
|
|
987
|
+
checkpoints: yield* decodeRows(Schema.Array(CheckpointRow), "effect_agent_checkpoints", "startup_scan", checkpoints)
|
|
988
|
+
};
|
|
989
|
+
}));
|
|
990
|
+
}),
|
|
991
|
+
withWriteTransaction,
|
|
992
|
+
withReadTransaction
|
|
993
|
+
};
|
|
994
|
+
});
|
|
995
|
+
//#endregion
|
|
996
|
+
export { initializeSqliteJournal as a, decodeRows as i, RawCheckpoint as n, RawReadRequest as r, RawAppendRequest as t };
|
|
997
|
+
|
|
998
|
+
//# sourceMappingURL=sqlite-journal-CBOQDySe.mjs.map
|