@effect-agent/storage-sqlite 0.1.0-beta.39 → 0.1.0-beta.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +19 -2
- package/dist/index.mjs +713 -157
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -4
- package/src/index.ts +2 -0
- package/src/migrations.ts +9 -68
- package/src/sqlite-activity-store.ts +503 -0
- package/src/sqlite-memory-store.ts +532 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@effect-agent/storage-sqlite",
|
|
3
|
-
"version": "0.1.0-beta.
|
|
3
|
+
"version": "0.1.0-beta.40",
|
|
4
4
|
"exports": {
|
|
5
5
|
".": {
|
|
6
6
|
"types": "./dist/index.d.mts",
|
|
@@ -12,14 +12,15 @@
|
|
|
12
12
|
}
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"@effect-agent/
|
|
15
|
+
"@effect-agent/core": "0.1.0-beta.40",
|
|
16
|
+
"@effect-agent/thread": "0.1.0-beta.40",
|
|
16
17
|
"@effect/platform-node": "4.0.0-rc.111",
|
|
17
18
|
"@effect/sql-sqlite-node": "4.0.0-rc.111"
|
|
18
19
|
},
|
|
19
20
|
"peerDependencies": {
|
|
20
21
|
"effect": "^4.0.0-rc.111"
|
|
21
22
|
},
|
|
22
|
-
"description": "Node SQLite
|
|
23
|
+
"description": "Node SQLite persistence adapters for the Effect Agent durable runtime and optional memory lifecycle.",
|
|
23
24
|
"license": "MIT",
|
|
24
25
|
"repository": {
|
|
25
26
|
"type": "git",
|
|
@@ -43,6 +44,6 @@
|
|
|
43
44
|
"@effect/vitest": "4.0.0-rc.111",
|
|
44
45
|
"effect": "4.0.0-rc.111",
|
|
45
46
|
"typescript": "7.0.2",
|
|
46
|
-
"vite-plus": "0.
|
|
47
|
+
"vite-plus": "0.3.0"
|
|
47
48
|
}
|
|
48
49
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
export * from "./errors.ts";
|
|
2
2
|
export { CurrentSqliteStorageVersion } from "./migrations.ts";
|
|
3
|
+
export * from "./sqlite-activity-store.ts";
|
|
3
4
|
export * from "./sqlite-thread-store.ts";
|
|
4
5
|
export * from "./sqlite-ledger.ts";
|
|
6
|
+
export * from "./sqlite-memory-store.ts";
|
|
5
7
|
export * from "./sqlite-schedule-store.ts";
|
|
6
8
|
export * from "./sqlite-subscription-store.ts";
|
|
7
9
|
export * from "./sqlite-storage-config.ts";
|
package/src/migrations.ts
CHANGED
|
@@ -4,8 +4,9 @@ import * as SqlClient from "effect/unstable/sql/SqlClient";
|
|
|
4
4
|
|
|
5
5
|
export const CurrentSqliteStorageVersion = 7;
|
|
6
6
|
|
|
7
|
+
/** Initialize empty storage with the complete current schema. */
|
|
7
8
|
export const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
8
|
-
"
|
|
9
|
+
"1_current_thread_storage": Effect.gen(function* () {
|
|
9
10
|
const sql = yield* SqlClient.SqlClient;
|
|
10
11
|
|
|
11
12
|
yield* sql`
|
|
@@ -67,11 +68,6 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
67
68
|
)
|
|
68
69
|
`.withoutTransform;
|
|
69
70
|
|
|
70
|
-
yield* sql`PRAGMA user_version = 1`.withoutTransform;
|
|
71
|
-
}),
|
|
72
|
-
"2_durable_submission_ledger": Effect.gen(function* () {
|
|
73
|
-
const sql = yield* SqlClient.SqlClient;
|
|
74
|
-
|
|
75
71
|
// Admission rows exist before Thread materialization (durability §4), so
|
|
76
72
|
// thread_id intentionally carries no foreign key into effect_agent_threads.
|
|
77
73
|
yield* sql`
|
|
@@ -93,6 +89,13 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
93
89
|
ready_at TEXT,
|
|
94
90
|
input_applied_record_id TEXT,
|
|
95
91
|
input_applied_sequence INTEGER,
|
|
92
|
+
joined_host_submission_id TEXT,
|
|
93
|
+
suspended_reason_json TEXT,
|
|
94
|
+
suspended_at TEXT,
|
|
95
|
+
unknown_reason TEXT,
|
|
96
|
+
unknown_tool_call_ids_json TEXT,
|
|
97
|
+
parent_submission_id TEXT,
|
|
98
|
+
parent_tool_call_id TEXT,
|
|
96
99
|
UNIQUE (thread_id, principal, idempotency_key),
|
|
97
100
|
UNIQUE (thread_id, queue_sequence)
|
|
98
101
|
)
|
|
@@ -155,42 +158,6 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
155
158
|
)
|
|
156
159
|
`.withoutTransform;
|
|
157
160
|
|
|
158
|
-
yield* sql`PRAGMA user_version = 2`.withoutTransform;
|
|
159
|
-
}),
|
|
160
|
-
"3_durable_tools_and_joined_input": Effect.gen(function* () {
|
|
161
|
-
const sql = yield* SqlClient.SqlClient;
|
|
162
|
-
|
|
163
|
-
// Joined queued input (plan §2.5, DUR-016): a joining/joined Submission records which host
|
|
164
|
-
// Run claimed it. The canonical input marker reuses input_applied_record_id/_sequence
|
|
165
|
-
// because the joined input IS the Submission's own canonical `input:{sid}` record.
|
|
166
|
-
yield* sql`
|
|
167
|
-
ALTER TABLE effect_agent_submissions
|
|
168
|
-
ADD COLUMN joined_host_submission_id TEXT
|
|
169
|
-
`.withoutTransform;
|
|
170
|
-
|
|
171
|
-
// Durable approval suspension (plan §2.6): the reason is the Schema-encoded
|
|
172
|
-
// SuspensionReason union so later suspension families stay additive.
|
|
173
|
-
yield* sql`
|
|
174
|
-
ALTER TABLE effect_agent_submissions
|
|
175
|
-
ADD COLUMN suspended_reason_json TEXT
|
|
176
|
-
`.withoutTransform;
|
|
177
|
-
yield* sql`
|
|
178
|
-
ALTER TABLE effect_agent_submissions
|
|
179
|
-
ADD COLUMN suspended_at TEXT
|
|
180
|
-
`.withoutTransform;
|
|
181
|
-
|
|
182
|
-
// Unknown Outcome marking (DUR-009/DUR-017): the marked open Tool Call identities are
|
|
183
|
-
// stored so `recordUnknownResolution` can reopen the lane only when every marked call
|
|
184
|
-
// has a durable resolution intent.
|
|
185
|
-
yield* sql`
|
|
186
|
-
ALTER TABLE effect_agent_submissions
|
|
187
|
-
ADD COLUMN unknown_reason TEXT
|
|
188
|
-
`.withoutTransform;
|
|
189
|
-
yield* sql`
|
|
190
|
-
ALTER TABLE effect_agent_submissions
|
|
191
|
-
ADD COLUMN unknown_tool_call_ids_json TEXT
|
|
192
|
-
`.withoutTransform;
|
|
193
|
-
|
|
194
161
|
yield* sql`
|
|
195
162
|
CREATE INDEX effect_agent_submissions_joined_host
|
|
196
163
|
ON effect_agent_submissions (joined_host_submission_id)
|
|
@@ -226,21 +193,8 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
226
193
|
)
|
|
227
194
|
`.withoutTransform;
|
|
228
195
|
|
|
229
|
-
yield* sql`PRAGMA user_version = 3`.withoutTransform;
|
|
230
|
-
}),
|
|
231
|
-
"4_durable_subagents": Effect.gen(function* () {
|
|
232
|
-
const sql = yield* SqlClient.SqlClient;
|
|
233
|
-
|
|
234
196
|
// Durable attached children (spec §12, SUB-004): a child Submission records its immutable
|
|
235
197
|
// parent linkage at admission; the parent-side index serves the recovery attachment view.
|
|
236
|
-
yield* sql`
|
|
237
|
-
ALTER TABLE effect_agent_submissions
|
|
238
|
-
ADD COLUMN parent_submission_id TEXT
|
|
239
|
-
`.withoutTransform;
|
|
240
|
-
yield* sql`
|
|
241
|
-
ALTER TABLE effect_agent_submissions
|
|
242
|
-
ADD COLUMN parent_tool_call_id TEXT
|
|
243
|
-
`.withoutTransform;
|
|
244
198
|
yield* sql`
|
|
245
199
|
CREATE INDEX effect_agent_submissions_parent
|
|
246
200
|
ON effect_agent_submissions (parent_submission_id)
|
|
@@ -270,11 +224,6 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
270
224
|
)
|
|
271
225
|
`.withoutTransform;
|
|
272
226
|
|
|
273
|
-
yield* sql`PRAGMA user_version = 4`.withoutTransform;
|
|
274
|
-
}),
|
|
275
|
-
"5_durable_schedules": Effect.gen(function* () {
|
|
276
|
-
const sql = yield* SqlClient.SqlClient;
|
|
277
|
-
|
|
278
227
|
// record_json is authoritative. The remaining columns support owner keyset paging and
|
|
279
228
|
// deadline queries without decoding unrelated future schedules.
|
|
280
229
|
yield* sql`
|
|
@@ -300,10 +249,6 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
300
249
|
WHERE deadline_at_millis IS NOT NULL
|
|
301
250
|
`.withoutTransform;
|
|
302
251
|
|
|
303
|
-
yield* sql`PRAGMA user_version = 5`.withoutTransform;
|
|
304
|
-
}),
|
|
305
|
-
"6_durable_subscriptions": Effect.gen(function* () {
|
|
306
|
-
const sql = yield* SqlClient.SqlClient;
|
|
307
252
|
yield* sql`
|
|
308
253
|
CREATE TABLE effect_agent_subscription_sequences (
|
|
309
254
|
tenant_id TEXT NOT NULL,
|
|
@@ -377,10 +322,6 @@ export const sqliteMigrations = SqliteMigrator.fromRecord({
|
|
|
377
322
|
.withoutTransform;
|
|
378
323
|
yield* sql`CREATE INDEX effect_agent_subscription_deliveries_registration ON effect_agent_subscription_deliveries (tenant_id, source_address, owner_id, subscription_id, delivery_key)`
|
|
379
324
|
.withoutTransform;
|
|
380
|
-
yield* sql`PRAGMA user_version = 6`.withoutTransform;
|
|
381
|
-
}),
|
|
382
|
-
"7_thread_terminology_cutover": Effect.gen(function* () {
|
|
383
|
-
const sql = yield* SqlClient.SqlClient;
|
|
384
325
|
yield* sql`PRAGMA user_version = 7`.withoutTransform;
|
|
385
326
|
}),
|
|
386
327
|
});
|
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ActivityBusy,
|
|
3
|
+
ActivityClaim,
|
|
4
|
+
ActivityClaimRequest,
|
|
5
|
+
ActivityMutationFailpoint,
|
|
6
|
+
type ActivityMutationFailure,
|
|
7
|
+
ActivityOwnershipLost,
|
|
8
|
+
ActivityProcessorKey,
|
|
9
|
+
ActivityProcessorStore,
|
|
10
|
+
ActivityProgress,
|
|
11
|
+
ActivityStoreError,
|
|
12
|
+
ActivityWorkConflict,
|
|
13
|
+
Digest,
|
|
14
|
+
PreparedActivity,
|
|
15
|
+
} from "@effect-agent/thread";
|
|
16
|
+
import { Clock, Effect, Layer, Schema } from "effect";
|
|
17
|
+
import * as SqlClientService from "effect/unstable/sql/SqlClient";
|
|
18
|
+
import type { SqlError } from "effect/unstable/sql/SqlError";
|
|
19
|
+
|
|
20
|
+
const STORAGE_VERSION = 1 as const;
|
|
21
|
+
const METADATA_COMPONENT = "activity";
|
|
22
|
+
const STATE_TABLE = "effect_agent_activity_processor_state_v1";
|
|
23
|
+
const StoredJson = Schema.String.check(Schema.isMaxLength(16 * 1024 * 1024));
|
|
24
|
+
const sameKey = Schema.toEquivalence(ActivityProcessorKey);
|
|
25
|
+
const sameWork = Schema.toEquivalence(PreparedActivity);
|
|
26
|
+
|
|
27
|
+
class ActivityMetadataRow extends Schema.Class<ActivityMetadataRow>(
|
|
28
|
+
"@effect-agent/storage-sqlite/ActivityMetadataRow",
|
|
29
|
+
)({
|
|
30
|
+
version: Schema.Int,
|
|
31
|
+
}) {}
|
|
32
|
+
|
|
33
|
+
class ActivityTableRow extends Schema.Class<ActivityTableRow>(
|
|
34
|
+
"@effect-agent/storage-sqlite/ActivityTableRow",
|
|
35
|
+
)({
|
|
36
|
+
name: Schema.NonEmptyString,
|
|
37
|
+
}) {}
|
|
38
|
+
|
|
39
|
+
class ActivityStateRow extends Schema.Class<ActivityStateRow>(
|
|
40
|
+
"@effect-agent/storage-sqlite/ActivityStateRow",
|
|
41
|
+
)({
|
|
42
|
+
processor_id: ActivityProcessorKey.fields.processorId,
|
|
43
|
+
processor_version: ActivityProcessorKey.fields.processorVersion,
|
|
44
|
+
thread_id: ActivityProcessorKey.fields.threadId,
|
|
45
|
+
format_version: Schema.Int,
|
|
46
|
+
through_sequence: ActivityProgress.fields.throughSequence,
|
|
47
|
+
epoch: ActivityProgress.fields.epoch,
|
|
48
|
+
owner: ActivityProgress.fields.owner,
|
|
49
|
+
lease_expires_at: ActivityProgress.fields.leaseExpiresAt,
|
|
50
|
+
progress_json: StoredJson,
|
|
51
|
+
}) {}
|
|
52
|
+
|
|
53
|
+
class ActivityChangeCountRow extends Schema.Class<ActivityChangeCountRow>(
|
|
54
|
+
"@effect-agent/storage-sqlite/ActivityChangeCountRow",
|
|
55
|
+
)({
|
|
56
|
+
changed: Schema.Int,
|
|
57
|
+
}) {}
|
|
58
|
+
|
|
59
|
+
const StoredVersionHeader = Schema.Struct({ version: Schema.Int });
|
|
60
|
+
|
|
61
|
+
export type SqliteActivityInitializationError = ActivityStoreError | ActivityMutationFailure;
|
|
62
|
+
|
|
63
|
+
const storeError = (
|
|
64
|
+
operation: string,
|
|
65
|
+
reason: ActivityStoreError["reason"] = "unavailable",
|
|
66
|
+
): ActivityStoreError => ActivityStoreError.make({ operation, reason });
|
|
67
|
+
|
|
68
|
+
const query = <A extends object>(
|
|
69
|
+
effect: Effect.Effect<ReadonlyArray<A>, SqlError>,
|
|
70
|
+
operation: string,
|
|
71
|
+
) => effect.pipe(Effect.mapError(() => storeError(operation)));
|
|
72
|
+
|
|
73
|
+
const decodeRows = Effect.fn("SqliteActivityStore.decodeRows")(function* <A, I>(
|
|
74
|
+
schema: Schema.Codec<A, I, never>,
|
|
75
|
+
rows: ReadonlyArray<unknown>,
|
|
76
|
+
operation: string,
|
|
77
|
+
): Effect.fn.Return<ReadonlyArray<A>, ActivityStoreError> {
|
|
78
|
+
return yield* Schema.decodeUnknownEffect(Schema.Array(schema))(rows).pipe(
|
|
79
|
+
Effect.mapError(() => storeError(operation, "corrupt")),
|
|
80
|
+
);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
const decodeInput = Effect.fn("SqliteActivityStore.decodeInput")(function* <A, I>(
|
|
84
|
+
schema: Schema.Codec<A, I, never>,
|
|
85
|
+
value: unknown,
|
|
86
|
+
operation: string,
|
|
87
|
+
): Effect.fn.Return<A, ActivityStoreError> {
|
|
88
|
+
return yield* Schema.decodeUnknownEffect(schema)(value).pipe(
|
|
89
|
+
Effect.mapError(() => storeError(operation, "invalid-input")),
|
|
90
|
+
);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
const encodeProgress = Effect.fn("SqliteActivityStore.encodeProgress")(function* (
|
|
94
|
+
progress: ActivityProgress,
|
|
95
|
+
operation: string,
|
|
96
|
+
): Effect.fn.Return<string, ActivityStoreError> {
|
|
97
|
+
return yield* Schema.encodeEffect(Schema.fromJsonString(ActivityProgress))(progress).pipe(
|
|
98
|
+
Effect.mapError(() => storeError(operation, "corrupt")),
|
|
99
|
+
Effect.flatMap((encoded) =>
|
|
100
|
+
Schema.decodeEffect(StoredJson)(encoded).pipe(
|
|
101
|
+
Effect.mapError(() => storeError(operation, "invalid-input")),
|
|
102
|
+
),
|
|
103
|
+
),
|
|
104
|
+
);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
const decodeProgress = Effect.fn("SqliteActivityStore.decodeProgress")(function* (
|
|
108
|
+
value: string,
|
|
109
|
+
operation: string,
|
|
110
|
+
): Effect.fn.Return<ActivityProgress, ActivityStoreError> {
|
|
111
|
+
const header = yield* Schema.decodeEffect(Schema.fromJsonString(StoredVersionHeader))(value).pipe(
|
|
112
|
+
Effect.mapError(() => storeError(operation, "corrupt")),
|
|
113
|
+
);
|
|
114
|
+
if (header.version !== STORAGE_VERSION) {
|
|
115
|
+
return yield* storeError(operation, "incompatible");
|
|
116
|
+
}
|
|
117
|
+
const progress = yield* Schema.decodeEffect(Schema.fromJsonString(ActivityProgress))(value).pipe(
|
|
118
|
+
Effect.mapError(() => storeError(operation, "corrupt")),
|
|
119
|
+
);
|
|
120
|
+
const canonical = yield* encodeProgress(progress, operation);
|
|
121
|
+
if (canonical !== value) return yield* storeError(operation, "corrupt");
|
|
122
|
+
return progress;
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
const validateProgress = Effect.fn("SqliteActivityStore.validateProgress")(function* (
|
|
126
|
+
progress: ActivityProgress,
|
|
127
|
+
operation: string,
|
|
128
|
+
): Effect.fn.Return<ActivityProgress, ActivityStoreError> {
|
|
129
|
+
if (
|
|
130
|
+
progress.epoch < 1 ||
|
|
131
|
+
(progress.owner === null && progress.leaseExpiresAt !== 0) ||
|
|
132
|
+
(progress.pending !== null &&
|
|
133
|
+
(!sameKey(progress.pending.key, progress.key) ||
|
|
134
|
+
progress.pending.sequence !== progress.throughSequence + 1))
|
|
135
|
+
) {
|
|
136
|
+
return yield* storeError(operation, "corrupt");
|
|
137
|
+
}
|
|
138
|
+
return progress;
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
const makeClaim = (progress: ActivityProgress): ActivityClaim | null =>
|
|
142
|
+
progress.owner === null
|
|
143
|
+
? null
|
|
144
|
+
: ActivityClaim.make({
|
|
145
|
+
key: progress.key,
|
|
146
|
+
owner: progress.owner,
|
|
147
|
+
epoch: progress.epoch,
|
|
148
|
+
throughSequence: progress.throughSequence,
|
|
149
|
+
leaseExpiresAt: progress.leaseExpiresAt,
|
|
150
|
+
pending: progress.pending,
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
const ownershipLost = (claim: ActivityClaim) =>
|
|
154
|
+
ActivityOwnershipLost.make({ key: claim.key, owner: claim.owner, epoch: claim.epoch });
|
|
155
|
+
|
|
156
|
+
// The pinned Node SQLite client's writable withTransaction begins with BEGIN IMMEDIATE.
|
|
157
|
+
// Each mutation therefore locks before reading progress, which serializes independent
|
|
158
|
+
// connections without a deferred read-to-write upgrade.
|
|
159
|
+
const makeActivityStore = Effect.fn("SqliteActivityStore.make")(function* () {
|
|
160
|
+
const sql = yield* SqlClientService.SqlClient;
|
|
161
|
+
const failpoint = yield* ActivityMutationFailpoint;
|
|
162
|
+
|
|
163
|
+
yield* failpoint.hit("activity:initialize:before");
|
|
164
|
+
yield* sql
|
|
165
|
+
.withTransaction(
|
|
166
|
+
Effect.gen(function* () {
|
|
167
|
+
yield* sql`
|
|
168
|
+
CREATE TABLE IF NOT EXISTS effect_agent_activity_metadata (
|
|
169
|
+
component TEXT PRIMARY KEY NOT NULL,
|
|
170
|
+
version INTEGER NOT NULL
|
|
171
|
+
)
|
|
172
|
+
`;
|
|
173
|
+
const metadataRows = yield* sql<Record<string, unknown>>`
|
|
174
|
+
SELECT version FROM effect_agent_activity_metadata
|
|
175
|
+
WHERE component = ${METADATA_COMPONENT}
|
|
176
|
+
`;
|
|
177
|
+
const metadata = yield* decodeRows(
|
|
178
|
+
ActivityMetadataRow,
|
|
179
|
+
metadataRows,
|
|
180
|
+
"decode activity schema version",
|
|
181
|
+
);
|
|
182
|
+
if (metadata.length > 1) {
|
|
183
|
+
return yield* storeError("decode activity schema version", "corrupt");
|
|
184
|
+
}
|
|
185
|
+
const currentVersion = metadata[0]?.version;
|
|
186
|
+
if (currentVersion !== undefined && currentVersion !== STORAGE_VERSION) {
|
|
187
|
+
return yield* storeError("initialize activity schema", "incompatible");
|
|
188
|
+
}
|
|
189
|
+
if (currentVersion === undefined) {
|
|
190
|
+
const tableRows = yield* sql<Record<string, unknown>>`
|
|
191
|
+
SELECT name FROM sqlite_master
|
|
192
|
+
WHERE type = 'table' AND name = ${STATE_TABLE}
|
|
193
|
+
`;
|
|
194
|
+
const existing = yield* decodeRows(
|
|
195
|
+
ActivityTableRow,
|
|
196
|
+
tableRows,
|
|
197
|
+
"inspect activity schema",
|
|
198
|
+
);
|
|
199
|
+
if (existing.length > 0) {
|
|
200
|
+
return yield* storeError("initialize activity schema", "incompatible");
|
|
201
|
+
}
|
|
202
|
+
yield* sql`
|
|
203
|
+
CREATE TABLE effect_agent_activity_processor_state_v1 (
|
|
204
|
+
processor_id TEXT NOT NULL,
|
|
205
|
+
processor_version TEXT NOT NULL,
|
|
206
|
+
thread_id TEXT NOT NULL,
|
|
207
|
+
format_version INTEGER NOT NULL,
|
|
208
|
+
through_sequence INTEGER NOT NULL,
|
|
209
|
+
epoch INTEGER NOT NULL,
|
|
210
|
+
owner TEXT,
|
|
211
|
+
lease_expires_at REAL NOT NULL,
|
|
212
|
+
progress_json TEXT NOT NULL,
|
|
213
|
+
PRIMARY KEY (processor_id, processor_version, thread_id)
|
|
214
|
+
)
|
|
215
|
+
`;
|
|
216
|
+
yield* sql`
|
|
217
|
+
INSERT INTO effect_agent_activity_metadata (component, version)
|
|
218
|
+
VALUES (${METADATA_COMPONENT}, ${STORAGE_VERSION})
|
|
219
|
+
`;
|
|
220
|
+
}
|
|
221
|
+
yield* sql`
|
|
222
|
+
SELECT processor_id, processor_version, thread_id, format_version,
|
|
223
|
+
through_sequence, epoch, owner, lease_expires_at, progress_json
|
|
224
|
+
FROM effect_agent_activity_processor_state_v1
|
|
225
|
+
LIMIT 0
|
|
226
|
+
`;
|
|
227
|
+
}),
|
|
228
|
+
)
|
|
229
|
+
.pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError("initialize activity schema"))));
|
|
230
|
+
yield* failpoint.hit("activity:initialize:after");
|
|
231
|
+
|
|
232
|
+
const readProgress = Effect.fn("SqliteActivityStore.readProgress")(function* (
|
|
233
|
+
key: ActivityProcessorKey,
|
|
234
|
+
operation: string,
|
|
235
|
+
): Effect.fn.Return<ActivityProgress | null, ActivityStoreError> {
|
|
236
|
+
const rawRows = yield* query(
|
|
237
|
+
sql<Record<string, unknown>>`
|
|
238
|
+
SELECT processor_id, processor_version, thread_id, format_version,
|
|
239
|
+
through_sequence, epoch, owner, lease_expires_at, progress_json
|
|
240
|
+
FROM effect_agent_activity_processor_state_v1
|
|
241
|
+
WHERE processor_id = ${key.processorId}
|
|
242
|
+
AND processor_version = ${key.processorVersion}
|
|
243
|
+
AND thread_id = ${key.threadId}
|
|
244
|
+
`,
|
|
245
|
+
operation,
|
|
246
|
+
);
|
|
247
|
+
const rows = yield* decodeRows(ActivityStateRow, rawRows, operation);
|
|
248
|
+
if (rows.length === 0) return null;
|
|
249
|
+
if (rows.length !== 1) return yield* storeError(operation, "corrupt");
|
|
250
|
+
const row = rows[0];
|
|
251
|
+
if (row.format_version !== STORAGE_VERSION) {
|
|
252
|
+
return yield* storeError(operation, "incompatible");
|
|
253
|
+
}
|
|
254
|
+
const progress = yield* decodeProgress(row.progress_json, operation);
|
|
255
|
+
yield* validateProgress(progress, operation);
|
|
256
|
+
if (
|
|
257
|
+
!sameKey(progress.key, key) ||
|
|
258
|
+
row.processor_id !== key.processorId ||
|
|
259
|
+
row.processor_version !== key.processorVersion ||
|
|
260
|
+
row.thread_id !== key.threadId ||
|
|
261
|
+
row.through_sequence !== progress.throughSequence ||
|
|
262
|
+
row.epoch !== progress.epoch ||
|
|
263
|
+
row.owner !== progress.owner ||
|
|
264
|
+
row.lease_expires_at !== progress.leaseExpiresAt
|
|
265
|
+
) {
|
|
266
|
+
return yield* storeError(operation, "corrupt");
|
|
267
|
+
}
|
|
268
|
+
return progress;
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
const checkChanged = Effect.fn("SqliteActivityStore.checkChanged")(function* (
|
|
272
|
+
operation: string,
|
|
273
|
+
): Effect.fn.Return<void, ActivityStoreError> {
|
|
274
|
+
const rawRows = yield* query(
|
|
275
|
+
sql<Record<string, unknown>>`SELECT changes() AS changed`,
|
|
276
|
+
operation,
|
|
277
|
+
);
|
|
278
|
+
const rows = yield* decodeRows(ActivityChangeCountRow, rawRows, operation);
|
|
279
|
+
if (rows.length !== 1 || rows[0].changed !== 1) {
|
|
280
|
+
return yield* storeError(operation, "corrupt");
|
|
281
|
+
}
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
const insertProgress = Effect.fn("SqliteActivityStore.insertProgress")(function* (
|
|
285
|
+
progress: ActivityProgress,
|
|
286
|
+
operation: string,
|
|
287
|
+
) {
|
|
288
|
+
const progressJson = yield* encodeProgress(progress, operation);
|
|
289
|
+
yield* sql`
|
|
290
|
+
INSERT INTO effect_agent_activity_processor_state_v1 (
|
|
291
|
+
processor_id, processor_version, thread_id, format_version, through_sequence,
|
|
292
|
+
epoch, owner, lease_expires_at, progress_json
|
|
293
|
+
) VALUES (
|
|
294
|
+
${progress.key.processorId}, ${progress.key.processorVersion}, ${progress.key.threadId},
|
|
295
|
+
${STORAGE_VERSION}, ${progress.throughSequence}, ${progress.epoch}, ${progress.owner},
|
|
296
|
+
${progress.leaseExpiresAt}, ${progressJson}
|
|
297
|
+
)
|
|
298
|
+
`;
|
|
299
|
+
yield* checkChanged(operation);
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
const updateProgress = Effect.fn("SqliteActivityStore.updateProgress")(function* (
|
|
303
|
+
current: ActivityProgress,
|
|
304
|
+
next: ActivityProgress,
|
|
305
|
+
operation: string,
|
|
306
|
+
) {
|
|
307
|
+
const progressJson = yield* encodeProgress(next, operation);
|
|
308
|
+
yield* sql`
|
|
309
|
+
UPDATE effect_agent_activity_processor_state_v1
|
|
310
|
+
SET format_version = ${STORAGE_VERSION},
|
|
311
|
+
through_sequence = ${next.throughSequence},
|
|
312
|
+
epoch = ${next.epoch},
|
|
313
|
+
owner = ${next.owner},
|
|
314
|
+
lease_expires_at = ${next.leaseExpiresAt},
|
|
315
|
+
progress_json = ${progressJson}
|
|
316
|
+
WHERE processor_id = ${current.key.processorId}
|
|
317
|
+
AND processor_version = ${current.key.processorVersion}
|
|
318
|
+
AND thread_id = ${current.key.threadId}
|
|
319
|
+
AND through_sequence = ${current.throughSequence}
|
|
320
|
+
AND epoch = ${current.epoch}
|
|
321
|
+
`;
|
|
322
|
+
yield* checkChanged(operation);
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
const requireLive = Effect.fn("SqliteActivityStore.requireLive")(function* (
|
|
326
|
+
progress: ActivityProgress | null,
|
|
327
|
+
claim: ActivityClaim,
|
|
328
|
+
requireSequence: boolean,
|
|
329
|
+
): Effect.fn.Return<ActivityProgress, ActivityOwnershipLost> {
|
|
330
|
+
const now = yield* Clock.currentTimeMillis;
|
|
331
|
+
if (
|
|
332
|
+
progress === null ||
|
|
333
|
+
!sameKey(progress.key, claim.key) ||
|
|
334
|
+
progress.owner !== claim.owner ||
|
|
335
|
+
progress.epoch !== claim.epoch ||
|
|
336
|
+
progress.leaseExpiresAt <= now ||
|
|
337
|
+
(requireSequence && progress.throughSequence !== claim.throughSequence)
|
|
338
|
+
) {
|
|
339
|
+
return yield* ownershipLost(claim);
|
|
340
|
+
}
|
|
341
|
+
return progress;
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
const inspect: ActivityProcessorStore["Service"]["inspect"] = Effect.fn(
|
|
345
|
+
"SqliteActivityStore.inspect",
|
|
346
|
+
)(function* (key) {
|
|
347
|
+
const decodedKey = yield* decodeInput(ActivityProcessorKey, key, "inspect activity progress");
|
|
348
|
+
return yield* readProgress(decodedKey, "inspect activity progress");
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
const claim: ActivityProcessorStore["Service"]["claim"] = Effect.fn("SqliteActivityStore.claim")(
|
|
352
|
+
function* (request) {
|
|
353
|
+
const operation = "claim activity progress";
|
|
354
|
+
const decoded = yield* decodeInput(ActivityClaimRequest, request, operation);
|
|
355
|
+
yield* failpoint.hit("activity:claim:before");
|
|
356
|
+
const claimed = yield* sql
|
|
357
|
+
.withTransaction(
|
|
358
|
+
Effect.gen(function* () {
|
|
359
|
+
const current = yield* readProgress(decoded.key, operation);
|
|
360
|
+
const now = yield* Clock.currentTimeMillis;
|
|
361
|
+
if (current !== null && current.owner !== null && current.leaseExpiresAt > now) {
|
|
362
|
+
return yield* ActivityBusy.make({
|
|
363
|
+
key: decoded.key,
|
|
364
|
+
leaseExpiresAt: current.leaseExpiresAt,
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
const next = yield* Schema.decodeUnknownEffect(ActivityProgress)({
|
|
368
|
+
version: STORAGE_VERSION,
|
|
369
|
+
key: decoded.key,
|
|
370
|
+
throughSequence: current?.throughSequence ?? 0,
|
|
371
|
+
epoch: (current?.epoch ?? 0) + 1,
|
|
372
|
+
owner: decoded.owner,
|
|
373
|
+
leaseExpiresAt: now + decoded.leaseMillis,
|
|
374
|
+
pending: current?.pending ?? null,
|
|
375
|
+
advancedAt: current?.advancedAt ?? null,
|
|
376
|
+
}).pipe(Effect.mapError(() => storeError(operation, "corrupt")));
|
|
377
|
+
if (current === null) yield* insertProgress(next, operation);
|
|
378
|
+
else yield* updateProgress(current, next, operation);
|
|
379
|
+
yield* failpoint.hit("activity:claim:after-state");
|
|
380
|
+
const result = makeClaim(next);
|
|
381
|
+
if (result === null) return yield* storeError(operation, "corrupt");
|
|
382
|
+
return result;
|
|
383
|
+
}),
|
|
384
|
+
)
|
|
385
|
+
.pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError(operation))));
|
|
386
|
+
yield* failpoint.hit("activity:claim:after");
|
|
387
|
+
return claimed;
|
|
388
|
+
},
|
|
389
|
+
);
|
|
390
|
+
|
|
391
|
+
const prepare: ActivityProcessorStore["Service"]["prepare"] = Effect.fn(
|
|
392
|
+
"SqliteActivityStore.prepare",
|
|
393
|
+
)(function* (request) {
|
|
394
|
+
const operation = "prepare activity output";
|
|
395
|
+
const claim = yield* decodeInput(ActivityClaim, request.claim, operation);
|
|
396
|
+
const work = yield* decodeInput(PreparedActivity, request.work, operation);
|
|
397
|
+
yield* failpoint.hit("activity:prepare:before");
|
|
398
|
+
const result = yield* sql
|
|
399
|
+
.withTransaction(
|
|
400
|
+
Effect.gen(function* () {
|
|
401
|
+
const current = yield* requireLive(
|
|
402
|
+
yield* readProgress(claim.key, operation),
|
|
403
|
+
claim,
|
|
404
|
+
true,
|
|
405
|
+
);
|
|
406
|
+
if (current.pending !== null) {
|
|
407
|
+
if (sameWork(current.pending, work)) {
|
|
408
|
+
return { work: current.pending, changed: false } as const;
|
|
409
|
+
}
|
|
410
|
+
return yield* ActivityWorkConflict.make({ key: claim.key, workId: work.workId });
|
|
411
|
+
}
|
|
412
|
+
if (!sameKey(work.key, claim.key) || work.sequence !== current.throughSequence + 1) {
|
|
413
|
+
return yield* ActivityWorkConflict.make({ key: claim.key, workId: work.workId });
|
|
414
|
+
}
|
|
415
|
+
const next = ActivityProgress.make({ ...current, pending: work });
|
|
416
|
+
yield* updateProgress(current, next, operation);
|
|
417
|
+
yield* failpoint.hit("activity:prepare:after-state");
|
|
418
|
+
return { work, changed: true } as const;
|
|
419
|
+
}),
|
|
420
|
+
)
|
|
421
|
+
.pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError(operation))));
|
|
422
|
+
if (result.changed) yield* failpoint.hit("activity:prepare:after");
|
|
423
|
+
return result.work;
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
const advance: ActivityProcessorStore["Service"]["advance"] = Effect.fn(
|
|
427
|
+
"SqliteActivityStore.advance",
|
|
428
|
+
)(function* (request) {
|
|
429
|
+
const operation = "advance activity progress";
|
|
430
|
+
const claim = yield* decodeInput(ActivityClaim, request.claim, operation);
|
|
431
|
+
const workId = yield* decodeInput(Digest, request.workId, operation);
|
|
432
|
+
yield* failpoint.hit("activity:advance:before");
|
|
433
|
+
const nextClaim = yield* sql
|
|
434
|
+
.withTransaction(
|
|
435
|
+
Effect.gen(function* () {
|
|
436
|
+
const current = yield* requireLive(
|
|
437
|
+
yield* readProgress(claim.key, operation),
|
|
438
|
+
claim,
|
|
439
|
+
true,
|
|
440
|
+
);
|
|
441
|
+
if (current.pending === null || current.pending.workId !== workId) {
|
|
442
|
+
return yield* ActivityWorkConflict.make({ key: claim.key, workId });
|
|
443
|
+
}
|
|
444
|
+
const next = ActivityProgress.make({
|
|
445
|
+
...current,
|
|
446
|
+
throughSequence: current.pending.sequence,
|
|
447
|
+
pending: null,
|
|
448
|
+
advancedAt: yield* Clock.currentTimeMillis,
|
|
449
|
+
});
|
|
450
|
+
yield* updateProgress(current, next, operation);
|
|
451
|
+
yield* failpoint.hit("activity:advance:after-state");
|
|
452
|
+
const result = makeClaim(next);
|
|
453
|
+
if (result === null) return yield* storeError(operation, "corrupt");
|
|
454
|
+
return result;
|
|
455
|
+
}),
|
|
456
|
+
)
|
|
457
|
+
.pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError(operation))));
|
|
458
|
+
yield* failpoint.hit("activity:advance:after");
|
|
459
|
+
return nextClaim;
|
|
460
|
+
});
|
|
461
|
+
|
|
462
|
+
const release: ActivityProcessorStore["Service"]["release"] = Effect.fn(
|
|
463
|
+
"SqliteActivityStore.release",
|
|
464
|
+
)(function* (claim) {
|
|
465
|
+
const operation = "release activity claim";
|
|
466
|
+
const decoded = yield* decodeInput(ActivityClaim, claim, operation);
|
|
467
|
+
yield* failpoint.hit("activity:release:before");
|
|
468
|
+
yield* sql
|
|
469
|
+
.withTransaction(
|
|
470
|
+
Effect.gen(function* () {
|
|
471
|
+
const current = yield* readProgress(decoded.key, operation);
|
|
472
|
+
if (
|
|
473
|
+
current === null ||
|
|
474
|
+
current.owner !== decoded.owner ||
|
|
475
|
+
current.epoch !== decoded.epoch
|
|
476
|
+
) {
|
|
477
|
+
return yield* ownershipLost(decoded);
|
|
478
|
+
}
|
|
479
|
+
const next = ActivityProgress.make({ ...current, owner: null, leaseExpiresAt: 0 });
|
|
480
|
+
yield* updateProgress(current, next, operation);
|
|
481
|
+
yield* failpoint.hit("activity:release:after-state");
|
|
482
|
+
}),
|
|
483
|
+
)
|
|
484
|
+
.pipe(Effect.catchTag("SqlError", () => Effect.fail(storeError(operation))));
|
|
485
|
+
yield* failpoint.hit("activity:release:after");
|
|
486
|
+
});
|
|
487
|
+
|
|
488
|
+
return ActivityProcessorStore.of({ inspect, claim, prepare, advance, release });
|
|
489
|
+
});
|
|
490
|
+
|
|
491
|
+
/** SQLite activity progress with mutation failpoints kept injectable for recovery tests. */
|
|
492
|
+
export const activityProcessorStoreLayerWithFailpoints: Layer.Layer<
|
|
493
|
+
ActivityProcessorStore,
|
|
494
|
+
SqliteActivityInitializationError,
|
|
495
|
+
SqlClientService.SqlClient | ActivityMutationFailpoint
|
|
496
|
+
> = Layer.effect(ActivityProcessorStore, makeActivityStore());
|
|
497
|
+
|
|
498
|
+
/** SQLite activity progress with the production no-op mutation failpoint. */
|
|
499
|
+
export const activityProcessorStoreLayer: Layer.Layer<
|
|
500
|
+
ActivityProcessorStore,
|
|
501
|
+
SqliteActivityInitializationError,
|
|
502
|
+
SqlClientService.SqlClient
|
|
503
|
+
> = activityProcessorStoreLayerWithFailpoints.pipe(Layer.provide(ActivityMutationFailpoint.layer));
|