@streamsy/storage 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +134 -2
- package/dist/bun.d.ts +20 -0
- package/dist/bun.js +109 -0
- package/dist/durable-object.d.ts +24 -0
- package/dist/durable-object.js +33 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +13 -0
- package/dist/rolldown-runtime-D7D4PA-g.js +13 -0
- package/dist/storage-CpnFxwO5.js +838 -0
- package/dist/storage-CrMmTyyG.d.ts +43 -0
- package/package.json +81 -4
|
@@ -0,0 +1,838 @@
|
|
|
1
|
+
import { Clock, Context, Deferred, Effect, Fiber, Layer, Option, Predicate, Queue, Schedule, Schema, Stream } from "effect";
|
|
2
|
+
import * as Scheduler from "effect/Scheduler";
|
|
3
|
+
import * as Reactivity from "effect/reactivity/Reactivity";
|
|
4
|
+
import * as SqlClient from "effect/sql/SqlClient";
|
|
5
|
+
import { isSqlError } from "effect/sql/SqlError";
|
|
6
|
+
import { OwnerRecords, attachOwner, ownerOf } from "@streamsy/core/internal";
|
|
7
|
+
import { CommitBoundary, MutationRejected, Mutations, Offset, OperationResults, Storage, StorageFault, StreamId, ZERO_OFFSET } from "@streamsy/core";
|
|
8
|
+
import { Protocol } from "@streamsy/core/protocol";
|
|
9
|
+
const transactionRetryPolicy = (options) => {
|
|
10
|
+
const attempts = options.transactionRetryAttempts ?? 16;
|
|
11
|
+
const delayMs = options.transactionRetryDelayMs ?? 25;
|
|
12
|
+
if (!Number.isSafeInteger(attempts) || attempts <= 0) throw new RangeError("transactionRetryAttempts must be a positive safe integer");
|
|
13
|
+
if (!Number.isFinite(delayMs) || delayMs <= 0) throw new RangeError("transactionRetryDelayMs must be positive");
|
|
14
|
+
return {
|
|
15
|
+
attempts,
|
|
16
|
+
delayMs
|
|
17
|
+
};
|
|
18
|
+
};
|
|
19
|
+
/** Retry only explicitly classified failures; defects and interruption are never retried. */
|
|
20
|
+
const retryWithPolicy = (transaction, policy, isRetryable) => transaction.pipe(Effect.retry({
|
|
21
|
+
times: policy.attempts - 1,
|
|
22
|
+
schedule: Schedule.spaced(policy.delayMs),
|
|
23
|
+
while: isRetryable
|
|
24
|
+
}));
|
|
25
|
+
//#endregion
|
|
26
|
+
//#region src/boundary.ts
|
|
27
|
+
var PendingInvalidations = class extends Context.Service()("@streamsy/storage/PendingInvalidations") {};
|
|
28
|
+
const noop = () => {};
|
|
29
|
+
var BoundaryRuntimeService = class extends Context.Service()("@streamsy/storage/BoundaryRuntime") {};
|
|
30
|
+
const sharedSqlClientLayer = (makeClient) => Context.empty().pipe(Effect.succeed, Effect.flatMap(() => Reactivity.make), Effect.flatMap((reactivity) => makeClient.pipe(Effect.provideService(Reactivity.Reactivity, reactivity), Effect.map((client) => Context.empty().pipe(Context.add(SqlClient.SqlClient, client), Context.add(Reactivity.Reactivity, reactivity))))));
|
|
31
|
+
/**
|
|
32
|
+
* Build a `SqlClient` and provide it with the `Reactivity` it captured, so the
|
|
33
|
+
* client and the storage boundary share one change graph.
|
|
34
|
+
*
|
|
35
|
+
* ```ts
|
|
36
|
+
* const storage = Storage.layer(streams).pipe(
|
|
37
|
+
* Layer.provide(Storage.clientLayer(PgClient.make({ url: Redacted.make(url) }))),
|
|
38
|
+
* );
|
|
39
|
+
* ```
|
|
40
|
+
*/
|
|
41
|
+
const clientLayer = (makeClient) => Layer.effectContext(sharedSqlClientLayer(makeClient));
|
|
42
|
+
const makeBoundary = (repairIntervalMs, transactionRetry, probe, transactionMaxOpsBeforeYield) => Effect.gen(function* () {
|
|
43
|
+
if (!Number.isFinite(repairIntervalMs) || repairIntervalMs <= 0) return yield* Effect.die(/* @__PURE__ */ new RangeError("repairIntervalMs must be positive"));
|
|
44
|
+
const sql = yield* SqlClient.SqlClient;
|
|
45
|
+
const reactivity = yield* Reactivity.Reactivity;
|
|
46
|
+
const subscriptions = /* @__PURE__ */ new Set();
|
|
47
|
+
let closed = false;
|
|
48
|
+
yield* Effect.addFinalizer(() => Effect.gen(function* () {
|
|
49
|
+
const active = yield* Effect.sync(() => {
|
|
50
|
+
closed = true;
|
|
51
|
+
if (probe !== void 0) probe.closed = true;
|
|
52
|
+
return [...subscriptions];
|
|
53
|
+
});
|
|
54
|
+
for (const subscription of active) yield* subscription.cleanup;
|
|
55
|
+
}));
|
|
56
|
+
const ownedTransaction = (body) => Effect.suspend(() => {
|
|
57
|
+
if (probe !== void 0) probe.transactionAttempts += 1;
|
|
58
|
+
const pending = /* @__PURE__ */ new Set();
|
|
59
|
+
let bodyCompleted = false;
|
|
60
|
+
const transactionBody = body.pipe(Effect.provideService(PendingInvalidations, pending), Effect.tap(() => Effect.sync(() => {
|
|
61
|
+
bodyCompleted = true;
|
|
62
|
+
})));
|
|
63
|
+
const transaction = sql.withTransaction(transactionBody).pipe(Effect.catchIf((error) => bodyCompleted && isSqlError(error), (error) => Effect.die(error)));
|
|
64
|
+
return (transactionMaxOpsBeforeYield === void 0 ? transaction : transaction.pipe(Effect.provideService(Scheduler.MaxOpsBeforeYield, transactionMaxOpsBeforeYield))).pipe(Effect.tap(() => {
|
|
65
|
+
if (pending.size === 0) return Effect.void;
|
|
66
|
+
if (probe !== void 0) probe.invalidations += 1;
|
|
67
|
+
return reactivity.invalidate([...pending]);
|
|
68
|
+
}));
|
|
69
|
+
});
|
|
70
|
+
const withTransaction = (body) => Effect.flatMap(Effect.serviceOption(PendingInvalidations), (owned) => {
|
|
71
|
+
if (Option.isSome(owned)) return body;
|
|
72
|
+
return Effect.flatMap(Effect.serviceOption(sql.transactionService), (ambient) => {
|
|
73
|
+
if (Option.isSome(ambient)) return Effect.die(/* @__PURE__ */ new Error("Commit boundary cannot enter an unowned ambient SQL transaction"));
|
|
74
|
+
return ownedTransaction(body);
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
const mutation = ({ keys, effect, retryable }) => Effect.flatMap(Effect.serviceOption(sql.transactionService), (ambient) => Effect.flatMap(Effect.serviceOption(PendingInvalidations), (pendingBefore) => {
|
|
78
|
+
if (Option.isSome(ambient) && Option.isNone(pendingBefore)) return Effect.die(/* @__PURE__ */ new Error("Ambient SQL transaction is not owned by CommitBoundary"));
|
|
79
|
+
const run = Effect.tap(effect, () => Effect.flatMap(Effect.serviceOption(PendingInvalidations), (pending) => {
|
|
80
|
+
if (Option.isNone(pending)) return Effect.die(/* @__PURE__ */ new Error("CommitBoundary owner context disappeared"));
|
|
81
|
+
return Effect.sync(() => {
|
|
82
|
+
for (const key of keys) pending.value.add(key);
|
|
83
|
+
});
|
|
84
|
+
}));
|
|
85
|
+
return Option.isSome(ambient) ? run : retryWithPolicy(ownedTransaction(run), transactionRetry, (error) => isSqlError(error) ? error.isRetryable : retryable(error));
|
|
86
|
+
})).pipe(Effect.onExit(() => Effect.sync(() => {
|
|
87
|
+
if (probe !== void 0) probe.completedMutations += 1;
|
|
88
|
+
})));
|
|
89
|
+
const changes = ({ keys, read }) => Stream.unwrap(Effect.gen(function* () {
|
|
90
|
+
const subscription = yield* Effect.acquireRelease(Effect.gen(function* () {
|
|
91
|
+
const queue = yield* Queue.make({
|
|
92
|
+
capacity: 1,
|
|
93
|
+
strategy: "dropping"
|
|
94
|
+
});
|
|
95
|
+
const repairStarted = yield* Deferred.make();
|
|
96
|
+
const repairFiber = yield* Effect.acquireUseRelease(Effect.sync(() => {
|
|
97
|
+
if (probe !== void 0) probe.activeRepairFibers += 1;
|
|
98
|
+
}), () => Deferred.succeed(repairStarted, void 0).pipe(Effect.andThen(Effect.sleep(repairIntervalMs)), Effect.andThen(Effect.suspend(() => {
|
|
99
|
+
if (probe?.repairsPaused === true) return Effect.void;
|
|
100
|
+
if (probe !== void 0) probe.repairPasses += 1;
|
|
101
|
+
return Queue.offer(queue, void 0);
|
|
102
|
+
})), Effect.repeat(Schedule.forever)), () => Effect.sync(() => {
|
|
103
|
+
if (probe !== void 0) probe.activeRepairFibers -= 1;
|
|
104
|
+
})).pipe(Effect.forkScoped);
|
|
105
|
+
yield* Deferred.await(repairStarted);
|
|
106
|
+
let cleaned = false;
|
|
107
|
+
let cancel = noop;
|
|
108
|
+
const active = {
|
|
109
|
+
queue,
|
|
110
|
+
cleanup: Effect.suspend(() => {
|
|
111
|
+
if (cleaned) return Effect.void;
|
|
112
|
+
cleaned = true;
|
|
113
|
+
subscriptions.delete(active);
|
|
114
|
+
probe?.queues.delete(queue);
|
|
115
|
+
if (probe !== void 0) probe.activeRegistrations -= 1;
|
|
116
|
+
cancel();
|
|
117
|
+
return Fiber.interrupt(repairFiber).pipe(Effect.andThen(Queue.shutdown(queue)));
|
|
118
|
+
})
|
|
119
|
+
};
|
|
120
|
+
if (!(yield* Effect.sync(() => {
|
|
121
|
+
if (closed) return false;
|
|
122
|
+
cancel = reactivity.registerUnsafe(keys, () => Queue.offerUnsafe(queue, void 0));
|
|
123
|
+
subscriptions.add(active);
|
|
124
|
+
probe?.queues.add(queue);
|
|
125
|
+
if (probe !== void 0) probe.activeRegistrations += 1;
|
|
126
|
+
return true;
|
|
127
|
+
}))) {
|
|
128
|
+
yield* Fiber.interrupt(repairFiber);
|
|
129
|
+
yield* Queue.shutdown(queue);
|
|
130
|
+
return yield* Effect.interrupt;
|
|
131
|
+
}
|
|
132
|
+
if (probe?.afterRegister !== void 0) yield* probe.afterRegister;
|
|
133
|
+
return active;
|
|
134
|
+
}), (active) => active.cleanup);
|
|
135
|
+
return Stream.concat(Stream.fromEffect(read), Stream.fromQueue(subscription.queue).pipe(Stream.mapEffect(() => read)));
|
|
136
|
+
}));
|
|
137
|
+
return {
|
|
138
|
+
withTransaction,
|
|
139
|
+
mutation,
|
|
140
|
+
changes
|
|
141
|
+
};
|
|
142
|
+
});
|
|
143
|
+
const boundaryLayer = (repairIntervalMs, transactionRetry, probe, transactionMaxOpsBeforeYield) => Layer.effect(BoundaryRuntimeService, makeBoundary(repairIntervalMs, transactionRetry, probe, transactionMaxOpsBeforeYield));
|
|
144
|
+
//#endregion
|
|
145
|
+
//#region src/dialect.ts
|
|
146
|
+
/** The client's dialect. Other dialects fail with a non-retryable fault. */
|
|
147
|
+
const storageDialect = (sql) => sql.onDialectOrElse({
|
|
148
|
+
sqlite: () => Effect.succeed("sqlite"),
|
|
149
|
+
pg: () => Effect.succeed("pg"),
|
|
150
|
+
mysql: () => Effect.succeed("mysql"),
|
|
151
|
+
orElse: () => Effect.fail(new StorageFault({
|
|
152
|
+
operation: "storage.dialect",
|
|
153
|
+
message: "Unsupported SQL dialect; Streamsy storage supports sqlite, pg and mysql",
|
|
154
|
+
retryable: false
|
|
155
|
+
}))
|
|
156
|
+
});
|
|
157
|
+
//#endregion
|
|
158
|
+
//#region src/schema.ts
|
|
159
|
+
const STORAGE_VERSION_TABLE = "streamsy_schema";
|
|
160
|
+
/** Version tables of earlier formats. A database that holds one must not be migrated. */
|
|
161
|
+
const LEGACY_VERSION_TABLES = ["streamsy_schema_version", "streamsy_storage_schema_version"];
|
|
162
|
+
const STORAGE_TABLES = [
|
|
163
|
+
"streamsy_streams",
|
|
164
|
+
"streamsy_messages",
|
|
165
|
+
"streamsy_producers",
|
|
166
|
+
"streamsy_owner_records"
|
|
167
|
+
];
|
|
168
|
+
const STORAGE_VERSION_TABLE_DDL = {
|
|
169
|
+
sqlite: `CREATE TABLE ${STORAGE_VERSION_TABLE} (version INTEGER PRIMARY KEY, applied_at_ms INTEGER NOT NULL)`,
|
|
170
|
+
pg: `CREATE TABLE ${STORAGE_VERSION_TABLE} (version INTEGER PRIMARY KEY, applied_at_ms BIGINT NOT NULL)`,
|
|
171
|
+
mysql: `CREATE TABLE IF NOT EXISTS ${STORAGE_VERSION_TABLE} (version INT PRIMARY KEY, applied_at_ms BIGINT NOT NULL) ENGINE=InnoDB`
|
|
172
|
+
};
|
|
173
|
+
const MYSQL_STREAM_ID = "VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin";
|
|
174
|
+
const MYSQL_PRODUCER_ID = "VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin";
|
|
175
|
+
const MYSQL_OFFSET = "VARCHAR(33) CHARACTER SET ascii COLLATE ascii_bin";
|
|
176
|
+
/**
|
|
177
|
+
* Migration versions in order. Version `n` is entry `n - 1`. SQLite and
|
|
178
|
+
* Postgres run one version inside the migration transaction. MySQL commits
|
|
179
|
+
* each DDL statement by itself, so every MySQL statement must be safe to run
|
|
180
|
+
* twice.
|
|
181
|
+
*/
|
|
182
|
+
const STORAGE_MIGRATIONS = [{
|
|
183
|
+
sqlite: [
|
|
184
|
+
`CREATE TABLE streamsy_streams (
|
|
185
|
+
stream_id TEXT PRIMARY KEY, content_type TEXT NOT NULL, ttl_seconds REAL, expires_at TEXT,
|
|
186
|
+
created_at REAL NOT NULL, current_offset TEXT NOT NULL, last_seq TEXT, closed INTEGER NOT NULL,
|
|
187
|
+
closed_at REAL, forked_from TEXT, fork_offset TEXT, fork_sub_offset REAL,
|
|
188
|
+
soft_deleted INTEGER NOT NULL, expires_at_ms REAL
|
|
189
|
+
)`,
|
|
190
|
+
`CREATE TABLE streamsy_messages (
|
|
191
|
+
stream_id TEXT NOT NULL, msg_offset TEXT NOT NULL, timestamp REAL NOT NULL, data BLOB NOT NULL,
|
|
192
|
+
PRIMARY KEY(stream_id, msg_offset),
|
|
193
|
+
FOREIGN KEY(stream_id) REFERENCES streamsy_streams(stream_id) ON DELETE CASCADE
|
|
194
|
+
)`,
|
|
195
|
+
`CREATE TABLE streamsy_producers (
|
|
196
|
+
stream_id TEXT NOT NULL, producer_id TEXT NOT NULL, epoch REAL NOT NULL, last_seq REAL NOT NULL,
|
|
197
|
+
PRIMARY KEY(stream_id, producer_id),
|
|
198
|
+
FOREIGN KEY(stream_id) REFERENCES streamsy_streams(stream_id) ON DELETE CASCADE
|
|
199
|
+
)`,
|
|
200
|
+
`CREATE INDEX idx_streamsy_streams_forked_from ON streamsy_streams(forked_from)
|
|
201
|
+
WHERE forked_from IS NOT NULL`,
|
|
202
|
+
`CREATE INDEX idx_streamsy_streams_expiry ON streamsy_streams(expires_at_ms, stream_id)
|
|
203
|
+
WHERE expires_at_ms IS NOT NULL AND soft_deleted = 0`,
|
|
204
|
+
`CREATE TABLE streamsy_owner_records (owner_key TEXT PRIMARY KEY, owner_value TEXT NOT NULL)`
|
|
205
|
+
],
|
|
206
|
+
pg: [
|
|
207
|
+
`CREATE TABLE streamsy_streams (
|
|
208
|
+
stream_id TEXT COLLATE "C" PRIMARY KEY, content_type TEXT NOT NULL,
|
|
209
|
+
ttl_seconds DOUBLE PRECISION, expires_at TEXT, created_at DOUBLE PRECISION NOT NULL,
|
|
210
|
+
current_offset TEXT COLLATE "C" NOT NULL, last_seq TEXT, closed SMALLINT NOT NULL,
|
|
211
|
+
closed_at DOUBLE PRECISION, forked_from TEXT COLLATE "C", fork_offset TEXT COLLATE "C",
|
|
212
|
+
fork_sub_offset DOUBLE PRECISION, soft_deleted SMALLINT NOT NULL, expires_at_ms DOUBLE PRECISION
|
|
213
|
+
)`,
|
|
214
|
+
`CREATE TABLE streamsy_messages (
|
|
215
|
+
stream_id TEXT COLLATE "C" NOT NULL, msg_offset TEXT COLLATE "C" NOT NULL,
|
|
216
|
+
timestamp DOUBLE PRECISION NOT NULL, data BYTEA NOT NULL,
|
|
217
|
+
PRIMARY KEY(stream_id, msg_offset),
|
|
218
|
+
FOREIGN KEY(stream_id) REFERENCES streamsy_streams(stream_id) ON DELETE CASCADE
|
|
219
|
+
)`,
|
|
220
|
+
`CREATE TABLE streamsy_producers (
|
|
221
|
+
stream_id TEXT COLLATE "C" NOT NULL, producer_id TEXT COLLATE "C" NOT NULL,
|
|
222
|
+
epoch DOUBLE PRECISION NOT NULL, last_seq DOUBLE PRECISION NOT NULL,
|
|
223
|
+
PRIMARY KEY(stream_id, producer_id),
|
|
224
|
+
FOREIGN KEY(stream_id) REFERENCES streamsy_streams(stream_id) ON DELETE CASCADE
|
|
225
|
+
)`,
|
|
226
|
+
`CREATE INDEX idx_streamsy_streams_forked_from ON streamsy_streams(forked_from)
|
|
227
|
+
WHERE forked_from IS NOT NULL`,
|
|
228
|
+
`CREATE INDEX idx_streamsy_streams_expiry ON streamsy_streams(expires_at_ms, stream_id)
|
|
229
|
+
WHERE expires_at_ms IS NOT NULL AND soft_deleted = 0`,
|
|
230
|
+
`CREATE TABLE streamsy_owner_records (
|
|
231
|
+
owner_key TEXT COLLATE "C" PRIMARY KEY, owner_value TEXT NOT NULL
|
|
232
|
+
)`
|
|
233
|
+
],
|
|
234
|
+
mysql: [
|
|
235
|
+
`CREATE TABLE IF NOT EXISTS streamsy_streams (
|
|
236
|
+
stream_id ${MYSQL_STREAM_ID} PRIMARY KEY, content_type TEXT NOT NULL, ttl_seconds DOUBLE,
|
|
237
|
+
expires_at TEXT, created_at DOUBLE NOT NULL, current_offset ${MYSQL_OFFSET} NOT NULL,
|
|
238
|
+
last_seq TEXT, closed TINYINT NOT NULL, closed_at DOUBLE, forked_from ${MYSQL_STREAM_ID},
|
|
239
|
+
fork_offset ${MYSQL_OFFSET}, fork_sub_offset DOUBLE, soft_deleted TINYINT NOT NULL,
|
|
240
|
+
expires_at_ms DOUBLE,
|
|
241
|
+
INDEX idx_streamsy_streams_forked_from (forked_from),
|
|
242
|
+
INDEX idx_streamsy_streams_expiry (expires_at_ms, stream_id)
|
|
243
|
+
) ENGINE=InnoDB`,
|
|
244
|
+
`CREATE TABLE IF NOT EXISTS streamsy_messages (
|
|
245
|
+
stream_id ${MYSQL_STREAM_ID} NOT NULL, msg_offset ${MYSQL_OFFSET} NOT NULL,
|
|
246
|
+
timestamp DOUBLE NOT NULL, data LONGBLOB NOT NULL,
|
|
247
|
+
PRIMARY KEY(stream_id, msg_offset),
|
|
248
|
+
FOREIGN KEY(stream_id) REFERENCES streamsy_streams(stream_id) ON DELETE CASCADE
|
|
249
|
+
) ENGINE=InnoDB`,
|
|
250
|
+
`CREATE TABLE IF NOT EXISTS streamsy_producers (
|
|
251
|
+
stream_id ${MYSQL_STREAM_ID} NOT NULL, producer_id ${MYSQL_PRODUCER_ID} NOT NULL,
|
|
252
|
+
epoch DOUBLE NOT NULL, last_seq DOUBLE NOT NULL,
|
|
253
|
+
PRIMARY KEY(stream_id, producer_id),
|
|
254
|
+
FOREIGN KEY(stream_id) REFERENCES streamsy_streams(stream_id) ON DELETE CASCADE
|
|
255
|
+
) ENGINE=InnoDB`,
|
|
256
|
+
`CREATE TABLE IF NOT EXISTS streamsy_owner_records (
|
|
257
|
+
owner_key VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_bin PRIMARY KEY, owner_value LONGTEXT NOT NULL
|
|
258
|
+
) ENGINE=InnoDB`
|
|
259
|
+
]
|
|
260
|
+
}];
|
|
261
|
+
//#endregion
|
|
262
|
+
//#region src/migrations.ts
|
|
263
|
+
/** Fixed `pg_advisory_xact_lock` key: "streamsy" in ASCII, read as a 64-bit integer. */
|
|
264
|
+
const PG_MIGRATION_LOCK_KEY = "8319400191411975033";
|
|
265
|
+
/** Named lock that serialises MySQL migrations, whose DDL cannot share one transaction. */
|
|
266
|
+
const MYSQL_MIGRATION_LOCK = STORAGE_VERSION_TABLE;
|
|
267
|
+
const MYSQL_MIGRATION_LOCK_TIMEOUT_SECONDS = 30;
|
|
268
|
+
const fault = (operation, error) => new StorageFault({
|
|
269
|
+
operation,
|
|
270
|
+
message: error.message,
|
|
271
|
+
retryable: error.isRetryable,
|
|
272
|
+
cause: error
|
|
273
|
+
});
|
|
274
|
+
const sqlAttempt = (operation, effect) => effect.pipe(Effect.mapError((error) => fault(operation, error)));
|
|
275
|
+
/** Return whether the database already holds this format, or fail on an older format. */
|
|
276
|
+
const inspectFormat = (tables) => {
|
|
277
|
+
const ownsFormat = tables.has(STORAGE_VERSION_TABLE);
|
|
278
|
+
return LEGACY_VERSION_TABLES.some((name) => tables.has(name)) || !ownsFormat && STORAGE_TABLES.some((name) => tables.has(name)) ? Effect.fail(new StorageFault({
|
|
279
|
+
operation: "migration.open",
|
|
280
|
+
message: "Unsupported older Streamsy storage format; use a fresh database path (the database was not modified)",
|
|
281
|
+
retryable: false
|
|
282
|
+
})) : Effect.succeed(ownsFormat);
|
|
283
|
+
};
|
|
284
|
+
const newerVersionFault = (version) => new StorageFault({
|
|
285
|
+
operation: "migration.open",
|
|
286
|
+
message: `Unsupported newer Streamsy storage schema version ${version}; supported version is 1`,
|
|
287
|
+
retryable: false
|
|
288
|
+
});
|
|
289
|
+
const listTables = (sql, dialect) => sqlAttempt("migration.inspect", {
|
|
290
|
+
sqlite: () => sql`SELECT name FROM sqlite_master WHERE type = 'table'`,
|
|
291
|
+
pg: () => sql`SELECT table_name AS name FROM information_schema.tables
|
|
292
|
+
WHERE table_schema = current_schema()`,
|
|
293
|
+
mysql: () => sql`SELECT table_name AS name FROM information_schema.tables
|
|
294
|
+
WHERE table_schema = DATABASE()`
|
|
295
|
+
}[dialect]()).pipe(Effect.map((rows) => new Set(rows.map(({ name }) => name))));
|
|
296
|
+
const migrateVersions = (sql, dialect, hooks) => Effect.gen(function* () {
|
|
297
|
+
if (!(yield* inspectFormat(yield* listTables(sql, dialect)))) yield* sqlAttempt("migration.create-version-table", sql.unsafe(STORAGE_VERSION_TABLE_DDL[dialect]).pipe(Effect.asVoid));
|
|
298
|
+
const current = yield* sqlAttempt("migration.read-version", sql`SELECT MAX(version) AS version FROM ${sql(STORAGE_VERSION_TABLE)}`.pipe(Effect.map((rows) => Number(rows[0]?.version ?? 0))));
|
|
299
|
+
if (current > 1) return yield* newerVersionFault(current);
|
|
300
|
+
for (let version = current + 1; version <= 1; version++) {
|
|
301
|
+
if (hooks.beforeVersion !== void 0) yield* hooks.beforeVersion(version);
|
|
302
|
+
const statements = STORAGE_MIGRATIONS[version - 1]?.[dialect];
|
|
303
|
+
if (statements === void 0) return yield* Effect.die(/* @__PURE__ */ new Error("Missing migration"));
|
|
304
|
+
for (const statement of statements) yield* sqlAttempt(`migration.apply.${version}`, sql.unsafe(statement).pipe(Effect.asVoid));
|
|
305
|
+
const appliedAtMs = yield* Clock.currentTimeMillis;
|
|
306
|
+
yield* sqlAttempt(`migration.record.${version}`, (dialect === "mysql" ? sql`INSERT INTO ${sql(STORAGE_VERSION_TABLE)} (version, applied_at_ms) VALUES (${version}, ${appliedAtMs})
|
|
307
|
+
ON DUPLICATE KEY UPDATE applied_at_ms = applied_at_ms` : sql`INSERT INTO ${sql(STORAGE_VERSION_TABLE)} (version, applied_at_ms) VALUES (${version}, ${appliedAtMs})`).pipe(Effect.asVoid));
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
/**
|
|
311
|
+
* MySQL commits each DDL statement by itself, so one transaction cannot hold
|
|
312
|
+
* the migration. A named lock on one reserved connection serialises
|
|
313
|
+
* instances instead, and every MySQL statement is safe to run twice.
|
|
314
|
+
*/
|
|
315
|
+
const migrateMysql = (sql, hooks) => Effect.scoped(Effect.gen(function* () {
|
|
316
|
+
const connection = yield* sqlAttempt("migration.connection", sql.reserve);
|
|
317
|
+
return yield* Effect.provideService(Effect.acquireUseRelease(sqlAttempt("migration.lock", sql`SELECT GET_LOCK(${MYSQL_MIGRATION_LOCK}, ${MYSQL_MIGRATION_LOCK_TIMEOUT_SECONDS}) AS acquired`).pipe(Effect.flatMap((rows) => Number(rows[0]?.acquired ?? 0) === 1 ? Effect.void : Effect.fail(new StorageFault({
|
|
318
|
+
operation: "migration.lock",
|
|
319
|
+
message: `Timed out waiting for the MySQL migration lock ${MYSQL_MIGRATION_LOCK}`,
|
|
320
|
+
retryable: false
|
|
321
|
+
})))), () => migrateVersions(sql, "mysql", hooks), () => sql`SELECT RELEASE_LOCK(${MYSQL_MIGRATION_LOCK})`.pipe(Effect.ignore)), sql.transactionService, [connection, 0]);
|
|
322
|
+
}));
|
|
323
|
+
/**
|
|
324
|
+
* Migrate one fresh-format database. SQLite and Postgres run under a single
|
|
325
|
+
* write transaction: the legacy check runs before the first write, and every
|
|
326
|
+
* schema and version statement rolls back together if acquisition is
|
|
327
|
+
* interrupted or a migration fails. Postgres takes an advisory lock first, so
|
|
328
|
+
* concurrent instances migrate one after the other. MySQL uses a named lock
|
|
329
|
+
* and statements that are safe to run twice.
|
|
330
|
+
*/
|
|
331
|
+
const migrate = (sql, hooks = {}, transactionRetry = transactionRetryPolicy({})) => Effect.flatMap(storageDialect(sql), (dialect) => retryWithPolicy(Effect.suspend(() => {
|
|
332
|
+
hooks.onTransactionAttempt?.();
|
|
333
|
+
if (dialect === "mysql") return migrateMysql(sql, hooks);
|
|
334
|
+
return sql.withTransaction(dialect === "pg" ? sqlAttempt("migration.lock", sql`SELECT pg_advisory_xact_lock(${sql.literal(PG_MIGRATION_LOCK_KEY)})`).pipe(Effect.andThen(migrateVersions(sql, dialect, hooks))) : migrateVersions(sql, dialect, hooks));
|
|
335
|
+
}), transactionRetry, (error) => isSqlError(error) ? error.isRetryable : error.retryable).pipe(Effect.catchIf(isSqlError, (error) => Effect.fail(fault("migration.transaction", error)))));
|
|
336
|
+
//#endregion
|
|
337
|
+
//#region src/storage.ts
|
|
338
|
+
const DEFAULT_REPAIR_INTERVAL_MS = 1e3;
|
|
339
|
+
const STORAGE_KEY = "streamsy:storage";
|
|
340
|
+
const RecordRow = Schema.Struct({
|
|
341
|
+
stream_id: StreamId,
|
|
342
|
+
content_type: Schema.String,
|
|
343
|
+
ttl_seconds: Schema.NullOr(Schema.Finite),
|
|
344
|
+
expires_at: Schema.NullOr(Schema.String),
|
|
345
|
+
created_at: Schema.Finite,
|
|
346
|
+
current_offset: Offset,
|
|
347
|
+
last_seq: Schema.NullOr(Schema.String),
|
|
348
|
+
closed: Schema.Literals([0, 1]),
|
|
349
|
+
closed_at: Schema.NullOr(Schema.Finite),
|
|
350
|
+
forked_from: Schema.NullOr(StreamId),
|
|
351
|
+
fork_offset: Schema.NullOr(Offset),
|
|
352
|
+
fork_sub_offset: Schema.NullOr(Schema.Finite),
|
|
353
|
+
soft_deleted: Schema.Literals([0, 1]),
|
|
354
|
+
expires_at_ms: Schema.NullOr(Schema.Finite)
|
|
355
|
+
});
|
|
356
|
+
const ProducerRow = Schema.Struct({
|
|
357
|
+
epoch: Schema.Finite,
|
|
358
|
+
last_seq: Schema.Finite
|
|
359
|
+
});
|
|
360
|
+
const ExpiryRow = Schema.Struct({
|
|
361
|
+
at: Schema.Finite,
|
|
362
|
+
stream_id: StreamId
|
|
363
|
+
});
|
|
364
|
+
const DependentsRow = Schema.Struct({ present: Schema.Literals([
|
|
365
|
+
0,
|
|
366
|
+
1,
|
|
367
|
+
0n,
|
|
368
|
+
1n,
|
|
369
|
+
false,
|
|
370
|
+
true
|
|
371
|
+
]) });
|
|
372
|
+
const SnapshotRow = Schema.Struct({
|
|
373
|
+
current_offset: Schema.NullOr(Offset),
|
|
374
|
+
closed: Schema.NullOr(Schema.Literals([0, 1])),
|
|
375
|
+
soft_deleted: Schema.NullOr(Schema.Literals([0, 1]))
|
|
376
|
+
});
|
|
377
|
+
const decodeRecordRow = (row) => Schema.decodeEffect(RecordRow)(row).pipe(Effect.orDie);
|
|
378
|
+
const decodeProducerRow = (row) => Schema.decodeEffect(ProducerRow)(row).pipe(Effect.orDie);
|
|
379
|
+
const decodeExpiryRow = (row) => Schema.decodeEffect(ExpiryRow)(row).pipe(Effect.orDie);
|
|
380
|
+
const decodeSnapshotRow = (row) => Schema.decodeEffect(SnapshotRow)(row).pipe(Effect.orDie);
|
|
381
|
+
const decodeDependentsRow = (row) => Schema.decodeEffect(DependentsRow)(row).pipe(Effect.orDie);
|
|
382
|
+
const toRecord = (row) => {
|
|
383
|
+
const config = {
|
|
384
|
+
contentType: row.content_type,
|
|
385
|
+
createdAt: row.created_at
|
|
386
|
+
};
|
|
387
|
+
if (row.ttl_seconds !== null) config.ttlSeconds = row.ttl_seconds;
|
|
388
|
+
if (row.expires_at !== null) config.expiresAt = row.expires_at;
|
|
389
|
+
const lifecycle = {
|
|
390
|
+
closed: row.closed === 1,
|
|
391
|
+
softDeleted: row.soft_deleted === 1
|
|
392
|
+
};
|
|
393
|
+
if (row.last_seq !== null) lifecycle.lastSeq = row.last_seq;
|
|
394
|
+
if (row.closed_at !== null) lifecycle.closedAt = row.closed_at;
|
|
395
|
+
if (row.forked_from !== null) lifecycle.forkedFrom = row.forked_from;
|
|
396
|
+
if (row.fork_offset !== null) lifecycle.forkOffset = row.fork_offset;
|
|
397
|
+
if (row.fork_sub_offset !== null) lifecycle.forkSubOffset = row.fork_sub_offset;
|
|
398
|
+
if (row.expires_at_ms !== null) lifecycle.expiresAtMs = row.expires_at_ms;
|
|
399
|
+
return {
|
|
400
|
+
id: row.stream_id,
|
|
401
|
+
config,
|
|
402
|
+
lifecycle,
|
|
403
|
+
currentOffset: row.current_offset
|
|
404
|
+
};
|
|
405
|
+
};
|
|
406
|
+
const patchRecord = (record, patch) => {
|
|
407
|
+
const config = {
|
|
408
|
+
...record.config,
|
|
409
|
+
...patch.config
|
|
410
|
+
};
|
|
411
|
+
const lifecycle = {
|
|
412
|
+
...record.lifecycle,
|
|
413
|
+
...patch.lifecycle
|
|
414
|
+
};
|
|
415
|
+
for (const field of patch.clear ?? []) {
|
|
416
|
+
if (field === "ttlSeconds") delete config.ttlSeconds;
|
|
417
|
+
if (field === "expiresAt") delete config.expiresAt;
|
|
418
|
+
if (field === "expiresAtMs") delete lifecycle.expiresAtMs;
|
|
419
|
+
if (field === "lastSeq") delete lifecycle.lastSeq;
|
|
420
|
+
}
|
|
421
|
+
return {
|
|
422
|
+
id: record.id,
|
|
423
|
+
config,
|
|
424
|
+
lifecycle,
|
|
425
|
+
currentOffset: patch.currentOffset ?? record.currentOffset
|
|
426
|
+
};
|
|
427
|
+
};
|
|
428
|
+
const copyRecord = (record) => ({
|
|
429
|
+
id: record.id,
|
|
430
|
+
currentOffset: record.currentOffset,
|
|
431
|
+
config: { ...record.config },
|
|
432
|
+
lifecycle: { ...record.lifecycle }
|
|
433
|
+
});
|
|
434
|
+
const sqlFault = (operation, error) => new StorageFault({
|
|
435
|
+
operation,
|
|
436
|
+
message: error.message,
|
|
437
|
+
retryable: error.isRetryable,
|
|
438
|
+
cause: error
|
|
439
|
+
});
|
|
440
|
+
const createFault = (error) => Predicate.isTagged(error.reason, "UniqueViolation") ? new StorageFault({
|
|
441
|
+
operation: "mutate.create-record",
|
|
442
|
+
message: error.message,
|
|
443
|
+
retryable: true,
|
|
444
|
+
cause: error
|
|
445
|
+
}) : sqlFault("mutate.create-record", error);
|
|
446
|
+
const attempt = (operation, effect) => effect.pipe(Effect.mapError((error) => sqlFault(operation, error)));
|
|
447
|
+
const execute = (operation, statement) => attempt(operation, statement).pipe(Effect.asVoid);
|
|
448
|
+
const tooLong = (operation, label, value, limit) => {
|
|
449
|
+
const length = [...value].length;
|
|
450
|
+
return length > limit ? Option.some(new StorageFault({
|
|
451
|
+
operation,
|
|
452
|
+
message: `${label} has ${length} characters; the limit is ${limit}`,
|
|
453
|
+
retryable: false
|
|
454
|
+
})) : Option.none();
|
|
455
|
+
};
|
|
456
|
+
const oversizedKey = (mutation) => {
|
|
457
|
+
for (const operation of mutation.operations) {
|
|
458
|
+
const fault = Predicate.isTagged(operation, "Create") ? tooLong("mutate.stream-id", "Stream id", operation.record.id, 512) : Predicate.isTagged(operation, "Append") && operation.producer !== void 0 ? tooLong("mutate.producer-id", "Producer id", operation.producer.producerId, 255) : Option.none();
|
|
459
|
+
if (Option.isSome(fault)) return fault;
|
|
460
|
+
}
|
|
461
|
+
return Option.none();
|
|
462
|
+
};
|
|
463
|
+
/**
|
|
464
|
+
* Serialise writers on the stream rows. This is the first statement of a
|
|
465
|
+
* mutation transaction. On MySQL, the reads after it are locking reads, so they
|
|
466
|
+
* see commits made while it waited, even when the caller read first.
|
|
467
|
+
* Sorted ids give every writer one lock order. SQLite already has one writer.
|
|
468
|
+
*/
|
|
469
|
+
const lockStreams = (sql, ids) => {
|
|
470
|
+
const sorted = [...new Set(ids)].toSorted();
|
|
471
|
+
return sql.onDialectOrElse({
|
|
472
|
+
sqlite: () => Effect.void,
|
|
473
|
+
orElse: () => execute("mutate.lock", sql`SELECT stream_id FROM streamsy_streams WHERE stream_id IN ${sql.in(sorted)}
|
|
474
|
+
ORDER BY stream_id FOR UPDATE`)
|
|
475
|
+
});
|
|
476
|
+
};
|
|
477
|
+
/**
|
|
478
|
+
* A mutation reads with a row lock on MySQL. At REPEATABLE READ a plain read uses
|
|
479
|
+
* the snapshot of the transaction's first plain read, which app SQL in a fused
|
|
480
|
+
* commit may have fixed before the stream lock. A locking read sees the latest
|
|
481
|
+
* committed row. SQLite has one writer, and Postgres at READ COMMITTED reads the
|
|
482
|
+
* latest commit in each statement, so both keep plain reads.
|
|
483
|
+
*/
|
|
484
|
+
const lockingRead = (sql, lock) => sql.onDialectOrElse({
|
|
485
|
+
mysql: () => sql.literal(lock === void 0 ? "" : lock === "update" ? " FOR UPDATE" : " FOR SHARE"),
|
|
486
|
+
orElse: () => sql.literal("")
|
|
487
|
+
});
|
|
488
|
+
/**
|
|
489
|
+
* The lock for a read the protocol plans a mutation from. Inside an open
|
|
490
|
+
* transaction on this client, such as a fused commit, it is `update`: on MySQL a
|
|
491
|
+
* plain read would return the snapshot fixed by the transaction's first read, so
|
|
492
|
+
* every replan would see the same stale tail. The row then stays locked until the
|
|
493
|
+
* mutation. Outside a transaction the read stays plain and takes no lock.
|
|
494
|
+
*/
|
|
495
|
+
const planningLock = (sql) => Effect.map(Effect.serviceOption(sql.transactionService), (open) => Option.isSome(open) ? "update" : void 0);
|
|
496
|
+
const lockIds = (mutation) => mutation.operations.flatMap((operation) => Predicate.isTagged(operation, "Create") ? operation.forkSource === void 0 ? [operation.record.id] : [operation.record.id, operation.forkSource.id] : [operation.streamId]);
|
|
497
|
+
const recordColumns = (sql) => sql.literal("stream_id,content_type,ttl_seconds,expires_at,created_at,current_offset,last_seq,closed,closed_at,forked_from,fork_offset,fork_sub_offset,soft_deleted,expires_at_ms");
|
|
498
|
+
const readRecord = (sql, id, lock) => attempt("record", sql`SELECT ${recordColumns(sql)} FROM streamsy_streams WHERE stream_id = ${id}${lockingRead(sql, lock)}`).pipe(Effect.flatMap((rows) => {
|
|
499
|
+
const row = rows[0];
|
|
500
|
+
return row === void 0 ? Effect.succeed(Option.none()) : decodeRecordRow(row).pipe(Effect.map((decoded) => Option.some(toRecord(decoded))));
|
|
501
|
+
}));
|
|
502
|
+
const recordValues = (record) => ({
|
|
503
|
+
content_type: record.config.contentType,
|
|
504
|
+
ttl_seconds: record.config.ttlSeconds ?? null,
|
|
505
|
+
expires_at: record.config.expiresAt ?? null,
|
|
506
|
+
created_at: record.config.createdAt,
|
|
507
|
+
current_offset: record.currentOffset,
|
|
508
|
+
last_seq: record.lifecycle.lastSeq ?? null,
|
|
509
|
+
closed: record.lifecycle.closed ? 1 : 0,
|
|
510
|
+
closed_at: record.lifecycle.closedAt ?? null,
|
|
511
|
+
forked_from: record.lifecycle.forkedFrom ?? null,
|
|
512
|
+
fork_offset: record.lifecycle.forkOffset ?? null,
|
|
513
|
+
fork_sub_offset: record.lifecycle.forkSubOffset ?? null,
|
|
514
|
+
soft_deleted: record.lifecycle.softDeleted ? 1 : 0,
|
|
515
|
+
expires_at_ms: record.lifecycle.expiresAtMs ?? null
|
|
516
|
+
});
|
|
517
|
+
const insertRecord = (sql, record) => sql`INSERT INTO streamsy_streams ${sql.insert({
|
|
518
|
+
stream_id: record.id,
|
|
519
|
+
...recordValues(record)
|
|
520
|
+
})}`.pipe(Effect.mapError(createFault), Effect.asVoid);
|
|
521
|
+
const updateRecord = (sql, record) => execute("mutate.update-record", sql`UPDATE streamsy_streams SET ${sql.update(recordValues(record))} WHERE stream_id = ${record.id}`);
|
|
522
|
+
const insertMessage = (sql, operation, id, message) => execute(operation, sql`INSERT INTO streamsy_messages ${sql.insert({
|
|
523
|
+
stream_id: id,
|
|
524
|
+
msg_offset: message.offset,
|
|
525
|
+
timestamp: message.timestamp,
|
|
526
|
+
data: new Uint8Array(message.data)
|
|
527
|
+
})}`);
|
|
528
|
+
const readProducer = (sql, operation, id, producerId, lock) => attempt(operation, sql`SELECT epoch, last_seq FROM streamsy_producers
|
|
529
|
+
WHERE stream_id = ${id} AND producer_id = ${producerId}${lockingRead(sql, lock)}`).pipe(Effect.flatMap((rows) => rows[0] === void 0 ? Effect.succeed(Option.none()) : decodeProducerRow(rows[0]).pipe(Effect.map(({ epoch, last_seq: lastSeq }) => Option.some({
|
|
530
|
+
epoch,
|
|
531
|
+
lastSeq
|
|
532
|
+
})))));
|
|
533
|
+
const upsertProducer = (sql, id, producerId, next) => {
|
|
534
|
+
const row = sql.insert({
|
|
535
|
+
stream_id: id,
|
|
536
|
+
producer_id: producerId,
|
|
537
|
+
epoch: next.epoch,
|
|
538
|
+
last_seq: next.lastSeq
|
|
539
|
+
});
|
|
540
|
+
return execute("mutate.producer", sql.onDialectOrElse({
|
|
541
|
+
mysql: () => sql`INSERT INTO streamsy_producers ${row}
|
|
542
|
+
ON DUPLICATE KEY UPDATE epoch = VALUES(epoch), last_seq = VALUES(last_seq)`,
|
|
543
|
+
orElse: () => sql`INSERT INTO streamsy_producers ${row}
|
|
544
|
+
ON CONFLICT (stream_id, producer_id) DO UPDATE SET epoch = excluded.epoch, last_seq = excluded.last_seq`
|
|
545
|
+
}));
|
|
546
|
+
};
|
|
547
|
+
const preflight = (sql, mutation) => Effect.gen(function* () {
|
|
548
|
+
for (const [index, operation] of mutation.operations.entries()) {
|
|
549
|
+
const id = Predicate.isTagged(operation, "Create") ? operation.record.id : operation.streamId;
|
|
550
|
+
const current = yield* readRecord(sql, id, "update");
|
|
551
|
+
if (Predicate.isTagged(operation, "Create")) {
|
|
552
|
+
if (Option.isSome(current)) return yield* new MutationRejected({
|
|
553
|
+
index,
|
|
554
|
+
reason: "exists",
|
|
555
|
+
record: current
|
|
556
|
+
});
|
|
557
|
+
if (operation.forkSource !== void 0) {
|
|
558
|
+
const source = yield* readRecord(sql, operation.forkSource.id, "share");
|
|
559
|
+
if (Option.isNone(source) || source.value.lifecycle.softDeleted || source.value.currentOffset < operation.forkSource.liveAtOffset) return yield* new MutationRejected({
|
|
560
|
+
index,
|
|
561
|
+
reason: "fork-source-gone",
|
|
562
|
+
record: Option.none()
|
|
563
|
+
});
|
|
564
|
+
}
|
|
565
|
+
continue;
|
|
566
|
+
}
|
|
567
|
+
if (Option.isNone(current)) return yield* new MutationRejected({
|
|
568
|
+
index,
|
|
569
|
+
reason: "not-found",
|
|
570
|
+
record: Option.none()
|
|
571
|
+
});
|
|
572
|
+
const record = current.value;
|
|
573
|
+
if (record.lifecycle.softDeleted) return yield* new MutationRejected({
|
|
574
|
+
index,
|
|
575
|
+
reason: "gone",
|
|
576
|
+
record: current
|
|
577
|
+
});
|
|
578
|
+
if (Predicate.isTagged(operation, "Delete")) {
|
|
579
|
+
if (operation.reason === "expiry" && (operation.expectedExpiresAtMs === void 0 || operation.expectedExpiresAtMs !== record.lifecycle.expiresAtMs)) return yield* new MutationRejected({
|
|
580
|
+
index,
|
|
581
|
+
reason: "expiry-mismatch",
|
|
582
|
+
record: current
|
|
583
|
+
});
|
|
584
|
+
continue;
|
|
585
|
+
}
|
|
586
|
+
if (operation.expectedOffset !== void 0 && operation.expectedOffset !== record.currentOffset) return yield* new MutationRejected({
|
|
587
|
+
index,
|
|
588
|
+
reason: "offset",
|
|
589
|
+
record: current
|
|
590
|
+
});
|
|
591
|
+
if (operation.expectedClosed !== void 0 && operation.expectedClosed !== record.lifecycle.closed) return yield* new MutationRejected({
|
|
592
|
+
index,
|
|
593
|
+
reason: "closed",
|
|
594
|
+
record: current
|
|
595
|
+
});
|
|
596
|
+
if (operation.producer !== void 0) {
|
|
597
|
+
const actual = yield* readProducer(sql, "mutate.preflight-producer", id, operation.producer.producerId, "update");
|
|
598
|
+
const expected = operation.producer.expected;
|
|
599
|
+
if (Option.isSome(actual) !== Option.isSome(expected) || Option.isSome(actual) && Option.isSome(expected) && (actual.value.epoch !== expected.value.epoch || actual.value.lastSeq !== expected.value.lastSeq)) return yield* new MutationRejected({
|
|
600
|
+
index,
|
|
601
|
+
reason: "producer",
|
|
602
|
+
record: current
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
});
|
|
607
|
+
const hasDependents = (sql, id) => attempt("mutate.dependents", sql`SELECT CASE WHEN EXISTS(
|
|
608
|
+
SELECT 1 FROM streamsy_streams WHERE forked_from = ${id}${lockingRead(sql, "share")}
|
|
609
|
+
) THEN 1 ELSE 0 END AS present`).pipe(Effect.flatMap((rows) => decodeDependentsRow(rows[0] ?? { present: 0 })), Effect.map(({ present }) => present === 1 || present === 1n || present === true));
|
|
610
|
+
const purge = (sql, start) => Effect.gen(function* () {
|
|
611
|
+
let current = start;
|
|
612
|
+
while (current !== void 0) {
|
|
613
|
+
const parentId = current.lifecycle.forkedFrom;
|
|
614
|
+
yield* execute("mutate.purge-messages", sql`DELETE FROM streamsy_messages WHERE stream_id = ${current.id}`);
|
|
615
|
+
yield* execute("mutate.purge-producers", sql`DELETE FROM streamsy_producers WHERE stream_id = ${current.id}`);
|
|
616
|
+
yield* execute("mutate.purge", sql`DELETE FROM streamsy_streams WHERE stream_id = ${current.id}`);
|
|
617
|
+
if (parentId === void 0) return;
|
|
618
|
+
yield* lockStreams(sql, [parentId]);
|
|
619
|
+
const parent = yield* readRecord(sql, parentId, "update");
|
|
620
|
+
if (Option.isNone(parent) || !parent.value.lifecycle.softDeleted || (yield* hasDependents(sql, parentId))) return;
|
|
621
|
+
current = parent.value;
|
|
622
|
+
}
|
|
623
|
+
});
|
|
624
|
+
const applyMutation = (sql, mutation) => Effect.gen(function* () {
|
|
625
|
+
yield* lockStreams(sql, lockIds(mutation));
|
|
626
|
+
yield* preflight(sql, mutation);
|
|
627
|
+
const records = /* @__PURE__ */ new Map();
|
|
628
|
+
const results = [];
|
|
629
|
+
for (const operation of mutation.operations) {
|
|
630
|
+
if (!Predicate.isTagged(operation, "Create")) continue;
|
|
631
|
+
yield* insertRecord(sql, operation.record);
|
|
632
|
+
for (const message of operation.initialMessages) yield* insertMessage(sql, "mutate.create-message", operation.record.id, message);
|
|
633
|
+
records.set(operation.record.id, operation.record);
|
|
634
|
+
}
|
|
635
|
+
for (const operation of mutation.operations) {
|
|
636
|
+
const id = Predicate.isTagged(operation, "Create") ? operation.record.id : operation.streamId;
|
|
637
|
+
const current = records.get(id) ?? Option.getOrThrow(yield* readRecord(sql, id, "update"));
|
|
638
|
+
if (Predicate.isTagged(operation, "Create")) {
|
|
639
|
+
results.push(OperationResults.Created({ record: copyRecord(current) }));
|
|
640
|
+
continue;
|
|
641
|
+
}
|
|
642
|
+
if (Predicate.isTagged(operation, "Append")) {
|
|
643
|
+
for (const message of operation.messages) yield* insertMessage(sql, "mutate.append-message", id, message);
|
|
644
|
+
const next = patchRecord(current, operation.patch);
|
|
645
|
+
yield* updateRecord(sql, next);
|
|
646
|
+
if (operation.producer !== void 0) yield* upsertProducer(sql, id, operation.producer.producerId, operation.producer.next);
|
|
647
|
+
records.set(id, next);
|
|
648
|
+
results.push(OperationResults.Appended({ record: next }));
|
|
649
|
+
continue;
|
|
650
|
+
}
|
|
651
|
+
if (yield* hasDependents(sql, id)) {
|
|
652
|
+
const next = patchRecord(current, { lifecycle: { softDeleted: true } });
|
|
653
|
+
yield* updateRecord(sql, next);
|
|
654
|
+
results.push(OperationResults.SoftDeleted({ record: next }));
|
|
655
|
+
} else {
|
|
656
|
+
results.push(OperationResults.Purged({ record: current }));
|
|
657
|
+
yield* purge(sql, current);
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
const [first, ...rest] = results;
|
|
661
|
+
if (first === void 0) return yield* Effect.die(/* @__PURE__ */ new Error("Empty mutation"));
|
|
662
|
+
return Mutations.Applied({ results: [first, ...rest] });
|
|
663
|
+
});
|
|
664
|
+
const directMessage = (row) => {
|
|
665
|
+
if (!Predicate.isString(row.msg_offset) || !/^\d{16}_\d{16}$/.test(row.msg_offset) || !Predicate.isNumber(row.timestamp) || !Number.isFinite(row.timestamp) || !Predicate.isUint8Array(row.data)) throw new Error("Corrupt persisted message row");
|
|
666
|
+
return {
|
|
667
|
+
offset: Offset.make(row.msg_offset),
|
|
668
|
+
timestamp: row.timestamp,
|
|
669
|
+
data: new Uint8Array(row.data)
|
|
670
|
+
};
|
|
671
|
+
};
|
|
672
|
+
const readMessages = (sql, id, window) => Effect.gen(function* () {
|
|
673
|
+
let remaining = window.limit === void 0 ? void 0 : Math.max(0, Math.trunc(window.limit));
|
|
674
|
+
if (remaining === 0) return [];
|
|
675
|
+
const visited = /* @__PURE__ */ new Set();
|
|
676
|
+
const chain = [];
|
|
677
|
+
let cursor = id;
|
|
678
|
+
let until;
|
|
679
|
+
while (cursor !== void 0) {
|
|
680
|
+
if (visited.has(cursor)) return yield* Effect.die(/* @__PURE__ */ new Error("Corrupt cyclic stream lineage"));
|
|
681
|
+
visited.add(cursor);
|
|
682
|
+
chain.push(until === void 0 ? { id: cursor } : {
|
|
683
|
+
id: cursor,
|
|
684
|
+
until
|
|
685
|
+
});
|
|
686
|
+
const record = yield* readRecord(sql, cursor);
|
|
687
|
+
if (Option.isNone(record)) break;
|
|
688
|
+
cursor = record.value.lifecycle.forkedFrom;
|
|
689
|
+
until = record.value.lifecycle.forkOffset;
|
|
690
|
+
}
|
|
691
|
+
const messages = [];
|
|
692
|
+
for (const segment of chain.toReversed()) {
|
|
693
|
+
if (remaining === 0) break;
|
|
694
|
+
const upper = segment.until === void 0 ? window.until : window.until === void 0 || segment.until < window.until ? segment.until : window.until;
|
|
695
|
+
const predicates = [sql`stream_id = ${segment.id}`];
|
|
696
|
+
if (window.after !== void 0) predicates.push(sql`msg_offset > ${window.after}`);
|
|
697
|
+
if (upper !== void 0) predicates.push(sql`msg_offset <= ${upper}`);
|
|
698
|
+
const rows = yield* attempt("messages", sql`SELECT msg_offset, timestamp, data FROM streamsy_messages
|
|
699
|
+
WHERE ${sql.and(predicates)} ORDER BY msg_offset${remaining === void 0 ? sql.literal("") : sql` LIMIT ${remaining}`}`);
|
|
700
|
+
for (const row of rows) messages.push(yield* Effect.try(() => directMessage(row)).pipe(Effect.orDie));
|
|
701
|
+
if (remaining !== void 0) remaining -= rows.length;
|
|
702
|
+
}
|
|
703
|
+
return messages;
|
|
704
|
+
});
|
|
705
|
+
const snapshot = (sql, id) => attempt("changes", sql`SELECT current_offset, closed, soft_deleted FROM streamsy_streams
|
|
706
|
+
WHERE stream_id = ${id}`).pipe(Effect.flatMap((rows) => decodeSnapshotRow(rows[0] ?? {
|
|
707
|
+
current_offset: null,
|
|
708
|
+
closed: null,
|
|
709
|
+
soft_deleted: null
|
|
710
|
+
})), Effect.map((row) => ({
|
|
711
|
+
present: row.current_offset !== null,
|
|
712
|
+
currentOffset: row.current_offset ?? ZERO_OFFSET,
|
|
713
|
+
closed: row.closed === 1,
|
|
714
|
+
softDeleted: row.soft_deleted === 1
|
|
715
|
+
})));
|
|
716
|
+
const makeStorage = (sql, boundary) => Storage.of({
|
|
717
|
+
capabilities: {
|
|
718
|
+
fork: "chain",
|
|
719
|
+
atomicScope: "store",
|
|
720
|
+
wake: "push",
|
|
721
|
+
expiryIndex: "indexed"
|
|
722
|
+
},
|
|
723
|
+
record: Effect.fn("SqlStorage.record")((id) => Effect.flatMap(planningLock(sql), (lock) => readRecord(sql, id, lock))),
|
|
724
|
+
messages: Effect.fn("SqlStorage.messages")((id, window) => readMessages(sql, id, window)),
|
|
725
|
+
producer: Effect.fn("SqlStorage.producer")((id, producerId) => Effect.flatMap(planningLock(sql), (lock) => readProducer(sql, "producer", id, producerId, lock))),
|
|
726
|
+
mutate: Effect.fn("SqlStorage.mutate")((mutation) => {
|
|
727
|
+
const ids = mutation.operations.map((operation) => Predicate.isTagged(operation, "Create") ? operation.record.id : operation.streamId);
|
|
728
|
+
if (ids.length === 0 || new Set(ids).size !== ids.length) return Effect.die(/* @__PURE__ */ new Error("Mutation requires distinct streams within atomicScope"));
|
|
729
|
+
const oversized = oversizedKey(mutation);
|
|
730
|
+
if (Option.isSome(oversized)) return Effect.fail(oversized.value);
|
|
731
|
+
return boundary.mutation({
|
|
732
|
+
keys: [STORAGE_KEY],
|
|
733
|
+
effect: applyMutation(sql, mutation),
|
|
734
|
+
retryable: (error) => Predicate.isTagged(error, "StorageFault") && error.retryable
|
|
735
|
+
}).pipe(Effect.catchIf(isSqlError, (error) => Effect.fail(sqlFault("mutate.transaction", error))));
|
|
736
|
+
}),
|
|
737
|
+
changes: (id) => boundary.changes({
|
|
738
|
+
keys: [STORAGE_KEY],
|
|
739
|
+
read: snapshot(sql, id)
|
|
740
|
+
}),
|
|
741
|
+
nextExpiry: attempt("nextExpiry", sql`SELECT expires_at_ms AS at, stream_id FROM streamsy_streams
|
|
742
|
+
WHERE expires_at_ms IS NOT NULL AND soft_deleted = 0
|
|
743
|
+
ORDER BY expires_at_ms, stream_id LIMIT 1`).pipe(Effect.flatMap((rows) => rows[0] === void 0 ? Effect.succeed(Option.none()) : decodeExpiryRow(rows[0]).pipe(Effect.map(({ at, stream_id: streamId }) => Option.some({
|
|
744
|
+
at,
|
|
745
|
+
streamId
|
|
746
|
+
})))))
|
|
747
|
+
});
|
|
748
|
+
/**
|
|
749
|
+
* The fault operation of an owner record write that lost a first-write race.
|
|
750
|
+
* `CheckpointStore.layer` in `@streamsy/projection` reports it as a token conflict.
|
|
751
|
+
*/
|
|
752
|
+
const OWNER_RECORD_CONFLICT = "records.conflict";
|
|
753
|
+
/**
|
|
754
|
+
* Encoded owner records; statements join the boundary transaction when one is open.
|
|
755
|
+
*
|
|
756
|
+
* The checkpoint store reads a record, compares its token and writes it back in one
|
|
757
|
+
* transaction. Inside a transaction, a found record is read again with
|
|
758
|
+
* `FOR UPDATE` on Postgres and MySQL, so a concurrent save waits and then sees the
|
|
759
|
+
* new token. An absent key has no row to lock, and on MySQL a locking read of it
|
|
760
|
+
* takes a gap lock that turns two first saves into a deadlock. So the first read
|
|
761
|
+
* stays plain, the transaction remembers the absent key, and its write is a plain
|
|
762
|
+
* insert: a concurrent first save then fails on the primary key and is not lost.
|
|
763
|
+
*/
|
|
764
|
+
const makeOwnerRecords = (sql) => {
|
|
765
|
+
const absentKeys = /* @__PURE__ */ new WeakMap();
|
|
766
|
+
const select = (key, forUpdate) => attempt("records.read", sql`SELECT owner_value FROM streamsy_owner_records WHERE owner_key = ${key}${sql.literal(forUpdate ? " FOR UPDATE" : "")}`).pipe(Effect.map((rows) => Option.fromUndefinedOr(rows[0]?.owner_value)));
|
|
767
|
+
const lockFound = (key, found) => Option.isNone(found) ? Effect.succeed(found) : sql.onDialectOrElse({
|
|
768
|
+
sqlite: () => Effect.succeed(found),
|
|
769
|
+
orElse: () => select(key, true)
|
|
770
|
+
});
|
|
771
|
+
const upsert = (key, encoded) => {
|
|
772
|
+
const row = sql.insert({
|
|
773
|
+
owner_key: key,
|
|
774
|
+
owner_value: encoded
|
|
775
|
+
});
|
|
776
|
+
return execute("records.write", sql.onDialectOrElse({
|
|
777
|
+
mysql: () => sql`INSERT INTO streamsy_owner_records ${row}
|
|
778
|
+
ON DUPLICATE KEY UPDATE owner_value = VALUES(owner_value)`,
|
|
779
|
+
orElse: () => sql`INSERT INTO streamsy_owner_records ${row}
|
|
780
|
+
ON CONFLICT (owner_key) DO UPDATE SET owner_value = excluded.owner_value`
|
|
781
|
+
}));
|
|
782
|
+
};
|
|
783
|
+
const insert = (key, encoded) => sql`INSERT INTO streamsy_owner_records ${sql.insert({
|
|
784
|
+
owner_key: key,
|
|
785
|
+
owner_value: encoded
|
|
786
|
+
})}`.pipe(Effect.mapError((error) => Predicate.isTagged(error.reason, "UniqueViolation") ? new StorageFault({
|
|
787
|
+
operation: OWNER_RECORD_CONFLICT,
|
|
788
|
+
message: `Owner record ${key} was created by a concurrent writer`,
|
|
789
|
+
retryable: false,
|
|
790
|
+
cause: error
|
|
791
|
+
}) : sqlFault("records.write", error)), Effect.asVoid);
|
|
792
|
+
return OwnerRecords.of({
|
|
793
|
+
read: Effect.fn("SqlStorage.records.read")(function* (key) {
|
|
794
|
+
const transaction = yield* Effect.serviceOption(sql.transactionService);
|
|
795
|
+
if (Option.isNone(transaction)) return yield* select(key, false);
|
|
796
|
+
const found = yield* Effect.flatMap(select(key, false), (first) => lockFound(key, first));
|
|
797
|
+
const absent = absentKeys.get(transaction.value) ?? /* @__PURE__ */ new Set();
|
|
798
|
+
absentKeys.set(transaction.value, absent);
|
|
799
|
+
if (Option.isNone(found)) absent.add(key);
|
|
800
|
+
else absent.delete(key);
|
|
801
|
+
return found;
|
|
802
|
+
}),
|
|
803
|
+
write: Effect.fn("SqlStorage.records.write")(function* (key, encoded) {
|
|
804
|
+
const transaction = yield* Effect.serviceOption(sql.transactionService);
|
|
805
|
+
const absent = Option.isSome(transaction) ? absentKeys.get(transaction.value) : void 0;
|
|
806
|
+
if (absent?.has(key) === true) {
|
|
807
|
+
yield* insert(key, encoded);
|
|
808
|
+
absent.delete(key);
|
|
809
|
+
} else yield* upsert(key, encoded);
|
|
810
|
+
}),
|
|
811
|
+
remove: Effect.fn("SqlStorage.records.remove")((key) => execute("records.remove", sql`DELETE FROM streamsy_owner_records WHERE owner_key = ${key}`))
|
|
812
|
+
});
|
|
813
|
+
};
|
|
814
|
+
const makeLayer = (options, probe, transactionMaxOpsBeforeYield) => {
|
|
815
|
+
const transactionRetry = transactionRetryPolicy(options);
|
|
816
|
+
return Layer.effectContext(Effect.gen(function* () {
|
|
817
|
+
const sql = yield* SqlClient.SqlClient;
|
|
818
|
+
yield* migrate(sql, { onTransactionAttempt: probe === void 0 ? void 0 : () => {
|
|
819
|
+
probe.migrationAttempts += 1;
|
|
820
|
+
} }, transactionRetry);
|
|
821
|
+
const boundary = yield* BoundaryRuntimeService;
|
|
822
|
+
const owner = ownerOf(boundary);
|
|
823
|
+
return Context.empty().pipe(Context.add(Storage, attachOwner(makeStorage(sql, boundary), owner)), Context.add(CommitBoundary, CommitBoundary.of(boundary)), Context.add(OwnerRecords, attachOwner(makeOwnerRecords(sql), owner)));
|
|
824
|
+
})).pipe(Layer.provide(boundaryLayer(options.repairIntervalMs ?? 1e3, transactionRetry, probe, transactionMaxOpsBeforeYield)));
|
|
825
|
+
};
|
|
826
|
+
/** Generic SQLite-stream SQL layer. Hosts supply one matching SqlClient/Reactivity graph. */
|
|
827
|
+
const storageLayer = (options = {}) => makeLayer(options);
|
|
828
|
+
/** Internal host seam for the official Durable Object transaction boundary. */
|
|
829
|
+
const layerWithDurableObjectTransactions = (options) => makeLayer(options, void 0, Number.MAX_SAFE_INTEGER);
|
|
830
|
+
/** Protocol over SQL storage. */
|
|
831
|
+
const layerProtocol = (options = {}) => Protocol.layer(options).pipe(Layer.provideMerge(storageLayer(options)));
|
|
832
|
+
function layer(group, options = {}) {
|
|
833
|
+
const storage = storageLayer(options);
|
|
834
|
+
const protocol = Protocol.layer(options).pipe(Layer.provideMerge(storage));
|
|
835
|
+
return Protocol.bind(protocol, group).pipe(Layer.provideMerge(storage));
|
|
836
|
+
}
|
|
837
|
+
//#endregion
|
|
838
|
+
export { storageLayer as a, STORAGE_VERSION_TABLE as c, retryWithPolicy as d, transactionRetryPolicy as f, layerWithDurableObjectTransactions as i, clientLayer as l, layer as n, inspectFormat as o, layerProtocol as r, newerVersionFault as s, DEFAULT_REPAIR_INTERVAL_MS as t, sharedSqlClientLayer as u };
|