@spooky-sync/core 0.0.1-canary.326 → 0.0.1-canary.328
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +5 -3
- package/dist/index.d.ts +12 -2
- package/dist/index.js +461 -344
- package/dist/types.d.ts +10 -1
- package/package.json +5 -4
- package/scripts/check-worker-entries.mjs +1 -0
- package/skills/sp00ky-core/SKILL.md +1 -1
- package/skills/sp00ky-core/references/config.md +1 -1
- package/src/client/sp00ky-client.test.ts +35 -0
- package/src/client/sp00ky-client.ts +8 -2
- package/src/modules/auth/auth.local-first.test.ts +14 -0
- package/src/modules/auth/index.ts +3 -3
- package/src/modules/crdt/crdt.test.ts +49 -1
- package/src/modules/crdt/index.ts +35 -13
- package/src/query/id-set-plan.test.ts +2 -2
- package/src/query/lifecycle.saga.ts +28 -1
- package/src/query/materialize.test.ts +1 -1
- package/src/query/membership.saga.test.ts +57 -0
- package/src/query/membership.saga.ts +4 -0
- package/src/query/window-query.test.ts +6 -6
- package/src/query/window-query.ts +14 -1
- package/src/services/database/database.ts +10 -0
- package/src/services/database/local.test.ts +86 -0
- package/src/services/database/local.ts +22 -1
- package/src/services/database/sqlite-cache-engine.test.ts +11 -0
- package/src/services/database/surql-translate.ts +4 -3
- package/src/services/database/surreal-cache-engine.ts +3 -1
- package/src/state/selectors.ts +24 -0
- package/src/testing/fake-services.ts +16 -1
package/dist/index.js
CHANGED
|
@@ -2188,9 +2188,9 @@ function trimHistory(history, cap) {
|
|
|
2188
2188
|
|
|
2189
2189
|
//#endregion
|
|
2190
2190
|
//#region src/modules/devtools/index.ts
|
|
2191
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
2192
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
2193
|
-
const SURREAL_VERSION = "3.0
|
|
2191
|
+
const CORE_VERSION = "0.0.1-canary.328";
|
|
2192
|
+
const WASM_VERSION = "0.0.1-canary.328";
|
|
2193
|
+
const SURREAL_VERSION = "3.1.0";
|
|
2194
2194
|
var DevToolsService = class DevToolsService {
|
|
2195
2195
|
version = CORE_VERSION;
|
|
2196
2196
|
backendInfo = emptyBackendInfo();
|
|
@@ -3604,6 +3604,28 @@ function retained(s) {
|
|
|
3604
3604
|
const known = vouched(s);
|
|
3605
3605
|
return (id) => id.startsWith("_00_") || known.has(id) || held.has(id);
|
|
3606
3606
|
}
|
|
3607
|
+
/**
|
|
3608
|
+
* Ids something live still needs: a mounted query's membership or subquery
|
|
3609
|
+
* children, or a local write not yet settled. Unlike `retained`, durable view
|
|
3610
|
+
* rows of queries nobody runs right now do not count: a row the server just
|
|
3611
|
+
* took out of a live query must stop showing even if an old view named it.
|
|
3612
|
+
*/
|
|
3613
|
+
function activelyHeld(s) {
|
|
3614
|
+
const held = /* @__PURE__ */ new Set();
|
|
3615
|
+
for (const e of s.queries.values()) {
|
|
3616
|
+
for (const [id] of e.remoteArray) held.add(id);
|
|
3617
|
+
for (const [id] of e.subqueryRemoteArray) held.add(id);
|
|
3618
|
+
}
|
|
3619
|
+
for (const item of s.outbox) held.add(item.recordId);
|
|
3620
|
+
for (const write of s.pendingWrites.values()) held.add(write.recordId);
|
|
3621
|
+
return held;
|
|
3622
|
+
}
|
|
3623
|
+
/** Ids of `before` that `after` no longer names. */
|
|
3624
|
+
function departedIds(before, after) {
|
|
3625
|
+
if (before.length === 0) return [];
|
|
3626
|
+
const kept = new Set(after.map(([id]) => id));
|
|
3627
|
+
return before.filter(([id]) => !kept.has(id)).map(([id]) => id);
|
|
3628
|
+
}
|
|
3607
3629
|
const pendingDeleteIds = (s) => new Set(s.outbox.filter((i) => i.type === "delete").map((i) => i.recordId));
|
|
3608
3630
|
const hasAckedWrites = (s) => s.outbox.some((i) => i.status === "acked");
|
|
3609
3631
|
const pendingMutationCount = (s) => s.outbox.filter((i) => i.status === "pending").length;
|
|
@@ -4213,6 +4235,343 @@ function suspectHashes(snapshots, heldCounts, edgeCount) {
|
|
|
4213
4235
|
return out;
|
|
4214
4236
|
}
|
|
4215
4237
|
|
|
4238
|
+
//#endregion
|
|
4239
|
+
//#region src/query/sql.ts
|
|
4240
|
+
/** Every SurrealQL statement the query side sends, as pure builders. */
|
|
4241
|
+
const VIEW_TABLE = "_00_view";
|
|
4242
|
+
const LEGACY_VIEW_TABLE = "_00_window";
|
|
4243
|
+
const viewRecordId = (viewKey) => new RecordId$1(VIEW_TABLE, viewKey);
|
|
4244
|
+
/**
|
|
4245
|
+
* The durable `_00_view` row: the confirmed membership plus the subquery
|
|
4246
|
+
* children, which the GC must keep and a cold scan may paint as much as the
|
|
4247
|
+
* members.
|
|
4248
|
+
*/
|
|
4249
|
+
const viewRow = (ids, children, confirmed, now) => ({
|
|
4250
|
+
ids,
|
|
4251
|
+
children,
|
|
4252
|
+
confirmed,
|
|
4253
|
+
updatedAt: now
|
|
4254
|
+
});
|
|
4255
|
+
/** Every `_00_view` row: the durable view index and the GC read it. */
|
|
4256
|
+
const readViewRows = () => `SELECT * FROM ${VIEW_TABLE}`;
|
|
4257
|
+
/** Copy every legacy `_00_window` row into `_00_view` once. */
|
|
4258
|
+
const readLegacyViewRows = () => `SELECT * FROM ${LEGACY_VIEW_TABLE}`;
|
|
4259
|
+
const countViewRows = () => `SELECT count() FROM ${VIEW_TABLE} GROUP ALL`;
|
|
4260
|
+
const listRefSelect = (table) => `SELECT out, version FROM $in->${table} WHERE parent IS NONE`;
|
|
4261
|
+
const subqueryListRefSelect = (table) => `SELECT out, version FROM $in->${table} WHERE parent IS NOT NONE`;
|
|
4262
|
+
const queryRowCountSelect = () => "SELECT VALUE { rowCount: rowCount, state: state } FROM ONLY $in";
|
|
4263
|
+
const listRefBatchSelect = (table) => `SELECT in, out, version, parent FROM $ins->${table}`;
|
|
4264
|
+
const queryRowCountBatchSelect = () => "SELECT VALUE { id: id, rowCount: rowCount, state: state } FROM $ins";
|
|
4265
|
+
/** The single-query read: edges, meta, subquery children in one request. */
|
|
4266
|
+
const singleSnapshotSelect = (table) => `${listRefSelect(table)};\n${queryRowCountSelect()};\n${subqueryListRefSelect(table)}`;
|
|
4267
|
+
/** The many-query read: all edges + all metas in one request. */
|
|
4268
|
+
const batchSnapshotSelect = (table) => `${listRefBatchSelect(table)};\n${queryRowCountBatchSelect()}`;
|
|
4269
|
+
/**
|
|
4270
|
+
* The session's one LIVE subscription on its `_00_list_ref` table.
|
|
4271
|
+
*
|
|
4272
|
+
* `inlineBodies` adds `FETCH out`, which resolves the edge's target into the
|
|
4273
|
+
* notification so the body arrives with the doorbell.
|
|
4274
|
+
*
|
|
4275
|
+
* On the anon table it is filtered to this client's own edges. Every
|
|
4276
|
+
* signed-out session shares that table, so unfiltered each one received (and
|
|
4277
|
+
* the server fetched a body for) every anonymous edge change of every other
|
|
4278
|
+
* session, and the client then dropped all but its own. An edge carries the
|
|
4279
|
+
* `session::id()` of whichever socket last registered its view, which is a
|
|
4280
|
+
* socket of this client but not necessarily the current one: a view stays
|
|
4281
|
+
* registered across a reconnect, and a reload keeps its salt and its views,
|
|
4282
|
+
* so their older edges still name an earlier socket. `sessionIds` are those
|
|
4283
|
+
* earlier sockets. The current one is matched by `session::id()` itself,
|
|
4284
|
+
* which SurrealDB evaluates in the subscriber's own session, so the filter
|
|
4285
|
+
* holds before the client has learned its new id.
|
|
4286
|
+
*
|
|
4287
|
+
* The cast is load-bearing: `session::id()` is a uuid, `clientId` a string,
|
|
4288
|
+
* and without it nothing matches. Per-user tables need no filter.
|
|
4289
|
+
*/
|
|
4290
|
+
function listRefLive(table, opts) {
|
|
4291
|
+
const join = opts.inlineBodies ? " FETCH out" : "";
|
|
4292
|
+
if (table !== ANON_LIST_REF_TABLE) return { sql: `LIVE SELECT * FROM ${table}${join}` };
|
|
4293
|
+
return {
|
|
4294
|
+
sql: `LIVE SELECT * FROM ${table} WHERE clientId = <string>session::id() OR clientId IN $sids${join}`,
|
|
4295
|
+
vars: { sids: [...opts.sessionIds] }
|
|
4296
|
+
};
|
|
4297
|
+
}
|
|
4298
|
+
/** Register + read back edges/meta/children in ONE request. */
|
|
4299
|
+
const registerSelect = (table) => `fn::query::register($config);\n${singleSnapshotSelect(table)}`;
|
|
4300
|
+
const registerVars = (payload) => ({
|
|
4301
|
+
config: payload,
|
|
4302
|
+
in: payload.id
|
|
4303
|
+
});
|
|
4304
|
+
/** One heartbeat statement per view id; result index i answers id i. */
|
|
4305
|
+
function heartbeatBatch(ids) {
|
|
4306
|
+
const vars = {};
|
|
4307
|
+
return {
|
|
4308
|
+
sql: ids.map((id, i) => {
|
|
4309
|
+
vars[`id${i}`] = id;
|
|
4310
|
+
return `fn::query::heartbeat($id${i})`;
|
|
4311
|
+
}).join(";\n"),
|
|
4312
|
+
vars
|
|
4313
|
+
};
|
|
4314
|
+
}
|
|
4315
|
+
/** A heartbeat answer whose statement matched no row: the view was reclaimed. */
|
|
4316
|
+
const heartbeatRowGone = (result) => Array.isArray(result) && result.length === 0;
|
|
4317
|
+
const QUERY_ID = /^_00_query:(?:[0-9a-f]{64}|⟨[0-9a-f]{64}⟩)$/;
|
|
4318
|
+
const SESSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
4319
|
+
/**
|
|
4320
|
+
* The pagehide release: one bare statement for `beaconSql`, which has no params
|
|
4321
|
+
* channel, so ids are inlined. Only shapes the client minted itself get in, a
|
|
4322
|
+
* `_00_query:<sha256>` id and a UUID session id, so nothing else is ever
|
|
4323
|
+
* interpolated. `null` when there is nothing the server could act on: with no
|
|
4324
|
+
* session ids it could not tell this page's entries from anyone else's.
|
|
4325
|
+
*/
|
|
4326
|
+
function leaveViewsBeacon(ids, sessionIds) {
|
|
4327
|
+
const list = ids.map((id) => String(id)).filter((id) => QUERY_ID.test(id));
|
|
4328
|
+
const sids = sessionIds.filter((sid) => SESSION_ID.test(sid));
|
|
4329
|
+
if (list.length === 0 || sids.length === 0) return null;
|
|
4330
|
+
const sidList = sids.map((sid) => `'${sid}'`).join(", ");
|
|
4331
|
+
return `FOR $id IN [${list.join(", ")}] { LET $_left = fn::query::leave($id, [${sidList}]); };`;
|
|
4332
|
+
}
|
|
4333
|
+
const bodySelect = () => "SELECT * FROM $ids";
|
|
4334
|
+
/** Local transaction that MERGEs many fetched bodies. */
|
|
4335
|
+
function upsertBodiesTx(records) {
|
|
4336
|
+
const vars = {};
|
|
4337
|
+
const stmts = records.map((r, i) => {
|
|
4338
|
+
vars[`id${i}`] = r.id;
|
|
4339
|
+
vars[`content${i}`] = r.content;
|
|
4340
|
+
return surql.upsertMerge(`id${i}`, `content${i}`);
|
|
4341
|
+
});
|
|
4342
|
+
return {
|
|
4343
|
+
query: surql.seal(surql.tx(stmts)),
|
|
4344
|
+
vars
|
|
4345
|
+
};
|
|
4346
|
+
}
|
|
4347
|
+
|
|
4348
|
+
//#endregion
|
|
4349
|
+
//#region src/query/lifecycle.saga.ts
|
|
4350
|
+
/**
|
|
4351
|
+
* One tick for every query in state: evict the ones nobody has watched for a
|
|
4352
|
+
* ttl, heartbeat the rest in one request, notice reclaimed rows. Reschedules
|
|
4353
|
+
* itself at half the shortest ttl.
|
|
4354
|
+
*/
|
|
4355
|
+
function* lifecycleTick(env) {
|
|
4356
|
+
const now = yield fx.now();
|
|
4357
|
+
const state = yield fx.state.read((s) => s);
|
|
4358
|
+
for (const hash of evictable(state, now)) yield* evictQuery(hash);
|
|
4359
|
+
const remaining = yield fx.state.read((s) => [...s.queries.values()].filter((e) => e.lifecycle.remote === "registered"));
|
|
4360
|
+
if (remaining.length > 0) {
|
|
4361
|
+
const beat = heartbeatBatch(remaining.map((e) => e.def.id));
|
|
4362
|
+
try {
|
|
4363
|
+
const results = yield fx.remote.query(beat.sql, beat.vars, env.remoteTimeoutMs);
|
|
4364
|
+
let reclaimed = 0;
|
|
4365
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
4366
|
+
const r = results[i];
|
|
4367
|
+
if (r?.status === "OK" && heartbeatRowGone(r.result)) {
|
|
4368
|
+
reclaimed++;
|
|
4369
|
+
yield fx.state.update(applyLifecycle(remaining[i].def.hash, { type: "remote-dropped" }));
|
|
4370
|
+
}
|
|
4371
|
+
}
|
|
4372
|
+
yield fx.state.update(stampHeartbeat(remaining.map((e) => e.def.hash), now));
|
|
4373
|
+
if (reclaimed > 0) {
|
|
4374
|
+
yield fx.emit({
|
|
4375
|
+
type: "log",
|
|
4376
|
+
level: "warn",
|
|
4377
|
+
message: "query rows reclaimed by the server; re-registering",
|
|
4378
|
+
data: { reclaimed }
|
|
4379
|
+
});
|
|
4380
|
+
yield fx.dispatch({ type: "EnsureRegistered" });
|
|
4381
|
+
}
|
|
4382
|
+
yield fx.dispatch({
|
|
4383
|
+
type: "SyncOutcome",
|
|
4384
|
+
ok: true
|
|
4385
|
+
});
|
|
4386
|
+
} catch (error) {
|
|
4387
|
+
yield fx.dispatch({
|
|
4388
|
+
type: "SyncOutcome",
|
|
4389
|
+
ok: false,
|
|
4390
|
+
error
|
|
4391
|
+
});
|
|
4392
|
+
}
|
|
4393
|
+
}
|
|
4394
|
+
const ttl = yield fx.state.read((s) => shortestTtlMs(s));
|
|
4395
|
+
yield fx.timer.set("lifecycle", Math.floor((ttl ?? env.defaultTtlMs) * TTL_HEARTBEAT_FRACTION), { type: "LifecycleTick" });
|
|
4396
|
+
}
|
|
4397
|
+
/** Free a query's local view and forget it. The server row expires by TTL. */
|
|
4398
|
+
function* evictQuery(hash) {
|
|
4399
|
+
if (!(yield fx.state.read((s) => s.queries.has(hash)))) return;
|
|
4400
|
+
try {
|
|
4401
|
+
yield fx.ssp.unregister(hash);
|
|
4402
|
+
} catch (error) {
|
|
4403
|
+
yield fx.emit({
|
|
4404
|
+
type: "log",
|
|
4405
|
+
level: "debug",
|
|
4406
|
+
message: "unregister failed",
|
|
4407
|
+
data: {
|
|
4408
|
+
hash,
|
|
4409
|
+
error
|
|
4410
|
+
}
|
|
4411
|
+
});
|
|
4412
|
+
}
|
|
4413
|
+
yield fx.state.update(removeQuery(hash));
|
|
4414
|
+
yield fx.emit({
|
|
4415
|
+
type: "query:evicted",
|
|
4416
|
+
hash
|
|
4417
|
+
});
|
|
4418
|
+
}
|
|
4419
|
+
/** Drop acked outbox items membership never named within the grace window. */
|
|
4420
|
+
function* ackPrune() {
|
|
4421
|
+
const now = yield fx.now();
|
|
4422
|
+
yield fx.state.update(outboxPruneAcked(now, ACK_GRACE_MS));
|
|
4423
|
+
if (yield fx.state.read((s) => s.outbox.some((i) => i.status === "acked"))) yield fx.timer.set("ack-prune", ACK_GRACE_MS, { type: "AckPrune" });
|
|
4424
|
+
}
|
|
4425
|
+
/** Every `_00_view` row, parsed. Throws when the read fails. */
|
|
4426
|
+
function* readViews() {
|
|
4427
|
+
const res = yield fx.local.query(readViewRows());
|
|
4428
|
+
return parseViewIndex(Array.isArray(res) ? res[0] : void 0);
|
|
4429
|
+
}
|
|
4430
|
+
const indexOf = (rows) => new Map(rows.map((r) => [r.key, r.ids]));
|
|
4431
|
+
/**
|
|
4432
|
+
* Load the durable view index (boot, bucket switch). A failed read leaves it
|
|
4433
|
+
* empty, which fails closed: cold scans paint nothing until the server answers.
|
|
4434
|
+
*/
|
|
4435
|
+
function* loadViews() {
|
|
4436
|
+
try {
|
|
4437
|
+
yield fx.state.update(reloadViews(indexOf(yield* readViews())));
|
|
4438
|
+
} catch (error) {
|
|
4439
|
+
yield fx.emit({
|
|
4440
|
+
type: "log",
|
|
4441
|
+
level: "warn",
|
|
4442
|
+
message: "view index read failed",
|
|
4443
|
+
data: { error }
|
|
4444
|
+
});
|
|
4445
|
+
}
|
|
4446
|
+
}
|
|
4447
|
+
const tableOf$1 = (id) => id.slice(0, id.indexOf(":"));
|
|
4448
|
+
/**
|
|
4449
|
+
* Orphan collection, `GC_BOOT_DELAY_MS` after boot and then every
|
|
4450
|
+
* `GC_INTERVAL_MS`. Re-reads every `_00_view` row into the index, retires the
|
|
4451
|
+
* rows no query has resolved for `VIEW_RETENTION_MS`, then deletes the bodies
|
|
4452
|
+
* nothing retains (see `retained`) from the store and the circuit, a chunk at
|
|
4453
|
+
* a time. Only a leader or solo tab deletes, and a leader relays the deletes
|
|
4454
|
+
* so followers drop the rows too; a follower only refreshes its index.
|
|
4455
|
+
*/
|
|
4456
|
+
function* gcTick() {
|
|
4457
|
+
yield fx.state.wait((s) => s.primed);
|
|
4458
|
+
try {
|
|
4459
|
+
const rows = yield* readViews();
|
|
4460
|
+
const [role, bucket, held] = yield fx.state.read((s) => [
|
|
4461
|
+
s.tabRole,
|
|
4462
|
+
s.bucketId,
|
|
4463
|
+
new Set([...s.queries.values()].map((e) => e.def.viewKey))
|
|
4464
|
+
]);
|
|
4465
|
+
const now = yield fx.now();
|
|
4466
|
+
const expired = role === "follower" ? [] : rows.filter((r) => !held.has(r.key) && now - r.updatedAt >= VIEW_RETENTION_MS);
|
|
4467
|
+
const retired = /* @__PURE__ */ new Set();
|
|
4468
|
+
for (let i = 0; i < expired.length; i += GC_CHUNK) {
|
|
4469
|
+
const chunk = expired.slice(i, i + GC_CHUNK);
|
|
4470
|
+
const settled = yield fx.all(chunk.map((r) => fx.local.delete(VIEW_TABLE, viewRecordId(r.key))));
|
|
4471
|
+
chunk.forEach((r, j) => settled[j].ok && retired.add(r.key));
|
|
4472
|
+
}
|
|
4473
|
+
yield fx.state.update(reloadViews(indexOf(rows.filter((r) => !retired.has(r.key)))));
|
|
4474
|
+
if (role === "follower") return;
|
|
4475
|
+
const removed = yield* collectOrphans(bucket, role === "leader");
|
|
4476
|
+
yield fx.emit({
|
|
4477
|
+
type: "log",
|
|
4478
|
+
level: "info",
|
|
4479
|
+
message: "orphan gc done",
|
|
4480
|
+
data: {
|
|
4481
|
+
removed,
|
|
4482
|
+
retiredViews: retired.size
|
|
4483
|
+
}
|
|
4484
|
+
});
|
|
4485
|
+
} catch (error) {
|
|
4486
|
+
yield fx.emit({
|
|
4487
|
+
type: "log",
|
|
4488
|
+
level: "warn",
|
|
4489
|
+
message: "orphan gc failed",
|
|
4490
|
+
data: { error }
|
|
4491
|
+
});
|
|
4492
|
+
} finally {
|
|
4493
|
+
yield fx.timer.set("gc", GC_INTERVAL_MS, { type: "GcTick" });
|
|
4494
|
+
}
|
|
4495
|
+
}
|
|
4496
|
+
/**
|
|
4497
|
+
* Forget rows the server just took out of `hash` (its membership or its
|
|
4498
|
+
* subquery children) when nothing live still holds them: delete the local
|
|
4499
|
+
* body and the in-browser circuit's copy, and re-render the query.
|
|
4500
|
+
*
|
|
4501
|
+
* Primary rows already stop rendering once they leave membership, but
|
|
4502
|
+
* subquery children are rendered from the local store (a thread's comments
|
|
4503
|
+
* are read back by the relation), so a comment deleted on another device
|
|
4504
|
+
* stayed on screen until the hourly GC or a reload. Only a leader or solo tab
|
|
4505
|
+
* deletes, and a leader relays it to its followers, as the GC does.
|
|
4506
|
+
*/
|
|
4507
|
+
function* forgetDeparted(hash, departed) {
|
|
4508
|
+
if (departed.length === 0) return;
|
|
4509
|
+
const [gone, role] = yield fx.state.read((s) => {
|
|
4510
|
+
const held = activelyHeld(s);
|
|
4511
|
+
return [departed.filter((id) => !id.startsWith("_00_") && s.versions.has(id) && !held.has(id)), s.tabRole];
|
|
4512
|
+
});
|
|
4513
|
+
if (gone.length === 0 || role === "follower") return;
|
|
4514
|
+
const settled = yield fx.all(gone.map((id) => fx.local.delete(tableOf$1(id), id)));
|
|
4515
|
+
const done = gone.filter((_, j) => settled[j].ok);
|
|
4516
|
+
if (done.length === 0) return;
|
|
4517
|
+
const records = done.map((id) => ({
|
|
4518
|
+
table: tableOf$1(id),
|
|
4519
|
+
op: "DELETE",
|
|
4520
|
+
id,
|
|
4521
|
+
record: {}
|
|
4522
|
+
}));
|
|
4523
|
+
yield fx.state.update(compose(deleteVersions(done), markDirty([hash])));
|
|
4524
|
+
yield fx.ssp.ingest(records);
|
|
4525
|
+
if (role === "leader") yield fx.emit({
|
|
4526
|
+
type: "tabs:broadcast",
|
|
4527
|
+
message: {
|
|
4528
|
+
type: "ingest",
|
|
4529
|
+
records
|
|
4530
|
+
}
|
|
4531
|
+
});
|
|
4532
|
+
}
|
|
4533
|
+
/**
|
|
4534
|
+
* Delete every body nothing retains, `GC_CHUNK` at a time. Each chunk is
|
|
4535
|
+
* re-checked against fresh state (a membership committed meanwhile may name
|
|
4536
|
+
* an id again) and the sweep stops if the bucket moved under it.
|
|
4537
|
+
*/
|
|
4538
|
+
function* collectOrphans(bucket, relay) {
|
|
4539
|
+
const candidates = yield fx.state.read((s) => {
|
|
4540
|
+
const keep = retained(s);
|
|
4541
|
+
return [...s.versions.keys()].filter((id) => !keep(id));
|
|
4542
|
+
});
|
|
4543
|
+
let removed = 0;
|
|
4544
|
+
for (let i = 0; i < candidates.length; i += GC_CHUNK) {
|
|
4545
|
+
const chunk = yield fx.state.read((s) => {
|
|
4546
|
+
if (!s.primed || s.bucketId !== bucket) return null;
|
|
4547
|
+
const keep = retained(s);
|
|
4548
|
+
return candidates.slice(i, i + GC_CHUNK).filter((id) => s.versions.has(id) && !keep(id));
|
|
4549
|
+
});
|
|
4550
|
+
if (chunk === null) break;
|
|
4551
|
+
const settled = yield fx.all(chunk.map((id) => fx.local.delete(tableOf$1(id), id)));
|
|
4552
|
+
const done = chunk.filter((_, j) => settled[j].ok);
|
|
4553
|
+
if (done.length === 0) continue;
|
|
4554
|
+
const records = done.map((id) => ({
|
|
4555
|
+
table: tableOf$1(id),
|
|
4556
|
+
op: "DELETE",
|
|
4557
|
+
id,
|
|
4558
|
+
record: {}
|
|
4559
|
+
}));
|
|
4560
|
+
yield fx.state.update(deleteVersions(done));
|
|
4561
|
+
yield fx.ssp.ingest(records);
|
|
4562
|
+
if (relay) yield fx.emit({
|
|
4563
|
+
type: "tabs:broadcast",
|
|
4564
|
+
message: {
|
|
4565
|
+
type: "ingest",
|
|
4566
|
+
records
|
|
4567
|
+
}
|
|
4568
|
+
});
|
|
4569
|
+
removed += done.length;
|
|
4570
|
+
}
|
|
4571
|
+
if (removed > 0) yield fx.dispatch({ type: "FetchRows" });
|
|
4572
|
+
return removed;
|
|
4573
|
+
}
|
|
4574
|
+
|
|
4216
4575
|
//#endregion
|
|
4217
4576
|
//#region src/sync/policy.ts
|
|
4218
4577
|
/** Known edges one poll round trip may carry before the cycle is split. */
|
|
@@ -4339,141 +4698,31 @@ function nextHealth(input, ok, error, degradeAfter) {
|
|
|
4339
4698
|
}
|
|
4340
4699
|
const consecutiveFailures = input.consecutiveFailures + 1;
|
|
4341
4700
|
const kind = classifySyncError(error);
|
|
4342
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
4343
|
-
if (input.health.status !== "degraded" && consecutiveFailures >= degradeAfter) return {
|
|
4344
|
-
...base,
|
|
4345
|
-
consecutiveFailures,
|
|
4346
|
-
health: {
|
|
4347
|
-
...input.health,
|
|
4348
|
-
status: "degraded",
|
|
4349
|
-
consecutiveFailures,
|
|
4350
|
-
kind,
|
|
4351
|
-
error: message
|
|
4352
|
-
},
|
|
4353
|
-
degradedNow: true
|
|
4354
|
-
};
|
|
4355
|
-
return {
|
|
4356
|
-
...base,
|
|
4357
|
-
consecutiveFailures,
|
|
4358
|
-
health: {
|
|
4359
|
-
...input.health,
|
|
4360
|
-
consecutiveFailures,
|
|
4361
|
-
kind,
|
|
4362
|
-
error: message
|
|
4363
|
-
}
|
|
4364
|
-
};
|
|
4365
|
-
}
|
|
4366
|
-
const selfHealDelayMs = (attempt) => Math.min(SELF_HEAL_MAX_MS, SELF_HEAL_BASE_MS * 2 ** Math.min(attempt, 30));
|
|
4367
|
-
|
|
4368
|
-
//#endregion
|
|
4369
|
-
//#region src/query/sql.ts
|
|
4370
|
-
/** Every SurrealQL statement the query side sends, as pure builders. */
|
|
4371
|
-
const VIEW_TABLE = "_00_view";
|
|
4372
|
-
const LEGACY_VIEW_TABLE = "_00_window";
|
|
4373
|
-
const viewRecordId = (viewKey) => new RecordId$1(VIEW_TABLE, viewKey);
|
|
4374
|
-
/**
|
|
4375
|
-
* The durable `_00_view` row: the confirmed membership plus the subquery
|
|
4376
|
-
* children, which the GC must keep and a cold scan may paint as much as the
|
|
4377
|
-
* members.
|
|
4378
|
-
*/
|
|
4379
|
-
const viewRow = (ids, children, confirmed, now) => ({
|
|
4380
|
-
ids,
|
|
4381
|
-
children,
|
|
4382
|
-
confirmed,
|
|
4383
|
-
updatedAt: now
|
|
4384
|
-
});
|
|
4385
|
-
/** Every `_00_view` row: the durable view index and the GC read it. */
|
|
4386
|
-
const readViewRows = () => `SELECT * FROM ${VIEW_TABLE}`;
|
|
4387
|
-
/** Copy every legacy `_00_window` row into `_00_view` once. */
|
|
4388
|
-
const readLegacyViewRows = () => `SELECT * FROM ${LEGACY_VIEW_TABLE}`;
|
|
4389
|
-
const countViewRows = () => `SELECT count() FROM ${VIEW_TABLE} GROUP ALL`;
|
|
4390
|
-
const listRefSelect = (table) => `SELECT out, version FROM $in->${table} WHERE parent IS NONE`;
|
|
4391
|
-
const subqueryListRefSelect = (table) => `SELECT out, version FROM $in->${table} WHERE parent IS NOT NONE`;
|
|
4392
|
-
const queryRowCountSelect = () => "SELECT VALUE { rowCount: rowCount, state: state } FROM ONLY $in";
|
|
4393
|
-
const listRefBatchSelect = (table) => `SELECT in, out, version, parent FROM $ins->${table}`;
|
|
4394
|
-
const queryRowCountBatchSelect = () => "SELECT VALUE { id: id, rowCount: rowCount, state: state } FROM $ins";
|
|
4395
|
-
/** The single-query read: edges, meta, subquery children in one request. */
|
|
4396
|
-
const singleSnapshotSelect = (table) => `${listRefSelect(table)};\n${queryRowCountSelect()};\n${subqueryListRefSelect(table)}`;
|
|
4397
|
-
/** The many-query read: all edges + all metas in one request. */
|
|
4398
|
-
const batchSnapshotSelect = (table) => `${listRefBatchSelect(table)};\n${queryRowCountBatchSelect()}`;
|
|
4399
|
-
/**
|
|
4400
|
-
* The session's one LIVE subscription on its `_00_list_ref` table.
|
|
4401
|
-
*
|
|
4402
|
-
* `inlineBodies` adds `FETCH out`, which resolves the edge's target into the
|
|
4403
|
-
* notification so the body arrives with the doorbell.
|
|
4404
|
-
*
|
|
4405
|
-
* On the anon table it is filtered to this client's own edges. Every
|
|
4406
|
-
* signed-out session shares that table, so unfiltered each one received (and
|
|
4407
|
-
* the server fetched a body for) every anonymous edge change of every other
|
|
4408
|
-
* session, and the client then dropped all but its own. An edge carries the
|
|
4409
|
-
* `session::id()` of whichever socket last registered its view, which is a
|
|
4410
|
-
* socket of this client but not necessarily the current one: a view stays
|
|
4411
|
-
* registered across a reconnect, and a reload keeps its salt and its views,
|
|
4412
|
-
* so their older edges still name an earlier socket. `sessionIds` are those
|
|
4413
|
-
* earlier sockets. The current one is matched by `session::id()` itself,
|
|
4414
|
-
* which SurrealDB evaluates in the subscriber's own session, so the filter
|
|
4415
|
-
* holds before the client has learned its new id.
|
|
4416
|
-
*
|
|
4417
|
-
* The cast is load-bearing: `session::id()` is a uuid, `clientId` a string,
|
|
4418
|
-
* and without it nothing matches. Per-user tables need no filter.
|
|
4419
|
-
*/
|
|
4420
|
-
function listRefLive(table, opts) {
|
|
4421
|
-
const join = opts.inlineBodies ? " FETCH out" : "";
|
|
4422
|
-
if (table !== ANON_LIST_REF_TABLE) return { sql: `LIVE SELECT * FROM ${table}${join}` };
|
|
4423
|
-
return {
|
|
4424
|
-
sql: `LIVE SELECT * FROM ${table} WHERE clientId = <string>session::id() OR clientId IN $sids${join}`,
|
|
4425
|
-
vars: { sids: [...opts.sessionIds] }
|
|
4426
|
-
};
|
|
4427
|
-
}
|
|
4428
|
-
/** Register + read back edges/meta/children in ONE request. */
|
|
4429
|
-
const registerSelect = (table) => `fn::query::register($config);\n${singleSnapshotSelect(table)}`;
|
|
4430
|
-
const registerVars = (payload) => ({
|
|
4431
|
-
config: payload,
|
|
4432
|
-
in: payload.id
|
|
4433
|
-
});
|
|
4434
|
-
/** One heartbeat statement per view id; result index i answers id i. */
|
|
4435
|
-
function heartbeatBatch(ids) {
|
|
4436
|
-
const vars = {};
|
|
4437
|
-
return {
|
|
4438
|
-
sql: ids.map((id, i) => {
|
|
4439
|
-
vars[`id${i}`] = id;
|
|
4440
|
-
return `fn::query::heartbeat($id${i})`;
|
|
4441
|
-
}).join(";\n"),
|
|
4442
|
-
vars
|
|
4701
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
4702
|
+
if (input.health.status !== "degraded" && consecutiveFailures >= degradeAfter) return {
|
|
4703
|
+
...base,
|
|
4704
|
+
consecutiveFailures,
|
|
4705
|
+
health: {
|
|
4706
|
+
...input.health,
|
|
4707
|
+
status: "degraded",
|
|
4708
|
+
consecutiveFailures,
|
|
4709
|
+
kind,
|
|
4710
|
+
error: message
|
|
4711
|
+
},
|
|
4712
|
+
degradedNow: true
|
|
4443
4713
|
};
|
|
4444
|
-
}
|
|
4445
|
-
/** A heartbeat answer whose statement matched no row: the view was reclaimed. */
|
|
4446
|
-
const heartbeatRowGone = (result) => Array.isArray(result) && result.length === 0;
|
|
4447
|
-
const QUERY_ID = /^_00_query:(?:[0-9a-f]{64}|⟨[0-9a-f]{64}⟩)$/;
|
|
4448
|
-
const SESSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
4449
|
-
/**
|
|
4450
|
-
* The pagehide release: one bare statement for `beaconSql`, which has no params
|
|
4451
|
-
* channel, so ids are inlined. Only shapes the client minted itself get in, a
|
|
4452
|
-
* `_00_query:<sha256>` id and a UUID session id, so nothing else is ever
|
|
4453
|
-
* interpolated. `null` when there is nothing the server could act on: with no
|
|
4454
|
-
* session ids it could not tell this page's entries from anyone else's.
|
|
4455
|
-
*/
|
|
4456
|
-
function leaveViewsBeacon(ids, sessionIds) {
|
|
4457
|
-
const list = ids.map((id) => String(id)).filter((id) => QUERY_ID.test(id));
|
|
4458
|
-
const sids = sessionIds.filter((sid) => SESSION_ID.test(sid));
|
|
4459
|
-
if (list.length === 0 || sids.length === 0) return null;
|
|
4460
|
-
const sidList = sids.map((sid) => `'${sid}'`).join(", ");
|
|
4461
|
-
return `FOR $id IN [${list.join(", ")}] { LET $_left = fn::query::leave($id, [${sidList}]); };`;
|
|
4462
|
-
}
|
|
4463
|
-
const bodySelect = () => "SELECT * FROM $ids";
|
|
4464
|
-
/** Local transaction that MERGEs many fetched bodies. */
|
|
4465
|
-
function upsertBodiesTx(records) {
|
|
4466
|
-
const vars = {};
|
|
4467
|
-
const stmts = records.map((r, i) => {
|
|
4468
|
-
vars[`id${i}`] = r.id;
|
|
4469
|
-
vars[`content${i}`] = r.content;
|
|
4470
|
-
return surql.upsertMerge(`id${i}`, `content${i}`);
|
|
4471
|
-
});
|
|
4472
4714
|
return {
|
|
4473
|
-
|
|
4474
|
-
|
|
4715
|
+
...base,
|
|
4716
|
+
consecutiveFailures,
|
|
4717
|
+
health: {
|
|
4718
|
+
...input.health,
|
|
4719
|
+
consecutiveFailures,
|
|
4720
|
+
kind,
|
|
4721
|
+
error: message
|
|
4722
|
+
}
|
|
4475
4723
|
};
|
|
4476
4724
|
}
|
|
4725
|
+
const selfHealDelayMs = (attempt) => Math.min(SELF_HEAL_MAX_MS, SELF_HEAL_BASE_MS * 2 ** Math.min(attempt, 30));
|
|
4477
4726
|
|
|
4478
4727
|
//#endregion
|
|
4479
4728
|
//#region src/query/membership.saga.ts
|
|
@@ -4530,6 +4779,7 @@ function* applyMembership(hash, remoteArray, meta, verifiedRemoval) {
|
|
|
4530
4779
|
}
|
|
4531
4780
|
const wasAuthoritative = isAuthoritative(entry.lifecycle);
|
|
4532
4781
|
yield fx.state.update(commitMembership(hash, remoteArray, meta?.present ?? true));
|
|
4782
|
+
yield* forgetDeparted(hash, departedIds(entry.remoteArray, remoteArray));
|
|
4533
4783
|
if (!wasAuthoritative) yield fx.emit({
|
|
4534
4784
|
type: "query:authority",
|
|
4535
4785
|
hash,
|
|
@@ -4576,6 +4826,7 @@ function* applySubqueryChildren(hash, children) {
|
|
|
4576
4826
|
const entry = yield fx.state.read((s) => s.queries.get(hash));
|
|
4577
4827
|
if (!entry || recordVersionArraysEqual(entry.subqueryRemoteArray, children)) return;
|
|
4578
4828
|
yield fx.state.update(setSubqueryRemoteArray(hash, children));
|
|
4829
|
+
yield* forgetDeparted(hash, departedIds(entry.subqueryRemoteArray, children));
|
|
4579
4830
|
if (entry.lifecycle.phase !== "cold") try {
|
|
4580
4831
|
yield fx.local.upsert(VIEW_TABLE, viewRecordId(entry.def.viewKey), { children }, "merge");
|
|
4581
4832
|
} catch (err) {
|
|
@@ -4949,196 +5200,6 @@ function* registerRemote(env, hash, retry = false) {
|
|
|
4949
5200
|
}
|
|
4950
5201
|
}
|
|
4951
5202
|
|
|
4952
|
-
//#endregion
|
|
4953
|
-
//#region src/query/lifecycle.saga.ts
|
|
4954
|
-
/**
|
|
4955
|
-
* One tick for every query in state: evict the ones nobody has watched for a
|
|
4956
|
-
* ttl, heartbeat the rest in one request, notice reclaimed rows. Reschedules
|
|
4957
|
-
* itself at half the shortest ttl.
|
|
4958
|
-
*/
|
|
4959
|
-
function* lifecycleTick(env) {
|
|
4960
|
-
const now = yield fx.now();
|
|
4961
|
-
const state = yield fx.state.read((s) => s);
|
|
4962
|
-
for (const hash of evictable(state, now)) yield* evictQuery(hash);
|
|
4963
|
-
const remaining = yield fx.state.read((s) => [...s.queries.values()].filter((e) => e.lifecycle.remote === "registered"));
|
|
4964
|
-
if (remaining.length > 0) {
|
|
4965
|
-
const beat = heartbeatBatch(remaining.map((e) => e.def.id));
|
|
4966
|
-
try {
|
|
4967
|
-
const results = yield fx.remote.query(beat.sql, beat.vars, env.remoteTimeoutMs);
|
|
4968
|
-
let reclaimed = 0;
|
|
4969
|
-
for (let i = 0; i < remaining.length; i++) {
|
|
4970
|
-
const r = results[i];
|
|
4971
|
-
if (r?.status === "OK" && heartbeatRowGone(r.result)) {
|
|
4972
|
-
reclaimed++;
|
|
4973
|
-
yield fx.state.update(applyLifecycle(remaining[i].def.hash, { type: "remote-dropped" }));
|
|
4974
|
-
}
|
|
4975
|
-
}
|
|
4976
|
-
yield fx.state.update(stampHeartbeat(remaining.map((e) => e.def.hash), now));
|
|
4977
|
-
if (reclaimed > 0) {
|
|
4978
|
-
yield fx.emit({
|
|
4979
|
-
type: "log",
|
|
4980
|
-
level: "warn",
|
|
4981
|
-
message: "query rows reclaimed by the server; re-registering",
|
|
4982
|
-
data: { reclaimed }
|
|
4983
|
-
});
|
|
4984
|
-
yield fx.dispatch({ type: "EnsureRegistered" });
|
|
4985
|
-
}
|
|
4986
|
-
yield fx.dispatch({
|
|
4987
|
-
type: "SyncOutcome",
|
|
4988
|
-
ok: true
|
|
4989
|
-
});
|
|
4990
|
-
} catch (error) {
|
|
4991
|
-
yield fx.dispatch({
|
|
4992
|
-
type: "SyncOutcome",
|
|
4993
|
-
ok: false,
|
|
4994
|
-
error
|
|
4995
|
-
});
|
|
4996
|
-
}
|
|
4997
|
-
}
|
|
4998
|
-
const ttl = yield fx.state.read((s) => shortestTtlMs(s));
|
|
4999
|
-
yield fx.timer.set("lifecycle", Math.floor((ttl ?? env.defaultTtlMs) * TTL_HEARTBEAT_FRACTION), { type: "LifecycleTick" });
|
|
5000
|
-
}
|
|
5001
|
-
/** Free a query's local view and forget it. The server row expires by TTL. */
|
|
5002
|
-
function* evictQuery(hash) {
|
|
5003
|
-
if (!(yield fx.state.read((s) => s.queries.has(hash)))) return;
|
|
5004
|
-
try {
|
|
5005
|
-
yield fx.ssp.unregister(hash);
|
|
5006
|
-
} catch (error) {
|
|
5007
|
-
yield fx.emit({
|
|
5008
|
-
type: "log",
|
|
5009
|
-
level: "debug",
|
|
5010
|
-
message: "unregister failed",
|
|
5011
|
-
data: {
|
|
5012
|
-
hash,
|
|
5013
|
-
error
|
|
5014
|
-
}
|
|
5015
|
-
});
|
|
5016
|
-
}
|
|
5017
|
-
yield fx.state.update(removeQuery(hash));
|
|
5018
|
-
yield fx.emit({
|
|
5019
|
-
type: "query:evicted",
|
|
5020
|
-
hash
|
|
5021
|
-
});
|
|
5022
|
-
}
|
|
5023
|
-
/** Drop acked outbox items membership never named within the grace window. */
|
|
5024
|
-
function* ackPrune() {
|
|
5025
|
-
const now = yield fx.now();
|
|
5026
|
-
yield fx.state.update(outboxPruneAcked(now, ACK_GRACE_MS));
|
|
5027
|
-
if (yield fx.state.read((s) => s.outbox.some((i) => i.status === "acked"))) yield fx.timer.set("ack-prune", ACK_GRACE_MS, { type: "AckPrune" });
|
|
5028
|
-
}
|
|
5029
|
-
/** Every `_00_view` row, parsed. Throws when the read fails. */
|
|
5030
|
-
function* readViews() {
|
|
5031
|
-
const res = yield fx.local.query(readViewRows());
|
|
5032
|
-
return parseViewIndex(Array.isArray(res) ? res[0] : void 0);
|
|
5033
|
-
}
|
|
5034
|
-
const indexOf = (rows) => new Map(rows.map((r) => [r.key, r.ids]));
|
|
5035
|
-
/**
|
|
5036
|
-
* Load the durable view index (boot, bucket switch). A failed read leaves it
|
|
5037
|
-
* empty, which fails closed: cold scans paint nothing until the server answers.
|
|
5038
|
-
*/
|
|
5039
|
-
function* loadViews() {
|
|
5040
|
-
try {
|
|
5041
|
-
yield fx.state.update(reloadViews(indexOf(yield* readViews())));
|
|
5042
|
-
} catch (error) {
|
|
5043
|
-
yield fx.emit({
|
|
5044
|
-
type: "log",
|
|
5045
|
-
level: "warn",
|
|
5046
|
-
message: "view index read failed",
|
|
5047
|
-
data: { error }
|
|
5048
|
-
});
|
|
5049
|
-
}
|
|
5050
|
-
}
|
|
5051
|
-
const tableOf$1 = (id) => id.slice(0, id.indexOf(":"));
|
|
5052
|
-
/**
|
|
5053
|
-
* Orphan collection, `GC_BOOT_DELAY_MS` after boot and then every
|
|
5054
|
-
* `GC_INTERVAL_MS`. Re-reads every `_00_view` row into the index, retires the
|
|
5055
|
-
* rows no query has resolved for `VIEW_RETENTION_MS`, then deletes the bodies
|
|
5056
|
-
* nothing retains (see `retained`) from the store and the circuit, a chunk at
|
|
5057
|
-
* a time. Only a leader or solo tab deletes, and a leader relays the deletes
|
|
5058
|
-
* so followers drop the rows too; a follower only refreshes its index.
|
|
5059
|
-
*/
|
|
5060
|
-
function* gcTick() {
|
|
5061
|
-
yield fx.state.wait((s) => s.primed);
|
|
5062
|
-
try {
|
|
5063
|
-
const rows = yield* readViews();
|
|
5064
|
-
const [role, bucket, held] = yield fx.state.read((s) => [
|
|
5065
|
-
s.tabRole,
|
|
5066
|
-
s.bucketId,
|
|
5067
|
-
new Set([...s.queries.values()].map((e) => e.def.viewKey))
|
|
5068
|
-
]);
|
|
5069
|
-
const now = yield fx.now();
|
|
5070
|
-
const expired = role === "follower" ? [] : rows.filter((r) => !held.has(r.key) && now - r.updatedAt >= VIEW_RETENTION_MS);
|
|
5071
|
-
const retired = /* @__PURE__ */ new Set();
|
|
5072
|
-
for (let i = 0; i < expired.length; i += GC_CHUNK) {
|
|
5073
|
-
const chunk = expired.slice(i, i + GC_CHUNK);
|
|
5074
|
-
const settled = yield fx.all(chunk.map((r) => fx.local.delete(VIEW_TABLE, viewRecordId(r.key))));
|
|
5075
|
-
chunk.forEach((r, j) => settled[j].ok && retired.add(r.key));
|
|
5076
|
-
}
|
|
5077
|
-
yield fx.state.update(reloadViews(indexOf(rows.filter((r) => !retired.has(r.key)))));
|
|
5078
|
-
if (role === "follower") return;
|
|
5079
|
-
const removed = yield* collectOrphans(bucket, role === "leader");
|
|
5080
|
-
yield fx.emit({
|
|
5081
|
-
type: "log",
|
|
5082
|
-
level: "info",
|
|
5083
|
-
message: "orphan gc done",
|
|
5084
|
-
data: {
|
|
5085
|
-
removed,
|
|
5086
|
-
retiredViews: retired.size
|
|
5087
|
-
}
|
|
5088
|
-
});
|
|
5089
|
-
} catch (error) {
|
|
5090
|
-
yield fx.emit({
|
|
5091
|
-
type: "log",
|
|
5092
|
-
level: "warn",
|
|
5093
|
-
message: "orphan gc failed",
|
|
5094
|
-
data: { error }
|
|
5095
|
-
});
|
|
5096
|
-
} finally {
|
|
5097
|
-
yield fx.timer.set("gc", GC_INTERVAL_MS, { type: "GcTick" });
|
|
5098
|
-
}
|
|
5099
|
-
}
|
|
5100
|
-
/**
|
|
5101
|
-
* Delete every body nothing retains, `GC_CHUNK` at a time. Each chunk is
|
|
5102
|
-
* re-checked against fresh state (a membership committed meanwhile may name
|
|
5103
|
-
* an id again) and the sweep stops if the bucket moved under it.
|
|
5104
|
-
*/
|
|
5105
|
-
function* collectOrphans(bucket, relay) {
|
|
5106
|
-
const candidates = yield fx.state.read((s) => {
|
|
5107
|
-
const keep = retained(s);
|
|
5108
|
-
return [...s.versions.keys()].filter((id) => !keep(id));
|
|
5109
|
-
});
|
|
5110
|
-
let removed = 0;
|
|
5111
|
-
for (let i = 0; i < candidates.length; i += GC_CHUNK) {
|
|
5112
|
-
const chunk = yield fx.state.read((s) => {
|
|
5113
|
-
if (!s.primed || s.bucketId !== bucket) return null;
|
|
5114
|
-
const keep = retained(s);
|
|
5115
|
-
return candidates.slice(i, i + GC_CHUNK).filter((id) => s.versions.has(id) && !keep(id));
|
|
5116
|
-
});
|
|
5117
|
-
if (chunk === null) break;
|
|
5118
|
-
const settled = yield fx.all(chunk.map((id) => fx.local.delete(tableOf$1(id), id)));
|
|
5119
|
-
const done = chunk.filter((_, j) => settled[j].ok);
|
|
5120
|
-
if (done.length === 0) continue;
|
|
5121
|
-
const records = done.map((id) => ({
|
|
5122
|
-
table: tableOf$1(id),
|
|
5123
|
-
op: "DELETE",
|
|
5124
|
-
id,
|
|
5125
|
-
record: {}
|
|
5126
|
-
}));
|
|
5127
|
-
yield fx.state.update(deleteVersions(done));
|
|
5128
|
-
yield fx.ssp.ingest(records);
|
|
5129
|
-
if (relay) yield fx.emit({
|
|
5130
|
-
type: "tabs:broadcast",
|
|
5131
|
-
message: {
|
|
5132
|
-
type: "ingest",
|
|
5133
|
-
records
|
|
5134
|
-
}
|
|
5135
|
-
});
|
|
5136
|
-
removed += done.length;
|
|
5137
|
-
}
|
|
5138
|
-
if (removed > 0) yield fx.dispatch({ type: "FetchRows" });
|
|
5139
|
-
return removed;
|
|
5140
|
-
}
|
|
5141
|
-
|
|
5142
5203
|
//#endregion
|
|
5143
5204
|
//#region src/mutation/write.saga.ts
|
|
5144
5205
|
const debounceKey = (input) => {
|
|
@@ -6307,7 +6368,17 @@ function renderIdSetSelect(surql, kw, idsParam) {
|
|
|
6307
6368
|
].filter((n) => n !== null && n > orderByIndex);
|
|
6308
6369
|
orderBy = " " + surql.slice(orderByIndex, Math.min(...ends)).trim();
|
|
6309
6370
|
}
|
|
6310
|
-
return `${selectClause} FROM
|
|
6371
|
+
return `${selectClause} FROM ${idSetSource(selectClause, idsParam, orderBy !== "")}${orderBy}`;
|
|
6372
|
+
}
|
|
6373
|
+
/**
|
|
6374
|
+
* The FROM source for an id-set read. The in-browser SurrealDB (wasm, 3.x)
|
|
6375
|
+
* reads bare record ids without loading them: a field list over them throws
|
|
6376
|
+
* ("Specify a database to use") and an ORDER BY is silently ignored, while a
|
|
6377
|
+
* plain `SELECT *` works. Those read from `(SELECT * FROM $ids)` instead, the
|
|
6378
|
+
* same rows as documents. The SQLite translator accepts both shapes.
|
|
6379
|
+
*/
|
|
6380
|
+
function idSetSource(selectClause, idsParam, ordered = false) {
|
|
6381
|
+
return !ordered && /^SELECT\s+(VALUE\s+)?\*/i.test(selectClause.trim()) ? `$${idsParam}` : `(SELECT * FROM $${idsParam})`;
|
|
6311
6382
|
}
|
|
6312
6383
|
/**
|
|
6313
6384
|
* Restrict any plan's base rows to an explicit id-set, regardless of offset.
|
|
@@ -7777,6 +7848,15 @@ var AbstractDatabaseService = class {
|
|
|
7777
7848
|
if (next) next();
|
|
7778
7849
|
}
|
|
7779
7850
|
/**
|
|
7851
|
+
* Resolve once every statement issued before this call has settled: take
|
|
7852
|
+
* every slot (FIFO, so earlier statements go first), then hand them back.
|
|
7853
|
+
* Bounded by {@link queryTimeoutMs}, like the statements it waits on.
|
|
7854
|
+
*/
|
|
7855
|
+
async drainStatements() {
|
|
7856
|
+
for (let i = 0; i < this.maxConcurrentQueries; i++) await this.acquireQuerySlot();
|
|
7857
|
+
for (let i = 0; i < this.maxConcurrentQueries; i++) this.releaseQuerySlot();
|
|
7858
|
+
}
|
|
7859
|
+
/**
|
|
7780
7860
|
* Hook run right before a statement is sent, inside its slot. The remote
|
|
7781
7861
|
* service uses it to hold statements while a connect is in flight, so a
|
|
7782
7862
|
* query never runs on a socket that is open but has not had its namespace
|
|
@@ -7989,6 +8069,8 @@ var LocalDatabaseService = class extends AbstractDatabaseService {
|
|
|
7989
8069
|
gate = Promise.resolve();
|
|
7990
8070
|
/** The incoming client while a switch is in flight (for unload cleanup). */
|
|
7991
8071
|
pendingSwitchClient = null;
|
|
8072
|
+
/** The open `connect()` started, until it settles. */
|
|
8073
|
+
opening = null;
|
|
7992
8074
|
constructor(config, logger) {
|
|
7993
8075
|
const events = createDatabaseEventSystem();
|
|
7994
8076
|
super(createBareSurrealClient(), logger, events);
|
|
@@ -8036,6 +8118,15 @@ var LocalDatabaseService = class extends AbstractDatabaseService {
|
|
|
8036
8118
|
return super.query(query, vars);
|
|
8037
8119
|
}
|
|
8038
8120
|
async connect(bucketId = ANON_USER_ID) {
|
|
8121
|
+
const opening = this.openBucket(bucketId);
|
|
8122
|
+
this.opening = opening;
|
|
8123
|
+
try {
|
|
8124
|
+
await opening;
|
|
8125
|
+
} finally {
|
|
8126
|
+
if (this.opening === opening) this.opening = null;
|
|
8127
|
+
}
|
|
8128
|
+
}
|
|
8129
|
+
async openBucket(bucketId) {
|
|
8039
8130
|
const { namespace, database } = this.getConfig();
|
|
8040
8131
|
const store = this.getConfig().store ?? "memory";
|
|
8041
8132
|
this.bucketId = bucketId;
|
|
@@ -8058,14 +8149,20 @@ var LocalDatabaseService = class extends AbstractDatabaseService {
|
|
|
8058
8149
|
* async chains can detect they're stale.
|
|
8059
8150
|
*
|
|
8060
8151
|
* Callers own the drain/rebind choreography (close the gate, quiesce sync +
|
|
8061
|
-
* timers BEFORE calling this; re-provision + rebind AFTER).
|
|
8152
|
+
* timers BEFORE calling this; re-provision + rebind AFTER). Statements that
|
|
8153
|
+
* passed the gate before it closed are let finish before the old client
|
|
8154
|
+
* closes: the engine streams results from its worker, and a client closed
|
|
8155
|
+
* under a statement leaves that statement pending until its deadline, with
|
|
8156
|
+
* every later local statement queued behind it.
|
|
8062
8157
|
*/
|
|
8063
8158
|
async switchStore(bucketId) {
|
|
8159
|
+
if (this.opening) await this.opening.catch(() => {});
|
|
8064
8160
|
if (bucketId === this.bucketId) return;
|
|
8065
8161
|
const { namespace, database } = this.getConfig();
|
|
8066
8162
|
const store = this.getConfig().store ?? "memory";
|
|
8067
8163
|
this.storeEpoch++;
|
|
8068
8164
|
if (store === "memory") {
|
|
8165
|
+
await this.drainStatements();
|
|
8069
8166
|
try {
|
|
8070
8167
|
await this.client.close();
|
|
8071
8168
|
} catch {}
|
|
@@ -8084,6 +8181,7 @@ var LocalDatabaseService = class extends AbstractDatabaseService {
|
|
|
8084
8181
|
} finally {
|
|
8085
8182
|
this.pendingSwitchClient = null;
|
|
8086
8183
|
}
|
|
8184
|
+
await this.drainStatements();
|
|
8087
8185
|
const old = this.client;
|
|
8088
8186
|
this.client = next;
|
|
8089
8187
|
this.bucketId = bucketId;
|
|
@@ -9130,7 +9228,7 @@ var SurrealCacheEngine = class extends LocalDatabaseService {
|
|
|
9130
9228
|
}
|
|
9131
9229
|
async selectByIds(table, ids, opts) {
|
|
9132
9230
|
if (ids.length === 0) return [];
|
|
9133
|
-
let sql = `SELECT ${opts?.select && opts.select.length > 0 ? ["id", ...opts.select].join(", ") : "*"} FROM $__ids`;
|
|
9231
|
+
let sql = `SELECT ${opts?.select && opts.select.length > 0 ? ["id", ...opts.select].join(", ") : "*"} FROM (SELECT * FROM $__ids)`;
|
|
9134
9232
|
if (opts?.orderBy && opts.orderBy.length > 0) sql += ` ORDER BY ${opts.orderBy.map(([f, d]) => `${f} ${d}`).join(", ")}`;
|
|
9135
9233
|
const [rows] = await this.query(`${sql};`, { __ids: ids });
|
|
9136
9234
|
const result = rows ?? [];
|
|
@@ -9305,12 +9403,12 @@ function translateStatement(stmt, vars) {
|
|
|
9305
9403
|
value
|
|
9306
9404
|
};
|
|
9307
9405
|
}
|
|
9308
|
-
m = /^SELECT\s+(VALUE\s+)?(.+?)\s+FROM\s+\$(\w+)$/i.exec(head);
|
|
9406
|
+
m = /^SELECT\s+(VALUE\s+)?(.+?)\s+FROM\s+(?:\$(\w+)|\(\s*SELECT\s+\*\s+FROM\s+\$(\w+)\s*\))$/i.exec(head);
|
|
9309
9407
|
if (m) {
|
|
9310
9408
|
const value = m[1] ? m[2].trim() : void 0;
|
|
9311
9409
|
return {
|
|
9312
9410
|
kind: "selectByIds",
|
|
9313
|
-
ids: vars[m[3]] ?? [],
|
|
9411
|
+
ids: vars[m[3] ?? m[4]] ?? [],
|
|
9314
9412
|
select: projFields(m[2], !!m[1]),
|
|
9315
9413
|
value,
|
|
9316
9414
|
limit,
|
|
@@ -11784,9 +11882,10 @@ var AuthService = class {
|
|
|
11784
11882
|
/**
|
|
11785
11883
|
* Subscribe to auth state changes.
|
|
11786
11884
|
* callback is called immediately with current value and whenever validation status changes.
|
|
11885
|
+
* `immediate: false` skips that first call: only changes are reported.
|
|
11787
11886
|
*/
|
|
11788
|
-
subscribe(cb) {
|
|
11789
|
-
cb(this.announcedUserId());
|
|
11887
|
+
subscribe(cb, opts = {}) {
|
|
11888
|
+
if (opts.immediate !== false) cb(this.announcedUserId());
|
|
11790
11889
|
const id = this.events.subscribe(AuthEventTypes.AuthStateChanged, (event) => {
|
|
11791
11890
|
cb(event.payload);
|
|
11792
11891
|
});
|
|
@@ -12483,14 +12582,17 @@ const MAX_WAIT_MS = 2e3;
|
|
|
12483
12582
|
const PERSIST_DEBOUNCE_MS = 400;
|
|
12484
12583
|
const PRESENCE_THROTTLE_MS = 250;
|
|
12485
12584
|
const CLAIM_RETRY_MS = 5e3;
|
|
12486
|
-
/**
|
|
12487
|
-
*
|
|
12488
|
-
|
|
12585
|
+
/** Longest an open or close waits on the server (the first answer for a
|
|
12586
|
+
* field with nothing local, the last edits going out) before going on
|
|
12587
|
+
* without it. Offline it does not wait at all. */
|
|
12588
|
+
const SERVER_WAIT_MS = 4e3;
|
|
12489
12589
|
/** Pause after a fold is noticed before checking the published document. */
|
|
12490
12590
|
const VERIFY_DELAY_MS = 1e3;
|
|
12491
12591
|
var CrdtManager = class {
|
|
12492
12592
|
sessions = /* @__PURE__ */ new Map();
|
|
12493
12593
|
opening = /* @__PURE__ */ new Map();
|
|
12594
|
+
/** Closes still saving their field's local state, by session key. */
|
|
12595
|
+
saving = /* @__PURE__ */ new Map();
|
|
12494
12596
|
ports = null;
|
|
12495
12597
|
logger;
|
|
12496
12598
|
peer = randomPeer();
|
|
@@ -12546,7 +12648,7 @@ var CrdtManager = class {
|
|
|
12546
12648
|
session = await inFlight;
|
|
12547
12649
|
session.refs++;
|
|
12548
12650
|
} else {
|
|
12549
|
-
const pending = this.start(key, table, recordId, field, kind, cursor).finally(() => this.opening.delete(key));
|
|
12651
|
+
const pending = this.start(key, table, recordId, field, kind, cursor, !!draft).finally(() => this.opening.delete(key));
|
|
12550
12652
|
this.opening.set(key, pending);
|
|
12551
12653
|
session = await pending;
|
|
12552
12654
|
}
|
|
@@ -12572,7 +12674,8 @@ var CrdtManager = class {
|
|
|
12572
12674
|
this.scheduleFlush(s, 0);
|
|
12573
12675
|
}
|
|
12574
12676
|
/** Close one open. The last close of a field resolves once its unsent
|
|
12575
|
-
* edits are in the outbox
|
|
12677
|
+
* edits are in the outbox, or once they are saved on the device when the
|
|
12678
|
+
* server does not answer in time (offline: at once). */
|
|
12576
12679
|
close(table, recordId, field) {
|
|
12577
12680
|
const s = this.sessions.get(`${table}|${recordId}|${field}`);
|
|
12578
12681
|
if (!s || --s.refs > 0) return Promise.resolve();
|
|
@@ -12595,10 +12698,11 @@ var CrdtManager = class {
|
|
|
12595
12698
|
cursor: !!col.cursor
|
|
12596
12699
|
};
|
|
12597
12700
|
}
|
|
12598
|
-
async start(key, table, recordId, field, kind, cursor) {
|
|
12701
|
+
async start(key, table, recordId, field, kind, cursor, fresh = false) {
|
|
12599
12702
|
if (!this.ports) throw new Error("CrdtManager: not attached to a client");
|
|
12600
12703
|
const doc = new (await (loadLoro())).LoroDoc();
|
|
12601
12704
|
doc.setPeerId(this.peer);
|
|
12705
|
+
await this.saving.get(key);
|
|
12602
12706
|
const local = await this.readLocal(recordId, field);
|
|
12603
12707
|
if (local.snapshot) this.tryImport(doc, local.snapshot, key);
|
|
12604
12708
|
const session = {
|
|
@@ -12630,7 +12734,7 @@ var CrdtManager = class {
|
|
|
12630
12734
|
session.handle = new CrdtField(table, recordId, field, kind, cursor, doc, { pushPresence: (state) => this.pushPresence(session, state) });
|
|
12631
12735
|
this.sessions.set(key, session);
|
|
12632
12736
|
this.ensureClaimed();
|
|
12633
|
-
if (!local.snapshot) await this.refetchCanonical(session);
|
|
12737
|
+
if (!local.snapshot && !fresh) await this.serverWait(this.refetchCanonical(session));
|
|
12634
12738
|
session.disposers.push(doc.subscribeLocalUpdates(() => {
|
|
12635
12739
|
session.local.peers[this.peer] ??= { enqueued: 0 };
|
|
12636
12740
|
this.scheduleFlush(session);
|
|
@@ -12651,7 +12755,7 @@ var CrdtManager = class {
|
|
|
12651
12755
|
}, (rows) => this.ingestPresence(session, rows));
|
|
12652
12756
|
session.disposers.push(presence.stop);
|
|
12653
12757
|
}
|
|
12654
|
-
if (!session.handle.hasContent()) await
|
|
12758
|
+
if (!session.handle.hasContent() && !fresh) await this.serverWait(log.settled);
|
|
12655
12759
|
await session.ingesting;
|
|
12656
12760
|
this.scheduleFlush(session, 0);
|
|
12657
12761
|
this.logger.debug({
|
|
@@ -12672,10 +12776,23 @@ var CrdtManager = class {
|
|
|
12672
12776
|
if (s.verifyTimer) clearTimeout(s.verifyTimer);
|
|
12673
12777
|
if (s.presenceSent) this.dropPresence(s);
|
|
12674
12778
|
if (flush) {
|
|
12675
|
-
|
|
12676
|
-
|
|
12779
|
+
const saved = this.persist(s);
|
|
12780
|
+
this.saving.set(s.key, saved);
|
|
12781
|
+
await saved;
|
|
12782
|
+
if (this.saving.get(s.key) === saved) this.saving.delete(s.key);
|
|
12783
|
+
await this.serverWait(this.flush(s));
|
|
12677
12784
|
}
|
|
12678
12785
|
}
|
|
12786
|
+
/**
|
|
12787
|
+
* Wait for server-bound `work`, at most {@link SERVER_WAIT_MS}, and not at
|
|
12788
|
+
* all while disconnected: a remote call made then queues behind the
|
|
12789
|
+
* reconnect for up to the query timeout, and an offline "new thread" form
|
|
12790
|
+
* must not hang on it. The work itself goes on either way.
|
|
12791
|
+
*/
|
|
12792
|
+
serverWait(work) {
|
|
12793
|
+
if (this.remote.getStatus() !== "connected") return Promise.resolve();
|
|
12794
|
+
return Promise.race([work, delay(SERVER_WAIT_MS)]);
|
|
12795
|
+
}
|
|
12679
12796
|
async ingest(s, rows) {
|
|
12680
12797
|
const loro = await loadLoro();
|
|
12681
12798
|
const ids = new Set(rows.map((row) => String(row.id ?? "")));
|
|
@@ -15275,7 +15392,7 @@ function createTabsCoordinator(config, s, host) {
|
|
|
15275
15392
|
return new TabsCoordinator({
|
|
15276
15393
|
tabId: s.tabId,
|
|
15277
15394
|
fingerprint: computeTabsFingerprint({
|
|
15278
|
-
coreVersion: "0.0.1-canary.
|
|
15395
|
+
coreVersion: "0.0.1-canary.328",
|
|
15279
15396
|
schemaHash: hash53(config.schemaSurql),
|
|
15280
15397
|
endpoint: config.database.endpoint ?? "",
|
|
15281
15398
|
namespace: config.database.namespace,
|
|
@@ -16045,7 +16162,7 @@ var Sp00kyClient = class {
|
|
|
16045
16162
|
s.auth.subscribe((userId) => void this.runtime.dispatch({
|
|
16046
16163
|
type: "AuthFlip",
|
|
16047
16164
|
userId
|
|
16048
|
-
}));
|
|
16165
|
+
}), { immediate: false });
|
|
16049
16166
|
this.impersonationBanner = new ImpersonationBanner(() => s.auth.stopImpersonating(), this.config.impersonationBanner);
|
|
16050
16167
|
const banner = this.impersonationBanner;
|
|
16051
16168
|
this.disposers.push(s.auth.subscribeImpersonation((info) => banner.update(info)), () => banner.unmount());
|
|
@@ -16381,7 +16498,7 @@ var Sp00kyClient = class {
|
|
|
16381
16498
|
return this.services.crdt.createDraft(table, field);
|
|
16382
16499
|
}
|
|
16383
16500
|
/** Close a `@crdt` field. The last close resolves once its unsent edits
|
|
16384
|
-
* are queued in the outbox. */
|
|
16501
|
+
* are queued in the outbox; offline, once they are saved on the device. */
|
|
16385
16502
|
closeCrdtField(table, recordId, field) {
|
|
16386
16503
|
return this.services.crdt.close(table, recordId, field);
|
|
16387
16504
|
}
|