@spooky-sync/core 0.0.1-canary.283 → 0.0.1-canary.287
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.js +306 -53
- package/dist/live.js +1 -1
- package/dist/pure.d.ts +9 -2
- package/dist/pure.js +2 -2
- package/dist/sqlite-open.js +28 -1
- package/dist/sqlite-worker.js +1 -1
- package/dist/worker-edges.js +10 -2
- package/package.json +3 -3
- package/skills/sp00ky-core/SKILL.md +9 -6
- package/src/boot/boot.saga.test.ts +9 -2
- package/src/boot/boot.saga.ts +4 -1
- package/src/boot/bucket-switch.saga.ts +2 -0
- package/src/client/runtime.test.ts +4 -1
- package/src/client/sp00ky-client.test.ts +8 -1
- package/src/kernel/constants.ts +15 -2
- package/src/query/fetch.saga.test.ts +44 -1
- package/src/query/fetch.saga.ts +7 -2
- package/src/query/lifecycle.saga.test.ts +169 -17
- package/src/query/lifecycle.saga.ts +86 -28
- package/src/query/materialize.saga.test.ts +36 -3
- package/src/query/materialize.saga.ts +15 -6
- package/src/query/materialize.test.ts +9 -1
- package/src/query/materialize.ts +14 -1
- package/src/query/membership.saga.test.ts +30 -2
- package/src/query/membership.saga.ts +18 -2
- package/src/query/membership.ts +30 -0
- package/src/query/render-set.test.ts +12 -1
- package/src/query/render-set.ts +11 -0
- package/src/query/sql.test.ts +2 -1
- package/src/query/sql.ts +15 -1
- package/src/services/database/landed-body-clear.integration.test.ts +236 -0
- package/src/services/database/sqlite-cache-engine.ts +9 -5
- package/src/services/database/sqlite-plan-sql.test.ts +35 -0
- package/src/services/database/sqlite-plan-sql.ts +26 -0
- package/src/state/client-state.ts +7 -0
- package/src/state/reducers.test.ts +19 -0
- package/src/state/reducers.ts +41 -3
- package/src/state/selectors.test.ts +18 -0
- package/src/state/selectors.ts +32 -0
- package/src/sync/tabs.saga.test.ts +4 -0
- package/src/sync/tabs.saga.ts +3 -0
- package/src/utils/parser.test.ts +37 -1
- package/src/utils/parser.ts +32 -0
package/dist/sqlite-open.js
CHANGED
|
@@ -182,6 +182,33 @@ function serializeRow(row) {
|
|
|
182
182
|
return v;
|
|
183
183
|
});
|
|
184
184
|
}
|
|
185
|
+
/**
|
|
186
|
+
* Serialize a MERGE row ONCE for both halves of the merge upsert: the row a
|
|
187
|
+
* fresh INSERT stores and the RFC 7396 delta `json_patch` applies on conflict.
|
|
188
|
+
*
|
|
189
|
+
* A top-level own key holding `undefined` means "clear this field", which is
|
|
190
|
+
* what SurrealDB's MERGE does with it (the SDK sends it as NONE). JSON drops
|
|
191
|
+
* such a key, so a patch built by {@link serializeRow} alone never mentioned
|
|
192
|
+
* it and the stored value survived every merge: a field cleared on the server
|
|
193
|
+
* stayed set locally. The patch therefore carries each one as a `null` member,
|
|
194
|
+
* which `json_patch` removes; the insert row simply lacks it, exactly as
|
|
195
|
+
* before. Appended to the already-serialized text, so the common case costs no
|
|
196
|
+
* second stringify. Nested `undefined` stays dropped (never a clear here).
|
|
197
|
+
*/
|
|
198
|
+
function serializeMergeRow(row) {
|
|
199
|
+
const insert = serializeRow(row);
|
|
200
|
+
const cleared = [];
|
|
201
|
+
for (const key of Object.keys(row)) if (row[key] === void 0) cleared.push(`${JSON.stringify(key)}:null`);
|
|
202
|
+
if (cleared.length === 0 || insert[0] !== "{") return {
|
|
203
|
+
insert,
|
|
204
|
+
patch: insert
|
|
205
|
+
};
|
|
206
|
+
const members = cleared.join(",");
|
|
207
|
+
return {
|
|
208
|
+
insert,
|
|
209
|
+
patch: insert === "{}" ? `{${members}}` : `${insert.slice(0, -1)},${members}}`
|
|
210
|
+
};
|
|
211
|
+
}
|
|
185
212
|
function reviveRow(json) {
|
|
186
213
|
if (json.indexOf("\"__u8\"") === -1) return JSON.parse(json);
|
|
187
214
|
return JSON.parse(json, (_k, v) => {
|
|
@@ -328,4 +355,4 @@ async function openDb(sqlite3, dbName, useOpfs, opts = {}) {
|
|
|
328
355
|
}
|
|
329
356
|
|
|
330
357
|
//#endregion
|
|
331
|
-
export { renderOrderSql as a,
|
|
358
|
+
export { renderOrderSql as a, serializeMergeRow as c, stableKey as d, projectedDataSql as i, serializeRow as l, openDb as n, renderWhereSql as o, project as r, reviveRow as s, PROMOTION_OPEN_OPTIONS as t, resolveRelations as u };
|
package/dist/sqlite-worker.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { a as renderOrderSql,
|
|
1
|
+
import { a as renderOrderSql, d as stableKey, i as projectedDataSql, n as openDb, o as renderWhereSql, r as project, s as reviveRow, u as resolveRelations } from "./sqlite-open.js";
|
|
2
2
|
import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
|
|
3
3
|
|
|
4
4
|
//#region src/services/database/sqlite-lock-verify.ts
|
package/dist/worker-edges.js
CHANGED
|
@@ -103,11 +103,19 @@ const surql = {
|
|
|
103
103
|
const VIEW_TABLE = "_00_view";
|
|
104
104
|
const LEGACY_VIEW_TABLE = "_00_window";
|
|
105
105
|
const viewRecordId = (viewKey) => new RecordId(VIEW_TABLE, viewKey);
|
|
106
|
-
|
|
106
|
+
/**
|
|
107
|
+
* The durable `_00_view` row: the confirmed membership plus the subquery
|
|
108
|
+
* children, which the GC must keep and a cold scan may paint as much as the
|
|
109
|
+
* members.
|
|
110
|
+
*/
|
|
111
|
+
const viewRow = (ids, children, confirmed, now) => ({
|
|
107
112
|
ids,
|
|
113
|
+
children,
|
|
108
114
|
confirmed,
|
|
109
115
|
updatedAt: now
|
|
110
116
|
});
|
|
117
|
+
/** Every `_00_view` row: the durable view index and the GC read it. */
|
|
118
|
+
const readViewRows = () => `SELECT * FROM ${VIEW_TABLE}`;
|
|
111
119
|
/** Copy every legacy `_00_window` row into `_00_view` once. */
|
|
112
120
|
const readLegacyViewRows = () => `SELECT * FROM ${LEGACY_VIEW_TABLE}`;
|
|
113
121
|
const countViewRows = () => `SELECT count() FROM ${VIEW_TABLE} GROUP ALL`;
|
|
@@ -412,4 +420,4 @@ function rowOfEdge(value) {
|
|
|
412
420
|
}
|
|
413
421
|
|
|
414
422
|
//#endregion
|
|
415
|
-
export {
|
|
423
|
+
export { registerSelect as A, viewKeyInput as B, heartbeatRowGone as C, queryRowCountSelect as D, queryRowCountBatchSelect as E, viewRecordId as F, viewRow as I, surql as L, singleSnapshotSelect as M, subqueryListRefSelect as N, readLegacyViewRows as O, upsertBodiesTx as P, sha256Hex as R, heartbeatBatch as S, listRefSelect as T, LEGACY_VIEW_TABLE as _, metaFromRow as a, bodySelect as b, parseDuration as c, parseQueryParams as d, ANON_USER_ID as f, sanitizeUserId as g, listRefTableFor as h, dedupeRecordVersions as i, registerVars as j, readViewRows as k, parseRecordIdString as l, bucketIdForUser as m, rowOfEdge as n, snapshotFromSingle as o, DEFAULT_REF_MODE as p, FAILED as r, encodeRecordId as s, hashOfEdge as t, withTimeout as u, VIEW_TABLE as v, listRefBatchSelect as w, countViewRows as x, batchSnapshotSelect as y, queryHashInput as z };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.287",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -75,8 +75,8 @@
|
|
|
75
75
|
}
|
|
76
76
|
},
|
|
77
77
|
"dependencies": {
|
|
78
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
79
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
78
|
+
"@spooky-sync/query-builder": "0.0.1-canary.287",
|
|
79
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.287",
|
|
80
80
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
81
81
|
"@surrealdb/wasm": "^3.0.3",
|
|
82
82
|
"blurhash": "^2.0.5",
|
|
@@ -182,12 +182,15 @@ Backend runs let you trigger server-side HTTP operations via an **outbox pattern
|
|
|
182
182
|
### `db.run()` Signature
|
|
183
183
|
|
|
184
184
|
```typescript
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
185
|
+
// A method of the client: called as `db.run(backend, route, payload, options)`.
|
|
186
|
+
interface Sp00kyClient<S> {
|
|
187
|
+
run<B extends BackendNames<S>, R extends BackendRoutes<S, B>>(
|
|
188
|
+
backend: B, // Backend name from sp00ky.yml (e.g., 'api')
|
|
189
|
+
route: R, // Route path from OpenAPI spec (e.g., '/spookify')
|
|
190
|
+
payload: RoutePayload<S, B, R>, // Typed args for the route
|
|
191
|
+
options?: RunOptions
|
|
192
|
+
): Promise<void>;
|
|
193
|
+
}
|
|
191
194
|
```
|
|
192
195
|
|
|
193
196
|
All parameters are type-safe — `backend`, `route`, and `payload` are inferred from the generated schema.
|
|
@@ -19,7 +19,11 @@ describe('boot', () => {
|
|
|
19
19
|
'auth.access': () => 'account',
|
|
20
20
|
});
|
|
21
21
|
const out = await runPure(boot(env, { sharedTabs: false }), {
|
|
22
|
-
handlers: {
|
|
22
|
+
handlers: {
|
|
23
|
+
service: svc.handler,
|
|
24
|
+
'local.query': (e: any) =>
|
|
25
|
+
e.sql.startsWith('SELECT count()') ? [[{ count: 1 }]] : e.sql === 'SELECT * FROM _00_view' ? [[{ id: '_00_view:k', ids: [['t:1', 1]] }]] : [],
|
|
26
|
+
},
|
|
23
27
|
});
|
|
24
28
|
expect(svc.names()).toEqual([
|
|
25
29
|
'hint.read',
|
|
@@ -39,7 +43,10 @@ describe('boot', () => {
|
|
|
39
43
|
]);
|
|
40
44
|
expect(svc.calls[1]).toEqual(['local.connect', ['u1']]);
|
|
41
45
|
expect(out.state).toMatchObject({ bucketId: 'u1', userId: 'user:u1', saltUserId: 'user:u1', sessionId: 'salt-1', localReady: true });
|
|
42
|
-
expect(out.dispatched.map((d) => d.type)).toEqual(['WarmBlobs', 'PrimeCircuit', 'LifecycleTick', '
|
|
46
|
+
expect(out.dispatched.map((d) => d.type)).toEqual(['WarmBlobs', 'PrimeCircuit', 'LifecycleTick', 'StartRemote']);
|
|
47
|
+
// The view index is loaded before anything can paint; the first GC waits out the boot.
|
|
48
|
+
expect(out.state.views).toEqual(new Map([['k', ['t:1']]]));
|
|
49
|
+
expect(out.timers.get('gc')).toEqual({ ms: 30_000, event: { type: 'GcTick' } });
|
|
43
50
|
expect(svc.names()).not.toContain('remote.connect');
|
|
44
51
|
});
|
|
45
52
|
it('shared tabs: takes the role from the coordinator, or falls back to solo; schemaless engines skip provisioning; anonymous skips session auth', async () => {
|
package/src/boot/boot.saga.ts
CHANGED
|
@@ -6,6 +6,8 @@ import { ANON_USER_ID } from '../modules/ref-tables';
|
|
|
6
6
|
import type { SagaEnv } from '../query/env';
|
|
7
7
|
import * as sql from '../query/sql';
|
|
8
8
|
import { loadOutbox } from '../mutation/push.saga';
|
|
9
|
+
import { loadViews } from '../query/lifecycle.saga';
|
|
10
|
+
import { GC_BOOT_DELAY_MS } from '../kernel/constants';
|
|
9
11
|
|
|
10
12
|
export interface BootOptions {
|
|
11
13
|
/** A shared-tabs coordinator is available: ask it for a role instead of opening the store alone. */
|
|
@@ -34,6 +36,7 @@ export function* boot(env: SagaEnv, opts: BootOptions): Saga<void> {
|
|
|
34
36
|
yield fx.state.update(R.setIdentity({ bucketId: bucket }));
|
|
35
37
|
if ((yield fx.service('local.usesSurqlSchema')) as boolean) yield fx.service('migrator.provision');
|
|
36
38
|
yield* migrateWindowToView();
|
|
39
|
+
yield* loadViews();
|
|
37
40
|
yield fx.dispatch({ type: 'WarmBlobs' });
|
|
38
41
|
yield fx.service('ssp.init');
|
|
39
42
|
yield fx.service('ssp.setPermissions');
|
|
@@ -51,7 +54,7 @@ export function* boot(env: SagaEnv, opts: BootOptions): Saga<void> {
|
|
|
51
54
|
yield fx.service('features.init');
|
|
52
55
|
yield fx.service('releases.init');
|
|
53
56
|
yield fx.dispatch({ type: 'LifecycleTick' });
|
|
54
|
-
yield fx.
|
|
57
|
+
yield fx.timer.set('gc', GC_BOOT_DELAY_MS, { type: 'GcTick' });
|
|
55
58
|
yield fx.state.update(R.setIdentity({ localReady: true }));
|
|
56
59
|
yield fx.dispatch({ type: 'StartRemote' });
|
|
57
60
|
}
|
|
@@ -10,6 +10,7 @@ import type { SagaEnv } from '../query/env';
|
|
|
10
10
|
import { queryHashInput } from '../query/hash';
|
|
11
11
|
import { isResolvedBefore, parseViewRow } from '../query/membership';
|
|
12
12
|
import * as sql from '../query/sql';
|
|
13
|
+
import { loadViews } from '../query/lifecycle.saga';
|
|
13
14
|
import { loadOutbox } from '../mutation/push.saga';
|
|
14
15
|
|
|
15
16
|
export interface BucketSwitchOptions {
|
|
@@ -71,6 +72,7 @@ export function* bucketSwitch(env: SagaEnv, target: string, release: (() => void
|
|
|
71
72
|
yield fx.emit({ type: 'log', level: 'warn', message: 'failed to re-persist the auth token', data: { error } });
|
|
72
73
|
}
|
|
73
74
|
}
|
|
75
|
+
yield* loadViews();
|
|
74
76
|
yield* rebindQueries();
|
|
75
77
|
yield* loadOutbox(env);
|
|
76
78
|
yield fx.dispatch({ type: 'EnsureRegistered' });
|
|
@@ -73,7 +73,10 @@ describe('Runtime lanes', () => {
|
|
|
73
73
|
|
|
74
74
|
describe('Runtime state hooks', () => {
|
|
75
75
|
it('dirty queries are materialized after the debounce, once per burst', async () => {
|
|
76
|
-
const { a, rt } = make(
|
|
76
|
+
const { a, rt } = make(
|
|
77
|
+
{ local: { select: async () => [{ id: 'thing:1' }] } },
|
|
78
|
+
buildState([buildEntry({ def: { hash: 'q', plan: { table: 'thing' } as any } })], R.reloadViews(new Map([['v', ['thing:1']]])))
|
|
79
|
+
);
|
|
77
80
|
const seen: unknown[] = [];
|
|
78
81
|
rt.subscribe('q', (rows) => seen.push(rows));
|
|
79
82
|
rt.update(R.markDirty(['q']));
|
|
@@ -49,7 +49,14 @@ describe('Sp00kyClient facade', () => {
|
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
it('queryRaw registers locally and returns the hash before any remote work; subscribers paint from the local store', async () => {
|
|
52
|
-
|
|
52
|
+
// Boot loads the view index: `thing:1` is vouched for, `thing:2` is an orphan and stays hidden.
|
|
53
|
+
const stored = (sql: string) =>
|
|
54
|
+
sql.startsWith('SELECT * FROM thing')
|
|
55
|
+
? [[{ id: 'thing:1' }, { id: 'thing:2' }]]
|
|
56
|
+
: sql === 'SELECT * FROM _00_view'
|
|
57
|
+
? [[{ id: '_00_view:k', ids: [['thing:1', 1]] }]]
|
|
58
|
+
: [];
|
|
59
|
+
const { client, a } = makeClient({ local: { query: (async (sql: string) => stored(sql)) as any } });
|
|
53
60
|
await client.init();
|
|
54
61
|
const hash = await client.queryRaw('SELECT * FROM thing', {}, '10m');
|
|
55
62
|
expect(client.state.queries.get(hash)!.lifecycle.phase).toBe('cold');
|
package/src/kernel/constants.ts
CHANGED
|
@@ -48,8 +48,21 @@ export const AUTH_READY_RETRY_MS = 500;
|
|
|
48
48
|
export const AUTH_READY_MAX_ATTEMPTS = 10;
|
|
49
49
|
/** Heartbeat at this fraction of the shortest ttl in state. */
|
|
50
50
|
export const TTL_HEARTBEAT_FRACTION = 0.5;
|
|
51
|
-
/**
|
|
52
|
-
|
|
51
|
+
/**
|
|
52
|
+
* Orphan body garbage collection: the first sweep waits this long after boot
|
|
53
|
+
* (off the first paint and the first sync round), then repeats on the interval.
|
|
54
|
+
*/
|
|
55
|
+
export const GC_BOOT_DELAY_MS = 30_000;
|
|
56
|
+
export const GC_INTERVAL_MS = 60 * 60 * 1000;
|
|
57
|
+
/** Ids deleted per GC step. */
|
|
58
|
+
export const GC_CHUNK = 200;
|
|
59
|
+
/**
|
|
60
|
+
* A `_00_view` row no query has resolved for this long stops vouching for its
|
|
61
|
+
* ids (the sweep deletes it, so its bodies can be collected). Each session
|
|
62
|
+
* that resolves a query rewrites its row, so this only retires query shapes
|
|
63
|
+
* the app no longer runs.
|
|
64
|
+
*/
|
|
65
|
+
export const VIEW_RETENTION_MS = 14 * 24 * 60 * 60 * 1000;
|
|
53
66
|
/** Remote adapter slot limit. */
|
|
54
67
|
export const REMOTE_CONCURRENCY = 16;
|
|
55
68
|
/** Rolling telemetry sample window per query. */
|
|
@@ -4,7 +4,7 @@ import { runPure } from '../testing/run-pure';
|
|
|
4
4
|
import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
|
|
5
5
|
import * as R from '../state/reducers';
|
|
6
6
|
import { defaultEnv } from './env';
|
|
7
|
-
import { fetchRows } from './fetch.saga';
|
|
7
|
+
import { fetchRows, landChunk } from './fetch.saga';
|
|
8
8
|
import type { StatementResult } from '../kernel/effects';
|
|
9
9
|
|
|
10
10
|
const env = defaultEnv({ tables: [{ name: 'thing', columns: { a: {} } }] } as any);
|
|
@@ -154,3 +154,46 @@ describe('fetchRows', () => {
|
|
|
154
154
|
expect(out.state.versions.get('thing:2')).toBe(1);
|
|
155
155
|
});
|
|
156
156
|
});
|
|
157
|
+
|
|
158
|
+
describe('landChunk', () => {
|
|
159
|
+
const typed = defaultEnv({
|
|
160
|
+
tables: [
|
|
161
|
+
{
|
|
162
|
+
name: 'broadcast',
|
|
163
|
+
columns: {
|
|
164
|
+
title: { type: 'string', optional: false },
|
|
165
|
+
owner: { type: 'string', recordId: true, optional: false },
|
|
166
|
+
club: { type: 'string', recordId: true, optional: true },
|
|
167
|
+
note: { type: 'string', optional: true },
|
|
168
|
+
},
|
|
169
|
+
},
|
|
170
|
+
],
|
|
171
|
+
} as any);
|
|
172
|
+
|
|
173
|
+
// `UPDATE broadcast:x UNSET club` leaves no `club` key in the body the server
|
|
174
|
+
// returns, and a MERGE that does not name it kept the stale local club.
|
|
175
|
+
it('names each optional column the body lacks as undefined so the MERGE clears it', async () => {
|
|
176
|
+
const rid = new RecordId('broadcast', 'x');
|
|
177
|
+
const out = await runPure(landChunk(typed, ['broadcast:x'], [{ id: rid, title: 't', note: 'n', _00_extra: 1 }], new Map([['broadcast:x', 4]]), buildState(), 3), {
|
|
178
|
+
handlers: { 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
|
|
179
|
+
});
|
|
180
|
+
expect(out.result).toBe(true);
|
|
181
|
+
const exec = out.log.find((e) => e.kind === 'local.execute') as any;
|
|
182
|
+
expect(exec.query.sql).toContain('UPSERT ONLY $id0 MERGE $content0');
|
|
183
|
+
expect(exec.vars.content0).toStrictEqual({ club: undefined, title: 't', note: 'n', _00_extra: 1, _00_rv: 4 });
|
|
184
|
+
expect(Object.prototype.hasOwnProperty.call(exec.vars.content0, 'club')).toBe(true);
|
|
185
|
+
// A required column (`owner`) is never cleared: its absence means hidden, not unset.
|
|
186
|
+
expect(Object.prototype.hasOwnProperty.call(exec.vars.content0, 'owner')).toBe(false);
|
|
187
|
+
// The circuit record carries no clear: CREATE/UPDATE replace the circuit row.
|
|
188
|
+
const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
|
|
189
|
+
expect(ingest.records[0].record).toStrictEqual({ id: rid, title: 't', note: 'n', _00_extra: 1, _00_rv: 4 });
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
it('names nothing for a table without a schema', async () => {
|
|
193
|
+
const out = await runPure(landChunk(typed, ['other:1'], [{ id: new RecordId('other', '1'), z: 1 }], new Map([['other:1', 1]]), buildState(), 0), {
|
|
194
|
+
handlers: { 'local.execute': () => undefined, 'ssp.ingest': () => undefined },
|
|
195
|
+
});
|
|
196
|
+
const exec = out.log.find((e) => e.kind === 'local.execute') as any;
|
|
197
|
+
expect(exec.vars.content0).toStrictEqual({ z: 1, _00_rv: 1 });
|
|
198
|
+
});
|
|
199
|
+
});
|
package/src/query/fetch.saga.ts
CHANGED
|
@@ -8,7 +8,7 @@ import type { ClientState } from '../state/client-state';
|
|
|
8
8
|
import * as R from '../state/reducers';
|
|
9
9
|
import { planFetch } from '../state/selectors';
|
|
10
10
|
import { encodeRecordId, parseRecordIdString } from '../utils/index';
|
|
11
|
-
import { cleanRecord } from '../utils/parser';
|
|
11
|
+
import { absentOptionalColumns, cleanRecord } from '../utils/parser';
|
|
12
12
|
import type { SagaEnv } from './env';
|
|
13
13
|
import * as sql from './sql';
|
|
14
14
|
|
|
@@ -87,7 +87,12 @@ export function* landChunk(
|
|
|
87
87
|
const columns = env.schema.tables.find((t) => t.name === table)?.columns;
|
|
88
88
|
const cleaned = columns ? cleanRecord(columns, row) : row;
|
|
89
89
|
const { id: _id, ...content } = cleaned;
|
|
90
|
-
|
|
90
|
+
// The row is the server's whole body, so an optional column it lacks was
|
|
91
|
+
// cleared upstream: name it (as `undefined`) so the MERGE removes the local
|
|
92
|
+
// copy instead of keeping it. `_00_*` local bookkeeping is never named, so
|
|
93
|
+
// MERGE (not REPLACE) still preserves it. The circuit record below needs no
|
|
94
|
+
// such key: CREATE/UPDATE replace the circuit's row outright.
|
|
95
|
+
bodies.push({ id: rid, content: { ...absentOptionalColumns(columns, row), ...content, _00_rv: version } });
|
|
91
96
|
ingest.push({ table, op: state.versions.has(id) ? 'UPDATE' : 'CREATE', id, record: { ...cleaned, _00_rv: version } });
|
|
92
97
|
seen.add(id);
|
|
93
98
|
}
|
|
@@ -4,7 +4,7 @@ import { runPure } from '../testing/run-pure';
|
|
|
4
4
|
import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
|
|
5
5
|
import * as R from '../state/reducers';
|
|
6
6
|
import { defaultEnv } from './env';
|
|
7
|
-
import { ackPrune, evictQuery, gcTick, lifecycleTick } from './lifecycle.saga';
|
|
7
|
+
import { ackPrune, evictQuery, gcTick, lifecycleTick, loadViews } from './lifecycle.saga';
|
|
8
8
|
import type { StatementResult } from '../kernel/effects';
|
|
9
9
|
|
|
10
10
|
const env = defaultEnv({ tables: [] } as any);
|
|
@@ -75,31 +75,158 @@ describe('ackPrune', () => {
|
|
|
75
75
|
});
|
|
76
76
|
});
|
|
77
77
|
|
|
78
|
+
describe('loadViews', () => {
|
|
79
|
+
it('fills the index from every _00_view row (members and children); a failed read leaves it empty', async () => {
|
|
80
|
+
const rows = [
|
|
81
|
+
{ id: '_00_view:a', ids: [['t:1', 1]], children: [['c:1', 1], 'junk'] },
|
|
82
|
+
{ id: new RecordId('_00_view', 'b'), ids: [] },
|
|
83
|
+
{ id: '', ids: [] },
|
|
84
|
+
{ ids: [['t:no-key', 1]] },
|
|
85
|
+
null,
|
|
86
|
+
];
|
|
87
|
+
const out = await runPure(loadViews(), { state: buildState(), handlers: { 'local.query': () => [rows] } });
|
|
88
|
+
expect(out.state.views).toEqual(new Map([['a', ['t:1', 'c:1']], ['b', []]]));
|
|
89
|
+
const odd = await runPure(loadViews(), { state: buildState(), handlers: { 'local.query': () => undefined } });
|
|
90
|
+
expect(odd.state.views.size).toBe(0);
|
|
91
|
+
const failed = await runPure(loadViews(), {
|
|
92
|
+
state: buildState(),
|
|
93
|
+
handlers: {
|
|
94
|
+
'local.query': () => {
|
|
95
|
+
throw new Error('no table');
|
|
96
|
+
},
|
|
97
|
+
},
|
|
98
|
+
});
|
|
99
|
+
expect(failed.state.views.size).toBe(0);
|
|
100
|
+
expect(failed.emitted).toEqual([expect.objectContaining({ level: 'warn', message: 'view index read failed' })]);
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
78
104
|
describe('gcTick', () => {
|
|
79
|
-
|
|
105
|
+
const day = 24 * 60 * 60 * 1000;
|
|
106
|
+
const now = 100 * day;
|
|
107
|
+
const ready = R.setIdentity({ primed: true, bucketId: 'u1' });
|
|
108
|
+
const deletedIds = (log: Array<{ kind: string }>) =>
|
|
109
|
+
log.filter((e: any) => e.kind === 'local.delete').map((e: any) => (e.table === '_00_view' ? `view:${e.id.id}` : e.id));
|
|
110
|
+
|
|
111
|
+
it('retires stale view rows, deletes the bodies nothing retains from the store and the circuit, re-fetches, reschedules', async () => {
|
|
80
112
|
const s = buildState(
|
|
81
|
-
[],
|
|
82
|
-
|
|
113
|
+
[buildEntry({ def: { hash: 'h', viewKey: 'held' }, lifecycle: { phase: 'cached' }, remoteArray: [['thing:mem', 1]] })],
|
|
114
|
+
ready,
|
|
115
|
+
R.setVersions(
|
|
116
|
+
['thing:keep', 'child:1', 'thing:stale', 'thing:held', 'thing:stuck', 'thing:mem', 'thing:gone', 'thing:pending', '_00_view:x', 'thing:failed'].map(
|
|
117
|
+
(id) => [id, 1] as const
|
|
118
|
+
)
|
|
119
|
+
),
|
|
83
120
|
R.outboxReplace([buildOutboxItem({ recordId: 'thing:pending' })])
|
|
84
121
|
);
|
|
85
|
-
const
|
|
122
|
+
const rows = [
|
|
123
|
+
{ id: '_00_view:fresh', ids: [['thing:keep', 1]], children: [['child:1', 1]], updatedAt: now - day },
|
|
124
|
+
// Nobody resolved this query for 15 days: the row is retired and stops vouching.
|
|
125
|
+
{ id: new RecordId('_00_view', 'old'), ids: [['thing:stale', 1]], updatedAt: now - 15 * day },
|
|
126
|
+
// As old, but a query in state holds it.
|
|
127
|
+
{ id: '_00_view:held', ids: [['thing:held', 1]] },
|
|
128
|
+
// As old, but its delete fails: it keeps vouching.
|
|
129
|
+
{ id: '_00_view:stuck', ids: [['thing:stuck', 1]], updatedAt: 0 },
|
|
130
|
+
{ ids: 'bad' },
|
|
131
|
+
];
|
|
86
132
|
const out = await runPure(gcTick(), {
|
|
87
133
|
state: s,
|
|
134
|
+
now,
|
|
88
135
|
handlers: {
|
|
89
|
-
'local.query': () =>
|
|
136
|
+
'local.query': (e: any) => {
|
|
137
|
+
expect(e.sql).toBe('SELECT * FROM _00_view');
|
|
138
|
+
return [rows];
|
|
139
|
+
},
|
|
90
140
|
'local.delete': (e: any) => {
|
|
91
|
-
|
|
92
|
-
if (e.id === 'thing:failed') throw new Error('busy');
|
|
141
|
+
if (e.id === 'thing:failed' || e.id?.id === 'stuck') throw new Error('busy');
|
|
93
142
|
},
|
|
94
143
|
'ssp.ingest': () => undefined,
|
|
95
144
|
},
|
|
96
145
|
});
|
|
97
|
-
expect(
|
|
98
|
-
expect(
|
|
146
|
+
expect(deletedIds(out.log)).toEqual(['view:old', 'view:stuck', 'thing:stale', 'thing:gone', 'thing:failed']);
|
|
147
|
+
expect(out.state.views).toEqual(
|
|
148
|
+
new Map([
|
|
149
|
+
['fresh', ['thing:keep', 'child:1']],
|
|
150
|
+
['held', ['thing:held']],
|
|
151
|
+
['stuck', ['thing:stuck']],
|
|
152
|
+
])
|
|
153
|
+
);
|
|
154
|
+
expect([...out.state.versions.keys()]).toEqual(['thing:keep', 'child:1', 'thing:held', 'thing:stuck', 'thing:mem', 'thing:pending', '_00_view:x', 'thing:failed']);
|
|
99
155
|
const ingest = out.log.find((e) => e.kind === 'ssp.ingest') as any;
|
|
100
|
-
expect(ingest.records).toEqual([
|
|
101
|
-
|
|
102
|
-
|
|
156
|
+
expect(ingest.records).toEqual([
|
|
157
|
+
{ table: 'thing', op: 'DELETE', id: 'thing:stale', record: {} },
|
|
158
|
+
{ table: 'thing', op: 'DELETE', id: 'thing:gone', record: {} },
|
|
159
|
+
]);
|
|
160
|
+
// Solo: nobody to relay to.
|
|
161
|
+
expect(out.emitted.filter((e) => e.type === 'tabs:broadcast')).toEqual([]);
|
|
162
|
+
expect(out.emitted).toContainEqual(expect.objectContaining({ message: 'orphan gc done', data: { removed: 2, retiredViews: 1 } }));
|
|
163
|
+
expect(out.dispatched).toEqual([{ type: 'FetchRows' }]);
|
|
164
|
+
expect(out.timers.get('gc')).toEqual({ ms: 60 * 60 * 1000, event: { type: 'GcTick' } });
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
it('waits for the circuit prime (the versions); a leader relays its deletes to the other tabs', async () => {
|
|
168
|
+
const s = buildState([], R.setIdentity({ bucketId: 'u1' }), R.setTabRole('leader'), R.setVersions([['thing:gone', 1]]));
|
|
169
|
+
const out = await runPure(gcTick(), {
|
|
170
|
+
state: s,
|
|
171
|
+
handlers: {
|
|
172
|
+
'state.wait': (e: any, ctx) => {
|
|
173
|
+
expect(e.until(ctx.state)).toBe(false);
|
|
174
|
+
ctx.state = R.setIdentity({ primed: true })(ctx.state);
|
|
175
|
+
},
|
|
176
|
+
'local.query': () => [[]],
|
|
177
|
+
'local.delete': () => undefined,
|
|
178
|
+
'ssp.ingest': () => undefined,
|
|
179
|
+
},
|
|
180
|
+
});
|
|
181
|
+
expect(out.emitted).toContainEqual({
|
|
182
|
+
type: 'tabs:broadcast',
|
|
183
|
+
message: { type: 'ingest', records: [{ table: 'thing', op: 'DELETE', id: 'thing:gone', record: {} }] },
|
|
184
|
+
});
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it('a follower only refreshes its index: no view row or body is deleted', async () => {
|
|
188
|
+
const s = buildState([], ready, R.setTabRole('follower'), R.setVersions([['thing:gone', 1]]));
|
|
189
|
+
const out = await runPure(gcTick(), {
|
|
190
|
+
state: s,
|
|
191
|
+
now,
|
|
192
|
+
handlers: { 'local.query': () => [[{ id: '_00_view:old', ids: [['thing:x', 1]], updatedAt: 0 }]] },
|
|
193
|
+
});
|
|
194
|
+
expect(out.state.views).toEqual(new Map([['old', ['thing:x']]]));
|
|
195
|
+
expect(deletedIds(out.log)).toEqual([]);
|
|
196
|
+
expect(out.state.versions.has('thing:gone')).toBe(true);
|
|
197
|
+
expect(out.timers.has('gc')).toBe(true);
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
it('each chunk is re-checked against fresh state, and the sweep stops when the bucket moves', async () => {
|
|
201
|
+
const ids = Array.from({ length: 250 }, (_, i) => `thing:o${i}`);
|
|
202
|
+
const s = buildState([], ready, R.setVersions(ids.map((id) => [id, 1] as const)));
|
|
203
|
+
const handlers = (onFirst: (ctx: any) => void) => ({
|
|
204
|
+
'local.query': () => [[]],
|
|
205
|
+
'local.delete': (e: any, ctx: any) => {
|
|
206
|
+
if (e.id === 'thing:o0') onFirst(ctx);
|
|
207
|
+
},
|
|
208
|
+
'ssp.ingest': () => undefined,
|
|
209
|
+
});
|
|
210
|
+
// Meanwhile a query commits o210 and o220's body goes another way: neither is deleted.
|
|
211
|
+
const recheck = await runPure(gcTick(), {
|
|
212
|
+
state: s,
|
|
213
|
+
handlers: handlers((ctx) => {
|
|
214
|
+
ctx.state = R.compose(R.putQuery(buildEntry({ def: { hash: 'q' }, remoteArray: [['thing:o210', 1]] })), R.deleteVersions(['thing:o220']))(ctx.state);
|
|
215
|
+
}),
|
|
216
|
+
});
|
|
217
|
+
const removed = deletedIds(recheck.log);
|
|
218
|
+
expect(removed).toHaveLength(248);
|
|
219
|
+
expect(removed).not.toContain('thing:o210');
|
|
220
|
+
expect(removed).not.toContain('thing:o220');
|
|
221
|
+
expect(recheck.log.filter((e) => e.kind === 'ssp.ingest')).toHaveLength(2);
|
|
222
|
+
const moved = await runPure(gcTick(), { state: s, handlers: handlers((ctx) => (ctx.state = R.setIdentity({ bucketId: 'u2' })(ctx.state))) });
|
|
223
|
+
expect(deletedIds(moved.log)).toHaveLength(200);
|
|
224
|
+
expect(moved.timers.has('gc')).toBe(true);
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
it('failures are logged and the tick still reschedules; a sweep that removes nothing asks for nothing', async () => {
|
|
228
|
+
const s = buildState([], ready, R.setVersions([['thing:gone', 1]]));
|
|
229
|
+
const readFails = await runPure(gcTick(), {
|
|
103
230
|
state: s,
|
|
104
231
|
handlers: {
|
|
105
232
|
'local.query': () => {
|
|
@@ -107,10 +234,35 @@ describe('gcTick', () => {
|
|
|
107
234
|
},
|
|
108
235
|
},
|
|
109
236
|
});
|
|
110
|
-
expect(
|
|
111
|
-
expect(
|
|
112
|
-
const
|
|
113
|
-
|
|
237
|
+
expect(readFails.emitted).toEqual([expect.objectContaining({ message: 'orphan gc failed' })]);
|
|
238
|
+
expect(readFails.timers.has('gc')).toBe(true);
|
|
239
|
+
const ingestFails = await runPure(gcTick(), {
|
|
240
|
+
state: s,
|
|
241
|
+
handlers: {
|
|
242
|
+
'local.query': () => [[]],
|
|
243
|
+
'local.delete': () => undefined,
|
|
244
|
+
'ssp.ingest': () => {
|
|
245
|
+
throw new Error('wasm');
|
|
246
|
+
},
|
|
247
|
+
},
|
|
248
|
+
});
|
|
249
|
+
expect(ingestFails.emitted).toEqual([expect.objectContaining({ message: 'orphan gc failed' })]);
|
|
250
|
+
expect(ingestFails.timers.has('gc')).toBe(true);
|
|
251
|
+
const deleteFails = await runPure(gcTick(), {
|
|
252
|
+
state: s,
|
|
253
|
+
handlers: {
|
|
254
|
+
'local.query': () => [[]],
|
|
255
|
+
'local.delete': () => {
|
|
256
|
+
throw new Error('busy');
|
|
257
|
+
},
|
|
258
|
+
},
|
|
259
|
+
});
|
|
260
|
+
expect(deleteFails.log.filter((e) => e.kind === 'ssp.ingest')).toHaveLength(0);
|
|
261
|
+
expect(deleteFails.dispatched).toEqual([]);
|
|
262
|
+
expect(deleteFails.emitted).toContainEqual(expect.objectContaining({ message: 'orphan gc done', data: { removed: 0, retiredViews: 0 } }));
|
|
263
|
+
const nothing = await runPure(gcTick(), { state: buildState([], ready), handlers: { 'local.query': () => [] } });
|
|
264
|
+
expect(nothing.log.filter((e) => e.kind === 'local.delete')).toHaveLength(0);
|
|
265
|
+
expect(nothing.dispatched).toEqual([]);
|
|
114
266
|
});
|
|
115
267
|
});
|
|
116
268
|
|