@spooky-sync/core 0.0.1-canary.38 → 0.0.1-canary.381
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +91 -0
- package/README.md +59 -16
- package/dist/index.d.ts +2238 -144
- package/dist/index.js +18149 -2475
- package/dist/live.d.ts +149 -0
- package/dist/live.js +739 -0
- package/dist/otel/index.d.ts +1 -1
- package/dist/pure.d.ts +401 -0
- package/dist/pure.js +24 -0
- package/dist/sqlite-open.js +392 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +439 -0
- package/dist/sw.d.ts +207 -0
- package/dist/sw.js +393 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +477 -0
- package/dist/types.d.ts +2082 -39
- package/dist/worker-edges.js +498 -0
- package/dist/worker-impersonation.js +60 -0
- package/dist/worker-ref-tables.d.ts +48 -0
- package/dist/worker-types.d.ts +112 -0
- package/dist/worker-types.js +73 -0
- package/package.json +28 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/scripts/check-worker-entries.mjs +173 -0
- package/skills/sp00ky-core/SKILL.md +42 -8
- package/skills/sp00ky-core/references/config.md +8 -2
- package/src/boot/auth-flip.saga.test.ts +174 -0
- package/src/boot/auth-flip.saga.ts +77 -0
- package/src/boot/boot.saga.test.ts +203 -0
- package/src/boot/boot.saga.ts +146 -0
- package/src/boot/bucket-switch.saga.test.ts +191 -0
- package/src/boot/bucket-switch.saga.ts +151 -0
- package/src/boot/preload.saga.test.ts +96 -0
- package/src/boot/preload.saga.ts +45 -0
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/client/bucket-handle.test.ts +195 -0
- package/src/client/bucket-handle.ts +352 -0
- package/src/client/router.test.ts +59 -0
- package/src/client/router.ts +104 -0
- package/src/client/runtime.test.ts +319 -0
- package/src/client/runtime.ts +307 -0
- package/src/client/services.test.ts +74 -0
- package/src/client/services.ts +437 -0
- package/src/client/sp00ky-client.test.ts +428 -0
- package/src/client/sp00ky-client.ts +737 -0
- package/src/events/index.ts +3 -0
- package/src/index.ts +86 -1
- package/src/kernel/constants.test.ts +19 -0
- package/src/kernel/constants.ts +94 -0
- package/src/kernel/effects.test.ts +40 -0
- package/src/kernel/effects.ts +171 -0
- package/src/kernel/events.ts +60 -0
- package/src/kernel/interpreter.test.ts +116 -0
- package/src/kernel/interpreter.ts +148 -0
- package/src/kernel/saga.test.ts +89 -0
- package/src/kernel/saga.ts +73 -0
- package/src/live/index.test.ts +512 -0
- package/src/live/index.ts +1059 -0
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +209 -0
- package/src/modules/auth/auth.local-first.test.ts +228 -0
- package/src/modules/auth/auth.session-recheck.test.ts +140 -0
- package/src/modules/auth/events/index.ts +7 -1
- package/src/modules/auth/impersonation-banner.ts +443 -0
- package/src/modules/auth/impersonation.integration.test.ts +209 -0
- package/src/modules/auth/impersonation.test.ts +377 -0
- package/src/modules/auth/impersonation.ts +179 -0
- package/src/modules/auth/index.ts +538 -14
- package/src/modules/channel/channel.test.ts +752 -0
- package/src/modules/channel/index.ts +1153 -0
- package/src/modules/channel/webrtc.ts +472 -0
- package/src/modules/crdt/acceptance.ts +75 -0
- package/src/modules/crdt/bytes.ts +50 -0
- package/src/modules/crdt/crdt-field.ts +125 -0
- package/src/modules/crdt/crdt.test.ts +546 -0
- package/src/modules/crdt/hash.ts +103 -0
- package/src/modules/crdt/index.ts +821 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/crdt/mirror.ts +37 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +746 -93
- package/src/modules/devtools/mutations.test.ts +92 -0
- package/src/modules/devtools/mutations.ts +160 -0
- package/src/modules/devtools/notify-throttle.test.ts +166 -0
- package/src/modules/devtools/state-shape.test.ts +263 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +298 -0
- package/src/modules/feature-flag/index.ts +316 -0
- package/src/modules/ref-tables.test.ts +93 -0
- package/src/modules/ref-tables.ts +91 -0
- package/src/modules/web-push/index.test.ts +824 -0
- package/src/modules/web-push/index.ts +1027 -0
- package/src/mutation/ingest-outage.integration.test.ts +400 -0
- package/src/mutation/jobs.test.ts +25 -0
- package/src/mutation/jobs.ts +41 -0
- package/src/mutation/mutation-id.test.ts +37 -0
- package/src/mutation/mutation-id.ts +35 -0
- package/src/mutation/pending-writes.integration.test.ts +175 -0
- package/src/mutation/push.saga.test.ts +361 -0
- package/src/mutation/push.saga.ts +245 -0
- package/src/mutation/rows.test.ts +278 -0
- package/src/mutation/rows.ts +515 -0
- package/src/mutation/tray.saga.test.ts +63 -0
- package/src/mutation/tray.saga.ts +48 -0
- package/src/mutation/write.saga.test.ts +436 -0
- package/src/mutation/write.saga.ts +384 -0
- package/src/pure.ts +66 -0
- package/src/push/types.ts +174 -0
- package/src/query/anon-query-isolation.integration.test.ts +531 -0
- package/src/query/env.test.ts +23 -0
- package/src/query/env.ts +65 -0
- package/src/query/fetch.saga.test.ts +199 -0
- package/src/query/fetch.saga.ts +120 -0
- package/src/query/hash.test.ts +14 -0
- package/src/query/hash.ts +18 -0
- package/src/query/id-set-plan.test.ts +132 -0
- package/src/query/lifecycle.saga.test.ts +274 -0
- package/src/query/lifecycle.saga.ts +186 -0
- package/src/query/list-ref-live.integration.test.ts +203 -0
- package/src/query/materialize.saga.test.ts +200 -0
- package/src/query/materialize.saga.ts +102 -0
- package/src/query/materialize.test.ts +58 -0
- package/src/query/materialize.ts +57 -0
- package/src/query/membership.saga.test.ts +514 -0
- package/src/query/membership.saga.ts +339 -0
- package/src/query/membership.test.ts +151 -0
- package/src/query/membership.ts +234 -0
- package/src/query/register.saga.test.ts +369 -0
- package/src/query/register.saga.ts +293 -0
- package/src/query/render-set.test.ts +47 -0
- package/src/query/render-set.ts +71 -0
- package/src/query/session-guard.integration.test.ts +304 -0
- package/src/query/sql.test.ts +118 -0
- package/src/query/sql.ts +174 -0
- package/src/query/window-query.test.ts +70 -0
- package/src/query/window-query.ts +208 -0
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +209 -0
- package/src/services/database/connection-supervisor.test.ts +318 -0
- package/src/services/database/connection-supervisor.ts +422 -0
- package/src/services/database/database.query-timeout.test.ts +140 -0
- package/src/services/database/database.responses.test.ts +43 -0
- package/src/services/database/database.ts +151 -56
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/index.ts +7 -0
- package/src/services/database/landed-body-clear.integration.test.ts +236 -0
- package/src/services/database/local-migrator.integration.test.ts +110 -0
- package/src/services/database/local-migrator.ts +54 -9
- package/src/services/database/local.test.ts +150 -0
- package/src/services/database/local.ts +503 -59
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +484 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +283 -4
- package/src/services/database/sqlite-cache-engine.test.ts +628 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +1401 -0
- package/src/services/database/sqlite-column-reset.integration.test.ts +100 -0
- package/src/services/database/sqlite-column-reset.ts +91 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +164 -0
- package/src/services/database/sqlite-plan-sql.ts +196 -0
- package/src/services/database/sqlite-projection.test.ts +121 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +261 -0
- package/src/services/database/sqlite-select.ts +131 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +224 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +417 -0
- package/src/services/database/surreal-cache-engine.ts +163 -0
- package/src/services/logger/index.test.ts +132 -0
- package/src/services/logger/index.ts +187 -6
- package/src/services/persistence/resilient.ts +8 -1
- package/src/services/persistence/surrealdb.ts +2 -2
- package/src/services/stream-processor/index.ts +801 -63
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +234 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
- package/src/services/stream-processor/wasm-types.ts +57 -1
- package/src/services/tabs/broker-client.ts +284 -0
- package/src/services/tabs/broker.test.ts +381 -0
- package/src/services/tabs/coordinator.test.ts +386 -0
- package/src/services/tabs/coordinator.ts +646 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +262 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +652 -0
- package/src/sp00ky.ts +5 -394
- package/src/state/client-state.ts +231 -0
- package/src/state/lifecycle.test.ts +184 -0
- package/src/state/lifecycle.ts +127 -0
- package/src/state/reducers.test.ts +337 -0
- package/src/state/reducers.ts +571 -0
- package/src/state/selectors.test.ts +203 -0
- package/src/state/selectors.ts +264 -0
- package/src/sw/index.test.ts +582 -0
- package/src/sw/index.ts +607 -0
- package/src/sw/store.ts +96 -0
- package/src/sync/connection.saga.test.ts +97 -0
- package/src/sync/connection.saga.ts +89 -0
- package/src/sync/live.saga.test.ts +229 -0
- package/src/sync/live.saga.ts +116 -0
- package/src/sync/policy.health.test.ts +56 -0
- package/src/sync/policy.test.ts +663 -0
- package/src/sync/policy.ts +515 -0
- package/src/sync/poll.saga.test.ts +51 -0
- package/src/sync/poll.saga.ts +36 -0
- package/src/sync/tabs.saga.test.ts +78 -0
- package/src/sync/tabs.saga.ts +96 -0
- package/src/testing/adapters.ts +137 -0
- package/src/testing/build.ts +63 -0
- package/src/testing/fake-services.ts +63 -0
- package/src/testing/run-pure.test.ts +92 -0
- package/src/testing/run-pure.ts +128 -0
- package/src/testing/services.ts +17 -0
- package/src/types.ts +694 -7
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/edges.ts +57 -0
- package/src/utils/error-classification.test.ts +116 -0
- package/src/utils/error-classification.ts +74 -0
- package/src/utils/index.ts +83 -6
- package/src/utils/parser.test.ts +67 -102
- package/src/utils/parser.ts +63 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/sha256.ts +15 -0
- package/src/utils/surql.ts +22 -11
- package/src/utils/utils.test.ts +3 -2
- package/tsdown.config.ts +117 -3
- package/vitest.config.ts +48 -0
- package/src/modules/cache/index.ts +0 -241
- package/src/modules/cache/types.ts +0 -19
- package/src/modules/data/data.test.ts +0 -58
- package/src/modules/data/index.ts +0 -882
- package/src/modules/sync/engine.ts +0 -173
- package/src/modules/sync/events/index.ts +0 -78
- package/src/modules/sync/index.ts +0 -3
- package/src/modules/sync/queue/index.ts +0 -2
- package/src/modules/sync/queue/queue-down.ts +0 -90
- package/src/modules/sync/queue/queue-up.ts +0 -224
- package/src/modules/sync/scheduler.ts +0 -84
- package/src/modules/sync/sync.ts +0 -422
- package/src/modules/sync/utils.test.ts +0 -311
- package/src/modules/sync/utils.ts +0 -169
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { runPure } from '../testing/run-pure';
|
|
4
|
+
import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
|
|
5
|
+
import * as R from '../state/reducers';
|
|
6
|
+
import { materialize, streamUpdate } from './materialize.saga';
|
|
7
|
+
|
|
8
|
+
describe('materialize', () => {
|
|
9
|
+
const plan = { table: 'thing' } as any;
|
|
10
|
+
it('no entry: nothing', async () => {
|
|
11
|
+
const out = await runPure(materialize('zz'), { state: buildState() });
|
|
12
|
+
expect(out.log.filter((e) => e.kind !== 'state.read')).toHaveLength(0);
|
|
13
|
+
});
|
|
14
|
+
const vouch = (...ids: string[]) => R.reloadViews(new Map([['other-view', ids]]));
|
|
15
|
+
it('cold: predicate scan through the plan; rows land, dirt clears, subscribers hear once', async () => {
|
|
16
|
+
const s = R.markDirty(['a'])(buildState([buildEntry({ def: { hash: 'a', plan } })], vouch('thing:1')));
|
|
17
|
+
const out = await runPure(materialize('a'), { state: s, handlers: { 'local.select': (e: any) => (e.plan.ids ? [] : [{ id: 'thing:1' }]) } });
|
|
18
|
+
const en = out.state.queries.get('a')!;
|
|
19
|
+
expect(en.records).toEqual([{ id: 'thing:1' }]);
|
|
20
|
+
expect(en.lifecycle.notified).toBe(true);
|
|
21
|
+
expect(en.telemetry.updateCount).toBe(1);
|
|
22
|
+
expect(en.telemetry.lastUpdatedAt).toBe(1_700_000_000_000);
|
|
23
|
+
expect(out.state.dirty.has('a')).toBe(false);
|
|
24
|
+
expect(out.emitted).toEqual([{ type: 'query:records', hash: 'a', records: [{ id: 'thing:1' }] }]);
|
|
25
|
+
const again = await runPure(materialize('a'), { state: out.state, handlers: { 'local.select': () => [{ id: 'thing:1' }] } });
|
|
26
|
+
expect(again.emitted).toEqual([]);
|
|
27
|
+
expect(again.state.queries.get('a')!.telemetry.updateCount).toBe(1);
|
|
28
|
+
});
|
|
29
|
+
it('cold: a body no view vouches for (an orphan) does not paint; vouched, optimistic, _00_ and id-less rows do', async () => {
|
|
30
|
+
const s = buildState(
|
|
31
|
+
[buildEntry({ def: { hash: 'a', plan } })],
|
|
32
|
+
vouch('thing:1', 'thing:4'),
|
|
33
|
+
R.outboxReplace([buildOutboxItem({ id: 'c', type: 'create', recordId: 'thing:3' })])
|
|
34
|
+
);
|
|
35
|
+
const scanned = [
|
|
36
|
+
{ id: 'thing:1' },
|
|
37
|
+
{ id: 'thing:2' },
|
|
38
|
+
{ id: 'thing:3' },
|
|
39
|
+
{ id: '_00_user_feature:f' },
|
|
40
|
+
{ name: 'no id' },
|
|
41
|
+
{ id: new RecordId('thing', '4') },
|
|
42
|
+
{ id: new RecordId('thing', '5') },
|
|
43
|
+
];
|
|
44
|
+
const out = await runPure(materialize('a'), { state: s, handlers: { 'local.select': () => scanned } });
|
|
45
|
+
expect(out.state.queries.get('a')!.records).toEqual([
|
|
46
|
+
{ id: 'thing:1' },
|
|
47
|
+
{ id: 'thing:3' },
|
|
48
|
+
{ id: '_00_user_feature:f' },
|
|
49
|
+
{ name: 'no id' },
|
|
50
|
+
{ id: new RecordId('thing', '4') },
|
|
51
|
+
]);
|
|
52
|
+
// Once the server has answered, membership decides and the index is not consulted.
|
|
53
|
+
const live = buildState([buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'live' }, remoteArray: [['thing:2', 1]] })]);
|
|
54
|
+
const l = await runPure(materialize('a'), { state: live, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
|
|
55
|
+
expect(l.state.queries.get('a')!.records).toEqual([{ id: 'thing:2' }]);
|
|
56
|
+
});
|
|
57
|
+
it("cold: a peer's unconfirmed delete hides the row the store still holds", async () => {
|
|
58
|
+
const s = buildState(
|
|
59
|
+
[buildEntry({ def: { hash: 'a', plan } })],
|
|
60
|
+
vouch('thing:1', 'thing:2'),
|
|
61
|
+
R.setPeerWrite('thing:2', { kind: 'delete', table: 'thing', expiresAt: 1 })
|
|
62
|
+
);
|
|
63
|
+
const out = await runPure(materialize('a'), {
|
|
64
|
+
state: s,
|
|
65
|
+
handlers: { 'local.select': () => [{ id: 'thing:1' }, { id: 'thing:2' }, { name: 'no id' }] },
|
|
66
|
+
});
|
|
67
|
+
expect(out.state.queries.get('a')!.records).toEqual([{ id: 'thing:1' }, { name: 'no id' }]);
|
|
68
|
+
});
|
|
69
|
+
it('cold window renders the SSP local window (orphans dropped); live renders membership with the overlay', async () => {
|
|
70
|
+
const win = buildState(
|
|
71
|
+
[buildEntry({ def: { hash: 'w', plan: { ...plan, offset: 5 }, surql: 'SELECT * FROM thing LIMIT 5 START 5' }, localArray: [['thing:7', 1], ['thing:8', 1]] })],
|
|
72
|
+
vouch('thing:7')
|
|
73
|
+
);
|
|
74
|
+
const w = await runPure(materialize('w'), { state: win, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
|
|
75
|
+
expect(w.state.queries.get('w')!.records).toEqual([{ id: 'thing:7' }]);
|
|
76
|
+
const live = buildState(
|
|
77
|
+
[buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'live' }, remoteArray: [['thing:2', 1], ['thing:1', 1]], localArray: [['thing:9', 1]] })],
|
|
78
|
+
R.outboxReplace([buildOutboxItem({ id: 'c', type: 'create', recordId: 'thing:9' }), buildOutboxItem({ id: 'd', type: 'delete', recordId: 'thing:2' })])
|
|
79
|
+
);
|
|
80
|
+
const l = await runPure(materialize('a'), { state: live, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
|
|
81
|
+
expect(l.state.queries.get('a')!.records).toEqual([{ id: 'thing:1' }, { id: 'thing:9' }]);
|
|
82
|
+
const ordered = buildState([buildEntry({ def: { hash: 'o', plan: { ...plan, orderBy: [['a', 'asc']] } }, lifecycle: { phase: 'live' }, remoteArray: [['thing:2', 1], ['thing:1', 1]] })]);
|
|
83
|
+
const o = await runPure(materialize('o'), { state: ordered, handlers: { 'local.select': (e: any) => e.plan.ids.map((id: any) => ({ id: `${id.table}:${id.id}` })) } });
|
|
84
|
+
expect(o.state.queries.get('o')!.records).toEqual([{ id: 'thing:2' }, { id: 'thing:1' }]);
|
|
85
|
+
});
|
|
86
|
+
it('a failing read counts an error and clears dirt; an entry removed mid-read is left alone', async () => {
|
|
87
|
+
const s = R.markDirty(['a'])(buildState([buildEntry({ def: { hash: 'a', plan } })]));
|
|
88
|
+
const fail = await runPure(materialize('a'), {
|
|
89
|
+
state: s,
|
|
90
|
+
handlers: {
|
|
91
|
+
'local.select': () => {
|
|
92
|
+
throw new Error('locked');
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
});
|
|
96
|
+
expect(fail.state.queries.get('a')!.telemetry.errorCount).toBe(1);
|
|
97
|
+
expect(fail.state.dirty.has('a')).toBe(false);
|
|
98
|
+
expect(fail.emitted).toEqual([expect.objectContaining({ level: 'warn' })]);
|
|
99
|
+
const gone = await runPure(materialize('a'), {
|
|
100
|
+
state: s,
|
|
101
|
+
handlers: {
|
|
102
|
+
'local.select': (_e, ctx) => {
|
|
103
|
+
ctx.state = R.removeQuery('a')(ctx.state);
|
|
104
|
+
return [];
|
|
105
|
+
},
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
expect(gone.emitted).toEqual([]);
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
describe('materialize answers the query', () => {
|
|
113
|
+
const plan = { table: 'thing' } as any;
|
|
114
|
+
const byIds = { 'local.select': (e: any) => (e.plan.ids ?? []).map((id: any) => ({ id: `${id.table}:${id.id}` })) };
|
|
115
|
+
const authority = (out: { emitted: Array<{ type: string }> }) => out.emitted.filter((e) => e.type === 'query:authority');
|
|
116
|
+
|
|
117
|
+
it('a cached re-seed: rows first, then authority; the next render flips nothing', async () => {
|
|
118
|
+
// The bucket-switch window: a durable view was found, `records` is not
|
|
119
|
+
// painted yet. Authority comes from this render, after the rows.
|
|
120
|
+
const s = R.markDirty(['a'])(
|
|
121
|
+
buildState([buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'cached' }, remoteArray: [['thing:1', 1]] })])
|
|
122
|
+
);
|
|
123
|
+
const out = await runPure(materialize('a'), { state: s, handlers: byIds });
|
|
124
|
+
expect(out.state.queries.get('a')!.lifecycle).toMatchObject({ phase: 'cached', notified: true, answered: true });
|
|
125
|
+
expect(out.emitted).toEqual([
|
|
126
|
+
{ type: 'query:records', hash: 'a', records: [{ id: 'thing:1' }] },
|
|
127
|
+
{ type: 'query:authority', hash: 'a', known: true },
|
|
128
|
+
]);
|
|
129
|
+
const again = await runPure(materialize('a'), { state: R.markDirty(['a'])(out.state), handlers: byIds });
|
|
130
|
+
expect(again.emitted).toEqual([]);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
it('a known empty set answers without a rows emission', async () => {
|
|
134
|
+
const s = buildState([buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'live' }, remoteArray: [] })]);
|
|
135
|
+
const out = await runPure(materialize('a'), { state: s, handlers: byIds });
|
|
136
|
+
expect(out.emitted).toEqual([{ type: 'query:authority', hash: 'a', known: true }]);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('a cold render never answers', async () => {
|
|
140
|
+
const s = buildState([buildEntry({ def: { hash: 'a', plan } })], R.reloadViews(new Map([['v', ['thing:1']]])));
|
|
141
|
+
const out = await runPure(materialize('a'), { state: s, handlers: { 'local.select': () => [{ id: 'thing:1' }] } });
|
|
142
|
+
expect(out.state.queries.get('a')!.lifecycle).toMatchObject({ notified: true, answered: false });
|
|
143
|
+
expect(authority(out)).toEqual([]);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it('a server set whose bodies are still downloading is not an answer yet', async () => {
|
|
147
|
+
const s = buildState(
|
|
148
|
+
[buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'live' }, remoteArray: [['thing:1', 1], ['thing:2', 1]] })],
|
|
149
|
+
R.setVersions([['thing:1', 1]])
|
|
150
|
+
);
|
|
151
|
+
const out = await runPure(materialize('a'), { state: s, handlers: byIds });
|
|
152
|
+
expect(out.state.queries.get('a')!.lifecycle.answered).toBe(false);
|
|
153
|
+
expect(authority(out)).toEqual([]);
|
|
154
|
+
const landed = await runPure(materialize('a'), { state: R.setVersions([['thing:2', 1]])(out.state), handlers: byIds });
|
|
155
|
+
expect(landed.state.queries.get('a')!.lifecycle.answered).toBe(true);
|
|
156
|
+
expect(authority(landed)).toEqual([{ type: 'query:authority', hash: 'a', known: true }]);
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('a membership replaced while the read was out does not answer: the next render does', async () => {
|
|
160
|
+
const s = buildState([buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'cached' }, remoteArray: [['thing:1', 1]] })]);
|
|
161
|
+
const out = await runPure(materialize('a'), {
|
|
162
|
+
state: s,
|
|
163
|
+
handlers: {
|
|
164
|
+
'local.select': (e: any, ctx) => {
|
|
165
|
+
ctx.state = R.rebindQuery('a', { id: new RecordId('_00_query', 'b'), lifecycle: { phase: 'cached', remote: 'unregistered', fetchDepth: 0, backgroundDepth: 0, notified: false, answered: false }, remoteArray: [['thing:2', 1]], localArray: [] })(ctx.state);
|
|
166
|
+
return byIds['local.select'](e);
|
|
167
|
+
},
|
|
168
|
+
},
|
|
169
|
+
});
|
|
170
|
+
expect(out.state.queries.get('a')!.lifecycle.answered).toBe(false);
|
|
171
|
+
expect(authority(out)).toEqual([]);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it('after a rebind, a new store that renders nothing clears what the old principal painted', async () => {
|
|
175
|
+
const s = buildState([buildEntry({ def: { hash: 'a', plan }, lifecycle: { phase: 'live', answered: true }, remoteArray: [['thing:1', 1]], records: [{ id: 'thing:1' }] })]);
|
|
176
|
+
const rebound = R.rebindQuery('a', { id: new RecordId('_00_query', 'b'), lifecycle: { phase: 'cached', remote: 'unregistered', fetchDepth: 0, backgroundDepth: 0, notified: false, answered: false }, remoteArray: [], localArray: [] })(s);
|
|
177
|
+
const out = await runPure(materialize('a'), { state: rebound, handlers: byIds });
|
|
178
|
+
expect(out.emitted).toEqual([
|
|
179
|
+
{ type: 'query:records', hash: 'a', records: [] },
|
|
180
|
+
{ type: 'query:authority', hash: 'a', known: true },
|
|
181
|
+
]);
|
|
182
|
+
});
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
describe('streamUpdate', () => {
|
|
186
|
+
it('takes the local id-set (dirtying the query) and records ingest timings; unknown hashes are ignored', async () => {
|
|
187
|
+
const s = buildState([buildEntry({ def: { hash: 'a' } })]);
|
|
188
|
+
const out = await runPure(streamUpdate({ queryHash: 'a', localArray: [['thing:1', 1]], op: 'CREATE', materializationTimeMs: 4, storeApplyMs: 1, circuitStepMs: 2 }), { state: s });
|
|
189
|
+
const en = out.state.queries.get('a')!;
|
|
190
|
+
expect(en.localArray).toEqual([['thing:1', 1]]);
|
|
191
|
+
expect(out.state.dirty.has('a')).toBe(true);
|
|
192
|
+
expect(en.telemetry.lastIngestLatencyMs).toBe(4);
|
|
193
|
+
expect(en.telemetry.materializationSamples).toEqual([4]);
|
|
194
|
+
expect(en.telemetry.phaseLast).toEqual({ sspStoreApply: 1, sspCircuitStep: 2 });
|
|
195
|
+
const bare = await runPure(streamUpdate({ queryHash: 'a', localArray: [] }), { state: s });
|
|
196
|
+
expect(bare.state.queries.get('a')!.telemetry.lastIngestLatencyMs).toBeNull();
|
|
197
|
+
const unknown = await runPure(streamUpdate({ queryHash: 'zz', localArray: [] }), { state: s });
|
|
198
|
+
expect(unknown.log.filter((e) => e.kind === 'state.update')).toHaveLength(0);
|
|
199
|
+
});
|
|
200
|
+
});
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import type { QueryHash } from '../types';
|
|
2
|
+
import type { StreamUpdate } from '../services/stream-processor/index';
|
|
3
|
+
import type { Saga } from '../kernel/saga';
|
|
4
|
+
import { fx } from '../kernel/effects';
|
|
5
|
+
import type { QueryEntry } from '../state/client-state';
|
|
6
|
+
import * as R from '../state/reducers';
|
|
7
|
+
import { needed, overlay, vouched } from '../state/selectors';
|
|
8
|
+
import { isAnswered } from '../state/lifecycle';
|
|
9
|
+
import type { Overlay } from '../state/selectors';
|
|
10
|
+
import { isWindowed, materializeEffect, rowKey, rowsEqual, rowsFromResult } from './materialize';
|
|
11
|
+
import { buildRenderIds, paintsCold, resolveMembership } from './render-set';
|
|
12
|
+
|
|
13
|
+
const NONE: ReadonlySet<string> = new Set();
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Render one query from state: membership (or a cold scan), the outbox
|
|
17
|
+
* overlay, one local read, then notify subscribers if the rows changed. The
|
|
18
|
+
* only writer of `records`. Runs on the `mat:<hash>` dedupe lane whenever the
|
|
19
|
+
* query is dirty. Before the server has answered (`cold`) only bodies some
|
|
20
|
+
* durable view vouches for paint (see `paintsCold`).
|
|
21
|
+
*
|
|
22
|
+
* Also the only place a query becomes answered (see `QueryLifecycle`): the
|
|
23
|
+
* authority flip is emitted here, after the rows it vouches for.
|
|
24
|
+
*/
|
|
25
|
+
export function* materialize(hash: QueryHash): Saga<void> {
|
|
26
|
+
const [entry, ov, known, complete] = (yield fx.state.read((s) => {
|
|
27
|
+
const e = s.queries.get(hash);
|
|
28
|
+
return [e, overlay(s), e?.lifecycle.phase === 'cold' ? vouched(s) : NONE, needed(s, hash).length === 0];
|
|
29
|
+
})) as [QueryEntry | undefined, Overlay, ReadonlySet<string>, boolean];
|
|
30
|
+
if (!entry) return;
|
|
31
|
+
const t0 = (yield fx.now()) as number;
|
|
32
|
+
const isWindow = isWindowed(entry.def.surql);
|
|
33
|
+
const paints = (id: string | null) => id === null || paintsCold(id, known, ov);
|
|
34
|
+
const coldWindow = entry.lifecycle.phase === 'cold' && isWindow;
|
|
35
|
+
const membership = resolveMembership({
|
|
36
|
+
phase: entry.lifecycle.phase,
|
|
37
|
+
remoteArray: entry.remoteArray,
|
|
38
|
+
localArray: coldWindow ? entry.localArray.filter(([id]) => paints(id)) : entry.localArray,
|
|
39
|
+
isWindow,
|
|
40
|
+
});
|
|
41
|
+
const ids = membership
|
|
42
|
+
? buildRenderIds(membership, entry.localArray, ov, {
|
|
43
|
+
hasExplicitOrder: (entry.def.plan?.orderBy?.length ?? 0) > 0,
|
|
44
|
+
isWindow,
|
|
45
|
+
})
|
|
46
|
+
: null;
|
|
47
|
+
const effect = materializeEffect(entry.def, ids);
|
|
48
|
+
let rows: Record<string, unknown>[];
|
|
49
|
+
try {
|
|
50
|
+
rows = rowsFromResult(effect, yield effect);
|
|
51
|
+
// A peer's delete leaves the row in the store until it is confirmed, so
|
|
52
|
+
// the predicate scan must drop it here (an own delete removed it already).
|
|
53
|
+
if (ids === null)
|
|
54
|
+
rows = rows.filter((row) => {
|
|
55
|
+
const key = rowKey(row);
|
|
56
|
+
return paints(key) && !(key !== null && ov.deletes.has(key));
|
|
57
|
+
});
|
|
58
|
+
} catch (error) {
|
|
59
|
+
yield fx.state.update(R.compose(R.recordError(hash), R.clearDirty(hash)));
|
|
60
|
+
yield fx.emit({ type: 'log', level: 'warn', message: 'materialize failed', data: { hash, error } });
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const current = (yield fx.state.read((s) => s.queries.get(hash))) as QueryEntry | undefined;
|
|
64
|
+
if (!current) return;
|
|
65
|
+
const t1 = (yield fx.now()) as number;
|
|
66
|
+
const changed = !rowsEqual(rows, current.records);
|
|
67
|
+
// These rows answer the query when they drew a known membership with no
|
|
68
|
+
// body of a server set still to download, and that membership is still
|
|
69
|
+
// the one in state. A server set applied, or a bucket switch, while the
|
|
70
|
+
// read was out dirtied the query again: the next render answers instead.
|
|
71
|
+
const answered =
|
|
72
|
+
complete && entry.lifecycle.phase !== 'cold' && current.lifecycle.phase !== 'cold' && current.remoteArray === entry.remoteArray;
|
|
73
|
+
yield fx.state.update(
|
|
74
|
+
R.compose(R.setRecords(hash, rows, changed, t1 - t0, answered), changed ? R.stampUpdated(hash, t1) : (s) => s)
|
|
75
|
+
);
|
|
76
|
+
if (changed) yield fx.emit({ type: 'query:records', hash, records: rows });
|
|
77
|
+
// After the rows: a subscriber told "known" must already hold them.
|
|
78
|
+
if (answered && !isAnswered(current.lifecycle)) yield fx.emit({ type: 'query:authority', hash, known: true });
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* A view update from the in-browser SSP: the query's local id-set moved.
|
|
83
|
+
* State takes the array (which dirties the query); the ingest timings feed
|
|
84
|
+
* telemetry. Synthetic updates no longer exist: re-materialization is dirt.
|
|
85
|
+
*/
|
|
86
|
+
export function* streamUpdate(update: StreamUpdate): Saga<void> {
|
|
87
|
+
const hash = update.queryHash;
|
|
88
|
+
const exists = (yield fx.state.read((s) => s.queries.has(hash))) as boolean;
|
|
89
|
+
if (!exists) return;
|
|
90
|
+
const phases: Array<[string, number | undefined]> = [
|
|
91
|
+
['sspStoreApply', update.storeApplyMs],
|
|
92
|
+
['sspCircuitStep', update.circuitStepMs],
|
|
93
|
+
['sspTransform', update.transformMs],
|
|
94
|
+
];
|
|
95
|
+
yield fx.state.update(
|
|
96
|
+
R.compose(
|
|
97
|
+
R.setLocalArray(hash, update.localArray),
|
|
98
|
+
typeof update.materializationTimeMs === 'number' ? R.recordIngest(hash, update.materializationTimeMs) : (s) => s,
|
|
99
|
+
...phases.filter((p): p is [string, number] => typeof p[1] === 'number').map(([phase, ms]) => R.recordPhase(hash, phase, ms))
|
|
100
|
+
)
|
|
101
|
+
);
|
|
102
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { isWindowed, materializeEffect, rowKey, rowsEqual, rowsFromResult } from './materialize';
|
|
4
|
+
|
|
5
|
+
describe('materializeEffect', () => {
|
|
6
|
+
const plan = { table: 'thing', where: [{ field: 'a', op: '=', value: 1 }] } as any;
|
|
7
|
+
it('plan + ids: id-set plan select without where/limit/offset', () => {
|
|
8
|
+
const e = materializeEffect({ surql: 'SELECT * FROM thing', params: { a: 1 }, plan: { ...plan, limit: 5, offset: 2 } }, ['thing:1']);
|
|
9
|
+
expect(e.kind).toBe('local.select');
|
|
10
|
+
if (e.kind !== 'local.select') throw new Error();
|
|
11
|
+
expect(e.plan.ids).toEqual([new RecordId('thing', '1')]);
|
|
12
|
+
expect(e.plan.where).toBeUndefined();
|
|
13
|
+
expect(e.plan.limit).toBeUndefined();
|
|
14
|
+
expect(e.params).toEqual({ a: 1 });
|
|
15
|
+
});
|
|
16
|
+
it('surql + ids: id-set rewrite bound to __win; unparseable surql falls back to the raw query', () => {
|
|
17
|
+
const e = materializeEffect({ surql: 'SELECT * FROM thing WHERE a = $a ORDER BY b LIMIT 3', params: { a: 1 } }, ['thing:1']);
|
|
18
|
+
expect(e).toMatchObject({ kind: 'local.query', sql: 'SELECT * FROM (SELECT * FROM $__win) ORDER BY b' });
|
|
19
|
+
if (e.kind !== 'local.query') throw new Error();
|
|
20
|
+
expect(e.vars).toEqual({ a: 1, __win: [new RecordId('thing', '1')] });
|
|
21
|
+
const raw = materializeEffect({ surql: 'RETURN 1', params: {} }, ['thing:1']);
|
|
22
|
+
expect(raw).toMatchObject({ kind: 'local.query', sql: 'RETURN 1' });
|
|
23
|
+
});
|
|
24
|
+
it('null ids: predicate scan via plan or raw surql', () => {
|
|
25
|
+
expect(materializeEffect({ surql: 's', params: {}, plan }, null)).toMatchObject({ kind: 'local.select', plan });
|
|
26
|
+
expect(materializeEffect({ surql: 'SELECT * FROM thing', params: { x: 1 } }, null)).toMatchObject({
|
|
27
|
+
kind: 'local.query',
|
|
28
|
+
sql: 'SELECT * FROM thing',
|
|
29
|
+
vars: { x: 1 },
|
|
30
|
+
});
|
|
31
|
+
});
|
|
32
|
+
it('rowsFromResult reads both effect shapes; rowsEqual compares structurally', () => {
|
|
33
|
+
const sel = materializeEffect({ surql: 's', params: {}, plan }, null);
|
|
34
|
+
const q = materializeEffect({ surql: 'SELECT * FROM thing', params: {} }, null);
|
|
35
|
+
expect(rowsFromResult(sel, [{ id: 1 }])).toEqual([{ id: 1 }]);
|
|
36
|
+
expect(rowsFromResult(sel, undefined)).toEqual([]);
|
|
37
|
+
expect(rowsFromResult(q, [[{ id: 1 }]])).toEqual([{ id: 1 }]);
|
|
38
|
+
expect(rowsFromResult(q, [null])).toEqual([]);
|
|
39
|
+
expect(rowsFromResult(q, 'x')).toEqual([]);
|
|
40
|
+
const rows = [{ id: 1 }];
|
|
41
|
+
expect(rowsEqual(rows, rows)).toBe(true);
|
|
42
|
+
expect(rowsEqual(rows, [{ id: 1 }])).toBe(true);
|
|
43
|
+
expect(rowsEqual(rows, [{ id: 2 }])).toBe(false);
|
|
44
|
+
expect(rowsEqual(rows, [])).toBe(false);
|
|
45
|
+
});
|
|
46
|
+
it('isWindowed', () => {
|
|
47
|
+
expect(isWindowed('SELECT * FROM t LIMIT 10 START 10')).toBe(true);
|
|
48
|
+
expect(isWindowed('SELECT * FROM t LIMIT 10')).toBe(false);
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
describe('rowKey', () => {
|
|
53
|
+
it('reads a string id (SQLite) or a RecordId (SurrealDB); null when the row carries none', () => {
|
|
54
|
+
expect(rowKey({ id: 'thing:1' })).toBe('thing:1');
|
|
55
|
+
expect(rowKey({ id: new RecordId('thing', '2') })).toBe('thing:2');
|
|
56
|
+
for (const row of [{ name: 'x' }, { id: null }, { id: { other: 1 } }, null, 5]) expect(rowKey(row)).toBeNull();
|
|
57
|
+
});
|
|
58
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import type { QueryPlan } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { RecordId } from 'surrealdb';
|
|
3
|
+
import type { Effect } from '../kernel/effects';
|
|
4
|
+
import { fx } from '../kernel/effects';
|
|
5
|
+
import { encodeRecordId, parseRecordIdString } from '../utils/index';
|
|
6
|
+
import { buildIdSetPlan, buildIdSetSurql, buildWindowMaterialization } from './window-query';
|
|
7
|
+
|
|
8
|
+
export interface MaterializeSource {
|
|
9
|
+
surql: string;
|
|
10
|
+
params: Record<string, unknown>;
|
|
11
|
+
plan?: QueryPlan;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export const isWindowed = (surql: string): boolean => buildWindowMaterialization(surql) !== null;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The one read that materializes a query. `ids` is the render set (membership
|
|
18
|
+
* with the overlay applied); `null` means "no membership yet, scan the local
|
|
19
|
+
* store with the query's own predicate".
|
|
20
|
+
*/
|
|
21
|
+
export function materializeEffect(src: MaterializeSource, ids: string[] | null): Effect {
|
|
22
|
+
if (ids !== null) {
|
|
23
|
+
const parsed = ids.map((id) => parseRecordIdString(id));
|
|
24
|
+
if (src.plan) return fx.local.select(buildIdSetPlan(src.plan, parsed), src.params);
|
|
25
|
+
const idSet = buildIdSetSurql(src.surql);
|
|
26
|
+
if (idSet) return fx.local.query(idSet.query, { ...src.params, __win: parsed });
|
|
27
|
+
return fx.local.query(src.surql, src.params);
|
|
28
|
+
}
|
|
29
|
+
if (src.plan) return fx.local.select(src.plan, src.params);
|
|
30
|
+
return fx.local.query(src.surql, src.params);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Rows out of either read effect's result. */
|
|
34
|
+
export function rowsFromResult(effect: Effect, result: unknown): Record<string, unknown>[] {
|
|
35
|
+
if (effect.kind === 'local.select') return (result as Record<string, unknown>[] | undefined) ?? [];
|
|
36
|
+
const first = Array.isArray(result) ? result[0] : undefined;
|
|
37
|
+
return Array.isArray(first) ? (first as Record<string, unknown>[]) : [];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* A row's encoded record id: a string on the SQLite engine, a `RecordId` on
|
|
42
|
+
* the SurrealDB one. `null` for a row that carries none (a projection
|
|
43
|
+
* without `id`, a `SELECT VALUE`).
|
|
44
|
+
*/
|
|
45
|
+
export function rowKey(row: unknown): string | null {
|
|
46
|
+
const id = row !== null && typeof row === 'object' ? (row as { id?: unknown }).id : undefined;
|
|
47
|
+
if (typeof id === 'string') return id;
|
|
48
|
+
if (id !== null && typeof id === 'object' && 'table' in id && 'id' in id) return encodeRecordId(id as RecordId<string>);
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Cheap structural equality for the "did the rows change" check. */
|
|
53
|
+
export function rowsEqual(a: ReadonlyArray<unknown>, b: ReadonlyArray<unknown>): boolean {
|
|
54
|
+
if (a === b) return true;
|
|
55
|
+
if (a.length !== b.length) return false;
|
|
56
|
+
return JSON.stringify(a) === JSON.stringify(b);
|
|
57
|
+
}
|