@spooky-sync/core 0.0.1-canary.20 → 0.0.1-canary.201
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -0
- package/dist/index.d.ts +2184 -54
- package/dist/index.js +11515 -2399
- package/dist/otel/index.d.ts +2 -2
- package/dist/otel/index.js +6 -6
- package/dist/sqlite-open.js +276 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +421 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +434 -0
- package/dist/types.d.ts +688 -11
- package/package.json +11 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
- package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
- package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +35 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +112 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +210 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +463 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.membership.test.ts +462 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.settled-writes.test.ts +206 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +1580 -130
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +194 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +386 -37
- package/src/modules/devtools/notify-throttle.test.ts +149 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +251 -0
- package/src/modules/feature-flag/index.ts +308 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.test.ts +107 -0
- package/src/modules/sync/queue/queue-down.ts +35 -6
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
- package/src/modules/sync/queue/queue-up.ts +241 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.retry.test.ts +156 -0
- package/src/modules/sync/scheduler.ts +158 -11
- package/src/modules/sync/sync.cleanup.test.ts +116 -0
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.heartbeat.test.ts +80 -0
- package/src/modules/sync/sync.live-removal.test.ts +134 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1558 -99
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +201 -17
- package/src/otel/index.ts +13 -10
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +160 -0
- package/src/services/database/connection-supervisor.test.ts +289 -0
- package/src/services/database/connection-supervisor.ts +415 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +32 -12
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +7 -0
- package/src/services/database/local-migrator.ts +30 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +478 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +110 -14
- package/src/services/database/sqlite-cache-engine.test.ts +558 -0
- package/src/services/database/sqlite-cache-engine.ts +1257 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +106 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +121 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +221 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +416 -0
- package/src/services/database/surreal-cache-engine.ts +141 -0
- package/src/services/logger/index.ts +3 -2
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +444 -52
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +216 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +23 -2
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +278 -0
- package/src/services/tabs/coordinator.test.ts +244 -0
- package/src/services/tabs/coordinator.ts +576 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +242 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +586 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.ts +1543 -0
- package/src/types.ts +496 -13
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +73 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { UpQueue } from './queue-up';
|
|
3
|
+
import { translateSurql } from '../../../services/database/surql-translate';
|
|
4
|
+
import { surql, classifySyncError } from '../../../utils/index';
|
|
5
|
+
|
|
6
|
+
function makeLogger(): any {
|
|
7
|
+
const noop = () => {};
|
|
8
|
+
const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
9
|
+
l.child = () => l;
|
|
10
|
+
return l;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const ROW = {
|
|
14
|
+
id: '_00_pending_mutations:0001753875000000_0001_tabB',
|
|
15
|
+
mutationType: 'create',
|
|
16
|
+
recordId: 'game:g1',
|
|
17
|
+
data: { title: 'x' },
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
// Shared-tabs: followers commit outbox rows into the shared store and notify
|
|
21
|
+
// the leader by id. The notify may be REPLAYED (failover buffering), so the
|
|
22
|
+
// enqueue must be idempotent, and only the leader ever drains.
|
|
23
|
+
describe('UpQueue.enqueueFromDatabase', () => {
|
|
24
|
+
function makeQueue(rows: Record<string, unknown[]>) {
|
|
25
|
+
const query = vi.fn(async (_sql: string, vars?: Record<string, unknown>) => {
|
|
26
|
+
const ids = (vars?.mutation_ids as unknown[]) ?? [];
|
|
27
|
+
const id = String(ids[0] ?? '');
|
|
28
|
+
return [rows[id] ?? []];
|
|
29
|
+
});
|
|
30
|
+
const local: any = { query };
|
|
31
|
+
return { queue: new UpQueue(local, makeLogger()), query };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
it('loads exactly the forwarded row and enqueues it once', async () => {
|
|
35
|
+
const { queue } = makeQueue({ [ROW.id]: [ROW] });
|
|
36
|
+
await queue.enqueueFromDatabase(ROW.id);
|
|
37
|
+
expect(queue.size).toBe(1);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it('is idempotent: a replayed notify does not double-enqueue', async () => {
|
|
41
|
+
const { queue, query } = makeQueue({ [ROW.id]: [ROW] });
|
|
42
|
+
await queue.enqueueFromDatabase(ROW.id);
|
|
43
|
+
await queue.enqueueFromDatabase(ROW.id);
|
|
44
|
+
expect(queue.size).toBe(1);
|
|
45
|
+
// The second call short-circuits before even touching the store.
|
|
46
|
+
expect(query).toHaveBeenCalledTimes(1);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
it('a row that never committed is a silent no-op (the app promise already rejected)', async () => {
|
|
50
|
+
const { queue } = makeQueue({});
|
|
51
|
+
await queue.enqueueFromDatabase(ROW.id);
|
|
52
|
+
expect(queue.size).toBe(0);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
// The mock above replaces `local.query` wholesale, so on its own it proves
|
|
56
|
+
// nothing about the SQLite engine actually being able to RUN the statement.
|
|
57
|
+
// That gap shipped a bug: the emitted `SELECT * FROM $mutation_id` passed a
|
|
58
|
+
// single RecordId where the engine's `selectByIds` lowering expects an array,
|
|
59
|
+
// so it threw, the surrounding catch swallowed it at `error` level, and every
|
|
60
|
+
// follower mutation was silently never pushed. Drive the real translator.
|
|
61
|
+
it('discards a create with no payload instead of queueing a guaranteed failure', async () => {
|
|
62
|
+
// `processUpEvent` does `Object.keys(event.data)`, so this row can never be
|
|
63
|
+
// sent. Queued, it sat at the HEAD and blocked every later mutation for the
|
|
64
|
+
// whole app: `next()` re-queues on failure, so the backlog never drained.
|
|
65
|
+
const dropped: any[] = [];
|
|
66
|
+
const rows: Record<string, unknown[]> = {
|
|
67
|
+
[ROW.id]: [{ id: ROW.id, mutationType: 'create', recordId: 'comment:c1' }],
|
|
68
|
+
};
|
|
69
|
+
const query = vi.fn(async (sql: string, vars?: Record<string, unknown>) => {
|
|
70
|
+
if (sql.startsWith('DELETE')) return [[]];
|
|
71
|
+
const ids = (vars?.mutation_ids as unknown[]) ?? [];
|
|
72
|
+
return [rows[String(ids[0] ?? '')] ?? []];
|
|
73
|
+
});
|
|
74
|
+
const queue = new UpQueue({ query } as any, makeLogger(), (d) => dropped.push(d));
|
|
75
|
+
|
|
76
|
+
await queue.enqueueFromDatabase(ROW.id);
|
|
77
|
+
|
|
78
|
+
expect(queue.size).toBe(0);
|
|
79
|
+
expect(dropped).toHaveLength(1);
|
|
80
|
+
// And the row is deleted, so it cannot re-poison the next boot.
|
|
81
|
+
expect(query.mock.calls.some(([sql]) => String(sql).startsWith('DELETE'))).toBe(true);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it('loadFromDatabase drains the replayable rows even when one is unsendable', async () => {
|
|
85
|
+
const dropped: any[] = [];
|
|
86
|
+
const query = vi.fn(async (sql: string) => {
|
|
87
|
+
if (sql.startsWith('DELETE')) return [[]];
|
|
88
|
+
return [
|
|
89
|
+
[
|
|
90
|
+
{ id: '_00_pending_mutations:a', mutationType: 'create', recordId: 'comment:c1' },
|
|
91
|
+
{
|
|
92
|
+
id: '_00_pending_mutations:b',
|
|
93
|
+
mutationType: 'update',
|
|
94
|
+
recordId: 'game:g1',
|
|
95
|
+
data: { title: 'x' },
|
|
96
|
+
},
|
|
97
|
+
],
|
|
98
|
+
];
|
|
99
|
+
});
|
|
100
|
+
const queue = new UpQueue({ query } as any, makeLogger(), (d) => dropped.push(d));
|
|
101
|
+
|
|
102
|
+
await queue.loadFromDatabase();
|
|
103
|
+
|
|
104
|
+
// The good row still loads; the poison one is dropped and reported.
|
|
105
|
+
expect(queue.size).toBe(1);
|
|
106
|
+
expect(dropped).toHaveLength(1);
|
|
107
|
+
expect(dropped[0].mutationId).toBe('_00_pending_mutations:a');
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it('persists the payload for a create, so it can be replayed at all', () => {
|
|
111
|
+
// The create branch used to accept `dataVar` and ignore it, so the outbox
|
|
112
|
+
// row was the ONLY copy of a pending create and it carried no data.
|
|
113
|
+
const stmt = surql.createMutation('create', 'mid', 'id', 'data');
|
|
114
|
+
expect(stmt).toContain('data = $data');
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('deletes the row using the STORED id, not its escaped display form', async () => {
|
|
118
|
+
// Outbox ids contain `_` and `-`, so the store returns them escaped:
|
|
119
|
+
// `_00_pending_mutations:⟨1785…_0005_c960…⟩`. Parsing that verbatim and
|
|
120
|
+
// re-encoding escapes the brackets AGAIN, so the DELETE targeted
|
|
121
|
+
// `⟨⟨…\⟩⟩` and matched nothing: a mutation replayed from the store could
|
|
122
|
+
// never have its row removed, even after a SUCCESSFUL push, so it was
|
|
123
|
+
// re-sent on every boot.
|
|
124
|
+
const storedId = '_00_pending_mutations:⟨1785540216679_0005_c960958c-d6ed⟩';
|
|
125
|
+
const deletes: any[] = [];
|
|
126
|
+
const query = vi.fn(async (sql: string, vars?: Record<string, unknown>) => {
|
|
127
|
+
if (sql.startsWith('DELETE')) {
|
|
128
|
+
deletes.push(vars?.mutation_id);
|
|
129
|
+
return [[]];
|
|
130
|
+
}
|
|
131
|
+
return [[{ id: storedId, mutationType: 'create', recordId: 'comment:c1' }]];
|
|
132
|
+
});
|
|
133
|
+
const queue = new UpQueue({ query } as any, makeLogger(), () => {});
|
|
134
|
+
|
|
135
|
+
await queue.enqueueFromDatabase(storedId);
|
|
136
|
+
|
|
137
|
+
expect(deletes).toHaveLength(1);
|
|
138
|
+
const rid: any = deletes[0];
|
|
139
|
+
expect(String(rid.table)).toBe('_00_pending_mutations');
|
|
140
|
+
expect(rid.id).toBe('1785540216679_0005_c960958c-d6ed');
|
|
141
|
+
expect(String(rid.id)).not.toContain('⟨');
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
it('a push timeout classifies as network, so it retries instead of rolling back', () => {
|
|
145
|
+
const err = new Error('Mutation push timed out after 30000ms (create)');
|
|
146
|
+
expect(classifySyncError(err)).toBe('network');
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it('emits a statement the SQLite engine can actually translate', async () => {
|
|
150
|
+
const { queue, query } = makeQueue({ [ROW.id]: [ROW] });
|
|
151
|
+
await queue.enqueueFromDatabase(ROW.id);
|
|
152
|
+
|
|
153
|
+
const [sql, vars] = query.mock.calls[0] as [string, Record<string, unknown>];
|
|
154
|
+
const translated = translateSurql(sql, vars);
|
|
155
|
+
const op: any = translated.ops[0];
|
|
156
|
+
|
|
157
|
+
expect(op.kind).toBe('selectByIds');
|
|
158
|
+
// The engine does `ids.length` then `ids.map(...)`: a non-array silently
|
|
159
|
+
// skips the empty-guard and then throws.
|
|
160
|
+
expect(Array.isArray(op.ids)).toBe(true);
|
|
161
|
+
expect(op.ids).toHaveLength(1);
|
|
162
|
+
expect(() => (op.ids as unknown[]).map((x) => x)).not.toThrow();
|
|
163
|
+
});
|
|
164
|
+
});
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { RecordId } from 'surrealdb';
|
|
2
|
-
import {
|
|
2
|
+
import type { LocalStore } from '../../../services/database/index';
|
|
3
|
+
import type { SyncQueueEventSystem } from '../events/index';
|
|
4
|
+
import { createSyncQueueEventSystem, SyncQueueEventTypes } from '../events/index';
|
|
3
5
|
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
import { Logger } from '../../../services/logger/index';
|
|
10
|
-
import { PushEventOptions } from '../../../events/index';
|
|
6
|
+
parseRecordIdString,
|
|
7
|
+
extractTablePart,
|
|
8
|
+
classifySyncError,
|
|
9
|
+
encodeRecordId,
|
|
10
|
+
} from '../../../utils/index';
|
|
11
|
+
import type { Logger } from '../../../services/logger/index';
|
|
12
|
+
import type { PushEventOptions } from '../../../events/index';
|
|
11
13
|
|
|
12
14
|
export type CreateEvent = {
|
|
13
15
|
type: 'create';
|
|
@@ -40,25 +42,80 @@ export type UpEvent = CreateEvent | UpdateEvent | DeleteEvent;
|
|
|
40
42
|
|
|
41
43
|
export type RollbackCallback = (event: UpEvent, error: Error) => Promise<void>;
|
|
42
44
|
|
|
45
|
+
/**
|
|
46
|
+
* A pending mutation that can never be sent, so it was discarded instead of
|
|
47
|
+
* being retried forever at the head of the queue.
|
|
48
|
+
*/
|
|
49
|
+
export type DroppedMutation = {
|
|
50
|
+
mutationId: string;
|
|
51
|
+
recordId?: string;
|
|
52
|
+
mutationType?: string;
|
|
53
|
+
reason: string;
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
export type DroppedCallback = (dropped: DroppedMutation) => void;
|
|
57
|
+
|
|
43
58
|
export class UpQueue {
|
|
44
59
|
private queue: UpEvent[] = [];
|
|
45
60
|
private _events: SyncQueueEventSystem;
|
|
46
61
|
private logger: Logger;
|
|
47
|
-
private debouncedMutations: Map<
|
|
62
|
+
private debouncedMutations: Map<
|
|
63
|
+
string,
|
|
64
|
+
{ timer: any; firstBeforeRecord?: Record<string, unknown> }
|
|
65
|
+
>;
|
|
48
66
|
|
|
49
67
|
get events(): SyncQueueEventSystem {
|
|
50
68
|
return this._events;
|
|
51
69
|
}
|
|
52
70
|
|
|
53
71
|
constructor(
|
|
54
|
-
private local:
|
|
55
|
-
logger: Logger
|
|
72
|
+
private local: LocalStore,
|
|
73
|
+
logger: Logger,
|
|
74
|
+
private onDropped?: DroppedCallback
|
|
56
75
|
) {
|
|
57
76
|
this._events = createSyncQueueEventSystem();
|
|
58
77
|
this.logger = logger.child({ service: 'UpQueue' });
|
|
59
78
|
this.debouncedMutations = new Map();
|
|
60
79
|
}
|
|
61
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Discard an outbox row that can never be replayed and report it.
|
|
83
|
+
*
|
|
84
|
+
* Silently skipping such a row leaves it in the store to be re-read on every
|
|
85
|
+
* boot; leaving it QUEUED is worse, since `next()` re-queues it on failure and
|
|
86
|
+
* one unsendable row then blocks every later mutation for the whole app. A
|
|
87
|
+
* lost write must also be loud: this is the only signal a caller gets.
|
|
88
|
+
*/
|
|
89
|
+
private async discardUnreplayable(row: any, reason: string): Promise<void> {
|
|
90
|
+
const mutationId = typeof row?.id === 'string' ? row.id : encodeRecordId(row?.id);
|
|
91
|
+
this.logger.error(
|
|
92
|
+
{
|
|
93
|
+
mutationId,
|
|
94
|
+
recordId: row?.recordId,
|
|
95
|
+
mutationType: row?.mutationType,
|
|
96
|
+
reason,
|
|
97
|
+
Category: 'sp00ky-client::UpQueue::discardUnreplayable',
|
|
98
|
+
},
|
|
99
|
+
'Discarding an unsendable pending mutation'
|
|
100
|
+
);
|
|
101
|
+
try {
|
|
102
|
+
await this.local.query(`DELETE $mutation_id`, {
|
|
103
|
+
mutation_id: parseStoredRecordId(mutationId),
|
|
104
|
+
});
|
|
105
|
+
} catch (error) {
|
|
106
|
+
this.logger.error(
|
|
107
|
+
{ error, mutationId, Category: 'sp00ky-client::UpQueue::discardUnreplayable' },
|
|
108
|
+
'Failed to delete an unsendable pending mutation'
|
|
109
|
+
);
|
|
110
|
+
}
|
|
111
|
+
this.onDropped?.({
|
|
112
|
+
mutationId,
|
|
113
|
+
recordId: row?.recordId,
|
|
114
|
+
mutationType: row?.mutationType,
|
|
115
|
+
reason,
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
|
|
62
119
|
get size(): number {
|
|
63
120
|
return this.queue.length;
|
|
64
121
|
}
|
|
@@ -104,7 +161,32 @@ export class UpQueue {
|
|
|
104
161
|
this.debouncedMutations.set(key, { timer, firstBeforeRecord });
|
|
105
162
|
}
|
|
106
163
|
|
|
107
|
-
|
|
164
|
+
/**
|
|
165
|
+
* Cancel all pending debounce timers WITHOUT enqueueing their events. Used
|
|
166
|
+
* on local-bucket switches: the mutation's `_00_pending_mutations` row was
|
|
167
|
+
* already persisted at mutation time, so dropping the in-memory push only
|
|
168
|
+
* defers it to that bucket's next `loadFromDatabase` — it must NOT be pushed
|
|
169
|
+
* now, the remote session already belongs to the next user.
|
|
170
|
+
*/
|
|
171
|
+
clearDebounceTimers(): void {
|
|
172
|
+
for (const { timer } of this.debouncedMutations.values()) {
|
|
173
|
+
clearTimeout(timer);
|
|
174
|
+
}
|
|
175
|
+
this.debouncedMutations.clear();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* @param onSettled Reports a mutation the server ACCEPTED, after its outbox
|
|
180
|
+
* row is gone. Deliberately not called on the rollback path: a rejected
|
|
181
|
+
* mutation must stop being rendered immediately, while an accepted one has
|
|
182
|
+
* to stay visible until its membership arrives (see
|
|
183
|
+
* `DataModule.noteWriteSettled`).
|
|
184
|
+
*/
|
|
185
|
+
async next(
|
|
186
|
+
fn: (event: UpEvent) => Promise<void>,
|
|
187
|
+
onRollback?: RollbackCallback,
|
|
188
|
+
onSettled?: (event: UpEvent) => void
|
|
189
|
+
): Promise<void> {
|
|
108
190
|
const event = this.queue.shift();
|
|
109
191
|
if (event) {
|
|
110
192
|
try {
|
|
@@ -114,7 +196,7 @@ export class UpQueue {
|
|
|
114
196
|
|
|
115
197
|
if (errorType === 'network') {
|
|
116
198
|
this.logger.error(
|
|
117
|
-
{ error, event, Category: '
|
|
199
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
118
200
|
'Network error processing mutation, re-queuing'
|
|
119
201
|
);
|
|
120
202
|
this.queue.unshift(event);
|
|
@@ -123,14 +205,14 @@ export class UpQueue {
|
|
|
123
205
|
|
|
124
206
|
// Application error — rollback instead of re-queuing
|
|
125
207
|
this.logger.error(
|
|
126
|
-
{ error, event, Category: '
|
|
208
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
127
209
|
'Application error processing mutation, rolling back'
|
|
128
210
|
);
|
|
129
211
|
try {
|
|
130
212
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
131
213
|
} catch (removeError) {
|
|
132
214
|
this.logger.error(
|
|
133
|
-
{ error: removeError, event, Category: '
|
|
215
|
+
{ error: removeError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
134
216
|
'Failed to remove rolled-back mutation from database'
|
|
135
217
|
);
|
|
136
218
|
}
|
|
@@ -139,7 +221,7 @@ export class UpQueue {
|
|
|
139
221
|
await onRollback(event, error instanceof Error ? error : new Error(String(error)));
|
|
140
222
|
} catch (rollbackError) {
|
|
141
223
|
this.logger.error(
|
|
142
|
-
{ error: rollbackError, event, Category: '
|
|
224
|
+
{ error: rollbackError, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
143
225
|
'Rollback handler failed'
|
|
144
226
|
);
|
|
145
227
|
}
|
|
@@ -154,10 +236,23 @@ export class UpQueue {
|
|
|
154
236
|
await this.removeEventFromDatabase(event.mutation_id);
|
|
155
237
|
} catch (error) {
|
|
156
238
|
this.logger.error(
|
|
157
|
-
{ error, event, Category: '
|
|
239
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
158
240
|
'Failed to remove mutation from database after successful processing'
|
|
159
241
|
);
|
|
160
242
|
}
|
|
243
|
+
// Report AFTER the outbox row is gone: that delete is exactly what drops
|
|
244
|
+
// the row out of the render set, so this is the moment the grace window
|
|
245
|
+
// has to start covering.
|
|
246
|
+
if (onSettled) {
|
|
247
|
+
try {
|
|
248
|
+
onSettled(event);
|
|
249
|
+
} catch (error) {
|
|
250
|
+
this.logger.error(
|
|
251
|
+
{ error, event, Category: 'sp00ky-client::UpQueue::next' },
|
|
252
|
+
'Settled-write handler failed'
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
161
256
|
this._events.addEvent({
|
|
162
257
|
type: SyncQueueEventTypes.MutationDequeued,
|
|
163
258
|
payload: { queueSize: this.queue.length },
|
|
@@ -169,55 +264,144 @@ export class UpQueue {
|
|
|
169
264
|
return this.local.query(`DELETE $mutation_id`, { mutation_id });
|
|
170
265
|
}
|
|
171
266
|
|
|
267
|
+
/**
|
|
268
|
+
* Shared-tabs: a follower committed an outbox row into the shared store and
|
|
269
|
+
* notified the leader; enqueue exactly that row. Idempotent: a replayed
|
|
270
|
+
* notify (failover) or a row already queued is a no-op, and a row that never
|
|
271
|
+
* committed simply is not found (the follower's own write promise already
|
|
272
|
+
* rejected in that case).
|
|
273
|
+
*/
|
|
274
|
+
async enqueueFromDatabase(mutationId: string): Promise<void> {
|
|
275
|
+
if (this.queue.some((e) => encodeUpEventId(e) === mutationId)) return;
|
|
276
|
+
try {
|
|
277
|
+
// ARRAY param, matching SyncEngine's `SELECT * FROM $idsToFetch`. A bare
|
|
278
|
+
// `FROM $singleRecordId` looks fine against SurrealDB but the SQLite
|
|
279
|
+
// engine lowers any `FROM $param` to `selectByIds` and calls `.map` on
|
|
280
|
+
// the param (surql-translate.ts, SqliteCacheEngine.selectByIds), so a
|
|
281
|
+
// single RecordId threw. The throw landed in the catch below, which logs
|
|
282
|
+
// at `error` — invisible to an app running `logLevel: 'fatal'` — so every
|
|
283
|
+
// forwarded mutation was silently dropped: the follower's optimistic
|
|
284
|
+
// write stuck locally, was never pushed, and the next down-sync reverted it.
|
|
285
|
+
const [records] = await this.local.query<any>(`SELECT * FROM $mutation_ids`, {
|
|
286
|
+
mutation_ids: [parseRecordIdString(mutationId)],
|
|
287
|
+
});
|
|
288
|
+
const row = Array.isArray(records) ? records[0] : undefined;
|
|
289
|
+
if (!row) return;
|
|
290
|
+
const event = rowToUpEvent(row, this.logger);
|
|
291
|
+
if (event) {
|
|
292
|
+
this.addToQueue(event);
|
|
293
|
+
return;
|
|
294
|
+
}
|
|
295
|
+
await this.discardUnreplayable(row, 'forwarded mutation is not replayable');
|
|
296
|
+
} catch (error) {
|
|
297
|
+
this.logger.error(
|
|
298
|
+
{ error, mutationId, Category: 'sp00ky-client::UpQueue::enqueueFromDatabase' },
|
|
299
|
+
'Failed to load a forwarded mutation'
|
|
300
|
+
);
|
|
301
|
+
this.onDropped?.({
|
|
302
|
+
mutationId,
|
|
303
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
304
|
+
});
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
172
308
|
async loadFromDatabase() {
|
|
173
309
|
try {
|
|
310
|
+
// ORDER BY id: mutation ids are zero-padded-timestamp-prefixed (see
|
|
311
|
+
// modules/data/mutation-id.ts), so lexicographic id order IS chronological
|
|
312
|
+
// order. (The previous created_at was never a defined field on this
|
|
313
|
+
// table, so the old ordering was undefined.)
|
|
174
314
|
const [records] = await this.local.query<any>(
|
|
175
|
-
`SELECT * FROM
|
|
315
|
+
`SELECT * FROM _00_pending_mutations ORDER BY id ASC`
|
|
176
316
|
);
|
|
177
317
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
mutation_id: parseRecordIdString(r.id),
|
|
193
|
-
record_id: parseRecordIdString(r.recordId),
|
|
194
|
-
data: r.data,
|
|
195
|
-
beforeRecord: r.beforeRecord,
|
|
196
|
-
};
|
|
197
|
-
case 'delete':
|
|
198
|
-
return {
|
|
199
|
-
type: 'delete',
|
|
200
|
-
mutation_id: parseRecordIdString(r.id),
|
|
201
|
-
record_id: parseRecordIdString(r.recordId),
|
|
202
|
-
};
|
|
203
|
-
default:
|
|
204
|
-
this.logger.warn(
|
|
205
|
-
{
|
|
206
|
-
mutationType: r.mutationType,
|
|
207
|
-
record: r,
|
|
208
|
-
Category: 'spooky-client::UpQueue::loadFromDatabase',
|
|
209
|
-
},
|
|
210
|
-
'Unknown mutation type'
|
|
211
|
-
);
|
|
212
|
-
return null;
|
|
213
|
-
}
|
|
214
|
-
})
|
|
215
|
-
.filter((e: UpEvent | null): e is UpEvent => e !== null);
|
|
318
|
+
const loaded: UpEvent[] = [];
|
|
319
|
+
const unreplayable: any[] = [];
|
|
320
|
+
for (const row of records as any[]) {
|
|
321
|
+
const event = rowToUpEvent(row, this.logger);
|
|
322
|
+
if (event) loaded.push(event);
|
|
323
|
+
else unreplayable.push(row);
|
|
324
|
+
}
|
|
325
|
+
this.queue = loaded;
|
|
326
|
+
// Drop them AFTER the queue is populated: one unsendable row must not
|
|
327
|
+
// stop the rest of the backlog from draining, and leaving it in the store
|
|
328
|
+
// would just re-poison the next boot.
|
|
329
|
+
for (const row of unreplayable) {
|
|
330
|
+
await this.discardUnreplayable(row, 'pending mutation is not replayable');
|
|
331
|
+
}
|
|
216
332
|
} catch (error) {
|
|
217
333
|
this.logger.error(
|
|
218
|
-
{ error, Category: '
|
|
334
|
+
{ error, Category: 'sp00ky-client::UpQueue::loadFromDatabase' },
|
|
219
335
|
'Failed to load pending mutations from database'
|
|
220
336
|
);
|
|
221
337
|
}
|
|
222
338
|
}
|
|
223
339
|
}
|
|
340
|
+
|
|
341
|
+
/** The stable string id of an event's outbox row. */
|
|
342
|
+
function encodeUpEventId(event: UpEvent): string {
|
|
343
|
+
return encodeRecordId(event.mutation_id);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
/**
|
|
347
|
+
* Parse a record id READ BACK FROM THE STORE, stripping SurrealDB's `⟨⟩`
|
|
348
|
+
* escaping.
|
|
349
|
+
*
|
|
350
|
+
* Outbox ids contain `_` and `-`, so the store hands them back in escaped
|
|
351
|
+
* display form (`_00_pending_mutations:⟨1785…_0005_c960…⟩`). `parseRecordIdString`
|
|
352
|
+
* keeps that verbatim, and re-encoding then escapes the brackets AGAIN, so a
|
|
353
|
+
* `DELETE $mutation_id` built from a stored id targets
|
|
354
|
+
* `_00_pending_mutations:⟨⟨1785…\⟩⟩` and matches nothing. Every mutation
|
|
355
|
+
* replayed from the store was therefore un-deletable: its row survived a
|
|
356
|
+
* SUCCESSFUL push and got re-sent on the next boot.
|
|
357
|
+
*/
|
|
358
|
+
function parseStoredRecordId(id: string): RecordId<string> {
|
|
359
|
+
const [table, ...rest] = id.split(':');
|
|
360
|
+
let raw = rest.join(':');
|
|
361
|
+
if (raw.startsWith('⟨') && raw.endsWith('⟩')) raw = raw.slice(1, -1);
|
|
362
|
+
return new RecordId(table, raw);
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/** Materialize one `_00_pending_mutations` row into an UpEvent. */
|
|
366
|
+
function rowToUpEvent(r: any, logger: Logger): UpEvent | null {
|
|
367
|
+
switch (r.mutationType) {
|
|
368
|
+
case 'create':
|
|
369
|
+
// `processUpEvent` does `Object.keys(event.data)`, so a create with no
|
|
370
|
+
// payload throws before it reaches the network and can never succeed.
|
|
371
|
+
// Rows written before the create branch of `surql.createMutation`
|
|
372
|
+
// persisted `data` are exactly that, so refuse them here instead of
|
|
373
|
+
// queueing a guaranteed failure.
|
|
374
|
+
if (r.data === undefined || r.data === null) return null;
|
|
375
|
+
return {
|
|
376
|
+
type: 'create',
|
|
377
|
+
mutation_id: parseStoredRecordId(r.id),
|
|
378
|
+
record_id: parseRecordIdString(r.recordId),
|
|
379
|
+
data: r.data,
|
|
380
|
+
tableName: extractTablePart(r.recordId),
|
|
381
|
+
};
|
|
382
|
+
case 'update':
|
|
383
|
+
return {
|
|
384
|
+
type: 'update',
|
|
385
|
+
mutation_id: parseStoredRecordId(r.id),
|
|
386
|
+
record_id: parseRecordIdString(r.recordId),
|
|
387
|
+
data: r.data,
|
|
388
|
+
beforeRecord: r.beforeRecord,
|
|
389
|
+
};
|
|
390
|
+
case 'delete':
|
|
391
|
+
return {
|
|
392
|
+
type: 'delete',
|
|
393
|
+
mutation_id: parseStoredRecordId(r.id),
|
|
394
|
+
record_id: parseRecordIdString(r.recordId),
|
|
395
|
+
};
|
|
396
|
+
default:
|
|
397
|
+
logger.warn(
|
|
398
|
+
{
|
|
399
|
+
mutationType: r.mutationType,
|
|
400
|
+
record: r,
|
|
401
|
+
Category: 'sp00ky-client::UpQueue::rowToUpEvent',
|
|
402
|
+
},
|
|
403
|
+
'Unknown mutation type'
|
|
404
|
+
);
|
|
405
|
+
return null;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { SyncScheduler } from './scheduler';
|
|
3
|
+
import type { UpQueue, DownQueue, UpEvent } from './queue/index';
|
|
4
|
+
|
|
5
|
+
// `SyncScheduler.pause()` is the drain step of a local-bucket switch: it must
|
|
6
|
+
// (a) let an IN-FLIGHT queue item finish — including its outbox-row delete,
|
|
7
|
+
// which has to land in the old bucket — and only then resolve, and (b) refuse
|
|
8
|
+
// to start new rounds until `resume()`. The pause point is between items.
|
|
9
|
+
|
|
10
|
+
const silentLogger = {
|
|
11
|
+
child: () => silentLogger,
|
|
12
|
+
debug: () => {},
|
|
13
|
+
info: () => {},
|
|
14
|
+
warn: () => {},
|
|
15
|
+
error: () => {},
|
|
16
|
+
} as any;
|
|
17
|
+
|
|
18
|
+
function makeQueues(items: UpEvent[]) {
|
|
19
|
+
const upQueue = {
|
|
20
|
+
queue: [...items],
|
|
21
|
+
get size() {
|
|
22
|
+
return this.queue.length;
|
|
23
|
+
},
|
|
24
|
+
events: { subscribe: () => {} },
|
|
25
|
+
loadFromDatabase: async () => {},
|
|
26
|
+
async next(fn: (event: UpEvent) => Promise<void>) {
|
|
27
|
+
const event = this.queue.shift();
|
|
28
|
+
if (event) await fn(event);
|
|
29
|
+
},
|
|
30
|
+
} as unknown as UpQueue & { queue: UpEvent[] };
|
|
31
|
+
|
|
32
|
+
const downQueue = {
|
|
33
|
+
queue: [] as unknown[],
|
|
34
|
+
get size() {
|
|
35
|
+
return this.queue.length;
|
|
36
|
+
},
|
|
37
|
+
events: { subscribe: () => {} },
|
|
38
|
+
async next() {},
|
|
39
|
+
clear() {
|
|
40
|
+
this.queue = [];
|
|
41
|
+
},
|
|
42
|
+
} as unknown as DownQueue;
|
|
43
|
+
|
|
44
|
+
return { upQueue, downQueue };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const upEvent = (n: number) => ({ type: 'delete', mutation_id: n, record_id: n }) as unknown as UpEvent;
|
|
48
|
+
|
|
49
|
+
describe('SyncScheduler.pause', () => {
|
|
50
|
+
it('waits for the in-flight item (push + outbox delete) before resolving', async () => {
|
|
51
|
+
const { upQueue, downQueue } = makeQueues([upEvent(1), upEvent(2)]);
|
|
52
|
+
let release!: () => void;
|
|
53
|
+
const gate = new Promise<void>((r) => (release = r));
|
|
54
|
+
const processed: unknown[] = [];
|
|
55
|
+
const scheduler = new SyncScheduler(
|
|
56
|
+
upQueue,
|
|
57
|
+
downQueue,
|
|
58
|
+
async (event) => {
|
|
59
|
+
await gate; // simulate a slow remote push
|
|
60
|
+
processed.push(event);
|
|
61
|
+
},
|
|
62
|
+
async () => {},
|
|
63
|
+
silentLogger
|
|
64
|
+
);
|
|
65
|
+
|
|
66
|
+
const round = scheduler.syncUp();
|
|
67
|
+
const pauseDone = vi.fn();
|
|
68
|
+
const pause = scheduler.pause().then(pauseDone);
|
|
69
|
+
|
|
70
|
+
// The in-flight item hasn't finished — pause must not have resolved.
|
|
71
|
+
await Promise.resolve();
|
|
72
|
+
expect(pauseDone).not.toHaveBeenCalled();
|
|
73
|
+
|
|
74
|
+
release();
|
|
75
|
+
await pause;
|
|
76
|
+
await round;
|
|
77
|
+
// Exactly the in-flight item completed; the second stayed queued.
|
|
78
|
+
expect(processed).toHaveLength(1);
|
|
79
|
+
expect(upQueue.size).toBe(1);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('refuses new rounds while paused and drains them on resume', async () => {
|
|
83
|
+
const { upQueue, downQueue } = makeQueues([upEvent(1)]);
|
|
84
|
+
const processed: unknown[] = [];
|
|
85
|
+
const scheduler = new SyncScheduler(
|
|
86
|
+
upQueue,
|
|
87
|
+
downQueue,
|
|
88
|
+
async (event) => {
|
|
89
|
+
processed.push(event);
|
|
90
|
+
},
|
|
91
|
+
async () => {},
|
|
92
|
+
silentLogger
|
|
93
|
+
);
|
|
94
|
+
|
|
95
|
+
await scheduler.pause();
|
|
96
|
+
await scheduler.syncUp();
|
|
97
|
+
expect(processed).toHaveLength(0);
|
|
98
|
+
|
|
99
|
+
scheduler.resume();
|
|
100
|
+
await vi.waitFor(() => expect(processed).toHaveLength(1));
|
|
101
|
+
expect(upQueue.size).toBe(0);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it('resolves immediately when nothing is in flight', async () => {
|
|
105
|
+
const { upQueue, downQueue } = makeQueues([]);
|
|
106
|
+
const scheduler = new SyncScheduler(upQueue, downQueue, async () => {}, async () => {}, silentLogger);
|
|
107
|
+
await expect(scheduler.pause()).resolves.toBeUndefined();
|
|
108
|
+
});
|
|
109
|
+
});
|