@firtoz/db-helpers 2.0.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -5
- package/src/deferred-write-queue.ts +202 -0
- package/src/generic-sync.ts +390 -91
- package/src/index.ts +6 -0
- package/src/memoryCollection.ts +43 -8
- package/src/sync-types.ts +16 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@firtoz/db-helpers",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.1.1",
|
|
4
4
|
"description": "TanStack DB helpers and utilities",
|
|
5
5
|
"main": "./src/index.ts",
|
|
6
6
|
"module": "./src/index.ts",
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"README.md"
|
|
19
19
|
],
|
|
20
20
|
"scripts": {
|
|
21
|
-
"typecheck": "
|
|
21
|
+
"typecheck": "tsgo --noEmit -p ./tsconfig.json",
|
|
22
22
|
"test": "bun test",
|
|
23
23
|
"lint": "biome check --write src",
|
|
24
24
|
"lint:ci": "biome ci src",
|
|
@@ -49,12 +49,12 @@
|
|
|
49
49
|
},
|
|
50
50
|
"peerDependencies": {
|
|
51
51
|
"@standard-schema/spec": ">=1.1.0",
|
|
52
|
-
"@tanstack/db": ">=0.
|
|
52
|
+
"@tanstack/db": ">=0.6.3"
|
|
53
53
|
},
|
|
54
54
|
"devDependencies": {
|
|
55
55
|
"@standard-schema/spec": "^1.1.0",
|
|
56
|
-
"@tanstack/db": "^0.
|
|
57
|
-
"bun-types": "^1.3.
|
|
56
|
+
"@tanstack/db": "^0.6.4",
|
|
57
|
+
"bun-types": "^1.3.11",
|
|
58
58
|
"zod": "^4.3.6"
|
|
59
59
|
},
|
|
60
60
|
"dependencies": {
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
import type { GenericSyncBackend } from "./generic-sync";
|
|
2
|
+
|
|
3
|
+
export type DeferredUpdateMutation<TItem extends object> = {
|
|
4
|
+
key: string;
|
|
5
|
+
changes: Partial<TItem>;
|
|
6
|
+
original: TItem;
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
export type DeferredDeleteMutation<TItem extends object> = {
|
|
10
|
+
key: string;
|
|
11
|
+
modified: TItem;
|
|
12
|
+
original: TItem;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
type PendingRow<TItem extends object> =
|
|
16
|
+
| { kind: "row"; value: TItem; insertedOnly: boolean }
|
|
17
|
+
| { kind: "delete" };
|
|
18
|
+
|
|
19
|
+
function mergeUpdate<TItem extends object>(
|
|
20
|
+
m: DeferredUpdateMutation<TItem>,
|
|
21
|
+
): TItem {
|
|
22
|
+
return { ...m.original, ...m.changes } as TItem;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Write-behind queue for local mutations: coalesces by persist key and flushes to a
|
|
27
|
+
* {@link GenericSyncBackend} on an interval or when {@link flush} is called explicitly.
|
|
28
|
+
*/
|
|
29
|
+
export class DeferredWriteQueue<TItem extends object> {
|
|
30
|
+
readonly #backend: GenericSyncBackend<TItem>;
|
|
31
|
+
readonly #getPersistKey: (item: TItem) => string;
|
|
32
|
+
readonly #flushIntervalMs: number;
|
|
33
|
+
#pending = new Map<string, PendingRow<TItem>>();
|
|
34
|
+
#intervalId: ReturnType<typeof setInterval> | null = null;
|
|
35
|
+
#flushTail: Promise<void> = Promise.resolve();
|
|
36
|
+
#disposed = false;
|
|
37
|
+
|
|
38
|
+
constructor(options: {
|
|
39
|
+
backend: GenericSyncBackend<TItem>;
|
|
40
|
+
getPersistKey: (item: TItem) => string;
|
|
41
|
+
flushIntervalMs?: number;
|
|
42
|
+
}) {
|
|
43
|
+
this.#backend = options.backend;
|
|
44
|
+
this.#getPersistKey = options.getPersistKey;
|
|
45
|
+
this.#flushIntervalMs = options.flushIntervalMs ?? 100;
|
|
46
|
+
|
|
47
|
+
if (typeof globalThis !== "undefined") {
|
|
48
|
+
globalThis.addEventListener?.("beforeunload", this.#onBeforeUnload);
|
|
49
|
+
globalThis.addEventListener?.(
|
|
50
|
+
"visibilitychange",
|
|
51
|
+
this.#onVisibilityChange,
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
this.#intervalId = setInterval(() => {
|
|
56
|
+
void this.flush();
|
|
57
|
+
}, this.#flushIntervalMs);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
#onBeforeUnload = (): void => {
|
|
61
|
+
void this.flush();
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
#onVisibilityChange = (): void => {
|
|
65
|
+
const doc = (
|
|
66
|
+
globalThis as typeof globalThis & {
|
|
67
|
+
document?: { visibilityState?: string };
|
|
68
|
+
}
|
|
69
|
+
).document;
|
|
70
|
+
if (doc?.visibilityState === "hidden") {
|
|
71
|
+
void this.flush();
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
|
|
75
|
+
enqueueInsert(items: TItem[]): void {
|
|
76
|
+
if (this.#disposed || items.length === 0) return;
|
|
77
|
+
for (const item of items) {
|
|
78
|
+
const key = this.#getPersistKey(item);
|
|
79
|
+
const cur = this.#pending.get(key);
|
|
80
|
+
if (cur?.kind === "delete") {
|
|
81
|
+
this.#pending.set(key, {
|
|
82
|
+
kind: "row",
|
|
83
|
+
value: item,
|
|
84
|
+
insertedOnly: true,
|
|
85
|
+
});
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (cur?.kind === "row" && !cur.insertedOnly) {
|
|
89
|
+
this.#pending.set(key, {
|
|
90
|
+
kind: "row",
|
|
91
|
+
value: item,
|
|
92
|
+
insertedOnly: false,
|
|
93
|
+
});
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
this.#pending.set(key, { kind: "row", value: item, insertedOnly: true });
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
enqueueUpdate(mutations: DeferredUpdateMutation<TItem>[]): void {
|
|
101
|
+
if (this.#disposed || mutations.length === 0) return;
|
|
102
|
+
for (const m of mutations) {
|
|
103
|
+
const key = m.key;
|
|
104
|
+
const value = mergeUpdate(m);
|
|
105
|
+
const cur = this.#pending.get(key);
|
|
106
|
+
if (cur?.kind === "delete") {
|
|
107
|
+
this.#pending.set(key, { kind: "row", value, insertedOnly: false });
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (cur?.kind === "row") {
|
|
111
|
+
this.#pending.set(key, {
|
|
112
|
+
kind: "row",
|
|
113
|
+
value,
|
|
114
|
+
insertedOnly: cur.insertedOnly,
|
|
115
|
+
});
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
this.#pending.set(key, { kind: "row", value, insertedOnly: false });
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
enqueueDelete(mutations: DeferredDeleteMutation<TItem>[]): void {
|
|
123
|
+
if (this.#disposed || mutations.length === 0) return;
|
|
124
|
+
for (const m of mutations) {
|
|
125
|
+
this.#pending.set(m.key, { kind: "delete" });
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Drains pending ops into the backend. Serialized so concurrent flushes chain.
|
|
131
|
+
*/
|
|
132
|
+
flush(): Promise<void> {
|
|
133
|
+
this.#flushTail = this.#flushTail
|
|
134
|
+
.catch(() => {})
|
|
135
|
+
.then(() => this.#flushImpl());
|
|
136
|
+
return this.#flushTail;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async #flushImpl(): Promise<void> {
|
|
140
|
+
if (this.#pending.size === 0) return;
|
|
141
|
+
const entries = [...this.#pending.entries()];
|
|
142
|
+
this.#pending.clear();
|
|
143
|
+
|
|
144
|
+
const deletePayload: DeferredDeleteMutation<TItem>[] = [];
|
|
145
|
+
const toInsert: TItem[] = [];
|
|
146
|
+
const toUpsert: TItem[] = [];
|
|
147
|
+
|
|
148
|
+
for (const [key, op] of entries) {
|
|
149
|
+
if (op.kind === "delete") {
|
|
150
|
+
const id =
|
|
151
|
+
Number.isFinite(Number(key)) && String(Number(key)) === key
|
|
152
|
+
? Number(key)
|
|
153
|
+
: key;
|
|
154
|
+
const stub = { id } as TItem;
|
|
155
|
+
deletePayload.push({
|
|
156
|
+
key,
|
|
157
|
+
modified: stub,
|
|
158
|
+
original: stub,
|
|
159
|
+
});
|
|
160
|
+
} else if (op.insertedOnly) {
|
|
161
|
+
toInsert.push(op.value);
|
|
162
|
+
} else {
|
|
163
|
+
toUpsert.push(op.value);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if (deletePayload.length > 0) {
|
|
168
|
+
await this.#backend.handleDelete(deletePayload);
|
|
169
|
+
}
|
|
170
|
+
if (toInsert.length > 0) {
|
|
171
|
+
await this.#backend.handleInsert(toInsert);
|
|
172
|
+
}
|
|
173
|
+
if (toUpsert.length > 0) {
|
|
174
|
+
if (this.#backend.handleBatchPut !== undefined) {
|
|
175
|
+
await this.#backend.handleBatchPut(toUpsert);
|
|
176
|
+
} else {
|
|
177
|
+
await this.#backend.handleUpdate(
|
|
178
|
+
toUpsert.map((value) => ({
|
|
179
|
+
key: this.#getPersistKey(value),
|
|
180
|
+
changes: value as Partial<TItem>,
|
|
181
|
+
original: value,
|
|
182
|
+
})),
|
|
183
|
+
);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
dispose(): void {
|
|
189
|
+
if (this.#disposed) return;
|
|
190
|
+
this.#disposed = true;
|
|
191
|
+
if (this.#intervalId !== null) {
|
|
192
|
+
clearInterval(this.#intervalId);
|
|
193
|
+
this.#intervalId = null;
|
|
194
|
+
}
|
|
195
|
+
globalThis.removeEventListener?.("beforeunload", this.#onBeforeUnload);
|
|
196
|
+
globalThis.removeEventListener?.(
|
|
197
|
+
"visibilitychange",
|
|
198
|
+
this.#onVisibilityChange,
|
|
199
|
+
);
|
|
200
|
+
void this.flush();
|
|
201
|
+
}
|
|
202
|
+
}
|
package/src/generic-sync.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type {
|
|
2
|
+
CollectionUtils,
|
|
3
|
+
ReceiveSyncDurableOp,
|
|
4
|
+
SyncMessage,
|
|
5
|
+
} from "./sync-types";
|
|
6
|
+
import { DeferredWriteQueue } from "./deferred-write-queue";
|
|
2
7
|
import { exhaustiveGuard } from "@firtoz/maybe-error";
|
|
3
8
|
import type { StandardSchemaV1 } from "@standard-schema/spec";
|
|
4
9
|
import type {
|
|
@@ -9,21 +14,31 @@ import type {
|
|
|
9
14
|
SyncMode,
|
|
10
15
|
LoadSubsetOptions,
|
|
11
16
|
} from "@tanstack/db";
|
|
12
|
-
import { DeduplicatedLoadSubset } from "@tanstack/db";
|
|
17
|
+
import { BasicIndex, DeduplicatedLoadSubset } from "@tanstack/db";
|
|
13
18
|
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
|
|
18
|
-
export const USE_DEDUPE = false as boolean;
|
|
19
|
+
// DeduplicatedLoadSubset dedupes overlapping loadSubset calls. Previously disabled for TanStack/db#828
|
|
20
|
+
// (invalid OR expressions when toggling isNull/isNotNull). Re-enabled with @tanstack/db 0.6.4; if
|
|
21
|
+
// regressions appear, set back to false and add a regression test.
|
|
22
|
+
export const USE_DEDUPE = true as boolean;
|
|
19
23
|
|
|
20
24
|
/**
|
|
21
25
|
* Base configuration for sync lifecycle management (generic, no Drizzle dependency).
|
|
22
26
|
*/
|
|
23
|
-
export interface GenericBaseSyncConfig {
|
|
27
|
+
export interface GenericBaseSyncConfig<TItem extends object = object> {
|
|
24
28
|
readyPromise: Promise<void>;
|
|
25
29
|
syncMode?: SyncMode;
|
|
26
30
|
debug?: boolean;
|
|
31
|
+
/**
|
|
32
|
+
* Row key for durable storage when applying {@link CollectionUtils.receiveSync} updates.
|
|
33
|
+
* If omitted, `id` on the item (string or number) is used.
|
|
34
|
+
*/
|
|
35
|
+
getSyncPersistKey?: (item: TItem) => string;
|
|
36
|
+
/**
|
|
37
|
+
* When set, local `onInsert` / `onUpdate` / `onDelete` confirm TanStack sync state immediately
|
|
38
|
+
* and enqueue durable backend writes (coalesced, flushed on an interval). `receiveSync`,
|
|
39
|
+
* `loadSubset`, and `truncate` flush the queue first so reads stay consistent.
|
|
40
|
+
*/
|
|
41
|
+
deferLocalPersistence?: boolean | { flushIntervalMs?: number };
|
|
27
42
|
}
|
|
28
43
|
|
|
29
44
|
/**
|
|
@@ -48,6 +63,18 @@ export interface GenericSyncBackend<TItem extends object> {
|
|
|
48
63
|
}>,
|
|
49
64
|
) => Promise<void>;
|
|
50
65
|
handleTruncate?: () => Promise<void>;
|
|
66
|
+
/**
|
|
67
|
+
* When set, {@link CollectionUtils.receiveSync} persists an entire message batch with one call
|
|
68
|
+
* (e.g. one SQLite transaction) instead of awaiting {@link handleInsert}/handleUpdate per
|
|
69
|
+
* message. TanStack `syncWrite`/`syncTruncate` still run once per message in order.
|
|
70
|
+
*/
|
|
71
|
+
applyReceiveSyncDurableWrites?: (
|
|
72
|
+
ops: ReceiveSyncDurableOp<TItem>[],
|
|
73
|
+
) => Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* Optional batch upsert for deferred local persistence flushes (e.g. IndexedDB `put` in one tx).
|
|
76
|
+
*/
|
|
77
|
+
handleBatchPut?: (items: Array<TItem>) => Promise<void>;
|
|
51
78
|
}
|
|
52
79
|
|
|
53
80
|
/**
|
|
@@ -81,7 +108,7 @@ export type GenericSyncFunctionResult<TItem extends object> = {
|
|
|
81
108
|
* Generic version -- no Drizzle dependency.
|
|
82
109
|
*/
|
|
83
110
|
export function createGenericSyncFunction<TItem extends object>(
|
|
84
|
-
config: GenericBaseSyncConfig
|
|
111
|
+
config: GenericBaseSyncConfig<TItem>,
|
|
85
112
|
backend: GenericSyncBackend<TItem>,
|
|
86
113
|
): GenericSyncFunctionResult<TItem> {
|
|
87
114
|
type CollectionType = CollectionConfig<
|
|
@@ -101,6 +128,57 @@ export function createGenericSyncFunction<TItem extends object>(
|
|
|
101
128
|
| null = null;
|
|
102
129
|
let syncCommit: (() => void) | null = null;
|
|
103
130
|
let syncTruncate: (() => void) | null = null;
|
|
131
|
+
/** Resolves when eager `initialSync` has finished (or immediately in on-demand mode). Used so `receiveSync` cannot interleave with initial inserts. */
|
|
132
|
+
let initialSyncDone: Promise<void> | null = null;
|
|
133
|
+
/**
|
|
134
|
+
* TanStack DB allows only one pending sync transaction per collection. Every path that calls
|
|
135
|
+
* `begin`/`commit` — `initialSync`, `loadSubset`, `onInsert`/`onUpdate`/`onDelete`, `receiveSync`,
|
|
136
|
+
* and `truncate` — must run through this queue so async backends (e.g. SQLite WASM) cannot
|
|
137
|
+
* leave a transaction open across an `await` while another path starts a second transaction.
|
|
138
|
+
*/
|
|
139
|
+
let syncLayerSerial: Promise<void> = Promise.resolve();
|
|
140
|
+
|
|
141
|
+
const enqueueSyncLayer = (run: () => void | Promise<void>): Promise<void> => {
|
|
142
|
+
const next = syncLayerSerial.catch(() => {}).then(run);
|
|
143
|
+
syncLayerSerial = next;
|
|
144
|
+
return next;
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
function resolveDeferLocalPersistence(
|
|
148
|
+
opts: GenericBaseSyncConfig<TItem>["deferLocalPersistence"],
|
|
149
|
+
): { enabled: boolean; flushIntervalMs: number } {
|
|
150
|
+
if (opts === true) return { enabled: true, flushIntervalMs: 100 };
|
|
151
|
+
if (typeof opts === "object" && opts !== null) {
|
|
152
|
+
return { enabled: true, flushIntervalMs: opts.flushIntervalMs ?? 100 };
|
|
153
|
+
}
|
|
154
|
+
return { enabled: false, flushIntervalMs: 100 };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const deferOpts = resolveDeferLocalPersistence(config.deferLocalPersistence);
|
|
158
|
+
|
|
159
|
+
const resolveDeferredPersistKey = (item: TItem): string => {
|
|
160
|
+
if (config.getSyncPersistKey !== undefined) {
|
|
161
|
+
return config.getSyncPersistKey(item);
|
|
162
|
+
}
|
|
163
|
+
if (item !== null && typeof item === "object" && "id" in item) {
|
|
164
|
+
const id = (item as { id: unknown }).id;
|
|
165
|
+
if (typeof id === "string" || typeof id === "number") {
|
|
166
|
+
return String(id);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
throw new Error(
|
|
170
|
+
"[deferLocalPersistence] Persist key missing: set GenericBaseSyncConfig.getSyncPersistKey or use items with string/number `id`",
|
|
171
|
+
);
|
|
172
|
+
};
|
|
173
|
+
|
|
174
|
+
let deferQueue: DeferredWriteQueue<TItem> | null = null;
|
|
175
|
+
if (deferOpts.enabled) {
|
|
176
|
+
deferQueue = new DeferredWriteQueue({
|
|
177
|
+
backend,
|
|
178
|
+
getPersistKey: resolveDeferredPersistKey,
|
|
179
|
+
flushIntervalMs: deferOpts.flushIntervalMs,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
104
182
|
|
|
105
183
|
const syncFn: SyncConfig<TItem, string>["sync"] = (params) => {
|
|
106
184
|
const { begin, write, commit, markReady, truncate } = params;
|
|
@@ -111,88 +189,156 @@ export function createGenericSyncFunction<TItem extends object>(
|
|
|
111
189
|
syncTruncate = truncate;
|
|
112
190
|
|
|
113
191
|
const initialSync = async () => {
|
|
114
|
-
await
|
|
192
|
+
await enqueueSyncLayer(async () => {
|
|
193
|
+
await config.readyPromise;
|
|
115
194
|
|
|
116
|
-
|
|
117
|
-
|
|
195
|
+
try {
|
|
196
|
+
const items = await backend.initialLoad();
|
|
118
197
|
|
|
119
|
-
|
|
198
|
+
begin();
|
|
120
199
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
200
|
+
for (const item of items) {
|
|
201
|
+
write({
|
|
202
|
+
type: "insert",
|
|
203
|
+
value: item,
|
|
204
|
+
});
|
|
205
|
+
}
|
|
127
206
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
207
|
+
commit();
|
|
208
|
+
} finally {
|
|
209
|
+
markReady();
|
|
210
|
+
}
|
|
211
|
+
});
|
|
132
212
|
};
|
|
133
213
|
|
|
134
214
|
if (config.syncMode === "eager" || !config.syncMode) {
|
|
135
|
-
initialSync();
|
|
215
|
+
initialSyncDone = initialSync();
|
|
136
216
|
} else {
|
|
137
217
|
markReady();
|
|
218
|
+
initialSyncDone = Promise.resolve();
|
|
138
219
|
}
|
|
139
220
|
|
|
140
221
|
insertListener = async (params) => {
|
|
141
|
-
|
|
142
|
-
params.transaction.mutations.map((m) => m.modified)
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
222
|
+
await enqueueSyncLayer(async () => {
|
|
223
|
+
const items = params.transaction.mutations.map((m) => m.modified);
|
|
224
|
+
if (deferQueue !== null) {
|
|
225
|
+
begin();
|
|
226
|
+
for (const item of items) {
|
|
227
|
+
write({
|
|
228
|
+
type: "insert",
|
|
229
|
+
value: item,
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
commit();
|
|
233
|
+
deferQueue.enqueueInsert(items);
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
const results = await backend.handleInsert(items);
|
|
238
|
+
|
|
239
|
+
begin();
|
|
240
|
+
for (const result of results) {
|
|
241
|
+
write({
|
|
242
|
+
type: "insert",
|
|
243
|
+
value: result,
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
commit();
|
|
247
|
+
});
|
|
153
248
|
};
|
|
154
249
|
|
|
155
250
|
updateListener = async (params) => {
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
251
|
+
await enqueueSyncLayer(async () => {
|
|
252
|
+
if (deferQueue !== null) {
|
|
253
|
+
const mutations = params.transaction.mutations.map((m) => ({
|
|
254
|
+
key: String(m.key),
|
|
255
|
+
changes: m.changes as Partial<TItem>,
|
|
256
|
+
original: m.original as TItem,
|
|
257
|
+
}));
|
|
258
|
+
const results = mutations.map(
|
|
259
|
+
(m) => ({ ...m.original, ...m.changes }) as TItem,
|
|
260
|
+
);
|
|
261
|
+
begin();
|
|
262
|
+
for (const result of results) {
|
|
263
|
+
write({
|
|
264
|
+
type: "update",
|
|
265
|
+
value: result,
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
commit();
|
|
269
|
+
deferQueue.enqueueUpdate(mutations);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const results = await backend.handleUpdate(
|
|
274
|
+
params.transaction.mutations,
|
|
275
|
+
);
|
|
276
|
+
|
|
277
|
+
begin();
|
|
278
|
+
for (const result of results) {
|
|
279
|
+
write({
|
|
280
|
+
type: "update",
|
|
281
|
+
value: result,
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
commit();
|
|
285
|
+
});
|
|
166
286
|
};
|
|
167
287
|
|
|
168
288
|
deleteListener = async (params) => {
|
|
169
|
-
await
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
289
|
+
await enqueueSyncLayer(async () => {
|
|
290
|
+
if (deferQueue !== null) {
|
|
291
|
+
const mutations = params.transaction.mutations.map((m) => ({
|
|
292
|
+
key: String(m.key),
|
|
293
|
+
modified: m.modified as TItem,
|
|
294
|
+
original: m.original as TItem,
|
|
295
|
+
}));
|
|
296
|
+
begin();
|
|
297
|
+
for (const item of mutations) {
|
|
298
|
+
write({
|
|
299
|
+
type: "delete",
|
|
300
|
+
value: item.modified,
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
commit();
|
|
304
|
+
deferQueue.enqueueDelete(mutations);
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
await backend.handleDelete(params.transaction.mutations);
|
|
309
|
+
|
|
310
|
+
begin();
|
|
311
|
+
for (const item of params.transaction.mutations) {
|
|
312
|
+
write({
|
|
313
|
+
type: "delete",
|
|
314
|
+
value: item.modified,
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
commit();
|
|
318
|
+
});
|
|
179
319
|
};
|
|
180
320
|
|
|
181
321
|
const loadSubset = async (options: LoadSubsetOptions) => {
|
|
182
|
-
await
|
|
322
|
+
await enqueueSyncLayer(async () => {
|
|
323
|
+
await config.readyPromise;
|
|
183
324
|
|
|
184
|
-
|
|
325
|
+
if (deferQueue !== null) {
|
|
326
|
+
await deferQueue.flush();
|
|
327
|
+
}
|
|
185
328
|
|
|
186
|
-
|
|
329
|
+
const items = await backend.loadSubset(options);
|
|
187
330
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
331
|
+
begin();
|
|
332
|
+
|
|
333
|
+
for (const item of items) {
|
|
334
|
+
write({
|
|
335
|
+
type: "insert",
|
|
336
|
+
value: item,
|
|
337
|
+
});
|
|
338
|
+
}
|
|
194
339
|
|
|
195
|
-
|
|
340
|
+
commit();
|
|
341
|
+
});
|
|
196
342
|
};
|
|
197
343
|
|
|
198
344
|
let loadSubsetDedupe: DeduplicatedLoadSubset | null = null;
|
|
@@ -204,6 +350,8 @@ export function createGenericSyncFunction<TItem extends object>(
|
|
|
204
350
|
|
|
205
351
|
return {
|
|
206
352
|
cleanup: () => {
|
|
353
|
+
deferQueue?.dispose();
|
|
354
|
+
deferQueue = null;
|
|
207
355
|
insertListener = undefined;
|
|
208
356
|
updateListener = undefined;
|
|
209
357
|
deleteListener = undefined;
|
|
@@ -213,45 +361,179 @@ export function createGenericSyncFunction<TItem extends object>(
|
|
|
213
361
|
} satisfies SyncConfigRes;
|
|
214
362
|
};
|
|
215
363
|
|
|
216
|
-
const
|
|
217
|
-
if (
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
);
|
|
364
|
+
const resolveReceiveSyncPersistKey = (item: TItem): string => {
|
|
365
|
+
if (config.getSyncPersistKey !== undefined) {
|
|
366
|
+
return config.getSyncPersistKey(item);
|
|
367
|
+
}
|
|
368
|
+
if (item !== null && typeof item === "object" && "id" in item) {
|
|
369
|
+
const id = (item as { id: unknown }).id;
|
|
370
|
+
if (typeof id === "string" || typeof id === "number") {
|
|
371
|
+
return String(id);
|
|
224
372
|
}
|
|
225
|
-
return;
|
|
226
373
|
}
|
|
227
|
-
|
|
374
|
+
throw new Error(
|
|
375
|
+
"[receiveSync] Persist key missing: set GenericBaseSyncConfig.getSyncPersistKey or use items with string/number `id`",
|
|
376
|
+
);
|
|
377
|
+
};
|
|
378
|
+
|
|
379
|
+
const shallowRecordDiff = (previous: TItem, next: TItem): Partial<TItem> => {
|
|
380
|
+
const out: Partial<TItem> = {};
|
|
381
|
+
if (
|
|
382
|
+
previous !== null &&
|
|
383
|
+
typeof previous === "object" &&
|
|
384
|
+
next !== null &&
|
|
385
|
+
typeof next === "object"
|
|
386
|
+
) {
|
|
387
|
+
const prevRec = previous as Record<string, unknown>;
|
|
388
|
+
const nextRec = next as Record<string, unknown>;
|
|
389
|
+
for (const k of Object.keys(nextRec)) {
|
|
390
|
+
if (prevRec[k] !== nextRec[k]) {
|
|
391
|
+
(out as Record<string, unknown>)[k] = nextRec[k];
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
return out;
|
|
396
|
+
};
|
|
397
|
+
|
|
398
|
+
const toReceiveSyncDurableOps = (
|
|
399
|
+
messages: SyncMessage<TItem>[],
|
|
400
|
+
): ReceiveSyncDurableOp<TItem>[] => {
|
|
401
|
+
const out: ReceiveSyncDurableOp<TItem>[] = [];
|
|
228
402
|
for (const msg of messages) {
|
|
229
403
|
switch (msg.type) {
|
|
230
404
|
case "insert":
|
|
231
|
-
|
|
405
|
+
out.push({ type: "insert", value: msg.value });
|
|
232
406
|
break;
|
|
233
407
|
case "update":
|
|
234
|
-
|
|
408
|
+
out.push({
|
|
409
|
+
type: "update",
|
|
410
|
+
key: resolveReceiveSyncPersistKey(msg.value),
|
|
411
|
+
changes: shallowRecordDiff(
|
|
412
|
+
msg.previousValue,
|
|
413
|
+
msg.value,
|
|
414
|
+
) as Partial<TItem>,
|
|
415
|
+
original: msg.previousValue,
|
|
416
|
+
});
|
|
235
417
|
break;
|
|
236
418
|
case "delete":
|
|
237
|
-
|
|
238
|
-
type: "delete",
|
|
239
|
-
value: { id: msg.key } as TItem,
|
|
240
|
-
});
|
|
419
|
+
out.push({ type: "delete", key: String(msg.key) });
|
|
241
420
|
break;
|
|
242
421
|
case "truncate":
|
|
243
|
-
|
|
422
|
+
out.push({ type: "truncate" });
|
|
244
423
|
break;
|
|
245
424
|
default:
|
|
246
425
|
exhaustiveGuard(msg);
|
|
247
426
|
}
|
|
248
427
|
}
|
|
249
|
-
|
|
428
|
+
return out;
|
|
429
|
+
};
|
|
430
|
+
|
|
431
|
+
const receiveSync = async (messages: SyncMessage<TItem>[]) => {
|
|
432
|
+
if (messages.length === 0) return;
|
|
433
|
+
|
|
434
|
+
await enqueueSyncLayer(async () => {
|
|
435
|
+
if (initialSyncDone) {
|
|
436
|
+
await initialSyncDone;
|
|
437
|
+
}
|
|
438
|
+
if (!syncBegin || !syncWrite || !syncCommit || !syncTruncate) {
|
|
439
|
+
if (config.debug) {
|
|
440
|
+
console.warn(
|
|
441
|
+
"[receiveSync] Sync functions not initialized yet - messages will be dropped",
|
|
442
|
+
messages.length,
|
|
443
|
+
);
|
|
444
|
+
}
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
if (deferQueue !== null) {
|
|
448
|
+
await deferQueue.flush();
|
|
449
|
+
}
|
|
450
|
+
syncBegin();
|
|
451
|
+
|
|
452
|
+
try {
|
|
453
|
+
const applyBatch = backend.applyReceiveSyncDurableWrites;
|
|
454
|
+
if (applyBatch !== undefined) {
|
|
455
|
+
await applyBatch(toReceiveSyncDurableOps(messages));
|
|
456
|
+
for (const msg of messages) {
|
|
457
|
+
switch (msg.type) {
|
|
458
|
+
case "insert":
|
|
459
|
+
syncWrite({ type: "insert", value: msg.value });
|
|
460
|
+
break;
|
|
461
|
+
case "update":
|
|
462
|
+
syncWrite({ type: "update", value: msg.value });
|
|
463
|
+
break;
|
|
464
|
+
case "delete":
|
|
465
|
+
syncWrite({
|
|
466
|
+
type: "delete",
|
|
467
|
+
value: { id: msg.key } as TItem,
|
|
468
|
+
});
|
|
469
|
+
break;
|
|
470
|
+
case "truncate":
|
|
471
|
+
syncTruncate();
|
|
472
|
+
break;
|
|
473
|
+
default:
|
|
474
|
+
exhaustiveGuard(msg);
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
} else {
|
|
478
|
+
for (const msg of messages) {
|
|
479
|
+
switch (msg.type) {
|
|
480
|
+
case "insert":
|
|
481
|
+
await backend.handleInsert([msg.value]);
|
|
482
|
+
syncWrite({ type: "insert", value: msg.value });
|
|
483
|
+
break;
|
|
484
|
+
case "update": {
|
|
485
|
+
const key = resolveReceiveSyncPersistKey(msg.value);
|
|
486
|
+
await backend.handleUpdate([
|
|
487
|
+
{
|
|
488
|
+
key,
|
|
489
|
+
changes: shallowRecordDiff(
|
|
490
|
+
msg.previousValue,
|
|
491
|
+
msg.value,
|
|
492
|
+
) as Partial<TItem>,
|
|
493
|
+
original: msg.previousValue,
|
|
494
|
+
},
|
|
495
|
+
]);
|
|
496
|
+
syncWrite({ type: "update", value: msg.value });
|
|
497
|
+
break;
|
|
498
|
+
}
|
|
499
|
+
case "delete":
|
|
500
|
+
await backend.handleDelete([
|
|
501
|
+
{
|
|
502
|
+
key: String(msg.key),
|
|
503
|
+
modified: { id: msg.key } as TItem,
|
|
504
|
+
original: { id: msg.key } as TItem,
|
|
505
|
+
},
|
|
506
|
+
]);
|
|
507
|
+
syncWrite({
|
|
508
|
+
type: "delete",
|
|
509
|
+
value: { id: msg.key } as TItem,
|
|
510
|
+
});
|
|
511
|
+
break;
|
|
512
|
+
case "truncate":
|
|
513
|
+
if (backend.handleTruncate) {
|
|
514
|
+
await backend.handleTruncate();
|
|
515
|
+
}
|
|
516
|
+
syncTruncate();
|
|
517
|
+
break;
|
|
518
|
+
default:
|
|
519
|
+
exhaustiveGuard(msg);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
} catch (err) {
|
|
524
|
+
console.error(
|
|
525
|
+
"[receiveSync] error during sync writes, committing partial batch to avoid leaving transaction open",
|
|
526
|
+
err,
|
|
527
|
+
);
|
|
528
|
+
}
|
|
529
|
+
syncCommit();
|
|
530
|
+
});
|
|
250
531
|
};
|
|
251
532
|
|
|
252
533
|
const utils: CollectionUtils<TItem> = {
|
|
253
534
|
truncate: async () => {
|
|
254
|
-
|
|
535
|
+
const handleTruncate = backend.handleTruncate;
|
|
536
|
+
if (!handleTruncate) {
|
|
255
537
|
throw new Error("Truncate not supported by this backend");
|
|
256
538
|
}
|
|
257
539
|
if (!syncBegin || !syncTruncate || !syncCommit) {
|
|
@@ -259,10 +541,23 @@ export function createGenericSyncFunction<TItem extends object>(
|
|
|
259
541
|
"Sync functions not initialized - sync function may not have been called yet",
|
|
260
542
|
);
|
|
261
543
|
}
|
|
262
|
-
await
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
544
|
+
await enqueueSyncLayer(async () => {
|
|
545
|
+
if (deferQueue !== null) {
|
|
546
|
+
await deferQueue.flush();
|
|
547
|
+
}
|
|
548
|
+
await handleTruncate();
|
|
549
|
+
const begin = syncBegin;
|
|
550
|
+
const trunc = syncTruncate;
|
|
551
|
+
const commit = syncCommit;
|
|
552
|
+
if (!begin || !trunc || !commit) {
|
|
553
|
+
throw new Error(
|
|
554
|
+
"Sync functions not initialized - sync function may not have been called yet",
|
|
555
|
+
);
|
|
556
|
+
}
|
|
557
|
+
begin();
|
|
558
|
+
trunc();
|
|
559
|
+
commit();
|
|
560
|
+
});
|
|
266
561
|
},
|
|
267
562
|
receiveSync,
|
|
268
563
|
};
|
|
@@ -332,8 +627,8 @@ export function createGenericCollectionConfig<
|
|
|
332
627
|
CollectionConfig<
|
|
333
628
|
TItem,
|
|
334
629
|
string,
|
|
335
|
-
|
|
336
|
-
|
|
630
|
+
TSchema,
|
|
631
|
+
CollectionUtils<InferSchemaOutput<TSchema>>
|
|
337
632
|
>,
|
|
338
633
|
"utils"
|
|
339
634
|
> & {
|
|
@@ -350,6 +645,10 @@ export function createGenericCollectionConfig<
|
|
|
350
645
|
onUpdate: config.onUpdate ?? config.syncResult.onUpdate,
|
|
351
646
|
onDelete: config.onDelete ?? config.syncResult.onDelete,
|
|
352
647
|
syncMode: config.syncMode,
|
|
648
|
+
// TanStack DB 0.6+: indexing is opt-in; eager BasicIndex restores pre-0.6 behavior for
|
|
649
|
+
// orderBy/limit live queries (playground pagination, usePredicateFilteredRows, etc.).
|
|
650
|
+
defaultIndexType: BasicIndex,
|
|
651
|
+
autoIndex: "eager",
|
|
353
652
|
utils: config.syncResult.utils as CollectionUtils<
|
|
354
653
|
InferSchemaOutput<TSchema>
|
|
355
654
|
>,
|
package/src/index.ts
CHANGED
|
@@ -2,6 +2,7 @@ export type {
|
|
|
2
2
|
CollectionUtils,
|
|
3
3
|
ExternalSyncEvent,
|
|
4
4
|
ExternalSyncHandler,
|
|
5
|
+
ReceiveSyncDurableOp,
|
|
5
6
|
SyncMessage,
|
|
6
7
|
} from "./sync-types";
|
|
7
8
|
export {
|
|
@@ -20,3 +21,8 @@ export {
|
|
|
20
21
|
type GenericSyncBackend,
|
|
21
22
|
type GenericSyncFunctionResult,
|
|
22
23
|
} from "./generic-sync";
|
|
24
|
+
export {
|
|
25
|
+
DeferredWriteQueue,
|
|
26
|
+
type DeferredDeleteMutation,
|
|
27
|
+
type DeferredUpdateMutation,
|
|
28
|
+
} from "./deferred-write-queue";
|
package/src/memoryCollection.ts
CHANGED
|
@@ -46,10 +46,27 @@ export function memoryCollectionOptions<TSchema extends StandardSchemaV1>(
|
|
|
46
46
|
type TItem = InferSchemaOutput<TSchema>;
|
|
47
47
|
type TKey = string | number;
|
|
48
48
|
let syncParams: Parameters<SyncConfig<TItem>["sync"]>[0] | null = null;
|
|
49
|
+
/** Batches from `receiveSync` that arrived before TanStack called `sync`. */
|
|
50
|
+
const pendingReceiveSyncBatches: SyncMessage<TItem, TKey>[][] = [];
|
|
51
|
+
/**
|
|
52
|
+
* One TanStack sync transaction at a time: `receiveSync`, local mutations, and `truncate` all
|
|
53
|
+
* call `begin`/`commit` — overlapping calls cause SyncTransactionAlreadyCommittedWriteError.
|
|
54
|
+
*/
|
|
55
|
+
let syncWriteChain: Promise<void> = Promise.resolve();
|
|
56
|
+
|
|
57
|
+
const enqueueSyncWrite = async (fn: () => void): Promise<void> => {
|
|
58
|
+
const next = syncWriteChain.catch(() => {}).then(fn);
|
|
59
|
+
syncWriteChain = next;
|
|
60
|
+
await next;
|
|
61
|
+
};
|
|
49
62
|
|
|
50
63
|
const sync: SyncConfig<TItem>["sync"] = (params) => {
|
|
51
64
|
syncParams = params;
|
|
52
65
|
params.markReady();
|
|
66
|
+
for (const batch of pendingReceiveSyncBatches) {
|
|
67
|
+
writeChanges(batch);
|
|
68
|
+
}
|
|
69
|
+
pendingReceiveSyncBatches.length = 0;
|
|
53
70
|
return () => {};
|
|
54
71
|
};
|
|
55
72
|
|
|
@@ -88,7 +105,9 @@ export function memoryCollectionOptions<TSchema extends StandardSchemaV1>(
|
|
|
88
105
|
for (const mutation of params.transaction.mutations) {
|
|
89
106
|
writes.push({ type: "insert", value: mutation.modified });
|
|
90
107
|
}
|
|
91
|
-
|
|
108
|
+
await enqueueSyncWrite(() => {
|
|
109
|
+
writeChanges(writes);
|
|
110
|
+
});
|
|
92
111
|
config.onBroadcast?.(writes);
|
|
93
112
|
};
|
|
94
113
|
|
|
@@ -101,7 +120,9 @@ export function memoryCollectionOptions<TSchema extends StandardSchemaV1>(
|
|
|
101
120
|
previousValue: mutation.original,
|
|
102
121
|
});
|
|
103
122
|
}
|
|
104
|
-
|
|
123
|
+
await enqueueSyncWrite(() => {
|
|
124
|
+
writeChanges(writes);
|
|
125
|
+
});
|
|
105
126
|
config.onBroadcast?.(writes);
|
|
106
127
|
};
|
|
107
128
|
|
|
@@ -110,22 +131,36 @@ export function memoryCollectionOptions<TSchema extends StandardSchemaV1>(
|
|
|
110
131
|
for (const mutation of params.transaction.mutations) {
|
|
111
132
|
writes.push({ type: "delete", key: mutation.key as TKey });
|
|
112
133
|
}
|
|
113
|
-
|
|
134
|
+
await enqueueSyncWrite(() => {
|
|
135
|
+
writeChanges(writes);
|
|
136
|
+
});
|
|
114
137
|
config.onBroadcast?.(writes);
|
|
115
138
|
};
|
|
116
139
|
|
|
117
140
|
const truncate = async () => {
|
|
118
141
|
if (!syncParams) {
|
|
119
|
-
|
|
142
|
+
// TanStack may not have invoked `sync` yet (e.g. first paint / effect). Nothing to clear.
|
|
143
|
+
pendingReceiveSyncBatches.length = 0;
|
|
144
|
+
return;
|
|
120
145
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
146
|
+
await enqueueSyncWrite(() => {
|
|
147
|
+
const p = syncParams;
|
|
148
|
+
if (!p) return;
|
|
149
|
+
p.begin();
|
|
150
|
+
p.truncate();
|
|
151
|
+
p.commit();
|
|
152
|
+
});
|
|
124
153
|
};
|
|
125
154
|
|
|
126
155
|
const receiveSync = async (messages: SyncMessage<TItem, TKey>[]) => {
|
|
127
156
|
if (messages.length === 0) return;
|
|
128
|
-
|
|
157
|
+
if (!syncParams) {
|
|
158
|
+
pendingReceiveSyncBatches.push(messages);
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
await enqueueSyncWrite(() => {
|
|
162
|
+
writeChanges(messages);
|
|
163
|
+
});
|
|
129
164
|
};
|
|
130
165
|
|
|
131
166
|
return {
|
package/src/sync-types.ts
CHANGED
|
@@ -15,6 +15,22 @@ export type SyncMessage<
|
|
|
15
15
|
| { type: "delete"; key: TKey }
|
|
16
16
|
| { type: "truncate" };
|
|
17
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Normalized durable ops for a {@link SyncMessage} batch. SQLite-style backends can implement
|
|
20
|
+
* `GenericSyncBackend.applyReceiveSyncDurableWrites` to persist the whole batch in one store
|
|
21
|
+
* transaction instead of one transaction per message.
|
|
22
|
+
*/
|
|
23
|
+
export type ReceiveSyncDurableOp<TItem extends object> =
|
|
24
|
+
| { type: "insert"; value: TItem }
|
|
25
|
+
| {
|
|
26
|
+
type: "update";
|
|
27
|
+
key: string;
|
|
28
|
+
changes: Partial<TItem>;
|
|
29
|
+
original: TItem;
|
|
30
|
+
}
|
|
31
|
+
| { type: "delete"; key: string }
|
|
32
|
+
| { type: "truncate" };
|
|
33
|
+
|
|
18
34
|
/**
|
|
19
35
|
* External sync event (batched). Used internally by the sync layer.
|
|
20
36
|
*/
|