@syncular/server 0.15.48 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -9
- package/dist/admin.js +1 -5
- package/dist/authoritative-query.d.ts +14 -6
- package/dist/authoritative-query.js +60 -82
- package/dist/d1-storage.d.ts +13 -1
- package/dist/d1-storage.js +468 -121
- package/dist/operations.d.ts +2 -0
- package/dist/operations.js +23 -4
- package/dist/postgres-storage.d.ts +6 -1
- package/dist/postgres-storage.js +80 -24
- package/dist/prune.d.ts +3 -1
- package/dist/prune.js +18 -14
- package/dist/pull.js +79 -39
- package/dist/push.js +5 -5
- package/dist/realtime.d.ts +1 -1
- package/dist/realtime.js +16 -10
- package/dist/relational-rows.d.ts +7 -1
- package/dist/relational-rows.js +17 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +3 -3
- package/dist/sqlite-image.js +10 -6
- package/dist/sqlite-node.js +2 -2
- package/dist/sqlite-storage.d.ts +6 -1
- package/dist/sqlite-storage.js +93 -32
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +7 -0
- package/dist/storage.d.ts +29 -5
- package/package.json +2 -2
- package/src/admin.ts +4 -6
- package/src/authoritative-query.ts +89 -94
- package/src/d1-storage.ts +641 -159
- package/src/operations.ts +31 -3
- package/src/postgres-storage.ts +146 -46
- package/src/prune.ts +29 -15
- package/src/pull.ts +102 -49
- package/src/push.ts +5 -5
- package/src/realtime.ts +20 -9
- package/src/relational-rows.ts +18 -2
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +26 -15
- package/src/sqlite-node.ts +2 -2
- package/src/sqlite-storage.ts +131 -37
- package/src/storage-errors.ts +22 -1
- package/src/storage.ts +50 -5
package/src/operations.ts
CHANGED
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type AuthoritativeRelationPlan,
|
|
3
|
+
validateAuthoritativeRelationPlan,
|
|
4
|
+
} from './authoritative-query';
|
|
1
5
|
import {
|
|
2
6
|
decodeRow,
|
|
3
7
|
decodeRemoteOperationRequest,
|
|
@@ -42,6 +46,7 @@ export interface AuthoritativeQueryDescriptor<Params = undefined> {
|
|
|
42
46
|
readonly hasParams: boolean;
|
|
43
47
|
readonly sql: string;
|
|
44
48
|
readonly tables: readonly string[];
|
|
49
|
+
readonly relationPlans: readonly AuthoritativeRelationPlan[];
|
|
45
50
|
readonly resultColumns: readonly {
|
|
46
51
|
readonly name: string;
|
|
47
52
|
readonly type:
|
|
@@ -280,6 +285,9 @@ export function registerRemoteQuery<Params>(
|
|
|
280
285
|
options: RemoteQueryOptions<Params>,
|
|
281
286
|
): RegisteredRemoteQuery {
|
|
282
287
|
if (
|
|
288
|
+
!Array.isArray(descriptor.relationPlans) ||
|
|
289
|
+
!descriptor.relationPlans.some((plan) => plan.sql === descriptor.sql) ||
|
|
290
|
+
descriptor.relationPlans.some((plan) => !Array.isArray(plan.relations)) ||
|
|
283
291
|
descriptor.id.length === 0 ||
|
|
284
292
|
new Set(descriptor.tables).size !== descriptor.tables.length ||
|
|
285
293
|
!Array.isArray(descriptor.resultColumns) ||
|
|
@@ -288,9 +296,12 @@ export function registerRemoteQuery<Params>(
|
|
|
288
296
|
descriptor.resultColumns.length
|
|
289
297
|
) {
|
|
290
298
|
throw new Error(
|
|
291
|
-
'remote query requires a non-empty id and unique tables and result columns',
|
|
299
|
+
'remote query requires generated relation plans, a non-empty id, and unique tables and result columns; regenerate queries',
|
|
292
300
|
);
|
|
293
301
|
}
|
|
302
|
+
for (const plan of descriptor.relationPlans) {
|
|
303
|
+
validateAuthoritativeRelationPlan(plan, descriptor.tables);
|
|
304
|
+
}
|
|
294
305
|
if (
|
|
295
306
|
!Number.isSafeInteger(options.maxRows) ||
|
|
296
307
|
options.maxRows < 1 ||
|
|
@@ -394,12 +405,29 @@ export function registerRemoteQuery<Params>(
|
|
|
394
405
|
'configured storage does not implement authoritative queries',
|
|
395
406
|
);
|
|
396
407
|
}
|
|
397
|
-
await ctx.storage.ensureSchema(schema);
|
|
398
408
|
const selectedSql = descriptor.sqlFor?.(params) ?? descriptor.sql;
|
|
409
|
+
const plan = descriptor.relationPlans.find(
|
|
410
|
+
(candidate) => candidate.sql === selectedSql,
|
|
411
|
+
);
|
|
412
|
+
if (plan === undefined) {
|
|
413
|
+
throw syncError(
|
|
414
|
+
'operation.invalid_request',
|
|
415
|
+
'selected SQL has no generated relation plan; regenerate queries',
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
await ctx.storage.ensureSchema(schema);
|
|
419
|
+
const prefix = 'SELECT * FROM (';
|
|
399
420
|
let result;
|
|
400
421
|
try {
|
|
401
422
|
result = await ctx.storage.queryAuthoritative(ctx.partition, {
|
|
402
|
-
|
|
423
|
+
plan: {
|
|
424
|
+
sql: `${prefix}${selectedSql}) AS "_syncular_registered_query" LIMIT ?`,
|
|
425
|
+
relations: plan.relations.map((relation) => ({
|
|
426
|
+
...relation,
|
|
427
|
+
start: relation.start + prefix.length,
|
|
428
|
+
end: relation.end + prefix.length,
|
|
429
|
+
})),
|
|
430
|
+
},
|
|
403
431
|
params: [...descriptor.bind(params), options.maxRows + 1],
|
|
404
432
|
tables: descriptor.tables,
|
|
405
433
|
});
|
package/src/postgres-storage.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { validateCommitPruneQuery } from './prune';
|
|
2
|
+
import { StorageQueryError } from './storage-errors';
|
|
3
|
+
import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
1
4
|
/**
|
|
2
5
|
* Postgres server storage: the production database path.
|
|
3
6
|
*
|
|
@@ -223,6 +226,28 @@ interface SerializedResult {
|
|
|
223
226
|
details?: import('@syncular/core').RejectionDetails;
|
|
224
227
|
}
|
|
225
228
|
|
|
229
|
+
async function lockPartitionOn(
|
|
230
|
+
client: PgQueryable,
|
|
231
|
+
partition: string,
|
|
232
|
+
): Promise<void> {
|
|
233
|
+
const locked = await client.query(
|
|
234
|
+
'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
|
|
235
|
+
[partition],
|
|
236
|
+
);
|
|
237
|
+
if (locked.rows.length > 0) return;
|
|
238
|
+
// The first writer initializes the partition. Concurrent initializers may
|
|
239
|
+
// wait on this insert, so acquire the row lock again before applying writes.
|
|
240
|
+
await client.query(
|
|
241
|
+
`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
|
|
242
|
+
ON CONFLICT (partition) DO NOTHING`,
|
|
243
|
+
[partition],
|
|
244
|
+
);
|
|
245
|
+
await client.query(
|
|
246
|
+
'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
|
|
247
|
+
[partition],
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
|
|
226
251
|
function toBase64(bytes: Uint8Array): string {
|
|
227
252
|
return Buffer.from(bytes).toString('base64');
|
|
228
253
|
}
|
|
@@ -649,15 +674,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
649
674
|
|
|
650
675
|
async lockPartitionForPush(): Promise<void> {
|
|
651
676
|
this.#assertOpen();
|
|
652
|
-
await this.#client
|
|
653
|
-
`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
|
|
654
|
-
ON CONFLICT (partition) DO NOTHING`,
|
|
655
|
-
[this.#partition],
|
|
656
|
-
);
|
|
657
|
-
await this.#client.query(
|
|
658
|
-
'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
|
|
659
|
-
[this.#partition],
|
|
660
|
-
);
|
|
677
|
+
await lockPartitionOn(this.#client, this.#partition);
|
|
661
678
|
await this.#client.query('SAVEPOINT syncular_push_candidate');
|
|
662
679
|
this.#pushApplySavepoint = true;
|
|
663
680
|
}
|
|
@@ -743,32 +760,40 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
743
760
|
// Allocate the next dense commitSeq under a per-partition row lock: the
|
|
744
761
|
// UPDATE … RETURNING serializes concurrent pushes to this partition and
|
|
745
762
|
// never leaves a gap on rollback (see the file header).
|
|
746
|
-
const { rows } = await q.query<{
|
|
747
|
-
`
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
VALUES ($1,$2,$3,$4,$5,$6)`,
|
|
763
|
+
const { rows } = await q.query<{ commit_seq: unknown }>(
|
|
764
|
+
`WITH allocated AS (
|
|
765
|
+
INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 1)
|
|
766
|
+
ON CONFLICT (partition) DO UPDATE
|
|
767
|
+
SET max_commit_seq = sync_partitions.max_commit_seq + 1
|
|
768
|
+
RETURNING max_commit_seq
|
|
769
|
+
)
|
|
770
|
+
INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms)
|
|
771
|
+
SELECT $1, max_commit_seq, $2, $3, $4, $5 FROM allocated
|
|
772
|
+
RETURNING commit_seq`,
|
|
757
773
|
[
|
|
758
774
|
p,
|
|
759
|
-
commitSeq,
|
|
760
775
|
commit.clientId,
|
|
761
776
|
commit.clientCommitId,
|
|
762
777
|
commit.actorId,
|
|
763
778
|
commit.createdAtMs,
|
|
764
779
|
],
|
|
765
780
|
);
|
|
781
|
+
const commitSeq = asNumber(rows[0]?.commit_seq);
|
|
766
782
|
for (let idx = 0; idx < commit.changes.length; idx++) {
|
|
767
783
|
const change = commit.changes[idx];
|
|
768
784
|
if (change === undefined) continue;
|
|
785
|
+
// Bind serialized scopes as text before parsing JSONB. Drivers that
|
|
786
|
+
// encode JSONB parameters would otherwise store this string as a scalar.
|
|
769
787
|
await q.query(
|
|
770
|
-
`
|
|
771
|
-
|
|
788
|
+
`WITH inserted AS (
|
|
789
|
+
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
790
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7,$8::text::jsonb,$9)
|
|
791
|
+
RETURNING partition, tbl, commit_seq, scopes
|
|
792
|
+
)
|
|
793
|
+
INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
|
|
794
|
+
SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
|
|
795
|
+
FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
|
|
796
|
+
ON CONFLICT DO NOTHING`,
|
|
772
797
|
[
|
|
773
798
|
p,
|
|
774
799
|
commitSeq,
|
|
@@ -781,13 +806,6 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
781
806
|
change.payload ?? null,
|
|
782
807
|
],
|
|
783
808
|
);
|
|
784
|
-
for (const [variable, value] of Object.entries(change.scopes)) {
|
|
785
|
-
await q.query(
|
|
786
|
-
`INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
|
|
787
|
-
VALUES ($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`,
|
|
788
|
-
[p, change.table, variable, value, commitSeq],
|
|
789
|
-
);
|
|
790
|
-
}
|
|
791
809
|
}
|
|
792
810
|
return commitSeq;
|
|
793
811
|
}
|
|
@@ -976,6 +994,9 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
976
994
|
await client.query('DELETE FROM sync_row_scopes WHERE tbl=$1', [
|
|
977
995
|
tableName,
|
|
978
996
|
]);
|
|
997
|
+
await client.query('DELETE FROM sync_blob_refs WHERE tbl=$1', [
|
|
998
|
+
tableName,
|
|
999
|
+
]);
|
|
979
1000
|
await client.query(dropTableDdl(tableName));
|
|
980
1001
|
}
|
|
981
1002
|
for (const statement of schemaDdl(
|
|
@@ -1047,6 +1068,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1047
1068
|
): Promise<PartitionRegistryEntry> {
|
|
1048
1069
|
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
1049
1070
|
await this.#exec.transaction(async (client) => {
|
|
1071
|
+
await lockPartitionOn(client, partition);
|
|
1050
1072
|
await client.query(
|
|
1051
1073
|
`INSERT INTO sync_partition_registry(
|
|
1052
1074
|
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
@@ -1148,7 +1170,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1148
1170
|
}
|
|
1149
1171
|
const prepared = bindAuthoritativePartition(
|
|
1150
1172
|
prepareAuthoritativeQuery(
|
|
1151
|
-
query.
|
|
1173
|
+
query.plan,
|
|
1152
1174
|
query.params,
|
|
1153
1175
|
query.tables,
|
|
1154
1176
|
this.#tables,
|
|
@@ -1177,6 +1199,14 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1177
1199
|
});
|
|
1178
1200
|
}
|
|
1179
1201
|
|
|
1202
|
+
async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
|
|
1203
|
+
const { rows } = await this.#exec.query<{ log_epoch: string }>(
|
|
1204
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1',
|
|
1205
|
+
[partition],
|
|
1206
|
+
);
|
|
1207
|
+
return rows[0]?.log_epoch;
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1180
1210
|
async getHorizonSeq(partition: string): Promise<number> {
|
|
1181
1211
|
const { rows } = await this.#exec.query<{ horizon_seq: unknown }>(
|
|
1182
1212
|
'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
|
|
@@ -1188,25 +1218,52 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1188
1218
|
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
1189
1219
|
await this.#exec.query(
|
|
1190
1220
|
`INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
|
|
1191
|
-
ON CONFLICT (partition) DO UPDATE SET horizon_seq=EXCLUDED.horizon_seq`,
|
|
1221
|
+
ON CONFLICT (partition) DO UPDATE SET horizon_seq=GREATEST(sync_partitions.horizon_seq,EXCLUDED.horizon_seq)`,
|
|
1192
1222
|
[partition, seq],
|
|
1193
1223
|
);
|
|
1194
1224
|
}
|
|
1195
1225
|
|
|
1196
|
-
async pruneCommitsThrough(
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
);
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
[
|
|
1208
|
-
|
|
1209
|
-
|
|
1226
|
+
async pruneCommitsThrough(
|
|
1227
|
+
partition: string,
|
|
1228
|
+
query: CommitPruneQuery,
|
|
1229
|
+
): Promise<CommitPruneResult> {
|
|
1230
|
+
validateCommitPruneQuery(query);
|
|
1231
|
+
return this.#exec.transaction(async (client) => {
|
|
1232
|
+
await lockPartitionOn(client, partition);
|
|
1233
|
+
const epoch = await client.query<{ log_epoch: string }>(
|
|
1234
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1 FOR UPDATE',
|
|
1235
|
+
[partition],
|
|
1236
|
+
);
|
|
1237
|
+
if (epoch.rows[0]?.log_epoch !== query.logEpoch)
|
|
1238
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
1239
|
+
const previous = await client.query<{ horizon_seq: unknown }>(
|
|
1240
|
+
'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
|
|
1241
|
+
[partition],
|
|
1242
|
+
);
|
|
1243
|
+
const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
|
|
1244
|
+
const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
|
|
1245
|
+
await client.query(
|
|
1246
|
+
'UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1',
|
|
1247
|
+
[partition, horizonSeq],
|
|
1248
|
+
);
|
|
1249
|
+
const removed = await client.query(
|
|
1250
|
+
'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2',
|
|
1251
|
+
[partition, horizonSeq],
|
|
1252
|
+
);
|
|
1253
|
+
await client.query(
|
|
1254
|
+
'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
|
|
1255
|
+
[partition, horizonSeq],
|
|
1256
|
+
);
|
|
1257
|
+
await client.query(
|
|
1258
|
+
'DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2',
|
|
1259
|
+
[partition, horizonSeq],
|
|
1260
|
+
);
|
|
1261
|
+
return {
|
|
1262
|
+
previousHorizonSeq,
|
|
1263
|
+
horizonSeq,
|
|
1264
|
+
removedCommits: removed.rowCount,
|
|
1265
|
+
};
|
|
1266
|
+
});
|
|
1210
1267
|
}
|
|
1211
1268
|
|
|
1212
1269
|
async getCommitSeqBefore(
|
|
@@ -1630,6 +1687,49 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1630
1687
|
);
|
|
1631
1688
|
}
|
|
1632
1689
|
|
|
1690
|
+
async advanceClientCursor(
|
|
1691
|
+
partition: string,
|
|
1692
|
+
clientId: string,
|
|
1693
|
+
actorId: string,
|
|
1694
|
+
logEpoch: string,
|
|
1695
|
+
cursor: number,
|
|
1696
|
+
updatedAtMs: number,
|
|
1697
|
+
): Promise<void> {
|
|
1698
|
+
await this.#exec.query(
|
|
1699
|
+
`UPDATE sync_clients
|
|
1700
|
+
SET cursor=GREATEST(cursor, $1), updated_at_ms=GREATEST(updated_at_ms, $2)
|
|
1701
|
+
WHERE partition=$3 AND client_id=$4 AND actor_id=$5
|
|
1702
|
+
AND EXISTS (SELECT 1 FROM sync_partition_registry
|
|
1703
|
+
WHERE partition=sync_clients.partition AND log_epoch=$6)`,
|
|
1704
|
+
[cursor, updatedAtMs, partition, clientId, actorId, logEpoch],
|
|
1705
|
+
);
|
|
1706
|
+
}
|
|
1707
|
+
|
|
1708
|
+
async updateClientCursor(
|
|
1709
|
+
partition: string,
|
|
1710
|
+
clientId: string,
|
|
1711
|
+
cursor: number,
|
|
1712
|
+
updatedAtMs: number,
|
|
1713
|
+
): Promise<void> {
|
|
1714
|
+
await this.#exec.query(
|
|
1715
|
+
`UPDATE sync_clients
|
|
1716
|
+
SET cursor=GREATEST(cursor, $3), updated_at_ms=GREATEST(updated_at_ms, $4)
|
|
1717
|
+
WHERE partition=$1 AND client_id=$2`,
|
|
1718
|
+
[partition, clientId, cursor, updatedAtMs],
|
|
1719
|
+
);
|
|
1720
|
+
}
|
|
1721
|
+
|
|
1722
|
+
async getActiveClientCursorFloor(
|
|
1723
|
+
partition: string,
|
|
1724
|
+
cutoffMs: number,
|
|
1725
|
+
): Promise<number | null> {
|
|
1726
|
+
const { rows } = await this.#exec.query<{ cursor: unknown }>(
|
|
1727
|
+
'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2',
|
|
1728
|
+
[partition, cutoffMs],
|
|
1729
|
+
);
|
|
1730
|
+
return rows[0]!.cursor === null ? null : asNumber(rows[0]!.cursor);
|
|
1731
|
+
}
|
|
1732
|
+
|
|
1633
1733
|
async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
|
|
1634
1734
|
const { rows } = await this.#exec.query<{
|
|
1635
1735
|
client_id: string;
|
package/src/prune.ts
CHANGED
|
@@ -7,7 +7,20 @@
|
|
|
7
7
|
* least the newest `minRetainedCommits` commits are always retained.
|
|
8
8
|
*/
|
|
9
9
|
import { emitEvent, type SyncularServerEvents } from './events';
|
|
10
|
-
import type { ServerStorage } from './storage';
|
|
10
|
+
import type { CommitPruneQuery, ServerStorage } from './storage';
|
|
11
|
+
import { StorageQueryError } from './storage-errors';
|
|
12
|
+
|
|
13
|
+
/** Shared validation for the built-in atomic pruning adapters. */
|
|
14
|
+
export function validateCommitPruneQuery(query: CommitPruneQuery): void {
|
|
15
|
+
if (
|
|
16
|
+
!Number.isSafeInteger(query.throughSeq) ||
|
|
17
|
+
query.throughSeq < 0 ||
|
|
18
|
+
typeof query.logEpoch !== 'string' ||
|
|
19
|
+
query.logEpoch.length === 0
|
|
20
|
+
) {
|
|
21
|
+
throw new StorageQueryError('sync.storage.invalid_prune_cursor');
|
|
22
|
+
}
|
|
23
|
+
}
|
|
11
24
|
|
|
12
25
|
export interface RetentionPolicy {
|
|
13
26
|
/** Active window for laggard cursors (default 14 days). */
|
|
@@ -37,28 +50,29 @@ export interface PruneOptions {
|
|
|
37
50
|
export async function pruneCommitLog(options: PruneOptions): Promise<number> {
|
|
38
51
|
const { storage, partition, nowMs } = options;
|
|
39
52
|
const policy = { ...DEFAULT_RETENTION, ...options.retention };
|
|
53
|
+
const logEpoch = await storage.getPartitionLogEpoch(partition);
|
|
54
|
+
if (logEpoch === undefined)
|
|
55
|
+
throw new StorageQueryError('sync.storage.partition_unregistered');
|
|
40
56
|
const maxSeq = await storage.getMaxCommitSeq(partition);
|
|
41
|
-
const cursors = await storage.listClientCursors(partition);
|
|
42
|
-
const activeCursors = cursors
|
|
43
|
-
.filter((c) => c.updatedAtMs >= nowMs - policy.activeWindowMs)
|
|
44
|
-
.map((c) => c.cursor);
|
|
45
57
|
const cursorFloor =
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
58
|
+
(await storage.getActiveClientCursorFloor(
|
|
59
|
+
partition,
|
|
60
|
+
nowMs - policy.activeWindowMs,
|
|
61
|
+
)) ?? Number.MAX_SAFE_INTEGER;
|
|
49
62
|
const forcedSeq = await storage.getCommitSeqBefore(
|
|
50
63
|
partition,
|
|
51
64
|
nowMs - policy.ageForceMs,
|
|
52
65
|
);
|
|
53
66
|
const retainFloor = maxSeq - policy.minRetainedCommits;
|
|
54
67
|
const target = Math.min(Math.max(cursorFloor, forcedSeq), retainFloor);
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
68
|
+
const {
|
|
69
|
+
previousHorizonSeq: current,
|
|
70
|
+
horizonSeq: horizon,
|
|
71
|
+
removedCommits,
|
|
72
|
+
} = await storage.pruneCommitsThrough(partition, {
|
|
73
|
+
logEpoch,
|
|
74
|
+
throughSeq: Math.max(0, target),
|
|
75
|
+
});
|
|
62
76
|
const events = options.events;
|
|
63
77
|
if (events !== undefined) {
|
|
64
78
|
emitEvent(events, {
|
package/src/pull.ts
CHANGED
|
@@ -17,10 +17,16 @@ import { clockOf, limitsOf } from './context';
|
|
|
17
17
|
import type { PullSegmentSummary } from './events';
|
|
18
18
|
import type { CompiledSchema, CompiledTable } from './schema';
|
|
19
19
|
import { scopeDigest } from './scopes';
|
|
20
|
-
import type { SegmentRecord } from './segment-store';
|
|
20
|
+
import type { SegmentRecord, SegmentStore } from './segment-store';
|
|
21
21
|
import { issueSegmentUrl } from './signed-url';
|
|
22
22
|
import type { SqliteImageBuilder } from './sqlite-image';
|
|
23
|
-
import type { StoredCommit, StoredRow } from './storage';
|
|
23
|
+
import type { ServerStorage, StoredCommit, StoredRow } from './storage';
|
|
24
|
+
|
|
25
|
+
// One artifact build per owning storage pair and complete immutable identity.
|
|
26
|
+
const imageBuilds = new WeakMap<
|
|
27
|
+
ServerStorage,
|
|
28
|
+
WeakMap<SegmentStore, Map<string, Promise<SegmentRecord>>>
|
|
29
|
+
>();
|
|
24
30
|
|
|
25
31
|
/**
|
|
26
32
|
* Resolve the §5.3 image builder: the host-injected one if present, else the
|
|
@@ -260,57 +266,89 @@ async function* sqliteImageSegment(
|
|
|
260
266
|
// support floor, not a fallback (§5.3: sqlite is an *accept*, not a demand).
|
|
261
267
|
const buildImage = await resolveImageBuilder(ctx);
|
|
262
268
|
if (buildImage === undefined) return false;
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
let
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
scopeFilter: plan.effective,
|
|
273
|
-
afterRowId,
|
|
274
|
-
limit: 50_000,
|
|
275
|
-
});
|
|
276
|
-
rows.push(...scanned);
|
|
277
|
-
const last = scanned[scanned.length - 1];
|
|
278
|
-
if (scanned.length < 50_000 || last === undefined) break;
|
|
279
|
-
afterRowId = last.rowId;
|
|
269
|
+
let stores = imageBuilds.get(storage);
|
|
270
|
+
if (stores === undefined) {
|
|
271
|
+
stores = new WeakMap();
|
|
272
|
+
imageBuilds.set(storage, stores);
|
|
273
|
+
}
|
|
274
|
+
let builds = stores.get(segments);
|
|
275
|
+
if (builds === undefined) {
|
|
276
|
+
builds = new Map();
|
|
277
|
+
stores.set(segments, builds);
|
|
280
278
|
}
|
|
281
|
-
const
|
|
282
|
-
|
|
279
|
+
const identity = {
|
|
280
|
+
partition,
|
|
281
|
+
logEpoch,
|
|
282
|
+
table: plan.table.name,
|
|
283
283
|
schemaVersion: schema.version,
|
|
284
|
-
|
|
284
|
+
mediaType: 'sqlite' as const,
|
|
285
285
|
scopeDigest: digest,
|
|
286
|
-
|
|
287
|
-
}
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
rowCount
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
286
|
+
asOfCommitSeq: asOf,
|
|
287
|
+
};
|
|
288
|
+
const key = JSON.stringify(identity);
|
|
289
|
+
let building = builds.get(key);
|
|
290
|
+
let builtHere = false;
|
|
291
|
+
if (building === undefined) {
|
|
292
|
+
building = (async () => {
|
|
293
|
+
// Another request can finish while this one's eligibility probe awaits.
|
|
294
|
+
const cached = await segments.find(identity, clockOf(ctx)());
|
|
295
|
+
if (cached !== undefined) return cached;
|
|
296
|
+
builtHere = true;
|
|
297
|
+
let rowCount = 0;
|
|
298
|
+
const bytes = await buildImage({
|
|
299
|
+
table: plan.table,
|
|
300
|
+
schemaVersion: schema.version,
|
|
301
|
+
asOfCommitSeq: asOf,
|
|
302
|
+
scopeDigest: digest,
|
|
303
|
+
rowBatches: (async function* () {
|
|
304
|
+
rowCount += probe.length;
|
|
305
|
+
yield probe;
|
|
306
|
+
let afterRowId = probe[probe.length - 1]!.rowId;
|
|
307
|
+
for (;;) {
|
|
308
|
+
const rows = await storage.scanRows(partition, {
|
|
309
|
+
table: plan.table.name,
|
|
310
|
+
scopeFilter: plan.effective,
|
|
311
|
+
afterRowId,
|
|
312
|
+
limit: 5_000,
|
|
313
|
+
});
|
|
314
|
+
rowCount += rows.length;
|
|
315
|
+
yield rows;
|
|
316
|
+
const last = rows[rows.length - 1];
|
|
317
|
+
if (rows.length < 5_000 || last === undefined) break;
|
|
318
|
+
afterRowId = last.rowId;
|
|
319
|
+
}
|
|
320
|
+
})(),
|
|
321
|
+
});
|
|
322
|
+
return segments.put(
|
|
323
|
+
{ ...identity, rowCount, rowCursor: null, nextRowCursor: null },
|
|
324
|
+
bytes,
|
|
325
|
+
clockOf(ctx)(),
|
|
326
|
+
);
|
|
327
|
+
})();
|
|
328
|
+
builds.set(key, building);
|
|
329
|
+
}
|
|
330
|
+
let record: SegmentRecord;
|
|
331
|
+
try {
|
|
332
|
+
record = await building;
|
|
333
|
+
} finally {
|
|
334
|
+
if (builds.get(key) === building) builds.delete(key);
|
|
335
|
+
}
|
|
304
336
|
trace?.segments.push({
|
|
305
337
|
mediaType: 'sqlite',
|
|
306
338
|
delivery: 'ref',
|
|
307
|
-
origin: 'built',
|
|
339
|
+
origin: builtHere ? 'built' : 'reused',
|
|
308
340
|
bytes: record.byteLength,
|
|
309
341
|
rows: record.rowCount,
|
|
310
342
|
});
|
|
311
343
|
yield segmentRefFrame(
|
|
312
344
|
record,
|
|
313
|
-
await signedUrlFields(
|
|
345
|
+
await signedUrlFields(
|
|
346
|
+
ctx,
|
|
347
|
+
limits,
|
|
348
|
+
record.segmentId,
|
|
349
|
+
digest,
|
|
350
|
+
clockOf(ctx)(),
|
|
351
|
+
),
|
|
314
352
|
);
|
|
315
353
|
return true;
|
|
316
354
|
}
|
|
@@ -463,6 +501,28 @@ export async function* subscriptionSection(
|
|
|
463
501
|
|
|
464
502
|
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
465
503
|
|
|
504
|
+
let commits: StoredCommit[] = [];
|
|
505
|
+
if (
|
|
506
|
+
token === undefined &&
|
|
507
|
+
sub.cursor >= 0 &&
|
|
508
|
+
sub.cursor >= horizonSeq &&
|
|
509
|
+
sub.cursor <= maxSeq
|
|
510
|
+
) {
|
|
511
|
+
commits = await ctx.storage.readCommitWindow(ctx.partition, {
|
|
512
|
+
table: sub.table,
|
|
513
|
+
scopeFilter: plan.effective,
|
|
514
|
+
afterSeq: sub.cursor,
|
|
515
|
+
throughSeq: maxSeq,
|
|
516
|
+
limitChanges: limits.limitCommits + 1,
|
|
517
|
+
});
|
|
518
|
+
// Validate continuity before committing to an active section. A prune
|
|
519
|
+
// during a paged read can otherwise make an incomplete window look empty.
|
|
520
|
+
horizonSeq = Math.max(
|
|
521
|
+
horizonSeq,
|
|
522
|
+
await ctx.storage.getHorizonSeq(ctx.partition),
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
|
|
466
526
|
// §4.6: a cursor behind the horizon (and not resuming a bootstrap)
|
|
467
527
|
// cannot compute deltas — answer `reset` and echo the cursor.
|
|
468
528
|
if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
|
|
@@ -536,13 +596,6 @@ export async function* subscriptionSection(
|
|
|
536
596
|
effectiveScopes: plan.effective,
|
|
537
597
|
bootstrap: false,
|
|
538
598
|
};
|
|
539
|
-
const commits = await ctx.storage.readCommitWindow(ctx.partition, {
|
|
540
|
-
table: sub.table,
|
|
541
|
-
scopeFilter: plan.effective,
|
|
542
|
-
afterSeq: sub.cursor,
|
|
543
|
-
throughSeq: maxSeq,
|
|
544
|
-
limitChanges: limits.limitCommits + 1,
|
|
545
|
-
});
|
|
546
599
|
let delivered = 0;
|
|
547
600
|
let deliveredCommits = 0;
|
|
548
601
|
let lastDeliveredSeq = sub.cursor;
|
package/src/push.ts
CHANGED
|
@@ -185,7 +185,7 @@ async function runValidator(
|
|
|
185
185
|
return errorRecord(
|
|
186
186
|
opIndex,
|
|
187
187
|
'sync.constraint_violation',
|
|
188
|
-
|
|
188
|
+
'write validator failed',
|
|
189
189
|
);
|
|
190
190
|
}
|
|
191
191
|
return undefined;
|
|
@@ -220,7 +220,7 @@ async function mergeCrdtColumns(
|
|
|
220
220
|
return errorRecord(
|
|
221
221
|
opIndex,
|
|
222
222
|
'sync.crdt_merge_failed',
|
|
223
|
-
|
|
223
|
+
'no CRDT merger registered',
|
|
224
224
|
);
|
|
225
225
|
}
|
|
226
226
|
const storedRaw = storedValues?.[index];
|
|
@@ -228,11 +228,11 @@ async function mergeCrdtColumns(
|
|
|
228
228
|
let merged: Uint8Array;
|
|
229
229
|
try {
|
|
230
230
|
merged = await merger(stored, incoming);
|
|
231
|
-
} catch
|
|
231
|
+
} catch {
|
|
232
232
|
return errorRecord(
|
|
233
233
|
opIndex,
|
|
234
234
|
'sync.crdt_merge_failed',
|
|
235
|
-
|
|
235
|
+
'CRDT merger failed',
|
|
236
236
|
);
|
|
237
237
|
}
|
|
238
238
|
values[index] = merged;
|
|
@@ -732,7 +732,7 @@ async function runCommitValidator(
|
|
|
732
732
|
return errorRecord(
|
|
733
733
|
operations[0]?.opIndex ?? 0,
|
|
734
734
|
'sync.constraint_violation',
|
|
735
|
-
|
|
735
|
+
'whole-commit validator failed',
|
|
736
736
|
);
|
|
737
737
|
}
|
|
738
738
|
return undefined;
|