@syncular/server 0.15.47 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -5
- package/dist/admin.d.ts +1 -5
- package/dist/admin.js +3 -12
- package/dist/authoritative-query.d.ts +14 -6
- package/dist/authoritative-query.js +60 -82
- package/dist/blob-handlers.js +4 -1
- package/dist/context.d.ts +3 -1
- package/dist/context.js +4 -0
- package/dist/d1-storage.d.ts +8 -2
- package/dist/d1-storage.js +134 -26
- package/dist/errors.js +6 -0
- package/dist/events.d.ts +2 -1
- package/dist/frame-bytes.d.ts +2 -2
- package/dist/frame-bytes.js +33 -14
- package/dist/handler.js +58 -14
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/operations.d.ts +2 -0
- package/dist/operations.js +25 -5
- package/dist/postgres-storage.d.ts +9 -3
- package/dist/postgres-storage.js +110 -20
- package/dist/prune.d.ts +3 -1
- package/dist/prune.js +18 -14
- package/dist/pull.d.ts +1 -1
- package/dist/pull.js +74 -37
- package/dist/push.js +5 -5
- package/dist/realtime.d.ts +4 -1
- package/dist/realtime.js +33 -9
- package/dist/restore.d.ts +13 -0
- package/dist/restore.js +13 -0
- package/dist/s3-segment-store.js +10 -1
- package/dist/seed.js +36 -4
- package/dist/segment-download.js +5 -2
- package/dist/segment-store.d.ts +3 -0
- package/dist/segment-store.js +1 -0
- package/dist/sqlite-bun-driver.d.ts +2 -1
- package/dist/sqlite-bun-driver.js +5 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-dialect.d.ts +1 -1
- package/dist/sqlite-dialect.js +7 -0
- package/dist/sqlite-image.d.ts +3 -3
- package/dist/sqlite-image.js +10 -6
- package/dist/sqlite-node.js +2 -2
- package/dist/sqlite-segment-store.js +14 -5
- package/dist/sqlite-storage.d.ts +8 -2
- package/dist/sqlite-storage.js +147 -36
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +3 -0
- package/dist/storage.d.ts +38 -10
- package/package.json +2 -2
- package/src/admin.ts +7 -15
- package/src/authoritative-query.ts +89 -94
- package/src/blob-handlers.ts +8 -1
- package/src/context.ts +16 -1
- package/src/d1-storage.ts +193 -29
- package/src/errors.ts +6 -0
- package/src/events.ts +2 -1
- package/src/frame-bytes.ts +40 -14
- package/src/handler.ts +102 -29
- package/src/index.ts +1 -0
- package/src/operations.ts +37 -4
- package/src/postgres-storage.ts +184 -38
- package/src/prune.ts +29 -15
- package/src/pull.ts +90 -41
- package/src/push.ts +5 -5
- package/src/realtime.ts +46 -7
- package/src/restore.ts +28 -0
- package/src/s3-segment-store.ts +10 -1
- package/src/seed.ts +46 -4
- package/src/segment-download.ts +11 -2
- package/src/segment-store.ts +4 -0
- package/src/sqlite-bun-driver.ts +6 -2
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-dialect.ts +7 -0
- package/src/sqlite-image.ts +26 -15
- package/src/sqlite-node.ts +2 -2
- package/src/sqlite-segment-store.ts +18 -4
- package/src/sqlite-storage.ts +203 -39
- package/src/storage-errors.ts +10 -1
- package/src/storage.ts +57 -10
package/src/postgres-storage.ts
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { validateCommitPruneQuery } from './prune';
|
|
2
|
+
import { StorageQueryError } from './storage-errors';
|
|
3
|
+
import type { CommitPruneQuery, CommitPruneResult } from './storage';
|
|
1
4
|
/**
|
|
2
5
|
* Postgres server storage: the production database path.
|
|
3
6
|
*
|
|
@@ -85,6 +88,7 @@ import type {
|
|
|
85
88
|
IndexRowScanQuery,
|
|
86
89
|
NewCommit,
|
|
87
90
|
NewReaction,
|
|
91
|
+
PartitionRegistryEntry,
|
|
88
92
|
PrunedReactionCounts,
|
|
89
93
|
ReactionClaimQuery,
|
|
90
94
|
ReactionFailure,
|
|
@@ -136,6 +140,12 @@ CREATE TABLE IF NOT EXISTS sync_partitions(
|
|
|
136
140
|
max_commit_seq BIGINT NOT NULL DEFAULT 0,
|
|
137
141
|
horizon_seq BIGINT NOT NULL DEFAULT 0
|
|
138
142
|
);
|
|
143
|
+
CREATE TABLE IF NOT EXISTS sync_partition_registry(
|
|
144
|
+
partition TEXT PRIMARY KEY,
|
|
145
|
+
log_epoch TEXT NOT NULL,
|
|
146
|
+
epoch_required BOOLEAN NOT NULL DEFAULT FALSE,
|
|
147
|
+
last_authenticated_at_ms BIGINT NOT NULL
|
|
148
|
+
);
|
|
139
149
|
CREATE TABLE IF NOT EXISTS sync_row_scopes(
|
|
140
150
|
partition TEXT NOT NULL, tbl TEXT NOT NULL,
|
|
141
151
|
var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
@@ -189,10 +199,13 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
|
|
|
189
199
|
ON sync_reactions(partition, status, available_at_ms, idempotency_key);
|
|
190
200
|
CREATE TABLE IF NOT EXISTS sync_clients(
|
|
191
201
|
partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
|
|
202
|
+
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
192
203
|
cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,
|
|
193
204
|
updated_at_ms BIGINT NOT NULL,
|
|
194
205
|
PRIMARY KEY(partition, client_id)
|
|
195
206
|
);
|
|
207
|
+
ALTER TABLE sync_clients
|
|
208
|
+
ADD COLUMN IF NOT EXISTS wire_version INTEGER NOT NULL DEFAULT 1;
|
|
196
209
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
197
210
|
partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
198
211
|
blob_id TEXT NOT NULL,
|
|
@@ -213,6 +226,21 @@ interface SerializedResult {
|
|
|
213
226
|
details?: import('@syncular/core').RejectionDetails;
|
|
214
227
|
}
|
|
215
228
|
|
|
229
|
+
async function lockPartitionOn(
|
|
230
|
+
client: PgQueryable,
|
|
231
|
+
partition: string,
|
|
232
|
+
): Promise<void> {
|
|
233
|
+
await client.query(
|
|
234
|
+
`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
|
|
235
|
+
ON CONFLICT (partition) DO NOTHING`,
|
|
236
|
+
[partition],
|
|
237
|
+
);
|
|
238
|
+
await client.query(
|
|
239
|
+
'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
|
|
240
|
+
[partition],
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
|
|
216
244
|
function toBase64(bytes: Uint8Array): string {
|
|
217
245
|
return Buffer.from(bytes).toString('base64');
|
|
218
246
|
}
|
|
@@ -639,15 +667,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
639
667
|
|
|
640
668
|
async lockPartitionForPush(): Promise<void> {
|
|
641
669
|
this.#assertOpen();
|
|
642
|
-
await this.#client
|
|
643
|
-
`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
|
|
644
|
-
ON CONFLICT (partition) DO NOTHING`,
|
|
645
|
-
[this.#partition],
|
|
646
|
-
);
|
|
647
|
-
await this.#client.query(
|
|
648
|
-
'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
|
|
649
|
-
[this.#partition],
|
|
650
|
-
);
|
|
670
|
+
await lockPartitionOn(this.#client, this.#partition);
|
|
651
671
|
await this.#client.query('SAVEPOINT syncular_push_candidate');
|
|
652
672
|
this.#pushApplySavepoint = true;
|
|
653
673
|
}
|
|
@@ -998,6 +1018,87 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
998
1018
|
this.#schemaVersion = schema.version;
|
|
999
1019
|
}
|
|
1000
1020
|
|
|
1021
|
+
async touchPartition(
|
|
1022
|
+
partition: string,
|
|
1023
|
+
authenticatedAtMs: number,
|
|
1024
|
+
initialLogEpoch: string,
|
|
1025
|
+
): Promise<PartitionRegistryEntry> {
|
|
1026
|
+
if (initialLogEpoch.length === 0) {
|
|
1027
|
+
throw new Error('initial log epoch must be non-empty');
|
|
1028
|
+
}
|
|
1029
|
+
const { rows } = await this.#exec.query<{
|
|
1030
|
+
log_epoch: string;
|
|
1031
|
+
epoch_required: boolean;
|
|
1032
|
+
last_authenticated_at_ms: unknown;
|
|
1033
|
+
}>(
|
|
1034
|
+
`INSERT INTO sync_partition_registry(
|
|
1035
|
+
partition, log_epoch, last_authenticated_at_ms
|
|
1036
|
+
) VALUES ($1,$2,$3)
|
|
1037
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
1038
|
+
last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms
|
|
1039
|
+
RETURNING log_epoch, epoch_required, last_authenticated_at_ms`,
|
|
1040
|
+
[partition, initialLogEpoch, authenticatedAtMs],
|
|
1041
|
+
);
|
|
1042
|
+
const row = rows[0];
|
|
1043
|
+
if (row === undefined)
|
|
1044
|
+
throw new Error('partition registry write did not persist');
|
|
1045
|
+
return {
|
|
1046
|
+
partition,
|
|
1047
|
+
logEpoch: row.log_epoch,
|
|
1048
|
+
epochRequired: row.epoch_required,
|
|
1049
|
+
lastAuthenticatedAtMs: asNumber(row.last_authenticated_at_ms),
|
|
1050
|
+
};
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
async rotatePartitionLogEpoch(
|
|
1054
|
+
partition: string,
|
|
1055
|
+
logEpoch: string,
|
|
1056
|
+
authenticatedAtMs: number,
|
|
1057
|
+
): Promise<PartitionRegistryEntry> {
|
|
1058
|
+
if (logEpoch.length === 0) throw new Error('log epoch must be non-empty');
|
|
1059
|
+
await this.#exec.transaction(async (client) => {
|
|
1060
|
+
await lockPartitionOn(client, partition);
|
|
1061
|
+
await client.query(
|
|
1062
|
+
`INSERT INTO sync_partition_registry(
|
|
1063
|
+
partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
1064
|
+
) VALUES ($1,$2,TRUE,$3)
|
|
1065
|
+
ON CONFLICT(partition) DO UPDATE SET
|
|
1066
|
+
log_epoch=EXCLUDED.log_epoch,
|
|
1067
|
+
epoch_required=TRUE,
|
|
1068
|
+
last_authenticated_at_ms=EXCLUDED.last_authenticated_at_ms`,
|
|
1069
|
+
[partition, logEpoch, authenticatedAtMs],
|
|
1070
|
+
);
|
|
1071
|
+
await client.query('DELETE FROM sync_clients WHERE partition=$1', [
|
|
1072
|
+
partition,
|
|
1073
|
+
]);
|
|
1074
|
+
});
|
|
1075
|
+
return {
|
|
1076
|
+
partition,
|
|
1077
|
+
logEpoch,
|
|
1078
|
+
epochRequired: true,
|
|
1079
|
+
lastAuthenticatedAtMs: authenticatedAtMs,
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
async listPartitionRegistry(): Promise<PartitionRegistryEntry[]> {
|
|
1084
|
+
const { rows } = await this.#exec.query<{
|
|
1085
|
+
partition: string;
|
|
1086
|
+
log_epoch: string;
|
|
1087
|
+
epoch_required: boolean;
|
|
1088
|
+
last_authenticated_at_ms: unknown;
|
|
1089
|
+
}>(
|
|
1090
|
+
`SELECT partition, log_epoch, epoch_required, last_authenticated_at_ms
|
|
1091
|
+
FROM sync_partition_registry ORDER BY partition`,
|
|
1092
|
+
[],
|
|
1093
|
+
);
|
|
1094
|
+
return rows.map((row) => ({
|
|
1095
|
+
partition: row.partition,
|
|
1096
|
+
logEpoch: row.log_epoch,
|
|
1097
|
+
epochRequired: row.epoch_required,
|
|
1098
|
+
lastAuthenticatedAtMs: asNumber(row.last_authenticated_at_ms),
|
|
1099
|
+
}));
|
|
1100
|
+
}
|
|
1101
|
+
|
|
1001
1102
|
/**
|
|
1002
1103
|
* Open a real Postgres transaction. The push handler drives the returned
|
|
1003
1104
|
* `StorageTransaction` imperatively (getRow/upsert/…/commit), but the
|
|
@@ -1058,7 +1159,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1058
1159
|
}
|
|
1059
1160
|
const prepared = bindAuthoritativePartition(
|
|
1060
1161
|
prepareAuthoritativeQuery(
|
|
1061
|
-
query.
|
|
1162
|
+
query.plan,
|
|
1062
1163
|
query.params,
|
|
1063
1164
|
query.tables,
|
|
1064
1165
|
this.#tables,
|
|
@@ -1087,6 +1188,14 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1087
1188
|
});
|
|
1088
1189
|
}
|
|
1089
1190
|
|
|
1191
|
+
async getPartitionLogEpoch(partition: string): Promise<string | undefined> {
|
|
1192
|
+
const { rows } = await this.#exec.query<{ log_epoch: string }>(
|
|
1193
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1',
|
|
1194
|
+
[partition],
|
|
1195
|
+
);
|
|
1196
|
+
return rows[0]?.log_epoch;
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1090
1199
|
async getHorizonSeq(partition: string): Promise<number> {
|
|
1091
1200
|
const { rows } = await this.#exec.query<{ horizon_seq: unknown }>(
|
|
1092
1201
|
'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
|
|
@@ -1098,25 +1207,52 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1098
1207
|
async setHorizonSeq(partition: string, seq: number): Promise<void> {
|
|
1099
1208
|
await this.#exec.query(
|
|
1100
1209
|
`INSERT INTO sync_partitions(partition, horizon_seq) VALUES ($1,$2)
|
|
1101
|
-
ON CONFLICT (partition) DO UPDATE SET horizon_seq=EXCLUDED.horizon_seq`,
|
|
1210
|
+
ON CONFLICT (partition) DO UPDATE SET horizon_seq=GREATEST(sync_partitions.horizon_seq,EXCLUDED.horizon_seq)`,
|
|
1102
1211
|
[partition, seq],
|
|
1103
1212
|
);
|
|
1104
1213
|
}
|
|
1105
1214
|
|
|
1106
|
-
async pruneCommitsThrough(
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
);
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
[
|
|
1118
|
-
|
|
1119
|
-
|
|
1215
|
+
async pruneCommitsThrough(
|
|
1216
|
+
partition: string,
|
|
1217
|
+
query: CommitPruneQuery,
|
|
1218
|
+
): Promise<CommitPruneResult> {
|
|
1219
|
+
validateCommitPruneQuery(query);
|
|
1220
|
+
return this.#exec.transaction(async (client) => {
|
|
1221
|
+
await lockPartitionOn(client, partition);
|
|
1222
|
+
const epoch = await client.query<{ log_epoch: string }>(
|
|
1223
|
+
'SELECT log_epoch FROM sync_partition_registry WHERE partition=$1 FOR UPDATE',
|
|
1224
|
+
[partition],
|
|
1225
|
+
);
|
|
1226
|
+
if (epoch.rows[0]?.log_epoch !== query.logEpoch)
|
|
1227
|
+
throw new StorageQueryError('sync.storage.prune_epoch_mismatch');
|
|
1228
|
+
const previous = await client.query<{ horizon_seq: unknown }>(
|
|
1229
|
+
'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
|
|
1230
|
+
[partition],
|
|
1231
|
+
);
|
|
1232
|
+
const previousHorizonSeq = asNumber(previous.rows[0]?.horizon_seq);
|
|
1233
|
+
const horizonSeq = Math.max(previousHorizonSeq, query.throughSeq);
|
|
1234
|
+
await client.query(
|
|
1235
|
+
'UPDATE sync_partitions SET horizon_seq=$2 WHERE partition=$1',
|
|
1236
|
+
[partition, horizonSeq],
|
|
1237
|
+
);
|
|
1238
|
+
const removed = await client.query(
|
|
1239
|
+
'DELETE FROM sync_commits WHERE partition=$1 AND commit_seq<=$2',
|
|
1240
|
+
[partition, horizonSeq],
|
|
1241
|
+
);
|
|
1242
|
+
await client.query(
|
|
1243
|
+
'DELETE FROM sync_changes WHERE partition=$1 AND commit_seq<=$2',
|
|
1244
|
+
[partition, horizonSeq],
|
|
1245
|
+
);
|
|
1246
|
+
await client.query(
|
|
1247
|
+
'DELETE FROM sync_change_scopes WHERE partition=$1 AND commit_seq<=$2',
|
|
1248
|
+
[partition, horizonSeq],
|
|
1249
|
+
);
|
|
1250
|
+
return {
|
|
1251
|
+
previousHorizonSeq,
|
|
1252
|
+
horizonSeq,
|
|
1253
|
+
removedCommits: removed.rowCount,
|
|
1254
|
+
};
|
|
1255
|
+
});
|
|
1120
1256
|
}
|
|
1121
1257
|
|
|
1122
1258
|
async getCommitSeqBefore(
|
|
@@ -1496,11 +1632,12 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1496
1632
|
const { rows } = await this.#exec.query<{
|
|
1497
1633
|
client_id: string;
|
|
1498
1634
|
actor_id: string;
|
|
1635
|
+
wire_version: unknown;
|
|
1499
1636
|
cursor: unknown;
|
|
1500
1637
|
subscriptions: unknown;
|
|
1501
1638
|
updated_at_ms: unknown;
|
|
1502
1639
|
}>(
|
|
1503
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2',
|
|
1640
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 AND client_id=$2',
|
|
1504
1641
|
[partition, clientId],
|
|
1505
1642
|
);
|
|
1506
1643
|
const record = rows[0];
|
|
@@ -1508,6 +1645,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1508
1645
|
return {
|
|
1509
1646
|
clientId: record.client_id,
|
|
1510
1647
|
actorId: record.actor_id,
|
|
1648
|
+
wireVersion: asNumber(record.wire_version),
|
|
1511
1649
|
cursor: asNumber(record.cursor),
|
|
1512
1650
|
updatedAtMs: asNumber(record.updated_at_ms),
|
|
1513
1651
|
subscriptions: asJson<ClientSubscription[]>(record.subscriptions),
|
|
@@ -1519,16 +1657,18 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1519
1657
|
record: ClientRecord,
|
|
1520
1658
|
): Promise<void> {
|
|
1521
1659
|
await this.#exec.query(
|
|
1522
|
-
`INSERT INTO sync_clients(partition, client_id, actor_id, cursor, subscriptions, updated_at_ms)
|
|
1523
|
-
VALUES ($1,$2,$3,$4,$5,$6)
|
|
1660
|
+
`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms)
|
|
1661
|
+
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
|
1524
1662
|
ON CONFLICT (partition, client_id) DO UPDATE
|
|
1525
|
-
SET actor_id=EXCLUDED.actor_id,
|
|
1663
|
+
SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
|
|
1664
|
+
cursor=EXCLUDED.cursor,
|
|
1526
1665
|
subscriptions=EXCLUDED.subscriptions,
|
|
1527
1666
|
updated_at_ms=EXCLUDED.updated_at_ms`,
|
|
1528
1667
|
[
|
|
1529
1668
|
partition,
|
|
1530
1669
|
record.clientId,
|
|
1531
1670
|
record.actorId,
|
|
1671
|
+
record.wireVersion,
|
|
1532
1672
|
record.cursor,
|
|
1533
1673
|
JSON.stringify(record.subscriptions),
|
|
1534
1674
|
record.updatedAtMs,
|
|
@@ -1536,6 +1676,17 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1536
1676
|
);
|
|
1537
1677
|
}
|
|
1538
1678
|
|
|
1679
|
+
async getActiveClientCursorFloor(
|
|
1680
|
+
partition: string,
|
|
1681
|
+
cutoffMs: number,
|
|
1682
|
+
): Promise<number | null> {
|
|
1683
|
+
const { rows } = await this.#exec.query<{ cursor: unknown }>(
|
|
1684
|
+
'SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2',
|
|
1685
|
+
[partition, cutoffMs],
|
|
1686
|
+
);
|
|
1687
|
+
return rows[0]!.cursor === null ? null : asNumber(rows[0]!.cursor);
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1539
1690
|
async listClientCursors(partition: string): Promise<ClientCursorInfo[]> {
|
|
1540
1691
|
const { rows } = await this.#exec.query<{
|
|
1541
1692
|
client_id: string;
|
|
@@ -1608,16 +1759,18 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1608
1759
|
const { rows } = await this.#exec.query<{
|
|
1609
1760
|
client_id: string;
|
|
1610
1761
|
actor_id: string;
|
|
1762
|
+
wire_version: unknown;
|
|
1611
1763
|
cursor: unknown;
|
|
1612
1764
|
subscriptions: unknown;
|
|
1613
1765
|
updated_at_ms: unknown;
|
|
1614
1766
|
}>(
|
|
1615
|
-
'SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
|
|
1767
|
+
'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC',
|
|
1616
1768
|
[partition],
|
|
1617
1769
|
);
|
|
1618
1770
|
return rows.map((r) => ({
|
|
1619
1771
|
clientId: r.client_id,
|
|
1620
1772
|
actorId: r.actor_id,
|
|
1773
|
+
wireVersion: asNumber(r.wire_version),
|
|
1621
1774
|
cursor: asNumber(r.cursor),
|
|
1622
1775
|
updatedAtMs: asNumber(r.updated_at_ms),
|
|
1623
1776
|
subscriptions: (typeof r.subscriptions === 'string'
|
|
@@ -1742,13 +1895,6 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1742
1895
|
}
|
|
1743
1896
|
|
|
1744
1897
|
async listPartitions(): Promise<string[]> {
|
|
1745
|
-
|
|
1746
|
-
// first pull — a partition with only one of the two still shows up.
|
|
1747
|
-
const { rows } = await this.#exec.query<{ partition: string }>(
|
|
1748
|
-
`SELECT partition FROM sync_partitions
|
|
1749
|
-
UNION SELECT partition FROM sync_clients ORDER BY partition`,
|
|
1750
|
-
[],
|
|
1751
|
-
);
|
|
1752
|
-
return rows.map((r) => r.partition);
|
|
1898
|
+
return (await this.listPartitionRegistry()).map((entry) => entry.partition);
|
|
1753
1899
|
}
|
|
1754
1900
|
}
|
package/src/prune.ts
CHANGED
|
@@ -7,7 +7,20 @@
|
|
|
7
7
|
* least the newest `minRetainedCommits` commits are always retained.
|
|
8
8
|
*/
|
|
9
9
|
import { emitEvent, type SyncularServerEvents } from './events';
|
|
10
|
-
import type { ServerStorage } from './storage';
|
|
10
|
+
import type { CommitPruneQuery, ServerStorage } from './storage';
|
|
11
|
+
import { StorageQueryError } from './storage-errors';
|
|
12
|
+
|
|
13
|
+
/** Shared validation for the built-in atomic pruning adapters. */
|
|
14
|
+
export function validateCommitPruneQuery(query: CommitPruneQuery): void {
|
|
15
|
+
if (
|
|
16
|
+
!Number.isSafeInteger(query.throughSeq) ||
|
|
17
|
+
query.throughSeq < 0 ||
|
|
18
|
+
typeof query.logEpoch !== 'string' ||
|
|
19
|
+
query.logEpoch.length === 0
|
|
20
|
+
) {
|
|
21
|
+
throw new StorageQueryError('sync.storage.invalid_prune_cursor');
|
|
22
|
+
}
|
|
23
|
+
}
|
|
11
24
|
|
|
12
25
|
export interface RetentionPolicy {
|
|
13
26
|
/** Active window for laggard cursors (default 14 days). */
|
|
@@ -37,28 +50,29 @@ export interface PruneOptions {
|
|
|
37
50
|
export async function pruneCommitLog(options: PruneOptions): Promise<number> {
|
|
38
51
|
const { storage, partition, nowMs } = options;
|
|
39
52
|
const policy = { ...DEFAULT_RETENTION, ...options.retention };
|
|
53
|
+
const logEpoch = await storage.getPartitionLogEpoch(partition);
|
|
54
|
+
if (logEpoch === undefined)
|
|
55
|
+
throw new StorageQueryError('sync.storage.partition_unregistered');
|
|
40
56
|
const maxSeq = await storage.getMaxCommitSeq(partition);
|
|
41
|
-
const cursors = await storage.listClientCursors(partition);
|
|
42
|
-
const activeCursors = cursors
|
|
43
|
-
.filter((c) => c.updatedAtMs >= nowMs - policy.activeWindowMs)
|
|
44
|
-
.map((c) => c.cursor);
|
|
45
57
|
const cursorFloor =
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
58
|
+
(await storage.getActiveClientCursorFloor(
|
|
59
|
+
partition,
|
|
60
|
+
nowMs - policy.activeWindowMs,
|
|
61
|
+
)) ?? Number.MAX_SAFE_INTEGER;
|
|
49
62
|
const forcedSeq = await storage.getCommitSeqBefore(
|
|
50
63
|
partition,
|
|
51
64
|
nowMs - policy.ageForceMs,
|
|
52
65
|
);
|
|
53
66
|
const retainFloor = maxSeq - policy.minRetainedCommits;
|
|
54
67
|
const target = Math.min(Math.max(cursorFloor, forcedSeq), retainFloor);
|
|
55
|
-
const
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
68
|
+
const {
|
|
69
|
+
previousHorizonSeq: current,
|
|
70
|
+
horizonSeq: horizon,
|
|
71
|
+
removedCommits,
|
|
72
|
+
} = await storage.pruneCommitsThrough(partition, {
|
|
73
|
+
logEpoch,
|
|
74
|
+
throughSeq: Math.max(0, target),
|
|
75
|
+
});
|
|
62
76
|
const events = options.events;
|
|
63
77
|
if (events !== undefined) {
|
|
64
78
|
emitEvent(events, {
|
package/src/pull.ts
CHANGED
|
@@ -17,10 +17,16 @@ import { clockOf, limitsOf } from './context';
|
|
|
17
17
|
import type { PullSegmentSummary } from './events';
|
|
18
18
|
import type { CompiledSchema, CompiledTable } from './schema';
|
|
19
19
|
import { scopeDigest } from './scopes';
|
|
20
|
-
import type { SegmentRecord } from './segment-store';
|
|
20
|
+
import type { SegmentRecord, SegmentStore } from './segment-store';
|
|
21
21
|
import { issueSegmentUrl } from './signed-url';
|
|
22
22
|
import type { SqliteImageBuilder } from './sqlite-image';
|
|
23
|
-
import type { StoredCommit, StoredRow } from './storage';
|
|
23
|
+
import type { ServerStorage, StoredCommit, StoredRow } from './storage';
|
|
24
|
+
|
|
25
|
+
// One artifact build per owning storage pair and complete immutable identity.
|
|
26
|
+
const imageBuilds = new WeakMap<
|
|
27
|
+
ServerStorage,
|
|
28
|
+
WeakMap<SegmentStore, Map<string, Promise<SegmentRecord>>>
|
|
29
|
+
>();
|
|
24
30
|
|
|
25
31
|
/**
|
|
26
32
|
* Resolve the §5.3 image builder: the host-injected one if present, else the
|
|
@@ -212,12 +218,14 @@ async function* sqliteImageSegment(
|
|
|
212
218
|
asOf: number,
|
|
213
219
|
digest: string,
|
|
214
220
|
trace: PullSectionTrace | undefined,
|
|
221
|
+
logEpoch: string,
|
|
215
222
|
): AsyncGenerator<ResponseFrame, boolean> {
|
|
216
223
|
const { storage, segments, partition } = ctx;
|
|
217
224
|
const now = clockOf(ctx)();
|
|
218
225
|
const existing = await segments.find(
|
|
219
226
|
{
|
|
220
227
|
partition,
|
|
228
|
+
logEpoch,
|
|
221
229
|
table: plan.table.name,
|
|
222
230
|
schemaVersion: schema.version,
|
|
223
231
|
mediaType: 'sqlite',
|
|
@@ -258,56 +266,89 @@ async function* sqliteImageSegment(
|
|
|
258
266
|
// support floor, not a fallback (§5.3: sqlite is an *accept*, not a demand).
|
|
259
267
|
const buildImage = await resolveImageBuilder(ctx);
|
|
260
268
|
if (buildImage === undefined) return false;
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
const rows: StoredRow[] = [...probe];
|
|
266
|
-
let afterRowId: string | null = probe[probe.length - 1]?.rowId ?? null;
|
|
267
|
-
for (;;) {
|
|
268
|
-
const scanned: StoredRow[] = await storage.scanRows(partition, {
|
|
269
|
-
table: plan.table.name,
|
|
270
|
-
scopeFilter: plan.effective,
|
|
271
|
-
afterRowId,
|
|
272
|
-
limit: 50_000,
|
|
273
|
-
});
|
|
274
|
-
rows.push(...scanned);
|
|
275
|
-
const last = scanned[scanned.length - 1];
|
|
276
|
-
if (scanned.length < 50_000 || last === undefined) break;
|
|
277
|
-
afterRowId = last.rowId;
|
|
269
|
+
let stores = imageBuilds.get(storage);
|
|
270
|
+
if (stores === undefined) {
|
|
271
|
+
stores = new WeakMap();
|
|
272
|
+
imageBuilds.set(storage, stores);
|
|
278
273
|
}
|
|
279
|
-
|
|
280
|
-
|
|
274
|
+
let builds = stores.get(segments);
|
|
275
|
+
if (builds === undefined) {
|
|
276
|
+
builds = new Map();
|
|
277
|
+
stores.set(segments, builds);
|
|
278
|
+
}
|
|
279
|
+
const identity = {
|
|
280
|
+
partition,
|
|
281
|
+
logEpoch,
|
|
282
|
+
table: plan.table.name,
|
|
281
283
|
schemaVersion: schema.version,
|
|
282
|
-
|
|
284
|
+
mediaType: 'sqlite' as const,
|
|
283
285
|
scopeDigest: digest,
|
|
284
|
-
|
|
285
|
-
}
|
|
286
|
-
const
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
286
|
+
asOfCommitSeq: asOf,
|
|
287
|
+
};
|
|
288
|
+
const key = JSON.stringify(identity);
|
|
289
|
+
let building = builds.get(key);
|
|
290
|
+
let builtHere = false;
|
|
291
|
+
if (building === undefined) {
|
|
292
|
+
building = (async () => {
|
|
293
|
+
// Another request can finish while this one's eligibility probe awaits.
|
|
294
|
+
const cached = await segments.find(identity, clockOf(ctx)());
|
|
295
|
+
if (cached !== undefined) return cached;
|
|
296
|
+
builtHere = true;
|
|
297
|
+
let rowCount = 0;
|
|
298
|
+
const bytes = await buildImage({
|
|
299
|
+
table: plan.table,
|
|
300
|
+
schemaVersion: schema.version,
|
|
301
|
+
asOfCommitSeq: asOf,
|
|
302
|
+
scopeDigest: digest,
|
|
303
|
+
rowBatches: (async function* () {
|
|
304
|
+
rowCount += probe.length;
|
|
305
|
+
yield probe;
|
|
306
|
+
let afterRowId = probe[probe.length - 1]!.rowId;
|
|
307
|
+
for (;;) {
|
|
308
|
+
const rows = await storage.scanRows(partition, {
|
|
309
|
+
table: plan.table.name,
|
|
310
|
+
scopeFilter: plan.effective,
|
|
311
|
+
afterRowId,
|
|
312
|
+
limit: 5_000,
|
|
313
|
+
});
|
|
314
|
+
rowCount += rows.length;
|
|
315
|
+
yield rows;
|
|
316
|
+
const last = rows[rows.length - 1];
|
|
317
|
+
if (rows.length < 5_000 || last === undefined) break;
|
|
318
|
+
afterRowId = last.rowId;
|
|
319
|
+
}
|
|
320
|
+
})(),
|
|
321
|
+
});
|
|
322
|
+
return segments.put(
|
|
323
|
+
{ ...identity, rowCount, rowCursor: null, nextRowCursor: null },
|
|
324
|
+
bytes,
|
|
325
|
+
clockOf(ctx)(),
|
|
326
|
+
);
|
|
327
|
+
})();
|
|
328
|
+
builds.set(key, building);
|
|
329
|
+
}
|
|
330
|
+
let record: SegmentRecord;
|
|
331
|
+
try {
|
|
332
|
+
record = await building;
|
|
333
|
+
} finally {
|
|
334
|
+
if (builds.get(key) === building) builds.delete(key);
|
|
335
|
+
}
|
|
301
336
|
trace?.segments.push({
|
|
302
337
|
mediaType: 'sqlite',
|
|
303
338
|
delivery: 'ref',
|
|
304
|
-
origin: 'built',
|
|
339
|
+
origin: builtHere ? 'built' : 'reused',
|
|
305
340
|
bytes: record.byteLength,
|
|
306
341
|
rows: record.rowCount,
|
|
307
342
|
});
|
|
308
343
|
yield segmentRefFrame(
|
|
309
344
|
record,
|
|
310
|
-
await signedUrlFields(
|
|
345
|
+
await signedUrlFields(
|
|
346
|
+
ctx,
|
|
347
|
+
limits,
|
|
348
|
+
record.segmentId,
|
|
349
|
+
digest,
|
|
350
|
+
clockOf(ctx)(),
|
|
351
|
+
),
|
|
311
352
|
);
|
|
312
353
|
return true;
|
|
313
354
|
}
|
|
@@ -320,6 +361,7 @@ async function* bootstrapSegments(
|
|
|
320
361
|
asOf: number,
|
|
321
362
|
startRowCursor: string | null,
|
|
322
363
|
trace: PullSectionTrace | undefined,
|
|
364
|
+
logEpoch: string,
|
|
323
365
|
): AsyncGenerator<
|
|
324
366
|
ResponseFrame,
|
|
325
367
|
{ complete: boolean; rowCursor: string | null }
|
|
@@ -347,6 +389,7 @@ async function* bootstrapSegments(
|
|
|
347
389
|
asOf,
|
|
348
390
|
digest,
|
|
349
391
|
trace,
|
|
392
|
+
logEpoch,
|
|
350
393
|
);
|
|
351
394
|
if (imaged) return { complete: true, rowCursor: null };
|
|
352
395
|
}
|
|
@@ -393,6 +436,7 @@ async function* bootstrapSegments(
|
|
|
393
436
|
const record = await segments.put(
|
|
394
437
|
{
|
|
395
438
|
partition,
|
|
439
|
+
logEpoch,
|
|
396
440
|
table: plan.table.name,
|
|
397
441
|
schemaVersion: schema.version,
|
|
398
442
|
mediaType: 'rows',
|
|
@@ -435,8 +479,12 @@ export async function* subscriptionSection(
|
|
|
435
479
|
maxSeq: number,
|
|
436
480
|
horizonSeq: number,
|
|
437
481
|
trace?: PullSectionTrace,
|
|
482
|
+
logEpoch?: string,
|
|
438
483
|
): AsyncGenerator<ResponseFrame, SubscriptionResult> {
|
|
439
484
|
const sub = plan.frame;
|
|
485
|
+
if (logEpoch === undefined || logEpoch.length === 0) {
|
|
486
|
+
throw new Error('subscriptionSection requires a non-empty log epoch');
|
|
487
|
+
}
|
|
440
488
|
|
|
441
489
|
if (plan.status === 'revoked') {
|
|
442
490
|
yield {
|
|
@@ -496,6 +544,7 @@ export async function* subscriptionSection(
|
|
|
496
544
|
asOf,
|
|
497
545
|
startCursor,
|
|
498
546
|
trace,
|
|
547
|
+
logEpoch,
|
|
499
548
|
);
|
|
500
549
|
if (outcome.complete) {
|
|
501
550
|
yield { type: 'SUB_END', nextCursor: asOf };
|
package/src/push.ts
CHANGED
|
@@ -185,7 +185,7 @@ async function runValidator(
|
|
|
185
185
|
return errorRecord(
|
|
186
186
|
opIndex,
|
|
187
187
|
'sync.constraint_violation',
|
|
188
|
-
|
|
188
|
+
'write validator failed',
|
|
189
189
|
);
|
|
190
190
|
}
|
|
191
191
|
return undefined;
|
|
@@ -220,7 +220,7 @@ async function mergeCrdtColumns(
|
|
|
220
220
|
return errorRecord(
|
|
221
221
|
opIndex,
|
|
222
222
|
'sync.crdt_merge_failed',
|
|
223
|
-
|
|
223
|
+
'no CRDT merger registered',
|
|
224
224
|
);
|
|
225
225
|
}
|
|
226
226
|
const storedRaw = storedValues?.[index];
|
|
@@ -228,11 +228,11 @@ async function mergeCrdtColumns(
|
|
|
228
228
|
let merged: Uint8Array;
|
|
229
229
|
try {
|
|
230
230
|
merged = await merger(stored, incoming);
|
|
231
|
-
} catch
|
|
231
|
+
} catch {
|
|
232
232
|
return errorRecord(
|
|
233
233
|
opIndex,
|
|
234
234
|
'sync.crdt_merge_failed',
|
|
235
|
-
|
|
235
|
+
'CRDT merger failed',
|
|
236
236
|
);
|
|
237
237
|
}
|
|
238
238
|
values[index] = merged;
|
|
@@ -732,7 +732,7 @@ async function runCommitValidator(
|
|
|
732
732
|
return errorRecord(
|
|
733
733
|
operations[0]?.opIndex ?? 0,
|
|
734
734
|
'sync.constraint_violation',
|
|
735
|
-
|
|
735
|
+
'whole-commit validator failed',
|
|
736
736
|
);
|
|
737
737
|
}
|
|
738
738
|
return undefined;
|