@syncular/server 0.15.45 → 0.15.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +144 -7
- package/dist/admin.d.ts +10 -4
- package/dist/admin.js +10 -0
- package/dist/authoritative-query.d.ts +20 -0
- package/dist/authoritative-query.js +184 -0
- package/dist/context.d.ts +11 -1
- package/dist/context.js +2 -0
- package/dist/d1-storage.d.ts +10 -1
- package/dist/d1-storage.js +216 -0
- package/dist/errors.d.ts +1 -1
- package/dist/errors.js +43 -1
- package/dist/events.d.ts +52 -3
- package/dist/handler.js +4 -1
- package/dist/index-bun.d.ts +2 -0
- package/dist/index-bun.js +2 -0
- package/dist/index-node.d.ts +2 -0
- package/dist/index-node.js +2 -0
- package/dist/index.d.ts +7 -2
- package/dist/index.js +7 -6
- package/dist/operations-realtime.d.ts +16 -0
- package/dist/operations-realtime.js +196 -0
- package/dist/operations.d.ts +97 -0
- package/dist/operations.js +392 -0
- package/dist/postgres-storage.d.ts +11 -2
- package/dist/postgres-storage.js +220 -0
- package/dist/pull.js +1 -1
- package/dist/push.d.ts +8 -2
- package/dist/push.js +75 -21
- package/dist/reactions.d.ts +167 -0
- package/dist/reactions.js +442 -0
- package/dist/realtime.js +4 -1
- package/dist/sqlite-blob-store.d.ts +4 -9
- package/dist/sqlite-blob-store.js +5 -10
- package/dist/sqlite-bun-driver.d.ts +11 -0
- package/dist/sqlite-bun-driver.js +27 -0
- package/dist/sqlite-bun.d.ts +24 -0
- package/dist/sqlite-bun.js +40 -0
- package/dist/sqlite-dialect.d.ts +8 -8
- package/dist/sqlite-dialect.js +22 -2
- package/dist/sqlite-driver.d.ts +26 -0
- package/dist/sqlite-driver.js +8 -0
- package/dist/sqlite-image.d.ts +7 -9
- package/dist/sqlite-image.js +26 -28
- package/dist/sqlite-lease-store.d.ts +4 -9
- package/dist/sqlite-lease-store.js +5 -10
- package/dist/sqlite-node-driver.d.ts +10 -0
- package/dist/sqlite-node-driver.js +30 -0
- package/dist/sqlite-node.d.ts +24 -0
- package/dist/sqlite-node.js +50 -0
- package/dist/sqlite-segment-store.d.ts +4 -10
- package/dist/sqlite-segment-store.js +6 -9
- package/dist/sqlite-storage.d.ts +13 -12
- package/dist/sqlite-storage.js +223 -5
- package/dist/storage-errors.js +4 -1
- package/dist/storage.d.ts +109 -0
- package/dist/validate.js +1 -0
- package/package.json +18 -3
- package/src/admin.ts +27 -3
- package/src/authoritative-query.ts +218 -0
- package/src/context.ts +12 -1
- package/src/d1-storage.ts +352 -0
- package/src/errors.ts +43 -1
- package/src/events.ts +64 -2
- package/src/handler.ts +13 -1
- package/src/index-bun.ts +9 -0
- package/src/index-node.ts +9 -0
- package/src/index.ts +40 -6
- package/src/operations-realtime.ts +272 -0
- package/src/operations.ts +720 -0
- package/src/postgres-storage.ts +351 -0
- package/src/pull.ts +1 -1
- package/src/push.ts +97 -29
- package/src/reactions.ts +741 -0
- package/src/realtime.ts +7 -1
- package/src/sqlite-blob-store.ts +11 -10
- package/src/sqlite-bun-driver.ts +42 -0
- package/src/sqlite-bun.ts +53 -0
- package/src/sqlite-dialect.ts +27 -7
- package/src/sqlite-driver.ts +44 -0
- package/src/sqlite-image.ts +44 -49
- package/src/sqlite-lease-store.ts +11 -10
- package/src/sqlite-node-driver.ts +46 -0
- package/src/sqlite-node.ts +62 -0
- package/src/sqlite-segment-store.ts +11 -11
- package/src/sqlite-storage.ts +378 -7
- package/src/storage-errors.ts +4 -1
- package/src/storage.ts +165 -0
- package/src/validate.ts +1 -0
package/src/postgres-storage.ts
CHANGED
|
@@ -36,6 +36,11 @@
|
|
|
36
36
|
* does not tolerate). Cross-partition pushes never contend.
|
|
37
37
|
*/
|
|
38
38
|
import type { PushOperationResult } from '@syncular/core';
|
|
39
|
+
import {
|
|
40
|
+
bindAuthoritativePartition,
|
|
41
|
+
postgresPlaceholders,
|
|
42
|
+
prepareAuthoritativeQuery,
|
|
43
|
+
} from './authoritative-query';
|
|
39
44
|
import { syncError } from './errors';
|
|
40
45
|
import {
|
|
41
46
|
asBytes,
|
|
@@ -68,14 +73,24 @@ import {
|
|
|
68
73
|
import type { CompiledSchema, CompiledTable } from './schema';
|
|
69
74
|
import { matchesEffective } from './scopes';
|
|
70
75
|
import type {
|
|
76
|
+
AuthoritativeQueryRequest,
|
|
77
|
+
AuthoritativeQueryResult,
|
|
71
78
|
ClientCursorInfo,
|
|
72
79
|
ClientRecord,
|
|
73
80
|
ClientSubscription,
|
|
74
81
|
CommitMetadata,
|
|
75
82
|
CommitMetadataQuery,
|
|
76
83
|
CommitWindowQuery,
|
|
84
|
+
DurableJsonValue,
|
|
77
85
|
IndexRowScanQuery,
|
|
78
86
|
NewCommit,
|
|
87
|
+
NewReaction,
|
|
88
|
+
PrunedReactionCounts,
|
|
89
|
+
ReactionClaimQuery,
|
|
90
|
+
ReactionFailure,
|
|
91
|
+
ReactionFailureUpdate,
|
|
92
|
+
ReactionListQuery,
|
|
93
|
+
ReactionPruneQuery,
|
|
79
94
|
RowScanQuery,
|
|
80
95
|
ScopeActivityQuery,
|
|
81
96
|
ScopeCommitActivity,
|
|
@@ -84,6 +99,7 @@ import type {
|
|
|
84
99
|
StoredChange,
|
|
85
100
|
StoredCommit,
|
|
86
101
|
StoredPushResult,
|
|
102
|
+
StoredReaction,
|
|
87
103
|
StoredRow,
|
|
88
104
|
} from './storage';
|
|
89
105
|
import {
|
|
@@ -151,6 +167,26 @@ CREATE TABLE IF NOT EXISTS sync_push_results(
|
|
|
151
167
|
client_commit_id TEXT NOT NULL, result JSONB NOT NULL,
|
|
152
168
|
PRIMARY KEY(partition, client_id, client_commit_id)
|
|
153
169
|
);
|
|
170
|
+
CREATE TABLE IF NOT EXISTS sync_reactions(
|
|
171
|
+
partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,
|
|
172
|
+
type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,
|
|
173
|
+
source_client_id TEXT NOT NULL, source_client_commit_id TEXT NOT NULL,
|
|
174
|
+
source_commit_seq BIGINT NOT NULL, created_at_ms BIGINT NOT NULL,
|
|
175
|
+
available_at_ms BIGINT NOT NULL, status TEXT NOT NULL,
|
|
176
|
+
attempts INTEGER NOT NULL, max_attempts INTEGER NOT NULL,
|
|
177
|
+
lease_owner TEXT, lease_expires_at_ms BIGINT, completed_at_ms BIGINT,
|
|
178
|
+
last_failure JSONB,
|
|
179
|
+
PRIMARY KEY(partition, idempotency_key),
|
|
180
|
+
CHECK(status IN ('pending', 'leased', 'completed', 'dead-letter'))
|
|
181
|
+
);
|
|
182
|
+
CREATE INDEX IF NOT EXISTS sync_reactions_due
|
|
183
|
+
ON sync_reactions(partition, status, available_at_ms, created_at_ms, idempotency_key);
|
|
184
|
+
CREATE INDEX IF NOT EXISTS sync_reactions_lease
|
|
185
|
+
ON sync_reactions(partition, status, lease_expires_at_ms);
|
|
186
|
+
CREATE INDEX IF NOT EXISTS sync_reactions_completed
|
|
187
|
+
ON sync_reactions(partition, status, completed_at_ms, idempotency_key);
|
|
188
|
+
CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
|
|
189
|
+
ON sync_reactions(partition, status, available_at_ms, idempotency_key);
|
|
154
190
|
CREATE TABLE IF NOT EXISTS sync_clients(
|
|
155
191
|
partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
|
|
156
192
|
cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,
|
|
@@ -185,6 +221,52 @@ function fromBase64(text: string): Uint8Array {
|
|
|
185
221
|
return new Uint8Array(Buffer.from(text, 'base64'));
|
|
186
222
|
}
|
|
187
223
|
|
|
224
|
+
interface PostgresReactionRecord {
|
|
225
|
+
idempotency_key: string;
|
|
226
|
+
type: string;
|
|
227
|
+
version: unknown;
|
|
228
|
+
payload: unknown;
|
|
229
|
+
source_client_id: string;
|
|
230
|
+
source_client_commit_id: string;
|
|
231
|
+
source_commit_seq: unknown;
|
|
232
|
+
created_at_ms: unknown;
|
|
233
|
+
available_at_ms: unknown;
|
|
234
|
+
status: StoredReaction['status'];
|
|
235
|
+
attempts: unknown;
|
|
236
|
+
max_attempts: unknown;
|
|
237
|
+
lease_owner: string | null;
|
|
238
|
+
lease_expires_at_ms: unknown | null;
|
|
239
|
+
completed_at_ms: unknown | null;
|
|
240
|
+
last_failure: unknown | null;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function toStoredReaction(record: PostgresReactionRecord): StoredReaction {
|
|
244
|
+
return {
|
|
245
|
+
idempotencyKey: record.idempotency_key,
|
|
246
|
+
type: record.type,
|
|
247
|
+
version: asNumber(record.version),
|
|
248
|
+
payload: asJson(record.payload) as DurableJsonValue,
|
|
249
|
+
sourceClientId: record.source_client_id,
|
|
250
|
+
sourceClientCommitId: record.source_client_commit_id,
|
|
251
|
+
sourceCommitSeq: asNumber(record.source_commit_seq),
|
|
252
|
+
createdAtMs: asNumber(record.created_at_ms),
|
|
253
|
+
maxAttempts: asNumber(record.max_attempts),
|
|
254
|
+
status: record.status,
|
|
255
|
+
attempts: asNumber(record.attempts),
|
|
256
|
+
availableAtMs: asNumber(record.available_at_ms),
|
|
257
|
+
...(record.lease_owner !== null ? { leaseOwner: record.lease_owner } : {}),
|
|
258
|
+
...(record.lease_expires_at_ms !== null
|
|
259
|
+
? { leaseExpiresAtMs: asNumber(record.lease_expires_at_ms) }
|
|
260
|
+
: {}),
|
|
261
|
+
...(record.completed_at_ms !== null
|
|
262
|
+
? { completedAtMs: asNumber(record.completed_at_ms) }
|
|
263
|
+
: {}),
|
|
264
|
+
...(record.last_failure !== null
|
|
265
|
+
? { lastFailure: asJson(record.last_failure) as ReactionFailure }
|
|
266
|
+
: {}),
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
|
|
188
270
|
/**
|
|
189
271
|
* Serialize a push result to a JSON-able object (stored in a JSONB column).
|
|
190
272
|
* `serverRow` bytes are base64-encoded — JSONB cannot hold raw bytes.
|
|
@@ -719,6 +801,31 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
719
801
|
);
|
|
720
802
|
}
|
|
721
803
|
|
|
804
|
+
async enqueueReactions(reactions: readonly NewReaction[]): Promise<void> {
|
|
805
|
+
this.#assertOpen();
|
|
806
|
+
for (const reaction of reactions) {
|
|
807
|
+
await this.#client.query(
|
|
808
|
+
`INSERT INTO sync_reactions(
|
|
809
|
+
partition, idempotency_key, type, version, payload,
|
|
810
|
+
source_client_id, source_client_commit_id, source_commit_seq,
|
|
811
|
+
created_at_ms, available_at_ms, status, attempts, max_attempts
|
|
812
|
+
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$9,'pending',0,$10)`,
|
|
813
|
+
[
|
|
814
|
+
this.#partition,
|
|
815
|
+
reaction.idempotencyKey,
|
|
816
|
+
reaction.type,
|
|
817
|
+
reaction.version,
|
|
818
|
+
JSON.stringify(reaction.payload),
|
|
819
|
+
reaction.sourceClientId,
|
|
820
|
+
reaction.sourceClientCommitId,
|
|
821
|
+
reaction.sourceCommitSeq,
|
|
822
|
+
reaction.createdAtMs,
|
|
823
|
+
reaction.maxAttempts,
|
|
824
|
+
],
|
|
825
|
+
);
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
|
|
722
829
|
async commit(): Promise<void> {
|
|
723
830
|
this.#assertOpen();
|
|
724
831
|
this.#open = false;
|
|
@@ -940,6 +1047,46 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
940
1047
|
return rows[0] === undefined ? 0 : asNumber(rows[0].max_commit_seq);
|
|
941
1048
|
}
|
|
942
1049
|
|
|
1050
|
+
async queryAuthoritative(
|
|
1051
|
+
partition: string,
|
|
1052
|
+
query: AuthoritativeQueryRequest,
|
|
1053
|
+
): Promise<AuthoritativeQueryResult> {
|
|
1054
|
+
if (this.#tables === undefined) {
|
|
1055
|
+
throw new Error(
|
|
1056
|
+
'ensureSchema(schema) must run before registered queries',
|
|
1057
|
+
);
|
|
1058
|
+
}
|
|
1059
|
+
const prepared = bindAuthoritativePartition(
|
|
1060
|
+
prepareAuthoritativeQuery(
|
|
1061
|
+
query.sql,
|
|
1062
|
+
query.params,
|
|
1063
|
+
query.tables,
|
|
1064
|
+
this.#tables,
|
|
1065
|
+
),
|
|
1066
|
+
partition,
|
|
1067
|
+
);
|
|
1068
|
+
return this.#exec.transaction(async (client) => {
|
|
1069
|
+
await client.query(
|
|
1070
|
+
'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ READ ONLY',
|
|
1071
|
+
);
|
|
1072
|
+
const result = await client.query<Readonly<Record<string, unknown>>>(
|
|
1073
|
+
postgresPlaceholders(prepared.sql),
|
|
1074
|
+
prepared.params,
|
|
1075
|
+
);
|
|
1076
|
+
const cursor = await client.query<{ max_commit_seq: unknown }>(
|
|
1077
|
+
'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1',
|
|
1078
|
+
[partition],
|
|
1079
|
+
);
|
|
1080
|
+
return {
|
|
1081
|
+
rows: result.rows,
|
|
1082
|
+
maxCommitSeq:
|
|
1083
|
+
cursor.rows[0] === undefined
|
|
1084
|
+
? 0
|
|
1085
|
+
: asNumber(cursor.rows[0].max_commit_seq),
|
|
1086
|
+
};
|
|
1087
|
+
});
|
|
1088
|
+
}
|
|
1089
|
+
|
|
943
1090
|
async getHorizonSeq(partition: string): Promise<number> {
|
|
944
1091
|
const { rows } = await this.#exec.query<{ horizon_seq: unknown }>(
|
|
945
1092
|
'SELECT horizon_seq FROM sync_partitions WHERE partition=$1',
|
|
@@ -1000,6 +1147,210 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1000
1147
|
return getPushResultOn(this.#exec, partition, clientId, clientCommitId);
|
|
1001
1148
|
}
|
|
1002
1149
|
|
|
1150
|
+
async claimReactions(
|
|
1151
|
+
partition: string,
|
|
1152
|
+
query: ReactionClaimQuery,
|
|
1153
|
+
): Promise<StoredReaction[]> {
|
|
1154
|
+
if (query.types.length === 0 || query.limit <= 0) return [];
|
|
1155
|
+
const typeParams = query.types.map((_, index) => `$${index + 2}`).join(',');
|
|
1156
|
+
const nowParam = query.types.length + 2;
|
|
1157
|
+
const limitParam = nowParam + 1;
|
|
1158
|
+
const workerParam = limitParam + 1;
|
|
1159
|
+
const expiresParam = workerParam + 1;
|
|
1160
|
+
const { rows } = await this.#exec.query<PostgresReactionRecord>(
|
|
1161
|
+
`WITH due AS (
|
|
1162
|
+
SELECT partition, idempotency_key
|
|
1163
|
+
FROM sync_reactions
|
|
1164
|
+
WHERE partition=$1 AND type IN (${typeParams})
|
|
1165
|
+
AND ((status='pending' AND available_at_ms<=$${nowParam})
|
|
1166
|
+
OR (status='leased' AND lease_expires_at_ms<=$${nowParam}))
|
|
1167
|
+
ORDER BY CASE WHEN status='leased' THEN lease_expires_at_ms
|
|
1168
|
+
ELSE available_at_ms END,
|
|
1169
|
+
created_at_ms, idempotency_key
|
|
1170
|
+
FOR UPDATE SKIP LOCKED
|
|
1171
|
+
LIMIT $${limitParam}
|
|
1172
|
+
)
|
|
1173
|
+
UPDATE sync_reactions AS reaction
|
|
1174
|
+
SET status='leased', attempts=reaction.attempts+1,
|
|
1175
|
+
lease_owner=$${workerParam}, lease_expires_at_ms=$${expiresParam},
|
|
1176
|
+
completed_at_ms=NULL
|
|
1177
|
+
FROM due
|
|
1178
|
+
WHERE reaction.partition=due.partition
|
|
1179
|
+
AND reaction.idempotency_key=due.idempotency_key
|
|
1180
|
+
RETURNING reaction.*`,
|
|
1181
|
+
[
|
|
1182
|
+
partition,
|
|
1183
|
+
...query.types,
|
|
1184
|
+
query.nowMs,
|
|
1185
|
+
query.limit,
|
|
1186
|
+
query.leaseOwner,
|
|
1187
|
+
Math.min(Number.MAX_SAFE_INTEGER, query.nowMs + query.leaseDurationMs),
|
|
1188
|
+
],
|
|
1189
|
+
);
|
|
1190
|
+
return rows
|
|
1191
|
+
.map(toStoredReaction)
|
|
1192
|
+
.sort(
|
|
1193
|
+
(a, b) =>
|
|
1194
|
+
a.createdAtMs - b.createdAtMs ||
|
|
1195
|
+
a.idempotencyKey.localeCompare(b.idempotencyKey),
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
async completeReaction(
|
|
1200
|
+
partition: string,
|
|
1201
|
+
idempotencyKey: string,
|
|
1202
|
+
leaseOwner: string,
|
|
1203
|
+
completedAtMs: number,
|
|
1204
|
+
): Promise<boolean> {
|
|
1205
|
+
const result = await this.#exec.query(
|
|
1206
|
+
`UPDATE sync_reactions
|
|
1207
|
+
SET status='completed', completed_at_ms=$4,
|
|
1208
|
+
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1209
|
+
WHERE partition=$1 AND idempotency_key=$2
|
|
1210
|
+
AND status='leased' AND lease_owner=$3`,
|
|
1211
|
+
[partition, idempotencyKey, leaseOwner, completedAtMs],
|
|
1212
|
+
);
|
|
1213
|
+
return result.rowCount === 1;
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
async extendReactionLease(
|
|
1217
|
+
partition: string,
|
|
1218
|
+
idempotencyKey: string,
|
|
1219
|
+
leaseOwner: string,
|
|
1220
|
+
leaseExpiresAtMs: number,
|
|
1221
|
+
): Promise<boolean> {
|
|
1222
|
+
const result = await this.#exec.query(
|
|
1223
|
+
`UPDATE sync_reactions SET lease_expires_at_ms=$4
|
|
1224
|
+
WHERE partition=$1 AND idempotency_key=$2
|
|
1225
|
+
AND status='leased' AND lease_owner=$3`,
|
|
1226
|
+
[partition, idempotencyKey, leaseOwner, leaseExpiresAtMs],
|
|
1227
|
+
);
|
|
1228
|
+
return result.rowCount === 1;
|
|
1229
|
+
}
|
|
1230
|
+
|
|
1231
|
+
async failReaction(
|
|
1232
|
+
partition: string,
|
|
1233
|
+
idempotencyKey: string,
|
|
1234
|
+
update: ReactionFailureUpdate,
|
|
1235
|
+
): Promise<boolean> {
|
|
1236
|
+
const result = await this.#exec.query(
|
|
1237
|
+
`UPDATE sync_reactions
|
|
1238
|
+
SET status=$4, available_at_ms=$5, last_failure=$6,
|
|
1239
|
+
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
1240
|
+
WHERE partition=$1 AND idempotency_key=$2
|
|
1241
|
+
AND status='leased' AND lease_owner=$3`,
|
|
1242
|
+
[
|
|
1243
|
+
partition,
|
|
1244
|
+
idempotencyKey,
|
|
1245
|
+
update.leaseOwner,
|
|
1246
|
+
update.retryAtMs === undefined ? 'dead-letter' : 'pending',
|
|
1247
|
+
update.retryAtMs ?? update.failure.atMs,
|
|
1248
|
+
JSON.stringify(update.failure),
|
|
1249
|
+
],
|
|
1250
|
+
);
|
|
1251
|
+
return result.rowCount === 1;
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
async retryReaction(
|
|
1255
|
+
partition: string,
|
|
1256
|
+
idempotencyKey: string,
|
|
1257
|
+
nowMs: number,
|
|
1258
|
+
): Promise<boolean> {
|
|
1259
|
+
const result = await this.#exec.query(
|
|
1260
|
+
`UPDATE sync_reactions
|
|
1261
|
+
SET status='pending', attempts=0, available_at_ms=$3,
|
|
1262
|
+
last_failure=NULL, lease_owner=NULL, lease_expires_at_ms=NULL,
|
|
1263
|
+
completed_at_ms=NULL
|
|
1264
|
+
WHERE partition=$1 AND idempotency_key=$2 AND status='dead-letter'`,
|
|
1265
|
+
[partition, idempotencyKey, nowMs],
|
|
1266
|
+
);
|
|
1267
|
+
return result.rowCount === 1;
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
async getReaction(
|
|
1271
|
+
partition: string,
|
|
1272
|
+
idempotencyKey: string,
|
|
1273
|
+
): Promise<StoredReaction | undefined> {
|
|
1274
|
+
const { rows } = await this.#exec.query<PostgresReactionRecord>(
|
|
1275
|
+
'SELECT * FROM sync_reactions WHERE partition=$1 AND idempotency_key=$2',
|
|
1276
|
+
[partition, idempotencyKey],
|
|
1277
|
+
);
|
|
1278
|
+
return rows[0] === undefined ? undefined : toStoredReaction(rows[0]);
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
async listReactions(
|
|
1282
|
+
partition: string,
|
|
1283
|
+
query: ReactionListQuery,
|
|
1284
|
+
): Promise<StoredReaction[]> {
|
|
1285
|
+
const where = ['partition=$1'];
|
|
1286
|
+
const params: unknown[] = [partition];
|
|
1287
|
+
if (query.statuses !== undefined && query.statuses.length > 0) {
|
|
1288
|
+
const placeholders = query.statuses.map(
|
|
1289
|
+
(_, index) => `$${params.length + index + 1}`,
|
|
1290
|
+
);
|
|
1291
|
+
where.push(`status IN (${placeholders.join(',')})`);
|
|
1292
|
+
params.push(...query.statuses);
|
|
1293
|
+
}
|
|
1294
|
+
if (query.types !== undefined && query.types.length > 0) {
|
|
1295
|
+
const placeholders = query.types.map(
|
|
1296
|
+
(_, index) => `$${params.length + index + 1}`,
|
|
1297
|
+
);
|
|
1298
|
+
where.push(`type IN (${placeholders.join(',')})`);
|
|
1299
|
+
params.push(...query.types);
|
|
1300
|
+
}
|
|
1301
|
+
params.push(query.limit);
|
|
1302
|
+
const { rows } = await this.#exec.query<PostgresReactionRecord>(
|
|
1303
|
+
`SELECT * FROM sync_reactions WHERE ${where.join(' AND ')}
|
|
1304
|
+
ORDER BY created_at_ms DESC, idempotency_key DESC LIMIT $${params.length}`,
|
|
1305
|
+
params,
|
|
1306
|
+
);
|
|
1307
|
+
return rows.map(toStoredReaction);
|
|
1308
|
+
}
|
|
1309
|
+
|
|
1310
|
+
async pruneReactions(
|
|
1311
|
+
partition: string,
|
|
1312
|
+
query: ReactionPruneQuery,
|
|
1313
|
+
): Promise<PrunedReactionCounts> {
|
|
1314
|
+
if (query.limit <= 0) return { completed: 0, deadLetter: 0 };
|
|
1315
|
+
const { rows } = await this.#exec.query<{
|
|
1316
|
+
status: 'completed' | 'dead-letter';
|
|
1317
|
+
}>(
|
|
1318
|
+
`WITH targets AS (
|
|
1319
|
+
SELECT partition, idempotency_key
|
|
1320
|
+
FROM sync_reactions
|
|
1321
|
+
WHERE partition=$1
|
|
1322
|
+
AND ((status='completed' AND completed_at_ms IS NOT NULL
|
|
1323
|
+
AND completed_at_ms<$2)
|
|
1324
|
+
OR (status='dead-letter' AND available_at_ms<$3))
|
|
1325
|
+
ORDER BY CASE WHEN status='completed' THEN completed_at_ms
|
|
1326
|
+
ELSE available_at_ms END,
|
|
1327
|
+
idempotency_key
|
|
1328
|
+
FOR UPDATE SKIP LOCKED
|
|
1329
|
+
LIMIT $4
|
|
1330
|
+
)
|
|
1331
|
+
DELETE FROM sync_reactions AS reaction
|
|
1332
|
+
USING targets
|
|
1333
|
+
WHERE reaction.partition=targets.partition
|
|
1334
|
+
AND reaction.idempotency_key=targets.idempotency_key
|
|
1335
|
+
AND ((reaction.status='completed'
|
|
1336
|
+
AND reaction.completed_at_ms IS NOT NULL
|
|
1337
|
+
AND reaction.completed_at_ms<$2)
|
|
1338
|
+
OR (reaction.status='dead-letter'
|
|
1339
|
+
AND reaction.available_at_ms<$3))
|
|
1340
|
+
RETURNING reaction.status`,
|
|
1341
|
+
[
|
|
1342
|
+
partition,
|
|
1343
|
+
query.completedBeforeMs,
|
|
1344
|
+
query.deadLetterBeforeMs,
|
|
1345
|
+
query.limit,
|
|
1346
|
+
],
|
|
1347
|
+
);
|
|
1348
|
+
return {
|
|
1349
|
+
completed: rows.filter((row) => row.status === 'completed').length,
|
|
1350
|
+
deadLetter: rows.filter((row) => row.status === 'dead-letter').length,
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1003
1354
|
async readCommitWindow(
|
|
1004
1355
|
partition: string,
|
|
1005
1356
|
query: CommitWindowQuery,
|
package/src/pull.ts
CHANGED
|
@@ -36,7 +36,7 @@ async function resolveImageBuilder(
|
|
|
36
36
|
if (cachedDefaultBuilder === undefined) {
|
|
37
37
|
const hasBun = (globalThis as { Bun?: unknown }).Bun !== undefined;
|
|
38
38
|
cachedDefaultBuilder = hasBun
|
|
39
|
-
? (await import('./sqlite-
|
|
39
|
+
? (await import('./sqlite-bun')).buildSqliteImage
|
|
40
40
|
: null;
|
|
41
41
|
}
|
|
42
42
|
return cachedDefaultBuilder ?? undefined;
|
package/src/push.ts
CHANGED
|
@@ -33,6 +33,12 @@ import type { SyncRequestContext } from './context';
|
|
|
33
33
|
import { clockOf } from './context';
|
|
34
34
|
import type { CrdtMergerRegistry } from './crdt-merger';
|
|
35
35
|
import { SyncError } from './errors';
|
|
36
|
+
import { emitEvent } from './events';
|
|
37
|
+
import {
|
|
38
|
+
prepareReactions,
|
|
39
|
+
type PreparedReaction,
|
|
40
|
+
toNewReactions,
|
|
41
|
+
} from './reactions';
|
|
36
42
|
import type { CompiledSchema, CompiledTable } from './schema';
|
|
37
43
|
import type { ResolvedScopes } from './scopes';
|
|
38
44
|
import { authorizeWrite, renderScopeValue, storedScopesForRow } from './scopes';
|
|
@@ -837,6 +843,31 @@ export async function processPushCommitWithTrace(
|
|
|
837
843
|
resolved: ResolvedScopes,
|
|
838
844
|
clientId: string,
|
|
839
845
|
frame: PushCommitFrame,
|
|
846
|
+
): Promise<ProcessedPushCommit> {
|
|
847
|
+
return processPushOperationsWithTrace(
|
|
848
|
+
ctx,
|
|
849
|
+
schema,
|
|
850
|
+
resolved,
|
|
851
|
+
clientId,
|
|
852
|
+
frame.clientCommitId,
|
|
853
|
+
async () => frame.operations,
|
|
854
|
+
);
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
/**
|
|
858
|
+
* Shared serialized apply path for SSP2 commits and authoritative commands.
|
|
859
|
+
* The builder runs after the partition lock and idempotency re-check, so its
|
|
860
|
+
* reads and the returned operations share the transaction that is committed.
|
|
861
|
+
*/
|
|
862
|
+
export async function processPushOperationsWithTrace(
|
|
863
|
+
ctx: SyncRequestContext,
|
|
864
|
+
schema: CompiledSchema,
|
|
865
|
+
resolved: ResolvedScopes,
|
|
866
|
+
clientId: string,
|
|
867
|
+
clientCommitId: string,
|
|
868
|
+
buildOperations: (
|
|
869
|
+
tx: StorageTransaction,
|
|
870
|
+
) => Promise<readonly PushOperation[]>,
|
|
840
871
|
): Promise<ProcessedPushCommit> {
|
|
841
872
|
const { storage, partition } = ctx;
|
|
842
873
|
let persisted: StoredPushResult | undefined;
|
|
@@ -844,7 +875,7 @@ export async function processPushCommitWithTrace(
|
|
|
844
875
|
persisted = await storage.getPushResult(
|
|
845
876
|
partition,
|
|
846
877
|
clientId,
|
|
847
|
-
|
|
878
|
+
clientCommitId,
|
|
848
879
|
);
|
|
849
880
|
} catch (error) {
|
|
850
881
|
if (
|
|
@@ -854,14 +885,14 @@ export async function processPushCommitWithTrace(
|
|
|
854
885
|
// §6.3: answer the retryable cache-miss for this commit rather than
|
|
855
886
|
// re-applying. Not persisted — a retry may find a readable record.
|
|
856
887
|
return {
|
|
857
|
-
frame: idempotencyCacheMissFrame(
|
|
888
|
+
frame: idempotencyCacheMissFrame(clientCommitId, error),
|
|
858
889
|
replayed: false,
|
|
859
890
|
};
|
|
860
891
|
}
|
|
861
892
|
throw error;
|
|
862
893
|
}
|
|
863
894
|
if (persisted !== undefined) {
|
|
864
|
-
return processedPushCommit(
|
|
895
|
+
return processedPushCommit(clientCommitId, persisted, true);
|
|
865
896
|
}
|
|
866
897
|
|
|
867
898
|
const createdAtMs = clockOf(ctx)();
|
|
@@ -869,6 +900,7 @@ export async function processPushCommitWithTrace(
|
|
|
869
900
|
const crdtMergers = ctx.crdtMergers;
|
|
870
901
|
const validators = ctx.validators;
|
|
871
902
|
const commitValidator = ctx.commitValidator;
|
|
903
|
+
const reactionPlanner = ctx.reactionPlanner;
|
|
872
904
|
const tx = await storage.begin(partition);
|
|
873
905
|
const lockPartitionForPush =
|
|
874
906
|
tx.lockPartitionForPush?.bind(tx) ??
|
|
@@ -877,10 +909,11 @@ export async function processPushCommitWithTrace(
|
|
|
877
909
|
try {
|
|
878
910
|
if (
|
|
879
911
|
lockPartitionForPush === undefined ||
|
|
880
|
-
commitRejectedPushResult === undefined
|
|
912
|
+
commitRejectedPushResult === undefined ||
|
|
913
|
+
(reactionPlanner !== undefined && tx.enqueueReactions === undefined)
|
|
881
914
|
) {
|
|
882
915
|
throw new Error(
|
|
883
|
-
'storage transaction does not support serialized push apply
|
|
916
|
+
'storage transaction does not support serialized push apply, atomic rejection finalization, and configured durable reactions',
|
|
884
917
|
);
|
|
885
918
|
}
|
|
886
919
|
await lockPartitionForPush();
|
|
@@ -895,19 +928,11 @@ export async function processPushCommitWithTrace(
|
|
|
895
928
|
try {
|
|
896
929
|
const serializedPersisted =
|
|
897
930
|
tx.getPushResult !== undefined
|
|
898
|
-
? await tx.getPushResult(clientId,
|
|
899
|
-
: await storage.getPushResult(
|
|
900
|
-
partition,
|
|
901
|
-
clientId,
|
|
902
|
-
frame.clientCommitId,
|
|
903
|
-
);
|
|
931
|
+
? await tx.getPushResult(clientId, clientCommitId)
|
|
932
|
+
: await storage.getPushResult(partition, clientId, clientCommitId);
|
|
904
933
|
if (serializedPersisted !== undefined) {
|
|
905
934
|
await tx.rollback();
|
|
906
|
-
return processedPushCommit(
|
|
907
|
-
frame.clientCommitId,
|
|
908
|
-
serializedPersisted,
|
|
909
|
-
true,
|
|
910
|
-
);
|
|
935
|
+
return processedPushCommit(clientCommitId, serializedPersisted, true);
|
|
911
936
|
}
|
|
912
937
|
} catch (error) {
|
|
913
938
|
if (
|
|
@@ -916,7 +941,7 @@ export async function processPushCommitWithTrace(
|
|
|
916
941
|
) {
|
|
917
942
|
await tx.rollback();
|
|
918
943
|
return {
|
|
919
|
-
frame: idempotencyCacheMissFrame(
|
|
944
|
+
frame: idempotencyCacheMissFrame(clientCommitId, error),
|
|
920
945
|
replayed: false,
|
|
921
946
|
};
|
|
922
947
|
}
|
|
@@ -926,8 +951,9 @@ export async function processPushCommitWithTrace(
|
|
|
926
951
|
const changes: NewChange[] = [];
|
|
927
952
|
const validatedOperations: ValidateCommitOperation[] = [];
|
|
928
953
|
let terminated: PushOperationResult | undefined;
|
|
929
|
-
|
|
930
|
-
|
|
954
|
+
const operations = await buildOperations(tx);
|
|
955
|
+
for (let opIndex = 0; opIndex < operations.length; opIndex++) {
|
|
956
|
+
const op = operations[opIndex];
|
|
931
957
|
if (op === undefined) continue;
|
|
932
958
|
const outcome = await applyOperation(
|
|
933
959
|
tx,
|
|
@@ -956,7 +982,7 @@ export async function processPushCommitWithTrace(
|
|
|
956
982
|
tx,
|
|
957
983
|
schema,
|
|
958
984
|
clientId,
|
|
959
|
-
|
|
985
|
+
clientCommitId,
|
|
960
986
|
ctx.actorId,
|
|
961
987
|
partition,
|
|
962
988
|
validatedOperations,
|
|
@@ -966,6 +992,18 @@ export async function processPushCommitWithTrace(
|
|
|
966
992
|
}
|
|
967
993
|
}
|
|
968
994
|
|
|
995
|
+
let preparedReactions: PreparedReaction[] = [];
|
|
996
|
+
if (terminated === undefined && reactionPlanner !== undefined) {
|
|
997
|
+
preparedReactions = await prepareReactions(reactionPlanner, {
|
|
998
|
+
clientId,
|
|
999
|
+
clientCommitId,
|
|
1000
|
+
actorId: ctx.actorId,
|
|
1001
|
+
partition,
|
|
1002
|
+
operations: validatedOperations,
|
|
1003
|
+
read: commitValidationReader(tx, schema),
|
|
1004
|
+
});
|
|
1005
|
+
}
|
|
1006
|
+
|
|
969
1007
|
if (terminated !== undefined) {
|
|
970
1008
|
// §6.3 rejected: only the terminating operation's record; §6.4:
|
|
971
1009
|
// every write of the commit rolls back.
|
|
@@ -975,11 +1013,11 @@ export async function processPushCommitWithTrace(
|
|
|
975
1013
|
});
|
|
976
1014
|
// Discard candidates and persist the rejection while retaining the same
|
|
977
1015
|
// partition lock. There is no unlock gap in which a duplicate can rerun.
|
|
978
|
-
await commitRejectedPushResult(clientId,
|
|
1016
|
+
await commitRejectedPushResult(clientId, clientCommitId, stored);
|
|
979
1017
|
const canonical = await storage.getPushResult(
|
|
980
1018
|
partition,
|
|
981
1019
|
clientId,
|
|
982
|
-
|
|
1020
|
+
clientCommitId,
|
|
983
1021
|
);
|
|
984
1022
|
if (canonical === undefined) {
|
|
985
1023
|
throw new Error(
|
|
@@ -987,7 +1025,7 @@ export async function processPushCommitWithTrace(
|
|
|
987
1025
|
);
|
|
988
1026
|
}
|
|
989
1027
|
return processedPushCommit(
|
|
990
|
-
|
|
1028
|
+
clientCommitId,
|
|
991
1029
|
canonical,
|
|
992
1030
|
canonical.cacheIdentity !== stored.cacheIdentity,
|
|
993
1031
|
);
|
|
@@ -995,18 +1033,48 @@ export async function processPushCommitWithTrace(
|
|
|
995
1033
|
|
|
996
1034
|
const commitSeq = await tx.appendCommit({
|
|
997
1035
|
clientId,
|
|
998
|
-
clientCommitId
|
|
1036
|
+
clientCommitId,
|
|
999
1037
|
actorId: ctx.actorId,
|
|
1000
1038
|
createdAtMs,
|
|
1001
1039
|
changes,
|
|
1002
1040
|
});
|
|
1041
|
+
const newReactions = toNewReactions(preparedReactions, {
|
|
1042
|
+
clientId,
|
|
1043
|
+
clientCommitId,
|
|
1044
|
+
commitSeq,
|
|
1045
|
+
createdAtMs,
|
|
1046
|
+
});
|
|
1047
|
+
if (newReactions.length > 0) {
|
|
1048
|
+
const enqueueReactions = tx.enqueueReactions;
|
|
1049
|
+
if (enqueueReactions === undefined) {
|
|
1050
|
+
throw new Error('storage lost durable reaction enqueue support');
|
|
1051
|
+
}
|
|
1052
|
+
await enqueueReactions.call(tx, newReactions);
|
|
1053
|
+
}
|
|
1003
1054
|
const stored = newStoredPushResult(createdAtMs, {
|
|
1004
1055
|
status: 'applied',
|
|
1005
1056
|
commitSeq,
|
|
1006
1057
|
results,
|
|
1007
1058
|
});
|
|
1008
|
-
await tx.putPushResult(clientId,
|
|
1059
|
+
await tx.putPushResult(clientId, clientCommitId, stored);
|
|
1009
1060
|
await tx.commit();
|
|
1061
|
+
if (ctx.events !== undefined) {
|
|
1062
|
+
const atMs = clockOf(ctx)();
|
|
1063
|
+
for (const reaction of newReactions) {
|
|
1064
|
+
emitEvent(ctx.events, {
|
|
1065
|
+
type: 'reaction.queued',
|
|
1066
|
+
atMs,
|
|
1067
|
+
partition,
|
|
1068
|
+
actorId: ctx.actorId,
|
|
1069
|
+
clientId,
|
|
1070
|
+
clientCommitId,
|
|
1071
|
+
commitSeq,
|
|
1072
|
+
idempotencyKey: reaction.idempotencyKey,
|
|
1073
|
+
reactionType: reaction.type,
|
|
1074
|
+
version: reaction.version,
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1010
1078
|
if (ctx.realtime !== undefined && changes.length > 0) {
|
|
1011
1079
|
await ctx.realtime.notifyCommit(partition, {
|
|
1012
1080
|
commitSeq,
|
|
@@ -1015,7 +1083,7 @@ export async function processPushCommitWithTrace(
|
|
|
1015
1083
|
changes,
|
|
1016
1084
|
});
|
|
1017
1085
|
}
|
|
1018
|
-
return processedPushCommit(
|
|
1086
|
+
return processedPushCommit(clientCommitId, stored, false);
|
|
1019
1087
|
} catch (error) {
|
|
1020
1088
|
if (error instanceof StorageConstraintError) {
|
|
1021
1089
|
const stored = newStoredPushResult(createdAtMs, {
|
|
@@ -1037,11 +1105,11 @@ export async function processPushCommitWithTrace(
|
|
|
1037
1105
|
);
|
|
1038
1106
|
}
|
|
1039
1107
|
try {
|
|
1040
|
-
await commitRejectedPushResult(clientId,
|
|
1108
|
+
await commitRejectedPushResult(clientId, clientCommitId, stored);
|
|
1041
1109
|
const canonical = await storage.getPushResult(
|
|
1042
1110
|
partition,
|
|
1043
1111
|
clientId,
|
|
1044
|
-
|
|
1112
|
+
clientCommitId,
|
|
1045
1113
|
);
|
|
1046
1114
|
if (canonical === undefined) {
|
|
1047
1115
|
throw new Error(
|
|
@@ -1049,7 +1117,7 @@ export async function processPushCommitWithTrace(
|
|
|
1049
1117
|
);
|
|
1050
1118
|
}
|
|
1051
1119
|
return processedPushCommit(
|
|
1052
|
-
|
|
1120
|
+
clientCommitId,
|
|
1053
1121
|
canonical,
|
|
1054
1122
|
canonical.cacheIdentity !== stored.cacheIdentity,
|
|
1055
1123
|
);
|