or3-provider-sqlite 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -19
- package/dist/module.json +1 -1
- package/dist/runtime/server/admin/adapters/sync-sqlite.js +90 -16
- package/dist/runtime/server/admin/stores/sqlite-store.js +7 -8
- package/dist/runtime/server/auth/sqlite-auth-workspace-store.d.ts +5 -0
- package/dist/runtime/server/auth/sqlite-auth-workspace-store.js +368 -40
- package/dist/runtime/server/connect/sqlite-connect-store.js +19 -5
- package/dist/runtime/server/db/d1-dialect.d.ts +39 -0
- package/dist/runtime/server/db/d1-dialect.js +70 -0
- package/dist/runtime/server/db/d1.d.ts +13 -0
- package/dist/runtime/server/db/d1.js +30 -0
- package/dist/runtime/server/db/kysely.d.ts +53 -15
- package/dist/runtime/server/db/kysely.js +374 -16
- package/dist/runtime/server/db/migrate.js +3 -1
- package/dist/runtime/server/db/migrations/016_connect_runtime_binding.d.ts +4 -0
- package/dist/runtime/server/db/migrations/016_connect_runtime_binding.js +28 -0
- package/dist/runtime/server/db/schema.d.ts +3 -0
- package/dist/runtime/server/plugins/register.js +33 -13
- package/dist/runtime/server/rate-limit/sqlite-provider.d.ts +2 -0
- package/dist/runtime/server/rate-limit/sqlite-provider.js +70 -1
- package/dist/runtime/server/sync/maintenance-state.d.ts +26 -0
- package/dist/runtime/server/sync/maintenance-state.js +66 -0
- package/dist/runtime/server/sync/sqlite-sync-gateway-adapter.d.ts +15 -4
- package/dist/runtime/server/sync/sqlite-sync-gateway-adapter.js +780 -21
- package/dist/runtime/server/webhooks/sqlite-webhook-store.d.ts +3 -2
- package/dist/runtime/server/webhooks/sqlite-webhook-store.js +90 -39
- package/package.json +10 -6
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
import { canRunSyncHistoryGc } from "./history-gc-policy.js";
|
|
2
2
|
import { createError } from "h3";
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import {
|
|
4
|
+
beginSyncMaintenanceRun,
|
|
5
|
+
completeSyncMaintenanceRun,
|
|
6
|
+
computeSyncMaintenanceBacklog,
|
|
7
|
+
failSyncMaintenanceRun,
|
|
8
|
+
getSyncMaintenanceState,
|
|
9
|
+
SYNC_MAINTENANCE_RETENTION_SECONDS
|
|
10
|
+
} from "./maintenance-state.js";
|
|
11
|
+
import { getSqliteDb, getRawDb, isD1Driver } from "../db/kysely.js";
|
|
12
|
+
import { d1All, d1Batch, d1Run } from "../db/d1.js";
|
|
5
13
|
import { SYNCED_TABLE_MAP, ALLOWED_SYNC_TABLES } from "../db/schema.js";
|
|
6
14
|
import { emitWebhookSystemHook } from "~~/server/utils/webhooks/runtime";
|
|
7
15
|
const DEFAULT_PULL_LIMIT = 100;
|
|
@@ -14,7 +22,7 @@ const SESSION_CONTEXT_KEY_PREFIX = "__or3_session_context_";
|
|
|
14
22
|
const MIN_SYNC_RETENTION_SECONDS = 60 * 60;
|
|
15
23
|
const MAX_SYNC_RETENTION_SECONDS = 365 * 24 * 60 * 60;
|
|
16
24
|
function uid() {
|
|
17
|
-
return randomUUID();
|
|
25
|
+
return globalThis.crypto.randomUUID();
|
|
18
26
|
}
|
|
19
27
|
function nowEpoch() {
|
|
20
28
|
return Math.floor(Date.now() / 1e3);
|
|
@@ -302,6 +310,9 @@ export class SqliteSyncGatewayAdapter {
|
|
|
302
310
|
throw createError({ statusCode: 400, statusMessage: "Invalid workspace quota" });
|
|
303
311
|
}
|
|
304
312
|
const hash = normalizeStorageHash(input.hash);
|
|
313
|
+
if (isD1Driver()) {
|
|
314
|
+
return this.reserveUploadIntentInD1(input, hash);
|
|
315
|
+
}
|
|
305
316
|
const raw = getRawDb();
|
|
306
317
|
raw.transaction(() => {
|
|
307
318
|
const now = nowEpoch();
|
|
@@ -345,6 +356,27 @@ export class SqliteSyncGatewayAdapter {
|
|
|
345
356
|
}
|
|
346
357
|
async consumeUploadIntent(event, input) {
|
|
347
358
|
await assertWorkspaceScopeAuthorized(event, input.workspaceId);
|
|
359
|
+
if (isD1Driver()) {
|
|
360
|
+
const now = nowEpoch();
|
|
361
|
+
const result2 = await d1Run(
|
|
362
|
+
`UPDATE upload_intents
|
|
363
|
+
SET status = 'consumed', storage_id = ?, consumed_at = ?
|
|
364
|
+
WHERE id = ? AND workspace_id = ? AND status = 'active' AND expires_at > ?
|
|
365
|
+
AND hash = ? AND mime_type = ? AND size_bytes = ?`,
|
|
366
|
+
input.storageId,
|
|
367
|
+
now,
|
|
368
|
+
input.intentId,
|
|
369
|
+
input.workspaceId,
|
|
370
|
+
now,
|
|
371
|
+
normalizeStorageHash(input.hash),
|
|
372
|
+
input.mimeType,
|
|
373
|
+
input.sizeBytes
|
|
374
|
+
);
|
|
375
|
+
if ((result2.meta?.changes ?? 0) !== 1) {
|
|
376
|
+
throw createError({ statusCode: 409, statusMessage: "Upload intent expired, mismatched, or already consumed" });
|
|
377
|
+
}
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
348
380
|
const raw = getRawDb();
|
|
349
381
|
const result = raw.prepare(`UPDATE upload_intents
|
|
350
382
|
SET status = 'consumed', storage_id = ?, consumed_at = ?
|
|
@@ -365,6 +397,19 @@ export class SqliteSyncGatewayAdapter {
|
|
|
365
397
|
}
|
|
366
398
|
async cancelUploadIntent(event, input) {
|
|
367
399
|
await assertWorkspaceScopeAuthorized(event, input.workspaceId);
|
|
400
|
+
if (isD1Driver()) {
|
|
401
|
+
const result2 = await d1Run(
|
|
402
|
+
`UPDATE upload_intents SET status = 'cancelled', cancelled_at = ?
|
|
403
|
+
WHERE id = ? AND workspace_id = ? AND status = 'active'`,
|
|
404
|
+
nowEpoch(),
|
|
405
|
+
input.intentId,
|
|
406
|
+
input.workspaceId
|
|
407
|
+
);
|
|
408
|
+
if ((result2.meta?.changes ?? 0) !== 1) {
|
|
409
|
+
throw createError({ statusCode: 409, statusMessage: "Upload intent is not active" });
|
|
410
|
+
}
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
368
413
|
const result = getRawDb().prepare(`UPDATE upload_intents
|
|
369
414
|
SET status = 'cancelled', cancelled_at = ?
|
|
370
415
|
WHERE id = ? AND workspace_id = ? AND status = 'active'`).run(nowEpoch(), input.intentId, input.workspaceId);
|
|
@@ -381,17 +426,18 @@ export class SqliteSyncGatewayAdapter {
|
|
|
381
426
|
throw createError({ statusCode: 400, statusMessage: "Invalid storage hash" });
|
|
382
427
|
}
|
|
383
428
|
const after = decodeCanonicalStorageCursor(input.cursor, input.kind, hash);
|
|
384
|
-
const raw = getRawDb();
|
|
429
|
+
const raw = isD1Driver() ? null : getRawDb();
|
|
430
|
+
const all = async (sql, ...parameters) => raw ? raw.prepare(sql).all(...parameters) : d1All(sql, ...parameters);
|
|
385
431
|
if (input.kind === "active_reservations") {
|
|
386
432
|
if (after.length > 1) {
|
|
387
433
|
throw createError({ statusCode: 400, statusMessage: "Invalid canonical storage cursor" });
|
|
388
434
|
}
|
|
389
435
|
const now = input.now ?? nowEpoch();
|
|
390
|
-
const rows2 =
|
|
436
|
+
const rows2 = await all(`SELECT id, hash, reserved_bytes, expires_at
|
|
391
437
|
FROM upload_intents
|
|
392
438
|
WHERE workspace_id = ? AND status = 'active' AND expires_at > ?
|
|
393
439
|
AND id > ? AND reserved_bytes > 0 AND (? IS NULL OR hash = ?)
|
|
394
|
-
ORDER BY id ASC LIMIT
|
|
440
|
+
ORDER BY id ASC LIMIT ?`, workspaceId, now, after[0] ?? "", hash ?? null, hash ?? null, limit + 1);
|
|
395
441
|
const hasMore2 = rows2.length > limit;
|
|
396
442
|
const page2 = hasMore2 ? rows2.slice(0, limit) : rows2;
|
|
397
443
|
const last2 = page2[page2.length - 1];
|
|
@@ -416,7 +462,7 @@ export class SqliteSyncGatewayAdapter {
|
|
|
416
462
|
if (after.length > 1) {
|
|
417
463
|
throw createError({ statusCode: 400, statusMessage: "Invalid canonical storage cursor" });
|
|
418
464
|
}
|
|
419
|
-
const rows2 =
|
|
465
|
+
const rows2 = await all(`
|
|
420
466
|
SELECT id, data_json, updated_at
|
|
421
467
|
FROM s_file_meta
|
|
422
468
|
WHERE workspace_id = ?
|
|
@@ -431,7 +477,7 @@ export class SqliteSyncGatewayAdapter {
|
|
|
431
477
|
) = ?)
|
|
432
478
|
ORDER BY id ASC
|
|
433
479
|
LIMIT ?
|
|
434
|
-
|
|
480
|
+
`, workspaceId, after[0] ?? "", hash ?? null, hash ?? null, limit + 1);
|
|
435
481
|
const hasMore2 = rows2.length > limit;
|
|
436
482
|
const page2 = hasMore2 ? rows2.slice(0, limit) : rows2;
|
|
437
483
|
const items = page2.map((row) => {
|
|
@@ -472,7 +518,8 @@ export class SqliteSyncGatewayAdapter {
|
|
|
472
518
|
throw createError({ statusCode: 400, statusMessage: "Invalid canonical storage cursor" });
|
|
473
519
|
}
|
|
474
520
|
const [afterTable = "", afterId = "", afterHash = ""] = after;
|
|
475
|
-
const rows =
|
|
521
|
+
const rows = await all(
|
|
522
|
+
`
|
|
476
523
|
WITH reference_edges(source_table, source_id, raw_hash) AS (
|
|
477
524
|
SELECT 'messages', row.id, CAST(edge.value AS TEXT)
|
|
478
525
|
FROM s_messages AS row
|
|
@@ -516,7 +563,7 @@ export class SqliteSyncGatewayAdapter {
|
|
|
516
563
|
)
|
|
517
564
|
ORDER BY source_table ASC, source_id ASC, raw_hash ASC
|
|
518
565
|
LIMIT ?
|
|
519
|
-
|
|
566
|
+
`,
|
|
520
567
|
workspaceId,
|
|
521
568
|
workspaceId,
|
|
522
569
|
hash ?? null,
|
|
@@ -592,6 +639,15 @@ export class SqliteSyncGatewayAdapter {
|
|
|
592
639
|
uniqueOps.push(group.op);
|
|
593
640
|
indicesByOpId.set(group.op.stamp.opId, group.indices);
|
|
594
641
|
}
|
|
642
|
+
if (isD1Driver()) {
|
|
643
|
+
return this.pushInD1({
|
|
644
|
+
event,
|
|
645
|
+
workspaceId,
|
|
646
|
+
uniqueOps,
|
|
647
|
+
resultSlots,
|
|
648
|
+
indicesByOpId
|
|
649
|
+
});
|
|
650
|
+
}
|
|
595
651
|
const raw = getRawDb();
|
|
596
652
|
const now = nowEpoch();
|
|
597
653
|
const userId = resolveSessionUserId(event);
|
|
@@ -883,6 +939,9 @@ export class SqliteSyncGatewayAdapter {
|
|
|
883
939
|
const workspaceId = input.scope.workspaceId;
|
|
884
940
|
await assertWorkspaceScopeAuthorized(event, workspaceId);
|
|
885
941
|
const pageSize = resolveSnapshotPageSize(input.pageSize);
|
|
942
|
+
if (isD1Driver()) {
|
|
943
|
+
return this.snapshotInD1(workspaceId, input, pageSize);
|
|
944
|
+
}
|
|
886
945
|
const raw = getRawDb();
|
|
887
946
|
const now = nowEpoch();
|
|
888
947
|
let header;
|
|
@@ -1111,7 +1170,6 @@ export class SqliteSyncGatewayAdapter {
|
|
|
1111
1170
|
}
|
|
1112
1171
|
async updateCursor(event, input) {
|
|
1113
1172
|
await assertWorkspaceScopeAuthorized(event, input.scope.workspaceId);
|
|
1114
|
-
const raw = getRawDb();
|
|
1115
1173
|
const now = nowEpoch();
|
|
1116
1174
|
const deviceId = input.deviceId.trim();
|
|
1117
1175
|
if (!deviceId || deviceId.length > 256) {
|
|
@@ -1124,6 +1182,10 @@ export class SqliteSyncGatewayAdapter {
|
|
|
1124
1182
|
if (!ownerUserId) {
|
|
1125
1183
|
throw createError({ statusCode: 401, statusMessage: "Unauthorized" });
|
|
1126
1184
|
}
|
|
1185
|
+
if (isD1Driver()) {
|
|
1186
|
+
return this.updateCursorInD1(input.scope.workspaceId, deviceId, input.version, ownerUserId, now);
|
|
1187
|
+
}
|
|
1188
|
+
const raw = getRawDb();
|
|
1127
1189
|
const update = raw.transaction(() => {
|
|
1128
1190
|
const counter = raw.prepare(
|
|
1129
1191
|
"SELECT value FROM server_version_counter WHERE workspace_id = ?"
|
|
@@ -1161,17 +1223,59 @@ export class SqliteSyncGatewayAdapter {
|
|
|
1161
1223
|
}
|
|
1162
1224
|
if (!canRunSyncHistoryGc()) return;
|
|
1163
1225
|
const cutoff = nowEpoch() - input.retentionSeconds;
|
|
1226
|
+
if (isD1Driver()) {
|
|
1227
|
+
await d1Batch([
|
|
1228
|
+
{
|
|
1229
|
+
sql: `DELETE FROM device_cursors
|
|
1230
|
+
WHERE workspace_id = ? AND updated_at < ?`,
|
|
1231
|
+
parameters: [input.scope.workspaceId, cutoff]
|
|
1232
|
+
},
|
|
1233
|
+
{
|
|
1234
|
+
sql: `DELETE FROM tombstones
|
|
1235
|
+
WHERE workspace_id = ?
|
|
1236
|
+
AND created_at < ?
|
|
1237
|
+
AND (
|
|
1238
|
+
NOT EXISTS (
|
|
1239
|
+
SELECT 1 FROM device_cursors
|
|
1240
|
+
WHERE workspace_id = ?
|
|
1241
|
+
)
|
|
1242
|
+
OR server_version <= (
|
|
1243
|
+
SELECT MIN(last_seen_version)
|
|
1244
|
+
FROM device_cursors
|
|
1245
|
+
WHERE workspace_id = ?
|
|
1246
|
+
)
|
|
1247
|
+
)`,
|
|
1248
|
+
parameters: [
|
|
1249
|
+
input.scope.workspaceId,
|
|
1250
|
+
cutoff,
|
|
1251
|
+
input.scope.workspaceId,
|
|
1252
|
+
input.scope.workspaceId
|
|
1253
|
+
]
|
|
1254
|
+
}
|
|
1255
|
+
]);
|
|
1256
|
+
return;
|
|
1257
|
+
}
|
|
1164
1258
|
const raw = getRawDb();
|
|
1165
1259
|
raw.transaction(() => {
|
|
1260
|
+
raw.prepare(
|
|
1261
|
+
`DELETE FROM device_cursors
|
|
1262
|
+
WHERE workspace_id = ? AND updated_at < ?`
|
|
1263
|
+
).run(input.scope.workspaceId, cutoff);
|
|
1166
1264
|
const cursor = raw.prepare(
|
|
1167
1265
|
`SELECT MIN(last_seen_version) AS min_version
|
|
1168
1266
|
FROM device_cursors WHERE workspace_id = ?`
|
|
1169
1267
|
).get(input.scope.workspaceId);
|
|
1170
|
-
if (cursor.min_version === null)
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1268
|
+
if (cursor.min_version === null) {
|
|
1269
|
+
raw.prepare(
|
|
1270
|
+
`DELETE FROM tombstones
|
|
1271
|
+
WHERE workspace_id = ? AND created_at < ?`
|
|
1272
|
+
).run(input.scope.workspaceId, cutoff);
|
|
1273
|
+
} else {
|
|
1274
|
+
raw.prepare(
|
|
1275
|
+
`DELETE FROM tombstones
|
|
1276
|
+
WHERE workspace_id = ? AND created_at < ? AND server_version <= ?`
|
|
1277
|
+
).run(input.scope.workspaceId, cutoff, cursor.min_version);
|
|
1278
|
+
}
|
|
1175
1279
|
})();
|
|
1176
1280
|
}
|
|
1177
1281
|
async gcChangeLog(event, input) {
|
|
@@ -1181,19 +1285,674 @@ export class SqliteSyncGatewayAdapter {
|
|
|
1181
1285
|
}
|
|
1182
1286
|
if (!canRunSyncHistoryGc()) return;
|
|
1183
1287
|
const cutoff = nowEpoch() - input.retentionSeconds;
|
|
1288
|
+
if (isD1Driver()) {
|
|
1289
|
+
await d1Batch([
|
|
1290
|
+
{
|
|
1291
|
+
sql: `DELETE FROM device_cursors
|
|
1292
|
+
WHERE workspace_id = ? AND updated_at < ?`,
|
|
1293
|
+
parameters: [input.scope.workspaceId, cutoff]
|
|
1294
|
+
},
|
|
1295
|
+
{
|
|
1296
|
+
sql: `DELETE FROM change_log
|
|
1297
|
+
WHERE workspace_id = ?
|
|
1298
|
+
AND created_at < ?
|
|
1299
|
+
AND (
|
|
1300
|
+
NOT EXISTS (
|
|
1301
|
+
SELECT 1 FROM device_cursors
|
|
1302
|
+
WHERE workspace_id = ?
|
|
1303
|
+
)
|
|
1304
|
+
OR server_version <= (
|
|
1305
|
+
SELECT MIN(last_seen_version)
|
|
1306
|
+
FROM device_cursors
|
|
1307
|
+
WHERE workspace_id = ?
|
|
1308
|
+
)
|
|
1309
|
+
)`,
|
|
1310
|
+
parameters: [
|
|
1311
|
+
input.scope.workspaceId,
|
|
1312
|
+
cutoff,
|
|
1313
|
+
input.scope.workspaceId,
|
|
1314
|
+
input.scope.workspaceId
|
|
1315
|
+
]
|
|
1316
|
+
}
|
|
1317
|
+
]);
|
|
1318
|
+
return;
|
|
1319
|
+
}
|
|
1184
1320
|
const raw = getRawDb();
|
|
1185
1321
|
raw.transaction(() => {
|
|
1322
|
+
raw.prepare(
|
|
1323
|
+
`DELETE FROM device_cursors
|
|
1324
|
+
WHERE workspace_id = ? AND updated_at < ?`
|
|
1325
|
+
).run(input.scope.workspaceId, cutoff);
|
|
1186
1326
|
const cursor = raw.prepare(
|
|
1187
1327
|
`SELECT MIN(last_seen_version) AS min_version
|
|
1188
1328
|
FROM device_cursors WHERE workspace_id = ?`
|
|
1189
1329
|
).get(input.scope.workspaceId);
|
|
1190
|
-
if (cursor.min_version === null)
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1330
|
+
if (cursor.min_version === null) {
|
|
1331
|
+
raw.prepare(
|
|
1332
|
+
`DELETE FROM change_log
|
|
1333
|
+
WHERE workspace_id = ? AND created_at < ?`
|
|
1334
|
+
).run(input.scope.workspaceId, cutoff);
|
|
1335
|
+
} else {
|
|
1336
|
+
raw.prepare(
|
|
1337
|
+
`DELETE FROM change_log
|
|
1338
|
+
WHERE workspace_id = ? AND created_at < ? AND server_version <= ?`
|
|
1339
|
+
).run(input.scope.workspaceId, cutoff, cursor.min_version);
|
|
1340
|
+
}
|
|
1195
1341
|
})();
|
|
1196
1342
|
}
|
|
1343
|
+
async reserveUploadIntentInD1(input, hash) {
|
|
1344
|
+
const now = nowEpoch();
|
|
1345
|
+
const liveRows = await d1All(
|
|
1346
|
+
`SELECT id, data_json FROM s_file_meta
|
|
1347
|
+
WHERE workspace_id = ? AND deleted = 0`,
|
|
1348
|
+
input.workspaceId
|
|
1349
|
+
);
|
|
1350
|
+
let usedBytes = 0;
|
|
1351
|
+
let alreadyStored = false;
|
|
1352
|
+
for (const row of liveRows) {
|
|
1353
|
+
const payload = JSON.parse(row.data_json);
|
|
1354
|
+
const size = payload.size_bytes ?? payload.sizeBytes;
|
|
1355
|
+
if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0) {
|
|
1356
|
+
throw createError({ statusCode: 500, statusMessage: "Invalid canonical file size" });
|
|
1357
|
+
}
|
|
1358
|
+
usedBytes += size;
|
|
1359
|
+
if (normalizeStorageHash(row.id) === hash) alreadyStored = true;
|
|
1360
|
+
}
|
|
1361
|
+
const [totals] = await d1All(
|
|
1362
|
+
`SELECT COALESCE(SUM(reserved_bytes), 0) AS reserved_bytes,
|
|
1363
|
+
MAX(CASE WHEN hash = ? AND reserved_bytes > 0 THEN 1 ELSE 0 END) AS same_hash
|
|
1364
|
+
FROM upload_intents
|
|
1365
|
+
WHERE workspace_id = ? AND status = 'active' AND expires_at > ?`,
|
|
1366
|
+
hash,
|
|
1367
|
+
input.workspaceId,
|
|
1368
|
+
now
|
|
1369
|
+
);
|
|
1370
|
+
const reservedBytes = alreadyStored || totals?.same_hash === 1 ? 0 : input.sizeBytes;
|
|
1371
|
+
if (input.workspaceQuotaBytes !== void 0 && usedBytes + (totals?.reserved_bytes ?? 0) + reservedBytes > input.workspaceQuotaBytes) {
|
|
1372
|
+
throw createError({ statusCode: 413, statusMessage: "Workspace storage quota exceeded" });
|
|
1373
|
+
}
|
|
1374
|
+
await d1Batch([
|
|
1375
|
+
{
|
|
1376
|
+
sql: `UPDATE upload_intents SET status = 'expired'
|
|
1377
|
+
WHERE workspace_id = ? AND status = 'active' AND expires_at <= ?`,
|
|
1378
|
+
parameters: [input.workspaceId, now]
|
|
1379
|
+
},
|
|
1380
|
+
{
|
|
1381
|
+
sql: `INSERT INTO upload_intents (
|
|
1382
|
+
id, workspace_id, hash, mime_type, size_bytes, reserved_bytes,
|
|
1383
|
+
expires_at, status, created_at
|
|
1384
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?)`,
|
|
1385
|
+
parameters: [
|
|
1386
|
+
input.intentId,
|
|
1387
|
+
input.workspaceId,
|
|
1388
|
+
hash,
|
|
1389
|
+
input.mimeType,
|
|
1390
|
+
input.sizeBytes,
|
|
1391
|
+
reservedBytes,
|
|
1392
|
+
input.expiresAt,
|
|
1393
|
+
now
|
|
1394
|
+
]
|
|
1395
|
+
}
|
|
1396
|
+
]);
|
|
1397
|
+
}
|
|
1398
|
+
async snapshotInD1(workspaceId, input, pageSize) {
|
|
1399
|
+
const now = nowEpoch();
|
|
1400
|
+
let header;
|
|
1401
|
+
let after;
|
|
1402
|
+
if (input.pageToken) {
|
|
1403
|
+
if (input.pageToken.length > 4096) {
|
|
1404
|
+
throw createError({ statusCode: 400, statusMessage: "Invalid snapshot page token" });
|
|
1405
|
+
}
|
|
1406
|
+
const token = decodeSnapshotPageToken(input.pageToken);
|
|
1407
|
+
const [existing] = await d1All(
|
|
1408
|
+
`SELECT id, workspace_id, high_watermark, tables_json, expires_at
|
|
1409
|
+
FROM sync_snapshots WHERE id = ?`,
|
|
1410
|
+
token.snapshotId
|
|
1411
|
+
);
|
|
1412
|
+
if (!existing || existing.expires_at <= now) {
|
|
1413
|
+
if (existing) {
|
|
1414
|
+
await d1Run("DELETE FROM sync_snapshots WHERE id = ?", existing.id);
|
|
1415
|
+
}
|
|
1416
|
+
throw createError({ statusCode: 410, statusMessage: "Snapshot expired or unavailable" });
|
|
1417
|
+
}
|
|
1418
|
+
if (existing.workspace_id !== workspaceId) {
|
|
1419
|
+
throw createError({ statusCode: 403, statusMessage: "Forbidden" });
|
|
1420
|
+
}
|
|
1421
|
+
if (input.tables && existing.tables_json !== JSON.stringify(normalizeSnapshotTables(input.tables))) {
|
|
1422
|
+
throw createError({
|
|
1423
|
+
statusCode: 400,
|
|
1424
|
+
statusMessage: "Snapshot table filter cannot change between pages"
|
|
1425
|
+
});
|
|
1426
|
+
}
|
|
1427
|
+
header = existing;
|
|
1428
|
+
after = token.after;
|
|
1429
|
+
} else {
|
|
1430
|
+
const tables = normalizeSnapshotTables(input.tables);
|
|
1431
|
+
const snapshotId = uid();
|
|
1432
|
+
const expiresAt = now + SNAPSHOT_TTL_SECONDS;
|
|
1433
|
+
const tablesJson = JSON.stringify(tables);
|
|
1434
|
+
const statements = [
|
|
1435
|
+
{
|
|
1436
|
+
sql: "DELETE FROM sync_snapshots WHERE expires_at <= ?",
|
|
1437
|
+
parameters: [now]
|
|
1438
|
+
},
|
|
1439
|
+
{
|
|
1440
|
+
sql: `INSERT INTO sync_snapshots (
|
|
1441
|
+
id, workspace_id, high_watermark, tables_json, created_at, expires_at
|
|
1442
|
+
) VALUES (
|
|
1443
|
+
?, ?,
|
|
1444
|
+
COALESCE((
|
|
1445
|
+
SELECT value FROM server_version_counter WHERE workspace_id = ?
|
|
1446
|
+
), 0),
|
|
1447
|
+
?, ?, ?
|
|
1448
|
+
)`,
|
|
1449
|
+
parameters: [
|
|
1450
|
+
snapshotId,
|
|
1451
|
+
workspaceId,
|
|
1452
|
+
workspaceId,
|
|
1453
|
+
tablesJson,
|
|
1454
|
+
now,
|
|
1455
|
+
expiresAt
|
|
1456
|
+
]
|
|
1457
|
+
}
|
|
1458
|
+
];
|
|
1459
|
+
for (const tableName of tables) {
|
|
1460
|
+
const materializedTable = SYNCED_TABLE_MAP[tableName];
|
|
1461
|
+
if (!materializedTable) {
|
|
1462
|
+
throw new Error(`Missing materialized table for ${tableName}`);
|
|
1463
|
+
}
|
|
1464
|
+
statements.push(
|
|
1465
|
+
{
|
|
1466
|
+
sql: `INSERT INTO sync_snapshot_items (
|
|
1467
|
+
snapshot_id, table_name, pk, kind, payload_json,
|
|
1468
|
+
clock, hlc, op_id, server_deleted_at
|
|
1469
|
+
)
|
|
1470
|
+
SELECT
|
|
1471
|
+
?, ?, id, 'row', data_json,
|
|
1472
|
+
clock,
|
|
1473
|
+
CASE WHEN hlc <> '' THEN hlc ELSE 'legacy:0' END,
|
|
1474
|
+
CASE
|
|
1475
|
+
WHEN op_id <> '' THEN op_id
|
|
1476
|
+
ELSE 'legacy:${tableName}:' || id || ':' || clock || ':' || hlc
|
|
1477
|
+
END,
|
|
1478
|
+
NULL
|
|
1479
|
+
FROM "${materializedTable}"
|
|
1480
|
+
WHERE workspace_id = ? AND deleted = 0`,
|
|
1481
|
+
parameters: [snapshotId, tableName, workspaceId]
|
|
1482
|
+
},
|
|
1483
|
+
{
|
|
1484
|
+
sql: `INSERT INTO sync_snapshot_items (
|
|
1485
|
+
snapshot_id, table_name, pk, kind, payload_json,
|
|
1486
|
+
clock, hlc, op_id, server_deleted_at
|
|
1487
|
+
)
|
|
1488
|
+
SELECT
|
|
1489
|
+
?, ?, materialized.id, 'tombstone', NULL,
|
|
1490
|
+
materialized.clock,
|
|
1491
|
+
CASE
|
|
1492
|
+
WHEN materialized.hlc <> '' THEN materialized.hlc
|
|
1493
|
+
ELSE 'legacy:0'
|
|
1494
|
+
END,
|
|
1495
|
+
CASE
|
|
1496
|
+
WHEN materialized.op_id <> '' THEN materialized.op_id
|
|
1497
|
+
ELSE 'legacy:tombstone:${tableName}:' || materialized.id || ':' || materialized.clock
|
|
1498
|
+
END,
|
|
1499
|
+
MAX(0, COALESCE(
|
|
1500
|
+
(
|
|
1501
|
+
SELECT tombstone.deleted_at
|
|
1502
|
+
FROM tombstones AS tombstone
|
|
1503
|
+
WHERE tombstone.workspace_id = materialized.workspace_id
|
|
1504
|
+
AND tombstone.table_name = ?
|
|
1505
|
+
AND tombstone.pk = materialized.id
|
|
1506
|
+
LIMIT 1
|
|
1507
|
+
),
|
|
1508
|
+
materialized.updated_at,
|
|
1509
|
+
materialized.created_at,
|
|
1510
|
+
0
|
|
1511
|
+
))
|
|
1512
|
+
FROM "${materializedTable}" AS materialized
|
|
1513
|
+
WHERE materialized.workspace_id = ? AND materialized.deleted <> 0`,
|
|
1514
|
+
parameters: [snapshotId, tableName, tableName, workspaceId]
|
|
1515
|
+
},
|
|
1516
|
+
{
|
|
1517
|
+
sql: `INSERT INTO sync_snapshot_items (
|
|
1518
|
+
snapshot_id, table_name, pk, kind, payload_json,
|
|
1519
|
+
clock, hlc, op_id, server_deleted_at
|
|
1520
|
+
)
|
|
1521
|
+
SELECT
|
|
1522
|
+
?, ?, tombstone.pk, 'tombstone', NULL,
|
|
1523
|
+
tombstone.clock,
|
|
1524
|
+
CASE WHEN tombstone.hlc <> '' THEN tombstone.hlc ELSE 'legacy:0' END,
|
|
1525
|
+
CASE
|
|
1526
|
+
WHEN tombstone.op_id <> '' THEN tombstone.op_id
|
|
1527
|
+
ELSE 'legacy:tombstone:${tableName}:' || tombstone.pk || ':' || tombstone.clock
|
|
1528
|
+
END,
|
|
1529
|
+
MAX(0, tombstone.deleted_at)
|
|
1530
|
+
FROM tombstones AS tombstone
|
|
1531
|
+
WHERE tombstone.workspace_id = ?
|
|
1532
|
+
AND tombstone.table_name = ?
|
|
1533
|
+
AND NOT EXISTS (
|
|
1534
|
+
SELECT 1
|
|
1535
|
+
FROM "${materializedTable}" AS materialized
|
|
1536
|
+
WHERE materialized.workspace_id = tombstone.workspace_id
|
|
1537
|
+
AND materialized.id = tombstone.pk
|
|
1538
|
+
)`,
|
|
1539
|
+
parameters: [snapshotId, tableName, workspaceId, tableName]
|
|
1540
|
+
}
|
|
1541
|
+
);
|
|
1542
|
+
}
|
|
1543
|
+
await d1Batch(statements);
|
|
1544
|
+
const [created] = await d1All(
|
|
1545
|
+
`SELECT id, workspace_id, high_watermark, tables_json, expires_at
|
|
1546
|
+
FROM sync_snapshots WHERE id = ?`,
|
|
1547
|
+
snapshotId
|
|
1548
|
+
);
|
|
1549
|
+
if (!created) throw new Error("D1 did not create a sync snapshot.");
|
|
1550
|
+
header = created;
|
|
1551
|
+
}
|
|
1552
|
+
const rows = after ? await d1All(
|
|
1553
|
+
`SELECT table_name, pk, kind, payload_json, clock, hlc, op_id, server_deleted_at
|
|
1554
|
+
FROM sync_snapshot_items
|
|
1555
|
+
WHERE snapshot_id = ?
|
|
1556
|
+
AND (
|
|
1557
|
+
table_name > ?
|
|
1558
|
+
OR (table_name = ? AND pk > ?)
|
|
1559
|
+
OR (table_name = ? AND pk = ? AND kind > ?)
|
|
1560
|
+
)
|
|
1561
|
+
ORDER BY table_name ASC, pk ASC, kind ASC
|
|
1562
|
+
LIMIT ?`,
|
|
1563
|
+
header.id,
|
|
1564
|
+
after.tableName,
|
|
1565
|
+
after.tableName,
|
|
1566
|
+
after.pk,
|
|
1567
|
+
after.tableName,
|
|
1568
|
+
after.pk,
|
|
1569
|
+
after.kind,
|
|
1570
|
+
pageSize + 1
|
|
1571
|
+
) : await d1All(
|
|
1572
|
+
`SELECT table_name, pk, kind, payload_json, clock, hlc, op_id, server_deleted_at
|
|
1573
|
+
FROM sync_snapshot_items
|
|
1574
|
+
WHERE snapshot_id = ?
|
|
1575
|
+
ORDER BY table_name ASC, pk ASC, kind ASC
|
|
1576
|
+
LIMIT ?`,
|
|
1577
|
+
header.id,
|
|
1578
|
+
pageSize + 1
|
|
1579
|
+
);
|
|
1580
|
+
const hasMore = rows.length > pageSize;
|
|
1581
|
+
const pageRows = hasMore ? rows.slice(0, pageSize) : rows;
|
|
1582
|
+
const items = pageRows.map((row) => {
|
|
1583
|
+
const revision = { clock: row.clock, hlc: row.hlc, opId: row.op_id };
|
|
1584
|
+
if (row.kind === "row") {
|
|
1585
|
+
return {
|
|
1586
|
+
kind: "row",
|
|
1587
|
+
tableName: row.table_name,
|
|
1588
|
+
pk: row.pk,
|
|
1589
|
+
payload: JSON.parse(row.payload_json ?? "null"),
|
|
1590
|
+
revision
|
|
1591
|
+
};
|
|
1592
|
+
}
|
|
1593
|
+
return {
|
|
1594
|
+
kind: "tombstone",
|
|
1595
|
+
tableName: row.table_name,
|
|
1596
|
+
pk: row.pk,
|
|
1597
|
+
revision,
|
|
1598
|
+
serverDeletedAt: row.server_deleted_at ?? 0
|
|
1599
|
+
};
|
|
1600
|
+
});
|
|
1601
|
+
const lastRow = pageRows.at(-1);
|
|
1602
|
+
return {
|
|
1603
|
+
workspaceId: header.workspace_id,
|
|
1604
|
+
snapshotId: header.id,
|
|
1605
|
+
highWatermark: header.high_watermark,
|
|
1606
|
+
items,
|
|
1607
|
+
nextPageToken: hasMore && lastRow ? encodeSnapshotPageToken({
|
|
1608
|
+
version: 1,
|
|
1609
|
+
snapshotId: header.id,
|
|
1610
|
+
after: {
|
|
1611
|
+
tableName: lastRow.table_name,
|
|
1612
|
+
pk: lastRow.pk,
|
|
1613
|
+
kind: lastRow.kind
|
|
1614
|
+
}
|
|
1615
|
+
}) : null
|
|
1616
|
+
};
|
|
1617
|
+
}
|
|
1618
|
+
async updateCursorInD1(workspaceId, deviceId, version, ownerUserId, now) {
|
|
1619
|
+
const [counter] = await d1All(
|
|
1620
|
+
"SELECT value FROM server_version_counter WHERE workspace_id = ?",
|
|
1621
|
+
workspaceId
|
|
1622
|
+
);
|
|
1623
|
+
if (version > (counter?.value ?? 0)) {
|
|
1624
|
+
throw createError({ statusCode: 400, statusMessage: "Cursor exceeds workspace version" });
|
|
1625
|
+
}
|
|
1626
|
+
const [existing] = await d1All(
|
|
1627
|
+
`SELECT owner_user_id, last_seen_version
|
|
1628
|
+
FROM device_cursors WHERE workspace_id = ? AND device_id = ?`,
|
|
1629
|
+
workspaceId,
|
|
1630
|
+
deviceId
|
|
1631
|
+
);
|
|
1632
|
+
if (existing?.owner_user_id && existing.owner_user_id !== ownerUserId) {
|
|
1633
|
+
throw createError({ statusCode: 403, statusMessage: "Device cursor belongs to another user" });
|
|
1634
|
+
}
|
|
1635
|
+
if (existing && version < existing.last_seen_version) {
|
|
1636
|
+
throw createError({ statusCode: 409, statusMessage: "Device cursor cannot regress" });
|
|
1637
|
+
}
|
|
1638
|
+
const result = await d1Run(
|
|
1639
|
+
`INSERT INTO device_cursors (
|
|
1640
|
+
id, workspace_id, device_id, owner_user_id, last_seen_version, updated_at
|
|
1641
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
1642
|
+
ON CONFLICT(workspace_id, device_id) DO UPDATE SET
|
|
1643
|
+
owner_user_id = COALESCE(device_cursors.owner_user_id, excluded.owner_user_id),
|
|
1644
|
+
last_seen_version = excluded.last_seen_version,
|
|
1645
|
+
updated_at = excluded.updated_at
|
|
1646
|
+
WHERE (device_cursors.owner_user_id IS NULL OR device_cursors.owner_user_id = ?)
|
|
1647
|
+
AND excluded.last_seen_version >= device_cursors.last_seen_version`,
|
|
1648
|
+
uid(),
|
|
1649
|
+
workspaceId,
|
|
1650
|
+
deviceId,
|
|
1651
|
+
ownerUserId,
|
|
1652
|
+
version,
|
|
1653
|
+
now,
|
|
1654
|
+
ownerUserId
|
|
1655
|
+
);
|
|
1656
|
+
if ((result.meta?.changes ?? 0) !== 1) {
|
|
1657
|
+
throw createError({ statusCode: 409, statusMessage: "Device cursor changed concurrently" });
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
async pushInD1(input) {
|
|
1661
|
+
const { event, workspaceId, uniqueOps, resultSlots, indicesByOpId } = input;
|
|
1662
|
+
const existingOps = /* @__PURE__ */ new Map();
|
|
1663
|
+
const opIds = uniqueOps.map((op) => op.stamp.opId);
|
|
1664
|
+
for (let offset = 0; offset < opIds.length; offset += 500) {
|
|
1665
|
+
const chunk = opIds.slice(offset, offset + 500);
|
|
1666
|
+
if (chunk.length === 0) continue;
|
|
1667
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1668
|
+
const rows = await d1All(
|
|
1669
|
+
`SELECT op_id, server_version FROM change_log WHERE op_id IN (${placeholders})`,
|
|
1670
|
+
...chunk
|
|
1671
|
+
);
|
|
1672
|
+
for (const row of rows) existingOps.set(row.op_id, row.server_version);
|
|
1673
|
+
}
|
|
1674
|
+
const newOps = uniqueOps.filter((op) => !existingOps.has(op.stamp.opId));
|
|
1675
|
+
const versionOffsets = /* @__PURE__ */ new Map();
|
|
1676
|
+
newOps.forEach((op, index) => {
|
|
1677
|
+
versionOffsets.set(op.stamp.opId, index + 1);
|
|
1678
|
+
});
|
|
1679
|
+
const existingMaterializedKeys = /* @__PURE__ */ new Set();
|
|
1680
|
+
const opsByTable = /* @__PURE__ */ new Map();
|
|
1681
|
+
for (const op of newOps) {
|
|
1682
|
+
const materializedTable = SYNCED_TABLE_MAP[op.tableName];
|
|
1683
|
+
if (!materializedTable) continue;
|
|
1684
|
+
const tableOps = opsByTable.get(materializedTable) ?? [];
|
|
1685
|
+
tableOps.push(op);
|
|
1686
|
+
opsByTable.set(materializedTable, tableOps);
|
|
1687
|
+
}
|
|
1688
|
+
for (const [materializedTable, tableOps] of opsByTable) {
|
|
1689
|
+
const ids = [...new Set(tableOps.map((op) => op.pk))];
|
|
1690
|
+
for (let offset = 0; offset < ids.length; offset += 500) {
|
|
1691
|
+
const chunk = ids.slice(offset, offset + 500);
|
|
1692
|
+
if (chunk.length === 0) continue;
|
|
1693
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1694
|
+
const rows = await d1All(
|
|
1695
|
+
`SELECT id FROM "${materializedTable}"
|
|
1696
|
+
WHERE workspace_id = ? AND id IN (${placeholders})`,
|
|
1697
|
+
workspaceId,
|
|
1698
|
+
...chunk
|
|
1699
|
+
);
|
|
1700
|
+
for (const row of rows) {
|
|
1701
|
+
existingMaterializedKeys.add(`${materializedTable}\0${row.id}`);
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
}
|
|
1705
|
+
const now = nowEpoch();
|
|
1706
|
+
const userId = resolveSessionUserId(event);
|
|
1707
|
+
const statements = [];
|
|
1708
|
+
if (newOps.length > 0) {
|
|
1709
|
+
statements.push(
|
|
1710
|
+
{
|
|
1711
|
+
sql: `INSERT OR IGNORE INTO server_version_counter (workspace_id, value)
|
|
1712
|
+
VALUES (?, 0)`,
|
|
1713
|
+
parameters: [workspaceId]
|
|
1714
|
+
},
|
|
1715
|
+
{
|
|
1716
|
+
sql: "UPDATE server_version_counter SET value = value + ? WHERE workspace_id = ?",
|
|
1717
|
+
parameters: [newOps.length, workspaceId]
|
|
1718
|
+
}
|
|
1719
|
+
);
|
|
1720
|
+
}
|
|
1721
|
+
const materializedStatementIndex = /* @__PURE__ */ new Map();
|
|
1722
|
+
const materializedWasExisting = /* @__PURE__ */ new Map();
|
|
1723
|
+
const seenMaterializedKeys = /* @__PURE__ */ new Set();
|
|
1724
|
+
for (const op of newOps) {
|
|
1725
|
+
const materializedTable = SYNCED_TABLE_MAP[op.tableName];
|
|
1726
|
+
if (!materializedTable) continue;
|
|
1727
|
+
const versionOffset = versionOffsets.get(op.stamp.opId);
|
|
1728
|
+
if (versionOffset === void 0) continue;
|
|
1729
|
+
const payloadJson = op.payload == null ? null : JSON.stringify(op.payload);
|
|
1730
|
+
const materializedKey = `${materializedTable}\0${op.pk}`;
|
|
1731
|
+
materializedWasExisting.set(
|
|
1732
|
+
op.stamp.opId,
|
|
1733
|
+
existingMaterializedKeys.has(materializedKey) || seenMaterializedKeys.has(materializedKey)
|
|
1734
|
+
);
|
|
1735
|
+
seenMaterializedKeys.add(materializedKey);
|
|
1736
|
+
statements.push({
|
|
1737
|
+
sql: `INSERT OR IGNORE INTO change_log (
|
|
1738
|
+
id, workspace_id, server_version, table_name, pk, op,
|
|
1739
|
+
payload_json, clock, hlc, device_id, op_id, created_at
|
|
1740
|
+
) VALUES (
|
|
1741
|
+
?, ?,
|
|
1742
|
+
(SELECT value - ? + ? FROM server_version_counter WHERE workspace_id = ?),
|
|
1743
|
+
?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
1744
|
+
)`,
|
|
1745
|
+
parameters: [
|
|
1746
|
+
uid(),
|
|
1747
|
+
workspaceId,
|
|
1748
|
+
newOps.length,
|
|
1749
|
+
versionOffset,
|
|
1750
|
+
workspaceId,
|
|
1751
|
+
op.tableName,
|
|
1752
|
+
op.pk,
|
|
1753
|
+
op.operation,
|
|
1754
|
+
payloadJson,
|
|
1755
|
+
op.stamp.clock,
|
|
1756
|
+
op.stamp.hlc,
|
|
1757
|
+
op.stamp.deviceId,
|
|
1758
|
+
op.stamp.opId,
|
|
1759
|
+
now
|
|
1760
|
+
]
|
|
1761
|
+
});
|
|
1762
|
+
materializedStatementIndex.set(op.stamp.opId, statements.length);
|
|
1763
|
+
if (op.operation === "put") {
|
|
1764
|
+
statements.push({
|
|
1765
|
+
sql: `INSERT INTO "${materializedTable}" (
|
|
1766
|
+
id, workspace_id, data_json, clock, hlc, device_id,
|
|
1767
|
+
op_id, deleted, created_at, updated_at
|
|
1768
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?)
|
|
1769
|
+
ON CONFLICT(workspace_id, id) DO UPDATE SET
|
|
1770
|
+
data_json = excluded.data_json,
|
|
1771
|
+
clock = excluded.clock,
|
|
1772
|
+
hlc = excluded.hlc,
|
|
1773
|
+
device_id = excluded.device_id,
|
|
1774
|
+
op_id = excluded.op_id,
|
|
1775
|
+
deleted = 0,
|
|
1776
|
+
updated_at = excluded.updated_at
|
|
1777
|
+
WHERE excluded.clock > "${materializedTable}".clock
|
|
1778
|
+
OR (
|
|
1779
|
+
excluded.clock = "${materializedTable}".clock
|
|
1780
|
+
AND excluded.hlc > "${materializedTable}".hlc
|
|
1781
|
+
)`,
|
|
1782
|
+
parameters: [
|
|
1783
|
+
op.pk,
|
|
1784
|
+
workspaceId,
|
|
1785
|
+
payloadJson ?? "{}",
|
|
1786
|
+
op.stamp.clock,
|
|
1787
|
+
op.stamp.hlc,
|
|
1788
|
+
op.stamp.deviceId,
|
|
1789
|
+
op.stamp.opId,
|
|
1790
|
+
now,
|
|
1791
|
+
now
|
|
1792
|
+
]
|
|
1793
|
+
});
|
|
1794
|
+
} else {
|
|
1795
|
+
statements.push({
|
|
1796
|
+
sql: `INSERT INTO "${materializedTable}" (
|
|
1797
|
+
id, workspace_id, data_json, clock, hlc, device_id,
|
|
1798
|
+
op_id, deleted, created_at, updated_at
|
|
1799
|
+
) VALUES (?, ?, '{}', ?, ?, ?, ?, 1, ?, ?)
|
|
1800
|
+
ON CONFLICT(workspace_id, id) DO UPDATE SET
|
|
1801
|
+
clock = excluded.clock,
|
|
1802
|
+
hlc = excluded.hlc,
|
|
1803
|
+
device_id = excluded.device_id,
|
|
1804
|
+
op_id = excluded.op_id,
|
|
1805
|
+
deleted = 1,
|
|
1806
|
+
updated_at = excluded.updated_at
|
|
1807
|
+
WHERE excluded.clock > "${materializedTable}".clock
|
|
1808
|
+
OR (
|
|
1809
|
+
excluded.clock = "${materializedTable}".clock
|
|
1810
|
+
AND excluded.hlc > "${materializedTable}".hlc
|
|
1811
|
+
)`,
|
|
1812
|
+
parameters: [
|
|
1813
|
+
op.pk,
|
|
1814
|
+
workspaceId,
|
|
1815
|
+
op.stamp.clock,
|
|
1816
|
+
op.stamp.hlc,
|
|
1817
|
+
op.stamp.deviceId,
|
|
1818
|
+
op.stamp.opId,
|
|
1819
|
+
now,
|
|
1820
|
+
now
|
|
1821
|
+
]
|
|
1822
|
+
});
|
|
1823
|
+
statements.push({
|
|
1824
|
+
sql: `INSERT INTO tombstones (
|
|
1825
|
+
id, workspace_id, table_name, pk, deleted_at, clock,
|
|
1826
|
+
hlc, op_id, server_version, created_at
|
|
1827
|
+
) VALUES (
|
|
1828
|
+
?, ?, ?, ?, ?, ?, ?, ?,
|
|
1829
|
+
(SELECT value - ? + ? FROM server_version_counter WHERE workspace_id = ?),
|
|
1830
|
+
?
|
|
1831
|
+
)
|
|
1832
|
+
ON CONFLICT(workspace_id, table_name, pk) DO UPDATE SET
|
|
1833
|
+
deleted_at = excluded.deleted_at,
|
|
1834
|
+
clock = excluded.clock,
|
|
1835
|
+
hlc = excluded.hlc,
|
|
1836
|
+
op_id = excluded.op_id,
|
|
1837
|
+
server_version = excluded.server_version
|
|
1838
|
+
WHERE excluded.clock > tombstones.clock
|
|
1839
|
+
OR (
|
|
1840
|
+
excluded.clock = tombstones.clock
|
|
1841
|
+
AND excluded.server_version > tombstones.server_version
|
|
1842
|
+
)`,
|
|
1843
|
+
parameters: [
|
|
1844
|
+
uid(),
|
|
1845
|
+
workspaceId,
|
|
1846
|
+
op.tableName,
|
|
1847
|
+
op.pk,
|
|
1848
|
+
now,
|
|
1849
|
+
op.stamp.clock,
|
|
1850
|
+
op.stamp.hlc,
|
|
1851
|
+
op.stamp.opId,
|
|
1852
|
+
newOps.length,
|
|
1853
|
+
versionOffset,
|
|
1854
|
+
workspaceId,
|
|
1855
|
+
now
|
|
1856
|
+
]
|
|
1857
|
+
});
|
|
1858
|
+
}
|
|
1859
|
+
}
|
|
1860
|
+
const finalVersionResultIndex = newOps.length > 0 ? statements.length : void 0;
|
|
1861
|
+
if (finalVersionResultIndex !== void 0) {
|
|
1862
|
+
statements.push({
|
|
1863
|
+
sql: "SELECT value FROM server_version_counter WHERE workspace_id = ?",
|
|
1864
|
+
parameters: [workspaceId]
|
|
1865
|
+
});
|
|
1866
|
+
}
|
|
1867
|
+
const batchResults = statements.length > 0 ? await d1Batch(statements) : [];
|
|
1868
|
+
let finalServerVersion = 0;
|
|
1869
|
+
if (finalVersionResultIndex !== void 0) {
|
|
1870
|
+
const row = batchResults[finalVersionResultIndex]?.results?.[0];
|
|
1871
|
+
if (typeof row?.value !== "number") {
|
|
1872
|
+
throw new Error("D1 did not return a workspace version counter.");
|
|
1873
|
+
}
|
|
1874
|
+
finalServerVersion = row.value;
|
|
1875
|
+
} else if (uniqueOps.length > 0) {
|
|
1876
|
+
const [counter] = await d1All(
|
|
1877
|
+
"SELECT value FROM server_version_counter WHERE workspace_id = ?",
|
|
1878
|
+
workspaceId
|
|
1879
|
+
);
|
|
1880
|
+
finalServerVersion = counter?.value ?? 0;
|
|
1881
|
+
}
|
|
1882
|
+
const resolvedOps = /* @__PURE__ */ new Map();
|
|
1883
|
+
for (let offset = 0; offset < opIds.length; offset += 500) {
|
|
1884
|
+
const chunk = opIds.slice(offset, offset + 500);
|
|
1885
|
+
if (chunk.length === 0) continue;
|
|
1886
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1887
|
+
const rows = await d1All(
|
|
1888
|
+
`SELECT op_id, server_version FROM change_log WHERE op_id IN (${placeholders})`,
|
|
1889
|
+
...chunk
|
|
1890
|
+
);
|
|
1891
|
+
for (const row of rows) resolvedOps.set(row.op_id, row.server_version);
|
|
1892
|
+
}
|
|
1893
|
+
const emissions = [];
|
|
1894
|
+
for (const op of uniqueOps) {
|
|
1895
|
+
const serverVersion = resolvedOps.get(op.stamp.opId);
|
|
1896
|
+
if (serverVersion === void 0) {
|
|
1897
|
+
throw new Error(`D1 failed to resolve sync operation ${op.stamp.opId}.`);
|
|
1898
|
+
}
|
|
1899
|
+
const result = {
|
|
1900
|
+
opId: op.stamp.opId,
|
|
1901
|
+
success: true,
|
|
1902
|
+
serverVersion
|
|
1903
|
+
};
|
|
1904
|
+
for (const index of indicesByOpId.get(op.stamp.opId) ?? []) {
|
|
1905
|
+
resultSlots[index] = result;
|
|
1906
|
+
}
|
|
1907
|
+
const materializedResult = batchResults[materializedStatementIndex.get(op.stamp.opId) ?? -1];
|
|
1908
|
+
if ((materializedResult?.meta?.changes ?? 0) > 0) {
|
|
1909
|
+
const emission = resolveHookEmission({
|
|
1910
|
+
op,
|
|
1911
|
+
workspaceId,
|
|
1912
|
+
now,
|
|
1913
|
+
userId,
|
|
1914
|
+
wasExisting: materializedWasExisting.get(op.stamp.opId) ?? false,
|
|
1915
|
+
applied: true
|
|
1916
|
+
});
|
|
1917
|
+
if (emission) emissions.push(emission);
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
for (const emission of emissions) {
|
|
1921
|
+
await emitWebhookSystemHook(emission.hookName, emission.payload);
|
|
1922
|
+
}
|
|
1923
|
+
return {
|
|
1924
|
+
results: resultSlots.filter(
|
|
1925
|
+
(result) => Boolean(result)
|
|
1926
|
+
),
|
|
1927
|
+
serverVersion: finalServerVersion
|
|
1928
|
+
};
|
|
1929
|
+
}
|
|
1930
|
+
async listWorkspaceIds() {
|
|
1931
|
+
if (isD1Driver()) {
|
|
1932
|
+
const rows2 = await d1All(
|
|
1933
|
+
"SELECT id FROM workspaces WHERE deleted = 0 ORDER BY id ASC"
|
|
1934
|
+
);
|
|
1935
|
+
return rows2.map((row) => row.id);
|
|
1936
|
+
}
|
|
1937
|
+
const raw = getRawDb();
|
|
1938
|
+
const rows = raw.prepare(
|
|
1939
|
+
"SELECT id FROM workspaces WHERE deleted = 0 ORDER BY id ASC"
|
|
1940
|
+
).all();
|
|
1941
|
+
return rows.map((row) => row.id);
|
|
1942
|
+
}
|
|
1943
|
+
getMaintenanceState() {
|
|
1944
|
+
return getSyncMaintenanceState();
|
|
1945
|
+
}
|
|
1946
|
+
beginMaintenanceRun() {
|
|
1947
|
+
beginSyncMaintenanceRun();
|
|
1948
|
+
}
|
|
1949
|
+
completeMaintenanceRun(input) {
|
|
1950
|
+
const backlog = computeSyncMaintenanceBacklog(SYNC_MAINTENANCE_RETENTION_SECONDS, true);
|
|
1951
|
+
completeSyncMaintenanceRun({ lastRun: input.lastRun, backlog });
|
|
1952
|
+
}
|
|
1953
|
+
failMaintenanceRun(error) {
|
|
1954
|
+
failSyncMaintenanceRun(error);
|
|
1955
|
+
}
|
|
1197
1956
|
}
|
|
1198
1957
|
export function createSqliteSyncGatewayAdapter() {
|
|
1199
1958
|
return new SqliteSyncGatewayAdapter();
|