or3-provider-sqlite 0.0.4 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/README.md +112 -19
  2. package/dist/module.json +1 -1
  3. package/dist/runtime/server/admin/adapters/sync-sqlite.js +90 -16
  4. package/dist/runtime/server/admin/stores/sqlite-store.js +7 -8
  5. package/dist/runtime/server/auth/sqlite-auth-workspace-store.d.ts +5 -0
  6. package/dist/runtime/server/auth/sqlite-auth-workspace-store.js +368 -40
  7. package/dist/runtime/server/connect/sqlite-connect-store.js +19 -5
  8. package/dist/runtime/server/db/d1-dialect.d.ts +39 -0
  9. package/dist/runtime/server/db/d1-dialect.js +70 -0
  10. package/dist/runtime/server/db/d1.d.ts +13 -0
  11. package/dist/runtime/server/db/d1.js +30 -0
  12. package/dist/runtime/server/db/kysely.d.ts +53 -15
  13. package/dist/runtime/server/db/kysely.js +374 -16
  14. package/dist/runtime/server/db/migrate.js +3 -1
  15. package/dist/runtime/server/db/migrations/016_connect_runtime_binding.d.ts +4 -0
  16. package/dist/runtime/server/db/migrations/016_connect_runtime_binding.js +28 -0
  17. package/dist/runtime/server/db/schema.d.ts +3 -0
  18. package/dist/runtime/server/plugins/register.js +33 -13
  19. package/dist/runtime/server/rate-limit/sqlite-provider.d.ts +2 -0
  20. package/dist/runtime/server/rate-limit/sqlite-provider.js +70 -1
  21. package/dist/runtime/server/sync/maintenance-state.d.ts +26 -0
  22. package/dist/runtime/server/sync/maintenance-state.js +66 -0
  23. package/dist/runtime/server/sync/sqlite-sync-gateway-adapter.d.ts +15 -4
  24. package/dist/runtime/server/sync/sqlite-sync-gateway-adapter.js +780 -21
  25. package/dist/runtime/server/webhooks/sqlite-webhook-store.d.ts +3 -2
  26. package/dist/runtime/server/webhooks/sqlite-webhook-store.js +90 -39
  27. package/package.json +10 -6
@@ -1,7 +1,15 @@
1
1
  import { canRunSyncHistoryGc } from "./history-gc-policy.js";
2
2
  import { createError } from "h3";
3
- import { randomUUID } from "node:crypto";
4
- import { getSqliteDb, getRawDb } from "../db/kysely.js";
3
+ import {
4
+ beginSyncMaintenanceRun,
5
+ completeSyncMaintenanceRun,
6
+ computeSyncMaintenanceBacklog,
7
+ failSyncMaintenanceRun,
8
+ getSyncMaintenanceState,
9
+ SYNC_MAINTENANCE_RETENTION_SECONDS
10
+ } from "./maintenance-state.js";
11
+ import { getSqliteDb, getRawDb, isD1Driver } from "../db/kysely.js";
12
+ import { d1All, d1Batch, d1Run } from "../db/d1.js";
5
13
  import { SYNCED_TABLE_MAP, ALLOWED_SYNC_TABLES } from "../db/schema.js";
6
14
  import { emitWebhookSystemHook } from "~~/server/utils/webhooks/runtime";
7
15
  const DEFAULT_PULL_LIMIT = 100;
@@ -14,7 +22,7 @@ const SESSION_CONTEXT_KEY_PREFIX = "__or3_session_context_";
14
22
  const MIN_SYNC_RETENTION_SECONDS = 60 * 60;
15
23
  const MAX_SYNC_RETENTION_SECONDS = 365 * 24 * 60 * 60;
16
24
  function uid() {
17
- return randomUUID();
25
+ return globalThis.crypto.randomUUID();
18
26
  }
19
27
  function nowEpoch() {
20
28
  return Math.floor(Date.now() / 1e3);
@@ -302,6 +310,9 @@ export class SqliteSyncGatewayAdapter {
302
310
  throw createError({ statusCode: 400, statusMessage: "Invalid workspace quota" });
303
311
  }
304
312
  const hash = normalizeStorageHash(input.hash);
313
+ if (isD1Driver()) {
314
+ return this.reserveUploadIntentInD1(input, hash);
315
+ }
305
316
  const raw = getRawDb();
306
317
  raw.transaction(() => {
307
318
  const now = nowEpoch();
@@ -345,6 +356,27 @@ export class SqliteSyncGatewayAdapter {
345
356
  }
346
357
  async consumeUploadIntent(event, input) {
347
358
  await assertWorkspaceScopeAuthorized(event, input.workspaceId);
359
+ if (isD1Driver()) {
360
+ const now = nowEpoch();
361
+ const result2 = await d1Run(
362
+ `UPDATE upload_intents
363
+ SET status = 'consumed', storage_id = ?, consumed_at = ?
364
+ WHERE id = ? AND workspace_id = ? AND status = 'active' AND expires_at > ?
365
+ AND hash = ? AND mime_type = ? AND size_bytes = ?`,
366
+ input.storageId,
367
+ now,
368
+ input.intentId,
369
+ input.workspaceId,
370
+ now,
371
+ normalizeStorageHash(input.hash),
372
+ input.mimeType,
373
+ input.sizeBytes
374
+ );
375
+ if ((result2.meta?.changes ?? 0) !== 1) {
376
+ throw createError({ statusCode: 409, statusMessage: "Upload intent expired, mismatched, or already consumed" });
377
+ }
378
+ return;
379
+ }
348
380
  const raw = getRawDb();
349
381
  const result = raw.prepare(`UPDATE upload_intents
350
382
  SET status = 'consumed', storage_id = ?, consumed_at = ?
@@ -365,6 +397,19 @@ export class SqliteSyncGatewayAdapter {
365
397
  }
366
398
  async cancelUploadIntent(event, input) {
367
399
  await assertWorkspaceScopeAuthorized(event, input.workspaceId);
400
+ if (isD1Driver()) {
401
+ const result2 = await d1Run(
402
+ `UPDATE upload_intents SET status = 'cancelled', cancelled_at = ?
403
+ WHERE id = ? AND workspace_id = ? AND status = 'active'`,
404
+ nowEpoch(),
405
+ input.intentId,
406
+ input.workspaceId
407
+ );
408
+ if ((result2.meta?.changes ?? 0) !== 1) {
409
+ throw createError({ statusCode: 409, statusMessage: "Upload intent is not active" });
410
+ }
411
+ return;
412
+ }
368
413
  const result = getRawDb().prepare(`UPDATE upload_intents
369
414
  SET status = 'cancelled', cancelled_at = ?
370
415
  WHERE id = ? AND workspace_id = ? AND status = 'active'`).run(nowEpoch(), input.intentId, input.workspaceId);
@@ -381,17 +426,18 @@ export class SqliteSyncGatewayAdapter {
381
426
  throw createError({ statusCode: 400, statusMessage: "Invalid storage hash" });
382
427
  }
383
428
  const after = decodeCanonicalStorageCursor(input.cursor, input.kind, hash);
384
- const raw = getRawDb();
429
+ const raw = isD1Driver() ? null : getRawDb();
430
+ const all = async (sql, ...parameters) => raw ? raw.prepare(sql).all(...parameters) : d1All(sql, ...parameters);
385
431
  if (input.kind === "active_reservations") {
386
432
  if (after.length > 1) {
387
433
  throw createError({ statusCode: 400, statusMessage: "Invalid canonical storage cursor" });
388
434
  }
389
435
  const now = input.now ?? nowEpoch();
390
- const rows2 = raw.prepare(`SELECT id, hash, reserved_bytes, expires_at
436
+ const rows2 = await all(`SELECT id, hash, reserved_bytes, expires_at
391
437
  FROM upload_intents
392
438
  WHERE workspace_id = ? AND status = 'active' AND expires_at > ?
393
439
  AND id > ? AND reserved_bytes > 0 AND (? IS NULL OR hash = ?)
394
- ORDER BY id ASC LIMIT ?`).all(workspaceId, now, after[0] ?? "", hash ?? null, hash ?? null, limit + 1);
440
+ ORDER BY id ASC LIMIT ?`, workspaceId, now, after[0] ?? "", hash ?? null, hash ?? null, limit + 1);
395
441
  const hasMore2 = rows2.length > limit;
396
442
  const page2 = hasMore2 ? rows2.slice(0, limit) : rows2;
397
443
  const last2 = page2[page2.length - 1];
@@ -416,7 +462,7 @@ export class SqliteSyncGatewayAdapter {
416
462
  if (after.length > 1) {
417
463
  throw createError({ statusCode: 400, statusMessage: "Invalid canonical storage cursor" });
418
464
  }
419
- const rows2 = raw.prepare(`
465
+ const rows2 = await all(`
420
466
  SELECT id, data_json, updated_at
421
467
  FROM s_file_meta
422
468
  WHERE workspace_id = ?
@@ -431,7 +477,7 @@ export class SqliteSyncGatewayAdapter {
431
477
  ) = ?)
432
478
  ORDER BY id ASC
433
479
  LIMIT ?
434
- `).all(workspaceId, after[0] ?? "", hash ?? null, hash ?? null, limit + 1);
480
+ `, workspaceId, after[0] ?? "", hash ?? null, hash ?? null, limit + 1);
435
481
  const hasMore2 = rows2.length > limit;
436
482
  const page2 = hasMore2 ? rows2.slice(0, limit) : rows2;
437
483
  const items = page2.map((row) => {
@@ -472,7 +518,8 @@ export class SqliteSyncGatewayAdapter {
472
518
  throw createError({ statusCode: 400, statusMessage: "Invalid canonical storage cursor" });
473
519
  }
474
520
  const [afterTable = "", afterId = "", afterHash = ""] = after;
475
- const rows = raw.prepare(`
521
+ const rows = await all(
522
+ `
476
523
  WITH reference_edges(source_table, source_id, raw_hash) AS (
477
524
  SELECT 'messages', row.id, CAST(edge.value AS TEXT)
478
525
  FROM s_messages AS row
@@ -516,7 +563,7 @@ export class SqliteSyncGatewayAdapter {
516
563
  )
517
564
  ORDER BY source_table ASC, source_id ASC, raw_hash ASC
518
565
  LIMIT ?
519
- `).all(
566
+ `,
520
567
  workspaceId,
521
568
  workspaceId,
522
569
  hash ?? null,
@@ -592,6 +639,15 @@ export class SqliteSyncGatewayAdapter {
592
639
  uniqueOps.push(group.op);
593
640
  indicesByOpId.set(group.op.stamp.opId, group.indices);
594
641
  }
642
+ if (isD1Driver()) {
643
+ return this.pushInD1({
644
+ event,
645
+ workspaceId,
646
+ uniqueOps,
647
+ resultSlots,
648
+ indicesByOpId
649
+ });
650
+ }
595
651
  const raw = getRawDb();
596
652
  const now = nowEpoch();
597
653
  const userId = resolveSessionUserId(event);
@@ -883,6 +939,9 @@ export class SqliteSyncGatewayAdapter {
883
939
  const workspaceId = input.scope.workspaceId;
884
940
  await assertWorkspaceScopeAuthorized(event, workspaceId);
885
941
  const pageSize = resolveSnapshotPageSize(input.pageSize);
942
+ if (isD1Driver()) {
943
+ return this.snapshotInD1(workspaceId, input, pageSize);
944
+ }
886
945
  const raw = getRawDb();
887
946
  const now = nowEpoch();
888
947
  let header;
@@ -1111,7 +1170,6 @@ export class SqliteSyncGatewayAdapter {
1111
1170
  }
1112
1171
  async updateCursor(event, input) {
1113
1172
  await assertWorkspaceScopeAuthorized(event, input.scope.workspaceId);
1114
- const raw = getRawDb();
1115
1173
  const now = nowEpoch();
1116
1174
  const deviceId = input.deviceId.trim();
1117
1175
  if (!deviceId || deviceId.length > 256) {
@@ -1124,6 +1182,10 @@ export class SqliteSyncGatewayAdapter {
1124
1182
  if (!ownerUserId) {
1125
1183
  throw createError({ statusCode: 401, statusMessage: "Unauthorized" });
1126
1184
  }
1185
+ if (isD1Driver()) {
1186
+ return this.updateCursorInD1(input.scope.workspaceId, deviceId, input.version, ownerUserId, now);
1187
+ }
1188
+ const raw = getRawDb();
1127
1189
  const update = raw.transaction(() => {
1128
1190
  const counter = raw.prepare(
1129
1191
  "SELECT value FROM server_version_counter WHERE workspace_id = ?"
@@ -1161,17 +1223,59 @@ export class SqliteSyncGatewayAdapter {
1161
1223
  }
1162
1224
  if (!canRunSyncHistoryGc()) return;
1163
1225
  const cutoff = nowEpoch() - input.retentionSeconds;
1226
+ if (isD1Driver()) {
1227
+ await d1Batch([
1228
+ {
1229
+ sql: `DELETE FROM device_cursors
1230
+ WHERE workspace_id = ? AND updated_at < ?`,
1231
+ parameters: [input.scope.workspaceId, cutoff]
1232
+ },
1233
+ {
1234
+ sql: `DELETE FROM tombstones
1235
+ WHERE workspace_id = ?
1236
+ AND created_at < ?
1237
+ AND (
1238
+ NOT EXISTS (
1239
+ SELECT 1 FROM device_cursors
1240
+ WHERE workspace_id = ?
1241
+ )
1242
+ OR server_version <= (
1243
+ SELECT MIN(last_seen_version)
1244
+ FROM device_cursors
1245
+ WHERE workspace_id = ?
1246
+ )
1247
+ )`,
1248
+ parameters: [
1249
+ input.scope.workspaceId,
1250
+ cutoff,
1251
+ input.scope.workspaceId,
1252
+ input.scope.workspaceId
1253
+ ]
1254
+ }
1255
+ ]);
1256
+ return;
1257
+ }
1164
1258
  const raw = getRawDb();
1165
1259
  raw.transaction(() => {
1260
+ raw.prepare(
1261
+ `DELETE FROM device_cursors
1262
+ WHERE workspace_id = ? AND updated_at < ?`
1263
+ ).run(input.scope.workspaceId, cutoff);
1166
1264
  const cursor = raw.prepare(
1167
1265
  `SELECT MIN(last_seen_version) AS min_version
1168
1266
  FROM device_cursors WHERE workspace_id = ?`
1169
1267
  ).get(input.scope.workspaceId);
1170
- if (cursor.min_version === null) return;
1171
- raw.prepare(
1172
- `DELETE FROM tombstones
1173
- WHERE workspace_id = ? AND created_at < ? AND server_version <= ?`
1174
- ).run(input.scope.workspaceId, cutoff, cursor.min_version);
1268
+ if (cursor.min_version === null) {
1269
+ raw.prepare(
1270
+ `DELETE FROM tombstones
1271
+ WHERE workspace_id = ? AND created_at < ?`
1272
+ ).run(input.scope.workspaceId, cutoff);
1273
+ } else {
1274
+ raw.prepare(
1275
+ `DELETE FROM tombstones
1276
+ WHERE workspace_id = ? AND created_at < ? AND server_version <= ?`
1277
+ ).run(input.scope.workspaceId, cutoff, cursor.min_version);
1278
+ }
1175
1279
  })();
1176
1280
  }
1177
1281
  async gcChangeLog(event, input) {
@@ -1181,19 +1285,674 @@ export class SqliteSyncGatewayAdapter {
1181
1285
  }
1182
1286
  if (!canRunSyncHistoryGc()) return;
1183
1287
  const cutoff = nowEpoch() - input.retentionSeconds;
1288
+ if (isD1Driver()) {
1289
+ await d1Batch([
1290
+ {
1291
+ sql: `DELETE FROM device_cursors
1292
+ WHERE workspace_id = ? AND updated_at < ?`,
1293
+ parameters: [input.scope.workspaceId, cutoff]
1294
+ },
1295
+ {
1296
+ sql: `DELETE FROM change_log
1297
+ WHERE workspace_id = ?
1298
+ AND created_at < ?
1299
+ AND (
1300
+ NOT EXISTS (
1301
+ SELECT 1 FROM device_cursors
1302
+ WHERE workspace_id = ?
1303
+ )
1304
+ OR server_version <= (
1305
+ SELECT MIN(last_seen_version)
1306
+ FROM device_cursors
1307
+ WHERE workspace_id = ?
1308
+ )
1309
+ )`,
1310
+ parameters: [
1311
+ input.scope.workspaceId,
1312
+ cutoff,
1313
+ input.scope.workspaceId,
1314
+ input.scope.workspaceId
1315
+ ]
1316
+ }
1317
+ ]);
1318
+ return;
1319
+ }
1184
1320
  const raw = getRawDb();
1185
1321
  raw.transaction(() => {
1322
+ raw.prepare(
1323
+ `DELETE FROM device_cursors
1324
+ WHERE workspace_id = ? AND updated_at < ?`
1325
+ ).run(input.scope.workspaceId, cutoff);
1186
1326
  const cursor = raw.prepare(
1187
1327
  `SELECT MIN(last_seen_version) AS min_version
1188
1328
  FROM device_cursors WHERE workspace_id = ?`
1189
1329
  ).get(input.scope.workspaceId);
1190
- if (cursor.min_version === null) return;
1191
- raw.prepare(
1192
- `DELETE FROM change_log
1193
- WHERE workspace_id = ? AND created_at < ? AND server_version <= ?`
1194
- ).run(input.scope.workspaceId, cutoff, cursor.min_version);
1330
+ if (cursor.min_version === null) {
1331
+ raw.prepare(
1332
+ `DELETE FROM change_log
1333
+ WHERE workspace_id = ? AND created_at < ?`
1334
+ ).run(input.scope.workspaceId, cutoff);
1335
+ } else {
1336
+ raw.prepare(
1337
+ `DELETE FROM change_log
1338
+ WHERE workspace_id = ? AND created_at < ? AND server_version <= ?`
1339
+ ).run(input.scope.workspaceId, cutoff, cursor.min_version);
1340
+ }
1195
1341
  })();
1196
1342
  }
1343
+ async reserveUploadIntentInD1(input, hash) {
1344
+ const now = nowEpoch();
1345
+ const liveRows = await d1All(
1346
+ `SELECT id, data_json FROM s_file_meta
1347
+ WHERE workspace_id = ? AND deleted = 0`,
1348
+ input.workspaceId
1349
+ );
1350
+ let usedBytes = 0;
1351
+ let alreadyStored = false;
1352
+ for (const row of liveRows) {
1353
+ const payload = JSON.parse(row.data_json);
1354
+ const size = payload.size_bytes ?? payload.sizeBytes;
1355
+ if (typeof size !== "number" || !Number.isSafeInteger(size) || size < 0) {
1356
+ throw createError({ statusCode: 500, statusMessage: "Invalid canonical file size" });
1357
+ }
1358
+ usedBytes += size;
1359
+ if (normalizeStorageHash(row.id) === hash) alreadyStored = true;
1360
+ }
1361
+ const [totals] = await d1All(
1362
+ `SELECT COALESCE(SUM(reserved_bytes), 0) AS reserved_bytes,
1363
+ MAX(CASE WHEN hash = ? AND reserved_bytes > 0 THEN 1 ELSE 0 END) AS same_hash
1364
+ FROM upload_intents
1365
+ WHERE workspace_id = ? AND status = 'active' AND expires_at > ?`,
1366
+ hash,
1367
+ input.workspaceId,
1368
+ now
1369
+ );
1370
+ const reservedBytes = alreadyStored || totals?.same_hash === 1 ? 0 : input.sizeBytes;
1371
+ if (input.workspaceQuotaBytes !== void 0 && usedBytes + (totals?.reserved_bytes ?? 0) + reservedBytes > input.workspaceQuotaBytes) {
1372
+ throw createError({ statusCode: 413, statusMessage: "Workspace storage quota exceeded" });
1373
+ }
1374
+ await d1Batch([
1375
+ {
1376
+ sql: `UPDATE upload_intents SET status = 'expired'
1377
+ WHERE workspace_id = ? AND status = 'active' AND expires_at <= ?`,
1378
+ parameters: [input.workspaceId, now]
1379
+ },
1380
+ {
1381
+ sql: `INSERT INTO upload_intents (
1382
+ id, workspace_id, hash, mime_type, size_bytes, reserved_bytes,
1383
+ expires_at, status, created_at
1384
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?)`,
1385
+ parameters: [
1386
+ input.intentId,
1387
+ input.workspaceId,
1388
+ hash,
1389
+ input.mimeType,
1390
+ input.sizeBytes,
1391
+ reservedBytes,
1392
+ input.expiresAt,
1393
+ now
1394
+ ]
1395
+ }
1396
+ ]);
1397
+ }
1398
+ async snapshotInD1(workspaceId, input, pageSize) {
1399
+ const now = nowEpoch();
1400
+ let header;
1401
+ let after;
1402
+ if (input.pageToken) {
1403
+ if (input.pageToken.length > 4096) {
1404
+ throw createError({ statusCode: 400, statusMessage: "Invalid snapshot page token" });
1405
+ }
1406
+ const token = decodeSnapshotPageToken(input.pageToken);
1407
+ const [existing] = await d1All(
1408
+ `SELECT id, workspace_id, high_watermark, tables_json, expires_at
1409
+ FROM sync_snapshots WHERE id = ?`,
1410
+ token.snapshotId
1411
+ );
1412
+ if (!existing || existing.expires_at <= now) {
1413
+ if (existing) {
1414
+ await d1Run("DELETE FROM sync_snapshots WHERE id = ?", existing.id);
1415
+ }
1416
+ throw createError({ statusCode: 410, statusMessage: "Snapshot expired or unavailable" });
1417
+ }
1418
+ if (existing.workspace_id !== workspaceId) {
1419
+ throw createError({ statusCode: 403, statusMessage: "Forbidden" });
1420
+ }
1421
+ if (input.tables && existing.tables_json !== JSON.stringify(normalizeSnapshotTables(input.tables))) {
1422
+ throw createError({
1423
+ statusCode: 400,
1424
+ statusMessage: "Snapshot table filter cannot change between pages"
1425
+ });
1426
+ }
1427
+ header = existing;
1428
+ after = token.after;
1429
+ } else {
1430
+ const tables = normalizeSnapshotTables(input.tables);
1431
+ const snapshotId = uid();
1432
+ const expiresAt = now + SNAPSHOT_TTL_SECONDS;
1433
+ const tablesJson = JSON.stringify(tables);
1434
+ const statements = [
1435
+ {
1436
+ sql: "DELETE FROM sync_snapshots WHERE expires_at <= ?",
1437
+ parameters: [now]
1438
+ },
1439
+ {
1440
+ sql: `INSERT INTO sync_snapshots (
1441
+ id, workspace_id, high_watermark, tables_json, created_at, expires_at
1442
+ ) VALUES (
1443
+ ?, ?,
1444
+ COALESCE((
1445
+ SELECT value FROM server_version_counter WHERE workspace_id = ?
1446
+ ), 0),
1447
+ ?, ?, ?
1448
+ )`,
1449
+ parameters: [
1450
+ snapshotId,
1451
+ workspaceId,
1452
+ workspaceId,
1453
+ tablesJson,
1454
+ now,
1455
+ expiresAt
1456
+ ]
1457
+ }
1458
+ ];
1459
+ for (const tableName of tables) {
1460
+ const materializedTable = SYNCED_TABLE_MAP[tableName];
1461
+ if (!materializedTable) {
1462
+ throw new Error(`Missing materialized table for ${tableName}`);
1463
+ }
1464
+ statements.push(
1465
+ {
1466
+ sql: `INSERT INTO sync_snapshot_items (
1467
+ snapshot_id, table_name, pk, kind, payload_json,
1468
+ clock, hlc, op_id, server_deleted_at
1469
+ )
1470
+ SELECT
1471
+ ?, ?, id, 'row', data_json,
1472
+ clock,
1473
+ CASE WHEN hlc <> '' THEN hlc ELSE 'legacy:0' END,
1474
+ CASE
1475
+ WHEN op_id <> '' THEN op_id
1476
+ ELSE 'legacy:${tableName}:' || id || ':' || clock || ':' || hlc
1477
+ END,
1478
+ NULL
1479
+ FROM "${materializedTable}"
1480
+ WHERE workspace_id = ? AND deleted = 0`,
1481
+ parameters: [snapshotId, tableName, workspaceId]
1482
+ },
1483
+ {
1484
+ sql: `INSERT INTO sync_snapshot_items (
1485
+ snapshot_id, table_name, pk, kind, payload_json,
1486
+ clock, hlc, op_id, server_deleted_at
1487
+ )
1488
+ SELECT
1489
+ ?, ?, materialized.id, 'tombstone', NULL,
1490
+ materialized.clock,
1491
+ CASE
1492
+ WHEN materialized.hlc <> '' THEN materialized.hlc
1493
+ ELSE 'legacy:0'
1494
+ END,
1495
+ CASE
1496
+ WHEN materialized.op_id <> '' THEN materialized.op_id
1497
+ ELSE 'legacy:tombstone:${tableName}:' || materialized.id || ':' || materialized.clock
1498
+ END,
1499
+ MAX(0, COALESCE(
1500
+ (
1501
+ SELECT tombstone.deleted_at
1502
+ FROM tombstones AS tombstone
1503
+ WHERE tombstone.workspace_id = materialized.workspace_id
1504
+ AND tombstone.table_name = ?
1505
+ AND tombstone.pk = materialized.id
1506
+ LIMIT 1
1507
+ ),
1508
+ materialized.updated_at,
1509
+ materialized.created_at,
1510
+ 0
1511
+ ))
1512
+ FROM "${materializedTable}" AS materialized
1513
+ WHERE materialized.workspace_id = ? AND materialized.deleted <> 0`,
1514
+ parameters: [snapshotId, tableName, tableName, workspaceId]
1515
+ },
1516
+ {
1517
+ sql: `INSERT INTO sync_snapshot_items (
1518
+ snapshot_id, table_name, pk, kind, payload_json,
1519
+ clock, hlc, op_id, server_deleted_at
1520
+ )
1521
+ SELECT
1522
+ ?, ?, tombstone.pk, 'tombstone', NULL,
1523
+ tombstone.clock,
1524
+ CASE WHEN tombstone.hlc <> '' THEN tombstone.hlc ELSE 'legacy:0' END,
1525
+ CASE
1526
+ WHEN tombstone.op_id <> '' THEN tombstone.op_id
1527
+ ELSE 'legacy:tombstone:${tableName}:' || tombstone.pk || ':' || tombstone.clock
1528
+ END,
1529
+ MAX(0, tombstone.deleted_at)
1530
+ FROM tombstones AS tombstone
1531
+ WHERE tombstone.workspace_id = ?
1532
+ AND tombstone.table_name = ?
1533
+ AND NOT EXISTS (
1534
+ SELECT 1
1535
+ FROM "${materializedTable}" AS materialized
1536
+ WHERE materialized.workspace_id = tombstone.workspace_id
1537
+ AND materialized.id = tombstone.pk
1538
+ )`,
1539
+ parameters: [snapshotId, tableName, workspaceId, tableName]
1540
+ }
1541
+ );
1542
+ }
1543
+ await d1Batch(statements);
1544
+ const [created] = await d1All(
1545
+ `SELECT id, workspace_id, high_watermark, tables_json, expires_at
1546
+ FROM sync_snapshots WHERE id = ?`,
1547
+ snapshotId
1548
+ );
1549
+ if (!created) throw new Error("D1 did not create a sync snapshot.");
1550
+ header = created;
1551
+ }
1552
+ const rows = after ? await d1All(
1553
+ `SELECT table_name, pk, kind, payload_json, clock, hlc, op_id, server_deleted_at
1554
+ FROM sync_snapshot_items
1555
+ WHERE snapshot_id = ?
1556
+ AND (
1557
+ table_name > ?
1558
+ OR (table_name = ? AND pk > ?)
1559
+ OR (table_name = ? AND pk = ? AND kind > ?)
1560
+ )
1561
+ ORDER BY table_name ASC, pk ASC, kind ASC
1562
+ LIMIT ?`,
1563
+ header.id,
1564
+ after.tableName,
1565
+ after.tableName,
1566
+ after.pk,
1567
+ after.tableName,
1568
+ after.pk,
1569
+ after.kind,
1570
+ pageSize + 1
1571
+ ) : await d1All(
1572
+ `SELECT table_name, pk, kind, payload_json, clock, hlc, op_id, server_deleted_at
1573
+ FROM sync_snapshot_items
1574
+ WHERE snapshot_id = ?
1575
+ ORDER BY table_name ASC, pk ASC, kind ASC
1576
+ LIMIT ?`,
1577
+ header.id,
1578
+ pageSize + 1
1579
+ );
1580
+ const hasMore = rows.length > pageSize;
1581
+ const pageRows = hasMore ? rows.slice(0, pageSize) : rows;
1582
+ const items = pageRows.map((row) => {
1583
+ const revision = { clock: row.clock, hlc: row.hlc, opId: row.op_id };
1584
+ if (row.kind === "row") {
1585
+ return {
1586
+ kind: "row",
1587
+ tableName: row.table_name,
1588
+ pk: row.pk,
1589
+ payload: JSON.parse(row.payload_json ?? "null"),
1590
+ revision
1591
+ };
1592
+ }
1593
+ return {
1594
+ kind: "tombstone",
1595
+ tableName: row.table_name,
1596
+ pk: row.pk,
1597
+ revision,
1598
+ serverDeletedAt: row.server_deleted_at ?? 0
1599
+ };
1600
+ });
1601
+ const lastRow = pageRows.at(-1);
1602
+ return {
1603
+ workspaceId: header.workspace_id,
1604
+ snapshotId: header.id,
1605
+ highWatermark: header.high_watermark,
1606
+ items,
1607
+ nextPageToken: hasMore && lastRow ? encodeSnapshotPageToken({
1608
+ version: 1,
1609
+ snapshotId: header.id,
1610
+ after: {
1611
+ tableName: lastRow.table_name,
1612
+ pk: lastRow.pk,
1613
+ kind: lastRow.kind
1614
+ }
1615
+ }) : null
1616
+ };
1617
+ }
1618
+ async updateCursorInD1(workspaceId, deviceId, version, ownerUserId, now) {
1619
+ const [counter] = await d1All(
1620
+ "SELECT value FROM server_version_counter WHERE workspace_id = ?",
1621
+ workspaceId
1622
+ );
1623
+ if (version > (counter?.value ?? 0)) {
1624
+ throw createError({ statusCode: 400, statusMessage: "Cursor exceeds workspace version" });
1625
+ }
1626
+ const [existing] = await d1All(
1627
+ `SELECT owner_user_id, last_seen_version
1628
+ FROM device_cursors WHERE workspace_id = ? AND device_id = ?`,
1629
+ workspaceId,
1630
+ deviceId
1631
+ );
1632
+ if (existing?.owner_user_id && existing.owner_user_id !== ownerUserId) {
1633
+ throw createError({ statusCode: 403, statusMessage: "Device cursor belongs to another user" });
1634
+ }
1635
+ if (existing && version < existing.last_seen_version) {
1636
+ throw createError({ statusCode: 409, statusMessage: "Device cursor cannot regress" });
1637
+ }
1638
+ const result = await d1Run(
1639
+ `INSERT INTO device_cursors (
1640
+ id, workspace_id, device_id, owner_user_id, last_seen_version, updated_at
1641
+ ) VALUES (?, ?, ?, ?, ?, ?)
1642
+ ON CONFLICT(workspace_id, device_id) DO UPDATE SET
1643
+ owner_user_id = COALESCE(device_cursors.owner_user_id, excluded.owner_user_id),
1644
+ last_seen_version = excluded.last_seen_version,
1645
+ updated_at = excluded.updated_at
1646
+ WHERE (device_cursors.owner_user_id IS NULL OR device_cursors.owner_user_id = ?)
1647
+ AND excluded.last_seen_version >= device_cursors.last_seen_version`,
1648
+ uid(),
1649
+ workspaceId,
1650
+ deviceId,
1651
+ ownerUserId,
1652
+ version,
1653
+ now,
1654
+ ownerUserId
1655
+ );
1656
+ if ((result.meta?.changes ?? 0) !== 1) {
1657
+ throw createError({ statusCode: 409, statusMessage: "Device cursor changed concurrently" });
1658
+ }
1659
+ }
1660
+ async pushInD1(input) {
1661
+ const { event, workspaceId, uniqueOps, resultSlots, indicesByOpId } = input;
1662
+ const existingOps = /* @__PURE__ */ new Map();
1663
+ const opIds = uniqueOps.map((op) => op.stamp.opId);
1664
+ for (let offset = 0; offset < opIds.length; offset += 500) {
1665
+ const chunk = opIds.slice(offset, offset + 500);
1666
+ if (chunk.length === 0) continue;
1667
+ const placeholders = chunk.map(() => "?").join(",");
1668
+ const rows = await d1All(
1669
+ `SELECT op_id, server_version FROM change_log WHERE op_id IN (${placeholders})`,
1670
+ ...chunk
1671
+ );
1672
+ for (const row of rows) existingOps.set(row.op_id, row.server_version);
1673
+ }
1674
+ const newOps = uniqueOps.filter((op) => !existingOps.has(op.stamp.opId));
1675
+ const versionOffsets = /* @__PURE__ */ new Map();
1676
+ newOps.forEach((op, index) => {
1677
+ versionOffsets.set(op.stamp.opId, index + 1);
1678
+ });
1679
+ const existingMaterializedKeys = /* @__PURE__ */ new Set();
1680
+ const opsByTable = /* @__PURE__ */ new Map();
1681
+ for (const op of newOps) {
1682
+ const materializedTable = SYNCED_TABLE_MAP[op.tableName];
1683
+ if (!materializedTable) continue;
1684
+ const tableOps = opsByTable.get(materializedTable) ?? [];
1685
+ tableOps.push(op);
1686
+ opsByTable.set(materializedTable, tableOps);
1687
+ }
1688
+ for (const [materializedTable, tableOps] of opsByTable) {
1689
+ const ids = [...new Set(tableOps.map((op) => op.pk))];
1690
+ for (let offset = 0; offset < ids.length; offset += 500) {
1691
+ const chunk = ids.slice(offset, offset + 500);
1692
+ if (chunk.length === 0) continue;
1693
+ const placeholders = chunk.map(() => "?").join(",");
1694
+ const rows = await d1All(
1695
+ `SELECT id FROM "${materializedTable}"
1696
+ WHERE workspace_id = ? AND id IN (${placeholders})`,
1697
+ workspaceId,
1698
+ ...chunk
1699
+ );
1700
+ for (const row of rows) {
1701
+ existingMaterializedKeys.add(`${materializedTable}\0${row.id}`);
1702
+ }
1703
+ }
1704
+ }
1705
+ const now = nowEpoch();
1706
+ const userId = resolveSessionUserId(event);
1707
+ const statements = [];
1708
+ if (newOps.length > 0) {
1709
+ statements.push(
1710
+ {
1711
+ sql: `INSERT OR IGNORE INTO server_version_counter (workspace_id, value)
1712
+ VALUES (?, 0)`,
1713
+ parameters: [workspaceId]
1714
+ },
1715
+ {
1716
+ sql: "UPDATE server_version_counter SET value = value + ? WHERE workspace_id = ?",
1717
+ parameters: [newOps.length, workspaceId]
1718
+ }
1719
+ );
1720
+ }
1721
+ const materializedStatementIndex = /* @__PURE__ */ new Map();
1722
+ const materializedWasExisting = /* @__PURE__ */ new Map();
1723
+ const seenMaterializedKeys = /* @__PURE__ */ new Set();
1724
+ for (const op of newOps) {
1725
+ const materializedTable = SYNCED_TABLE_MAP[op.tableName];
1726
+ if (!materializedTable) continue;
1727
+ const versionOffset = versionOffsets.get(op.stamp.opId);
1728
+ if (versionOffset === void 0) continue;
1729
+ const payloadJson = op.payload == null ? null : JSON.stringify(op.payload);
1730
+ const materializedKey = `${materializedTable}\0${op.pk}`;
1731
+ materializedWasExisting.set(
1732
+ op.stamp.opId,
1733
+ existingMaterializedKeys.has(materializedKey) || seenMaterializedKeys.has(materializedKey)
1734
+ );
1735
+ seenMaterializedKeys.add(materializedKey);
1736
+ statements.push({
1737
+ sql: `INSERT OR IGNORE INTO change_log (
1738
+ id, workspace_id, server_version, table_name, pk, op,
1739
+ payload_json, clock, hlc, device_id, op_id, created_at
1740
+ ) VALUES (
1741
+ ?, ?,
1742
+ (SELECT value - ? + ? FROM server_version_counter WHERE workspace_id = ?),
1743
+ ?, ?, ?, ?, ?, ?, ?, ?, ?
1744
+ )`,
1745
+ parameters: [
1746
+ uid(),
1747
+ workspaceId,
1748
+ newOps.length,
1749
+ versionOffset,
1750
+ workspaceId,
1751
+ op.tableName,
1752
+ op.pk,
1753
+ op.operation,
1754
+ payloadJson,
1755
+ op.stamp.clock,
1756
+ op.stamp.hlc,
1757
+ op.stamp.deviceId,
1758
+ op.stamp.opId,
1759
+ now
1760
+ ]
1761
+ });
1762
+ materializedStatementIndex.set(op.stamp.opId, statements.length);
1763
+ if (op.operation === "put") {
1764
+ statements.push({
1765
+ sql: `INSERT INTO "${materializedTable}" (
1766
+ id, workspace_id, data_json, clock, hlc, device_id,
1767
+ op_id, deleted, created_at, updated_at
1768
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, ?)
1769
+ ON CONFLICT(workspace_id, id) DO UPDATE SET
1770
+ data_json = excluded.data_json,
1771
+ clock = excluded.clock,
1772
+ hlc = excluded.hlc,
1773
+ device_id = excluded.device_id,
1774
+ op_id = excluded.op_id,
1775
+ deleted = 0,
1776
+ updated_at = excluded.updated_at
1777
+ WHERE excluded.clock > "${materializedTable}".clock
1778
+ OR (
1779
+ excluded.clock = "${materializedTable}".clock
1780
+ AND excluded.hlc > "${materializedTable}".hlc
1781
+ )`,
1782
+ parameters: [
1783
+ op.pk,
1784
+ workspaceId,
1785
+ payloadJson ?? "{}",
1786
+ op.stamp.clock,
1787
+ op.stamp.hlc,
1788
+ op.stamp.deviceId,
1789
+ op.stamp.opId,
1790
+ now,
1791
+ now
1792
+ ]
1793
+ });
1794
+ } else {
1795
+ statements.push({
1796
+ sql: `INSERT INTO "${materializedTable}" (
1797
+ id, workspace_id, data_json, clock, hlc, device_id,
1798
+ op_id, deleted, created_at, updated_at
1799
+ ) VALUES (?, ?, '{}', ?, ?, ?, ?, 1, ?, ?)
1800
+ ON CONFLICT(workspace_id, id) DO UPDATE SET
1801
+ clock = excluded.clock,
1802
+ hlc = excluded.hlc,
1803
+ device_id = excluded.device_id,
1804
+ op_id = excluded.op_id,
1805
+ deleted = 1,
1806
+ updated_at = excluded.updated_at
1807
+ WHERE excluded.clock > "${materializedTable}".clock
1808
+ OR (
1809
+ excluded.clock = "${materializedTable}".clock
1810
+ AND excluded.hlc > "${materializedTable}".hlc
1811
+ )`,
1812
+ parameters: [
1813
+ op.pk,
1814
+ workspaceId,
1815
+ op.stamp.clock,
1816
+ op.stamp.hlc,
1817
+ op.stamp.deviceId,
1818
+ op.stamp.opId,
1819
+ now,
1820
+ now
1821
+ ]
1822
+ });
1823
+ statements.push({
1824
+ sql: `INSERT INTO tombstones (
1825
+ id, workspace_id, table_name, pk, deleted_at, clock,
1826
+ hlc, op_id, server_version, created_at
1827
+ ) VALUES (
1828
+ ?, ?, ?, ?, ?, ?, ?, ?,
1829
+ (SELECT value - ? + ? FROM server_version_counter WHERE workspace_id = ?),
1830
+ ?
1831
+ )
1832
+ ON CONFLICT(workspace_id, table_name, pk) DO UPDATE SET
1833
+ deleted_at = excluded.deleted_at,
1834
+ clock = excluded.clock,
1835
+ hlc = excluded.hlc,
1836
+ op_id = excluded.op_id,
1837
+ server_version = excluded.server_version
1838
+ WHERE excluded.clock > tombstones.clock
1839
+ OR (
1840
+ excluded.clock = tombstones.clock
1841
+ AND excluded.server_version > tombstones.server_version
1842
+ )`,
1843
+ parameters: [
1844
+ uid(),
1845
+ workspaceId,
1846
+ op.tableName,
1847
+ op.pk,
1848
+ now,
1849
+ op.stamp.clock,
1850
+ op.stamp.hlc,
1851
+ op.stamp.opId,
1852
+ newOps.length,
1853
+ versionOffset,
1854
+ workspaceId,
1855
+ now
1856
+ ]
1857
+ });
1858
+ }
1859
+ }
1860
+ const finalVersionResultIndex = newOps.length > 0 ? statements.length : void 0;
1861
+ if (finalVersionResultIndex !== void 0) {
1862
+ statements.push({
1863
+ sql: "SELECT value FROM server_version_counter WHERE workspace_id = ?",
1864
+ parameters: [workspaceId]
1865
+ });
1866
+ }
1867
+ const batchResults = statements.length > 0 ? await d1Batch(statements) : [];
1868
+ let finalServerVersion = 0;
1869
+ if (finalVersionResultIndex !== void 0) {
1870
+ const row = batchResults[finalVersionResultIndex]?.results?.[0];
1871
+ if (typeof row?.value !== "number") {
1872
+ throw new Error("D1 did not return a workspace version counter.");
1873
+ }
1874
+ finalServerVersion = row.value;
1875
+ } else if (uniqueOps.length > 0) {
1876
+ const [counter] = await d1All(
1877
+ "SELECT value FROM server_version_counter WHERE workspace_id = ?",
1878
+ workspaceId
1879
+ );
1880
+ finalServerVersion = counter?.value ?? 0;
1881
+ }
1882
+ const resolvedOps = /* @__PURE__ */ new Map();
1883
+ for (let offset = 0; offset < opIds.length; offset += 500) {
1884
+ const chunk = opIds.slice(offset, offset + 500);
1885
+ if (chunk.length === 0) continue;
1886
+ const placeholders = chunk.map(() => "?").join(",");
1887
+ const rows = await d1All(
1888
+ `SELECT op_id, server_version FROM change_log WHERE op_id IN (${placeholders})`,
1889
+ ...chunk
1890
+ );
1891
+ for (const row of rows) resolvedOps.set(row.op_id, row.server_version);
1892
+ }
1893
+ const emissions = [];
1894
+ for (const op of uniqueOps) {
1895
+ const serverVersion = resolvedOps.get(op.stamp.opId);
1896
+ if (serverVersion === void 0) {
1897
+ throw new Error(`D1 failed to resolve sync operation ${op.stamp.opId}.`);
1898
+ }
1899
+ const result = {
1900
+ opId: op.stamp.opId,
1901
+ success: true,
1902
+ serverVersion
1903
+ };
1904
+ for (const index of indicesByOpId.get(op.stamp.opId) ?? []) {
1905
+ resultSlots[index] = result;
1906
+ }
1907
+ const materializedResult = batchResults[materializedStatementIndex.get(op.stamp.opId) ?? -1];
1908
+ if ((materializedResult?.meta?.changes ?? 0) > 0) {
1909
+ const emission = resolveHookEmission({
1910
+ op,
1911
+ workspaceId,
1912
+ now,
1913
+ userId,
1914
+ wasExisting: materializedWasExisting.get(op.stamp.opId) ?? false,
1915
+ applied: true
1916
+ });
1917
+ if (emission) emissions.push(emission);
1918
+ }
1919
+ }
1920
+ for (const emission of emissions) {
1921
+ await emitWebhookSystemHook(emission.hookName, emission.payload);
1922
+ }
1923
+ return {
1924
+ results: resultSlots.filter(
1925
+ (result) => Boolean(result)
1926
+ ),
1927
+ serverVersion: finalServerVersion
1928
+ };
1929
+ }
1930
+ async listWorkspaceIds() {
1931
+ if (isD1Driver()) {
1932
+ const rows2 = await d1All(
1933
+ "SELECT id FROM workspaces WHERE deleted = 0 ORDER BY id ASC"
1934
+ );
1935
+ return rows2.map((row) => row.id);
1936
+ }
1937
+ const raw = getRawDb();
1938
+ const rows = raw.prepare(
1939
+ "SELECT id FROM workspaces WHERE deleted = 0 ORDER BY id ASC"
1940
+ ).all();
1941
+ return rows.map((row) => row.id);
1942
+ }
1943
+ getMaintenanceState() {
1944
+ return getSyncMaintenanceState();
1945
+ }
1946
+ beginMaintenanceRun() {
1947
+ beginSyncMaintenanceRun();
1948
+ }
1949
+ completeMaintenanceRun(input) {
1950
+ const backlog = computeSyncMaintenanceBacklog(SYNC_MAINTENANCE_RETENTION_SECONDS, true);
1951
+ completeSyncMaintenanceRun({ lastRun: input.lastRun, backlog });
1952
+ }
1953
+ failMaintenanceRun(error) {
1954
+ failSyncMaintenanceRun(error);
1955
+ }
1197
1956
  }
1198
1957
  export function createSqliteSyncGatewayAdapter() {
1199
1958
  return new SqliteSyncGatewayAdapter();