@syncular/server 0.74.0 → 0.76.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/dist/admin.d.ts +6 -3
  2. package/dist/admin.js +10 -7
  3. package/dist/blob-handlers.js +2 -0
  4. package/dist/bucket-segment-store.d.ts +4 -3
  5. package/dist/bucket-segment-store.js +4 -2
  6. package/dist/context.d.ts +29 -1
  7. package/dist/context.js +23 -2
  8. package/dist/handler.js +7 -2
  9. package/dist/index.d.ts +1 -1
  10. package/dist/lease-store.d.ts +10 -3
  11. package/dist/lease-store.js +0 -0
  12. package/dist/operations.js +2 -0
  13. package/dist/pull.d.ts +9 -4
  14. package/dist/pull.js +129 -76
  15. package/dist/realtime.d.ts +14 -1
  16. package/dist/realtime.js +14 -3
  17. package/dist/segment-download.js +1 -0
  18. package/dist/segment-store.d.ts +6 -4
  19. package/dist/segment-store.js +4 -2
  20. package/dist/signed-url.d.ts +15 -0
  21. package/dist/signed-url.js +7 -2
  22. package/dist/sqlite-bun.js +2 -2
  23. package/dist/sqlite-image.d.ts +18 -3
  24. package/dist/sqlite-image.js +27 -5
  25. package/dist/sqlite-lease-store.d.ts +1 -1
  26. package/dist/sqlite-lease-store.js +19 -6
  27. package/dist/sqlite-node.js +3 -2
  28. package/dist/sqlite-segment-store.js +7 -7
  29. package/package.json +2 -2
  30. package/src/admin.ts +11 -5
  31. package/src/blob-handlers.ts +2 -0
  32. package/src/bucket-segment-store.ts +8 -5
  33. package/src/context.ts +38 -2
  34. package/src/handler.ts +9 -1
  35. package/src/index.ts +1 -0
  36. package/src/lease-store.ts +0 -0
  37. package/src/operations.ts +2 -0
  38. package/src/pull.ts +200 -94
  39. package/src/realtime.ts +26 -12
  40. package/src/segment-download.ts +1 -0
  41. package/src/segment-store.ts +10 -6
  42. package/src/signed-url.ts +22 -2
  43. package/src/sqlite-bun.ts +2 -2
  44. package/src/sqlite-image.ts +42 -7
  45. package/src/sqlite-lease-store.ts +22 -4
  46. package/src/sqlite-node.ts +3 -2
  47. package/src/sqlite-segment-store.ts +29 -7
@@ -250,7 +250,8 @@ export function mergeSegmentRecord(
250
250
  /**
251
251
  * The §5.3 reuse key: sqlite images are not byte-deterministic, so
252
252
  * cross-client dedup is a metadata lookup, not hash convergence — one
253
- * stored image per (partition, table, schemaVersion, scope digest, pin).
253
+ * stored image per (partition, table, schemaVersion, scope digest, pin, range
254
+ * start).
254
255
  */
255
256
  export interface SegmentFindKey {
256
257
  readonly partition: string;
@@ -260,6 +261,8 @@ export interface SegmentFindKey {
260
261
  readonly mediaType: 'rows' | 'sqlite';
261
262
  readonly scopeDigest: string;
262
263
  readonly asOfCommitSeq: number;
264
+ /** The range start: the row id the image follows, `null` for the first. */
265
+ readonly rowCursor: string | null;
263
266
  }
264
267
 
265
268
  /**
@@ -313,9 +316,8 @@ export interface SegmentStore {
313
316
  segmentId: string,
314
317
  ): Promise<{ record: SegmentRecord; bytes: Uint8Array } | undefined>;
315
318
  /**
316
- * Unexpired record for the §5.3 reuse key (whole-table segments only:
317
- * `rowCursor` null), or undefined. Servers MUST reuse instead of
318
- * rebuilding sqlite images while one exists (§5.3).
319
+ * Unexpired record for the §5.3 reuse key, or undefined. Servers MUST
320
+ * reuse instead of rebuilding sqlite images while one exists (§5.3).
319
321
  *
320
322
  * The digest matches when it is one of the record's `scopeDigests`
321
323
  * (§5.1). That cannot hand a sqlite image to the wrong digest: a sqlite
@@ -393,7 +395,9 @@ export class MemorySegmentStore implements SegmentStore {
393
395
  key: SegmentFindKey,
394
396
  nowMs: number,
395
397
  ): Promise<SegmentRecord | undefined> {
396
- for (const { record } of this.#entries.values()) {
398
+ // The newest publication wins: a rebuild supersedes an image stored
399
+ // under the same key that the server no longer serves (§5.3 bound).
400
+ for (const { record } of [...this.#entries.values()].reverse()) {
397
401
  const publication = selectSegmentPublication(record, {
398
402
  partition: key.partition,
399
403
  logEpoch: key.logEpoch,
@@ -402,7 +406,7 @@ export class MemorySegmentStore implements SegmentStore {
402
406
  mediaType: key.mediaType,
403
407
  scopeDigest: key.scopeDigest,
404
408
  asOfCommitSeq: key.asOfCommitSeq,
405
- rowCursor: null,
409
+ rowCursor: key.rowCursor,
406
410
  nowMs,
407
411
  });
408
412
  if (publication !== undefined) {
package/src/signed-url.ts CHANGED
@@ -5,9 +5,10 @@
5
5
  *
6
6
  * Native: `st = base64url(payloadJson) + "." +
7
7
  * base64url(HMAC-SHA256(key, payloadJson))` with claims
8
- * `{v, seg, sd, aud, exp}`; `exp` is unix seconds. Issuance happens inside
8
+ * `{v, seg, sd, aud, exp, ctx?}`; `exp` is unix seconds and `ctx` the
9
+ * authentication context of the issuing pull (§1.1). Issuance happens inside
9
10
  * the pull immediately after scope resolution; verification checks MAC,
10
- * expiry (≤ 60 s skew), and `seg`/`sd`/`aud` equality. Actual CDN serving
11
+ * expiry (≤ 60 s skew), and `seg`/`sd`/`aud`/`ctx` equality. Actual CDN serving
11
12
  * is out of scope.
12
13
  *
13
14
  * Delegated presign: the provider mints the URL (§5.4 equivalence rule —
@@ -45,6 +46,8 @@ export interface SegmentPresignArgs {
45
46
  readonly nowMs: number;
46
47
  /** Resolved TTL (config value or the 900 s default) — already applied. */
47
48
  readonly ttlSeconds: number;
49
+ /** The authentication context (§1.1) the pull resolved scopes under. */
50
+ readonly context?: string;
48
51
  }
49
52
 
50
53
  /**
@@ -78,6 +81,7 @@ export async function issueSegmentUrl(
78
81
  readonly partition: string;
79
82
  readonly scopeDigest: string;
80
83
  readonly nowMs: number;
84
+ readonly context?: string;
81
85
  },
82
86
  ): Promise<SegmentUrlIssue> {
83
87
  const ttlSeconds = config.ttlSeconds ?? 900;
@@ -91,6 +95,7 @@ export async function issueSegmentUrl(
91
95
  sd: args.scopeDigest,
92
96
  aud: config.audience(args.partition),
93
97
  exp,
98
+ ...(args.context !== undefined ? { ctx: args.context } : {}),
94
99
  });
95
100
  return {
96
101
  url: `${config.baseUrl}/${args.segmentId}?st=${token}`,
@@ -114,6 +119,8 @@ export interface BlobPresignArgs {
114
119
  readonly nowMs: number;
115
120
  /** Resolved TTL (config value or the 900 s default) — already applied. */
116
121
  readonly ttlSeconds: number;
122
+ /** The authentication context (§1.1) the download was authorized under. */
123
+ readonly context?: string;
117
124
  }
118
125
 
119
126
  export interface BlobPresignConfig {
@@ -135,6 +142,7 @@ export async function issueBlobUrl(
135
142
  readonly partition: string;
136
143
  readonly blobId: string;
137
144
  readonly nowMs: number;
145
+ readonly context?: string;
138
146
  },
139
147
  ): Promise<SegmentUrlIssue> {
140
148
  const ttlSeconds = config.ttlSeconds ?? 900;
@@ -194,6 +202,8 @@ export interface SegmentTokenClaims {
194
202
  readonly aud: string;
195
203
  /** Unix seconds — the sole non-millisecond timestamp in the spec. */
196
204
  readonly exp: number;
205
+ /** The issuing pull's authentication context (§1.1); absent without one. */
206
+ readonly ctx?: string;
197
207
  }
198
208
 
199
209
  const encoder = new TextEncoder();
@@ -253,6 +263,7 @@ export async function signSegmentToken(
253
263
  sd: claims.sd,
254
264
  aud: claims.aud,
255
265
  exp: claims.exp,
266
+ ...(claims.ctx !== undefined ? { ctx: claims.ctx } : {}),
256
267
  });
257
268
  const payload = encoder.encode(payloadJson);
258
269
  const mac = await hmac(key, payload);
@@ -280,6 +291,13 @@ export async function verifySegmentToken(
280
291
  readonly audience: string;
281
292
  readonly nowMs: number;
282
293
  readonly skewSeconds?: number;
294
+ /**
295
+ * The authentication context (§1.1) of a download that authenticated.
296
+ * When set, a token minted under another context, or without one, is
297
+ * refused. A headerless signed download (§5.4) passes none: the
298
+ * MAC-bound `seg`, `sd` and `ctx` claims are the grant.
299
+ */
300
+ readonly context?: string;
283
301
  },
284
302
  ): Promise<SegmentTokenClaims> {
285
303
  const forbidden = (reason: string) =>
@@ -320,5 +338,7 @@ export async function verifySegmentToken(
320
338
  if (!scopeDigests.includes(claims.sd))
321
339
  throw forbidden('scope-digest mismatch');
322
340
  if (claims.aud !== expected.audience) throw forbidden('audience mismatch');
341
+ if (expected.context !== undefined && claims.ctx !== expected.context)
342
+ throw forbidden('context mismatch');
323
343
  return claims;
324
344
  }
package/src/sqlite-bun.ts CHANGED
@@ -43,8 +43,8 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
43
43
  export const buildSqliteImage: SqliteImageBuilder = async (input) => {
44
44
  const db = new BunSqliteDatabase();
45
45
  try {
46
- await writeSqliteImage(db, input);
47
- return db.serialize();
46
+ const rowCount = await writeSqliteImage(db, input);
47
+ return { bytes: db.serialize(), rowCount };
48
48
  } finally {
49
49
  db.close();
50
50
  }
@@ -1,8 +1,8 @@
1
1
  import { rollbackAfterError } from './storage-errors';
2
2
  /**
3
3
  * SQLite-image segment generation (SPEC.md §5.3): a complete SQLite
4
- * database file carrying one table's whole effective-scope snapshot at
5
- * the bootstrap pin, plus the single-row `_syncular_segment` metadata
4
+ * database file carrying one range of a table's effective-scope snapshot at
5
+ * the table's scope pin, plus the single-row `_syncular_segment` metadata
6
6
  * table. Runtime entries provide the concrete SQLite database.
7
7
  *
8
8
  * Images are NOT byte-deterministic (§5.3): the content address pins the
@@ -58,6 +58,13 @@ export interface SqliteImageInput {
58
58
  readonly schemaVersion: number;
59
59
  readonly asOfCommitSeq: number;
60
60
  readonly scopeDigest: string;
61
+ /**
62
+ * The image's file-size bound (`ServerLimits.maxImageSegmentBytes`). The
63
+ * builder stops before the row that could pass it; the first row always
64
+ * goes in.
65
+ */
66
+ readonly maxBytes: number;
67
+ /** The range's rows in row order, from its start to the table's end. */
61
68
  readonly rowBatches:
62
69
  | AsyncIterable<readonly StoredRow[]>
63
70
  | Iterable<readonly StoredRow[]>;
@@ -73,13 +80,22 @@ export interface SqliteImageInput {
73
80
  */
74
81
  export type SqliteImageBuilder = (
75
82
  input: SqliteImageInput,
76
- ) => Promise<Uint8Array>;
83
+ ) => Promise<SqliteImage>;
77
84
 
78
- /** Populate a §5.3 image database for a whole-table snapshot. */
85
+ /** A built image and the number of leading input rows it holds. */
86
+ export interface SqliteImage {
87
+ readonly bytes: Uint8Array;
88
+ readonly rowCount: number;
89
+ }
90
+
91
+ /**
92
+ * Populate a §5.3 image database with the leading input rows that fit
93
+ * `maxBytes`. Returns how many rows it consumed.
94
+ */
79
95
  export async function writeSqliteImage(
80
96
  db: SqliteDatabase,
81
97
  input: SqliteImageInput,
82
- ): Promise<void> {
98
+ ): Promise<number> {
83
99
  const { table, rowBatches } = input;
84
100
  const primaryKey = table.columns[table.primaryKeyIndex]?.name;
85
101
  const columnDefs = table.columns.map((column) => {
@@ -103,15 +119,33 @@ export async function writeSqliteImage(
103
119
  `INSERT INTO ${quoteIdent(table.name)} (${names.join(', ')})
104
120
  VALUES (${names.map(() => '?').join(', ')})`,
105
121
  );
122
+ const pageSize = db
123
+ .query<{ page_size: number }>('PRAGMA page_size')
124
+ .get()!.page_size;
125
+ const pageCount = db.query<{ page_count: number }>('PRAGMA page_count');
106
126
  db.exec('BEGIN');
107
127
  try {
108
128
  let rowCount = 0;
109
- for await (const rows of rowBatches) {
129
+ // An upper bound of the file size, re-measured when it nears the bound.
130
+ let size = pageCount.get()!.page_count * pageSize;
131
+ fill: for await (const rows of rowBatches) {
110
132
  for (const row of rows) {
133
+ // An appended row grows the table and its primary-key index by at
134
+ // most its cell (bounded by the payload plus a 9-byte varint per
135
+ // column, the version and the rowid), overflow rounding and a split
136
+ // per tree level.
137
+ const growth =
138
+ 2 * (row.payload.length + 9 * (table.columns.length + 2)) +
139
+ 16 * pageSize;
140
+ if (rowCount > 0 && size + growth > input.maxBytes) {
141
+ size = pageCount.get()!.page_count * pageSize;
142
+ if (size + growth > input.maxBytes) break fill;
143
+ }
111
144
  const values = decodeRow(table.columns, row.payload);
112
145
  insert.run(...values.map(toSql), row.serverVersion);
146
+ size += growth;
147
+ rowCount += 1;
113
148
  }
114
- rowCount += rows.length;
115
149
  }
116
150
  db.query(
117
151
  `INSERT INTO ${IMAGE_METADATA_TABLE} VALUES (1, ?, ?, ?, ?, ?)`,
@@ -124,6 +158,7 @@ export async function writeSqliteImage(
124
158
  );
125
159
 
126
160
  db.exec('COMMIT');
161
+ return rowCount;
127
162
  } catch (error) {
128
163
  rollbackAfterError(() => {
129
164
  db.exec('ROLLBACK');
@@ -31,6 +31,7 @@ export class SqliteLeaseStore implements LeaseStore {
31
31
  lease_id TEXT NOT NULL, actor_id TEXT NOT NULL,
32
32
  allowed_scopes TEXT NOT NULL,
33
33
  issued_at_ms INTEGER NOT NULL, expires_at_ms INTEGER NOT NULL,
34
+ context TEXT,
34
35
  PRIMARY KEY (partition, client_id)
35
36
  );
36
37
  CREATE TABLE IF NOT EXISTS sync_lease_revocations(
@@ -38,6 +39,16 @@ export class SqliteLeaseStore implements LeaseStore {
38
39
  PRIMARY KEY (partition, lease_id)
39
40
  );
40
41
  `);
42
+ // §7.3.1: a lease table created before leases recorded their
43
+ // authentication context gains the column; its rows carry none.
44
+ if (
45
+ !this.db
46
+ .query<{ name: string }, []>(`PRAGMA table_info(sync_leases)`)
47
+ .all()
48
+ .some((column) => column.name === 'context')
49
+ ) {
50
+ this.db.exec(`ALTER TABLE sync_leases ADD COLUMN context TEXT`);
51
+ }
41
52
  }
42
53
 
43
54
  async get(
@@ -49,6 +60,7 @@ export class SqliteLeaseStore implements LeaseStore {
49
60
  {
50
61
  lease_id: string;
51
62
  actor_id: string;
63
+ context: string | null;
52
64
  allowed_scopes: string;
53
65
  issued_at_ms: number;
54
66
  expires_at_ms: number;
@@ -58,7 +70,7 @@ export class SqliteLeaseStore implements LeaseStore {
58
70
  >(
59
71
  // §7.3.4: the revocation table is the only record of a revocation, so
60
72
  // a revoked id stays revoked whatever its lease row says.
61
- `SELECT l.lease_id, l.actor_id, l.allowed_scopes, l.issued_at_ms,
73
+ `SELECT l.lease_id, l.actor_id, l.context, l.allowed_scopes, l.issued_at_ms,
62
74
  l.expires_at_ms,
63
75
  EXISTS (SELECT 1 FROM sync_lease_revocations r
64
76
  WHERE r.partition=l.partition
@@ -70,6 +82,7 @@ export class SqliteLeaseStore implements LeaseStore {
70
82
  return {
71
83
  leaseId: row.lease_id,
72
84
  actorId: row.actor_id,
85
+ ...(row.context !== null ? { context: row.context } : {}),
73
86
  allowedScopes: JSON.parse(row.allowed_scopes) as ScopeMap,
74
87
  issuedAtMs: row.issued_at_ms,
75
88
  expiresAtMs: row.expires_at_ms,
@@ -84,16 +97,19 @@ export class SqliteLeaseStore implements LeaseStore {
84
97
  allowedScopes: ScopeMap,
85
98
  nowMs: number,
86
99
  ttlMs: number,
100
+ context?: string,
87
101
  ): Promise<LeaseRecord> {
88
102
  const existing = await this.get(partition, clientId);
89
103
  const reuse =
90
104
  existing !== undefined &&
91
105
  existing.actorId === actorId &&
106
+ existing.context === context &&
92
107
  !existing.revoked;
93
108
  const leaseId = reuse ? existing.leaseId : this.#newId();
94
109
  const record: LeaseRecord = {
95
110
  leaseId,
96
111
  actorId,
112
+ ...(context !== undefined ? { context } : {}),
97
113
  allowedScopes,
98
114
  issuedAtMs: nowMs,
99
115
  expiresAtMs: nowMs + ttlMs,
@@ -103,13 +119,14 @@ export class SqliteLeaseStore implements LeaseStore {
103
119
  .query(
104
120
  `INSERT INTO sync_leases(
105
121
  partition, client_id, lease_id, actor_id, allowed_scopes,
106
- issued_at_ms, expires_at_ms
107
- ) VALUES (?,?,?,?,?,?,?)
122
+ issued_at_ms, expires_at_ms, context
123
+ ) VALUES (?,?,?,?,?,?,?,?)
108
124
  ON CONFLICT(partition, client_id) DO UPDATE SET
109
125
  lease_id=excluded.lease_id, actor_id=excluded.actor_id,
110
126
  allowed_scopes=excluded.allowed_scopes,
111
127
  issued_at_ms=excluded.issued_at_ms,
112
- expires_at_ms=excluded.expires_at_ms`,
128
+ expires_at_ms=excluded.expires_at_ms,
129
+ context=excluded.context`,
113
130
  )
114
131
  .run(
115
132
  partition,
@@ -119,6 +136,7 @@ export class SqliteLeaseStore implements LeaseStore {
119
136
  JSON.stringify(allowedScopes),
120
137
  nowMs,
121
138
  record.expiresAtMs,
139
+ context ?? null,
122
140
  );
123
141
  return record;
124
142
  }
@@ -48,12 +48,13 @@ export const buildSqliteImage: SqliteImageBuilder = async (input) => {
48
48
  const path = join(directory, 'segment.db');
49
49
  const db = new NodeSqliteDatabase(path);
50
50
  try {
51
+ let rowCount: number;
51
52
  try {
52
- await writeSqliteImage(db, input);
53
+ rowCount = await writeSqliteImage(db, input);
53
54
  } finally {
54
55
  db.close();
55
56
  }
56
- return new Uint8Array(readFileSync(path));
57
+ return { bytes: new Uint8Array(readFileSync(path)), rowCount };
57
58
  } finally {
58
59
  rmSync(directory, { recursive: true, force: true });
59
60
  }
@@ -313,13 +313,23 @@ export class SqliteSegmentStore implements SegmentStore {
313
313
  const publication = this.db
314
314
  .query<
315
315
  { segment_id: string },
316
- [string, string, string, number, string, string, number, number]
316
+ [
317
+ string,
318
+ string,
319
+ string,
320
+ number,
321
+ string,
322
+ string,
323
+ number,
324
+ string | null,
325
+ number,
326
+ ]
317
327
  >(
318
328
  `SELECT segment_id FROM sync_segment_publications
319
329
  WHERE partition=? AND log_epoch=? AND tbl=? AND schema_version=? AND media_type=?
320
- AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS NULL
330
+ AND scope_digest=? AND as_of_commit_seq=? AND row_cursor IS ?
321
331
  AND expires_at_ms > ?
322
- LIMIT 1`,
332
+ ORDER BY created_at_ms DESC LIMIT 1`,
323
333
  )
324
334
  .get(
325
335
  key.partition,
@@ -329,20 +339,31 @@ export class SqliteSegmentStore implements SegmentStore {
329
339
  key.mediaType,
330
340
  key.scopeDigest,
331
341
  key.asOfCommitSeq,
342
+ key.rowCursor,
332
343
  nowMs,
333
344
  );
334
345
  const legacy = this.db
335
346
  .query<
336
347
  { segment_id: string },
337
- [string, string, string, number, string, string, number, number]
348
+ [
349
+ string,
350
+ string,
351
+ string,
352
+ number,
353
+ string,
354
+ string,
355
+ number,
356
+ string | null,
357
+ number,
358
+ ]
338
359
  >(
339
360
  `SELECT s.segment_id
340
361
  FROM sync_segments s
341
362
  JOIN sync_segment_scopes sc ON sc.segment_id = s.segment_id
342
363
  WHERE s.partition=? AND s.log_epoch=? AND s.tbl=? AND s.schema_version=? AND s.media_type=?
343
- AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS NULL
364
+ AND sc.scope_digest=? AND s.as_of_commit_seq=? AND s.row_cursor IS ?
344
365
  AND s.expires_at_ms > ?
345
- LIMIT 1`,
366
+ ORDER BY s.created_at_ms DESC LIMIT 1`,
346
367
  )
347
368
  .get(
348
369
  key.partition,
@@ -352,6 +373,7 @@ export class SqliteSegmentStore implements SegmentStore {
352
373
  key.mediaType,
353
374
  key.scopeDigest,
354
375
  key.asOfCommitSeq,
376
+ key.rowCursor,
355
377
  nowMs,
356
378
  );
357
379
  const segmentId = publication?.segment_id ?? legacy?.segment_id;
@@ -366,7 +388,7 @@ export class SqliteSegmentStore implements SegmentStore {
366
388
  mediaType: key.mediaType,
367
389
  scopeDigest: key.scopeDigest,
368
390
  asOfCommitSeq: key.asOfCommitSeq,
369
- rowCursor: null,
391
+ rowCursor: key.rowCursor,
370
392
  nowMs,
371
393
  });
372
394
  if (match === undefined) return undefined;