@syncular/server 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -36
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +46 -40
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +19 -5
- package/dist/context.js +19 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +249 -1119
- package/dist/errors.js +16 -0
- package/dist/events.d.ts +36 -18
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +503 -208
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +24 -12
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +420 -262
- package/dist/pull.d.ts +70 -47
- package/dist/pull.js +270 -301
- package/dist/push.d.ts +12 -3
- package/dist/push.js +361 -173
- package/dist/realtime.d.ts +85 -39
- package/dist/realtime.js +355 -219
- package/dist/relational-rows.d.ts +21 -66
- package/dist/relational-rows.js +28 -109
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +10 -16
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +22 -6
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +164 -84
- package/dist/sqlite-dialect.js +1145 -150
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +303 -1123
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +185 -63
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +99 -57
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +33 -6
- package/src/d1-storage.ts +350 -1700
- package/src/errors.ts +16 -0
- package/src/events.ts +37 -18
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +654 -261
- package/src/local-authority.ts +27 -14
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/operations.ts +44 -42
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +557 -354
- package/src/pull.ts +406 -440
- package/src/push.ts +466 -220
- package/src/realtime.ts +447 -290
- package/src/relational-rows.ts +43 -121
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +13 -20
- package/src/segment-download.ts +33 -7
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1851 -199
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +390 -1821
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +215 -67
package/src/admin.ts
CHANGED
|
@@ -19,6 +19,8 @@ import {
|
|
|
19
19
|
type PushOperationResult,
|
|
20
20
|
renderRowValues,
|
|
21
21
|
type ScopeMap,
|
|
22
|
+
unitId,
|
|
23
|
+
type UnitScopes,
|
|
22
24
|
} from '@syncular/core';
|
|
23
25
|
import type { BlobStore, BlobStoreStats } from './blob-store';
|
|
24
26
|
import {
|
|
@@ -52,6 +54,7 @@ import type {
|
|
|
52
54
|
CheckpointDeclaration,
|
|
53
55
|
ClientRecord,
|
|
54
56
|
CommitMetadata,
|
|
57
|
+
PushWatermark,
|
|
55
58
|
ReactionStatus,
|
|
56
59
|
StoredPushResult,
|
|
57
60
|
StoredRow,
|
|
@@ -61,13 +64,22 @@ import type {
|
|
|
61
64
|
StoredReaction,
|
|
62
65
|
} from './storage';
|
|
63
66
|
|
|
67
|
+
/** One registered unit of a client record. */
|
|
68
|
+
export interface AdminClientUnit {
|
|
69
|
+
/** The unit id (§4.1). */
|
|
70
|
+
readonly id: string;
|
|
71
|
+
readonly scopes: UnitScopes;
|
|
72
|
+
/** The unit's cursor while it catches up; absent for a live unit. */
|
|
73
|
+
readonly catchingUpCursor?: number;
|
|
74
|
+
}
|
|
75
|
+
|
|
64
76
|
/** A connected/known client as the console sees it (§4.5, §8.1). */
|
|
65
77
|
export interface AdminClient {
|
|
66
78
|
readonly clientId: string;
|
|
67
79
|
readonly actorId: string;
|
|
68
80
|
/** SSP2 wire version last accepted from this client. */
|
|
69
81
|
readonly wireVersion: number;
|
|
70
|
-
/**
|
|
82
|
+
/** Schema version of its latest completed round, when recorded. */
|
|
71
83
|
readonly schemaVersion?: number;
|
|
72
84
|
readonly cursor: number;
|
|
73
85
|
/**
|
|
@@ -76,11 +88,8 @@ export interface AdminClient {
|
|
|
76
88
|
*/
|
|
77
89
|
readonly lag: number;
|
|
78
90
|
readonly updatedAtMs: number;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
readonly table: string;
|
|
82
|
-
readonly scopes: Record<string, readonly string[]>;
|
|
83
|
-
}[];
|
|
91
|
+
/** The stored registration list (§4.1); empty when none is stored. */
|
|
92
|
+
readonly units: readonly AdminClientUnit[];
|
|
84
93
|
/** True when the cursor record was touched within the active window. */
|
|
85
94
|
readonly active: boolean;
|
|
86
95
|
/**
|
|
@@ -265,11 +274,11 @@ export interface AdminCommitDetail extends CommitMetadata {
|
|
|
265
274
|
* The trace verdict, evaluated in this order:
|
|
266
275
|
* `client-unknown`: no client record for the id.
|
|
267
276
|
* `row-missing`: no row and no retained change to it.
|
|
268
|
-
* `no-
|
|
277
|
+
* `no-unit`: no registered unit of the client covers the table.
|
|
269
278
|
* `row-deleted`: the row's newest retained change is a delete the client
|
|
270
279
|
* has pulled.
|
|
271
|
-
* `scope-mismatch`: no
|
|
272
|
-
* `scope-denied`: a
|
|
280
|
+
* `scope-mismatch`: no covering unit requests the row's stored scopes.
|
|
281
|
+
* `scope-denied`: a covering unit requests them, but the actor's allowed
|
|
273
282
|
* scopes (the host `resolveScopes`) exclude the row.
|
|
274
283
|
* `pending-pull`: the client's cursor is below the row's newest change.
|
|
275
284
|
* `delivered`: the client has pulled the row's newest change.
|
|
@@ -277,24 +286,24 @@ export interface AdminCommitDetail extends CommitMetadata {
|
|
|
277
286
|
export type AdminTraceVerdict =
|
|
278
287
|
| 'client-unknown'
|
|
279
288
|
| 'row-missing'
|
|
280
|
-
| 'no-
|
|
289
|
+
| 'no-unit'
|
|
281
290
|
| 'row-deleted'
|
|
282
291
|
| 'scope-mismatch'
|
|
283
292
|
| 'scope-denied'
|
|
284
293
|
| 'pending-pull'
|
|
285
294
|
| 'delivered';
|
|
286
295
|
|
|
287
|
-
/** One of the client's
|
|
288
|
-
export interface
|
|
296
|
+
/** One of the client's units covering the traced table. */
|
|
297
|
+
export interface AdminTraceUnit {
|
|
289
298
|
readonly id: string;
|
|
290
|
-
readonly scopes:
|
|
291
|
-
/** The row's stored scopes
|
|
299
|
+
readonly scopes: UnitScopes;
|
|
300
|
+
/** The row's stored scopes carry the unit's values. */
|
|
292
301
|
readonly requestedMatch: boolean;
|
|
293
|
-
/** Each
|
|
302
|
+
/** Each unit variable the row's stored value fails. */
|
|
294
303
|
readonly mismatches: readonly {
|
|
295
304
|
readonly variable: string;
|
|
296
305
|
readonly rowValue: string | null;
|
|
297
|
-
readonly requested:
|
|
306
|
+
readonly requested: string;
|
|
298
307
|
}[];
|
|
299
308
|
/**
|
|
300
309
|
* The row also satisfies requested ∩ allowed (§3.2). Absent when the
|
|
@@ -320,7 +329,7 @@ export interface AdminRowTrace {
|
|
|
320
329
|
* when it returned the §7.3.3 outage sentinel. Absent without a resolver.
|
|
321
330
|
*/
|
|
322
331
|
readonly allowed?: ScopeMap | 'outage';
|
|
323
|
-
readonly
|
|
332
|
+
readonly units: readonly AdminTraceUnit[];
|
|
324
333
|
/** The client's retained push rejections and conflicts, newest first. */
|
|
325
334
|
readonly rejections: readonly SyncularServerEvent[];
|
|
326
335
|
}
|
|
@@ -392,6 +401,13 @@ export interface AdminWriteLookup {
|
|
|
392
401
|
readonly clientCommitId: string;
|
|
393
402
|
/** One outcome per client that sent this clientCommitId. */
|
|
394
403
|
readonly outcomes: readonly AdminWriteOutcome[];
|
|
404
|
+
/**
|
|
405
|
+
* With no stored outcome: the §6.3.2 watermarks of the clients whose
|
|
406
|
+
* retained commit log carries this clientCommitId. A commit below an
|
|
407
|
+
* outbox's `throughSeq` lost its result to the watermark ("expired below
|
|
408
|
+
* sequence N").
|
|
409
|
+
*/
|
|
410
|
+
readonly watermarks: readonly PushWatermark[];
|
|
395
411
|
/** Retained ring events carrying this clientCommitId, newest first. */
|
|
396
412
|
readonly events: readonly SyncularServerEvent[];
|
|
397
413
|
}
|
|
@@ -525,7 +541,7 @@ function required<T>(value: T | undefined, what: string): T {
|
|
|
525
541
|
const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
|
|
526
542
|
'client-unknown',
|
|
527
543
|
'row-missing',
|
|
528
|
-
'no-
|
|
544
|
+
'no-unit',
|
|
529
545
|
'row-deleted',
|
|
530
546
|
'scope-mismatch',
|
|
531
547
|
'scope-denied',
|
|
@@ -533,39 +549,49 @@ const VERDICT_ORDER: readonly AdminTraceVerdict[] = [
|
|
|
533
549
|
'delivered',
|
|
534
550
|
];
|
|
535
551
|
|
|
536
|
-
/** One known client's
|
|
552
|
+
/** One known client's units covering the row's table and its verdict. */
|
|
537
553
|
function traceVerdict(
|
|
554
|
+
declared: ReadonlySet<string>,
|
|
538
555
|
record: ClientRecord,
|
|
539
556
|
row: AdminRowInspection,
|
|
540
557
|
lastChange: RowChangeRecord | undefined,
|
|
541
558
|
horizonSeq: number,
|
|
542
559
|
allowed: ScopeMap | 'outage' | undefined,
|
|
543
560
|
): {
|
|
544
|
-
readonly
|
|
561
|
+
readonly units: AdminTraceUnit[];
|
|
545
562
|
readonly verdict: AdminTraceVerdict;
|
|
546
563
|
} {
|
|
547
564
|
const rowScopes = row.scopes ?? {};
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
.
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
.map(([variable,
|
|
565
|
+
// §4.1: a unit covers every table that declares all of its variables.
|
|
566
|
+
const units = (record.registration?.units ?? [])
|
|
567
|
+
.filter((scopes) =>
|
|
568
|
+
Object.keys(scopes).every((variable) => declared.has(variable)),
|
|
569
|
+
)
|
|
570
|
+
.map((scopes): AdminTraceUnit => {
|
|
571
|
+
const mismatches = Object.entries(scopes)
|
|
572
|
+
.filter(([variable, value]) => rowScopes[variable] !== value)
|
|
573
|
+
.map(([variable, value]) => ({
|
|
557
574
|
variable,
|
|
558
575
|
rowValue: rowScopes[variable] ?? null,
|
|
559
|
-
requested:
|
|
576
|
+
requested: value,
|
|
560
577
|
}));
|
|
561
578
|
const traced = {
|
|
562
|
-
id:
|
|
563
|
-
scopes
|
|
579
|
+
id: unitId(scopes),
|
|
580
|
+
scopes,
|
|
564
581
|
requestedMatch: mismatches.length === 0,
|
|
565
582
|
mismatches,
|
|
566
583
|
};
|
|
567
584
|
if (allowed === undefined || allowed === 'outage') return traced;
|
|
568
|
-
const effective = computeEffective(
|
|
585
|
+
const effective = computeEffective(
|
|
586
|
+
declared,
|
|
587
|
+
Object.fromEntries(
|
|
588
|
+
Object.entries(scopes).map(([variable, value]) => [
|
|
589
|
+
variable,
|
|
590
|
+
[value],
|
|
591
|
+
]),
|
|
592
|
+
),
|
|
593
|
+
{ ok: true, allowed },
|
|
594
|
+
);
|
|
569
595
|
return {
|
|
570
596
|
...traced,
|
|
571
597
|
effectiveMatch:
|
|
@@ -579,20 +605,20 @@ function traceVerdict(
|
|
|
579
605
|
const verdict: AdminTraceVerdict =
|
|
580
606
|
!row.exists && lastChange === undefined
|
|
581
607
|
? 'row-missing'
|
|
582
|
-
:
|
|
583
|
-
? 'no-
|
|
608
|
+
: units.length === 0
|
|
609
|
+
? 'no-unit'
|
|
584
610
|
: !row.exists
|
|
585
611
|
? pulled
|
|
586
612
|
? 'row-deleted'
|
|
587
613
|
: 'pending-pull'
|
|
588
|
-
: !
|
|
614
|
+
: !units.some((unit) => unit.requestedMatch)
|
|
589
615
|
? 'scope-mismatch'
|
|
590
|
-
:
|
|
616
|
+
: units.every((unit) => unit.effectiveMatch === false)
|
|
591
617
|
? 'scope-denied'
|
|
592
618
|
: pulled
|
|
593
619
|
? 'delivered'
|
|
594
620
|
: 'pending-pull';
|
|
595
|
-
return {
|
|
621
|
+
return { units, verdict };
|
|
596
622
|
}
|
|
597
623
|
|
|
598
624
|
/** The console query surface plus its operator actions. One per process. */
|
|
@@ -866,7 +892,7 @@ export class SyncularAdmin {
|
|
|
866
892
|
|
|
867
893
|
/**
|
|
868
894
|
* Trace whether one client receives one row, combining the row's stored
|
|
869
|
-
* scopes and newest change with the client's
|
|
895
|
+
* scopes and newest change with the client's units, cursor, the
|
|
870
896
|
* actor's allowed scopes, and its retained push failures. See
|
|
871
897
|
* `AdminTraceVerdict` for the verdict order.
|
|
872
898
|
*/
|
|
@@ -899,14 +925,21 @@ export class SyncularAdmin {
|
|
|
899
925
|
...(lastChange !== undefined ? { lastChange } : {}),
|
|
900
926
|
};
|
|
901
927
|
if (record === undefined) {
|
|
902
|
-
return { ...base, verdict: 'client-unknown',
|
|
928
|
+
return { ...base, verdict: 'client-unknown', units: [] };
|
|
903
929
|
}
|
|
904
930
|
const allowed = await this.#allowedScopes(partition, record);
|
|
905
931
|
return {
|
|
906
932
|
...base,
|
|
907
933
|
client: (await this.#clientView(partition))(record),
|
|
908
934
|
...(allowed !== undefined ? { allowed } : {}),
|
|
909
|
-
...traceVerdict(
|
|
935
|
+
...traceVerdict(
|
|
936
|
+
this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set(),
|
|
937
|
+
record,
|
|
938
|
+
row,
|
|
939
|
+
lastChange,
|
|
940
|
+
horizonSeq,
|
|
941
|
+
allowed,
|
|
942
|
+
),
|
|
910
943
|
};
|
|
911
944
|
}
|
|
912
945
|
|
|
@@ -929,6 +962,8 @@ export class SyncularAdmin {
|
|
|
929
962
|
const lastChange = row.history[0];
|
|
930
963
|
const horizonSeq = await this.#storage.getHorizonSeq(partition);
|
|
931
964
|
const view = await this.#clientView(partition);
|
|
965
|
+
const declared =
|
|
966
|
+
this.#requireSchema().tables.get(table)?.declaredVariables ?? new Set();
|
|
932
967
|
const allowedByActor = new Map<string, ScopeMap | 'outage' | undefined>();
|
|
933
968
|
const clients: AdminRowDeliveryClient[] = [];
|
|
934
969
|
for (const record of await list(partition)) {
|
|
@@ -941,6 +976,7 @@ export class SyncularAdmin {
|
|
|
941
976
|
clients.push({
|
|
942
977
|
client: view(record),
|
|
943
978
|
verdict: traceVerdict(
|
|
979
|
+
declared,
|
|
944
980
|
record,
|
|
945
981
|
row,
|
|
946
982
|
lastChange,
|
|
@@ -1120,7 +1156,14 @@ export class SyncularAdmin {
|
|
|
1120
1156
|
(event) =>
|
|
1121
1157
|
'clientCommitId' in event && event.clientCommitId === clientCommitId,
|
|
1122
1158
|
);
|
|
1123
|
-
|
|
1159
|
+
const watermarks =
|
|
1160
|
+
outcomes.length > 0
|
|
1161
|
+
? []
|
|
1162
|
+
: await required(
|
|
1163
|
+
this.#storage.listPushWatermarks?.bind(this.#storage),
|
|
1164
|
+
'storage',
|
|
1165
|
+
)(partition, clientCommitId);
|
|
1166
|
+
return { clientCommitId, outcomes, watermarks, events };
|
|
1124
1167
|
}
|
|
1125
1168
|
|
|
1126
1169
|
/**
|
|
@@ -1610,30 +1653,29 @@ export class SyncularAdmin {
|
|
|
1610
1653
|
}
|
|
1611
1654
|
}
|
|
1612
1655
|
return (record) => {
|
|
1613
|
-
let schemaVersion: number | undefined;
|
|
1614
|
-
for (const sub of record.subscriptions) {
|
|
1615
|
-
if (
|
|
1616
|
-
sub.schemaVersion !== undefined &&
|
|
1617
|
-
(schemaVersion === undefined || sub.schemaVersion > schemaVersion)
|
|
1618
|
-
) {
|
|
1619
|
-
schemaVersion = sub.schemaVersion;
|
|
1620
|
-
}
|
|
1621
|
-
}
|
|
1622
1656
|
const connectedSinceMs = connectedSince.get(record.clientId);
|
|
1623
1657
|
return {
|
|
1624
1658
|
clientId: record.clientId,
|
|
1625
1659
|
actorId: record.actorId,
|
|
1626
1660
|
wireVersion: record.wireVersion,
|
|
1627
|
-
...(schemaVersion !== undefined
|
|
1661
|
+
...(record.schemaVersion !== undefined
|
|
1662
|
+
? { schemaVersion: record.schemaVersion }
|
|
1663
|
+
: {}),
|
|
1628
1664
|
cursor: record.cursor,
|
|
1629
1665
|
// A never-pulled cursor (-1) has seen nothing: it lags the whole log.
|
|
1630
1666
|
lag: Math.max(0, maxCommitSeq - Math.max(0, record.cursor)),
|
|
1631
1667
|
updatedAtMs: record.updatedAtMs,
|
|
1632
|
-
|
|
1633
|
-
id
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1668
|
+
units: (record.registration?.units ?? []).map((scopes) => {
|
|
1669
|
+
const id = unitId(scopes);
|
|
1670
|
+
const catchingUp = record.catchingUp.find((unit) => unit.id === id);
|
|
1671
|
+
return {
|
|
1672
|
+
id,
|
|
1673
|
+
scopes,
|
|
1674
|
+
...(catchingUp !== undefined
|
|
1675
|
+
? { catchingUpCursor: catchingUp.cursor }
|
|
1676
|
+
: {}),
|
|
1677
|
+
};
|
|
1678
|
+
}),
|
|
1637
1679
|
active: record.updatedAtMs >= activeFloorMs,
|
|
1638
1680
|
...(sockets !== undefined
|
|
1639
1681
|
? { connected: connectedSinceMs !== undefined }
|
package/src/blob-handlers.ts
CHANGED
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
* `blobId` is never a bearer capability. Framework-free; the hono/runtime
|
|
7
7
|
* adapters map these onto `PUT/GET <mount>/blobs/{blobId}`.
|
|
8
8
|
*/
|
|
9
|
-
import {
|
|
9
|
+
import { blobIdFor, isBlobId } from './blob-store';
|
|
10
|
+
import { decodeRequestBody } from './content-encoding';
|
|
10
11
|
import type { SyncRequestContext } from './context';
|
|
11
12
|
import {
|
|
12
13
|
clockOf,
|
|
@@ -17,36 +18,29 @@ import { SyncError, syncError } from './errors';
|
|
|
17
18
|
import { emitEvent } from './events';
|
|
18
19
|
import { compileSchema } from './schema';
|
|
19
20
|
import { authorizeWrite, type ResolvedScopes } from './scopes';
|
|
21
|
+
import { bytesStream } from './segment-store';
|
|
20
22
|
import { issueBlobUploadUrl, issueBlobUrl } from './signed-url';
|
|
21
23
|
|
|
22
24
|
const DEFAULT_MAX_BLOB_BYTES = 64 * 1024 * 1024;
|
|
23
25
|
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
export
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
* Present iff the host configured `blobSignedUrls`; when present, `bytes` is
|
|
43
|
-
* absent and the client MUST fetch the URL directly (no host auth), verify
|
|
44
|
-
* the content address, and on failure re-request this endpoint — never fall
|
|
45
|
-
* through (§5.9.5 recovery rule).
|
|
46
|
-
*/
|
|
47
|
-
readonly url?: string;
|
|
48
|
-
readonly urlExpiresAtMs?: number;
|
|
49
|
-
}
|
|
26
|
+
/**
|
|
27
|
+
* A §5.9.5 download: the stored bytes as a stream with their length (inline
|
|
28
|
+
* delivery), or, when the host configured `blobSignedUrls`, a provider-signed
|
|
29
|
+
* GET URL issued after the row-derived authorization check. The client
|
|
30
|
+
* fetches a URL directly (no host auth), verifies the content address, and on
|
|
31
|
+
* failure re-requests this endpoint (§5.9.5 recovery rule).
|
|
32
|
+
*/
|
|
33
|
+
export type BlobDownloadResult =
|
|
34
|
+
| {
|
|
35
|
+
readonly headers: Record<string, string>;
|
|
36
|
+
readonly body: ReadableStream<Uint8Array<ArrayBuffer>>;
|
|
37
|
+
readonly byteLength: number;
|
|
38
|
+
}
|
|
39
|
+
| {
|
|
40
|
+
readonly headers: Record<string, string>;
|
|
41
|
+
readonly url: string;
|
|
42
|
+
readonly urlExpiresAtMs: number;
|
|
43
|
+
};
|
|
50
44
|
|
|
51
45
|
/** `POST <mount>/blobs/{blobId}/upload-grant` request body (§5.9.3). */
|
|
52
46
|
export interface BlobUploadGrantRequest {
|
|
@@ -71,43 +65,67 @@ export interface BlobUploadGrantResult {
|
|
|
71
65
|
readonly present?: boolean;
|
|
72
66
|
}
|
|
73
67
|
|
|
74
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* `PUT <mount>/blobs/{blobId}` (§5.9.3). Host auth is the adapter's job. The
|
|
70
|
+
* size cap applies to the declared `Content-Length` before any byte is read
|
|
71
|
+
* and again to the counted decoded bytes while reading, so an oversized body
|
|
72
|
+
* is never buffered whole.
|
|
73
|
+
*/
|
|
75
74
|
export async function handleBlobUpload(
|
|
76
75
|
ctx: SyncRequestContext,
|
|
77
|
-
|
|
76
|
+
blobId: string,
|
|
77
|
+
request: Request,
|
|
78
78
|
): Promise<void> {
|
|
79
79
|
await touchAuthenticatedPartition(ctx);
|
|
80
80
|
const store = ctx.blobs;
|
|
81
81
|
if (store === undefined) {
|
|
82
82
|
throw syncError('blob.not_found', 'this server has no blob store (§5.9)');
|
|
83
83
|
}
|
|
84
|
-
if (!isBlobId(
|
|
84
|
+
if (!isBlobId(blobId)) {
|
|
85
85
|
throw syncError(
|
|
86
86
|
'blob.hash_mismatch',
|
|
87
87
|
'blobId path is not "sha256:<64 hex>" (§5.9.1)',
|
|
88
88
|
);
|
|
89
89
|
}
|
|
90
90
|
const maxBytes = ctx.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
91
|
+
const tooLarge = syncError(
|
|
92
|
+
'blob.too_large',
|
|
93
|
+
'blob exceeds the server size cap (§5.9.3)',
|
|
94
|
+
JSON.stringify({ max: maxBytes }),
|
|
95
|
+
);
|
|
96
|
+
if (Number(request.headers.get('content-length')) > maxBytes) throw tooLarge;
|
|
97
|
+
let bytes: Uint8Array | undefined;
|
|
98
|
+
try {
|
|
99
|
+
bytes = await decodeRequestBody(request, maxBytes);
|
|
100
|
+
} catch (error) {
|
|
101
|
+
throw error instanceof SyncError && error.code === 'sync.limit_exceeded'
|
|
102
|
+
? tooLarge
|
|
103
|
+
: error;
|
|
96
104
|
}
|
|
97
|
-
|
|
98
|
-
|
|
105
|
+
if (bytes === undefined) {
|
|
106
|
+
throw syncError('sync.invalid_request', 'unsupported content encoding');
|
|
107
|
+
}
|
|
108
|
+
if ((await blobIdFor(bytes)) !== blobId) {
|
|
99
109
|
throw syncError(
|
|
100
110
|
'blob.hash_mismatch',
|
|
101
111
|
'uploaded bytes do not hash to the blobId path (§5.9.3)',
|
|
102
112
|
);
|
|
103
113
|
}
|
|
114
|
+
const contentType = request.headers
|
|
115
|
+
.get('content-type')
|
|
116
|
+
?.split(';')[0]
|
|
117
|
+
?.trim();
|
|
104
118
|
const clock = clockOf(ctx);
|
|
105
119
|
await store.put(
|
|
106
120
|
ctx.partition,
|
|
107
|
-
|
|
108
|
-
|
|
121
|
+
blobId,
|
|
122
|
+
bytes,
|
|
109
123
|
clock(),
|
|
110
|
-
|
|
124
|
+
contentType === undefined ||
|
|
125
|
+
contentType === '' ||
|
|
126
|
+
contentType === 'application/octet-stream'
|
|
127
|
+
? undefined
|
|
128
|
+
: contentType,
|
|
111
129
|
);
|
|
112
130
|
if (ctx.events !== undefined) {
|
|
113
131
|
emitEvent(ctx.events, {
|
|
@@ -115,8 +133,8 @@ export async function handleBlobUpload(
|
|
|
115
133
|
atMs: clock(),
|
|
116
134
|
partition: ctx.partition,
|
|
117
135
|
actorId: ctx.actorId,
|
|
118
|
-
blobId
|
|
119
|
-
bytes:
|
|
136
|
+
blobId,
|
|
137
|
+
bytes: bytes.length,
|
|
120
138
|
});
|
|
121
139
|
}
|
|
122
140
|
}
|
|
@@ -146,7 +164,7 @@ export async function handleBlobDownload(
|
|
|
146
164
|
outcome: 'ok',
|
|
147
165
|
// Presigned delivery carries no inline bytes (§5.9.5 always-issue); the
|
|
148
166
|
// byte count is unknown to the sync server, which exited the egress path.
|
|
149
|
-
bytes: result.
|
|
167
|
+
bytes: 'byteLength' in result ? result.byteLength : 0,
|
|
150
168
|
durationMs: clock() - startedAtMs,
|
|
151
169
|
});
|
|
152
170
|
return result;
|
|
@@ -176,20 +194,10 @@ async function downloadBlob(
|
|
|
176
194
|
if (!isBlobId(blobId)) {
|
|
177
195
|
throw syncError('blob.not_found', 'malformed blobId (§5.9.1)');
|
|
178
196
|
}
|
|
179
|
-
//
|
|
180
|
-
//
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
let entry: { record: BlobRecord; bytes: Uint8Array } | undefined;
|
|
184
|
-
if (presignConfigured) {
|
|
185
|
-
if (!(await store.has(ctx.partition, blobId))) {
|
|
186
|
-
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
187
|
-
}
|
|
188
|
-
} else {
|
|
189
|
-
entry = await store.get(ctx.partition, blobId);
|
|
190
|
-
if (entry === undefined) {
|
|
191
|
-
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
192
|
-
}
|
|
197
|
+
// Existence first (a HEAD), authorization second, bytes last: no blob byte
|
|
198
|
+
// is read for an actor the reference index does not authorize (§5.9.5).
|
|
199
|
+
if (!(await store.has(ctx.partition, blobId))) {
|
|
200
|
+
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
193
201
|
}
|
|
194
202
|
|
|
195
203
|
// §5.9.5 authorization: resolve once, then test referencing rows.
|
|
@@ -264,25 +272,37 @@ async function downloadBlob(
|
|
|
264
272
|
headers: {
|
|
265
273
|
ETag: `"${blobId}"`,
|
|
266
274
|
'Cache-Control': 'private, max-age=0',
|
|
267
|
-
Vary: 'Authorization
|
|
275
|
+
Vary: 'Authorization',
|
|
268
276
|
},
|
|
269
277
|
url: issued.url,
|
|
270
278
|
urlExpiresAtMs: issued.urlExpiresAtMs,
|
|
271
279
|
};
|
|
272
280
|
}
|
|
273
281
|
|
|
274
|
-
// Inline delivery
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
282
|
+
// Inline delivery streams the stored bytes verbatim (identity encoding,
|
|
283
|
+
// §5.9.5). A store without `open` serves its `get` bytes as one chunk.
|
|
284
|
+
const opened =
|
|
285
|
+
store.open !== undefined
|
|
286
|
+
? await store.open(ctx.partition, blobId)
|
|
287
|
+
: await store.get(ctx.partition, blobId).then((entry) =>
|
|
288
|
+
entry === undefined
|
|
289
|
+
? undefined
|
|
290
|
+
: {
|
|
291
|
+
record: entry.record,
|
|
292
|
+
body: bytesStream(entry.bytes),
|
|
293
|
+
byteLength: entry.bytes.length,
|
|
294
|
+
},
|
|
295
|
+
);
|
|
296
|
+
if (opened === undefined) {
|
|
297
|
+
throw syncError('blob.not_found', 'unknown blob (§5.9.5)');
|
|
279
298
|
}
|
|
280
299
|
// §5.9.5: inline bytes carry the stored media type, so an <img> or <a>
|
|
281
300
|
// pointed at the route renders them. A JSON type stays octet-stream: a
|
|
282
301
|
// JSON body is reserved for the signed-URL envelope.
|
|
283
|
-
const mediaType =
|
|
302
|
+
const mediaType = opened.record.mediaType;
|
|
284
303
|
return {
|
|
285
|
-
|
|
304
|
+
body: opened.body,
|
|
305
|
+
byteLength: opened.byteLength,
|
|
286
306
|
headers: {
|
|
287
307
|
'Content-Type':
|
|
288
308
|
mediaType === undefined ||
|
|
@@ -293,7 +313,7 @@ async function downloadBlob(
|
|
|
293
313
|
'Content-Security-Policy': "sandbox; default-src 'none'",
|
|
294
314
|
ETag: `"${blobId}"`,
|
|
295
315
|
'Cache-Control': 'private, max-age=0',
|
|
296
|
-
Vary: 'Authorization
|
|
316
|
+
Vary: 'Authorization',
|
|
297
317
|
},
|
|
298
318
|
};
|
|
299
319
|
}
|
|
@@ -303,11 +323,9 @@ async function downloadBlob(
|
|
|
303
323
|
* auth is the adapter's job — uploading is host-auth-only, not scope-bearing,
|
|
304
324
|
* so any authenticated actor may obtain a grant within the size cap (§5.9.3).
|
|
305
325
|
*
|
|
306
|
-
* The size cap is enforced HERE, up front, against the declared `byteLength
|
|
307
|
-
*
|
|
308
|
-
*
|
|
309
|
-
* and the address is verified at reference time (§5.9.6 push existence) and on
|
|
310
|
-
* every download (§5.9.5/§5.1). Returns a presigned single PUT, or a
|
|
326
|
+
* The size cap is enforced HERE, up front, against the declared `byteLength`;
|
|
327
|
+
* the presigned PUT binds that length and the content address, so the object
|
|
328
|
+
* store rejects any other bytes (§5.9.3). Returns a presigned single PUT, or a
|
|
311
329
|
* `present` marker if the blob already exists (idempotent §5.9.3), or an empty
|
|
312
330
|
* result when no presigned-upload store is configured (client streams direct).
|
|
313
331
|
*/
|
|
@@ -335,7 +353,8 @@ export async function handleBlobUploadGrant(
|
|
|
335
353
|
// §5.9.3: cap enforced at grant issuance, before any URL is minted.
|
|
336
354
|
throw syncError(
|
|
337
355
|
'blob.too_large',
|
|
338
|
-
|
|
356
|
+
'blob byteLength exceeds the server size cap (§5.9.3)',
|
|
357
|
+
JSON.stringify({ max: maxBytes }),
|
|
339
358
|
);
|
|
340
359
|
}
|
|
341
360
|
// §5.9.3 no presigned-upload store: report no grant; the client streams
|
package/src/blob-store.ts
CHANGED
|
Binary file
|