@gusto/baerly-storage 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -124
- package/dist/API.md +161 -511
- package/dist/CHANGELOG.md +1148 -307
- package/dist/RECIPES.md +398 -80
- package/dist/THIRD-PARTY-LICENSES.txt +5 -26
- package/dist/{app-config-DyiFK3_k.d.ts → app-config-C6qW7m8S.d.ts} +119 -111
- package/dist/app-config.d.ts +31 -1
- package/dist/app-config.js +1 -2
- package/dist/app-config.js.map +1 -1
- package/dist/auth.d.ts +33 -28
- package/dist/auth.js +549 -985
- package/dist/auth.js.map +1 -1
- package/dist/baerly.js +130 -138
- package/dist/baerly.js.map +1 -1
- package/dist/baerly.spec.json +97 -22
- package/dist/capacity-warning-header-BlVopyiI.js +16 -0
- package/dist/capacity-warning-header-BlVopyiI.js.map +1 -0
- package/dist/{client-Fe8CwB73.d.ts → client-CHQTXpef.d.ts} +121 -36
- package/dist/client-react.d.ts +67 -20
- package/dist/client-react.js +240 -54
- package/dist/client-react.js.map +1 -1
- package/dist/client.d.ts +5 -4
- package/dist/client.js +51 -16
- package/dist/client.js.map +1 -1
- package/dist/{cloudflare-DzKBrUlg.js → cloudflare-CKHYXccw.js} +504 -124
- package/dist/cloudflare-CKHYXccw.js.map +1 -0
- package/dist/cloudflare.d.ts +106 -71
- package/dist/cloudflare.js +334 -271
- package/dist/cloudflare.js.map +1 -1
- package/dist/code-resolution-CL_7L0wS.js +22 -0
- package/dist/code-resolution-CL_7L0wS.js.map +1 -0
- package/dist/collection-api-DbyhLAY8.d.ts +1001 -0
- package/dist/config-DEzInG6M.js +4035 -0
- package/dist/config-DEzInG6M.js.map +1 -0
- package/dist/constants-D3GSH3tc.js +469 -0
- package/dist/constants-D3GSH3tc.js.map +1 -0
- package/dist/{context-DRZ3tZ5l.js → context-rlpSnHFA.js} +17 -17
- package/dist/context-rlpSnHFA.js.map +1 -0
- package/dist/cost-B52llVDx.js +330 -0
- package/dist/cost-B52llVDx.js.map +1 -0
- package/dist/{current-json-C6nhPOm1.js → current-json-CC6lE4Aj.js} +115 -32
- package/dist/current-json-CC6lE4Aj.js.map +1 -0
- package/dist/db-plFUNTSF.d.ts +130 -0
- package/dist/db-read-seams-BSs0m1z1.js +3261 -0
- package/dist/db-read-seams-BSs0m1z1.js.map +1 -0
- package/dist/{deploy-DonTr-CV.js → deploy-Obj4nE-Z.js} +12 -16
- package/dist/deploy-Obj4nE-Z.js.map +1 -0
- package/dist/{dev-banner-CWHIk8tH.d.ts → dev-banner-B5AUHmu1.d.ts} +1 -4
- package/dist/dev-landing-BI5UdmOb.d.ts +32 -0
- package/dist/dev-landing-CD-si4dW.js +50 -0
- package/dist/dev-landing-CD-si4dW.js.map +1 -0
- package/dist/dev-vite.d.ts +38 -39
- package/dist/dev-vite.js +67 -36
- package/dist/dev-vite.js.map +1 -1
- package/dist/dev.d.ts +3 -2
- package/dist/dev.js +2 -1
- package/dist/doctor-BA0i-YAN.js +270 -0
- package/dist/doctor-BA0i-YAN.js.map +1 -0
- package/dist/dump-D1vX8ozg.js +133 -0
- package/dist/dump-D1vX8ozg.js.map +1 -0
- package/dist/errors-7Rk1HGCE.js +160 -0
- package/dist/errors-7Rk1HGCE.js.map +1 -0
- package/dist/{validate-DtljsCh1.js → export-BVL2EvYE.js} +409 -19
- package/dist/export-BVL2EvYE.js.map +1 -0
- package/dist/fsck-BWkPgov4.js +145 -0
- package/dist/fsck-BWkPgov4.js.map +1 -0
- package/dist/gcs-admin-Dso1HWQj.d.ts +127 -0
- package/dist/gcs.d.ts +30 -0
- package/dist/gcs.js +1 -0
- package/dist/goog4-signer-CeLIABoE.js +331 -0
- package/dist/goog4-signer-CeLIABoE.js.map +1 -0
- package/dist/{http-DGd5tbNs.js → http-Z5rZyYMX.js} +898 -271
- package/dist/http-Z5rZyYMX.js.map +1 -0
- package/dist/http-transport-DNUM8kU-.js +1299 -0
- package/dist/http-transport-DNUM8kU-.js.map +1 -0
- package/dist/http.d.ts +366 -294
- package/dist/http.js +2 -2
- package/dist/index-JJrIu1pO.d.ts +135 -0
- package/dist/index-LEgHN0qJ.d.ts +50 -0
- package/dist/{index-CuA9Mi3K.d.ts → index-RiEYj8t0.d.ts} +291 -78
- package/dist/index.d.ts +27 -6
- package/dist/index.js +3 -5
- package/dist/inspect-BTsVYZKd.js +196 -0
- package/dist/inspect-BTsVYZKd.js.map +1 -0
- package/dist/{key-Di6yyxFu.js → key-C4FPZ9Io.js} +3 -4
- package/dist/{key-Di6yyxFu.js.map → key-C4FPZ9Io.js.map} +1 -1
- package/dist/ledger-Dj_bGgo0.js +192 -0
- package/dist/ledger-Dj_bGgo0.js.map +1 -0
- package/dist/legacy-names-DgZBtgOh.js +47 -0
- package/dist/legacy-names-DgZBtgOh.js.map +1 -0
- package/dist/log-C9W15Dfo.d.ts +94 -0
- package/dist/log-tail-DWJbneBF.js +142 -0
- package/dist/log-tail-DWJbneBF.js.map +1 -0
- package/dist/log-walk-C-vA-WGJ.js +111 -0
- package/dist/log-walk-C-vA-WGJ.js.map +1 -0
- package/dist/maintenance-Dd-Yakqj.js +1709 -0
- package/dist/maintenance-Dd-Yakqj.js.map +1 -0
- package/dist/maintenance-DevB6z4e.js +2237 -0
- package/dist/maintenance-DevB6z4e.js.map +1 -0
- package/dist/maintenance.d.ts +100 -3
- package/dist/maintenance.js +1 -3
- package/dist/node.d.ts +130 -142
- package/dist/node.js +4 -3
- package/dist/{observability-BmflLCdF.js → observability-C-ISCCLE.js} +1275 -332
- package/dist/observability-C-ISCCLE.js.map +1 -0
- package/dist/observability.d.ts +27 -3
- package/dist/observability.js +3 -3
- package/dist/path-segment-DBXG26PW.js +363 -0
- package/dist/path-segment-DBXG26PW.js.map +1 -0
- package/dist/{plan-a7PJHBCy.js → plan-BvQbaNMe.js} +19 -11
- package/dist/plan-BvQbaNMe.js.map +1 -0
- package/dist/request-CRWiTrza.js +272 -0
- package/dist/request-CRWiTrza.js.map +1 -0
- package/dist/restore-U6Im8V1V.js +1467 -0
- package/dist/restore-U6Im8V1V.js.map +1 -0
- package/dist/retry-OBtLB2GC.js +126 -0
- package/dist/retry-OBtLB2GC.js.map +1 -0
- package/dist/rolldown-runtime-D3KUYPsf.js +12 -0
- package/dist/{runtime-spec-Dsw2-n8S.js → runtime-spec-B5mg3pHF.js} +189 -28
- package/dist/runtime-spec-B5mg3pHF.js.map +1 -0
- package/dist/{s3-http-B-z8th6O.d.ts → s3-http-DBC_wiCL.d.ts} +18 -10
- package/dist/s3.d.ts +20 -7
- package/dist/s3.js +4 -5
- package/dist/s3.js.map +1 -1
- package/dist/sha256-4FfGQNPw.js +22 -0
- package/dist/sha256-4FfGQNPw.js.map +1 -0
- package/dist/{shared-secret-DgEicd3x.js → shared-secret-Vkur-8r0.js} +27 -15
- package/dist/shared-secret-Vkur-8r0.js.map +1 -0
- package/dist/sigv4-sign-5IUflUD_.js +507 -0
- package/dist/sigv4-sign-5IUflUD_.js.map +1 -0
- package/dist/snapshot-CknzkDM0.js +148 -0
- package/dist/snapshot-CknzkDM0.js.map +1 -0
- package/dist/{src-BL_JoPTn.js → src-Actga-bO.js} +1004 -378
- package/dist/src-Actga-bO.js.map +1 -0
- package/dist/src-D0MQfvOp.js +812 -0
- package/dist/src-D0MQfvOp.js.map +1 -0
- package/dist/src-Dr9uo3dc.js +501 -0
- package/dist/src-Dr9uo3dc.js.map +1 -0
- package/dist/{subcommand-DoDA-K7d.js → subcommand-BLoF66dG.js} +89 -16
- package/dist/subcommand-BLoF66dG.js.map +1 -0
- package/dist/{time-DUqvFMHq.js → time-DRxTP9iv.js} +15 -34
- package/dist/time-DRxTP9iv.js.map +1 -0
- package/dist/types-suVbsEWS.d.ts +116 -0
- package/dist/{usage-DBV1eQHC.js → usage-CPzVhrQr.js} +40 -45
- package/dist/usage-CPzVhrQr.js.map +1 -0
- package/dist/uuid-BuT_c404.js +333 -0
- package/dist/uuid-BuT_c404.js.map +1 -0
- package/dist/verifier-Bmlmndzo.d.ts +95 -0
- package/dist/{wire-izfGZDvr.js → wire-DndvKU-K.js} +2 -3
- package/dist/wire-DndvKU-K.js.map +1 -0
- package/package.json +96 -69
- package/dist/aws4fetch.esm-p7eEzLCs.js +0 -1702
- package/dist/aws4fetch.esm-p7eEzLCs.js.map +0 -1
- package/dist/bytes-BXdamO3L.js +0 -11
- package/dist/bytes-BXdamO3L.js.map +0 -1
- package/dist/cloudflare-DzKBrUlg.js.map +0 -1
- package/dist/config-1tv0-Yzm.js +0 -2693
- package/dist/config-1tv0-Yzm.js.map +0 -1
- package/dist/constants-CfeMWQWk.js +0 -222
- package/dist/constants-CfeMWQWk.js.map +0 -1
- package/dist/context-DRZ3tZ5l.js.map +0 -1
- package/dist/cost-DSMIVqzL.js +0 -203
- package/dist/cost-DSMIVqzL.js.map +0 -1
- package/dist/current-json-C6nhPOm1.js.map +0 -1
- package/dist/db-DsPkOVEH.d.ts +0 -254
- package/dist/deploy-DonTr-CV.js.map +0 -1
- package/dist/doctor-DSIyCDg_.js +0 -346
- package/dist/doctor-DSIyCDg_.js.map +0 -1
- package/dist/dump-Bh5k26VQ.js +0 -94
- package/dist/dump-Bh5k26VQ.js.map +0 -1
- package/dist/errors-Dfa3p78c.js +0 -78
- package/dist/errors-Dfa3p78c.js.map +0 -1
- package/dist/export-DtF5tDIM.js +0 -401
- package/dist/export-DtF5tDIM.js.map +0 -1
- package/dist/fsck-D_djMO5D.js +0 -279
- package/dist/fsck-D_djMO5D.js.map +0 -1
- package/dist/http-DGd5tbNs.js.map +0 -1
- package/dist/index-0rZcMBwy.d.ts +0 -115
- package/dist/index-BNRh23K9.d.ts +0 -162
- package/dist/index-BYk9oCNf.d.ts +0 -1112
- package/dist/indexes-Dn9hbGQG.d.ts +0 -22
- package/dist/inspect-6WZ-pC7-.js +0 -133
- package/dist/inspect-6WZ-pC7-.js.map +0 -1
- package/dist/log-tail-D98U1JBo.js +0 -522
- package/dist/log-tail-D98U1JBo.js.map +0 -1
- package/dist/maintenance-B4Dki6Ig.js +0 -2157
- package/dist/maintenance-B4Dki6Ig.js.map +0 -1
- package/dist/maintenance-B_e0T99L.d.ts +0 -537
- package/dist/maintenance-C7coQp4c.js +0 -1418
- package/dist/maintenance-C7coQp4c.js.map +0 -1
- package/dist/normalize-rjnZLwS4.js +0 -133
- package/dist/normalize-rjnZLwS4.js.map +0 -1
- package/dist/observability-BmflLCdF.js.map +0 -1
- package/dist/plan-a7PJHBCy.js.map +0 -1
- package/dist/query-rAtXNQQB.js +0 -1773
- package/dist/query-rAtXNQQB.js.map +0 -1
- package/dist/rebuild-index-BVzQSZlm.js +0 -81
- package/dist/rebuild-index-BVzQSZlm.js.map +0 -1
- package/dist/request-u0M51mhJ.js +0 -72
- package/dist/request-u0M51mhJ.js.map +0 -1
- package/dist/restore-R-4rbciV.js +0 -778
- package/dist/restore-R-4rbciV.js.map +0 -1
- package/dist/rolldown-runtime-EAsCxrDo.js +0 -14
- package/dist/runtime-spec-Dsw2-n8S.js.map +0 -1
- package/dist/shared-secret-DgEicd3x.js.map +0 -1
- package/dist/src-BL_JoPTn.js.map +0 -1
- package/dist/src-CYHFDrKz.js +0 -536
- package/dist/src-CYHFDrKz.js.map +0 -1
- package/dist/src-DYZ4H1aF.js +0 -392
- package/dist/src-DYZ4H1aF.js.map +0 -1
- package/dist/subcommand-DoDA-K7d.js.map +0 -1
- package/dist/time-DUqvFMHq.js.map +0 -1
- package/dist/usage-DBV1eQHC.js.map +0 -1
- package/dist/validate-DtljsCh1.js.map +0 -1
- package/dist/wire-izfGZDvr.js.map +0 -1
|
@@ -0,0 +1,4035 @@
|
|
|
1
|
+
import { c as isWriteConflict, s as BaerlyError } from "./baerly.js";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { link, mkdir, readFile, readdir, realpath, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
4
|
+
import { dirname, join, resolve, sep } from "node:path";
|
|
5
|
+
import { existsSync } from "node:fs";
|
|
6
|
+
import { pathToFileURL } from "node:url";
|
|
7
|
+
// ../protocol/src/auth-resolution.ts
|
|
8
|
+
/**
|
|
9
|
+
* Accepted form of a Cloudflare Access `teamDomain`: the bare team name
|
|
10
|
+
* (`acme`), not the `acme.cloudflareaccess.com` host. The
|
|
11
|
+
* `cloudflareAccess` preset interpolates it into the JWKS host
|
|
12
|
+
* `https://<teamDomain>.cloudflareaccess.com`, so a `/`, `@`, `.` or `:`
|
|
13
|
+
* could point the key fetch at another host. Shared by the preset, the
|
|
14
|
+
* CLI config loader and `baerly doctor` so all three accept the same set.
|
|
15
|
+
*
|
|
16
|
+
* Sited here rather than in `constants.ts`: this module is already in
|
|
17
|
+
* the `auth.js` closure, and importing `constants.ts` from the preset
|
|
18
|
+
* would add about 0.6 KB min-gz to it.
|
|
19
|
+
*/
|
|
20
|
+
const CF_ACCESS_TEAM_NAME_PATTERN = /^[a-z0-9-]+$/;
|
|
21
|
+
// ../protocol/src/constants.ts
|
|
22
|
+
/**
|
|
23
|
+
* Max encoded size of one committed document (UTF-8 bytes of
|
|
24
|
+
* `encodeJsonBytes(post-image)`, the bytes `LogEntry.after` is stored
|
|
25
|
+
* as): 1 MiB.
|
|
26
|
+
*
|
|
27
|
+
* The writer refuses an insert, update or replace whose post-image is
|
|
28
|
+
* larger, as `InvalidConfig` before any storage call. That bounds every
|
|
29
|
+
* write path the same way: in-process calls, which no HTTP body cap
|
|
30
|
+
* covers, and HTTP merge-patches, each of which fits `MAX_BODY_BYTES`
|
|
31
|
+
* but can grow a stored row past it. Only the post-image is checked, so
|
|
32
|
+
* a row stored larger by an earlier release stays readable, deletable,
|
|
33
|
+
* and shrinkable.
|
|
34
|
+
*
|
|
35
|
+
* 1 MiB matches `MAX_BODY_BYTES` (a document at the cap still fits one
|
|
36
|
+
* request body, with the envelope's few bytes of headroom lost).
|
|
37
|
+
* Firestore's document limit is the same 1 MiB. Any snapshot format must
|
|
38
|
+
* hold one row at this size: a later format sizes its storage unit to
|
|
39
|
+
* fit the row rather than shrinking this cap.
|
|
40
|
+
*/
|
|
41
|
+
const MAX_DOCUMENT_BYTES = 1048576;
|
|
42
|
+
/**
|
|
43
|
+
* Max total object-key length S3 and R2 accept (UTF-8 bytes). A key
|
|
44
|
+
* over this fails the PUT with an opaque provider 400; baerly surfaces
|
|
45
|
+
* it early as InvalidConfig. See docs/spec/storage-compatibility.md.
|
|
46
|
+
*/
|
|
47
|
+
const MAX_KEY_BYTES = 1024;
|
|
48
|
+
/**
|
|
49
|
+
* Upper bound, in milliseconds, on one backoff between storage
|
|
50
|
+
* attempts, jittered or server-hinted (`Retry-After` is clamped to
|
|
51
|
+
* it). 20 s is the AWS SDKs' backoff cap
|
|
52
|
+
* (https://aws.amazon.com/blogs/developer/announcing-updated-retry-behavior-for-aws-sdks-and-tools/).
|
|
53
|
+
*
|
|
54
|
+
* @see packages/protocol/src/storage/retry.ts (`storageRetryDelayMillis`)
|
|
55
|
+
*/
|
|
56
|
+
const STORAGE_RETRY_MAX_DELAY_MILLIS = 2e4;
|
|
57
|
+
/**
|
|
58
|
+
* Runaway guard on the tolerant forward-probe walk (`probeTailFrom`):
|
|
59
|
+
* max `log/<seq>` GETs past `tail_hint` before it THROWS `Internal`
|
|
60
|
+
* (rather than silently truncating reads). A runaway alarm only — the
|
|
61
|
+
* maintenance tick keeps `tail_hint` near the tail, so a >cap gap means
|
|
62
|
+
* maintenance is off or the collection is far past graduation.
|
|
63
|
+
*
|
|
64
|
+
* @see packages/server/src/log-tail.ts (`probeTailFrom`)
|
|
65
|
+
*/
|
|
66
|
+
const LOG_FORWARD_PROBE_CAP = 1e5;
|
|
67
|
+
/**
|
|
68
|
+
* Sequence distance retained below `log_seq_start` before `log/<seq>.json`
|
|
69
|
+
* objects become retirable: a rate limit on how fast retirement approaches the
|
|
70
|
+
* live floor.
|
|
71
|
+
*
|
|
72
|
+
* Folded entries stay readable below the floor for this long, which is what a
|
|
73
|
+
* windowed commit (`Writer#readFoldedWindow`, for an insert guard or an
|
|
74
|
+
* admission re-check whose read predates a fold) needs to finish rather than
|
|
75
|
+
* refuse with `WriteConflict`; it reads at most the entries a fold absorbed
|
|
76
|
+
* between two reads in one request, far fewer than 1024. The value is a
|
|
77
|
+
* literal: it carries no derivation, and shrinking it is a measured change of
|
|
78
|
+
* its own.
|
|
79
|
+
*
|
|
80
|
+
* **This window is NOT a paused-writer fence, and no safety property may be
|
|
81
|
+
* derived from it.** It is a cost margin and a rate limit on how fast
|
|
82
|
+
* retirement approaches the live floor — nothing more. The withdrawn argument
|
|
83
|
+
* was that a certified-deleted slot needs `log_seq_start` to advance by more
|
|
84
|
+
* than this window "during one in-flight commit". The arithmetic is correct and
|
|
85
|
+
* the quantifier is vacuous: one in-flight commit bounds no wall-clock and no
|
|
86
|
+
* commit count, so this is a rate x duration assumption expressed in sequences,
|
|
87
|
+
* the same class as `GC_GRACE_PERIOD_MILLIS` in seconds. Lowering the value on
|
|
88
|
+
* cost grounds costs windowed-commit headroom and retirement smoothness; it
|
|
89
|
+
* does not weaken a safety proof, because there was never one here.
|
|
90
|
+
*
|
|
91
|
+
* Withdrawing the claim does not leave the schedule open, because the commit
|
|
92
|
+
* path closes it directly. After the committing `log/<seq>` create resolves,
|
|
93
|
+
* `Writer#assertCommitAboveDeleteFloor` re-reads `current.json` and fails a
|
|
94
|
+
* create that won at a `seq` below `min(log_delete_floor, log_seq_start)` with
|
|
95
|
+
* `BaerlyError{code:"AmbiguousCommit"}`, rather than acknowledging a mutation
|
|
96
|
+
* no reader can see. That check keys on the certified delete floor, not on this
|
|
97
|
+
* window, which is exactly why the window may keep being a cost margin without
|
|
98
|
+
* carrying a safety obligation. Do not re-derive a fence from this value.
|
|
99
|
+
*
|
|
100
|
+
* @see packages/server/src/writer.ts (`Writer#assertCommitAboveDeleteFloor`)
|
|
101
|
+
* @see packages/server/src/log-retention.ts
|
|
102
|
+
* @see docs/spec/sync-protocol.md invariant 12
|
|
103
|
+
* @see docs/adr/002-ephemeral-coordination.md § Closed paths
|
|
104
|
+
*/
|
|
105
|
+
const LOG_RETENTION_SEQ_WINDOW = 1024;
|
|
106
|
+
/**
|
|
107
|
+
* MIME type written for `current.json` PUTs. S3 round-trips this on
|
|
108
|
+
* subsequent GETs; useful for diagnostics when staring at a bucket
|
|
109
|
+
* via the AWS console.
|
|
110
|
+
*
|
|
111
|
+
* @see packages/protocol/src/coordination/current-json.ts
|
|
112
|
+
*/
|
|
113
|
+
const CURRENT_JSON_CONTENT_TYPE = "application/json";
|
|
114
|
+
/**
|
|
115
|
+
* MIME type written for `gc/pending.json` PUTs.
|
|
116
|
+
*
|
|
117
|
+
* @see packages/protocol/src/coordination/gc-pending.ts
|
|
118
|
+
*/
|
|
119
|
+
const GC_PENDING_CONTENT_TYPE = "application/json";
|
|
120
|
+
/**
|
|
121
|
+
* Default grace period between "marking" a key for GC and "sweeping"
|
|
122
|
+
* (deleting) it. 7 days, chosen to span the worst plausible writer-
|
|
123
|
+
* retry window (a paused-process writer that resumes hours later
|
|
124
|
+
* should still find its idempotency anchor on the bucket). The
|
|
125
|
+
* `runGc()` function accepts an override for tests.
|
|
126
|
+
*
|
|
127
|
+
* Why 7 days specifically:
|
|
128
|
+
* - **1 day is too aggressive for batch workloads.** Worker isolate
|
|
129
|
+
* scheduling pauses, cross-region replication lag, and
|
|
130
|
+
* {@link STORAGE_RETRY_THROTTLE_BASE_MILLIS} retry cascades can plausibly
|
|
131
|
+
* exceed an hour under pathological conditions.
|
|
132
|
+
* - **30 days is conservative beyond the worst plausible pause.**
|
|
133
|
+
* Doubles `gc/pending.json` size at steady state and slows
|
|
134
|
+
* visibility into "did GC actually run?" by 4×.
|
|
135
|
+
* - **7 days spans the realistic upper bound** — long-running batch
|
|
136
|
+
* jobs, multi-region propagation delays, queue backlogs, and
|
|
137
|
+
* downstream outages the writer is retrying through. The protocol
|
|
138
|
+
* is unaffected by the choice of value within the [hours, weeks]
|
|
139
|
+
* range; this constant is the operator-tunable knob.
|
|
140
|
+
*
|
|
141
|
+
* Production code MUST NOT call `runGc` with `graceMillis` below the
|
|
142
|
+
* default outside maintenance windows — going below the longest
|
|
143
|
+
* plausible writer-retry latency risks deleting an anchor a writer
|
|
144
|
+
* is about to find on retry. Test code that sets `graceMillis: 0` is
|
|
145
|
+
* exercising the sweep path deliberately, not modelling production.
|
|
146
|
+
*
|
|
147
|
+
* @see packages/server/src/gc.ts
|
|
148
|
+
*/
|
|
149
|
+
const GC_GRACE_PERIOD_MILLIS = 6048e5;
|
|
150
|
+
/**
|
|
151
|
+
* Fraction of the GC grace one `runGc` pass may spend between the
|
|
152
|
+
* `current.json` read its DELETEs rest on and each DELETE batch. Once
|
|
153
|
+
* the elapsed time exceeds `graceMillis * GC_PASS_GRACE_FRACTION`, the
|
|
154
|
+
* pass issues no further DELETEs and leaves the remaining due
|
|
155
|
+
* candidates in `gc/pending.json` for a later pass, which re-reads the
|
|
156
|
+
* manifest.
|
|
157
|
+
*
|
|
158
|
+
* Why 1/10:
|
|
159
|
+
* - **A live pass never trips it.** Against the 7-day default the
|
|
160
|
+
* budget is ~16.8 hours, and a pass on every first-class host lasts
|
|
161
|
+
* seconds. Only a paused or starved process crosses it.
|
|
162
|
+
* - **It bounds how stale a DELETE's liveness proof can be.** It
|
|
163
|
+
* matters for the legacy `stale-log` arm, whose DELETE can rest on
|
|
164
|
+
* the pass's first `current.json` read: a candidate is due no earlier
|
|
165
|
+
* than its mark plus the grace, so a DELETE issued within `f` of the
|
|
166
|
+
* grace after that read rests on one taken at least `(1 - f)` of the
|
|
167
|
+
* grace after the mark. An `orphan-snapshot` DELETE always rests on
|
|
168
|
+
* the sweep-time re-read, taken after the candidate came due, so a
|
|
169
|
+
* full grace after its mark.
|
|
170
|
+
*
|
|
171
|
+
* In-band, by design: one clock comparison per DELETE batch, no lease
|
|
172
|
+
* and no knob. Elapsed time is the larger of one node's wall-clock and
|
|
173
|
+
* monotonic readings, so neither cross-node skew nor a backward clock
|
|
174
|
+
* correction shortens it, and a suspend the monotonic clock skips still
|
|
175
|
+
* shows on the wall clock. The after-the-mark figures rest on `due_at`
|
|
176
|
+
* and so keep invariant 14's wall-clock limitation. A pass with
|
|
177
|
+
* `graceMillis: 0` (tests only) has no grace to keep a fraction of and
|
|
178
|
+
* is not bounded.
|
|
179
|
+
*
|
|
180
|
+
* @see docs/spec/sync-protocol.md (invariant 14)
|
|
181
|
+
* @see packages/server/src/gc.ts
|
|
182
|
+
*/
|
|
183
|
+
const GC_PASS_GRACE_FRACTION = 1 / 10;
|
|
184
|
+
/**
|
|
185
|
+
* Cap on candidates kept in `gc/pending.json` to bound the size of
|
|
186
|
+
* the file. The compactor marks at most this many candidates per
|
|
187
|
+
* pass; subsequent passes pick up the rest. Larger collections will
|
|
188
|
+
* lag GC by one pass per `GC_MAX_PENDING_CANDIDATES` orphans, which
|
|
189
|
+
* is acceptable.
|
|
190
|
+
*
|
|
191
|
+
* @see packages/server/src/gc.ts
|
|
192
|
+
*/
|
|
193
|
+
const GC_MAX_PENDING_CANDIDATES = 1e3;
|
|
194
|
+
/**
|
|
195
|
+
* Keys per LIST response page on every in-tree object-store adapter:
|
|
196
|
+
* S3 and GCS `max-keys` and R2 binding `limit` all cap at 1000. `runGc`
|
|
197
|
+
* widens its orphan-snapshot mark window past already-pending keys only
|
|
198
|
+
* up to this size, so the widening never adds a full LIST page (a
|
|
199
|
+
* provider may still truncate a page short and cost another request).
|
|
200
|
+
*
|
|
201
|
+
* @see packages/server/src/gc.ts
|
|
202
|
+
* @see docs/spec/storage-compatibility.md
|
|
203
|
+
*/
|
|
204
|
+
const GC_MARK_LIST_PAGE_KEYS = 1e3;
|
|
205
|
+
/**
|
|
206
|
+
* Floor for the ratio denominator — avoid div-by-tiny on a fresh collection. Also
|
|
207
|
+
* sets the first-fold threshold: until a snapshot exists, fold fires at tail ≈ this.
|
|
208
|
+
*
|
|
209
|
+
* @see packages/server/src/maintenance.ts
|
|
210
|
+
*/
|
|
211
|
+
const MAINTENANCE_MIN_LIVE_BYTES = 65536;
|
|
212
|
+
/**
|
|
213
|
+
* Cloudflare Paid scheduled-compactor tail-discovery budget (`P`) for
|
|
214
|
+
* `CLOUDFLARE_PAID_TIER`. Without one, the scheduled probe may walk up to
|
|
215
|
+
* {@link LOG_FORWARD_PROBE_CAP} (100,000) GETs past a stale `tail_hint`,
|
|
216
|
+
* which no Paid invocation can issue: the request ledger refuses the pass
|
|
217
|
+
* at {@link CF_PAID_REQUEST_LIMIT} and nothing folds. With a budget, a
|
|
218
|
+
* pass that exhausts it checkpoints the certified lower bound in
|
|
219
|
+
* `current.json` and the next pass resumes there.
|
|
220
|
+
*
|
|
221
|
+
* With the Paid profile's per-pass slice
|
|
222
|
+
* `N` = {@link NODE_MAINTENANCE_FOLD_ENTRIES_PER_PASS} (200), the worst
|
|
223
|
+
* pass is `4 + N + P` = 1,204 storage calls, 3,612 with every call retried
|
|
224
|
+
* {@link STORAGE_RETRY_MAX_ATTEMPTS} times, well inside 9,000. 1,000 is
|
|
225
|
+
* five slices of discovery per pass: large enough that a pass behind by
|
|
226
|
+
* a normal backlog still folds, small enough that a scheduled compact, GC
|
|
227
|
+
* and retirement pass together stay under the limit at full retries.
|
|
228
|
+
*
|
|
229
|
+
* @see packages/server/src/maintenance.ts (`CLOUDFLARE_PAID_TIER`)
|
|
230
|
+
* @see packages/server/src/maintenance-budget.test.ts
|
|
231
|
+
*/
|
|
232
|
+
const CF_PAID_COMPACT_TAIL_PROBE_GETS = 1e3;
|
|
233
|
+
/**
|
|
234
|
+
* SNAPSHOT-rebuild ceiling `C` (the unsliceable axis — Decision 3a), memory. A
|
|
235
|
+
* conservative configured budget for the CF-free ~10 ms CPU limit; the ~5.5 ms
|
|
236
|
+
* rebuild-time figure behind it is a sizing estimate, not an observed platform
|
|
237
|
+
* limit. Raise via BAERLY_MAINTENANCE_MAX_FOLD_BYTES on capable
|
|
238
|
+
* hosts. Auto-maintained snapshot ceiling S_max = C. NOT snapshot+tail (tail is sliced).
|
|
239
|
+
*
|
|
240
|
+
* @see docs/spec/scale-ceilings.md
|
|
241
|
+
* @see packages/server/src/maintenance.ts
|
|
242
|
+
*/
|
|
243
|
+
const MAINTENANCE_MAX_FOLD_BYTES_DEFAULT = 524288;
|
|
244
|
+
/**
|
|
245
|
+
* SNAPSHOT-rebuild ceiling `E`, per-entry CPU axis: gates `snapshot_rows` (per-entry
|
|
246
|
+
* parse/merge/serialize is ~half of fold CPU and scales with ROW COUNT not bytes, VLDB 2021
|
|
247
|
+
* Sarkar — a tiny-doc snapshot can blow CPU under C).
|
|
248
|
+
*
|
|
249
|
+
* This is the **cf-free** value and the shared default. `pnpm bench:fold-ceiling`
|
|
250
|
+
* measures 2048 rows at ~1.5 ms against the ~10 ms free-tier budget (a ~6.6×
|
|
251
|
+
* margin), and the row axis does not reach 10 ms until ~16,384 rows. The value
|
|
252
|
+
* stays at 2048 as a conservative configured budget: a fold that overruns a free
|
|
253
|
+
* isolate's CPU may be terminated before the `current.json` CAS lands — see
|
|
254
|
+
* {@link CF_FREE_MAX_SAFE_FOLD_BYTES}. The
|
|
255
|
+
* larger hosts no longer inherit it; see {@link CF_PAID_MAINTENANCE_MAX_FOLD_ROWS}
|
|
256
|
+
* and {@link NODE_MAINTENANCE_MAX_FOLD_ROWS}.
|
|
257
|
+
*
|
|
258
|
+
* @see docs/spec/scale-ceilings.md
|
|
259
|
+
* @see packages/server/src/maintenance.ts
|
|
260
|
+
*/
|
|
261
|
+
const MAINTENANCE_MAX_FOLD_ROWS = 2048;
|
|
262
|
+
/**
|
|
263
|
+
* Snapshot-rebuild byte ceiling `C` for the **cf-paid** profile.
|
|
264
|
+
*
|
|
265
|
+
* A paid isolate's wall is ~128 MB of Worker memory, not CPU — at a 30 s CPU
|
|
266
|
+
* budget the rebuild is never CPU-bound, but a fold holds the old snapshot, the
|
|
267
|
+
* new snapshot, and the tail at once. `pnpm bench:fold-ceiling` measures peak
|
|
268
|
+
* heap at ~2.4× the snapshot on the byte axis, stable across 704 MB–4.3 GB
|
|
269
|
+
* heaps: 8 MiB peaks at ~19.5 MB, a 4× margin under 128 MB. The next grid cell
|
|
270
|
+
* up (16 MiB ⇒ 37.3 MB peak) exceeds that margin, which is what fixes this
|
|
271
|
+
* value.
|
|
272
|
+
*
|
|
273
|
+
* The margin is deliberately tight because overrun is NOT graceful here: an
|
|
274
|
+
* isolate OOM mid-fold would leave the `current.json` CAS unpublished, so that
|
|
275
|
+
* fold would not advance `log_seq_start`.
|
|
276
|
+
*
|
|
277
|
+
* @see docs/spec/scale-ceilings.md
|
|
278
|
+
* @see bench/measurement/fold-ceiling-probe.ts
|
|
279
|
+
*/
|
|
280
|
+
const CF_PAID_MAINTENANCE_MAX_FOLD_BYTES = 8388608;
|
|
281
|
+
/**
|
|
282
|
+
* Snapshot-rebuild row ceiling `E` for the **cf-paid** profile. Same 128 MB
|
|
283
|
+
* memory wall and same 4× margin as {@link CF_PAID_MAINTENANCE_MAX_FOLD_BYTES};
|
|
284
|
+
* the row axis costs ~600 B of peak heap per row (measured), so 32,768 rows
|
|
285
|
+
* peaks at ~19.7 MB. The next grid cell up (65,536 rows ⇒ 39.4 MB) exceeds the
|
|
286
|
+
* margin.
|
|
287
|
+
*
|
|
288
|
+
* @see docs/spec/scale-ceilings.md
|
|
289
|
+
*/
|
|
290
|
+
const CF_PAID_MAINTENANCE_MAX_FOLD_ROWS = 32768;
|
|
291
|
+
/**
|
|
292
|
+
* Snapshot-rebuild byte ceiling `C` for the **node** profile.
|
|
293
|
+
*
|
|
294
|
+
* Serverful Node has no per-request cap, and its fold is not abortable —
|
|
295
|
+
* `nodeMaintenanceDispatch` passes no `signal` — so an over-large fold
|
|
296
|
+
* COMPLETES. The failure mode is one slow write, not a stranded CAS. That
|
|
297
|
+
* asymmetry is why Node's ceiling is set generously where cf-free's and
|
|
298
|
+
* cf-paid's are set tightly: UNDER-setting `C` permanently defers the fold,
|
|
299
|
+
* which is the worse failure on every host. Inline fold latency is budgeted by
|
|
300
|
+
* {@link NODE_MAINTENANCE_FOLD_ENTRIES_PER_PASS} (the sliceable tail work), not
|
|
301
|
+
* by this ceiling (the unsliceable snapshot rebuild).
|
|
302
|
+
*
|
|
303
|
+
* 32 MiB is the top of the measured grid, NOT the wall: `pnpm bench:fold-ceiling`
|
|
304
|
+
* reports `grid-exhausted` for this profile at every margin and at every heap
|
|
305
|
+
* from 704 MB up. At a 704 MB heap — the smallest realistic container — a
|
|
306
|
+
* 32 MiB fold peaks at ~37 MB, a ~19× margin. The scaling rule past the grid is
|
|
307
|
+
* `C ≈ heap / 10` (peak ≈ 2.4× snapshot, 4× margin). Finding the real wall
|
|
308
|
+
* needs the byte axis extended past 32 MiB.
|
|
309
|
+
*
|
|
310
|
+
* A separate hard ceiling sits near 512 MB regardless of heap: `encodeJsonBytes`
|
|
311
|
+
* / `decodeJsonBytes` materialize the whole snapshot as a JS string, so a
|
|
312
|
+
* snapshot at V8's `MAX_STRING_LENGTH` throws mid-fold.
|
|
313
|
+
*
|
|
314
|
+
* @see docs/spec/scale-ceilings.md
|
|
315
|
+
* @see bench/measurement/fold-ceiling-probe.ts
|
|
316
|
+
*/
|
|
317
|
+
const NODE_MAINTENANCE_MAX_FOLD_BYTES = 33554432;
|
|
318
|
+
/**
|
|
319
|
+
* Snapshot-rebuild row ceiling `E` for the **node** profile. 65,536 rows is the
|
|
320
|
+
* largest row cell with a stable measured peak (~39.4 MB, reproduced within
|
|
321
|
+
* 0.05 MB across four runs); the 131,072-row cell's peak is not yet reliably
|
|
322
|
+
* measurable. Against even a 704 MB heap that is a ~17× margin, consistent with
|
|
323
|
+
* {@link NODE_MAINTENANCE_MAX_FOLD_BYTES} being a measured floor rather than a
|
|
324
|
+
* wall.
|
|
325
|
+
*
|
|
326
|
+
* @see docs/spec/scale-ceilings.md
|
|
327
|
+
*/
|
|
328
|
+
const NODE_MAINTENANCE_MAX_FOLD_ROWS = 65536;
|
|
329
|
+
/** CF-free profile; the absent-context default. */
|
|
330
|
+
const MAINTENANCE_PROFILE_CF_FREE = {
|
|
331
|
+
gcInterval: 4,
|
|
332
|
+
gcMaxMarks: 20,
|
|
333
|
+
gcMaxSweeps: 10,
|
|
334
|
+
maxFoldEntriesPerPass: 20,
|
|
335
|
+
maxFoldBytes: MAINTENANCE_MAX_FOLD_BYTES_DEFAULT,
|
|
336
|
+
maxFoldRows: MAINTENANCE_MAX_FOLD_ROWS
|
|
337
|
+
};
|
|
338
|
+
/** Node profile; 10× CF-free per-pass caps, and its own measured snapshot ceilings. */
|
|
339
|
+
const MAINTENANCE_PROFILE_NODE = {
|
|
340
|
+
gcInterval: 2,
|
|
341
|
+
gcMaxMarks: 200,
|
|
342
|
+
gcMaxSweeps: 100,
|
|
343
|
+
maxFoldEntriesPerPass: 200,
|
|
344
|
+
maxFoldBytes: NODE_MAINTENANCE_MAX_FOLD_BYTES,
|
|
345
|
+
maxFoldRows: NODE_MAINTENANCE_MAX_FOLD_ROWS
|
|
346
|
+
};
|
|
347
|
+
/**
|
|
348
|
+
* CF-paid profile — opt-in via `BAERLY_MAINTENANCE_PROFILE=cf-paid`. A paid
|
|
349
|
+
* isolate keeps the CF-free single-phase shape (one phase per request) but has a
|
|
350
|
+
* 9,000-call request ledger limit ({@link CF_PAID_REQUEST_LIMIT}), so it reuses the
|
|
351
|
+
* `NODE_MAINTENANCE_*` per-pass caps.
|
|
352
|
+
*
|
|
353
|
+
* A profile moves RATE and the DEFER THRESHOLD, never the stored data or the
|
|
354
|
+
* query semantics — that is the equivalence invariant, and it is what
|
|
355
|
+
* `tests/integration/maintenance-profile-equivalence.test.ts` proves: a read
|
|
356
|
+
* folds snapshot + live tail, so HOW MUCH a profile has folded is invisible to
|
|
357
|
+
* a reader. Snapshot ceilings therefore differ per host, sized to each host's
|
|
358
|
+
* real wall (see {@link CF_PAID_MAINTENANCE_MAX_FOLD_BYTES}). Operators can
|
|
359
|
+
* still raise `C` out-of-band with `BAERLY_MAINTENANCE_MAX_FOLD_BYTES`.
|
|
360
|
+
*
|
|
361
|
+
* @see packages/adapter-cloudflare/src/worker.ts
|
|
362
|
+
* @see docs/about/graduation.md
|
|
363
|
+
*/
|
|
364
|
+
const MAINTENANCE_PROFILE_CF_PAID = {
|
|
365
|
+
gcInterval: 2,
|
|
366
|
+
gcMaxMarks: 200,
|
|
367
|
+
gcMaxSweeps: 100,
|
|
368
|
+
maxFoldEntriesPerPass: 200,
|
|
369
|
+
maxFoldBytes: CF_PAID_MAINTENANCE_MAX_FOLD_BYTES,
|
|
370
|
+
maxFoldRows: CF_PAID_MAINTENANCE_MAX_FOLD_ROWS
|
|
371
|
+
};
|
|
372
|
+
/**
|
|
373
|
+
* Rate-limit the defer-warn off SHARED current.json.last_warned_seq (not per-isolate
|
|
374
|
+
* memory — CF recycles isolates). ~once per this many writes.
|
|
375
|
+
*
|
|
376
|
+
* @see packages/protocol/src/coordination/current-json.ts
|
|
377
|
+
* @see packages/server/src/maintenance.ts
|
|
378
|
+
*/
|
|
379
|
+
const MAINTENANCE_WARN_INTERVAL_WRITES = 1e3;
|
|
380
|
+
/**
|
|
381
|
+
* Canonical auth posture identifiers consumed by `BaerlyAppConfig.auth`,
|
|
382
|
+
* the adapter resolution path (`packages/adapter-cloudflare/src/worker.ts`,
|
|
383
|
+
* `packages/adapter-node/src/baerly-node.ts`), and `baerly doctor`.
|
|
384
|
+
*
|
|
385
|
+
* - `"none"` — no header check; pin every request to `config.tenant`.
|
|
386
|
+
* For local dev, intranet, CLI tools — contexts where the network
|
|
387
|
+
* seam itself is the trust boundary.
|
|
388
|
+
* - `"shared-secret"` — bearer-token check; reads `SHARED_SECRET` from
|
|
389
|
+
* the runtime env and pins every request to `config.tenant`.
|
|
390
|
+
*
|
|
391
|
+
* Custom verifiers (`cloudflareAccess`, `bearerJwt`, SigV4, …) bypass
|
|
392
|
+
* this enum — pass a `Verifier` to the adapter factory's `verifier:`
|
|
393
|
+
* option instead.
|
|
394
|
+
*/
|
|
395
|
+
const AUTH_CONFIG_VALUES = ["none", "shared-secret"];
|
|
396
|
+
/**
|
|
397
|
+
* Locked error wording for `auth: "shared-secret"` + missing env var.
|
|
398
|
+
* Same pinning rationale as {@link NO_AUTH_CONFIGURED_MESSAGE}.
|
|
399
|
+
*/
|
|
400
|
+
const SHARED_SECRET_MISSING_MESSAGE = "baerly: auth=\"shared-secret\" but SHARED_SECRET env is empty/unset. Cloudflare: `wrangler secret put SHARED_SECRET`, or add to .dev.vars for local dev. Node: set in process env.";
|
|
401
|
+
// ../protocol/src/env.ts
|
|
402
|
+
const PAAS_MARKERS = [
|
|
403
|
+
"RAILWAY_ENVIRONMENT",
|
|
404
|
+
"RENDER",
|
|
405
|
+
"FLY_APP_NAME",
|
|
406
|
+
"K_SERVICE",
|
|
407
|
+
"DYNO",
|
|
408
|
+
"KUBERNETES_SERVICE_HOST",
|
|
409
|
+
"ECS_CONTAINER_METADATA_URI_V4"
|
|
410
|
+
];
|
|
411
|
+
/**
|
|
412
|
+
* True when `env` looks like a deployed environment: `NODE_ENV=production`
|
|
413
|
+
* or any {@link PAAS_MARKERS} set to a non-empty value. Decides whether a
|
|
414
|
+
* non-durable store (in-memory, local-fs) should fail loud instead of
|
|
415
|
+
* silently serving a production workload. A CI runner (`CI` set to a
|
|
416
|
+
* non-empty, non-`false` value) is never treated as deployed, so PaaS
|
|
417
|
+
* markers from an in-cluster CI agent do not trip it.
|
|
418
|
+
*/
|
|
419
|
+
const isDeployedEnv = (env) => {
|
|
420
|
+
const inCi = (env["CI"] ?? "") !== "" && env["CI"] !== "false";
|
|
421
|
+
return env["NODE_ENV"] === "production" || !inCi && PAAS_MARKERS.some((m) => (env[m] ?? "") !== "");
|
|
422
|
+
};
|
|
423
|
+
// ../protocol/src/sha256.ts
|
|
424
|
+
/**
|
|
425
|
+
* SHA-256 of `bytes` as a 64-char lowercase hex string.
|
|
426
|
+
*
|
|
427
|
+
* Zero-import leaf module on purpose: co-locating this with heavier
|
|
428
|
+
* runtime (snapshot-hash.ts / hashing.ts) would drag that module into
|
|
429
|
+
* every consumer's bundle closure. Callers that want a truncated form
|
|
430
|
+
* (e.g. {@link versionFromContent}'s content keys) slice the return.
|
|
431
|
+
*/
|
|
432
|
+
const sha256Hex = async (bytes) => {
|
|
433
|
+
let view;
|
|
434
|
+
if (bytes.buffer instanceof ArrayBuffer) view = bytes;
|
|
435
|
+
else {
|
|
436
|
+
view = new Uint8Array(bytes.byteLength);
|
|
437
|
+
view.set(bytes);
|
|
438
|
+
}
|
|
439
|
+
const digest = await crypto.subtle.digest("SHA-256", view);
|
|
440
|
+
return new Uint8Array(digest).toHex();
|
|
441
|
+
};
|
|
442
|
+
// ../protocol/src/storage/key.ts
|
|
443
|
+
/**
|
|
444
|
+
* Validate a raw {@link Storage} object key against the portable baerly key
|
|
445
|
+
* grammar before it is used to address an object. A key MUST be non-empty
|
|
446
|
+
* and `/`-delimited, with **no path segment equal to `.` or `..`**.
|
|
447
|
+
*
|
|
448
|
+
* The dot-segment rule is not a vendor quirk — it is a *client-side*
|
|
449
|
+
* universal: RFC 3986 §5.2.4 "remove dot segments" is mandatory for every
|
|
450
|
+
* conformant URL parser, so `<endpoint>/<bucket>/.` normalizes to
|
|
451
|
+
* `<endpoint>/<bucket>/` and `<endpoint>/<bucket>/..` escapes the bucket
|
|
452
|
+
* entirely — *before* the request is signed or sent, in TypeScript, Go,
|
|
453
|
+
* Rust, Python, or any language a `Storage` port is written in. Such a key
|
|
454
|
+
* cannot be addressed over the S3/R2 HTTP API regardless of backend; a bare
|
|
455
|
+
* `.` PUT surfaces as a confusing bucket-root 403 rather than a clear
|
|
456
|
+
* rejection. The kernel never emits such keys (caller-controlled segments
|
|
457
|
+
* are already screened by `assertPathSegment` one layer up), so this is the
|
|
458
|
+
* boundary guard that makes every adapter reject them *identically* and
|
|
459
|
+
* gives ports a single, testable contract.
|
|
460
|
+
*
|
|
461
|
+
* Thrown as `BaerlyError{code:"InvalidConfig"}` to match the sibling
|
|
462
|
+
* key/segment guards (`assertKeyWithinLimit`, `assertPathSegment`). The
|
|
463
|
+
* full-key 1024-byte ceiling is enforced separately on the write path
|
|
464
|
+
* (`assertKeyWithinLimit`), where multi-segment keys are assembled.
|
|
465
|
+
*
|
|
466
|
+
* @see docs/spec/storage-compatibility.md — "Key namespace"
|
|
467
|
+
*/
|
|
468
|
+
function assertValidStorageKey(key) {
|
|
469
|
+
if (key.length === 0) throw new BaerlyError("InvalidConfig", "storage key must be a non-empty string");
|
|
470
|
+
for (const segment of key.split("/")) if (segment === "." || segment === "..") throw new BaerlyError("InvalidConfig", `storage key may not contain a "." or ".." path segment — RFC 3986 dot-segment removal makes it unaddressable over the S3/R2 HTTP API: ${JSON.stringify(key)}`);
|
|
471
|
+
}
|
|
472
|
+
// ../protocol/src/storage/memory.ts
|
|
473
|
+
const utf8Encoder$2 = new TextEncoder();
|
|
474
|
+
const EPHEMERAL_ENV_OPT_IN = "BAERLY_ALLOW_EPHEMERAL_STORAGE";
|
|
475
|
+
const readProcessEnv = () => globalThis.process?.env ?? {};
|
|
476
|
+
let warnedEphemeralInDeployment = false;
|
|
477
|
+
const assertEphemeralAllowed = (explicitOptIn) => {
|
|
478
|
+
const env = readProcessEnv();
|
|
479
|
+
if (!isDeployedEnv(env)) return;
|
|
480
|
+
if (!explicitOptIn && env[EPHEMERAL_ENV_OPT_IN] !== "true") throw new BaerlyError("InvalidConfig", `Refusing to use in-memory storage in a deployed environment — all data is lost on restart. MemoryStorage is for tests/local dev only; configure a durable bucket (AWS S3 / Cloudflare R2). To intentionally opt in: new MemoryStorage({ ephemeral: true }) or ${EPHEMERAL_ENV_OPT_IN}=true.`);
|
|
481
|
+
if (!warnedEphemeralInDeployment) {
|
|
482
|
+
warnedEphemeralInDeployment = true;
|
|
483
|
+
console.warn("[baerly] in-memory storage in a deployed environment — ALL DATA IS LOST ON RESTART.");
|
|
484
|
+
}
|
|
485
|
+
};
|
|
486
|
+
/**
|
|
487
|
+
* Compare two keys by their UTF-8 byte sequences — the order S3 and
|
|
488
|
+
* R2 use for `list`. JavaScript's default string sort compares UTF-16
|
|
489
|
+
* code units, which diverges from UTF-8 byte order for supplementary-
|
|
490
|
+
* plane characters (e.g. emoji sort before high-BMP characters under
|
|
491
|
+
* UTF-16 but after them in UTF-8). Using this keeps the in-memory
|
|
492
|
+
* reference backend faithful to the real adapters. (All kernel keys
|
|
493
|
+
* are ASCII base-32, where the two orders coincide — this only
|
|
494
|
+
* matters for adversarial / non-ASCII keys.)
|
|
495
|
+
*/
|
|
496
|
+
const compareKeysUtf8$1 = (a, b) => {
|
|
497
|
+
const ba = utf8Encoder$2.encode(a);
|
|
498
|
+
const bb = utf8Encoder$2.encode(b);
|
|
499
|
+
const n = Math.min(ba.length, bb.length);
|
|
500
|
+
for (let i = 0; i < n; i++) if (ba[i] !== bb[i]) return ba[i] - bb[i];
|
|
501
|
+
return ba.length - bb.length;
|
|
502
|
+
};
|
|
503
|
+
/**
|
|
504
|
+
* In-memory `Storage`. The randomized property test runs against
|
|
505
|
+
* this; it must be deterministic — no clocks beyond the caller's,
|
|
506
|
+
* no randomness, no I/O.
|
|
507
|
+
*
|
|
508
|
+
* ETags are a monotonically increasing hex counter formatted in the
|
|
509
|
+
* `"<hex>"` shape S3 returns (the surrounding double-quotes are part
|
|
510
|
+
* of the ETag header value). Keys are stored verbatim.
|
|
511
|
+
*
|
|
512
|
+
* **Test-confidence caveat:** every method resolves on the microtask
|
|
513
|
+
* queue with no real concurrency, so `put` reads `existing` and writes
|
|
514
|
+
* back with no interleaving. MemoryStorage therefore CANNOT exercise the
|
|
515
|
+
* compare-and-swap race the writer-fence model depends on — two writers
|
|
516
|
+
* contending on one `current.json` always serialize here, so a green run
|
|
517
|
+
* is necessary but not sufficient. That race is covered only by the
|
|
518
|
+
* randomized integration tests against real S3 / Minio
|
|
519
|
+
* (`tests/integration/randomized.test.ts`, `node-minio` variant).
|
|
520
|
+
* Relatedly, `serverDate` returns the local clock, not an independent
|
|
521
|
+
* server clock (see the `put` return below).
|
|
522
|
+
*/
|
|
523
|
+
var MemoryStorage = class {
|
|
524
|
+
#objects = /* @__PURE__ */ new Map();
|
|
525
|
+
#etagCounter = 0;
|
|
526
|
+
constructor(opts) {
|
|
527
|
+
assertEphemeralAllowed(opts?.ephemeral === true);
|
|
528
|
+
}
|
|
529
|
+
#nextEtag() {
|
|
530
|
+
this.#etagCounter += 1;
|
|
531
|
+
return `"${this.#etagCounter.toString(16)}"`;
|
|
532
|
+
}
|
|
533
|
+
async get(key, opts) {
|
|
534
|
+
assertValidStorageKey(key);
|
|
535
|
+
opts?.signal?.throwIfAborted();
|
|
536
|
+
const stored = this.#objects.get(key);
|
|
537
|
+
if (stored === void 0) return null;
|
|
538
|
+
return {
|
|
539
|
+
body: stored.body,
|
|
540
|
+
etag: stored.etag
|
|
541
|
+
};
|
|
542
|
+
}
|
|
543
|
+
async put(key, body, opts) {
|
|
544
|
+
assertValidStorageKey(key);
|
|
545
|
+
opts?.signal?.throwIfAborted();
|
|
546
|
+
const existing = this.#objects.get(key);
|
|
547
|
+
if (opts?.ifNoneMatch === "*" && existing !== void 0) throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed (ifNoneMatch="*" but key exists)`);
|
|
548
|
+
if (opts?.ifMatch !== void 0) {
|
|
549
|
+
if (existing === void 0) throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed (ifMatch=${opts.ifMatch} but key does not exist)`);
|
|
550
|
+
if (existing.etag !== opts.ifMatch) throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed (ifMatch=${opts.ifMatch} but current ETag is ${existing.etag})`);
|
|
551
|
+
}
|
|
552
|
+
const etag = this.#nextEtag();
|
|
553
|
+
this.#objects.set(key, {
|
|
554
|
+
body,
|
|
555
|
+
etag,
|
|
556
|
+
...opts?.contentType !== void 0 && { contentType: opts.contentType }
|
|
557
|
+
});
|
|
558
|
+
return {
|
|
559
|
+
etag,
|
|
560
|
+
serverDate: /* @__PURE__ */ new Date()
|
|
561
|
+
};
|
|
562
|
+
}
|
|
563
|
+
async delete(key, opts) {
|
|
564
|
+
assertValidStorageKey(key);
|
|
565
|
+
opts?.signal?.throwIfAborted();
|
|
566
|
+
this.#objects.delete(key);
|
|
567
|
+
}
|
|
568
|
+
async *list(prefix, opts) {
|
|
569
|
+
opts?.signal?.throwIfAborted();
|
|
570
|
+
const startAfter = opts?.startAfter ?? "";
|
|
571
|
+
const maxKeys = opts?.maxKeys ?? 1e400;
|
|
572
|
+
const sorted = [...this.#objects.keys()].filter((k) => k.startsWith(prefix) && compareKeysUtf8$1(k, startAfter) > 0).toSorted(compareKeysUtf8$1);
|
|
573
|
+
let yielded = 0;
|
|
574
|
+
for (const key of sorted) {
|
|
575
|
+
if (yielded >= maxKeys) return;
|
|
576
|
+
opts?.signal?.throwIfAborted();
|
|
577
|
+
const stored = this.#objects.get(key);
|
|
578
|
+
if (stored === void 0) continue;
|
|
579
|
+
yield {
|
|
580
|
+
key,
|
|
581
|
+
etag: stored.etag
|
|
582
|
+
};
|
|
583
|
+
yielded += 1;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* Test-only: drop all objects.
|
|
588
|
+
* @internal
|
|
589
|
+
*/
|
|
590
|
+
_clear() {
|
|
591
|
+
this.#objects.clear();
|
|
592
|
+
this.#etagCounter = 0;
|
|
593
|
+
}
|
|
594
|
+
};
|
|
595
|
+
const sharedPerBucket = /* @__PURE__ */ new Map();
|
|
596
|
+
/**
|
|
597
|
+
* Get the process-singleton {@link MemoryStorage} for the named
|
|
598
|
+
* bucket, creating one on first access. Use this when constructing
|
|
599
|
+
* a `Storage` directly so multiple consumers in the same process
|
|
600
|
+
* see each other's writes for the same bucket name.
|
|
601
|
+
*/
|
|
602
|
+
const getOrCreateMemoryStorageForBucket = (bucket) => {
|
|
603
|
+
return sharedPerBucket.getOrInsertComputed(bucket, () => new MemoryStorage());
|
|
604
|
+
};
|
|
605
|
+
// ../protocol/src/types.ts
|
|
606
|
+
/**
|
|
607
|
+
* Mint a fresh {@link UUID}. The cast lives here so callers don't sprinkle
|
|
608
|
+
* `<UUID>crypto.randomUUID()` throughout the codebase.
|
|
609
|
+
*/
|
|
610
|
+
const uuid = () => crypto.randomUUID();
|
|
611
|
+
/**
|
|
612
|
+
* Mint the trailing seq segment of an LSN (`<base32-time>_<sess>_<seq>`).
|
|
613
|
+
* Uses `COUNT_BIT_WIDTH` (see `packages/protocol/src/constants.ts`) to
|
|
614
|
+
* produce an 11-char descending base-32 string with a domain of
|
|
615
|
+
* 0 .. `Number.MAX_SAFE_INTEGER` (2^53 − 1).
|
|
616
|
+
*
|
|
617
|
+
* Used together with {@link timestamp} (the descending base-32
|
|
618
|
+
* encoding of millis-since-epoch). See {@link uint2strDesc} for the
|
|
619
|
+
* lex-reversal property both rely on.
|
|
620
|
+
*/
|
|
621
|
+
const countKey = (number) => uint2strDesc(number, 53);
|
|
622
|
+
const uint2str = (num, bits) => {
|
|
623
|
+
const maxBase32Length = Math.ceil(bits / 5);
|
|
624
|
+
return num.toString(32).padStart(maxBase32Length, "0");
|
|
625
|
+
};
|
|
626
|
+
/**
|
|
627
|
+
* Lex-reversed base-32 encoder for an unsigned integer in `[0, 2^bits)`.
|
|
628
|
+
* Pads to `Math.ceil(bits/5)` chars so all values in the domain compare
|
|
629
|
+
* lex against each other.
|
|
630
|
+
*
|
|
631
|
+
* **Order-reversal property:** for any pair of integers `a < b` (with
|
|
632
|
+
* `a, b ∈ [0, 2^bits)`), `uint2strDesc(a, bits) > uint2strDesc(b, bits)`
|
|
633
|
+
* lexicographically. Composed into an LSN of shape
|
|
634
|
+
* `<base32-time>_<session>_<seq>` (where both `<base32-time>` and
|
|
635
|
+
* `<seq>` use this encoder), a forward `Storage.list(prefix)`
|
|
636
|
+
* returns log entries in reverse-causal order — newer entries first.
|
|
637
|
+
*
|
|
638
|
+
* This is the structural property the read path leans on:
|
|
639
|
+
* `ListObjectsV2` is forward-only on S3, but the descending encoding
|
|
640
|
+
* makes "fetch the K newest entries" a single bounded LIST with
|
|
641
|
+
* `maxKeys=K` instead of a full-population LIST + in-memory reverse.
|
|
642
|
+
*
|
|
643
|
+
* Behaviourally verified across randomized populations in
|
|
644
|
+
* `packages/protocol/src/lsn-reverse-list.test.ts`. Quantified
|
|
645
|
+
* bytes-listed reduction is in
|
|
646
|
+
* `docs/spec/attachments/lsn-reverse-walk-baseline.json`
|
|
647
|
+
* (run `pnpm bench:lsn-reverse-walk` to reproduce). Spec rationale:
|
|
648
|
+
* `docs/spec/sync-protocol.md` §"LSNs, wall clocks, and downstream
|
|
649
|
+
* consumers".
|
|
650
|
+
*/
|
|
651
|
+
const uint2strDesc = (num, bits) => {
|
|
652
|
+
const maxValue = Math.pow(2, bits) - 1;
|
|
653
|
+
return uint2str(maxValue - num, bits);
|
|
654
|
+
};
|
|
655
|
+
// ../protocol/src/time.ts
|
|
656
|
+
/**
|
|
657
|
+
* Encode an epoch-millis instant as a lexicographically-DEScending
|
|
658
|
+
* base-32 key (newer sorts first). The load-bearing LSN ordering
|
|
659
|
+
* primitive — see {@link uint2strDesc}.
|
|
660
|
+
*
|
|
661
|
+
* `epoch` is REQUIRED and validated. There is no default: an argless
|
|
662
|
+
* call used to encode `0`, which descending-sorts as the most-ancient
|
|
663
|
+
* instant and would silently invert ordering. A non-finite, negative,
|
|
664
|
+
* or out-of-range `epoch` (e.g. a broken `Date.now()` returning `NaN`)
|
|
665
|
+
* would corrupt the key silently rather than fail loud, so it throws
|
|
666
|
+
* `Internal` instead — this is an invariant violation, not caller config
|
|
667
|
+
* (the only legitimate argument is `Date.now()`).
|
|
668
|
+
*/
|
|
669
|
+
const timestamp = (epoch) => {
|
|
670
|
+
if (!Number.isInteger(epoch) || epoch < 0 || epoch >= 2 ** 42) throw new BaerlyError("Internal", `timestamp(): epoch must be a non-negative integer < 2^42 (got ${epoch})`);
|
|
671
|
+
return uint2strDesc(epoch, 42);
|
|
672
|
+
};
|
|
673
|
+
/**
|
|
674
|
+
* Delay for `ms` milliseconds, with optional cancellation. If
|
|
675
|
+
* `signal` aborts before the delay elapses, the returned promise
|
|
676
|
+
* rejects with `signal.reason`. `retryStorageCall` uses it for its
|
|
677
|
+
* backoff without a signal, so a backoff it has started always runs
|
|
678
|
+
* to completion.
|
|
679
|
+
*/
|
|
680
|
+
const delay = (ms, signal) => new Promise((resolve, reject) => {
|
|
681
|
+
if (signal?.aborted) {
|
|
682
|
+
reject(signal.reason);
|
|
683
|
+
return;
|
|
684
|
+
}
|
|
685
|
+
const timer = setTimeout(resolve, ms);
|
|
686
|
+
signal?.addEventListener("abort", () => {
|
|
687
|
+
clearTimeout(timer);
|
|
688
|
+
reject(signal.reason);
|
|
689
|
+
}, { once: true });
|
|
690
|
+
});
|
|
691
|
+
// ../protocol/src/storage/ledger.ts
|
|
692
|
+
let resolveLedger = () => void 0;
|
|
693
|
+
/**
|
|
694
|
+
* Register how {@link currentRequestLedger} finds the active ledger. The
|
|
695
|
+
* server registers its observability context here, so adapters, which
|
|
696
|
+
* import only this package, charge the ledger of the invocation they run
|
|
697
|
+
* in. One resolver per process; a later call replaces it.
|
|
698
|
+
*/
|
|
699
|
+
const setRequestLedgerResolver = (resolver) => {
|
|
700
|
+
resolveLedger = resolver;
|
|
701
|
+
};
|
|
702
|
+
/** The ledger of the active invocation, or `undefined` outside one (nothing is charged). */
|
|
703
|
+
const currentRequestLedger = () => resolveLedger();
|
|
704
|
+
let resolveHold = () => void 0;
|
|
705
|
+
/**
|
|
706
|
+
* Register how {@link currentRequestHold} finds the active hold, as
|
|
707
|
+
* {@link setRequestLedgerResolver} does for the ledger. The server scopes a
|
|
708
|
+
* hold to the async context of the work it covers, so work started outside
|
|
709
|
+
* that context (a `ctx.waitUntil` tick) never draws on it.
|
|
710
|
+
*/
|
|
711
|
+
const setRequestHoldResolver = (resolver) => {
|
|
712
|
+
resolveHold = resolver;
|
|
713
|
+
};
|
|
714
|
+
/** The hold active in this async context, if any; `retryStorageCall` charges against it. */
|
|
715
|
+
const currentRequestHold = () => resolveHold();
|
|
716
|
+
// ../protocol/src/storage/retry.ts
|
|
717
|
+
/**
|
|
718
|
+
* Permanent {@link BaerlyError} codes that end `retryStorageCall` at
|
|
719
|
+
* once. These represent caller- or environment-level faults where
|
|
720
|
+
* retrying cannot succeed: `AccessDenied` (403 — credentials/policy),
|
|
721
|
+
* `InvalidConfig` (bad bucket / unsupported credential type),
|
|
722
|
+
* `InvalidResponse` (server returned unparseable data), and
|
|
723
|
+
* `WriteConflict` (a precondition was rejected). `NetworkError` is
|
|
724
|
+
* intentionally absent — it covers transient transport faults and stays
|
|
725
|
+
* retryable.
|
|
726
|
+
*
|
|
727
|
+
* Storage-layer set, distinct from `RETRIABLE_CODES` in `errors.ts`. A
|
|
728
|
+
* blind re-PUT of the same request re-loses a rejected precondition, so
|
|
729
|
+
* this layer never resends it. `WriteConflict` is still retriable for the
|
|
730
|
+
* logical caller: it re-reads and issues a new request, which is what
|
|
731
|
+
* `retriable` means.
|
|
732
|
+
*/
|
|
733
|
+
const PERMANENT_STORAGE_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
734
|
+
"AccessDenied",
|
|
735
|
+
"InvalidConfig",
|
|
736
|
+
"InvalidResponse",
|
|
737
|
+
"WriteConflict"
|
|
738
|
+
]);
|
|
739
|
+
/** `true` when an attempt uses the throttle backoff. */
|
|
740
|
+
const isThrottled = (e) => e instanceof BaerlyError && e.cause?.throttled === true;
|
|
741
|
+
/**
|
|
742
|
+
* `true` when an attempt was refused before it was issued and a resend
|
|
743
|
+
* in the same invocation is refused too (the Workers subrequest limit).
|
|
744
|
+
* This proves the attempt was not applied and ends the loop at once.
|
|
745
|
+
*/
|
|
746
|
+
const isRefused = (e) => e instanceof BaerlyError && e.cause?.refused === true;
|
|
747
|
+
/**
|
|
748
|
+
* `true` for an `AbortError`: a cancellation raised by a call that did
|
|
749
|
+
* not pass its signal in {@link StorageRetryOptions.signal}. A resend
|
|
750
|
+
* would fail the same way, so the loop ends and the error surfaces as is,
|
|
751
|
+
* as the other adapters raise it. Matched on `name` like the writer's
|
|
752
|
+
* predicate: workerd and undici reject with different `DOMException`
|
|
753
|
+
* classes. A passed signal covers every abort reason, not only this name.
|
|
754
|
+
*/
|
|
755
|
+
const isAbortError = (e) => typeof e === "object" && e !== null && "name" in e && e.name === "AbortError";
|
|
756
|
+
const retryAfterMillis = (e) => {
|
|
757
|
+
if (!(e instanceof BaerlyError)) return;
|
|
758
|
+
const seconds = e.cause?.retryAfterSeconds;
|
|
759
|
+
return seconds === void 0 ? void 0 : seconds * 1e3;
|
|
760
|
+
};
|
|
761
|
+
/**
|
|
762
|
+
* The wait before retry `retryIndex` (0-based) after `error`.
|
|
763
|
+
*
|
|
764
|
+
* A server `Retry-After` hint (`cause.retryAfterSeconds`) wins, clamped
|
|
765
|
+
* to `STORAGE_RETRY_MAX_DELAY_MILLIS`. Otherwise full jitter:
|
|
766
|
+
* `random() * min(cap, base * 2^retryIndex)`, with the throttle base for
|
|
767
|
+
* an error whose cause carries `throttled: true` and the transient base
|
|
768
|
+
* for everything else, unless `backoffMs` overrides both.
|
|
769
|
+
*/
|
|
770
|
+
const storageRetryDelayMillis = (error, retryIndex, opts = {}) => {
|
|
771
|
+
const hintMs = retryAfterMillis(error);
|
|
772
|
+
if (hintMs !== void 0) return Math.min(hintMs, STORAGE_RETRY_MAX_DELAY_MILLIS);
|
|
773
|
+
const base = opts.backoffMs ?? (isThrottled(error) ? 1e3 : 50);
|
|
774
|
+
const ceiling = Math.min(STORAGE_RETRY_MAX_DELAY_MILLIS, base * 2 ** retryIndex);
|
|
775
|
+
return (opts.random ?? Math.random)() * ceiling;
|
|
776
|
+
};
|
|
777
|
+
/**
|
|
778
|
+
* Run one storage request under the single adapter retry policy: at most
|
|
779
|
+
* `retries + 1` attempts (default {@link STORAGE_RETRY_MAX_ATTEMPTS}; one
|
|
780
|
+
* for a `compareAndSwap` call),
|
|
781
|
+
* one counter for throttles and transient failures alike, full-jitter
|
|
782
|
+
* backoff between attempts (see {@link storageRetryDelayMillis}). A
|
|
783
|
+
* {@link PERMANENT_STORAGE_ERROR_CODES} failure, an error whose cause
|
|
784
|
+
* carries `refused: true`, an `AbortError`, or a failure after
|
|
785
|
+
* `opts.signal` aborted ends the loop at once.
|
|
786
|
+
*
|
|
787
|
+
* Every adapter wraps each request — one object call, or one `list`
|
|
788
|
+
* page — in exactly one `retryStorageCall`, so a failing call costs at
|
|
789
|
+
* most `retries + 1` requests. Callers above the `Storage` boundary must
|
|
790
|
+
* not add their own blind resend loop on top.
|
|
791
|
+
*
|
|
792
|
+
* Each attempt is charged to the active invocation's request ledger
|
|
793
|
+
* ({@link currentRequestLedger}) on `opts.pool` before it is sent. An
|
|
794
|
+
* attempt the ledger refuses is never sent: the refusal (`CapacityExceeded`,
|
|
795
|
+
* `cause.refused`) ends the loop like any refusal.
|
|
796
|
+
*/
|
|
797
|
+
const retryStorageCall = async (fn, opts = {}) => {
|
|
798
|
+
const retries = opts.compareAndSwap === true ? 0 : opts.retries ?? 2;
|
|
799
|
+
if (!Number.isSafeInteger(retries) || retries < 0) throw new BaerlyError("InvalidConfig", `storage retries must be a non-negative integer, got ${String(retries)}`);
|
|
800
|
+
const sleep = opts.sleep ?? delay;
|
|
801
|
+
const ledger = currentRequestLedger();
|
|
802
|
+
const pool = opts.pool ?? "external";
|
|
803
|
+
let uncertain = false;
|
|
804
|
+
for (let attempt = 0; attempt <= retries; attempt++) try {
|
|
805
|
+
opts.signal?.throwIfAborted();
|
|
806
|
+
ledger?.acquire(pool, currentRequestHold());
|
|
807
|
+
return await fn();
|
|
808
|
+
} catch (error) {
|
|
809
|
+
const aborted = opts.signal?.aborted === true;
|
|
810
|
+
if (isAbortError(error) || aborted && !(error instanceof BaerlyError)) throw error;
|
|
811
|
+
if (error instanceof BaerlyError && PERMANENT_STORAGE_ERROR_CODES.has(error.code)) {
|
|
812
|
+
if (opts.conditionalCreate === true && uncertain && !isWriteConflict(error)) throw new BaerlyError("NetworkError", `${error.message} (an earlier attempt of this conditional create may have landed)`, { cause: error });
|
|
813
|
+
throw error;
|
|
814
|
+
}
|
|
815
|
+
const refused = isRefused(error);
|
|
816
|
+
const mayHaveApplied = error instanceof BaerlyError && error.cause?.mayHaveApplied === true;
|
|
817
|
+
if (!isThrottled(error) && !refused || mayHaveApplied) uncertain = true;
|
|
818
|
+
if (attempt >= retries || refused || aborted) {
|
|
819
|
+
if (opts.conditionalCreate === true && error instanceof BaerlyError) {
|
|
820
|
+
if (!uncertain) throw new BaerlyError(error.code, error.message, {
|
|
821
|
+
cause: {
|
|
822
|
+
...error.cause,
|
|
823
|
+
notApplied: true
|
|
824
|
+
},
|
|
825
|
+
...error.resolution !== void 0 && { resolution: error.resolution }
|
|
826
|
+
});
|
|
827
|
+
if (refused && error.code !== "NetworkError") throw new BaerlyError("NetworkError", `${error.message} (an earlier attempt of this conditional create may have landed)`, { cause: error.cause });
|
|
828
|
+
}
|
|
829
|
+
throw error;
|
|
830
|
+
}
|
|
831
|
+
await sleep(storageRetryDelayMillis(error, attempt, opts));
|
|
832
|
+
}
|
|
833
|
+
throw new BaerlyError("Internal", "storage retry loop exited without returning or throwing");
|
|
834
|
+
};
|
|
835
|
+
// ../dev/src/key-lock.ts
|
|
836
|
+
/**
|
|
837
|
+
* Live locks, keyed by caller-supplied string.
|
|
838
|
+
*
|
|
839
|
+
* Module-level rather than per-owner on purpose. `LocalFsStorage` is
|
|
840
|
+
* constructed freely — `localFsStorage()` mints a fresh instance on every
|
|
841
|
+
* call over the same default root, and the randomized cascade builds N
|
|
842
|
+
* instances over one directory specifically to make them contend — so a
|
|
843
|
+
* lock map hanging off the instance would serialize nothing in exactly the
|
|
844
|
+
* cases that need it. Callers key on the resolved root plus the storage
|
|
845
|
+
* key, which is the real identity of the thing being mutated.
|
|
846
|
+
*/
|
|
847
|
+
const locks = /* @__PURE__ */ new Map();
|
|
848
|
+
/**
|
|
849
|
+
* Run `fn` with exclusive access to `lockKey`, in FIFO order among
|
|
850
|
+
* contenders. Concurrent calls on *different* keys are unaffected.
|
|
851
|
+
*
|
|
852
|
+
* A rejecting `fn` releases the lock like any other completion and the
|
|
853
|
+
* rejection propagates to its own caller only — one failed critical
|
|
854
|
+
* section must not wedge the key or poison the next holder.
|
|
855
|
+
*
|
|
856
|
+
* Scope: one loaded copy of this module, since `locks` is module-level.
|
|
857
|
+
* That is one process in every normal setup. It says nothing about other
|
|
858
|
+
* processes, which need a lock file, a lease, or server-side CAS.
|
|
859
|
+
*
|
|
860
|
+
* `fn` must not itself call `withKeyLock` on the same key — the inner
|
|
861
|
+
* call would wait on the outer call's release and self-deadlock. Nothing
|
|
862
|
+
* in this package does; only one lock is ever held at a time, so there is
|
|
863
|
+
* no lock-ordering cycle either.
|
|
864
|
+
*/
|
|
865
|
+
const withKeyLock = async (lockKey, fn) => {
|
|
866
|
+
const lock = locks.getOrInsertComputed(lockKey, () => ({
|
|
867
|
+
tail: Promise.resolve(),
|
|
868
|
+
waiters: 0
|
|
869
|
+
}));
|
|
870
|
+
lock.waiters += 1;
|
|
871
|
+
const predecessor = lock.tail;
|
|
872
|
+
let release;
|
|
873
|
+
lock.tail = new Promise((resolve) => {
|
|
874
|
+
release = resolve;
|
|
875
|
+
});
|
|
876
|
+
await predecessor;
|
|
877
|
+
try {
|
|
878
|
+
return await fn();
|
|
879
|
+
} finally {
|
|
880
|
+
release();
|
|
881
|
+
lock.waiters -= 1;
|
|
882
|
+
if (lock.waiters === 0 && locks.get(lockKey) === lock) locks.delete(lockKey);
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
/** Longest component that still fits {@link CONTINUATION} under `NAME_MAX`. */
|
|
886
|
+
const MAX_PART_BYTES = 254;
|
|
887
|
+
const utf8Encoder$1 = new TextEncoder();
|
|
888
|
+
const utf8Decoder = new TextDecoder("utf-8", {
|
|
889
|
+
fatal: true,
|
|
890
|
+
ignoreBOM: true
|
|
891
|
+
});
|
|
892
|
+
/** Bytes that stay literal: `-` `.` `0-9` `_` `a-z`. `.` is special-cased. */
|
|
893
|
+
const isLiteralByte = (b) => b >= 97 && b <= 122 || b >= 48 && b <= 57 || b === 45 || b === 95 || b === 46;
|
|
894
|
+
const DOT = 46;
|
|
895
|
+
const escapeByte = (b) => `%${b.toString(16).padStart(2, "0")}`;
|
|
896
|
+
/** Tokens of one encoded segment: a literal char or a `%xx` triplet. */
|
|
897
|
+
const tokenize = (segment) => {
|
|
898
|
+
const tokens = [];
|
|
899
|
+
for (const b of utf8Encoder$1.encode(segment)) tokens.push(isLiteralByte(b) ? String.fromCharCode(b) : escapeByte(b));
|
|
900
|
+
return tokens;
|
|
901
|
+
};
|
|
902
|
+
/**
|
|
903
|
+
* Pack tokens into on-disk components. A component never starts with a
|
|
904
|
+
* literal `.`: such a token is written as `%2e` instead.
|
|
905
|
+
*/
|
|
906
|
+
const pack = (tokens, limit) => {
|
|
907
|
+
const parts = [];
|
|
908
|
+
let current = "";
|
|
909
|
+
for (let token of tokens) {
|
|
910
|
+
if (current.length + token.length > limit) {
|
|
911
|
+
parts.push(current);
|
|
912
|
+
current = "";
|
|
913
|
+
}
|
|
914
|
+
if (current === "" && token === ".") token = escapeByte(DOT);
|
|
915
|
+
current += token;
|
|
916
|
+
}
|
|
917
|
+
parts.push(current);
|
|
918
|
+
return parts;
|
|
919
|
+
};
|
|
920
|
+
/** Encode one non-empty `/`-segment into its on-disk component(s). */
|
|
921
|
+
const encodeSegment = (segment) => {
|
|
922
|
+
const tokens = tokenize(segment);
|
|
923
|
+
const single = pack(tokens, Number.POSITIVE_INFINITY);
|
|
924
|
+
if (single[0].length <= 255) return single;
|
|
925
|
+
const parts = pack(tokens, MAX_PART_BYTES);
|
|
926
|
+
return parts.map((p, i) => i < parts.length - 1 ? p + "~" : p);
|
|
927
|
+
};
|
|
928
|
+
/**
|
|
929
|
+
* Encode a key into on-disk path components. The caller has already
|
|
930
|
+
* validated the key's `/`-structure (no empty, `.`, or `..` segment).
|
|
931
|
+
* Rejects a key that is not well-formed UTF-16: `TextEncoder` maps every
|
|
932
|
+
* lone surrogate to U+FFFD, which would make the encoding non-injective.
|
|
933
|
+
*/
|
|
934
|
+
const encodeKey = (key) => {
|
|
935
|
+
if (!key.isWellFormed()) throw new BaerlyError("InvalidConfig", `LocalFsStorage: key is not well-formed UTF-16 (lone surrogate): ${JSON.stringify(key)}`);
|
|
936
|
+
return key.split("/").flatMap(encodeSegment);
|
|
937
|
+
};
|
|
938
|
+
const HEX = /^[0-9a-f]{2}$/;
|
|
939
|
+
/** Decode one component's bytes (continuation marker already stripped). */
|
|
940
|
+
const decodeBytes = (component, out) => {
|
|
941
|
+
for (let i = 0; i < component.length; i++) {
|
|
942
|
+
const c = component.charCodeAt(i);
|
|
943
|
+
if (c === 37) {
|
|
944
|
+
const hex = component.slice(i + 1, i + 3);
|
|
945
|
+
if (!HEX.test(hex)) return false;
|
|
946
|
+
out.push(Number.parseInt(hex, 16));
|
|
947
|
+
i += 2;
|
|
948
|
+
} else if (isLiteralByte(c)) out.push(c);
|
|
949
|
+
else return false;
|
|
950
|
+
}
|
|
951
|
+
return true;
|
|
952
|
+
};
|
|
953
|
+
/**
|
|
954
|
+
* Decode on-disk path components (relative to the root) back to a key.
|
|
955
|
+
* Returns `null` for anything that is not the canonical encoding of some
|
|
956
|
+
* key — a foreign file, a non-canonical escape, a dangling continuation —
|
|
957
|
+
* so the caller can decide how loudly to fail.
|
|
958
|
+
*/
|
|
959
|
+
const decodeComponents = (components) => {
|
|
960
|
+
const segments = [];
|
|
961
|
+
let bytes = [];
|
|
962
|
+
for (const component of components) {
|
|
963
|
+
const continued = component.endsWith("~");
|
|
964
|
+
const body = continued ? component.slice(0, -1) : component;
|
|
965
|
+
if (body === "" || !decodeBytes(body, bytes)) return null;
|
|
966
|
+
if (!continued) {
|
|
967
|
+
let segment;
|
|
968
|
+
try {
|
|
969
|
+
segment = utf8Decoder.decode(new Uint8Array(bytes));
|
|
970
|
+
} catch {
|
|
971
|
+
return null;
|
|
972
|
+
}
|
|
973
|
+
if (segment === "." || segment === "..") return null;
|
|
974
|
+
segments.push(segment);
|
|
975
|
+
bytes = [];
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
if (bytes.length > 0 || segments.length === 0) return null;
|
|
979
|
+
const key = segments.join("/");
|
|
980
|
+
let canonical;
|
|
981
|
+
try {
|
|
982
|
+
canonical = encodeKey(key);
|
|
983
|
+
} catch {
|
|
984
|
+
return null;
|
|
985
|
+
}
|
|
986
|
+
if (canonical.length !== components.length || canonical.some((c, i) => c !== components[i])) return null;
|
|
987
|
+
return key;
|
|
988
|
+
};
|
|
989
|
+
// ../dev/src/local-fs.ts
|
|
990
|
+
/**
|
|
991
|
+
* Prefix for the in-directory staging temps (see `put`). Dot-prefixed, so
|
|
992
|
+
* it can never collide with an encoded key (see `local-fs-key-codec.ts`)
|
|
993
|
+
* and `walk` skips it with every other adapter-private name.
|
|
994
|
+
*/
|
|
995
|
+
const TEMP_PREFIX = ".baerly-tmp-";
|
|
996
|
+
/**
|
|
997
|
+
* Root marker recording the on-disk key encoding. Dot-prefixed, so it is
|
|
998
|
+
* never a key. A directory without it is scanned once for names written
|
|
999
|
+
* by the pre-encoding layout (see `#ensureFormat`).
|
|
1000
|
+
*/
|
|
1001
|
+
const FORMAT_MARKER = ".baerly-localfs.json";
|
|
1002
|
+
/** Version of the key encoding in `local-fs-key-codec.ts`. */
|
|
1003
|
+
const KEY_ENCODING = 1;
|
|
1004
|
+
/** Legacy paths named in the refusal message before it truncates. */
|
|
1005
|
+
const LEGACY_PATHS_SHOWN = 10;
|
|
1006
|
+
/**
|
|
1007
|
+
* Staging path for a two-phase (write → publish) `put`. Always a sibling
|
|
1008
|
+
* of the destination: `rename(2)`/`link(2)` refuse to cross filesystems
|
|
1009
|
+
* (`EXDEV`), and the bucket root is routinely on a different volume from
|
|
1010
|
+
* `os.tmpdir()`. The leading `.` of `TEMP_PREFIX` is what makes staging
|
|
1011
|
+
* in-bucket safe — `walk` skips dot-prefixed names at every depth, and no
|
|
1012
|
+
* encoded key starts with one, so a crash before the publish leaves a file
|
|
1013
|
+
* invisible to `list` rather than a bogus key.
|
|
1014
|
+
*
|
|
1015
|
+
* The trade-off that invisibility buys: a temp stranded by a hard crash
|
|
1016
|
+
* now sits inside the bucket, and because every enumerator goes through
|
|
1017
|
+
* `list`, nothing — not `runGc`, not `admin fsck` — will ever reclaim it.
|
|
1018
|
+
* Staging in `os.tmpdir()` at least let the OS reap it. Repeated crashes
|
|
1019
|
+
* therefore grow the data directory, and clearing that is a manual
|
|
1020
|
+
* `find . -name '.baerly-tmp-*' -delete`. Non-crash failures are cleaned
|
|
1021
|
+
* up by `put`'s `finally`.
|
|
1022
|
+
*/
|
|
1023
|
+
const tempPathFor = (finalPath) => join(dirname(finalPath), `${TEMP_PREFIX}${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
1024
|
+
const utf8Encoder = new TextEncoder();
|
|
1025
|
+
/**
|
|
1026
|
+
* Compare two keys by their UTF-8 byte sequences — the order S3 and R2
|
|
1027
|
+
* use for `list`. JS's default string sort compares UTF-16 code units,
|
|
1028
|
+
* which diverges from UTF-8 byte order for supplementary-plane
|
|
1029
|
+
* characters; using this keeps `list` ordering faithful to the real
|
|
1030
|
+
* adapters. (All kernel keys are ASCII base-32, where both orders
|
|
1031
|
+
* coincide.)
|
|
1032
|
+
*/
|
|
1033
|
+
const compareKeysUtf8 = (a, b) => {
|
|
1034
|
+
const ba = utf8Encoder.encode(a);
|
|
1035
|
+
const bb = utf8Encoder.encode(b);
|
|
1036
|
+
const n = Math.min(ba.length, bb.length);
|
|
1037
|
+
for (let i = 0; i < n; i++) if (ba[i] !== bb[i]) return ba[i] - bb[i];
|
|
1038
|
+
return ba.length - bb.length;
|
|
1039
|
+
};
|
|
1040
|
+
/**
|
|
1041
|
+
* `Storage` backed by a directory tree. Keys may contain `/` and map
|
|
1042
|
+
* to nested directories on disk so `ls`/`cat` work as expected.
|
|
1043
|
+
*
|
|
1044
|
+
* Keys are byte-exact, as on S3 and R2: keys that differ only by case or
|
|
1045
|
+
* Unicode normalization are distinct objects even on a filesystem that
|
|
1046
|
+
* folds both (default macOS APFS). Each segment is escaped into a
|
|
1047
|
+
* fold-invariant alphabet on disk — `A` is stored as `%41`, NFC `é` as
|
|
1048
|
+
* `%c3%a9` — while kernel-written segments (lowercase ids, base-32 index
|
|
1049
|
+
* values, `current.json`) are stored verbatim. See
|
|
1050
|
+
* `local-fs-key-codec.ts`.
|
|
1051
|
+
*
|
|
1052
|
+
* The root carries a `.baerly-localfs.json` marker naming that encoding.
|
|
1053
|
+
* A directory written by an older release, before keys were escaped, is
|
|
1054
|
+
* adopted as-is when every name in it already reads the same under the
|
|
1055
|
+
* encoding; otherwise every operation refuses with `InvalidConfig`.
|
|
1056
|
+
* Such a directory is dev data: move it aside or delete it. Collections
|
|
1057
|
+
* worth keeping can be dumped with the older release (`baerly admin
|
|
1058
|
+
* dump`, rows only) and restored into a fresh directory (`baerly admin
|
|
1059
|
+
* restore`). Data a folding filesystem has already merged is not
|
|
1060
|
+
* recoverable.
|
|
1061
|
+
*
|
|
1062
|
+
* ETags are content-addressed: `"<sha-256-hex>"` (the surrounding
|
|
1063
|
+
* double-quotes match S3's wire format for non-multipart objects).
|
|
1064
|
+
* Identical bodies share an ETag across runs — the property that
|
|
1065
|
+
* makes this adapter useful for fixture-based tests.
|
|
1066
|
+
*
|
|
1067
|
+
* Writes are atomic via `writeFile(temp) + rename(final)`; readers
|
|
1068
|
+
* never see a partially-written file. The temp is always a sibling of
|
|
1069
|
+
* its destination — `rename(2)`/`link(2)` fail with `EXDEV` across
|
|
1070
|
+
* filesystems, and the bucket root is routinely on a different volume
|
|
1071
|
+
* from `os.tmpdir()` (see `tempPathFor`).
|
|
1072
|
+
*
|
|
1073
|
+
* `ifNoneMatch:"*"` uses `link(2)` (atomic exclusive create; `EEXIST` ⇒
|
|
1074
|
+
* key exists) so concurrent creates have exactly one winner. temp+`link`
|
|
1075
|
+
* over `open(…,"wx")` keeps a partially-written new key invisible to a
|
|
1076
|
+
* concurrent reader.
|
|
1077
|
+
*
|
|
1078
|
+
* `ifMatch` is a read-compare-write, so it is made atomic by serializing
|
|
1079
|
+
* every mutation of a key against every other — see `#lockKey`.
|
|
1080
|
+
*
|
|
1081
|
+
* Scope of that guarantee, precisely: **one loaded copy of this module**.
|
|
1082
|
+
* Within it, concurrent CAS on one key admits exactly one winner and the
|
|
1083
|
+
* losers get `WriteConflict`, across any number of `LocalFsStorage` instances
|
|
1084
|
+
* whose roots resolve — through symlinks — to the same directory. That is
|
|
1085
|
+
* one process in every normal setup, but the lock lives in a module-level
|
|
1086
|
+
* map, so a process that somehow loads both this source and the bundled
|
|
1087
|
+
* copy has two of them.
|
|
1088
|
+
*
|
|
1089
|
+
* Across processes it guarantees nothing: two `baerly` CLI runs, or two
|
|
1090
|
+
* servers, sharing a directory can both win, because nothing in POSIX
|
|
1091
|
+
* makes read-compare-write atomic without a lock file or a lease. That is
|
|
1092
|
+
* why this adapter is for single-process dev and self-hosting;
|
|
1093
|
+
* horizontally-scaled deploys need S3 / R2, whose server-side conditional
|
|
1094
|
+
* write is what the no-lease maintenance fold actually relies on.
|
|
1095
|
+
*
|
|
1096
|
+
* Node-only — imports `node:fs`, `node:path`, `node:crypto`. Lives in
|
|
1097
|
+
* `@baerly/dev` because the protocol kernel is pure-modules / no I/O
|
|
1098
|
+
* and must remain Worker-bundleable.
|
|
1099
|
+
*
|
|
1100
|
+
* ⚠️ **Hard termination and temp files.**
|
|
1101
|
+
*
|
|
1102
|
+
* Writes stage a sibling `.baerly-tmp-*` file before atomic rename/link.
|
|
1103
|
+
* Normal failures clean up the temp in `finally`, but hard termination
|
|
1104
|
+
* (`SIGKILL`, OOM, power loss) can strand a temp inside the root directory.
|
|
1105
|
+
*
|
|
1106
|
+
* These files are hidden from `list`, `gc`, `fsck`, and usage reporting
|
|
1107
|
+
* by design—the temp filter applies at every depth.
|
|
1108
|
+
*
|
|
1109
|
+
* **Manual cleanup procedure:**
|
|
1110
|
+
* 1. Stop all writers using the root
|
|
1111
|
+
* 2. Run: `find <root> -name '.baerly-tmp-*' -delete`
|
|
1112
|
+
*
|
|
1113
|
+
* Do not clean while writes are in progress—races with in-flight PUTs
|
|
1114
|
+
* can delete a file that a concurrent operation is about to rename.
|
|
1115
|
+
*/
|
|
1116
|
+
var LocalFsStorage = class {
|
|
1117
|
+
#root;
|
|
1118
|
+
/** Memoized `realpath` of {@link #root} — see {@link #canonicalRoot}. */
|
|
1119
|
+
#realRoot;
|
|
1120
|
+
/** Set once the root's key encoding is known to be current. */
|
|
1121
|
+
#formatChecked = false;
|
|
1122
|
+
constructor(opts) {
|
|
1123
|
+
this.#root = resolve(opts.root);
|
|
1124
|
+
}
|
|
1125
|
+
async get(key, opts) {
|
|
1126
|
+
opts?.signal?.throwIfAborted();
|
|
1127
|
+
const path = this.#pathFor(key);
|
|
1128
|
+
await this.#ensureFormat(false);
|
|
1129
|
+
let body;
|
|
1130
|
+
try {
|
|
1131
|
+
body = await readFile(path);
|
|
1132
|
+
} catch (error) {
|
|
1133
|
+
if (isErrnoException(error) && error.code === "ENOENT") return null;
|
|
1134
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage.get(${key}): ${error.message}`, { cause: error });
|
|
1135
|
+
}
|
|
1136
|
+
const etag = etagOf(body);
|
|
1137
|
+
return {
|
|
1138
|
+
body: toUint8Array(body),
|
|
1139
|
+
etag
|
|
1140
|
+
};
|
|
1141
|
+
}
|
|
1142
|
+
async put(key, body, opts) {
|
|
1143
|
+
opts?.signal?.throwIfAborted();
|
|
1144
|
+
const path = this.#pathFor(key);
|
|
1145
|
+
await this.#ensureFormat(true);
|
|
1146
|
+
await mkdir(dirname(path), { recursive: true });
|
|
1147
|
+
return withKeyLock(await this.#lockKey(path), () => this.#putSerialized(key, path, body, opts));
|
|
1148
|
+
}
|
|
1149
|
+
async #putSerialized(key, path, body, opts) {
|
|
1150
|
+
opts?.signal?.throwIfAborted();
|
|
1151
|
+
const newEtag = etagOf(body);
|
|
1152
|
+
if (opts?.ifNoneMatch === "*") {
|
|
1153
|
+
const tmp = tempPathFor(path);
|
|
1154
|
+
try {
|
|
1155
|
+
await writeFile(tmp, body);
|
|
1156
|
+
await link(tmp, path);
|
|
1157
|
+
} catch (error) {
|
|
1158
|
+
if (isErrnoException(error) && error.code === "EEXIST") throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed (ifNoneMatch="*" but key exists)`);
|
|
1159
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage.put(${key}): ${error.message}`, { cause: error });
|
|
1160
|
+
} finally {
|
|
1161
|
+
await rm(tmp, { force: true }).catch(() => {});
|
|
1162
|
+
}
|
|
1163
|
+
return {
|
|
1164
|
+
etag: newEtag,
|
|
1165
|
+
serverDate: /* @__PURE__ */ new Date()
|
|
1166
|
+
};
|
|
1167
|
+
}
|
|
1168
|
+
if (opts?.ifMatch !== void 0) {
|
|
1169
|
+
const existing = await readExisting(path);
|
|
1170
|
+
if (existing === null) throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed (ifMatch=${opts.ifMatch} but key does not exist)`);
|
|
1171
|
+
const currentEtag = etagOf(existing);
|
|
1172
|
+
if (currentEtag !== opts.ifMatch) throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed (ifMatch=${opts.ifMatch} but current ETag is ${currentEtag})`);
|
|
1173
|
+
}
|
|
1174
|
+
const tmp = tempPathFor(path);
|
|
1175
|
+
try {
|
|
1176
|
+
await writeFile(tmp, body);
|
|
1177
|
+
await rename(tmp, path);
|
|
1178
|
+
} catch (error) {
|
|
1179
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage.put(${key}): ${error.message}`, { cause: error });
|
|
1180
|
+
} finally {
|
|
1181
|
+
await rm(tmp, { force: true }).catch(() => {});
|
|
1182
|
+
}
|
|
1183
|
+
return {
|
|
1184
|
+
etag: newEtag,
|
|
1185
|
+
serverDate: /* @__PURE__ */ new Date()
|
|
1186
|
+
};
|
|
1187
|
+
}
|
|
1188
|
+
async delete(key, opts) {
|
|
1189
|
+
opts?.signal?.throwIfAborted();
|
|
1190
|
+
const path = this.#pathFor(key);
|
|
1191
|
+
await this.#ensureFormat(false);
|
|
1192
|
+
return withKeyLock(await this.#lockKey(path), async () => {
|
|
1193
|
+
opts?.signal?.throwIfAborted();
|
|
1194
|
+
try {
|
|
1195
|
+
await rm(path);
|
|
1196
|
+
} catch (error) {
|
|
1197
|
+
if (isErrnoException(error) && error.code === "ENOENT") return;
|
|
1198
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage.delete(${key}): ${error.message}`, { cause: error });
|
|
1199
|
+
}
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
1202
|
+
/**
|
|
1203
|
+
* Identity of the thing being mutated, for {@link withKeyLock}.
|
|
1204
|
+
*
|
|
1205
|
+
* Keyed on the *canonical filesystem path*, not on the key and not on
|
|
1206
|
+
* `this`. Not on `this` because separate instances over one directory
|
|
1207
|
+
* are the common case (`localFsStorage()` mints a fresh one per call),
|
|
1208
|
+
* so an instance-scoped lock would serialize nothing where it matters.
|
|
1209
|
+
* Not on the root spelling the caller passed, because several spellings
|
|
1210
|
+
* of a root can name one directory, and handing an alias its own lock
|
|
1211
|
+
* lets the race straight back in:
|
|
1212
|
+
*
|
|
1213
|
+
* - Symlinks. `resolve()` collapses `.`/`..` but not links, so
|
|
1214
|
+
* `/tmp/data` and `/private/tmp/data` — the same directory on every
|
|
1215
|
+
* Mac — look like two (measured at 2 winners before it was closed).
|
|
1216
|
+
* Hence `realpath`, memoized per instance and only cached once it
|
|
1217
|
+
* succeeds, so a root that does not exist yet cannot poison the entry.
|
|
1218
|
+
* - Case and Unicode in the root path. A caller can spell a root on a
|
|
1219
|
+
* folding filesystem as `/Users/me/Data` or `/users/me/data`, or in
|
|
1220
|
+
* NFC or NFD, and reach one directory.
|
|
1221
|
+
*
|
|
1222
|
+
* The key-derived tail of `path` cannot alias: it is the fold-invariant
|
|
1223
|
+
* encoding from `local-fs-key-codec.ts`, so distinct keys are distinct
|
|
1224
|
+
* files on every filesystem and the case/NFC fold below changes nothing
|
|
1225
|
+
* there. The fold is kept for the root-spelling case only. It
|
|
1226
|
+
* over-approximates on a case-sensitive filesystem, where two roots
|
|
1227
|
+
* differing only in case are really different directories and share a
|
|
1228
|
+
* lock; that costs a little concurrency, never correctness. And it is
|
|
1229
|
+
* not exact Unicode case folding (JS has no `toCaseFold()`), so a root
|
|
1230
|
+
* spelled with `Σ` and one with `σ` get separate locks — acceptable for
|
|
1231
|
+
* a root path, which the operator chooses once.
|
|
1232
|
+
*/
|
|
1233
|
+
async #lockKey(path) {
|
|
1234
|
+
return (await this.#canonicalRoot() + path.slice(this.#root.length)).normalize("NFC").toLowerCase();
|
|
1235
|
+
}
|
|
1236
|
+
/**
|
|
1237
|
+
* {@link #root} with symlinks resolved. Cached only on success: `delete`
|
|
1238
|
+
* against a bucket that was never created would otherwise memoize the
|
|
1239
|
+
* un-canonicalized spelling and defeat the aliasing guard for every
|
|
1240
|
+
* later `put`.
|
|
1241
|
+
*
|
|
1242
|
+
* Only `ENOENT` falls back. Every other `realpath` failure — `EACCES`,
|
|
1243
|
+
* `ELOOP`, `ENAMETOOLONG` — throws, because the fallback is a *different
|
|
1244
|
+
* lock key*, and absorbing those would let an instance that cannot
|
|
1245
|
+
* canonicalize run unserialized against one that can. That is the
|
|
1246
|
+
* exactly-one-winner guarantee this lock exists to provide, failing
|
|
1247
|
+
* silently, which is indistinguishable from the defect it fixed. A root
|
|
1248
|
+
* that is unreadable is a broken deployment either way; the only choice
|
|
1249
|
+
* is whether it says so.
|
|
1250
|
+
*
|
|
1251
|
+
* Not caching failure bounds that to one call but does not erase it.
|
|
1252
|
+
* `put` mkdirs before locking so it always canonicalizes; `delete` does
|
|
1253
|
+
* not, so against a not-yet-created symlinked root it locks the
|
|
1254
|
+
* unresolved spelling while a concurrent `put` locks the resolved one —
|
|
1255
|
+
* two locks, no serialization. That needs only ONE spelling in play, not
|
|
1256
|
+
* two: the split is resolved-vs-unresolved form of the same path.
|
|
1257
|
+
*
|
|
1258
|
+
* Known and kept, not overlooked. The hazard `delete` locks
|
|
1259
|
+
* against is slipping inside an `ifMatch` compare→publish, which needs
|
|
1260
|
+
* the key to exist, hence the root to exist, hence `realpath` to have
|
|
1261
|
+
* succeeded — so both conditions hold only if the root appears between
|
|
1262
|
+
* this call and the `rm`, and the next call self-heals. Fixes cost more
|
|
1263
|
+
* than that: mkdir-to-get-a-lock-key makes deleting from an absent
|
|
1264
|
+
* bucket create it, and walking to the deepest existing ancestor adds
|
|
1265
|
+
* real complexity for an unreachable window.
|
|
1266
|
+
*/
|
|
1267
|
+
async #canonicalRoot() {
|
|
1268
|
+
if (this.#realRoot !== void 0) return this.#realRoot;
|
|
1269
|
+
try {
|
|
1270
|
+
this.#realRoot = await realpath(this.#root);
|
|
1271
|
+
return this.#realRoot;
|
|
1272
|
+
} catch (error) {
|
|
1273
|
+
if (isErrnoException(error) && error.code === "ENOENT") return this.#root;
|
|
1274
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage: cannot canonicalize root (${this.#root}): ${error.message}`, { cause: error });
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
async *list(prefix, opts) {
|
|
1278
|
+
opts?.signal?.throwIfAborted();
|
|
1279
|
+
const startAfter = opts?.startAfter ?? "";
|
|
1280
|
+
const maxKeys = opts?.maxKeys ?? 1e400;
|
|
1281
|
+
await this.#ensureFormat(false);
|
|
1282
|
+
const keys = [];
|
|
1283
|
+
for (const components of await walk(this.#root) ?? []) {
|
|
1284
|
+
const key = decodeComponents(components);
|
|
1285
|
+
if (key !== null) {
|
|
1286
|
+
keys.push(key);
|
|
1287
|
+
continue;
|
|
1288
|
+
}
|
|
1289
|
+
const alias = await this.#aliasedKey(components);
|
|
1290
|
+
if (alias !== null) throw new BaerlyError("InvalidConfig", `LocalFsStorage.list: ${join(this.#root, ...components)} was not written by this adapter but the filesystem resolves key ${JSON.stringify(alias)} to it; move the file out of the data directory`);
|
|
1291
|
+
}
|
|
1292
|
+
keys.sort(compareKeysUtf8);
|
|
1293
|
+
let yielded = 0;
|
|
1294
|
+
for (const key of keys) {
|
|
1295
|
+
if (!key.startsWith(prefix)) continue;
|
|
1296
|
+
if (compareKeysUtf8(key, startAfter) <= 0) continue;
|
|
1297
|
+
if (yielded >= maxKeys) return;
|
|
1298
|
+
opts?.signal?.throwIfAborted();
|
|
1299
|
+
let buf;
|
|
1300
|
+
try {
|
|
1301
|
+
buf = await readFile(this.#pathFor(key));
|
|
1302
|
+
} catch (error) {
|
|
1303
|
+
if (isErrnoException(error) && error.code === "ENOENT") continue;
|
|
1304
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage.list(${key}): ${error.message}`, { cause: error });
|
|
1305
|
+
}
|
|
1306
|
+
yield {
|
|
1307
|
+
key,
|
|
1308
|
+
etag: etagOf(buf)
|
|
1309
|
+
};
|
|
1310
|
+
yielded += 1;
|
|
1311
|
+
}
|
|
1312
|
+
}
|
|
1313
|
+
/**
|
|
1314
|
+
* The key whose on-disk path the filesystem resolves to the stray file at
|
|
1315
|
+
* `components`, or `null` if none does. Only a case- or
|
|
1316
|
+
* normalization-folding filesystem produces one: the candidate is
|
|
1317
|
+
* {@link foldToEncodedAlphabet} of the name, which is a key only if it
|
|
1318
|
+
* decodes, and an alias only if both paths name the same inode.
|
|
1319
|
+
*/
|
|
1320
|
+
async #aliasedKey(components) {
|
|
1321
|
+
const folded = components.map(foldToEncodedAlphabet);
|
|
1322
|
+
const key = decodeComponents(folded);
|
|
1323
|
+
if (key === null) return null;
|
|
1324
|
+
try {
|
|
1325
|
+
const [a, b] = await Promise.all([stat(join(this.#root, ...components)), stat(join(this.#root, ...folded))]);
|
|
1326
|
+
return a.dev === b.dev && a.ino === b.ino ? key : null;
|
|
1327
|
+
} catch (error) {
|
|
1328
|
+
if (isErrnoException(error) && (error.code === "ENOENT" || error.code === "ENOTDIR" || error.code === "ENAMETOOLONG")) return null;
|
|
1329
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage.list: ${error.message}`, { cause: error });
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
/**
|
|
1333
|
+
* Make sure the root uses the current key encoding before any operation
|
|
1334
|
+
* touches it. With the marker present this is one small read per
|
|
1335
|
+
* instance. Without it the root is scanned once:
|
|
1336
|
+
*
|
|
1337
|
+
* - absent root: nothing to read, so reads return without stamping, and
|
|
1338
|
+
* the first `put` (`create`) makes the root and stamps it;
|
|
1339
|
+
* - every on-disk name already reads the same under the encoding (true
|
|
1340
|
+
* of kernel-written trees with lowercase ids that hold no snapshot;
|
|
1341
|
+
* the snapshot segment `L9` is escaped): stamp, nothing moves;
|
|
1342
|
+
* - otherwise refuse with `InvalidConfig`. There is no in-place
|
|
1343
|
+
* migration — renaming cannot undo merges a folding filesystem has
|
|
1344
|
+
* already made — so the remedy is dump with the older release and
|
|
1345
|
+
* restore into a fresh directory.
|
|
1346
|
+
*
|
|
1347
|
+
* Only success is cached, so a refusal repeats on every call.
|
|
1348
|
+
*/
|
|
1349
|
+
async #ensureFormat(create) {
|
|
1350
|
+
if (this.#formatChecked) return;
|
|
1351
|
+
await this.#canonicalRoot();
|
|
1352
|
+
const marker = join(this.#root, FORMAT_MARKER);
|
|
1353
|
+
if (await readFormatMarker(marker)) {
|
|
1354
|
+
this.#formatChecked = true;
|
|
1355
|
+
return;
|
|
1356
|
+
}
|
|
1357
|
+
if (create) await mkdir(this.#root, { recursive: true });
|
|
1358
|
+
const legacy = await findLegacyPaths(this.#root);
|
|
1359
|
+
if (legacy === null) return;
|
|
1360
|
+
if (legacy.length > 0 && !await readFormatMarker(marker)) {
|
|
1361
|
+
const shown = legacy.slice(0, LEGACY_PATHS_SHOWN).map((p) => ` ${p}`);
|
|
1362
|
+
if (legacy.length > LEGACY_PATHS_SHOWN) shown.push(` … and ${legacy.length - LEGACY_PATHS_SHOWN} more`);
|
|
1363
|
+
throw new BaerlyError("InvalidConfig", `LocalFsStorage: ${this.#root} holds dev data from an older release, which this release cannot read. Move the directory aside or delete it; the next write starts a fresh one. The usual causes are a folded collection (\`snapshot/L9/\`) and an uppercase letter in a collection name or id (a camelCase name such as \`boardInvitees\`). To keep a collection's rows, dump it with the older release (\`baerly admin dump\`) and restore into the fresh directory (\`baerly admin restore\`); a dump holds collection rows only, not other files the app stored here. Paths not valid under the current key encoding (no ${FORMAT_MARKER} marker):\n${shown.join("\n")}\nIf a listed path is a stray file that is not data, move it out of the directory instead. Keys that differed only by case or Unicode normalization may already have been merged by the filesystem.`);
|
|
1364
|
+
}
|
|
1365
|
+
await stampFormatMarker(marker);
|
|
1366
|
+
this.#formatChecked = true;
|
|
1367
|
+
}
|
|
1368
|
+
/**
|
|
1369
|
+
* Map an S3-style key to a filesystem path under `#root`, rejecting
|
|
1370
|
+
* anything that could escape the root (path traversal, absolute
|
|
1371
|
+
* paths, empty segments, backslashes) or that the encoding cannot
|
|
1372
|
+
* represent injectively (lone surrogates). All rejections throw
|
|
1373
|
+
* `BaerlyError("InvalidConfig", …)`.
|
|
1374
|
+
*/
|
|
1375
|
+
#pathFor(key) {
|
|
1376
|
+
assertValidStorageKey(key);
|
|
1377
|
+
if (key.startsWith("/")) throw new BaerlyError("InvalidConfig", `LocalFsStorage: leading "/" in key: ${key}`);
|
|
1378
|
+
if (key.includes("\\")) throw new BaerlyError("InvalidConfig", `LocalFsStorage: backslash in key: ${key}`);
|
|
1379
|
+
if (key.split("/").some((s) => s === "" || s === "." || s === "..")) throw new BaerlyError("InvalidConfig", `LocalFsStorage: illegal segment in key: ${key}`);
|
|
1380
|
+
const path = join(this.#root, ...encodeKey(key));
|
|
1381
|
+
if (path !== this.#root && !path.startsWith(this.#root + sep)) throw new BaerlyError("InvalidConfig", `LocalFsStorage: resolved path escapes root: ${key}`);
|
|
1382
|
+
return path;
|
|
1383
|
+
}
|
|
1384
|
+
};
|
|
1385
|
+
const etagOf = (body) => {
|
|
1386
|
+
return `"${createHash("sha256").update(body).digest("hex")}"`;
|
|
1387
|
+
};
|
|
1388
|
+
const toUint8Array = (buf) => new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
1389
|
+
const isErrnoException = (e) => typeof e === "object" && e !== null && "code" in e;
|
|
1390
|
+
const readExisting = async (path) => {
|
|
1391
|
+
try {
|
|
1392
|
+
return await readFile(path);
|
|
1393
|
+
} catch (error) {
|
|
1394
|
+
if (isErrnoException(error) && error.code === "ENOENT") return null;
|
|
1395
|
+
throw error;
|
|
1396
|
+
}
|
|
1397
|
+
};
|
|
1398
|
+
/**
|
|
1399
|
+
* `true` when the root's format marker exists and names the current
|
|
1400
|
+
* encoding, `false` when it is absent. A marker naming another encoding
|
|
1401
|
+
* means a newer release wrote this directory, so it is refused.
|
|
1402
|
+
*/
|
|
1403
|
+
const readFormatMarker = async (marker) => {
|
|
1404
|
+
const body = await readExisting(marker).catch((error) => {
|
|
1405
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage: cannot read ${marker}: ${error.message}`, { cause: error });
|
|
1406
|
+
});
|
|
1407
|
+
if (body === null) return false;
|
|
1408
|
+
let encoding;
|
|
1409
|
+
try {
|
|
1410
|
+
encoding = JSON.parse(new TextDecoder().decode(body)).key_encoding;
|
|
1411
|
+
} catch (error) {
|
|
1412
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage: ${marker} is not valid JSON`, { cause: error });
|
|
1413
|
+
}
|
|
1414
|
+
if (encoding !== KEY_ENCODING) throw new BaerlyError("InvalidConfig", `LocalFsStorage: ${marker} names key_encoding ${JSON.stringify(encoding)}; this release reads only key_encoding ${KEY_ENCODING}`);
|
|
1415
|
+
return true;
|
|
1416
|
+
};
|
|
1417
|
+
/**
|
|
1418
|
+
* Write the format marker with the same temp + `link(2)` exclusive create
|
|
1419
|
+
* as `ifNoneMatch:"*"`. Losing that race to a concurrent opener is fine as
|
|
1420
|
+
* long as what it wrote is the current encoding.
|
|
1421
|
+
*/
|
|
1422
|
+
const stampFormatMarker = async (marker) => {
|
|
1423
|
+
const tmp = tempPathFor(marker);
|
|
1424
|
+
try {
|
|
1425
|
+
await writeFile(tmp, `${JSON.stringify({ key_encoding: KEY_ENCODING })}\n`);
|
|
1426
|
+
await link(tmp, marker);
|
|
1427
|
+
} catch (error) {
|
|
1428
|
+
if (isErrnoException(error) && error.code === "EEXIST" && await readFormatMarker(marker)) return;
|
|
1429
|
+
throw new BaerlyError("InvalidResponse", `LocalFsStorage: cannot write ${marker}: ${error.message}`, { cause: error });
|
|
1430
|
+
} finally {
|
|
1431
|
+
await rm(tmp, { force: true }).catch(() => {});
|
|
1432
|
+
}
|
|
1433
|
+
};
|
|
1434
|
+
/**
|
|
1435
|
+
* Non-ASCII characters that APFS's full case folding maps to ASCII but
|
|
1436
|
+
* `toLowerCase` leaves alone (Unicode CaseFolding.txt status `F`, plus the
|
|
1437
|
+
* `C` mapping of long `ſ`). Each was checked on an APFS volume: a file
|
|
1438
|
+
* named with the left side is reached by the right side.
|
|
1439
|
+
*/
|
|
1440
|
+
const APFS_ASCII_FOLDS = {
|
|
1441
|
+
ß: "ss",
|
|
1442
|
+
ẞ: "ss",
|
|
1443
|
+
ſ: "s",
|
|
1444
|
+
ff: "ff",
|
|
1445
|
+
fi: "fi",
|
|
1446
|
+
fl: "fl",
|
|
1447
|
+
ffi: "ffi",
|
|
1448
|
+
ffl: "ffl",
|
|
1449
|
+
ſt: "st",
|
|
1450
|
+
st: "st"
|
|
1451
|
+
};
|
|
1452
|
+
const APFS_ASCII_FOLDABLE = /[ßẞſff-st]/g;
|
|
1453
|
+
/**
|
|
1454
|
+
* Fold an on-disk name the way a folding filesystem matches it, as far as
|
|
1455
|
+
* the result can land in the encoded alphabet (`a-z0-9-_.%`). A name
|
|
1456
|
+
* containing anything else after the fold cannot decode, so only folds to
|
|
1457
|
+
* ASCII matter: lowercase (which also maps Kelvin `K` to `k`) plus
|
|
1458
|
+
* {@link APFS_ASCII_FOLDS}. Normalization adds none: the only canonical
|
|
1459
|
+
* decompositions to ASCII are Kelvin `K`, U+037E (`;`) and U+1FEF (`` ` ``),
|
|
1460
|
+
* and the last two are escaped by the encoding.
|
|
1461
|
+
*/
|
|
1462
|
+
const foldToEncodedAlphabet = (name) => name.replace(APFS_ASCII_FOLDABLE, (c) => APFS_ASCII_FOLDS[c]).toLowerCase();
|
|
1463
|
+
/**
|
|
1464
|
+
* Paths under `root` that the pre-encoding layout wrote and the current
|
|
1465
|
+
* encoding would read differently (uppercase, non-ASCII, `%`, `~`, a
|
|
1466
|
+
* leading `.` …). Empty when the tree can be adopted as-is; `null` when
|
|
1467
|
+
* the root does not exist.
|
|
1468
|
+
*
|
|
1469
|
+
* Walks dot-prefixed names too: the old layout stored a key segment such
|
|
1470
|
+
* as an `_id` or collection named `.foo` verbatim, and the encoding treats
|
|
1471
|
+
* that name as adapter-private, so skipping it here would adopt the tree
|
|
1472
|
+
* and silently drop or orphan that data.
|
|
1473
|
+
*/
|
|
1474
|
+
const findLegacyPaths = async (root) => {
|
|
1475
|
+
const files = await walk(root, isLegacyScanNoise);
|
|
1476
|
+
if (files === null) return null;
|
|
1477
|
+
return files.filter((components) => decodeComponents(components) !== components.join("/")).map((components) => components.join("/")).toSorted();
|
|
1478
|
+
};
|
|
1479
|
+
/**
|
|
1480
|
+
* Entries the legacy scan may skip: staging temps (both layouts wrote them,
|
|
1481
|
+
* and the old one rejected keys with that prefix), and the files Finder
|
|
1482
|
+
* and git leave behind. Junk names are skipped as FILES only — a directory
|
|
1483
|
+
* called `.DS_Store` would be a legacy collection or tenant, i.e. data.
|
|
1484
|
+
* Nothing else is assumed not to be data.
|
|
1485
|
+
*
|
|
1486
|
+
* The format marker is deliberately not here. The scan runs only when the
|
|
1487
|
+
* marker was absent; if a concurrent opener stamps it mid-scan, the scan
|
|
1488
|
+
* flags it like any other name and `#ensureFormat`'s re-read of the marker
|
|
1489
|
+
* is what accepts the directory. A nested file with the marker's name is
|
|
1490
|
+
* legacy data (a key segment that old layout stored verbatim).
|
|
1491
|
+
*/
|
|
1492
|
+
const isLegacyScanNoise = (name, isDirectory) => name.startsWith(TEMP_PREFIX) || !isDirectory && (name === ".DS_Store" || name === ".gitkeep");
|
|
1493
|
+
/**
|
|
1494
|
+
* In the current layout every dot-prefixed name is not a key: no encoded
|
|
1495
|
+
* key component starts with `.`, so those are adapter-private (staging
|
|
1496
|
+
* temps, the format marker) or foreign (`.DS_Store`). Skipping temps is
|
|
1497
|
+
* what keeps a concurrent `list` from surfacing a half-published key.
|
|
1498
|
+
*/
|
|
1499
|
+
const isDotName = (name, _isDirectory) => name.startsWith(".");
|
|
1500
|
+
/**
|
|
1501
|
+
* Recursive directory walk. Returns each file's path components relative
|
|
1502
|
+
* to `root`, or `null` when `root` does not exist. Entries for which
|
|
1503
|
+
* `skip` holds are ignored at every depth, directories included.
|
|
1504
|
+
*/
|
|
1505
|
+
const walk = async (root, skip = isDotName) => {
|
|
1506
|
+
const out = [];
|
|
1507
|
+
const stack = [{
|
|
1508
|
+
dir: root,
|
|
1509
|
+
components: []
|
|
1510
|
+
}];
|
|
1511
|
+
while (stack.length > 0) {
|
|
1512
|
+
const { dir, components } = stack.pop();
|
|
1513
|
+
let entries;
|
|
1514
|
+
try {
|
|
1515
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
1516
|
+
} catch (error) {
|
|
1517
|
+
if (isErrnoException(error) && error.code === "ENOENT") {
|
|
1518
|
+
if (dir === root) return null;
|
|
1519
|
+
continue;
|
|
1520
|
+
}
|
|
1521
|
+
throw error;
|
|
1522
|
+
}
|
|
1523
|
+
for (const entry of entries) {
|
|
1524
|
+
if (skip(entry.name, entry.isDirectory())) continue;
|
|
1525
|
+
const next = [...components, entry.name];
|
|
1526
|
+
if (entry.isDirectory()) stack.push({
|
|
1527
|
+
dir: join(dir, entry.name),
|
|
1528
|
+
components: next
|
|
1529
|
+
});
|
|
1530
|
+
else if (entry.isFile()) out.push(next);
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
return out;
|
|
1534
|
+
};
|
|
1535
|
+
// ../../node_modules/.pnpm/@rgrove+parse-xml@5.2.0/node_modules/@rgrove/parse-xml/dist/browser.js
|
|
1536
|
+
/*! @rgrove/parse-xml v5.2.0 | ISC License | Copyright Ryan Grove */
|
|
1537
|
+
var emptyString = "";
|
|
1538
|
+
var surrogatePair = /[\uD800-\uDBFF][\uDC00-\uDFFF]/g;
|
|
1539
|
+
var StringScanner = class {
|
|
1540
|
+
constructor(string) {
|
|
1541
|
+
this.o = this.u(string, true);
|
|
1542
|
+
this.d = 0;
|
|
1543
|
+
this.length = string.length;
|
|
1544
|
+
this.p = this.o !== this.length;
|
|
1545
|
+
this.h = string;
|
|
1546
|
+
if (this.p) {
|
|
1547
|
+
let charsToBytes = [];
|
|
1548
|
+
for (let byteIndex = 0, charIndex = 0; charIndex <= this.o; ++charIndex) {
|
|
1549
|
+
charsToBytes[charIndex] = byteIndex;
|
|
1550
|
+
byteIndex += string.codePointAt(byteIndex) > 65535 ? 2 : 1;
|
|
1551
|
+
}
|
|
1552
|
+
this.C = charsToBytes;
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
/**
|
|
1556
|
+
* Whether the current character index is at the end of the input string.
|
|
1557
|
+
*/
|
|
1558
|
+
get D() {
|
|
1559
|
+
return this.d >= this.o;
|
|
1560
|
+
}
|
|
1561
|
+
/**
|
|
1562
|
+
* Returns the number of characters in the given string, which may differ from
|
|
1563
|
+
* the byte length if the string contains multibyte characters.
|
|
1564
|
+
*/
|
|
1565
|
+
u(string, multiByteSafe = this.p) {
|
|
1566
|
+
return multiByteSafe ? string.replace(surrogatePair, "_").length : string.length;
|
|
1567
|
+
}
|
|
1568
|
+
/**
|
|
1569
|
+
* Advances the scanner by the given number of characters, stopping if the end
|
|
1570
|
+
* of the string is reached.
|
|
1571
|
+
*/
|
|
1572
|
+
l(count = 1) {
|
|
1573
|
+
this.d = Math.min(this.o, this.d + count);
|
|
1574
|
+
}
|
|
1575
|
+
/**
|
|
1576
|
+
* Returns the byte index of the given character index in the string. The two
|
|
1577
|
+
* may differ in strings that contain multibyte characters.
|
|
1578
|
+
*/
|
|
1579
|
+
f(charIndex = this.d) {
|
|
1580
|
+
var _a;
|
|
1581
|
+
return this.p ? (_a = this.C[charIndex]) != null ? _a : 1e400 : charIndex;
|
|
1582
|
+
}
|
|
1583
|
+
/**
|
|
1584
|
+
* Consumes and returns the given number of characters if possible, advancing
|
|
1585
|
+
* the scanner and stopping if the end of the string is reached.
|
|
1586
|
+
*
|
|
1587
|
+
* If no characters could be consumed, an empty string will be returned.
|
|
1588
|
+
*/
|
|
1589
|
+
G(charCount = 1) {
|
|
1590
|
+
let chars = this.i(charCount);
|
|
1591
|
+
this.l(charCount);
|
|
1592
|
+
return chars;
|
|
1593
|
+
}
|
|
1594
|
+
/**
|
|
1595
|
+
* Consumes and returns the given number of bytes if possible, advancing the
|
|
1596
|
+
* scanner and stopping if the end of the string is reached.
|
|
1597
|
+
*
|
|
1598
|
+
* It's up to the caller to ensure that the given byte count doesn't split a
|
|
1599
|
+
* multibyte character.
|
|
1600
|
+
*
|
|
1601
|
+
* If no bytes could be consumed, an empty string will be returned.
|
|
1602
|
+
*/
|
|
1603
|
+
v(byteCount) {
|
|
1604
|
+
let byteIndex = this.f();
|
|
1605
|
+
let result = this.h.slice(byteIndex, byteIndex + byteCount);
|
|
1606
|
+
this.l(this.u(result));
|
|
1607
|
+
return result;
|
|
1608
|
+
}
|
|
1609
|
+
/**
|
|
1610
|
+
* Consumes and returns all characters for which the given function returns
|
|
1611
|
+
* `true`, stopping when `false` is returned or the end of the input is
|
|
1612
|
+
* reached.
|
|
1613
|
+
*/
|
|
1614
|
+
w(fn) {
|
|
1615
|
+
let { length, p: multiByteMode, h: string } = this;
|
|
1616
|
+
let startByteIndex = this.f();
|
|
1617
|
+
let endByteIndex = startByteIndex;
|
|
1618
|
+
if (multiByteMode) while (endByteIndex < length) {
|
|
1619
|
+
let char = string[endByteIndex];
|
|
1620
|
+
let isSurrogatePair = char >= "\ud800" && char <= "\udbff";
|
|
1621
|
+
if (isSurrogatePair) char += string[endByteIndex + 1];
|
|
1622
|
+
if (!fn(char)) break;
|
|
1623
|
+
endByteIndex += isSurrogatePair ? 2 : 1;
|
|
1624
|
+
}
|
|
1625
|
+
else while (endByteIndex < length && fn(string[endByteIndex])) ++endByteIndex;
|
|
1626
|
+
return this.v(endByteIndex - startByteIndex);
|
|
1627
|
+
}
|
|
1628
|
+
/**
|
|
1629
|
+
* Consumes the given string if it exists at the current character index, and
|
|
1630
|
+
* advances the scanner.
|
|
1631
|
+
*
|
|
1632
|
+
* If the given string doesn't exist at the current character index, an empty
|
|
1633
|
+
* string will be returned and the scanner will not be advanced.
|
|
1634
|
+
*/
|
|
1635
|
+
b(stringToConsume) {
|
|
1636
|
+
let { length } = stringToConsume;
|
|
1637
|
+
let byteIndex = this.f();
|
|
1638
|
+
if (stringToConsume === this.h.slice(byteIndex, byteIndex + length)) {
|
|
1639
|
+
this.l(length === 1 ? 1 : this.u(stringToConsume));
|
|
1640
|
+
return stringToConsume;
|
|
1641
|
+
}
|
|
1642
|
+
return emptyString;
|
|
1643
|
+
}
|
|
1644
|
+
/**
|
|
1645
|
+
* Consumes characters until the given global regex is matched, advancing the
|
|
1646
|
+
* scanner up to (but not beyond) the beginning of the match. If the regex
|
|
1647
|
+
* doesn't match, nothing will be consumed.
|
|
1648
|
+
*
|
|
1649
|
+
* Returns the consumed string, or an empty string if nothing was consumed.
|
|
1650
|
+
*/
|
|
1651
|
+
x(regex) {
|
|
1652
|
+
let matchByteIndex = this.h.slice(this.f()).search(regex);
|
|
1653
|
+
return matchByteIndex > 0 ? this.v(matchByteIndex) : emptyString;
|
|
1654
|
+
}
|
|
1655
|
+
/**
|
|
1656
|
+
* Consumes characters until the given string is found, advancing the scanner
|
|
1657
|
+
* up to (but not beyond) that point. If the string is never found, nothing
|
|
1658
|
+
* will be consumed.
|
|
1659
|
+
*
|
|
1660
|
+
* Returns the consumed string, or an empty string if nothing was consumed.
|
|
1661
|
+
*/
|
|
1662
|
+
s(searchString) {
|
|
1663
|
+
let byteIndex = this.f();
|
|
1664
|
+
let matchByteIndex = this.h.indexOf(searchString, byteIndex);
|
|
1665
|
+
return matchByteIndex > 0 ? this.v(matchByteIndex - byteIndex) : emptyString;
|
|
1666
|
+
}
|
|
1667
|
+
/**
|
|
1668
|
+
* Returns the given number of characters starting at the current character
|
|
1669
|
+
* index, without advancing the scanner and without exceeding the end of the
|
|
1670
|
+
* input string.
|
|
1671
|
+
*/
|
|
1672
|
+
i(count = 1) {
|
|
1673
|
+
let { d: charIndex, h: string } = this;
|
|
1674
|
+
return this.p ? string.slice(this.f(charIndex), this.f(charIndex + count)) : string.slice(charIndex, charIndex + count);
|
|
1675
|
+
}
|
|
1676
|
+
/**
|
|
1677
|
+
* Resets the scanner position to the given character _index_, or to the start
|
|
1678
|
+
* of the input string if no index is given.
|
|
1679
|
+
*
|
|
1680
|
+
* If _index_ is negative, the scanner position will be moved backward by that
|
|
1681
|
+
* many characters, stopping if the beginning of the string is reached.
|
|
1682
|
+
*/
|
|
1683
|
+
j(index = 0) {
|
|
1684
|
+
this.d = index >= 0 ? Math.min(this.o, index) : Math.max(0, this.d + index);
|
|
1685
|
+
}
|
|
1686
|
+
};
|
|
1687
|
+
var attValueCharDoubleQuote = /["&<]/;
|
|
1688
|
+
var attValueCharSingleQuote = /['&<]/;
|
|
1689
|
+
var attValueNormalizedWhitespace = /\r\n|[\n\r\t]/g;
|
|
1690
|
+
var endCharData = /<|&|]]>/;
|
|
1691
|
+
var lineBreakToNormalize = /\r\n?/g;
|
|
1692
|
+
var markupDeclarationDelimiter = /["'<>\]]|$/;
|
|
1693
|
+
var predefinedEntities = Object.freeze(Object.assign(/* @__PURE__ */ Object.create(null), {
|
|
1694
|
+
amp: "&",
|
|
1695
|
+
apos: "'",
|
|
1696
|
+
gt: ">",
|
|
1697
|
+
lt: "<",
|
|
1698
|
+
quot: "\""
|
|
1699
|
+
}));
|
|
1700
|
+
function isNameChar(char) {
|
|
1701
|
+
let cp = char.codePointAt(0);
|
|
1702
|
+
return cp >= 97 && cp <= 122 || cp >= 65 && cp <= 90 || cp >= 48 && cp <= 57 || cp === 45 || cp === 46 || cp === 183 || cp >= 768 && cp <= 879 || cp === 8255 || cp === 8256 || isNameStartChar(char, cp);
|
|
1703
|
+
}
|
|
1704
|
+
function isNameStartChar(char, cp = char.codePointAt(0)) {
|
|
1705
|
+
return cp >= 97 && cp <= 122 || cp >= 65 && cp <= 90 || cp === 58 || cp === 95 || cp >= 192 && cp <= 214 || cp >= 216 && cp <= 246 || cp >= 248 && cp <= 767 || cp >= 880 && cp <= 893 || cp >= 895 && cp <= 8191 || cp === 8204 || cp === 8205 || cp >= 8304 && cp <= 8591 || cp >= 11264 && cp <= 12271 || cp >= 12289 && cp <= 55295 || cp >= 63744 && cp <= 64975 || cp >= 65008 && cp <= 65533 || cp >= 65536 && cp <= 983039;
|
|
1706
|
+
}
|
|
1707
|
+
function isReferenceChar(char) {
|
|
1708
|
+
return char === "#" || isNameChar(char);
|
|
1709
|
+
}
|
|
1710
|
+
function isWhitespace(char) {
|
|
1711
|
+
let cp = char.codePointAt(0);
|
|
1712
|
+
return cp === 32 || cp === 9 || cp === 10 || cp === 13;
|
|
1713
|
+
}
|
|
1714
|
+
function isXmlCodePoint(cp) {
|
|
1715
|
+
return cp >= 32 && cp <= 55295 || cp === 10 || cp === 9 || cp === 13 || cp >= 57344 && cp <= 65533 || cp >= 65536 && cp <= 1114111;
|
|
1716
|
+
}
|
|
1717
|
+
var _XmlNode = class _XmlNode {
|
|
1718
|
+
constructor() {
|
|
1719
|
+
/**
|
|
1720
|
+
* Parent node of this node, or `null` if this node has no parent.
|
|
1721
|
+
*/
|
|
1722
|
+
this.parent = null;
|
|
1723
|
+
/**
|
|
1724
|
+
* Starting byte offset of this node in the original XML string, or `-1` if
|
|
1725
|
+
* the offset is unknown.
|
|
1726
|
+
*/
|
|
1727
|
+
this.start = -1;
|
|
1728
|
+
/**
|
|
1729
|
+
* Ending byte offset of this node in the original XML string, or `-1` if the
|
|
1730
|
+
* offset is unknown.
|
|
1731
|
+
*/
|
|
1732
|
+
this.end = -1;
|
|
1733
|
+
}
|
|
1734
|
+
/**
|
|
1735
|
+
* Document that contains this node, or `null` if this node is not associated
|
|
1736
|
+
* with a document.
|
|
1737
|
+
*/
|
|
1738
|
+
get document() {
|
|
1739
|
+
let node = this.parent;
|
|
1740
|
+
while (node !== null) {
|
|
1741
|
+
if (node.type === _XmlNode.TYPE_DOCUMENT) return node;
|
|
1742
|
+
node = node.parent;
|
|
1743
|
+
}
|
|
1744
|
+
return null;
|
|
1745
|
+
}
|
|
1746
|
+
/**
|
|
1747
|
+
* Whether this node is the root node of the document (also known as the
|
|
1748
|
+
* document element).
|
|
1749
|
+
*/
|
|
1750
|
+
get isRootNode() {
|
|
1751
|
+
var _a;
|
|
1752
|
+
return ((_a = this.parent) == null ? void 0 : _a.type) === _XmlNode.TYPE_DOCUMENT && this.type === _XmlNode.TYPE_ELEMENT;
|
|
1753
|
+
}
|
|
1754
|
+
/**
|
|
1755
|
+
* Whether whitespace should be preserved in the content of this element and
|
|
1756
|
+
* its children.
|
|
1757
|
+
*
|
|
1758
|
+
* This is influenced by the value of the special `xml:space` attribute, and
|
|
1759
|
+
* will be `true` for any node whose `xml:space` attribute is set to
|
|
1760
|
+
* "preserve". If a node has no such attribute, it will inherit the value of
|
|
1761
|
+
* the nearest ancestor that does (if any).
|
|
1762
|
+
*
|
|
1763
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#sec-white-space
|
|
1764
|
+
*/
|
|
1765
|
+
get preserveWhitespace() {
|
|
1766
|
+
var _a;
|
|
1767
|
+
return !!((_a = this.parent) == null ? void 0 : _a.preserveWhitespace);
|
|
1768
|
+
}
|
|
1769
|
+
/**
|
|
1770
|
+
* Type of this node.
|
|
1771
|
+
*
|
|
1772
|
+
* The value of this property is a string that matches one of the static
|
|
1773
|
+
* `TYPE_*` properties on the `XmlNode` class (e.g. `TYPE_ELEMENT`,
|
|
1774
|
+
* `TYPE_TEXT`, etc.).
|
|
1775
|
+
*
|
|
1776
|
+
* The `XmlNode` class itself is a base class and doesn't have its own type
|
|
1777
|
+
* name.
|
|
1778
|
+
*/
|
|
1779
|
+
get type() {
|
|
1780
|
+
return "";
|
|
1781
|
+
}
|
|
1782
|
+
/**
|
|
1783
|
+
* Returns a JSON-serializable object representing this node, minus properties
|
|
1784
|
+
* that could result in circular references.
|
|
1785
|
+
*/
|
|
1786
|
+
toJSON() {
|
|
1787
|
+
let json = { type: this.type };
|
|
1788
|
+
if (this.isRootNode) json.isRootNode = true;
|
|
1789
|
+
if (this.preserveWhitespace) json.preserveWhitespace = true;
|
|
1790
|
+
if (this.start !== -1) {
|
|
1791
|
+
json.start = this.start;
|
|
1792
|
+
json.end = this.end;
|
|
1793
|
+
}
|
|
1794
|
+
return json;
|
|
1795
|
+
}
|
|
1796
|
+
};
|
|
1797
|
+
/**
|
|
1798
|
+
* Type value for an `XmlCdata` node.
|
|
1799
|
+
*/
|
|
1800
|
+
_XmlNode.TYPE_CDATA = "cdata";
|
|
1801
|
+
/**
|
|
1802
|
+
* Type value for an `XmlComment` node.
|
|
1803
|
+
*/
|
|
1804
|
+
_XmlNode.TYPE_COMMENT = "comment";
|
|
1805
|
+
/**
|
|
1806
|
+
* Type value for an `XmlDocument` node.
|
|
1807
|
+
*/
|
|
1808
|
+
_XmlNode.TYPE_DOCUMENT = "document";
|
|
1809
|
+
/**
|
|
1810
|
+
* Type value for an `XmlDocumentType` node.
|
|
1811
|
+
*/
|
|
1812
|
+
_XmlNode.TYPE_DOCUMENT_TYPE = "doctype";
|
|
1813
|
+
/**
|
|
1814
|
+
* Type value for an `XmlElement` node.
|
|
1815
|
+
*/
|
|
1816
|
+
_XmlNode.TYPE_ELEMENT = "element";
|
|
1817
|
+
/**
|
|
1818
|
+
* Type value for an `XmlProcessingInstruction` node.
|
|
1819
|
+
*/
|
|
1820
|
+
_XmlNode.TYPE_PROCESSING_INSTRUCTION = "pi";
|
|
1821
|
+
/**
|
|
1822
|
+
* Type value for an `XmlText` node.
|
|
1823
|
+
*/
|
|
1824
|
+
_XmlNode.TYPE_TEXT = "text";
|
|
1825
|
+
/**
|
|
1826
|
+
* Type value for an `XmlDeclaration` node.
|
|
1827
|
+
*/
|
|
1828
|
+
_XmlNode.TYPE_XML_DECLARATION = "xmldecl";
|
|
1829
|
+
var XmlNode = _XmlNode;
|
|
1830
|
+
var XmlText = class extends XmlNode {
|
|
1831
|
+
constructor(text = "") {
|
|
1832
|
+
super();
|
|
1833
|
+
this.text = text;
|
|
1834
|
+
}
|
|
1835
|
+
get type() {
|
|
1836
|
+
return XmlNode.TYPE_TEXT;
|
|
1837
|
+
}
|
|
1838
|
+
toJSON() {
|
|
1839
|
+
return Object.assign(XmlNode.prototype.toJSON.call(this), { text: this.text });
|
|
1840
|
+
}
|
|
1841
|
+
};
|
|
1842
|
+
var XmlCdata = class extends XmlText {
|
|
1843
|
+
get type() {
|
|
1844
|
+
return XmlNode.TYPE_CDATA;
|
|
1845
|
+
}
|
|
1846
|
+
};
|
|
1847
|
+
var XmlComment = class extends XmlNode {
|
|
1848
|
+
constructor(content = "") {
|
|
1849
|
+
super();
|
|
1850
|
+
this.content = content;
|
|
1851
|
+
}
|
|
1852
|
+
get type() {
|
|
1853
|
+
return XmlNode.TYPE_COMMENT;
|
|
1854
|
+
}
|
|
1855
|
+
toJSON() {
|
|
1856
|
+
return Object.assign(XmlNode.prototype.toJSON.call(this), { content: this.content });
|
|
1857
|
+
}
|
|
1858
|
+
};
|
|
1859
|
+
var XmlDeclaration = class extends XmlNode {
|
|
1860
|
+
constructor(version, encoding, standalone) {
|
|
1861
|
+
super();
|
|
1862
|
+
this.version = version;
|
|
1863
|
+
this.encoding = encoding != null ? encoding : null;
|
|
1864
|
+
this.standalone = standalone != null ? standalone : null;
|
|
1865
|
+
}
|
|
1866
|
+
get type() {
|
|
1867
|
+
return XmlNode.TYPE_XML_DECLARATION;
|
|
1868
|
+
}
|
|
1869
|
+
toJSON() {
|
|
1870
|
+
let json = XmlNode.prototype.toJSON.call(this);
|
|
1871
|
+
json.version = this.version;
|
|
1872
|
+
for (let key of ["encoding", "standalone"]) if (this[key] !== null) json[key] = this[key];
|
|
1873
|
+
return json;
|
|
1874
|
+
}
|
|
1875
|
+
};
|
|
1876
|
+
var XmlElement = class _XmlElement extends XmlNode {
|
|
1877
|
+
constructor(name, attributes = /* @__PURE__ */ Object.create(null), children = []) {
|
|
1878
|
+
super();
|
|
1879
|
+
this.name = name;
|
|
1880
|
+
this.attributes = attributes;
|
|
1881
|
+
this.children = children;
|
|
1882
|
+
}
|
|
1883
|
+
/**
|
|
1884
|
+
* Whether this element is empty (meaning it has no children).
|
|
1885
|
+
*/
|
|
1886
|
+
get isEmpty() {
|
|
1887
|
+
return this.children.length === 0;
|
|
1888
|
+
}
|
|
1889
|
+
get preserveWhitespace() {
|
|
1890
|
+
let node = this;
|
|
1891
|
+
while (node instanceof _XmlElement) {
|
|
1892
|
+
if ("xml:space" in node.attributes) return node.attributes["xml:space"] === "preserve";
|
|
1893
|
+
node = node.parent;
|
|
1894
|
+
}
|
|
1895
|
+
return false;
|
|
1896
|
+
}
|
|
1897
|
+
/**
|
|
1898
|
+
* Text content of this element and all its descendants.
|
|
1899
|
+
*/
|
|
1900
|
+
get text() {
|
|
1901
|
+
return this.children.map((child) => "text" in child ? child.text : "").join("");
|
|
1902
|
+
}
|
|
1903
|
+
get type() {
|
|
1904
|
+
return XmlNode.TYPE_ELEMENT;
|
|
1905
|
+
}
|
|
1906
|
+
toJSON() {
|
|
1907
|
+
return Object.assign(XmlNode.prototype.toJSON.call(this), {
|
|
1908
|
+
name: this.name,
|
|
1909
|
+
attributes: this.attributes,
|
|
1910
|
+
children: this.children.map((child) => child.toJSON())
|
|
1911
|
+
});
|
|
1912
|
+
}
|
|
1913
|
+
};
|
|
1914
|
+
var XmlDocument = class extends XmlNode {
|
|
1915
|
+
constructor(children = []) {
|
|
1916
|
+
super();
|
|
1917
|
+
this.children = children;
|
|
1918
|
+
}
|
|
1919
|
+
get document() {
|
|
1920
|
+
return this;
|
|
1921
|
+
}
|
|
1922
|
+
/**
|
|
1923
|
+
* Root element of this document, or `null` if this document is empty.
|
|
1924
|
+
*/
|
|
1925
|
+
get root() {
|
|
1926
|
+
for (let child of this.children) if (child instanceof XmlElement) return child;
|
|
1927
|
+
return null;
|
|
1928
|
+
}
|
|
1929
|
+
/**
|
|
1930
|
+
* Text content of this document and all its descendants.
|
|
1931
|
+
*/
|
|
1932
|
+
get text() {
|
|
1933
|
+
return this.children.map((child) => "text" in child ? child.text : "").join("");
|
|
1934
|
+
}
|
|
1935
|
+
get type() {
|
|
1936
|
+
return XmlNode.TYPE_DOCUMENT;
|
|
1937
|
+
}
|
|
1938
|
+
toJSON() {
|
|
1939
|
+
return Object.assign(XmlNode.prototype.toJSON.call(this), { children: this.children.map((child) => child.toJSON()) });
|
|
1940
|
+
}
|
|
1941
|
+
};
|
|
1942
|
+
var XmlDocumentType = class extends XmlNode {
|
|
1943
|
+
constructor(name, publicId, systemId, internalSubset) {
|
|
1944
|
+
super();
|
|
1945
|
+
this.name = name;
|
|
1946
|
+
this.publicId = publicId != null ? publicId : null;
|
|
1947
|
+
this.systemId = systemId != null ? systemId : null;
|
|
1948
|
+
this.internalSubset = internalSubset != null ? internalSubset : null;
|
|
1949
|
+
}
|
|
1950
|
+
get type() {
|
|
1951
|
+
return XmlNode.TYPE_DOCUMENT_TYPE;
|
|
1952
|
+
}
|
|
1953
|
+
toJSON() {
|
|
1954
|
+
let json = XmlNode.prototype.toJSON.call(this);
|
|
1955
|
+
json.name = this.name;
|
|
1956
|
+
for (let key of [
|
|
1957
|
+
"publicId",
|
|
1958
|
+
"systemId",
|
|
1959
|
+
"internalSubset"
|
|
1960
|
+
]) if (this[key] !== null) json[key] = this[key];
|
|
1961
|
+
return json;
|
|
1962
|
+
}
|
|
1963
|
+
};
|
|
1964
|
+
var XmlError = class extends Error {
|
|
1965
|
+
constructor(message, charIndex, xml) {
|
|
1966
|
+
let column = 1;
|
|
1967
|
+
let excerpt = "";
|
|
1968
|
+
let line = 1;
|
|
1969
|
+
for (let i = 0; i < charIndex; ++i) {
|
|
1970
|
+
let char = xml[i];
|
|
1971
|
+
if (char === "\n") {
|
|
1972
|
+
column = 1;
|
|
1973
|
+
excerpt = "";
|
|
1974
|
+
line += 1;
|
|
1975
|
+
} else {
|
|
1976
|
+
column += 1;
|
|
1977
|
+
excerpt += char;
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
let eol = xml.indexOf("\n", charIndex);
|
|
1981
|
+
excerpt += eol === -1 ? xml.slice(charIndex) : xml.slice(charIndex, eol);
|
|
1982
|
+
let excerptStart = 0;
|
|
1983
|
+
if (excerpt.length > 50) {
|
|
1984
|
+
if (column < 40) excerpt = excerpt.slice(0, 50);
|
|
1985
|
+
else {
|
|
1986
|
+
excerptStart = column - 20;
|
|
1987
|
+
excerpt = excerpt.slice(excerptStart, column + 30);
|
|
1988
|
+
}
|
|
1989
|
+
}
|
|
1990
|
+
super(`${message} (line ${line}, column ${column})
|
|
1991
|
+
${excerpt}
|
|
1992
|
+
` + " ".repeat(column - excerptStart + 1) + "^\n");
|
|
1993
|
+
this.column = column;
|
|
1994
|
+
this.excerpt = excerpt;
|
|
1995
|
+
this.line = line;
|
|
1996
|
+
this.name = "XmlError";
|
|
1997
|
+
this.pos = charIndex;
|
|
1998
|
+
}
|
|
1999
|
+
};
|
|
2000
|
+
var XmlProcessingInstruction = class extends XmlNode {
|
|
2001
|
+
constructor(name, content = "") {
|
|
2002
|
+
super();
|
|
2003
|
+
this.name = name;
|
|
2004
|
+
this.content = content;
|
|
2005
|
+
}
|
|
2006
|
+
get type() {
|
|
2007
|
+
return XmlNode.TYPE_PROCESSING_INSTRUCTION;
|
|
2008
|
+
}
|
|
2009
|
+
toJSON() {
|
|
2010
|
+
return Object.assign(XmlNode.prototype.toJSON.call(this), {
|
|
2011
|
+
name: this.name,
|
|
2012
|
+
content: this.content
|
|
2013
|
+
});
|
|
2014
|
+
}
|
|
2015
|
+
};
|
|
2016
|
+
var defaultMaxElementDepth = 256;
|
|
2017
|
+
var emptyString2 = "";
|
|
2018
|
+
var Parser = class {
|
|
2019
|
+
/**
|
|
2020
|
+
* @param xml XML string to parse.
|
|
2021
|
+
* @param options Parser options.
|
|
2022
|
+
*/
|
|
2023
|
+
constructor(xml, options = {}) {
|
|
2024
|
+
this.depth = 0;
|
|
2025
|
+
var _a;
|
|
2026
|
+
let doc = this.document = new XmlDocument();
|
|
2027
|
+
this.m = doc;
|
|
2028
|
+
this.maxElementDepth = (_a = options.maxElementDepth) != null ? _a : defaultMaxElementDepth;
|
|
2029
|
+
this.g = options;
|
|
2030
|
+
this.c = new StringScanner(xml);
|
|
2031
|
+
if (this.g.includeOffsets) {
|
|
2032
|
+
doc.start = 0;
|
|
2033
|
+
doc.end = xml.length;
|
|
2034
|
+
}
|
|
2035
|
+
this.parse();
|
|
2036
|
+
}
|
|
2037
|
+
/**
|
|
2038
|
+
* Adds the given `XmlNode` as a child of `this.currentNode`.
|
|
2039
|
+
*/
|
|
2040
|
+
k(node, charIndex) {
|
|
2041
|
+
node.parent = this.m;
|
|
2042
|
+
if (this.g.includeOffsets) {
|
|
2043
|
+
node.start = this.c.f(charIndex);
|
|
2044
|
+
node.end = this.c.f();
|
|
2045
|
+
}
|
|
2046
|
+
this.m.children.push(node);
|
|
2047
|
+
return true;
|
|
2048
|
+
}
|
|
2049
|
+
/**
|
|
2050
|
+
* Adds the given _text_ to the document, either by appending it to a
|
|
2051
|
+
* preceding `XmlText` node (if possible) or by creating a new `XmlText` node.
|
|
2052
|
+
*
|
|
2053
|
+
* When _normalize_ is `true` (the default), line breaks in _text_ are
|
|
2054
|
+
* normalized per section 2.11 of the XML spec. This must be `false` for text
|
|
2055
|
+
* that comes from a character or entity reference, since references aren't
|
|
2056
|
+
* subject to line break normalization.
|
|
2057
|
+
*/
|
|
2058
|
+
y(text, charIndex, normalize = true) {
|
|
2059
|
+
let { children } = this.m;
|
|
2060
|
+
let { length } = children;
|
|
2061
|
+
if (normalize) text = normalizeLineBreaks(text);
|
|
2062
|
+
if (length > 0) {
|
|
2063
|
+
let prevNode = children[length - 1];
|
|
2064
|
+
if ((prevNode == null ? void 0 : prevNode.type) === XmlNode.TYPE_TEXT) {
|
|
2065
|
+
let textNode = prevNode;
|
|
2066
|
+
textNode.text += text;
|
|
2067
|
+
if (this.g.includeOffsets) textNode.end = this.c.f();
|
|
2068
|
+
return true;
|
|
2069
|
+
}
|
|
2070
|
+
}
|
|
2071
|
+
return this.k(new XmlText(text), charIndex);
|
|
2072
|
+
}
|
|
2073
|
+
/**
|
|
2074
|
+
* Consumes element attributes.
|
|
2075
|
+
*
|
|
2076
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#sec-starttags
|
|
2077
|
+
*/
|
|
2078
|
+
H() {
|
|
2079
|
+
let attributes = /* @__PURE__ */ Object.create(null);
|
|
2080
|
+
while (this.e()) {
|
|
2081
|
+
let attrName = this.q();
|
|
2082
|
+
if (!attrName) break;
|
|
2083
|
+
let attrValue = this.t() && this.I();
|
|
2084
|
+
if (attrValue === false) throw this.a("Attribute value expected");
|
|
2085
|
+
if (attrName in attributes) throw this.a(`Duplicate attribute: ${attrName}`);
|
|
2086
|
+
if (attrName === "xml:space" && attrValue !== "default" && attrValue !== "preserve") throw this.a("Value of the `xml:space` attribute must be \"default\" or \"preserve\"");
|
|
2087
|
+
attributes[attrName] = attrValue;
|
|
2088
|
+
}
|
|
2089
|
+
if (this.g.sortAttributes) {
|
|
2090
|
+
let attrNames = Object.keys(attributes).sort();
|
|
2091
|
+
let sortedAttributes = /* @__PURE__ */ Object.create(null);
|
|
2092
|
+
for (let i = 0; i < attrNames.length; ++i) {
|
|
2093
|
+
let attrName = attrNames[i];
|
|
2094
|
+
sortedAttributes[attrName] = attributes[attrName];
|
|
2095
|
+
}
|
|
2096
|
+
attributes = sortedAttributes;
|
|
2097
|
+
}
|
|
2098
|
+
return attributes;
|
|
2099
|
+
}
|
|
2100
|
+
/**
|
|
2101
|
+
* Consumes an `AttValue` (attribute value) if possible.
|
|
2102
|
+
*
|
|
2103
|
+
* @returns
|
|
2104
|
+
* Contents of the `AttValue` minus quotes, or `false` if nothing was
|
|
2105
|
+
* consumed. An empty string indicates that an `AttValue` was consumed but
|
|
2106
|
+
* was empty.
|
|
2107
|
+
*
|
|
2108
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-AttValue
|
|
2109
|
+
*/
|
|
2110
|
+
I() {
|
|
2111
|
+
let { c: scanner } = this;
|
|
2112
|
+
let quote = scanner.i();
|
|
2113
|
+
if (quote !== "\"" && quote !== "'") return false;
|
|
2114
|
+
scanner.l();
|
|
2115
|
+
let chars;
|
|
2116
|
+
let isClosed = false;
|
|
2117
|
+
let value = emptyString2;
|
|
2118
|
+
let regex = quote === "\"" ? attValueCharDoubleQuote : attValueCharSingleQuote;
|
|
2119
|
+
matchLoop: while (!scanner.D) {
|
|
2120
|
+
chars = scanner.x(regex);
|
|
2121
|
+
if (chars) {
|
|
2122
|
+
this.n(chars);
|
|
2123
|
+
value += chars.replace(attValueNormalizedWhitespace, " ");
|
|
2124
|
+
}
|
|
2125
|
+
switch (scanner.i()) {
|
|
2126
|
+
case quote:
|
|
2127
|
+
isClosed = true;
|
|
2128
|
+
break matchLoop;
|
|
2129
|
+
case "&":
|
|
2130
|
+
value += this.E();
|
|
2131
|
+
continue;
|
|
2132
|
+
case "<": throw this.a("Unescaped `<` is not allowed in an attribute value");
|
|
2133
|
+
default: break matchLoop;
|
|
2134
|
+
}
|
|
2135
|
+
}
|
|
2136
|
+
if (!isClosed) throw this.a("Unclosed attribute");
|
|
2137
|
+
scanner.l();
|
|
2138
|
+
return value;
|
|
2139
|
+
}
|
|
2140
|
+
/**
|
|
2141
|
+
* Consumes a CDATA section if possible.
|
|
2142
|
+
*
|
|
2143
|
+
* @returns Whether a CDATA section was consumed.
|
|
2144
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#sec-cdata-sect
|
|
2145
|
+
*/
|
|
2146
|
+
J() {
|
|
2147
|
+
let { c: scanner } = this;
|
|
2148
|
+
let startIndex = scanner.d;
|
|
2149
|
+
if (!scanner.b("<![CDATA[")) return false;
|
|
2150
|
+
let text = scanner.s("]]>");
|
|
2151
|
+
this.n(text);
|
|
2152
|
+
if (!scanner.b("]]>")) throw this.a("Unclosed CDATA section");
|
|
2153
|
+
return this.g.preserveCdata ? this.k(new XmlCdata(normalizeLineBreaks(text)), startIndex) : this.y(text, startIndex);
|
|
2154
|
+
}
|
|
2155
|
+
/**
|
|
2156
|
+
* Consumes character data if possible.
|
|
2157
|
+
*
|
|
2158
|
+
* @returns Whether character data was consumed.
|
|
2159
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#dt-chardata
|
|
2160
|
+
*/
|
|
2161
|
+
K() {
|
|
2162
|
+
let { c: scanner } = this;
|
|
2163
|
+
let startIndex = scanner.d;
|
|
2164
|
+
let charData = scanner.x(endCharData);
|
|
2165
|
+
if (!charData) return false;
|
|
2166
|
+
this.n(charData);
|
|
2167
|
+
if (scanner.i(3) === "]]>") throw this.a("Element content may not contain the CDATA section close delimiter `]]>`");
|
|
2168
|
+
return this.y(charData, startIndex);
|
|
2169
|
+
}
|
|
2170
|
+
/**
|
|
2171
|
+
* Consumes a comment if possible.
|
|
2172
|
+
*
|
|
2173
|
+
* @param addToDocument
|
|
2174
|
+
* Whether the comment may be added to the document (subject to the
|
|
2175
|
+
* `preserveComments` option). This should be `false` for comments inside a
|
|
2176
|
+
* DTD internal subset.
|
|
2177
|
+
*
|
|
2178
|
+
* @returns Whether a comment was consumed.
|
|
2179
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-Comment
|
|
2180
|
+
*/
|
|
2181
|
+
z(addToDocument = true) {
|
|
2182
|
+
let { c: scanner } = this;
|
|
2183
|
+
let startIndex = scanner.d;
|
|
2184
|
+
if (!scanner.b("<!--")) return false;
|
|
2185
|
+
let content = scanner.s("--");
|
|
2186
|
+
this.n(content);
|
|
2187
|
+
if (!scanner.b("-->")) {
|
|
2188
|
+
if (scanner.i(2) === "--") throw this.a("The string `--` isn't allowed inside a comment");
|
|
2189
|
+
throw this.a("Unclosed comment");
|
|
2190
|
+
}
|
|
2191
|
+
return addToDocument && this.g.preserveComments ? this.k(new XmlComment(normalizeLineBreaks(content)), startIndex) : true;
|
|
2192
|
+
}
|
|
2193
|
+
/**
|
|
2194
|
+
* Consumes a reference in a content context if possible.
|
|
2195
|
+
*
|
|
2196
|
+
* This differs from `consumeReference()` in that a consumed reference will be
|
|
2197
|
+
* added to the document as a text node instead of returned.
|
|
2198
|
+
*
|
|
2199
|
+
* @returns Whether a reference was consumed.
|
|
2200
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#entproc
|
|
2201
|
+
*/
|
|
2202
|
+
L() {
|
|
2203
|
+
let startIndex = this.c.d;
|
|
2204
|
+
let ref = this.E();
|
|
2205
|
+
return ref ? this.y(ref, startIndex, false) : false;
|
|
2206
|
+
}
|
|
2207
|
+
/**
|
|
2208
|
+
* Consumes a doctype declaration if possible.
|
|
2209
|
+
*
|
|
2210
|
+
* This is a loose implementation since DTDs are otherwise ignored. The
|
|
2211
|
+
* internal subset is only scanned closely enough to find where it ends.
|
|
2212
|
+
*
|
|
2213
|
+
* @returns Whether a doctype declaration was consumed.
|
|
2214
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#dtd
|
|
2215
|
+
*/
|
|
2216
|
+
M() {
|
|
2217
|
+
let { c: scanner } = this;
|
|
2218
|
+
let startIndex = scanner.d;
|
|
2219
|
+
if (!scanner.b("<!DOCTYPE")) return false;
|
|
2220
|
+
let name = this.e() && this.q();
|
|
2221
|
+
if (!name) throw this.a("Expected a name");
|
|
2222
|
+
let publicId;
|
|
2223
|
+
let systemId;
|
|
2224
|
+
if (this.e()) {
|
|
2225
|
+
if (scanner.b("PUBLIC")) {
|
|
2226
|
+
publicId = this.e() && this.N();
|
|
2227
|
+
if (publicId === false) throw this.a("Expected a public identifier");
|
|
2228
|
+
}
|
|
2229
|
+
if (publicId !== void 0 || scanner.b("SYSTEM")) {
|
|
2230
|
+
systemId = this.e() && this.r();
|
|
2231
|
+
if (systemId === false) throw this.a("Expected a system identifier");
|
|
2232
|
+
this.e();
|
|
2233
|
+
}
|
|
2234
|
+
}
|
|
2235
|
+
let internalSubset;
|
|
2236
|
+
if (scanner.b("[")) {
|
|
2237
|
+
internalSubset = this.O();
|
|
2238
|
+
if (!scanner.b("]")) throw this.a("Unclosed internal subset");
|
|
2239
|
+
this.e();
|
|
2240
|
+
}
|
|
2241
|
+
if (!scanner.b(">")) throw this.a("Unclosed doctype declaration");
|
|
2242
|
+
return this.g.preserveDocumentType ? this.k(new XmlDocumentType(name, publicId, systemId, internalSubset), startIndex) : true;
|
|
2243
|
+
}
|
|
2244
|
+
/**
|
|
2245
|
+
* Consumes an element if possible.
|
|
2246
|
+
*
|
|
2247
|
+
* @returns Whether an element was consumed.
|
|
2248
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-element
|
|
2249
|
+
*/
|
|
2250
|
+
F() {
|
|
2251
|
+
let { c: scanner } = this;
|
|
2252
|
+
let startIndex = scanner.d;
|
|
2253
|
+
if (!scanner.b("<")) return false;
|
|
2254
|
+
let name = this.q();
|
|
2255
|
+
if (!name) {
|
|
2256
|
+
scanner.j(startIndex);
|
|
2257
|
+
return false;
|
|
2258
|
+
}
|
|
2259
|
+
if (!(this.depth < this.maxElementDepth)) {
|
|
2260
|
+
scanner.j(startIndex);
|
|
2261
|
+
throw this.a(`Maximum element depth of ${this.maxElementDepth} exceeded (see the \`maxElementDepth\` option)`);
|
|
2262
|
+
}
|
|
2263
|
+
let attributes = this.H();
|
|
2264
|
+
let isEmpty = !!scanner.b("/>");
|
|
2265
|
+
let element = new XmlElement(name, attributes);
|
|
2266
|
+
element.parent = this.m;
|
|
2267
|
+
if (!isEmpty) {
|
|
2268
|
+
if (!scanner.b(">")) throw this.a(`Unclosed start tag for element \`${name}\``);
|
|
2269
|
+
this.m = element;
|
|
2270
|
+
this.depth += 1;
|
|
2271
|
+
do
|
|
2272
|
+
this.K();
|
|
2273
|
+
while (this.F() || this.L() || this.J() || this.A() || this.z());
|
|
2274
|
+
let endTagMark = scanner.d;
|
|
2275
|
+
let endTagName;
|
|
2276
|
+
if (!scanner.b("</") || !(endTagName = this.q()) || endTagName !== name) {
|
|
2277
|
+
scanner.j(endTagMark);
|
|
2278
|
+
throw this.a(`Missing end tag for element ${name}`);
|
|
2279
|
+
}
|
|
2280
|
+
this.e();
|
|
2281
|
+
if (!scanner.b(">")) throw this.a(`Unclosed end tag for element ${name}`);
|
|
2282
|
+
this.m = element.parent;
|
|
2283
|
+
this.depth -= 1;
|
|
2284
|
+
}
|
|
2285
|
+
return this.k(element, startIndex);
|
|
2286
|
+
}
|
|
2287
|
+
/**
|
|
2288
|
+
* Consumes an `Eq` production if possible.
|
|
2289
|
+
*
|
|
2290
|
+
* @returns Whether an `Eq` production was consumed.
|
|
2291
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-Eq
|
|
2292
|
+
*/
|
|
2293
|
+
t() {
|
|
2294
|
+
this.e();
|
|
2295
|
+
if (this.c.b("=")) {
|
|
2296
|
+
this.e();
|
|
2297
|
+
return true;
|
|
2298
|
+
}
|
|
2299
|
+
return false;
|
|
2300
|
+
}
|
|
2301
|
+
/**
|
|
2302
|
+
* Consumes the contents of a DTD internal subset, stopping before the closing
|
|
2303
|
+
* `]` or at the first content that isn't allowed in an internal subset.
|
|
2304
|
+
*
|
|
2305
|
+
* The end of the internal subset is the first `]` that isn't inside a
|
|
2306
|
+
* comment, processing instruction, or quoted literal, all of which may
|
|
2307
|
+
* legally contain `]>`. Since DTDs are otherwise ignored, markup declarations
|
|
2308
|
+
* are only scanned closely enough to find where they end.
|
|
2309
|
+
*
|
|
2310
|
+
* @returns Raw text of the internal subset.
|
|
2311
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-intSubset
|
|
2312
|
+
*/
|
|
2313
|
+
O() {
|
|
2314
|
+
let { c: scanner } = this;
|
|
2315
|
+
let startByteIndex = scanner.f();
|
|
2316
|
+
while (this.e() || this.z(false) || this.A(false) || this.P() || this.Q());
|
|
2317
|
+
return scanner.h.slice(startByteIndex, scanner.f());
|
|
2318
|
+
}
|
|
2319
|
+
/**
|
|
2320
|
+
* Consumes an element, attribute-list, entity, or notation declaration in a
|
|
2321
|
+
* DTD internal subset if possible.
|
|
2322
|
+
*
|
|
2323
|
+
* The declaration isn't fully parsed. Quoted literals are skipped as opaque
|
|
2324
|
+
* units, and the declaration ends at the first `>` outside a literal.
|
|
2325
|
+
*
|
|
2326
|
+
* @returns Whether a markup declaration was consumed.
|
|
2327
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-markupdecl
|
|
2328
|
+
*/
|
|
2329
|
+
P() {
|
|
2330
|
+
let { c: scanner } = this;
|
|
2331
|
+
let startIndex = scanner.d;
|
|
2332
|
+
if (scanner.b("<!ATTLIST")) {
|
|
2333
|
+
if (this.g.rejectAttributeListDeclarations) {
|
|
2334
|
+
scanner.j(startIndex);
|
|
2335
|
+
throw this.a("Attribute-list declarations aren't allowed when `rejectAttributeListDeclarations` is enabled");
|
|
2336
|
+
}
|
|
2337
|
+
} else if (!scanner.b("<!ELEMENT") && !scanner.b("<!ENTITY") && !scanner.b("<!NOTATION")) return false;
|
|
2338
|
+
for (;;) {
|
|
2339
|
+
this.n(scanner.x(markupDeclarationDelimiter));
|
|
2340
|
+
let char = scanner.i();
|
|
2341
|
+
if (char === ">") {
|
|
2342
|
+
scanner.l();
|
|
2343
|
+
return true;
|
|
2344
|
+
}
|
|
2345
|
+
if (char === "\"" || char === "'") {
|
|
2346
|
+
this.r();
|
|
2347
|
+
continue;
|
|
2348
|
+
}
|
|
2349
|
+
throw this.a("Unclosed markup declaration");
|
|
2350
|
+
}
|
|
2351
|
+
}
|
|
2352
|
+
/**
|
|
2353
|
+
* Consumes `Misc` content if possible.
|
|
2354
|
+
*
|
|
2355
|
+
* @returns Whether anything was consumed.
|
|
2356
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-Misc
|
|
2357
|
+
*/
|
|
2358
|
+
B() {
|
|
2359
|
+
return this.z() || this.A() || this.e();
|
|
2360
|
+
}
|
|
2361
|
+
/**
|
|
2362
|
+
* Consumes one or more `Name` characters if possible.
|
|
2363
|
+
*
|
|
2364
|
+
* @returns `Name` characters, or an empty string if none were consumed.
|
|
2365
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-Name
|
|
2366
|
+
*/
|
|
2367
|
+
q() {
|
|
2368
|
+
return isNameStartChar(this.c.i()) ? this.c.w(isNameChar) : emptyString2;
|
|
2369
|
+
}
|
|
2370
|
+
/**
|
|
2371
|
+
* Consumes a parameter entity reference in a DTD internal subset if possible.
|
|
2372
|
+
*
|
|
2373
|
+
* Since parse-xml doesn't expand parameter entities, a reference is rejected
|
|
2374
|
+
* when `rejectAttributeListDeclarations` is enabled, because its replacement
|
|
2375
|
+
* text could contain an attribute-list declaration.
|
|
2376
|
+
*
|
|
2377
|
+
* @returns Whether a parameter entity reference was consumed.
|
|
2378
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-PEReference
|
|
2379
|
+
*/
|
|
2380
|
+
Q() {
|
|
2381
|
+
let { c: scanner } = this;
|
|
2382
|
+
if (scanner.i() !== "%") return false;
|
|
2383
|
+
if (this.g.rejectAttributeListDeclarations) throw this.a("Parameter entity references aren't allowed when `rejectAttributeListDeclarations` is enabled");
|
|
2384
|
+
scanner.l();
|
|
2385
|
+
if (!this.q() || !scanner.b(";")) throw this.a("Invalid parameter entity reference");
|
|
2386
|
+
return true;
|
|
2387
|
+
}
|
|
2388
|
+
/**
|
|
2389
|
+
* Consumes a processing instruction if possible.
|
|
2390
|
+
*
|
|
2391
|
+
* @param addToDocument
|
|
2392
|
+
* Whether to add the processing instruction to the document. This should be
|
|
2393
|
+
* `false` for processing instructions inside a DTD internal subset.
|
|
2394
|
+
*
|
|
2395
|
+
* @returns Whether a processing instruction was consumed.
|
|
2396
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#sec-pi
|
|
2397
|
+
*/
|
|
2398
|
+
A(addToDocument = true) {
|
|
2399
|
+
let { c: scanner } = this;
|
|
2400
|
+
let startIndex = scanner.d;
|
|
2401
|
+
if (!scanner.b("<?")) return false;
|
|
2402
|
+
let name = this.q();
|
|
2403
|
+
if (name) {
|
|
2404
|
+
if (name.toLowerCase() === "xml") {
|
|
2405
|
+
scanner.j(startIndex);
|
|
2406
|
+
throw this.a("XML declaration isn't allowed here");
|
|
2407
|
+
}
|
|
2408
|
+
} else throw this.a("Invalid processing instruction");
|
|
2409
|
+
if (!this.e()) {
|
|
2410
|
+
if (scanner.b("?>")) return addToDocument ? this.k(new XmlProcessingInstruction(name), startIndex) : true;
|
|
2411
|
+
throw this.a("Whitespace is required after a processing instruction name");
|
|
2412
|
+
}
|
|
2413
|
+
let content = scanner.s("?>");
|
|
2414
|
+
this.n(content);
|
|
2415
|
+
if (!scanner.b("?>")) throw this.a("Unterminated processing instruction");
|
|
2416
|
+
return addToDocument ? this.k(new XmlProcessingInstruction(name, normalizeLineBreaks(content)), startIndex) : true;
|
|
2417
|
+
}
|
|
2418
|
+
/**
|
|
2419
|
+
* Consumes a prolog if possible.
|
|
2420
|
+
*
|
|
2421
|
+
* @returns Whether a prolog was consumed.
|
|
2422
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#sec-prolog-dtd
|
|
2423
|
+
*/
|
|
2424
|
+
R() {
|
|
2425
|
+
let { c: scanner } = this;
|
|
2426
|
+
let startIndex = scanner.d;
|
|
2427
|
+
this.S();
|
|
2428
|
+
while (this.B());
|
|
2429
|
+
if (this.M()) while (this.B());
|
|
2430
|
+
return startIndex < scanner.d;
|
|
2431
|
+
}
|
|
2432
|
+
/**
|
|
2433
|
+
* Consumes a public identifier literal if possible.
|
|
2434
|
+
*
|
|
2435
|
+
* @returns
|
|
2436
|
+
* Value of the public identifier literal minus quotes, or `false` if
|
|
2437
|
+
* nothing was consumed. An empty string indicates that a public id literal
|
|
2438
|
+
* was consumed but was empty.
|
|
2439
|
+
*
|
|
2440
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-PubidLiteral
|
|
2441
|
+
*/
|
|
2442
|
+
N() {
|
|
2443
|
+
let startIndex = this.c.d;
|
|
2444
|
+
let value = this.r();
|
|
2445
|
+
if (value !== false && !/^[-\x20\r\na-zA-Z0-9'()+,./:=?;!*#@$_%]*$/.test(value)) {
|
|
2446
|
+
this.c.j(startIndex);
|
|
2447
|
+
throw this.a("Invalid character in public identifier");
|
|
2448
|
+
}
|
|
2449
|
+
return value;
|
|
2450
|
+
}
|
|
2451
|
+
/**
|
|
2452
|
+
* Consumes a reference if possible.
|
|
2453
|
+
*
|
|
2454
|
+
* This differs from `consumeContentReference()` in that a consumed reference
|
|
2455
|
+
* will be returned rather than added to the document.
|
|
2456
|
+
*
|
|
2457
|
+
* @returns
|
|
2458
|
+
* Parsed reference value, or `false` if nothing was consumed (to
|
|
2459
|
+
* distinguish from a reference that resolves to an empty string).
|
|
2460
|
+
*
|
|
2461
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-Reference
|
|
2462
|
+
*/
|
|
2463
|
+
E() {
|
|
2464
|
+
let { c: scanner } = this;
|
|
2465
|
+
if (!scanner.b("&")) return false;
|
|
2466
|
+
let ref = scanner.w(isReferenceChar);
|
|
2467
|
+
if (scanner.G() !== ";") throw this.a("Unterminated reference (a reference must end with `;`)");
|
|
2468
|
+
let parsedValue;
|
|
2469
|
+
if (ref[0] === "#") {
|
|
2470
|
+
let codePoint = ref[1] === "x" ? parseInt(ref.slice(2), 16) : parseInt(ref.slice(1), 10);
|
|
2471
|
+
if (isNaN(codePoint) || !/^#(?:x[0-9A-Fa-f]+|[0-9]+)$/.test(ref)) throw this.a("Invalid character reference");
|
|
2472
|
+
if (!isXmlCodePoint(codePoint)) throw this.a("Character reference resolves to an invalid character");
|
|
2473
|
+
parsedValue = String.fromCodePoint(codePoint);
|
|
2474
|
+
} else {
|
|
2475
|
+
parsedValue = predefinedEntities[ref];
|
|
2476
|
+
if (parsedValue === void 0) {
|
|
2477
|
+
let { ignoreUndefinedEntities, resolveUndefinedEntity } = this.g;
|
|
2478
|
+
let wrappedRef = `&${ref};`;
|
|
2479
|
+
if (resolveUndefinedEntity) {
|
|
2480
|
+
let resolvedValue = resolveUndefinedEntity(wrappedRef);
|
|
2481
|
+
if (resolvedValue !== null && resolvedValue !== void 0) {
|
|
2482
|
+
let type = typeof resolvedValue;
|
|
2483
|
+
if (type !== "string") throw new TypeError(`\`resolveUndefinedEntity()\` must return a string, \`null\`, or \`undefined\`, but returned a value of type ${type}`);
|
|
2484
|
+
return resolvedValue;
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2487
|
+
if (ignoreUndefinedEntities) return wrappedRef;
|
|
2488
|
+
scanner.j(-wrappedRef.length);
|
|
2489
|
+
throw this.a(`Named entity isn't defined: ${wrappedRef}`);
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
return parsedValue;
|
|
2493
|
+
}
|
|
2494
|
+
/**
|
|
2495
|
+
* Consumes a `SystemLiteral` if possible.
|
|
2496
|
+
*
|
|
2497
|
+
* A `SystemLiteral` is similar to an attribute value, but allows the
|
|
2498
|
+
* characters `<` and `&` and doesn't replace references.
|
|
2499
|
+
*
|
|
2500
|
+
* @returns
|
|
2501
|
+
* Value of the `SystemLiteral` minus quotes, or `false` if nothing was
|
|
2502
|
+
* consumed. An empty string indicates that a `SystemLiteral` was consumed
|
|
2503
|
+
* but was empty.
|
|
2504
|
+
*
|
|
2505
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-SystemLiteral
|
|
2506
|
+
*/
|
|
2507
|
+
r() {
|
|
2508
|
+
let { c: scanner } = this;
|
|
2509
|
+
let quote = scanner.b("\"") || scanner.b("'");
|
|
2510
|
+
if (!quote) return false;
|
|
2511
|
+
let value = scanner.s(quote);
|
|
2512
|
+
this.n(value);
|
|
2513
|
+
if (!scanner.b(quote)) throw this.a("Missing end quote");
|
|
2514
|
+
return value;
|
|
2515
|
+
}
|
|
2516
|
+
/**
|
|
2517
|
+
* Consumes one or more whitespace characters if possible.
|
|
2518
|
+
*
|
|
2519
|
+
* @returns Whether any whitespace characters were consumed.
|
|
2520
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#white
|
|
2521
|
+
*/
|
|
2522
|
+
e() {
|
|
2523
|
+
return !!this.c.w(isWhitespace);
|
|
2524
|
+
}
|
|
2525
|
+
/**
|
|
2526
|
+
* Consumes an XML declaration if possible.
|
|
2527
|
+
*
|
|
2528
|
+
* @returns Whether an XML declaration was consumed.
|
|
2529
|
+
* @see https://www.w3.org/TR/2008/REC-xml-20081126/#NT-XMLDecl
|
|
2530
|
+
*/
|
|
2531
|
+
S() {
|
|
2532
|
+
let { c: scanner } = this;
|
|
2533
|
+
let startIndex = scanner.d;
|
|
2534
|
+
if (!scanner.b("<?xml")) return false;
|
|
2535
|
+
if (isNameChar(scanner.i())) {
|
|
2536
|
+
scanner.j(startIndex);
|
|
2537
|
+
return false;
|
|
2538
|
+
}
|
|
2539
|
+
if (!this.e()) throw this.a("Invalid XML declaration");
|
|
2540
|
+
let version = !!scanner.b("version") && this.t() && this.r();
|
|
2541
|
+
if (version === false) throw this.a("XML version is missing or invalid");
|
|
2542
|
+
else if (!/^1\.[0-9]+$/.test(version)) throw this.a("Invalid character in version number");
|
|
2543
|
+
let encoding;
|
|
2544
|
+
let standalone;
|
|
2545
|
+
let hasWhitespace = this.e();
|
|
2546
|
+
if (hasWhitespace && scanner.b("encoding")) {
|
|
2547
|
+
let value = this.t() && this.r();
|
|
2548
|
+
if (value === false) throw this.a("Invalid or unclosed XML declaration");
|
|
2549
|
+
else if (!/^[A-Za-z][\w.-]*$/.test(value)) throw this.a("Invalid character in encoding name");
|
|
2550
|
+
encoding = value;
|
|
2551
|
+
hasWhitespace = this.e();
|
|
2552
|
+
}
|
|
2553
|
+
if (hasWhitespace && scanner.b("standalone")) {
|
|
2554
|
+
let value = this.t() && this.r();
|
|
2555
|
+
if (value === false) throw this.a("Invalid or unclosed XML declaration");
|
|
2556
|
+
else if (value !== "yes" && value !== "no") throw this.a("Only \"yes\" and \"no\" are permitted as values of `standalone`");
|
|
2557
|
+
standalone = value;
|
|
2558
|
+
this.e();
|
|
2559
|
+
}
|
|
2560
|
+
if (!scanner.b("?>")) throw this.a("Invalid or unclosed XML declaration");
|
|
2561
|
+
return this.g.preserveXmlDeclaration ? this.k(new XmlDeclaration(version, encoding, standalone), startIndex) : true;
|
|
2562
|
+
}
|
|
2563
|
+
/**
|
|
2564
|
+
* Returns an `XmlError` for the current scanner position.
|
|
2565
|
+
*/
|
|
2566
|
+
a(message) {
|
|
2567
|
+
let { c: scanner } = this;
|
|
2568
|
+
return new XmlError(message, scanner.d, scanner.h);
|
|
2569
|
+
}
|
|
2570
|
+
/**
|
|
2571
|
+
* Parses the XML input.
|
|
2572
|
+
*/
|
|
2573
|
+
parse() {
|
|
2574
|
+
this.c.b("");
|
|
2575
|
+
this.R();
|
|
2576
|
+
if (!this.F()) throw this.a("Root element is missing or invalid");
|
|
2577
|
+
while (this.B());
|
|
2578
|
+
if (!this.c.D) throw this.a("Extra content at the end of the document");
|
|
2579
|
+
}
|
|
2580
|
+
/**
|
|
2581
|
+
* Throws an invalid character error if any character in the given _string_
|
|
2582
|
+
* isn't a valid XML character.
|
|
2583
|
+
*/
|
|
2584
|
+
n(string) {
|
|
2585
|
+
let { length } = string;
|
|
2586
|
+
for (let i = 0; i < length; ++i) {
|
|
2587
|
+
let cp = string.codePointAt(i);
|
|
2588
|
+
if (!isXmlCodePoint(cp)) {
|
|
2589
|
+
this.c.j(-[...string.slice(i)].length);
|
|
2590
|
+
throw this.a("Invalid character");
|
|
2591
|
+
}
|
|
2592
|
+
if (cp > 65535) i += 1;
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
2595
|
+
};
|
|
2596
|
+
function normalizeLineBreaks(text) {
|
|
2597
|
+
return text.includes("\r") ? text.replace(lineBreakToNormalize, "\n") : text;
|
|
2598
|
+
}
|
|
2599
|
+
function parseXml(xml, options) {
|
|
2600
|
+
return new Parser(xml, options).document;
|
|
2601
|
+
}
|
|
2602
|
+
// ../adapter-node/src/xml.ts
|
|
2603
|
+
const localName = (name) => name.includes(":") ? name.split(":")[1] : name;
|
|
2604
|
+
const isElement = (node) => node.type === "element";
|
|
2605
|
+
const isText = (node) => node.type === "text";
|
|
2606
|
+
const child = (el, name) => {
|
|
2607
|
+
for (const node of el.children) if (isElement(node) && localName(node.name) === name) return node;
|
|
2608
|
+
};
|
|
2609
|
+
const textOf = (el) => {
|
|
2610
|
+
if (el === void 0) return;
|
|
2611
|
+
let result = "";
|
|
2612
|
+
for (const node of el.children) if (isText(node)) result += node.text;
|
|
2613
|
+
return result === "" ? void 0 : result;
|
|
2614
|
+
};
|
|
2615
|
+
const docRoot = (doc) => doc.root ?? void 0;
|
|
2616
|
+
const xmlVal = (el, name) => {
|
|
2617
|
+
const v = el !== void 0 ? textOf(child(el, name)) : void 0;
|
|
2618
|
+
if (v === void 0 || v === "") return;
|
|
2619
|
+
try {
|
|
2620
|
+
return decodeURIComponent(v.replace(/\+/g, " "));
|
|
2621
|
+
} catch {
|
|
2622
|
+
throw new BaerlyError("InvalidResponse", `Malformed percent-escape in S3 <${name}>: ${v}`);
|
|
2623
|
+
}
|
|
2624
|
+
};
|
|
2625
|
+
const rawXmlVal = (el, name) => {
|
|
2626
|
+
return el !== void 0 ? textOf(child(el, name)) : void 0;
|
|
2627
|
+
};
|
|
2628
|
+
/**
|
|
2629
|
+
* Build a {@link ParsedS3Error} from the element that directly carries
|
|
2630
|
+
* the `<Code>` / `<Message>` children — `<Error>` for S3, the inner
|
|
2631
|
+
* `<Error>` for STS. Returns `undefined` when neither child is present
|
|
2632
|
+
* so callers can fall back to the raw status. Shared by
|
|
2633
|
+
* {@link parseS3Error} and {@link parseStsError}, which differ only in
|
|
2634
|
+
* how they locate that element.
|
|
2635
|
+
*/
|
|
2636
|
+
const codeMessageFrom = (el) => {
|
|
2637
|
+
const code = rawXmlVal(el, "Code");
|
|
2638
|
+
const message = rawXmlVal(el, "Message");
|
|
2639
|
+
if (code === void 0 && message === void 0) return;
|
|
2640
|
+
return {
|
|
2641
|
+
...code !== void 0 && { Code: code },
|
|
2642
|
+
...message !== void 0 && { Message: message }
|
|
2643
|
+
};
|
|
2644
|
+
};
|
|
2645
|
+
/**
|
|
2646
|
+
* Parse an S3 `<Error><Code>…</Code><Message>…</Message></Error>`
|
|
2647
|
+
* body. Returns `undefined` when the body is not a recognizable S3
|
|
2648
|
+
* error document (empty, HTML, a success payload, or a DTD), so
|
|
2649
|
+
* callers can fall back to the raw response text. This never throws:
|
|
2650
|
+
* it runs on a path that is already reporting an error.
|
|
2651
|
+
*/
|
|
2652
|
+
const parseS3Error = (xml) => {
|
|
2653
|
+
if (!/<Error\b/i.test(xml) || /<!DOCTYPE\b/i.test(xml)) return;
|
|
2654
|
+
let root;
|
|
2655
|
+
try {
|
|
2656
|
+
root = docRoot(parseXml(xml));
|
|
2657
|
+
} catch {
|
|
2658
|
+
return;
|
|
2659
|
+
}
|
|
2660
|
+
if (root === void 0) return;
|
|
2661
|
+
if (localName(root.name) !== "Error") return;
|
|
2662
|
+
return codeMessageFrom(root);
|
|
2663
|
+
};
|
|
2664
|
+
const parseLastModified = (lm) => {
|
|
2665
|
+
if (lm === void 0) return;
|
|
2666
|
+
const d = new Date(lm);
|
|
2667
|
+
return Number.isNaN(d.getTime()) ? void 0 : d;
|
|
2668
|
+
};
|
|
2669
|
+
const parseListObjectsV2CommandOutput = (xml) => {
|
|
2670
|
+
if (/<!DOCTYPE\b/i.test(xml)) throw new BaerlyError("InvalidResponse", "DTD not allowed in S3 XML responses");
|
|
2671
|
+
let root;
|
|
2672
|
+
try {
|
|
2673
|
+
root = docRoot(parseXml(xml));
|
|
2674
|
+
} catch {
|
|
2675
|
+
throw new BaerlyError("InvalidResponse", `Invalid XML: ${xml}`);
|
|
2676
|
+
}
|
|
2677
|
+
if (root === void 0) return {};
|
|
2678
|
+
if (localName(root.name) !== "ListBucketResult") return {};
|
|
2679
|
+
return {
|
|
2680
|
+
Contents: root.children.filter(isElement).filter((c) => localName(c.name) === "Contents").map((contentEl) => {
|
|
2681
|
+
return {
|
|
2682
|
+
ETag: rawXmlVal(contentEl, "ETag"),
|
|
2683
|
+
Key: xmlVal(contentEl, "Key"),
|
|
2684
|
+
LastModified: parseLastModified(rawXmlVal(contentEl, "LastModified")),
|
|
2685
|
+
Generation: rawXmlVal(contentEl, "Generation")
|
|
2686
|
+
};
|
|
2687
|
+
}),
|
|
2688
|
+
NextContinuationToken: rawXmlVal(root, "NextContinuationToken")
|
|
2689
|
+
};
|
|
2690
|
+
};
|
|
2691
|
+
// ../adapter-node/src/http-transport.ts
|
|
2692
|
+
/**
|
|
2693
|
+
* Parse an HTTP `Retry-After` header into a non-negative seconds value,
|
|
2694
|
+
* clamped to {@link RETRY_AFTER_MAX_SECONDS}. Returns `undefined` if
|
|
2695
|
+
* the header is absent, malformed, or whitespace.
|
|
2696
|
+
*
|
|
2697
|
+
* RFC 7231 §7.1.3 admits two forms: delta-seconds (non-negative
|
|
2698
|
+
* integer) and HTTP-date. Both are accepted; dates in the past return
|
|
2699
|
+
* `0`. Fractional seconds, negatives, and arbitrary strings reject.
|
|
2700
|
+
*
|
|
2701
|
+
* The `now` injection point exists for the unit test; production
|
|
2702
|
+
* callers should pass nothing.
|
|
2703
|
+
*/
|
|
2704
|
+
function parseRetryAfter(header, now = Date.now) {
|
|
2705
|
+
if (header === null) return;
|
|
2706
|
+
const trimmed = header.trim();
|
|
2707
|
+
if (trimmed === "") return;
|
|
2708
|
+
if (/^\d+$/.test(trimmed)) {
|
|
2709
|
+
const n = Number.parseInt(trimmed, 10);
|
|
2710
|
+
return Number.isFinite(n) ? Math.min(n, 60) : void 0;
|
|
2711
|
+
}
|
|
2712
|
+
if (!/[a-z]/i.test(trimmed)) return;
|
|
2713
|
+
const t = Date.parse(trimmed);
|
|
2714
|
+
if (Number.isNaN(t)) return;
|
|
2715
|
+
const seconds = Math.max(0, Math.ceil((t - now()) / 1e3));
|
|
2716
|
+
return Math.min(seconds, 60);
|
|
2717
|
+
}
|
|
2718
|
+
const retryAfterCause = (res) => {
|
|
2719
|
+
const hint = parseRetryAfter(res.headers.get("Retry-After"));
|
|
2720
|
+
return hint !== void 0 ? { retryAfterSeconds: hint } : {};
|
|
2721
|
+
};
|
|
2722
|
+
const xmlErrorDetail = (status, body) => {
|
|
2723
|
+
const parsed = parseS3Error(body);
|
|
2724
|
+
if (parsed !== void 0) {
|
|
2725
|
+
const code = parsed.Code ?? "UnknownError";
|
|
2726
|
+
return parsed.Message !== void 0 ? `${status} ${code}: ${parsed.Message}` : `${status} ${code}`;
|
|
2727
|
+
}
|
|
2728
|
+
return `${status} ${body}`;
|
|
2729
|
+
};
|
|
2730
|
+
/**
|
|
2731
|
+
* Shared status→{@link BaerlyError} mapping for the S3/GCS XML adapters, so
|
|
2732
|
+
* the retryable set (`429`/`≥500`) can't drift between them. Store-specific
|
|
2733
|
+
* statuses (GCS `412`; S3 `409`/`404`-with-`If-Match`) are handled inline
|
|
2734
|
+
* first; the rest route here: `403`→AccessDenied (permanent), `429`/`≥500`→
|
|
2735
|
+
* retryable NetworkError carrying `{status, retryAfterSeconds?}`, else
|
|
2736
|
+
* InvalidResponse. Callers `throw` the result; consumes `res.text()` on the
|
|
2737
|
+
* non-`403` paths. A throttle (`429`, or S3's `503 SlowDown`) also carries
|
|
2738
|
+
* `throttled: true`: the store refused the request without applying it.
|
|
2739
|
+
*/
|
|
2740
|
+
const mapStorageError = async (res, verb, key) => {
|
|
2741
|
+
if (res.status === 403) return new BaerlyError("AccessDenied", `${verb} ${key}: 403`);
|
|
2742
|
+
if (res.status === 429 || res.status >= 500) {
|
|
2743
|
+
const body = await res.text();
|
|
2744
|
+
return new BaerlyError("NetworkError", `${verb} ${key}: ${res.status} ${body}`, { cause: {
|
|
2745
|
+
status: res.status,
|
|
2746
|
+
...retryAfterCause(res),
|
|
2747
|
+
...isThrottle(res.status, body) && { throttled: true }
|
|
2748
|
+
} });
|
|
2749
|
+
}
|
|
2750
|
+
return new BaerlyError("InvalidResponse", `${verb} ${key}: ${xmlErrorDetail(res.status, await res.text())}`);
|
|
2751
|
+
};
|
|
2752
|
+
const isThrottle = (status, body) => status === 429 || status === 503 && parseS3Error(body)?.Code === "SlowDown";
|
|
2753
|
+
/**
|
|
2754
|
+
* S3 error codes for an expired STS session token. S3 answers these with
|
|
2755
|
+
* HTTP 400, not 403; AWS SDKs treat them as a signal to refresh credentials.
|
|
2756
|
+
*/
|
|
2757
|
+
const EXPIRED_TOKEN_CODES = /* @__PURE__ */ new Set(["ExpiredToken", "TokenRefreshRequired"]);
|
|
2758
|
+
/**
|
|
2759
|
+
* Read the S3 expired-token code from a `400` response, or `undefined` for
|
|
2760
|
+
* any other response. Reads a clone, so `res` stays readable for
|
|
2761
|
+
* {@link mapS3StorageError}.
|
|
2762
|
+
*/
|
|
2763
|
+
const s3ExpiredTokenCode = async (res) => {
|
|
2764
|
+
if (res.status !== 400) return;
|
|
2765
|
+
const code = parseS3Error(await res.clone().text())?.Code;
|
|
2766
|
+
return code !== void 0 && EXPIRED_TOKEN_CODES.has(code) ? code : void 0;
|
|
2767
|
+
};
|
|
2768
|
+
/**
|
|
2769
|
+
* {@link mapStorageError} for the S3 adapter: a `400` carrying an
|
|
2770
|
+
* expired-token code is a credentials failure, so it maps to the permanent
|
|
2771
|
+
* `AccessDenied` whose message carries the S3 code, the state and the remedy;
|
|
2772
|
+
* every other response maps as {@link mapStorageError} does.
|
|
2773
|
+
*/
|
|
2774
|
+
const mapS3StorageError = async (res, verb, key) => {
|
|
2775
|
+
if (await s3ExpiredTokenCode(res) !== void 0) return new BaerlyError("AccessDenied", `${verb} ${key}: ${xmlErrorDetail(400, await res.text())} — the credentials provider returned expired credentials even after a refresh; it should return fresh credentials with an accurate \`expiration\`.`, { resolution: "Return fresh credentials with an accurate `expiration` from the credentials provider; static session credentials must be replaced." });
|
|
2776
|
+
return mapStorageError(res, verb, key);
|
|
2777
|
+
};
|
|
2778
|
+
// ../adapter-node/src/s3-http.ts
|
|
2779
|
+
/**
|
|
2780
|
+
* Drop a weak-validator prefix from a response ETag. R2's S3 API compresses
|
|
2781
|
+
* `application/json` GET responses over ~159 bytes and returns the stored
|
|
2782
|
+
* ETag as `W/"<md5>"`; `If-Match` with that weak value always fails, while
|
|
2783
|
+
* the bare `"<md5>"` matches the stored object (and a wrong one is still
|
|
2784
|
+
* refused). `current.json` and `gc/pending.json` advance by exactly that
|
|
2785
|
+
* get-then-`ifMatch`, so without this no fold lands on R2 over S3.
|
|
2786
|
+
*/
|
|
2787
|
+
const strongEtag = (etag) => etag.startsWith("W/") ? etag.slice(2) : etag;
|
|
2788
|
+
let attachInvalidate;
|
|
2789
|
+
/**
|
|
2790
|
+
* Internal wiring (not re-exported from the package entries): make `storage`
|
|
2791
|
+
* drop the credentials that signed a request (`refreshingSigner(...).invalidate`)
|
|
2792
|
+
* and resend once per storage call when S3 answers `400 ExpiredToken` /
|
|
2793
|
+
* `TokenRefreshRequired`. The resend is charged to the request ledger like any
|
|
2794
|
+
* attempt; a second rejection, or storage without this wiring, is
|
|
2795
|
+
* `AccessDenied` naming the S3 code.
|
|
2796
|
+
*/
|
|
2797
|
+
const withCredentialInvalidation = (storage, invalidate) => {
|
|
2798
|
+
attachInvalidate(storage, invalidate);
|
|
2799
|
+
return storage;
|
|
2800
|
+
};
|
|
2801
|
+
/**
|
|
2802
|
+
* `Storage` impl that speaks the S3 REST API directly. Works against
|
|
2803
|
+
* any S3-compatible endpoint — AWS S3, R2 (S3-compat), Minio. Does not
|
|
2804
|
+
* depend on any AWS SDK. GCS is NOT supported over this S3-interop path
|
|
2805
|
+
* (its conditional-write semantics can't linearize the commit log); use
|
|
2806
|
+
* the native `GcsHttpStorage` adapter instead.
|
|
2807
|
+
*
|
|
2808
|
+
* Authentication is plugged in via the `sign` callback; this class
|
|
2809
|
+
* itself does no SigV4. For Cloudflare Workers, the R2 binding fast
|
|
2810
|
+
* path lives in `@baerly/adapter-cloudflare`.
|
|
2811
|
+
*/
|
|
2812
|
+
var S3HttpStorage = class {
|
|
2813
|
+
#endpoint;
|
|
2814
|
+
#bucket;
|
|
2815
|
+
#fetch;
|
|
2816
|
+
#sign;
|
|
2817
|
+
#invalidate;
|
|
2818
|
+
#retries;
|
|
2819
|
+
#backoffMs;
|
|
2820
|
+
constructor(options) {
|
|
2821
|
+
this.#endpoint = options.endpoint.replace(/\/+$/, "");
|
|
2822
|
+
this.#bucket = options.bucket;
|
|
2823
|
+
this.#fetch = options.fetch ?? globalThis.fetch.bind(globalThis);
|
|
2824
|
+
this.#sign = options.sign;
|
|
2825
|
+
this.#retries = options.retries;
|
|
2826
|
+
this.#backoffMs = options.backoffMs;
|
|
2827
|
+
}
|
|
2828
|
+
static {
|
|
2829
|
+
attachInvalidate = (storage, invalidate) => {
|
|
2830
|
+
storage.#invalidate = invalidate;
|
|
2831
|
+
};
|
|
2832
|
+
}
|
|
2833
|
+
#objectUrl(key) {
|
|
2834
|
+
assertValidStorageKey(key);
|
|
2835
|
+
return `${this.#endpoint}/${this.#bucket}/${encodeURIComponent(key)}`;
|
|
2836
|
+
}
|
|
2837
|
+
async #send(req) {
|
|
2838
|
+
const signed = this.#sign ? await this.#sign(req) : req;
|
|
2839
|
+
try {
|
|
2840
|
+
return {
|
|
2841
|
+
res: await this.#fetch(signed),
|
|
2842
|
+
signed
|
|
2843
|
+
};
|
|
2844
|
+
} catch (error) {
|
|
2845
|
+
if (error instanceof BaerlyError || signed.signal.aborted) throw error;
|
|
2846
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
2847
|
+
throw new BaerlyError("NetworkError", msg, { cause: error });
|
|
2848
|
+
}
|
|
2849
|
+
}
|
|
2850
|
+
/**
|
|
2851
|
+
* Send one request. An expired-session-token `400` drops the credentials
|
|
2852
|
+
* that signed it and resends once per storage call (`call` is shared by the
|
|
2853
|
+
* attempts of one `#retry`, so the retry policy cannot add a second
|
|
2854
|
+
* refresh). The resend is charged to the ledger here because it is a
|
|
2855
|
+
* request `retryStorageCall` did not count. A second expired-token
|
|
2856
|
+
* response is returned as is and maps to `AccessDenied`.
|
|
2857
|
+
*/
|
|
2858
|
+
async #dispatch(make, call) {
|
|
2859
|
+
const first = await this.#send(make());
|
|
2860
|
+
if (this.#invalidate === void 0 || call.refreshed || await s3ExpiredTokenCode(first.res) === void 0) return first.res;
|
|
2861
|
+
await first.res.body?.cancel();
|
|
2862
|
+
first.signed.signal.throwIfAborted();
|
|
2863
|
+
call.refreshed = true;
|
|
2864
|
+
this.#invalidate(first.signed);
|
|
2865
|
+
currentRequestLedger()?.acquire("external", currentRequestHold());
|
|
2866
|
+
return (await this.#send(make())).res;
|
|
2867
|
+
}
|
|
2868
|
+
#retry(fn, opts) {
|
|
2869
|
+
const call = { refreshed: false };
|
|
2870
|
+
return retryStorageCall(() => fn(call), {
|
|
2871
|
+
...this.#retries !== void 0 && { retries: this.#retries },
|
|
2872
|
+
...this.#backoffMs !== void 0 && { backoffMs: this.#backoffMs },
|
|
2873
|
+
conditionalCreate: opts?.ifNoneMatch === "*",
|
|
2874
|
+
compareAndSwap: opts?.ifMatch !== void 0 && opts.ifNoneMatch !== "*",
|
|
2875
|
+
pool: "external",
|
|
2876
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
2877
|
+
});
|
|
2878
|
+
}
|
|
2879
|
+
async get(key, opts) {
|
|
2880
|
+
opts?.signal?.throwIfAborted();
|
|
2881
|
+
const url = this.#objectUrl(key);
|
|
2882
|
+
return this.#retry(async (call) => {
|
|
2883
|
+
const res = await this.#dispatch(() => new Request(url, {
|
|
2884
|
+
method: "GET",
|
|
2885
|
+
signal: opts?.signal ?? null
|
|
2886
|
+
}), call);
|
|
2887
|
+
switch (res.status) {
|
|
2888
|
+
case 200: {
|
|
2889
|
+
const etag = res.headers.get("ETag");
|
|
2890
|
+
if (etag === null) throw new BaerlyError("InvalidResponse", `GET ${key}: missing ETag`);
|
|
2891
|
+
return {
|
|
2892
|
+
body: new Uint8Array(await res.arrayBuffer()),
|
|
2893
|
+
etag: strongEtag(etag)
|
|
2894
|
+
};
|
|
2895
|
+
}
|
|
2896
|
+
case 404: return null;
|
|
2897
|
+
default: throw await mapS3StorageError(res, "GET", key);
|
|
2898
|
+
}
|
|
2899
|
+
}, opts);
|
|
2900
|
+
}
|
|
2901
|
+
async put(key, body, opts) {
|
|
2902
|
+
opts?.signal?.throwIfAborted();
|
|
2903
|
+
const url = this.#objectUrl(key);
|
|
2904
|
+
const headers = new Headers();
|
|
2905
|
+
headers.set("Content-Type", opts?.contentType ?? "application/octet-stream");
|
|
2906
|
+
if (opts?.ifMatch !== void 0) headers.set("If-Match", opts.ifMatch);
|
|
2907
|
+
if (opts?.ifNoneMatch === "*") headers.set("If-None-Match", "*");
|
|
2908
|
+
return this.#retry(async (call) => {
|
|
2909
|
+
const res = await this.#dispatch(() => new Request(url, {
|
|
2910
|
+
method: "PUT",
|
|
2911
|
+
headers,
|
|
2912
|
+
body,
|
|
2913
|
+
signal: opts?.signal ?? null
|
|
2914
|
+
}), call);
|
|
2915
|
+
if (res.status === 412) throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed`);
|
|
2916
|
+
if (res.status === 409 && (opts?.ifNoneMatch === "*" || opts?.ifMatch !== void 0)) throw new BaerlyError("NetworkError", `PUT ${key}: 409 ConditionalRequestConflict (contended conditional write; retryable)`, { cause: {
|
|
2917
|
+
status: res.status,
|
|
2918
|
+
...retryAfterCause(res)
|
|
2919
|
+
} });
|
|
2920
|
+
if (res.status === 404 && opts?.ifMatch !== void 0) throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed (ifMatch=${opts.ifMatch} but key does not exist)`);
|
|
2921
|
+
if (res.status !== 200 && res.status !== 204) throw await mapS3StorageError(res, "PUT", key);
|
|
2922
|
+
const etag = res.headers.get("ETag");
|
|
2923
|
+
if (etag === null) throw new BaerlyError("InvalidResponse", `PUT ${key}: missing ETag`);
|
|
2924
|
+
const dateStr = res.headers.get("Date");
|
|
2925
|
+
const result = { etag: strongEtag(etag) };
|
|
2926
|
+
if (dateStr !== null) result.serverDate = new Date(dateStr);
|
|
2927
|
+
return result;
|
|
2928
|
+
}, opts);
|
|
2929
|
+
}
|
|
2930
|
+
async delete(key, opts) {
|
|
2931
|
+
opts?.signal?.throwIfAborted();
|
|
2932
|
+
const url = this.#objectUrl(key);
|
|
2933
|
+
await this.#retry(async (call) => {
|
|
2934
|
+
const res = await this.#dispatch(() => new Request(url, {
|
|
2935
|
+
method: "DELETE",
|
|
2936
|
+
signal: opts?.signal ?? null
|
|
2937
|
+
}), call);
|
|
2938
|
+
if (res.status !== 200 && res.status !== 204 && res.status !== 404) throw await mapS3StorageError(res, "DELETE", key);
|
|
2939
|
+
}, opts);
|
|
2940
|
+
}
|
|
2941
|
+
async *list(prefix, opts) {
|
|
2942
|
+
opts?.signal?.throwIfAborted();
|
|
2943
|
+
let yielded = 0;
|
|
2944
|
+
let continuationToken;
|
|
2945
|
+
const startAfter = opts?.startAfter ?? "";
|
|
2946
|
+
while (true) {
|
|
2947
|
+
opts?.signal?.throwIfAborted();
|
|
2948
|
+
const params = new URLSearchParams();
|
|
2949
|
+
params.set("list-type", "2");
|
|
2950
|
+
params.set("prefix", prefix);
|
|
2951
|
+
params.set("encoding-type", "url");
|
|
2952
|
+
if (continuationToken !== void 0) params.set("continuation-token", continuationToken);
|
|
2953
|
+
else if (startAfter !== "") params.set("start-after", startAfter);
|
|
2954
|
+
if (opts?.maxKeys !== void 0) {
|
|
2955
|
+
const remaining = opts.maxKeys - yielded;
|
|
2956
|
+
params.set("max-keys", String(Math.min(remaining, 1e3)));
|
|
2957
|
+
}
|
|
2958
|
+
const url = `${this.#endpoint}/${this.#bucket}/?${params.toString()}`;
|
|
2959
|
+
const parsed = parseListObjectsV2CommandOutput(await this.#retry(async (call) => {
|
|
2960
|
+
const res = await this.#dispatch(() => new Request(url, {
|
|
2961
|
+
method: "GET",
|
|
2962
|
+
signal: opts?.signal ?? null
|
|
2963
|
+
}), call);
|
|
2964
|
+
if (res.status !== 200) throw await mapS3StorageError(res, "LIST", prefix);
|
|
2965
|
+
return res.text();
|
|
2966
|
+
}, opts));
|
|
2967
|
+
for (const entry of parsed.Contents ?? []) {
|
|
2968
|
+
if (entry.Key === void 0) continue;
|
|
2969
|
+
yield {
|
|
2970
|
+
key: entry.Key,
|
|
2971
|
+
etag: entry.ETag ?? ""
|
|
2972
|
+
};
|
|
2973
|
+
yielded += 1;
|
|
2974
|
+
if (opts?.maxKeys !== void 0 && yielded >= opts.maxKeys) return;
|
|
2975
|
+
}
|
|
2976
|
+
continuationToken = parsed.NextContinuationToken;
|
|
2977
|
+
if (continuationToken === void 0 || continuationToken === "") return;
|
|
2978
|
+
}
|
|
2979
|
+
}
|
|
2980
|
+
};
|
|
2981
|
+
// ../adapter-node/src/gcs-http.ts
|
|
2982
|
+
/** Default GCS native XML API endpoint. */
|
|
2983
|
+
const DEFAULT_GCS_ENDPOINT = "https://storage.googleapis.com";
|
|
2984
|
+
/**
|
|
2985
|
+
* `Storage` impl that speaks the GCS native XML API and drives GCS's own
|
|
2986
|
+
* `x-goog-if-generation-match` conditional writes. Unlike GCS's S3-interop
|
|
2987
|
+
* endpoint (which treats `If-Match`/`If-None-Match` as read-scoped and can
|
|
2988
|
+
* never linearize the log), the native API returns 412 on any precondition
|
|
2989
|
+
* miss and the object `generation` in `x-goog-generation`. That generation
|
|
2990
|
+
* is carried verbatim in the opaque `Storage` etag.
|
|
2991
|
+
*
|
|
2992
|
+
* Authentication is plugged in via `sign` (GOOG4-HMAC-SHA256); this class
|
|
2993
|
+
* does no signing itself.
|
|
2994
|
+
*/
|
|
2995
|
+
var GcsHttpStorage = class {
|
|
2996
|
+
#endpoint;
|
|
2997
|
+
#bucket;
|
|
2998
|
+
#fetch;
|
|
2999
|
+
#sign;
|
|
3000
|
+
#retries;
|
|
3001
|
+
#backoffMs;
|
|
3002
|
+
constructor(options) {
|
|
3003
|
+
this.#endpoint = (options.endpoint ?? "https://storage.googleapis.com").replace(/\/+$/, "");
|
|
3004
|
+
this.#bucket = options.bucket;
|
|
3005
|
+
this.#fetch = options.fetch ?? globalThis.fetch.bind(globalThis);
|
|
3006
|
+
this.#sign = options.sign;
|
|
3007
|
+
this.#retries = options.retries;
|
|
3008
|
+
this.#backoffMs = options.backoffMs;
|
|
3009
|
+
}
|
|
3010
|
+
#objectUrl(key) {
|
|
3011
|
+
assertValidStorageKey(key);
|
|
3012
|
+
return `${this.#endpoint}/${this.#bucket}/${encodeURIComponent(key)}`;
|
|
3013
|
+
}
|
|
3014
|
+
async #dispatch(req) {
|
|
3015
|
+
const signed = this.#sign ? await this.#sign(req) : req;
|
|
3016
|
+
try {
|
|
3017
|
+
return await this.#fetch(signed);
|
|
3018
|
+
} catch (error) {
|
|
3019
|
+
if (error instanceof BaerlyError || signed.signal.aborted) throw error;
|
|
3020
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
3021
|
+
throw new BaerlyError("NetworkError", msg, { cause: error });
|
|
3022
|
+
}
|
|
3023
|
+
}
|
|
3024
|
+
#retry(fn, opts) {
|
|
3025
|
+
return retryStorageCall(fn, {
|
|
3026
|
+
...this.#retries !== void 0 && { retries: this.#retries },
|
|
3027
|
+
...this.#backoffMs !== void 0 && { backoffMs: this.#backoffMs },
|
|
3028
|
+
conditionalCreate: opts?.ifNoneMatch === "*",
|
|
3029
|
+
compareAndSwap: opts?.ifMatch !== void 0 && opts.ifNoneMatch !== "*",
|
|
3030
|
+
pool: "external",
|
|
3031
|
+
...opts?.signal !== void 0 && { signal: opts.signal }
|
|
3032
|
+
});
|
|
3033
|
+
}
|
|
3034
|
+
async get(key, opts) {
|
|
3035
|
+
opts?.signal?.throwIfAborted();
|
|
3036
|
+
const url = this.#objectUrl(key);
|
|
3037
|
+
return this.#retry(async () => {
|
|
3038
|
+
const res = await this.#dispatch(new Request(url, {
|
|
3039
|
+
method: "GET",
|
|
3040
|
+
signal: opts?.signal ?? null
|
|
3041
|
+
}));
|
|
3042
|
+
switch (res.status) {
|
|
3043
|
+
case 200: {
|
|
3044
|
+
const etag = res.headers.get("x-goog-generation");
|
|
3045
|
+
if (etag === null) throw new BaerlyError("InvalidResponse", `GET ${key}: missing x-goog-generation`);
|
|
3046
|
+
return {
|
|
3047
|
+
body: new Uint8Array(await res.arrayBuffer()),
|
|
3048
|
+
etag
|
|
3049
|
+
};
|
|
3050
|
+
}
|
|
3051
|
+
case 404: return null;
|
|
3052
|
+
default: throw await mapStorageError(res, "GET", key);
|
|
3053
|
+
}
|
|
3054
|
+
}, opts);
|
|
3055
|
+
}
|
|
3056
|
+
async put(key, body, opts) {
|
|
3057
|
+
opts?.signal?.throwIfAborted();
|
|
3058
|
+
const url = this.#objectUrl(key);
|
|
3059
|
+
const headers = new Headers();
|
|
3060
|
+
headers.set("Content-Type", opts?.contentType ?? "application/octet-stream");
|
|
3061
|
+
if (opts?.ifNoneMatch === "*") headers.set("x-goog-if-generation-match", "0");
|
|
3062
|
+
else if (opts?.ifMatch !== void 0) headers.set("x-goog-if-generation-match", opts.ifMatch);
|
|
3063
|
+
return this.#retry(async () => {
|
|
3064
|
+
const res = await this.#dispatch(new Request(url, {
|
|
3065
|
+
method: "PUT",
|
|
3066
|
+
headers,
|
|
3067
|
+
body,
|
|
3068
|
+
signal: opts?.signal ?? null
|
|
3069
|
+
}));
|
|
3070
|
+
if (res.status === 412) throw new BaerlyError("WriteConflict", `PUT ${key}: precondition failed`);
|
|
3071
|
+
if (res.status !== 200 && res.status !== 204) throw await mapStorageError(res, "PUT", key);
|
|
3072
|
+
const etag = res.headers.get("x-goog-generation");
|
|
3073
|
+
if (etag === null) throw new BaerlyError("InvalidResponse", `PUT ${key}: missing x-goog-generation`);
|
|
3074
|
+
const dateStr = res.headers.get("Date");
|
|
3075
|
+
const result = { etag };
|
|
3076
|
+
if (dateStr !== null) result.serverDate = new Date(dateStr);
|
|
3077
|
+
return result;
|
|
3078
|
+
}, opts);
|
|
3079
|
+
}
|
|
3080
|
+
async delete(key, opts) {
|
|
3081
|
+
opts?.signal?.throwIfAborted();
|
|
3082
|
+
const url = this.#objectUrl(key);
|
|
3083
|
+
await this.#retry(async () => {
|
|
3084
|
+
const res = await this.#dispatch(new Request(url, {
|
|
3085
|
+
method: "DELETE",
|
|
3086
|
+
signal: opts?.signal ?? null
|
|
3087
|
+
}));
|
|
3088
|
+
if (res.status !== 200 && res.status !== 204 && res.status !== 404) throw await mapStorageError(res, "DELETE", key);
|
|
3089
|
+
}, opts);
|
|
3090
|
+
}
|
|
3091
|
+
async *list(prefix, opts) {
|
|
3092
|
+
opts?.signal?.throwIfAborted();
|
|
3093
|
+
let yielded = 0;
|
|
3094
|
+
let continuationToken;
|
|
3095
|
+
const startAfter = opts?.startAfter ?? "";
|
|
3096
|
+
while (true) {
|
|
3097
|
+
opts?.signal?.throwIfAborted();
|
|
3098
|
+
const params = new URLSearchParams();
|
|
3099
|
+
params.set("list-type", "2");
|
|
3100
|
+
params.set("prefix", prefix);
|
|
3101
|
+
params.set("encoding-type", "url");
|
|
3102
|
+
if (continuationToken !== void 0) params.set("continuation-token", continuationToken);
|
|
3103
|
+
else if (startAfter !== "") params.set("start-after", startAfter);
|
|
3104
|
+
if (opts?.maxKeys !== void 0) {
|
|
3105
|
+
const remaining = opts.maxKeys - yielded;
|
|
3106
|
+
params.set("max-keys", String(Math.min(remaining, 1e3)));
|
|
3107
|
+
}
|
|
3108
|
+
const url = `${this.#endpoint}/${this.#bucket}/?${params.toString()}`;
|
|
3109
|
+
const parsed = parseListObjectsV2CommandOutput(await this.#retry(async () => {
|
|
3110
|
+
const res = await this.#dispatch(new Request(url, {
|
|
3111
|
+
method: "GET",
|
|
3112
|
+
signal: opts?.signal ?? null
|
|
3113
|
+
}));
|
|
3114
|
+
if (res.status !== 200) throw await mapStorageError(res, "LIST", prefix);
|
|
3115
|
+
return res.text();
|
|
3116
|
+
}, opts));
|
|
3117
|
+
for (const entry of parsed.Contents ?? []) {
|
|
3118
|
+
if (entry.Key === void 0) continue;
|
|
3119
|
+
yield {
|
|
3120
|
+
key: entry.Key,
|
|
3121
|
+
etag: entry.Generation ?? ""
|
|
3122
|
+
};
|
|
3123
|
+
yielded += 1;
|
|
3124
|
+
if (opts?.maxKeys !== void 0 && yielded >= opts.maxKeys) return;
|
|
3125
|
+
}
|
|
3126
|
+
continuationToken = parsed.NextContinuationToken;
|
|
3127
|
+
if (continuationToken === void 0 || continuationToken === "") return;
|
|
3128
|
+
}
|
|
3129
|
+
}
|
|
3130
|
+
};
|
|
3131
|
+
// ../adapter-node/src/credentials/resolve.ts
|
|
3132
|
+
const REFRESH_BUFFER_MS = 3e5;
|
|
3133
|
+
/** Cache a credential-derived signing value until five minutes before expiration. */
|
|
3134
|
+
function credentialResolver(credentials, now, build) {
|
|
3135
|
+
if (typeof credentials !== "function") {
|
|
3136
|
+
const value = build(credentials);
|
|
3137
|
+
return Object.assign(async () => value, { invalidate() {} });
|
|
3138
|
+
}
|
|
3139
|
+
let cached = null;
|
|
3140
|
+
let inFlight = null;
|
|
3141
|
+
const resolve = async () => {
|
|
3142
|
+
if (cached !== null && cached.expiresAt - REFRESH_BUFFER_MS > now()) return cached.value;
|
|
3143
|
+
if (inFlight === null) inFlight = Promise.resolve().then(credentials).then((creds) => {
|
|
3144
|
+
const value = build(creds);
|
|
3145
|
+
cached = {
|
|
3146
|
+
value,
|
|
3147
|
+
expiresAt: creds.expiration?.getTime() ?? Number.POSITIVE_INFINITY
|
|
3148
|
+
};
|
|
3149
|
+
return value;
|
|
3150
|
+
}).finally(() => {
|
|
3151
|
+
inFlight = null;
|
|
3152
|
+
});
|
|
3153
|
+
return inFlight;
|
|
3154
|
+
};
|
|
3155
|
+
return Object.assign(resolve, { invalidate(value) {
|
|
3156
|
+
if (cached?.value === value) cached = null;
|
|
3157
|
+
} });
|
|
3158
|
+
}
|
|
3159
|
+
// ../adapter-node/src/credentials/goog4-signer.ts
|
|
3160
|
+
/** GOOG4 signing-key prefix (vs AWS SigV4's "AWS4"). @see https://docs.cloud.google.com/storage/docs/authentication/signatures */
|
|
3161
|
+
const GOOG4_KEY_PREFIX = "GOOG4";
|
|
3162
|
+
/** Credential-scope terminator (vs SigV4's "aws4_request"). @see https://docs.cloud.google.com/storage/docs/authentication/signatures */
|
|
3163
|
+
const GOOG4_REQUEST = "goog4_request";
|
|
3164
|
+
/** Signing algorithm string. @see https://docs.cloud.google.com/storage/docs/authentication/signatures */
|
|
3165
|
+
const GOOG4_ALGORITHM = "GOOG4-HMAC-SHA256";
|
|
3166
|
+
/** GCS storage service name in the credential scope (vs SigV4's "s3"). @see https://docs.cloud.google.com/storage/docs/authentication/signatures */
|
|
3167
|
+
const GOOG4_SERVICE = "storage";
|
|
3168
|
+
/** GCS accepts "auto" as the signing region. @see https://docs.cloud.google.com/storage/docs/authentication/signatures */
|
|
3169
|
+
const GOOG4_REGION = "auto";
|
|
3170
|
+
const encoder$1 = new TextEncoder();
|
|
3171
|
+
/** RFC-3986-strict percent-encoding for canonical query components: encodes
|
|
3172
|
+
* everything except the unreserved set A-Za-z0-9-._~. encodeURIComponent
|
|
3173
|
+
* already handles that set but leaves !'()* unescaped, so escape those too.
|
|
3174
|
+
* @see https://docs.cloud.google.com/storage/docs/authentication/canonical-requests */
|
|
3175
|
+
function uriEncode(s) {
|
|
3176
|
+
return encodeURIComponent(s).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
3177
|
+
}
|
|
3178
|
+
/** Canonical-URI encoding for the request path. `URL.pathname` already
|
|
3179
|
+
* percent-encodes spaces / non-ASCII / control chars (and `^` `` ` `` `{` `}`),
|
|
3180
|
+
* but leaves GCS's enumerated reserved set raw — the sub-delims, `:@`, and
|
|
3181
|
+
* `[]`. GCS canonicalizes the path it receives by percent-encoding exactly
|
|
3182
|
+
* that set (`?=!#$&'()*+,:;@[]"`; `?#"` can't survive in `pathname`), so we
|
|
3183
|
+
* must reproduce the same delta or a `[`/`]`-bearing key signs differently
|
|
3184
|
+
* than GCS computes it (403 SignatureDoesNotMatch). Target only the delta so
|
|
3185
|
+
* already-encoded `%XX` and the unreserved set stay intact (no double-encode).
|
|
3186
|
+
* This is a denylist, not an allow-list, on purpose: over-encoding chars GCS
|
|
3187
|
+
* leaves raw (`|` `` ` `` `{}` …) would itself cause a mismatch.
|
|
3188
|
+
* @see https://docs.cloud.google.com/storage/docs/authentication/canonical-requests */
|
|
3189
|
+
function canonicalUri(pathname) {
|
|
3190
|
+
return pathname.replace(/[!$&'()*+,;=:@[\]]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
3191
|
+
}
|
|
3192
|
+
async function hmac$1(key, data) {
|
|
3193
|
+
const view = new Uint8Array(key.byteLength);
|
|
3194
|
+
view.set(key);
|
|
3195
|
+
const cryptoKey = await crypto.subtle.importKey("raw", view, {
|
|
3196
|
+
name: "HMAC",
|
|
3197
|
+
hash: "SHA-256"
|
|
3198
|
+
}, false, ["sign"]);
|
|
3199
|
+
const sig = await crypto.subtle.sign("HMAC", cryptoKey, encoder$1.encode(data));
|
|
3200
|
+
return new Uint8Array(sig);
|
|
3201
|
+
}
|
|
3202
|
+
/** SHA-256 hex of the empty payload: the `x-goog-content-sha256` of a bodyless request. @see https://docs.cloud.google.com/storage/docs/authentication/canonical-requests */
|
|
3203
|
+
const EMPTY_PAYLOAD_SHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
|
|
3204
|
+
/** The GOOG4 signing key for one secret and UTC day, imported once for reuse.
|
|
3205
|
+
* Same key chain as {@link sigV4Signature}, which stays the uncached oracle. */
|
|
3206
|
+
async function importSigningKey(secretAccessKey, yyyymmdd) {
|
|
3207
|
+
const signingKey = await hmac$1(await hmac$1(await hmac$1(await hmac$1(encoder$1.encode(GOOG4_KEY_PREFIX + secretAccessKey), yyyymmdd), GOOG4_REGION), GOOG4_SERVICE), GOOG4_REQUEST);
|
|
3208
|
+
return crypto.subtle.importKey("raw", signingKey, {
|
|
3209
|
+
name: "HMAC",
|
|
3210
|
+
hash: "SHA-256"
|
|
3211
|
+
}, false, ["sign"]);
|
|
3212
|
+
}
|
|
3213
|
+
/** Pure: build the GOOG4 canonical request string. Exported for the intermediates test. */
|
|
3214
|
+
function goog4CanonicalRequest(input) {
|
|
3215
|
+
const u = new URL(input.url);
|
|
3216
|
+
const normalized = {};
|
|
3217
|
+
for (const [name, value] of Object.entries(input.signedHeaders)) normalized[name.toLowerCase()] = value;
|
|
3218
|
+
const names = Object.keys(normalized).toSorted();
|
|
3219
|
+
const canonicalHeaders = names.map((n) => `${n}:${normalized[n].trim()}\n`).join("");
|
|
3220
|
+
const signedHeaderList = names.join(";");
|
|
3221
|
+
const query = [...u.searchParams].map(([k, v]) => [uriEncode(k), uriEncode(v)]).toSorted((a, b) => {
|
|
3222
|
+
const ka = `${a[0]}\0${a[1]}`;
|
|
3223
|
+
const kb = `${b[0]}\0${b[1]}`;
|
|
3224
|
+
if (ka < kb) return -1;
|
|
3225
|
+
return ka > kb ? 1 : 0;
|
|
3226
|
+
}).map(([k, v]) => `${k}=${v}`).join("&");
|
|
3227
|
+
return [
|
|
3228
|
+
input.method,
|
|
3229
|
+
canonicalUri(u.pathname),
|
|
3230
|
+
query,
|
|
3231
|
+
canonicalHeaders,
|
|
3232
|
+
signedHeaderList,
|
|
3233
|
+
input.hashedPayload
|
|
3234
|
+
].join("\n");
|
|
3235
|
+
}
|
|
3236
|
+
/** Pure: build the SigV4-family string-to-sign. Async because it hashes the canonical request. */
|
|
3237
|
+
async function goog4StringToSign(input) {
|
|
3238
|
+
const hash = await sha256Hex(encoder$1.encode(input.canonicalRequest));
|
|
3239
|
+
return [
|
|
3240
|
+
input.algorithm,
|
|
3241
|
+
input.amzDate,
|
|
3242
|
+
input.scope,
|
|
3243
|
+
hash
|
|
3244
|
+
].join("\n");
|
|
3245
|
+
}
|
|
3246
|
+
function amzDateFrom(ms) {
|
|
3247
|
+
const amzDate = new Date(ms).toISOString().replace(/[:-]/g, "").replace(/\.\d{3}Z$/, "Z");
|
|
3248
|
+
return {
|
|
3249
|
+
amzDate,
|
|
3250
|
+
yyyymmdd: amzDate.slice(0, 8)
|
|
3251
|
+
};
|
|
3252
|
+
}
|
|
3253
|
+
function goog4Signer(opts) {
|
|
3254
|
+
const now = opts.now ?? (() => Date.now());
|
|
3255
|
+
const resolve = credentialResolver(opts.credentials, now, (creds) => creds);
|
|
3256
|
+
let cachedKey;
|
|
3257
|
+
const signingKeyFor = (secret, yyyymmdd) => {
|
|
3258
|
+
if (cachedKey?.secret === secret && cachedKey.yyyymmdd === yyyymmdd) return cachedKey.key;
|
|
3259
|
+
const entry = {
|
|
3260
|
+
secret,
|
|
3261
|
+
yyyymmdd,
|
|
3262
|
+
key: importSigningKey(secret, yyyymmdd)
|
|
3263
|
+
};
|
|
3264
|
+
cachedKey = entry;
|
|
3265
|
+
entry.key.catch(() => {
|
|
3266
|
+
if (cachedKey === entry) cachedKey = void 0;
|
|
3267
|
+
});
|
|
3268
|
+
return entry.key;
|
|
3269
|
+
};
|
|
3270
|
+
return async (req) => {
|
|
3271
|
+
const creds = await resolve();
|
|
3272
|
+
const { amzDate, yyyymmdd } = amzDateFrom(now());
|
|
3273
|
+
const scope = `${yyyymmdd}/${GOOG4_REGION}/${GOOG4_SERVICE}/${GOOG4_REQUEST}`;
|
|
3274
|
+
const payloadHash = req.body === null ? EMPTY_PAYLOAD_SHA256 : await sha256Hex(new Uint8Array(await req.clone().arrayBuffer()));
|
|
3275
|
+
const url = new URL(req.url);
|
|
3276
|
+
const headers = new Headers(req.headers);
|
|
3277
|
+
headers.set("host", url.host);
|
|
3278
|
+
headers.set("x-goog-date", amzDate);
|
|
3279
|
+
headers.set("x-goog-content-sha256", payloadHash);
|
|
3280
|
+
const signedHeaders = {};
|
|
3281
|
+
for (const [name, value] of headers) {
|
|
3282
|
+
const lower = name.toLowerCase();
|
|
3283
|
+
if (lower === "host" || lower === "content-type" || lower.startsWith("x-goog-")) signedHeaders[lower] = value;
|
|
3284
|
+
}
|
|
3285
|
+
const canonical = goog4CanonicalRequest({
|
|
3286
|
+
method: req.method,
|
|
3287
|
+
url: req.url,
|
|
3288
|
+
signedHeaders,
|
|
3289
|
+
hashedPayload: payloadHash
|
|
3290
|
+
});
|
|
3291
|
+
const stringToSign = await goog4StringToSign({
|
|
3292
|
+
algorithm: GOOG4_ALGORITHM,
|
|
3293
|
+
amzDate,
|
|
3294
|
+
scope,
|
|
3295
|
+
canonicalRequest: canonical
|
|
3296
|
+
});
|
|
3297
|
+
const signingKey = await signingKeyFor(creds.secretAccessKey, yyyymmdd);
|
|
3298
|
+
const signature = new Uint8Array(await crypto.subtle.sign("HMAC", signingKey, encoder$1.encode(stringToSign))).toHex();
|
|
3299
|
+
const signedHeaderList = Object.keys(signedHeaders).map((n) => n.toLowerCase()).toSorted().join(";");
|
|
3300
|
+
headers.set("Authorization", `${GOOG4_ALGORITHM} Credential=${creds.accessKeyId}/${scope}, SignedHeaders=${signedHeaderList}, Signature=${signature}`);
|
|
3301
|
+
return new Request(req, { headers });
|
|
3302
|
+
};
|
|
3303
|
+
}
|
|
3304
|
+
// ../../node_modules/.pnpm/aws4fetch@1.0.20/node_modules/aws4fetch/dist/aws4fetch.esm.mjs
|
|
3305
|
+
/**
|
|
3306
|
+
* @license MIT <https://opensource.org/licenses/MIT>
|
|
3307
|
+
* @copyright Michael Hart 2024
|
|
3308
|
+
*/
|
|
3309
|
+
const encoder = new TextEncoder();
|
|
3310
|
+
const HOST_SERVICES = {
|
|
3311
|
+
appstream2: "appstream",
|
|
3312
|
+
cloudhsmv2: "cloudhsm",
|
|
3313
|
+
email: "ses",
|
|
3314
|
+
marketplace: "aws-marketplace",
|
|
3315
|
+
mobile: "AWSMobileHubService",
|
|
3316
|
+
pinpoint: "mobiletargeting",
|
|
3317
|
+
queue: "sqs",
|
|
3318
|
+
"git-codecommit": "codecommit",
|
|
3319
|
+
"mturk-requester-sandbox": "mturk-requester",
|
|
3320
|
+
"personalize-runtime": "personalize"
|
|
3321
|
+
};
|
|
3322
|
+
const UNSIGNABLE_HEADERS = /* @__PURE__ */ new Set([
|
|
3323
|
+
"authorization",
|
|
3324
|
+
"content-type",
|
|
3325
|
+
"content-length",
|
|
3326
|
+
"user-agent",
|
|
3327
|
+
"presigned-expires",
|
|
3328
|
+
"expect",
|
|
3329
|
+
"x-amzn-trace-id",
|
|
3330
|
+
"range",
|
|
3331
|
+
"connection"
|
|
3332
|
+
]);
|
|
3333
|
+
var AwsClient = class {
|
|
3334
|
+
constructor({ accessKeyId, secretAccessKey, sessionToken, service, region, cache, retries, initRetryMs }) {
|
|
3335
|
+
if (accessKeyId == null) throw new TypeError("accessKeyId is a required option");
|
|
3336
|
+
if (secretAccessKey == null) throw new TypeError("secretAccessKey is a required option");
|
|
3337
|
+
this.accessKeyId = accessKeyId;
|
|
3338
|
+
this.secretAccessKey = secretAccessKey;
|
|
3339
|
+
this.sessionToken = sessionToken;
|
|
3340
|
+
this.service = service;
|
|
3341
|
+
this.region = region;
|
|
3342
|
+
this.cache = cache || /* @__PURE__ */ new Map();
|
|
3343
|
+
this.retries = retries != null ? retries : 10;
|
|
3344
|
+
this.initRetryMs = initRetryMs || 50;
|
|
3345
|
+
}
|
|
3346
|
+
async sign(input, init) {
|
|
3347
|
+
if (input instanceof Request) {
|
|
3348
|
+
const { method, url, headers, body } = input;
|
|
3349
|
+
init = Object.assign({
|
|
3350
|
+
method,
|
|
3351
|
+
url,
|
|
3352
|
+
headers
|
|
3353
|
+
}, init);
|
|
3354
|
+
if (init.body == null && headers.has("Content-Type")) init.body = body != null && headers.has("X-Amz-Content-Sha256") ? body : await input.clone().arrayBuffer();
|
|
3355
|
+
input = url;
|
|
3356
|
+
}
|
|
3357
|
+
const signer = new AwsV4Signer(Object.assign({ url: input.toString() }, init, this, init && init.aws));
|
|
3358
|
+
const signed = Object.assign({}, init, await signer.sign());
|
|
3359
|
+
delete signed.aws;
|
|
3360
|
+
try {
|
|
3361
|
+
return new Request(signed.url.toString(), signed);
|
|
3362
|
+
} catch (e) {
|
|
3363
|
+
if (e instanceof TypeError) return new Request(signed.url.toString(), Object.assign({ duplex: "half" }, signed));
|
|
3364
|
+
throw e;
|
|
3365
|
+
}
|
|
3366
|
+
}
|
|
3367
|
+
async fetch(input, init) {
|
|
3368
|
+
for (let i = 0; i <= this.retries; i++) {
|
|
3369
|
+
const fetched = fetch(await this.sign(input, init));
|
|
3370
|
+
if (i === this.retries) return fetched;
|
|
3371
|
+
const res = await fetched;
|
|
3372
|
+
if (res.status < 500 && res.status !== 429) return res;
|
|
3373
|
+
await new Promise((resolve) => setTimeout(resolve, Math.random() * this.initRetryMs * Math.pow(2, i)));
|
|
3374
|
+
}
|
|
3375
|
+
throw new Error("An unknown error occurred, ensure retries is not negative");
|
|
3376
|
+
}
|
|
3377
|
+
};
|
|
3378
|
+
var AwsV4Signer = class {
|
|
3379
|
+
constructor({ method, url, headers, body, accessKeyId, secretAccessKey, sessionToken, service, region, cache, datetime, signQuery, appendSessionToken, allHeaders, singleEncode }) {
|
|
3380
|
+
if (url == null) throw new TypeError("url is a required option");
|
|
3381
|
+
if (accessKeyId == null) throw new TypeError("accessKeyId is a required option");
|
|
3382
|
+
if (secretAccessKey == null) throw new TypeError("secretAccessKey is a required option");
|
|
3383
|
+
this.method = method || (body ? "POST" : "GET");
|
|
3384
|
+
this.url = new URL(url);
|
|
3385
|
+
this.headers = new Headers(headers || {});
|
|
3386
|
+
this.body = body;
|
|
3387
|
+
this.accessKeyId = accessKeyId;
|
|
3388
|
+
this.secretAccessKey = secretAccessKey;
|
|
3389
|
+
this.sessionToken = sessionToken;
|
|
3390
|
+
let guessedService, guessedRegion;
|
|
3391
|
+
if (!service || !region) [guessedService, guessedRegion] = guessServiceRegion(this.url, this.headers);
|
|
3392
|
+
this.service = service || guessedService || "";
|
|
3393
|
+
this.region = region || guessedRegion || "us-east-1";
|
|
3394
|
+
this.cache = cache || /* @__PURE__ */ new Map();
|
|
3395
|
+
this.datetime = datetime || (/* @__PURE__ */ new Date()).toISOString().replace(/[:-]|\.\d{3}/g, "");
|
|
3396
|
+
this.signQuery = signQuery;
|
|
3397
|
+
this.appendSessionToken = appendSessionToken || this.service === "iotdevicegateway";
|
|
3398
|
+
this.headers.delete("Host");
|
|
3399
|
+
if (this.service === "s3" && !this.signQuery && !this.headers.has("X-Amz-Content-Sha256")) this.headers.set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD");
|
|
3400
|
+
const params = this.signQuery ? this.url.searchParams : this.headers;
|
|
3401
|
+
params.set("X-Amz-Date", this.datetime);
|
|
3402
|
+
if (this.sessionToken && !this.appendSessionToken) params.set("X-Amz-Security-Token", this.sessionToken);
|
|
3403
|
+
this.signableHeaders = ["host", ...this.headers.keys()].filter((header) => allHeaders || !UNSIGNABLE_HEADERS.has(header)).sort();
|
|
3404
|
+
this.signedHeaders = this.signableHeaders.join(";");
|
|
3405
|
+
this.canonicalHeaders = this.signableHeaders.map((header) => header + ":" + (header === "host" ? this.url.host : (this.headers.get(header) || "").replace(/\s+/g, " "))).join("\n");
|
|
3406
|
+
this.credentialString = [
|
|
3407
|
+
this.datetime.slice(0, 8),
|
|
3408
|
+
this.region,
|
|
3409
|
+
this.service,
|
|
3410
|
+
"aws4_request"
|
|
3411
|
+
].join("/");
|
|
3412
|
+
if (this.signQuery) {
|
|
3413
|
+
if (this.service === "s3" && !params.has("X-Amz-Expires")) params.set("X-Amz-Expires", "86400");
|
|
3414
|
+
params.set("X-Amz-Algorithm", "AWS4-HMAC-SHA256");
|
|
3415
|
+
params.set("X-Amz-Credential", this.accessKeyId + "/" + this.credentialString);
|
|
3416
|
+
params.set("X-Amz-SignedHeaders", this.signedHeaders);
|
|
3417
|
+
}
|
|
3418
|
+
if (this.service === "s3") try {
|
|
3419
|
+
this.encodedPath = decodeURIComponent(this.url.pathname.replace(/\+/g, " "));
|
|
3420
|
+
} catch (e) {
|
|
3421
|
+
this.encodedPath = this.url.pathname;
|
|
3422
|
+
}
|
|
3423
|
+
else this.encodedPath = this.url.pathname.replace(/\/+/g, "/");
|
|
3424
|
+
if (!singleEncode) this.encodedPath = encodeURIComponent(this.encodedPath).replace(/%2F/g, "/");
|
|
3425
|
+
this.encodedPath = encodeRfc3986(this.encodedPath);
|
|
3426
|
+
const seenKeys = /* @__PURE__ */ new Set();
|
|
3427
|
+
this.encodedSearch = [...this.url.searchParams].filter(([k]) => {
|
|
3428
|
+
if (!k) return false;
|
|
3429
|
+
if (this.service === "s3") {
|
|
3430
|
+
if (seenKeys.has(k)) return false;
|
|
3431
|
+
seenKeys.add(k);
|
|
3432
|
+
}
|
|
3433
|
+
return true;
|
|
3434
|
+
}).map((pair) => pair.map((p) => encodeRfc3986(encodeURIComponent(p)))).sort(([k1, v1], [k2, v2]) => k1 < k2 ? -1 : k1 > k2 ? 1 : v1 < v2 ? -1 : v1 > v2 ? 1 : 0).map((pair) => pair.join("=")).join("&");
|
|
3435
|
+
}
|
|
3436
|
+
async sign() {
|
|
3437
|
+
if (this.signQuery) {
|
|
3438
|
+
this.url.searchParams.set("X-Amz-Signature", await this.signature());
|
|
3439
|
+
if (this.sessionToken && this.appendSessionToken) this.url.searchParams.set("X-Amz-Security-Token", this.sessionToken);
|
|
3440
|
+
} else this.headers.set("Authorization", await this.authHeader());
|
|
3441
|
+
return {
|
|
3442
|
+
method: this.method,
|
|
3443
|
+
url: this.url,
|
|
3444
|
+
headers: this.headers,
|
|
3445
|
+
body: this.body
|
|
3446
|
+
};
|
|
3447
|
+
}
|
|
3448
|
+
async authHeader() {
|
|
3449
|
+
return [
|
|
3450
|
+
"AWS4-HMAC-SHA256 Credential=" + this.accessKeyId + "/" + this.credentialString,
|
|
3451
|
+
"SignedHeaders=" + this.signedHeaders,
|
|
3452
|
+
"Signature=" + await this.signature()
|
|
3453
|
+
].join(", ");
|
|
3454
|
+
}
|
|
3455
|
+
async signature() {
|
|
3456
|
+
const date = this.datetime.slice(0, 8);
|
|
3457
|
+
const cacheKey = [
|
|
3458
|
+
this.secretAccessKey,
|
|
3459
|
+
date,
|
|
3460
|
+
this.region,
|
|
3461
|
+
this.service
|
|
3462
|
+
].join();
|
|
3463
|
+
let kCredentials = this.cache.get(cacheKey);
|
|
3464
|
+
if (!kCredentials) {
|
|
3465
|
+
kCredentials = await hmac(await hmac(await hmac(await hmac("AWS4" + this.secretAccessKey, date), this.region), this.service), "aws4_request");
|
|
3466
|
+
this.cache.set(cacheKey, kCredentials);
|
|
3467
|
+
}
|
|
3468
|
+
return buf2hex(await hmac(kCredentials, await this.stringToSign()));
|
|
3469
|
+
}
|
|
3470
|
+
async stringToSign() {
|
|
3471
|
+
return [
|
|
3472
|
+
"AWS4-HMAC-SHA256",
|
|
3473
|
+
this.datetime,
|
|
3474
|
+
this.credentialString,
|
|
3475
|
+
buf2hex(await hash(await this.canonicalString()))
|
|
3476
|
+
].join("\n");
|
|
3477
|
+
}
|
|
3478
|
+
async canonicalString() {
|
|
3479
|
+
return [
|
|
3480
|
+
this.method.toUpperCase(),
|
|
3481
|
+
this.encodedPath,
|
|
3482
|
+
this.encodedSearch,
|
|
3483
|
+
this.canonicalHeaders + "\n",
|
|
3484
|
+
this.signedHeaders,
|
|
3485
|
+
await this.hexBodyHash()
|
|
3486
|
+
].join("\n");
|
|
3487
|
+
}
|
|
3488
|
+
async hexBodyHash() {
|
|
3489
|
+
let hashHeader = this.headers.get("X-Amz-Content-Sha256") || (this.service === "s3" && this.signQuery ? "UNSIGNED-PAYLOAD" : null);
|
|
3490
|
+
if (hashHeader == null) {
|
|
3491
|
+
if (this.body && typeof this.body !== "string" && !("byteLength" in this.body)) throw new Error("body must be a string, ArrayBuffer or ArrayBufferView, unless you include the X-Amz-Content-Sha256 header");
|
|
3492
|
+
hashHeader = buf2hex(await hash(this.body || ""));
|
|
3493
|
+
}
|
|
3494
|
+
return hashHeader;
|
|
3495
|
+
}
|
|
3496
|
+
};
|
|
3497
|
+
async function hmac(key, string) {
|
|
3498
|
+
const cryptoKey = await crypto.subtle.importKey("raw", typeof key === "string" ? encoder.encode(key) : key, {
|
|
3499
|
+
name: "HMAC",
|
|
3500
|
+
hash: { name: "SHA-256" }
|
|
3501
|
+
}, false, ["sign"]);
|
|
3502
|
+
return crypto.subtle.sign("HMAC", cryptoKey, encoder.encode(string));
|
|
3503
|
+
}
|
|
3504
|
+
async function hash(content) {
|
|
3505
|
+
return crypto.subtle.digest("SHA-256", typeof content === "string" ? encoder.encode(content) : content);
|
|
3506
|
+
}
|
|
3507
|
+
const HEX_CHARS = [
|
|
3508
|
+
"0",
|
|
3509
|
+
"1",
|
|
3510
|
+
"2",
|
|
3511
|
+
"3",
|
|
3512
|
+
"4",
|
|
3513
|
+
"5",
|
|
3514
|
+
"6",
|
|
3515
|
+
"7",
|
|
3516
|
+
"8",
|
|
3517
|
+
"9",
|
|
3518
|
+
"a",
|
|
3519
|
+
"b",
|
|
3520
|
+
"c",
|
|
3521
|
+
"d",
|
|
3522
|
+
"e",
|
|
3523
|
+
"f"
|
|
3524
|
+
];
|
|
3525
|
+
function buf2hex(arrayBuffer) {
|
|
3526
|
+
const buffer = new Uint8Array(arrayBuffer);
|
|
3527
|
+
let out = "";
|
|
3528
|
+
for (let idx = 0; idx < buffer.length; idx++) {
|
|
3529
|
+
const n = buffer[idx];
|
|
3530
|
+
out += HEX_CHARS[n >>> 4 & 15];
|
|
3531
|
+
out += HEX_CHARS[n & 15];
|
|
3532
|
+
}
|
|
3533
|
+
return out;
|
|
3534
|
+
}
|
|
3535
|
+
function encodeRfc3986(urlEncodedStr) {
|
|
3536
|
+
return urlEncodedStr.replace(/[!'()*]/g, (c) => "%" + c.charCodeAt(0).toString(16).toUpperCase());
|
|
3537
|
+
}
|
|
3538
|
+
function guessServiceRegion(url, headers) {
|
|
3539
|
+
const { hostname, pathname } = url;
|
|
3540
|
+
if (hostname.endsWith(".on.aws")) {
|
|
3541
|
+
const match = hostname.match(/^[^.]{1,63}\.lambda-url\.([^.]{1,63})\.on\.aws$/);
|
|
3542
|
+
return match != null ? ["lambda", match[1] || ""] : ["", ""];
|
|
3543
|
+
}
|
|
3544
|
+
if (hostname.endsWith(".r2.cloudflarestorage.com")) return ["s3", "auto"];
|
|
3545
|
+
if (hostname.endsWith(".backblazeb2.com")) {
|
|
3546
|
+
const match = hostname.match(/^(?:[^.]{1,63}\.)?s3\.([^.]{1,63})\.backblazeb2\.com$/);
|
|
3547
|
+
return match != null ? ["s3", match[1] || ""] : ["", ""];
|
|
3548
|
+
}
|
|
3549
|
+
const match = hostname.replace("dualstack.", "").match(/([^.]{1,63})\.(?:([^.]{0,63})\.)?amazonaws\.com(?:\.cn)?$/);
|
|
3550
|
+
let service = match && match[1] || "";
|
|
3551
|
+
let region = match && match[2];
|
|
3552
|
+
if (region === "us-gov") region = "us-gov-west-1";
|
|
3553
|
+
else if (region === "s3" || region === "s3-accelerate") {
|
|
3554
|
+
region = "us-east-1";
|
|
3555
|
+
service = "s3";
|
|
3556
|
+
} else if (service === "iot") {
|
|
3557
|
+
if (hostname.startsWith("iot.")) service = "execute-api";
|
|
3558
|
+
else if (hostname.startsWith("data.jobs.iot.")) service = "iot-jobs-data";
|
|
3559
|
+
else service = pathname === "/mqtt" ? "iotdevicegateway" : "iotdata";
|
|
3560
|
+
} else if (service === "autoscaling") {
|
|
3561
|
+
const targetPrefix = (headers.get("X-Amz-Target") || "").split(".")[0];
|
|
3562
|
+
if (targetPrefix === "AnyScaleFrontendService") service = "application-autoscaling";
|
|
3563
|
+
else if (targetPrefix === "AnyScaleScalingPlannerFrontendService") service = "autoscaling-plans";
|
|
3564
|
+
} else if (region == null && service.startsWith("s3-")) {
|
|
3565
|
+
region = service.slice(3).replace(/^fips-|^external-1/, "");
|
|
3566
|
+
service = "s3";
|
|
3567
|
+
} else if (service.endsWith("-fips")) service = service.slice(0, -5);
|
|
3568
|
+
else if (region && /-\d$/.test(service) && !/-\d$/.test(region)) [service, region] = [region, service];
|
|
3569
|
+
return [HOST_SERVICES[service] || service, region || ""];
|
|
3570
|
+
}
|
|
3571
|
+
// ../adapter-node/src/sigv4-sign.ts
|
|
3572
|
+
/**
|
|
3573
|
+
* Sign an S3 request through `aws4fetch` without touching its body.
|
|
3574
|
+
*
|
|
3575
|
+
* Given a `Request` with a `Content-Type` and no `X-Amz-Content-Sha256`,
|
|
3576
|
+
* `AwsClient.sign` reads the whole body through `req.clone().arrayBuffer()`
|
|
3577
|
+
* and rebuilds the request around the copy, although for S3 it signs
|
|
3578
|
+
* `UNSIGNED-PAYLOAD` and never hashes the body. Signing only the method,
|
|
3579
|
+
* URL and headers yields the same `Authorization` and header set without
|
|
3580
|
+
* that copy. The header defaults to `UNSIGNED-PAYLOAD` exactly as
|
|
3581
|
+
* `aws4fetch` defaults it for S3, so the signature is unchanged.
|
|
3582
|
+
*
|
|
3583
|
+
* The signed headers go back onto a `Request` built from the original, so
|
|
3584
|
+
* the original body source survives and Node still sends a `Content-Length`.
|
|
3585
|
+
* Handing `aws4fetch` the original `Request` with `X-Amz-Content-Sha256`
|
|
3586
|
+
* already set would skip the copy too, but it rebuilds the request around
|
|
3587
|
+
* `req.body` (a `ReadableStream`), and undici then sends
|
|
3588
|
+
* `Transfer-Encoding: chunked` with no length, which AWS S3 `PutObject`
|
|
3589
|
+
* rejects.
|
|
3590
|
+
*/
|
|
3591
|
+
async function signHeadersOnly(client, req) {
|
|
3592
|
+
const headers = new Headers(req.headers);
|
|
3593
|
+
if (!headers.has("X-Amz-Content-Sha256")) headers.set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD");
|
|
3594
|
+
const signed = await client.sign(req.url, {
|
|
3595
|
+
method: req.method,
|
|
3596
|
+
headers
|
|
3597
|
+
});
|
|
3598
|
+
return new Request(req, { headers: signed.headers });
|
|
3599
|
+
}
|
|
3600
|
+
// ../adapter-node/src/credentials/signer.ts
|
|
3601
|
+
function buildAwsClient(creds, region) {
|
|
3602
|
+
return new AwsClient({
|
|
3603
|
+
accessKeyId: creds.accessKeyId,
|
|
3604
|
+
secretAccessKey: creds.secretAccessKey,
|
|
3605
|
+
sessionToken: creds.sessionToken,
|
|
3606
|
+
region,
|
|
3607
|
+
service: "s3"
|
|
3608
|
+
});
|
|
3609
|
+
}
|
|
3610
|
+
function refreshingSigner(opts) {
|
|
3611
|
+
const region = opts.region;
|
|
3612
|
+
const now = opts.now ?? (() => Date.now());
|
|
3613
|
+
if (typeof opts.credentials !== "function") {
|
|
3614
|
+
const aws = buildAwsClient(opts.credentials, region);
|
|
3615
|
+
return (req) => signHeadersOnly(aws, req);
|
|
3616
|
+
}
|
|
3617
|
+
const resolve = credentialResolver(opts.credentials, now, (creds) => buildAwsClient(creds, region));
|
|
3618
|
+
const signedBy = /* @__PURE__ */ new WeakMap();
|
|
3619
|
+
const sign = async (req) => {
|
|
3620
|
+
const aws = await resolve();
|
|
3621
|
+
const signed = await signHeadersOnly(aws, req);
|
|
3622
|
+
signedBy.set(signed, aws);
|
|
3623
|
+
return signed;
|
|
3624
|
+
};
|
|
3625
|
+
return Object.assign(sign, { invalidate(signed) {
|
|
3626
|
+
const aws = signedBy.get(signed);
|
|
3627
|
+
if (aws !== void 0) resolve.invalidate(aws);
|
|
3628
|
+
} });
|
|
3629
|
+
}
|
|
3630
|
+
// ../adapter-node/src/storage-factories.ts
|
|
3631
|
+
function buildS3Storage(opts) {
|
|
3632
|
+
const sign = refreshingSigner({
|
|
3633
|
+
region: opts.region,
|
|
3634
|
+
credentials: opts.credentials
|
|
3635
|
+
});
|
|
3636
|
+
const storage = new S3HttpStorage({
|
|
3637
|
+
endpoint: opts.endpoint,
|
|
3638
|
+
bucket: opts.bucket,
|
|
3639
|
+
sign
|
|
3640
|
+
});
|
|
3641
|
+
return sign.invalidate === void 0 ? storage : withCredentialInvalidation(storage, sign.invalidate);
|
|
3642
|
+
}
|
|
3643
|
+
/**
|
|
3644
|
+
* AWS S3 `Storage` factory. Wraps `S3HttpStorage` with the standard
|
|
3645
|
+
* `aws4fetch` SigV4 signer. The endpoint is derived from the region as
|
|
3646
|
+
* `https://s3.<region>.amazonaws.com`.
|
|
3647
|
+
*
|
|
3648
|
+
* @example Static credentials
|
|
3649
|
+
* ```ts
|
|
3650
|
+
* import { s3Storage } from "@gusto/baerly-storage/node";
|
|
3651
|
+
*
|
|
3652
|
+
* const storage = s3Storage({
|
|
3653
|
+
* region: process.env["AWS_REGION"] ?? "us-east-1",
|
|
3654
|
+
* bucket: process.env["BUCKET"]!,
|
|
3655
|
+
* credentials: {
|
|
3656
|
+
* accessKeyId: process.env["AWS_ACCESS_KEY_ID"]!,
|
|
3657
|
+
* secretAccessKey: process.env["AWS_SECRET_ACCESS_KEY"]!,
|
|
3658
|
+
* },
|
|
3659
|
+
* });
|
|
3660
|
+
* ```
|
|
3661
|
+
*
|
|
3662
|
+
* @example EKS (refreshing) — auto-detects Pod Identity or IRSA
|
|
3663
|
+
* ```ts
|
|
3664
|
+
* import { s3Storage, fromEks } from "@gusto/baerly-storage/node";
|
|
3665
|
+
*
|
|
3666
|
+
* const storage = s3Storage({
|
|
3667
|
+
* region: process.env["AWS_REGION"] ?? "us-east-1",
|
|
3668
|
+
* bucket: process.env["BUCKET"]!,
|
|
3669
|
+
* credentials: fromEks(),
|
|
3670
|
+
* });
|
|
3671
|
+
* ```
|
|
3672
|
+
*/
|
|
3673
|
+
function s3Storage(opts) {
|
|
3674
|
+
return buildS3Storage({
|
|
3675
|
+
endpoint: `https://s3.${opts.region}.amazonaws.com`,
|
|
3676
|
+
region: opts.region,
|
|
3677
|
+
bucket: opts.bucket,
|
|
3678
|
+
credentials: opts.credentials
|
|
3679
|
+
});
|
|
3680
|
+
}
|
|
3681
|
+
/**
|
|
3682
|
+
* Cloudflare R2 `Storage` factory (S3-compat endpoint). Derives the
|
|
3683
|
+
* endpoint from `accountId` as
|
|
3684
|
+
* `https://<accountId>.r2.cloudflarestorage.com` and pins region
|
|
3685
|
+
* `"auto"` — R2 ignores region but `aws4fetch`'s signer requires
|
|
3686
|
+
* the field to be set.
|
|
3687
|
+
*
|
|
3688
|
+
* For Workers that have a native R2 binding (no HTTP hop), see
|
|
3689
|
+
* `r2BindingStorage` in `@baerly/adapter-cloudflare`. This factory
|
|
3690
|
+
* is for Node hosts that talk to R2 over its public S3-compat URL.
|
|
3691
|
+
*
|
|
3692
|
+
* @example
|
|
3693
|
+
* ```ts
|
|
3694
|
+
* import { r2Storage } from "@gusto/baerly-storage/node";
|
|
3695
|
+
*
|
|
3696
|
+
* const storage = r2Storage({
|
|
3697
|
+
* accountId: process.env["R2_ACCOUNT_ID"]!,
|
|
3698
|
+
* bucket: process.env["BUCKET"]!,
|
|
3699
|
+
* credentials: {
|
|
3700
|
+
* accessKeyId: process.env["AWS_ACCESS_KEY_ID"]!,
|
|
3701
|
+
* secretAccessKey: process.env["AWS_SECRET_ACCESS_KEY"]!,
|
|
3702
|
+
* },
|
|
3703
|
+
* });
|
|
3704
|
+
* ```
|
|
3705
|
+
*/
|
|
3706
|
+
function r2Storage(opts) {
|
|
3707
|
+
return buildS3Storage({
|
|
3708
|
+
endpoint: `https://${opts.accountId}.r2.cloudflarestorage.com`,
|
|
3709
|
+
region: "auto",
|
|
3710
|
+
bucket: opts.bucket,
|
|
3711
|
+
credentials: opts.credentials
|
|
3712
|
+
});
|
|
3713
|
+
}
|
|
3714
|
+
/**
|
|
3715
|
+
* Minio `Storage` factory — for local-dev / self-hosted S3-compat.
|
|
3716
|
+
* Endpoint is fully caller-supplied (e.g. `http://localhost:9102`)
|
|
3717
|
+
* because Minio deployments don't follow a URL template. Region is
|
|
3718
|
+
* pinned to `"us-east-1"`; Minio ignores it but `aws4fetch` requires
|
|
3719
|
+
* a value.
|
|
3720
|
+
*
|
|
3721
|
+
* @example
|
|
3722
|
+
* ```ts
|
|
3723
|
+
* import { minioStorage } from "@gusto/baerly-storage/node";
|
|
3724
|
+
*
|
|
3725
|
+
* const storage = minioStorage({
|
|
3726
|
+
* endpoint: process.env["BAERLY_S3_ENDPOINT"] ?? "http://localhost:9102",
|
|
3727
|
+
* bucket: "baerly",
|
|
3728
|
+
* credentials: {
|
|
3729
|
+
* accessKeyId: process.env["MINIO_ACCESS_KEY"]!,
|
|
3730
|
+
* secretAccessKey: process.env["MINIO_SECRET_KEY"]!,
|
|
3731
|
+
* },
|
|
3732
|
+
* });
|
|
3733
|
+
* ```
|
|
3734
|
+
*/
|
|
3735
|
+
function minioStorage(opts) {
|
|
3736
|
+
return buildS3Storage({
|
|
3737
|
+
endpoint: opts.endpoint,
|
|
3738
|
+
region: "us-east-1",
|
|
3739
|
+
bucket: opts.bucket,
|
|
3740
|
+
credentials: opts.credentials
|
|
3741
|
+
});
|
|
3742
|
+
}
|
|
3743
|
+
/**
|
|
3744
|
+
* Google Cloud Storage `Storage` factory over the native GCS XML API
|
|
3745
|
+
* at `https://storage.googleapis.com`. Wraps {@link GcsHttpStorage} with
|
|
3746
|
+
* a GOOG4-HMAC-SHA256 signer, driving GCS's own generation-precondition
|
|
3747
|
+
* conditional writes (`x-goog-if-generation-match: 0` for create-if-absent,
|
|
3748
|
+
* `x-goog-if-generation-match: <generation>` for compare-and-swap), which
|
|
3749
|
+
* return `412` on a lost write. The object `generation` is carried verbatim
|
|
3750
|
+
* as the opaque `Storage` etag. This is the sanctioned GCS path — the
|
|
3751
|
+
* S3-interop endpoint treats `If-Match`/`If-None-Match` as read-scoped only
|
|
3752
|
+
* and cannot linearize the log, so it is not supported.
|
|
3753
|
+
*
|
|
3754
|
+
* Authentication uses GCS HMAC keys (Console → Settings → Interoperability)
|
|
3755
|
+
* — mint an interop key, put the HMAC access key in `credentials.accessKeyId`
|
|
3756
|
+
* and the HMAC secret in `credentials.secretAccessKey`.
|
|
3757
|
+
*
|
|
3758
|
+
* @example
|
|
3759
|
+
* ```ts
|
|
3760
|
+
* import { gcsStorage } from "@gusto/baerly-storage/node";
|
|
3761
|
+
*
|
|
3762
|
+
* const storage = gcsStorage({
|
|
3763
|
+
* bucket: process.env["BUCKET"]!,
|
|
3764
|
+
* credentials: {
|
|
3765
|
+
* accessKeyId: process.env["GCS_HMAC_ACCESS_KEY_ID"]!,
|
|
3766
|
+
* secretAccessKey: process.env["GCS_HMAC_SECRET"]!,
|
|
3767
|
+
* },
|
|
3768
|
+
* });
|
|
3769
|
+
* ```
|
|
3770
|
+
*/
|
|
3771
|
+
function gcsStorage(opts) {
|
|
3772
|
+
const sign = goog4Signer({ credentials: opts.credentials });
|
|
3773
|
+
return new GcsHttpStorage({
|
|
3774
|
+
bucket: opts.bucket,
|
|
3775
|
+
sign,
|
|
3776
|
+
...opts.endpoint !== void 0 && { endpoint: opts.endpoint }
|
|
3777
|
+
});
|
|
3778
|
+
}
|
|
3779
|
+
// src/bucket-uri.ts
|
|
3780
|
+
/**
|
|
3781
|
+
* Bucket URI parsing for the CLI surface. Consumed by every
|
|
3782
|
+
* `baerly admin <cmd>`, `baerly inspect`, and `baerly export`.
|
|
3783
|
+
*
|
|
3784
|
+
* Grammar:
|
|
3785
|
+
* - `s3://<bucket>[/<prefix>]` — S3-compatible HTTP. Creds via env
|
|
3786
|
+
* vars (`BAERLY_S3_ENDPOINT`, `BAERLY_S3_ACCESS_KEY_ID`,
|
|
3787
|
+
* `BAERLY_S3_SECRET_ACCESS_KEY`, `BAERLY_S3_REGION` — region
|
|
3788
|
+
* defaults to `us-east-1`).
|
|
3789
|
+
* - `gcs://<bucket>[/<prefix>]` — native GCS XML API. Creds via
|
|
3790
|
+
* env vars `BAERLY_GCS_HMAC_ACCESS_KEY_ID` + `BAERLY_GCS_HMAC_SECRET`
|
|
3791
|
+
* (GCS HMAC interop keys: Console → Cloud Storage → Settings →
|
|
3792
|
+
* Interoperability). Drives GCS's native generation-precondition
|
|
3793
|
+
* path (`x-goog-if-generation-match`); the S3-interop endpoint is
|
|
3794
|
+
* NOT supported (it treats If-Match/If-None-Match as read-only).
|
|
3795
|
+
* - `file:///<absolute-path>` — `LocalFsStorage` rooted at the path.
|
|
3796
|
+
* Relative paths are rejected by the three-slash prefix.
|
|
3797
|
+
* - `memory://<bucket>` — `MemoryStorage` keyed via
|
|
3798
|
+
* `getOrCreateMemoryStorageForBucket`. Test-only.
|
|
3799
|
+
*/
|
|
3800
|
+
/**
|
|
3801
|
+
* Parse a bucket URI into a constructed `Storage` handle + optional
|
|
3802
|
+
* key prefix.
|
|
3803
|
+
*
|
|
3804
|
+
* @throws BaerlyError code="InvalidConfig" — unsupported scheme, or an
|
|
3805
|
+
* `s3://` / `gcs://` URI with a missing env var.
|
|
3806
|
+
*/
|
|
3807
|
+
const parseBucketUri = async (uri) => {
|
|
3808
|
+
if (uri.startsWith("s3://")) {
|
|
3809
|
+
const rest = uri.slice(5);
|
|
3810
|
+
const slash = rest.indexOf("/");
|
|
3811
|
+
const bucket = slash === -1 ? rest : rest.slice(0, slash);
|
|
3812
|
+
const prefix = slash === -1 ? "" : rest.slice(slash + 1);
|
|
3813
|
+
if (bucket.length === 0) throw new BaerlyError("InvalidConfig", `bucket URI: s3:// URI requires a bucket name (got ${JSON.stringify(uri)})`);
|
|
3814
|
+
const accessKeyId = requireEnv("BAERLY_S3_ACCESS_KEY_ID");
|
|
3815
|
+
const secretAccessKey = requireEnv("BAERLY_S3_SECRET_ACCESS_KEY");
|
|
3816
|
+
const region = process.env["BAERLY_S3_REGION"] ?? "us-east-1";
|
|
3817
|
+
const endpoint = requireEnv("BAERLY_S3_ENDPOINT");
|
|
3818
|
+
const r2Host = endpoint.match(/^https?:\/\/([^./]+)\.r2\.cloudflarestorage\.com\b/i);
|
|
3819
|
+
const isAws = /^https?:\/\/s3(\.[^.]+)?\.amazonaws\.com\b/i.test(endpoint);
|
|
3820
|
+
return {
|
|
3821
|
+
storage: pickS3Storage({
|
|
3822
|
+
r2Host,
|
|
3823
|
+
isAws,
|
|
3824
|
+
endpoint,
|
|
3825
|
+
bucket,
|
|
3826
|
+
accessKeyId,
|
|
3827
|
+
secretAccessKey,
|
|
3828
|
+
region
|
|
3829
|
+
}),
|
|
3830
|
+
keyPrefix: normalizeKeyPrefix(prefix)
|
|
3831
|
+
};
|
|
3832
|
+
}
|
|
3833
|
+
if (uri.startsWith("gcs://")) {
|
|
3834
|
+
const rest = uri.slice(6);
|
|
3835
|
+
const slash = rest.indexOf("/");
|
|
3836
|
+
const bucket = slash === -1 ? rest : rest.slice(0, slash);
|
|
3837
|
+
const prefix = slash === -1 ? "" : rest.slice(slash + 1);
|
|
3838
|
+
if (bucket.length === 0) throw new BaerlyError("InvalidConfig", `bucket URI: gcs:// URI requires a bucket name (got ${JSON.stringify(uri)})`);
|
|
3839
|
+
const accessKeyId = requireEnv("BAERLY_GCS_HMAC_ACCESS_KEY_ID");
|
|
3840
|
+
const secretAccessKey = requireEnv("BAERLY_GCS_HMAC_SECRET");
|
|
3841
|
+
const endpoint = DEFAULT_GCS_ENDPOINT;
|
|
3842
|
+
const credentials = {
|
|
3843
|
+
accessKeyId,
|
|
3844
|
+
secretAccessKey
|
|
3845
|
+
};
|
|
3846
|
+
return {
|
|
3847
|
+
storage: gcsStorage({
|
|
3848
|
+
endpoint,
|
|
3849
|
+
bucket,
|
|
3850
|
+
credentials
|
|
3851
|
+
}),
|
|
3852
|
+
keyPrefix: normalizeKeyPrefix(prefix),
|
|
3853
|
+
gcs: {
|
|
3854
|
+
endpoint,
|
|
3855
|
+
bucket,
|
|
3856
|
+
credentials
|
|
3857
|
+
}
|
|
3858
|
+
};
|
|
3859
|
+
}
|
|
3860
|
+
if (uri.startsWith("file:///")) return {
|
|
3861
|
+
storage: new LocalFsStorage({ root: uri.slice(7) }),
|
|
3862
|
+
keyPrefix: ""
|
|
3863
|
+
};
|
|
3864
|
+
if (uri.startsWith("memory://")) {
|
|
3865
|
+
const bucket = uri.slice(9);
|
|
3866
|
+
if (bucket.length === 0) throw new BaerlyError("InvalidConfig", `bucket URI: memory:// URI requires a bucket name (got ${JSON.stringify(uri)})`);
|
|
3867
|
+
return {
|
|
3868
|
+
storage: getOrCreateMemoryStorageForBucket(bucket),
|
|
3869
|
+
keyPrefix: ""
|
|
3870
|
+
};
|
|
3871
|
+
}
|
|
3872
|
+
throw new BaerlyError("InvalidConfig", `bucket URI: unsupported URI ${JSON.stringify(uri)}`);
|
|
3873
|
+
};
|
|
3874
|
+
const requireEnv = (name) => {
|
|
3875
|
+
const v = process.env[name];
|
|
3876
|
+
if (v === void 0 || v === "") throw new BaerlyError("InvalidConfig", `bucket URI: env var ${name} unset`);
|
|
3877
|
+
return v;
|
|
3878
|
+
};
|
|
3879
|
+
const pickS3Storage = (opts) => {
|
|
3880
|
+
const { r2Host, isAws, endpoint, bucket, accessKeyId, secretAccessKey, region } = opts;
|
|
3881
|
+
if (r2Host !== null) return r2Storage({
|
|
3882
|
+
accountId: r2Host[1],
|
|
3883
|
+
bucket,
|
|
3884
|
+
credentials: {
|
|
3885
|
+
accessKeyId,
|
|
3886
|
+
secretAccessKey
|
|
3887
|
+
}
|
|
3888
|
+
});
|
|
3889
|
+
if (isAws) return s3Storage({
|
|
3890
|
+
region,
|
|
3891
|
+
bucket,
|
|
3892
|
+
credentials: {
|
|
3893
|
+
accessKeyId,
|
|
3894
|
+
secretAccessKey
|
|
3895
|
+
}
|
|
3896
|
+
});
|
|
3897
|
+
return minioStorage({
|
|
3898
|
+
endpoint,
|
|
3899
|
+
bucket,
|
|
3900
|
+
credentials: {
|
|
3901
|
+
accessKeyId,
|
|
3902
|
+
secretAccessKey
|
|
3903
|
+
}
|
|
3904
|
+
});
|
|
3905
|
+
};
|
|
3906
|
+
const normalizeKeyPrefix = (prefix) => {
|
|
3907
|
+
if (prefix === "") return "";
|
|
3908
|
+
if (prefix.endsWith("/")) return prefix;
|
|
3909
|
+
return `${prefix}/`;
|
|
3910
|
+
};
|
|
3911
|
+
// src/config.ts
|
|
3912
|
+
/**
|
|
3913
|
+
* `baerly.config.ts` loader for `baerly deploy` / `baerly doctor`.
|
|
3914
|
+
*
|
|
3915
|
+
* Mirrors the shape of `BaerlyAppConfig` from `@gusto/baerly-storage/config`
|
|
3916
|
+
* but lives in `@baerly/cli` so the validator/loader doesn't depend on
|
|
3917
|
+
* the rolldown'd umbrella bundle. The scaffolder emits the config file;
|
|
3918
|
+
* the CLI reads it. Both sides agree on the wire shape (informally —
|
|
3919
|
+
* the duplication is small and intentional).
|
|
3920
|
+
*
|
|
3921
|
+
* Resolution order:
|
|
3922
|
+
* 1. `<cwd>/baerly.config.ts`
|
|
3923
|
+
* 2. `<cwd>/baerly.config.js`
|
|
3924
|
+
* 3. `<cwd>/baerly.config.mjs`
|
|
3925
|
+
* 4. `<cwd>/baerly.config.json`
|
|
3926
|
+
*
|
|
3927
|
+
* The `.ts` form relies on Node's built-in type stripping, which handles
|
|
3928
|
+
* only erasable syntax (no enums, namespaces or parameter properties);
|
|
3929
|
+
* a config using other TypeScript syntax fails with
|
|
3930
|
+
* `ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX`. If `import()` throws on the
|
|
3931
|
+
* `.ts` file, we re-wrap as `InvalidConfig` with a hint pointing at the
|
|
3932
|
+
* `.js` / `.json` fallbacks.
|
|
3933
|
+
*/
|
|
3934
|
+
const CONFIG_BASENAMES = [
|
|
3935
|
+
"baerly.config.ts",
|
|
3936
|
+
"baerly.config.js",
|
|
3937
|
+
"baerly.config.mjs",
|
|
3938
|
+
"baerly.config.json"
|
|
3939
|
+
];
|
|
3940
|
+
/**
|
|
3941
|
+
* Locate the first `baerly.config.*` file under `cwd`. Returns the
|
|
3942
|
+
* absolute path or `null` if none exists.
|
|
3943
|
+
*
|
|
3944
|
+
* Exported so the collection-aware loader (and future consumers) can
|
|
3945
|
+
* reuse the same resolution order without re-implementing it.
|
|
3946
|
+
*/
|
|
3947
|
+
const locateConfig = (cwd) => {
|
|
3948
|
+
for (const base of CONFIG_BASENAMES) {
|
|
3949
|
+
const abs = resolve(cwd, base);
|
|
3950
|
+
if (existsSync(abs)) return abs;
|
|
3951
|
+
}
|
|
3952
|
+
return null;
|
|
3953
|
+
};
|
|
3954
|
+
/**
|
|
3955
|
+
* Load `baerly.config.{ts,js,mjs,json}` from `cwd` (defaults to
|
|
3956
|
+
* `process.cwd()`). Validates the required fields and the
|
|
3957
|
+
* locked-shape optionals; returns an `AppConfig` decorated with the
|
|
3958
|
+
* `repoRoot` it was loaded from.
|
|
3959
|
+
*
|
|
3960
|
+
* @throws BaerlyError code="InvalidConfig" — file missing, default
|
|
3961
|
+
* export missing, or any field has the wrong shape / type.
|
|
3962
|
+
*/
|
|
3963
|
+
const loadAppConfig = async (cwd = process.cwd()) => {
|
|
3964
|
+
const cfgPath = locateConfig(cwd);
|
|
3965
|
+
if (cfgPath === null) throw new BaerlyError("InvalidConfig", `baerly: no baerly.config.{ts,js,mjs,json} at ${cwd}. Are you in a baerly app directory?`);
|
|
3966
|
+
let raw;
|
|
3967
|
+
if (cfgPath.endsWith(".json")) {
|
|
3968
|
+
let text;
|
|
3969
|
+
try {
|
|
3970
|
+
text = await readFile(cfgPath, "utf8");
|
|
3971
|
+
} catch (error) {
|
|
3972
|
+
throw new BaerlyError("InvalidConfig", `baerly: failed to read ${cfgPath}: ${error.message}`, { cause: error });
|
|
3973
|
+
}
|
|
3974
|
+
try {
|
|
3975
|
+
raw = JSON.parse(text);
|
|
3976
|
+
} catch (error) {
|
|
3977
|
+
throw new BaerlyError("InvalidConfig", `baerly: failed to parse ${cfgPath}: ${error.message}`, { cause: error });
|
|
3978
|
+
}
|
|
3979
|
+
} else {
|
|
3980
|
+
let mod;
|
|
3981
|
+
try {
|
|
3982
|
+
mod = await import(pathToFileURL(cfgPath).href);
|
|
3983
|
+
} catch (error) {
|
|
3984
|
+
const hint = cfgPath.endsWith(".ts") ? " (Node strips types only from erasable TypeScript syntax, so enums, namespaces and parameter properties fail with ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX — remove them, or point at a compiled .js or a .json config)" : "";
|
|
3985
|
+
throw new BaerlyError("InvalidConfig", `baerly: failed to load ${cfgPath}: ${error.message}${hint}`, { cause: error });
|
|
3986
|
+
}
|
|
3987
|
+
raw = mod.default;
|
|
3988
|
+
}
|
|
3989
|
+
if (raw === void 0 || typeof raw !== "object" || raw === null) throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath} must default-export a BaerlyAppConfig object`);
|
|
3990
|
+
const obj = raw;
|
|
3991
|
+
const app = obj["app"];
|
|
3992
|
+
const tenant = obj["tenant"];
|
|
3993
|
+
const target = obj["target"];
|
|
3994
|
+
if (typeof app !== "string" || app.length === 0) throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`app\` must be a non-empty string`);
|
|
3995
|
+
if (typeof tenant !== "string" || tenant.length === 0) throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`tenant\` must be a non-empty string`);
|
|
3996
|
+
if (target !== "cloudflare" && target !== "node") throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`target\` must be "cloudflare" or "node" (got ${JSON.stringify(target)})`);
|
|
3997
|
+
const authRaw = obj["auth"];
|
|
3998
|
+
if (typeof authRaw !== "string" || !AUTH_CONFIG_VALUES.includes(authRaw)) throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`auth\` is required and must be one of ${JSON.stringify(AUTH_CONFIG_VALUES)} (got ${JSON.stringify(authRaw)})`);
|
|
3999
|
+
const auth = authRaw;
|
|
4000
|
+
const domainRaw = obj["domain"];
|
|
4001
|
+
if (domainRaw !== void 0 && typeof domainRaw !== "string") throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`domain\` must be a string when set`);
|
|
4002
|
+
const secretsRaw = obj["requiredSecrets"];
|
|
4003
|
+
let requiredSecrets;
|
|
4004
|
+
if (secretsRaw !== void 0) {
|
|
4005
|
+
if (!Array.isArray(secretsRaw) || !secretsRaw.every((s) => typeof s === "string")) throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`requiredSecrets\` must be an array of strings`);
|
|
4006
|
+
requiredSecrets = secretsRaw;
|
|
4007
|
+
}
|
|
4008
|
+
const cfaRaw = obj["cloudflareAccess"];
|
|
4009
|
+
let cloudflareAccess;
|
|
4010
|
+
if (cfaRaw !== void 0) {
|
|
4011
|
+
if (typeof cfaRaw !== "object" || cfaRaw === null) throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`cloudflareAccess\` must be an object when set`);
|
|
4012
|
+
const cfa = cfaRaw;
|
|
4013
|
+
const td = cfa["teamDomain"];
|
|
4014
|
+
const aud = cfa["audienceTag"];
|
|
4015
|
+
if (typeof td !== "string" || !CF_ACCESS_TEAM_NAME_PATTERN.test(td)) throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`cloudflareAccess.teamDomain\` must be a bare team name of lowercase letters, digits and "-", such as "acme" (got ${JSON.stringify(td)})`);
|
|
4016
|
+
if (typeof aud !== "string" || aud.length === 0) throw new BaerlyError("InvalidConfig", `baerly: ${cfgPath}: \`cloudflareAccess.audienceTag\` must be a non-empty string`);
|
|
4017
|
+
cloudflareAccess = {
|
|
4018
|
+
teamDomain: td,
|
|
4019
|
+
audienceTag: aud
|
|
4020
|
+
};
|
|
4021
|
+
}
|
|
4022
|
+
return {
|
|
4023
|
+
app,
|
|
4024
|
+
tenant,
|
|
4025
|
+
target,
|
|
4026
|
+
auth,
|
|
4027
|
+
...domainRaw !== void 0 && { domain: domainRaw },
|
|
4028
|
+
...requiredSecrets !== void 0 && { requiredSecrets },
|
|
4029
|
+
...cloudflareAccess !== void 0 && { cloudflareAccess },
|
|
4030
|
+
repoRoot: cwd
|
|
4031
|
+
};
|
|
4032
|
+
};
|
|
4033
|
+
export { MAINTENANCE_PROFILE_NODE as C, SHARED_SECRET_MISSING_MESSAGE as D, MAX_KEY_BYTES as E, CF_ACCESS_TEAM_NAME_PATTERN as O, MAINTENANCE_PROFILE_CF_PAID as S, MAX_DOCUMENT_BYTES as T, GC_PENDING_CONTENT_TYPE as _, setRequestHoldResolver as a, MAINTENANCE_MIN_LIVE_BYTES as b, countKey as c, CF_PAID_COMPACT_TAIL_PROBE_GETS as d, CURRENT_JSON_CONTENT_TYPE as f, GC_PASS_GRACE_FRACTION as g, GC_MAX_PENDING_CANDIDATES as h, DEFAULT_GCS_ENDPOINT as i, uuid as l, GC_MARK_LIST_PAGE_KEYS as m, parseBucketUri as n, setRequestLedgerResolver as o, GC_GRACE_PERIOD_MILLIS as p, goog4Signer as r, timestamp as s, loadAppConfig as t, sha256Hex as u, LOG_FORWARD_PROBE_CAP as v, MAINTENANCE_WARN_INTERVAL_WRITES as w, MAINTENANCE_PROFILE_CF_FREE as x, LOG_RETENTION_SEQ_WINDOW as y };
|
|
4034
|
+
|
|
4035
|
+
//# sourceMappingURL=config-DEzInG6M.js.map
|