@narumitw/pi-sync 0.50.3 → 0.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/dist/chunks/{chunk-6LP3MCR2.ts → chunk-2JZFHNLM.ts} +3 -5
- package/dist/chunks/chunk-2JZFHNLM.ts.map +7 -0
- package/dist/chunks/{chunk-LCRI3ABW.ts → chunk-2RZSQWBH.ts} +8 -19
- package/dist/chunks/chunk-2RZSQWBH.ts.map +7 -0
- package/dist/chunks/{chunk-IJ6FJFA5.ts → chunk-4AIHXRKE.ts} +11 -17
- package/dist/chunks/chunk-4AIHXRKE.ts.map +7 -0
- package/dist/chunks/{chunk-C7XY5NF5.ts → chunk-4MPIH46N.ts} +29 -51
- package/dist/chunks/chunk-4MPIH46N.ts.map +7 -0
- package/dist/chunks/{chunk-5B3FXBLE.ts → chunk-6546TS54.ts} +16 -20
- package/dist/chunks/chunk-6546TS54.ts.map +7 -0
- package/dist/chunks/{chunk-Q2TUM5X6.ts → chunk-6NT3YFGY.ts} +2 -2
- package/dist/chunks/chunk-6NT3YFGY.ts.map +7 -0
- package/dist/chunks/{chunk-2B2ELUF5.ts → chunk-7D5SF7P3.ts} +1 -1
- package/dist/chunks/chunk-7D5SF7P3.ts.map +7 -0
- package/dist/chunks/{chunk-YDWUZVPN.ts → chunk-7KHKWL3Q.ts} +28 -47
- package/dist/chunks/chunk-7KHKWL3Q.ts.map +7 -0
- package/dist/chunks/{chunk-ZI5MXMEX.ts → chunk-BVVP33U5.ts} +2 -4
- package/dist/chunks/chunk-BVVP33U5.ts.map +7 -0
- package/dist/chunks/{chunk-SZWBGPID.ts → chunk-DM4X4FIA.ts} +8 -13
- package/dist/chunks/chunk-DM4X4FIA.ts.map +7 -0
- package/dist/chunks/{chunk-SL64RLXR.ts → chunk-DZUSTGQV.ts} +8 -16
- package/dist/chunks/chunk-DZUSTGQV.ts.map +7 -0
- package/dist/chunks/{chunk-YHEMENKV.ts → chunk-E42GXD6D.ts} +13 -19
- package/dist/chunks/chunk-E42GXD6D.ts.map +7 -0
- package/dist/chunks/{chunk-Z4DQF6L3.ts → chunk-ECYCNZOE.ts} +1 -1
- package/dist/chunks/chunk-ECYCNZOE.ts.map +7 -0
- package/dist/chunks/{chunk-OEX4ABY6.ts → chunk-EEJPYTWE.ts} +11 -21
- package/dist/chunks/chunk-EEJPYTWE.ts.map +7 -0
- package/dist/chunks/{chunk-CPEHGXJV.ts → chunk-G4RUXRWZ.ts} +3 -5
- package/dist/chunks/chunk-G4RUXRWZ.ts.map +7 -0
- package/dist/chunks/{chunk-T2B5IU74.ts → chunk-HGJH56KR.ts} +4 -4
- package/dist/chunks/chunk-HGJH56KR.ts.map +7 -0
- package/dist/chunks/{chunk-JAAKLL6M.ts → chunk-IVXEDPAD.ts} +10 -34
- package/dist/chunks/chunk-IVXEDPAD.ts.map +7 -0
- package/dist/chunks/chunk-K3RIEL6J.ts +20 -0
- package/dist/chunks/chunk-K3RIEL6J.ts.map +7 -0
- package/dist/chunks/{chunk-LTBWGSBV.ts → chunk-MSSW6F3R.ts} +1 -1
- package/dist/chunks/chunk-MSSW6F3R.ts.map +7 -0
- package/dist/chunks/{chunk-T3HH3UEJ.ts → chunk-OVDTF5CH.ts} +4 -10
- package/dist/chunks/chunk-OVDTF5CH.ts.map +7 -0
- package/dist/chunks/{chunk-NJFUIVYL.ts → chunk-PWMWOJP4.ts} +1 -1
- package/dist/chunks/chunk-PWMWOJP4.ts.map +7 -0
- package/dist/chunks/{chunk-HUDQATCL.ts → chunk-Q32CQWPM.ts} +6 -8
- package/dist/chunks/chunk-Q32CQWPM.ts.map +7 -0
- package/dist/chunks/{chunk-ETTRJDNP.ts → chunk-QLSN7AGX.ts} +6 -13
- package/dist/chunks/chunk-QLSN7AGX.ts.map +7 -0
- package/dist/chunks/{chunk-THMF4BVL.ts → chunk-SWUXA3MH.ts} +1 -1
- package/dist/chunks/chunk-SWUXA3MH.ts.map +7 -0
- package/dist/chunks/{chunk-TEN2ZBJV.ts → chunk-T2SJD3N7.ts} +2 -4
- package/dist/chunks/chunk-T2SJD3N7.ts.map +7 -0
- package/dist/chunks/{chunk-AUSDLD6F.ts → chunk-T56RNRZJ.ts} +10 -26
- package/dist/chunks/chunk-T56RNRZJ.ts.map +7 -0
- package/dist/chunks/{chunk-DHVFS4QK.ts → chunk-UA7RXOIT.ts} +37 -90
- package/dist/chunks/chunk-UA7RXOIT.ts.map +7 -0
- package/dist/chunks/{chunk-D5X27LCA.ts → chunk-UETBKTDI.ts} +1 -1
- package/dist/chunks/chunk-UETBKTDI.ts.map +7 -0
- package/dist/chunks/{chunk-CHH26D4F.ts → chunk-YFADF5ZQ.ts} +7 -6
- package/dist/chunks/chunk-YFADF5ZQ.ts.map +7 -0
- package/dist/chunks/{chunk-CCSN3F47.ts → chunk-YPXGU5OR.ts} +4 -4
- package/dist/chunks/chunk-YPXGU5OR.ts.map +7 -0
- package/dist/chunks/{file-selection-FTTKAOYX.ts → file-selection-RAVHRYFG.ts} +14 -26
- package/dist/chunks/file-selection-RAVHRYFG.ts.map +7 -0
- package/dist/chunks/{git-backend-GMVS4PD4.ts → git-backend-IAANDL3K.ts} +41 -104
- package/dist/chunks/git-backend-IAANDL3K.ts.map +7 -0
- package/dist/chunks/manager-result-dispatcher-OTUMQM26.ts +23 -0
- package/dist/chunks/{manager-ui-IPAOAWZA.ts → manager-ui-763NEKJY.ts} +124 -251
- package/dist/chunks/manager-ui-763NEKJY.ts.map +7 -0
- package/dist/chunks/remote-selection-ui-TZTJF32Y.ts +22 -0
- package/dist/chunks/{s3-backend-66O4FGBH.ts → s3-backend-XGXUOPUN.ts} +23 -54
- package/dist/chunks/s3-backend-XGXUOPUN.ts.map +7 -0
- package/dist/chunks/{setup-switch-IS2PEZ3J.ts → setup-switch-5G2VNJHL.ts} +9 -9
- package/dist/chunks/{setup-wizard-JUJZYNNM.ts → setup-wizard-YIEWGX32.ts} +12 -12
- package/dist/chunks/setup-wizard-YIEWGX32.ts.map +7 -0
- package/dist/chunks/{snapshot-HKEWCVK6.ts → snapshot-JL5DFSXI.ts} +5 -5
- package/dist/chunks/sync-inspection-P3ENJLNF.ts +20 -0
- package/dist/chunks/{sync-operations-LBX3GKCM.ts → sync-operations-AR76EB6C.ts} +85 -201
- package/dist/chunks/sync-operations-AR76EB6C.ts.map +7 -0
- package/dist/chunks/{sync-state-Q2FTXTW6.ts → sync-state-VHJUZ4YG.ts} +6 -6
- package/dist/chunks/{webdav-backend-ECIKD7SO.ts → webdav-backend-3EGUONAT.ts} +20 -51
- package/dist/chunks/webdav-backend-3EGUONAT.ts.map +7 -0
- package/dist/index.ts +49 -74
- package/dist/index.ts.map +3 -3
- package/docs/settings.md +9 -0
- package/package.json +61 -62
- package/src/backends/backend-factory.ts +15 -15
- package/src/backends/backend-identity.ts +31 -31
- package/src/backends/backend-types.ts +42 -42
- package/src/backends/git/git-backend.ts +714 -788
- package/src/backends/git/git-config.ts +104 -111
- package/src/backends/git/git-runner.ts +230 -247
- package/src/backends/git/git-storage.ts +228 -245
- package/src/backends/s3/s3-backend.ts +408 -441
- package/src/backends/s3/s3-client.ts +288 -333
- package/src/backends/sync-backend.ts +65 -65
- package/src/backends/webdav/webdav-backend.ts +542 -589
- package/src/backends/webdav/webdav-client.ts +292 -316
- package/src/backends/webdav/webdav-config.ts +57 -68
- package/src/commands/command-execution.ts +270 -289
- package/src/commands/command-handler.ts +160 -163
- package/src/commands/command-types.ts +13 -13
- package/src/commands/command.ts +190 -196
- package/src/commands/setup-completions.ts +1 -1
- package/src/paths.ts +47 -47
- package/src/settings/config-errors.ts +4 -5
- package/src/settings/config-file.ts +473 -506
- package/src/settings/config.ts +171 -174
- package/src/settings/settings-management.ts +191 -216
- package/src/settings/settings-store.ts +63 -77
- package/src/settings/settings-types.ts +78 -75
- package/src/settings/settings-validation.ts +328 -364
- package/src/snapshot/session-paths.ts +41 -55
- package/src/snapshot/snapshot-apply.ts +197 -229
- package/src/snapshot/snapshot-codec.ts +29 -31
- package/src/snapshot/snapshot-paths.ts +4 -6
- package/src/snapshot/snapshot-transaction.ts +161 -170
- package/src/snapshot/snapshot-types.ts +24 -24
- package/src/snapshot/snapshot.ts +310 -343
- package/src/state/json-file.ts +11 -11
- package/src/state/lock.ts +226 -242
- package/src/state/lockfile-fs.ts +20 -20
- package/src/state/operation-availability.ts +36 -41
- package/src/state/state-directory.ts +171 -173
- package/src/state/state-types.ts +15 -15
- package/src/state/sync-state-store.ts +23 -23
- package/src/sync/automatic-sync.ts +75 -80
- package/src/sync/remote-snapshot.ts +52 -60
- package/src/sync/setup-switch.ts +99 -114
- package/src/sync/signals.ts +3 -5
- package/src/sync/startup-check.ts +97 -111
- package/src/sync/sync-decision.ts +76 -80
- package/src/sync/sync-error-guidance.ts +41 -48
- package/src/sync/sync-errors.ts +10 -10
- package/src/sync/sync-inspection.ts +29 -33
- package/src/sync/sync-loaders.ts +28 -36
- package/src/sync/sync-mutations.ts +510 -622
- package/src/sync/sync-operations.ts +7 -7
- package/src/sync/sync-policy.ts +296 -326
- package/src/sync/sync-queries.ts +175 -194
- package/src/sync/sync-state.ts +74 -100
- package/src/sync-extension.ts +89 -101
- package/src/sync.ts +26 -26
- package/src/types.ts +37 -37
- package/src/ui/automatic-sync-summary.ts +1 -3
- package/src/ui/cancellable-operation.ts +89 -101
- package/src/ui/file-selection.ts +250 -274
- package/src/ui/manager-attention.ts +51 -63
- package/src/ui/manager-recovery.ts +48 -57
- package/src/ui/manager-result-dispatcher.ts +62 -76
- package/src/ui/manager-state.ts +200 -232
- package/src/ui/manager-ui.ts +264 -362
- package/src/ui/remote-selection-ui.ts +547 -588
- package/src/ui/secret-input.ts +197 -205
- package/src/ui/settings-ui.ts +175 -172
- package/src/ui/setup/git-ui.ts +232 -261
- package/src/ui/setup/s3-credentials-ui.ts +58 -65
- package/src/ui/setup/s3-ui.ts +171 -195
- package/src/ui/setup/setup-actions.ts +91 -101
- package/src/ui/setup/setup-location-ui.ts +121 -137
- package/src/ui/setup/setup-name-ui.ts +27 -27
- package/src/ui/setup/setup-prompts.ts +65 -65
- package/src/ui/setup/setup-review.ts +58 -59
- package/src/ui/setup/setup-switcher.ts +156 -160
- package/src/ui/setup/setup-wizard.ts +28 -28
- package/src/ui/setup/text-input.ts +54 -58
- package/src/ui/setup/webdav-ui.ts +246 -293
- package/src/ui/storage-connections-ui.ts +317 -369
- package/src/ui/sync-attention.ts +148 -164
- package/src/ui/sync-format.ts +110 -121
- package/src/ui/sync-resolution-ui.ts +147 -153
- package/src/ui/sync-setups-ui.ts +218 -238
- package/src/ui/sync-status.ts +17 -0
- package/src/ui/terminal-text.ts +2 -2
- package/dist/chunks/chunk-2B2ELUF5.ts.map +0 -7
- package/dist/chunks/chunk-5B3FXBLE.ts.map +0 -7
- package/dist/chunks/chunk-6LP3MCR2.ts.map +0 -7
- package/dist/chunks/chunk-AUSDLD6F.ts.map +0 -7
- package/dist/chunks/chunk-C7XY5NF5.ts.map +0 -7
- package/dist/chunks/chunk-CCSN3F47.ts.map +0 -7
- package/dist/chunks/chunk-CHH26D4F.ts.map +0 -7
- package/dist/chunks/chunk-CPEHGXJV.ts.map +0 -7
- package/dist/chunks/chunk-D5X27LCA.ts.map +0 -7
- package/dist/chunks/chunk-DHVFS4QK.ts.map +0 -7
- package/dist/chunks/chunk-ETTRJDNP.ts.map +0 -7
- package/dist/chunks/chunk-HUDQATCL.ts.map +0 -7
- package/dist/chunks/chunk-IJ6FJFA5.ts.map +0 -7
- package/dist/chunks/chunk-JAAKLL6M.ts.map +0 -7
- package/dist/chunks/chunk-LCRI3ABW.ts.map +0 -7
- package/dist/chunks/chunk-LTBWGSBV.ts.map +0 -7
- package/dist/chunks/chunk-NJFUIVYL.ts.map +0 -7
- package/dist/chunks/chunk-OEX4ABY6.ts.map +0 -7
- package/dist/chunks/chunk-Q2TUM5X6.ts.map +0 -7
- package/dist/chunks/chunk-SL64RLXR.ts.map +0 -7
- package/dist/chunks/chunk-SZWBGPID.ts.map +0 -7
- package/dist/chunks/chunk-T2B5IU74.ts.map +0 -7
- package/dist/chunks/chunk-T3HH3UEJ.ts.map +0 -7
- package/dist/chunks/chunk-TEN2ZBJV.ts.map +0 -7
- package/dist/chunks/chunk-THMF4BVL.ts.map +0 -7
- package/dist/chunks/chunk-YDWUZVPN.ts.map +0 -7
- package/dist/chunks/chunk-YHEMENKV.ts.map +0 -7
- package/dist/chunks/chunk-Z4DQF6L3.ts.map +0 -7
- package/dist/chunks/chunk-ZI5MXMEX.ts.map +0 -7
- package/dist/chunks/file-selection-FTTKAOYX.ts.map +0 -7
- package/dist/chunks/git-backend-GMVS4PD4.ts.map +0 -7
- package/dist/chunks/manager-result-dispatcher-ASKGOSW5.ts +0 -22
- package/dist/chunks/manager-ui-IPAOAWZA.ts.map +0 -7
- package/dist/chunks/remote-selection-ui-QDNR7WEI.ts +0 -21
- package/dist/chunks/s3-backend-66O4FGBH.ts.map +0 -7
- package/dist/chunks/setup-wizard-JUJZYNNM.ts.map +0 -7
- package/dist/chunks/sync-inspection-CST2KL65.ts +0 -20
- package/dist/chunks/sync-operations-LBX3GKCM.ts.map +0 -7
- package/dist/chunks/webdav-backend-ECIKD7SO.ts.map +0 -7
- /package/dist/chunks/{manager-result-dispatcher-ASKGOSW5.ts.map → manager-result-dispatcher-OTUMQM26.ts.map} +0 -0
- /package/dist/chunks/{remote-selection-ui-QDNR7WEI.ts.map → remote-selection-ui-TZTJF32Y.ts.map} +0 -0
- /package/dist/chunks/{setup-switch-IS2PEZ3J.ts.map → setup-switch-5G2VNJHL.ts.map} +0 -0
- /package/dist/chunks/{snapshot-HKEWCVK6.ts.map → snapshot-JL5DFSXI.ts.map} +0 -0
- /package/dist/chunks/{sync-inspection-CST2KL65.ts.map → sync-inspection-P3ENJLNF.ts.map} +0 -0
- /package/dist/chunks/{sync-state-Q2FTXTW6.ts.map → sync-state-VHJUZ4YG.ts.map} +0 -0
|
@@ -9,388 +9,343 @@ const MAX_ERROR_RESPONSE_BYTES = 64 * 1024;
|
|
|
9
9
|
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
10
10
|
|
|
11
11
|
function iso8601Basic(date: Date) {
|
|
12
|
-
|
|
12
|
+
return date.toISOString().replace(/[:-]|\.\d{3}/g, "");
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
function isCloudflareR2Endpoint(endpoint: string | undefined) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
return false;
|
|
25
|
-
}
|
|
16
|
+
const value = endpoint?.trim();
|
|
17
|
+
if (!value) return false;
|
|
18
|
+
try {
|
|
19
|
+
const hostname = new URL(value).hostname.toLowerCase();
|
|
20
|
+
return hostname === "r2.cloudflarestorage.com" || hostname.endsWith(".r2.cloudflarestorage.com");
|
|
21
|
+
} catch {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
26
24
|
}
|
|
27
25
|
|
|
28
26
|
export class S3ObjectAlreadyExistsError extends Error {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
27
|
+
constructor(readonly key: string) {
|
|
28
|
+
super(`S3 object already exists: ${key}`);
|
|
29
|
+
this.name = "S3ObjectAlreadyExistsError";
|
|
30
|
+
}
|
|
33
31
|
}
|
|
34
32
|
|
|
35
33
|
export class S3HttpError extends Error {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
34
|
+
constructor(
|
|
35
|
+
message: string,
|
|
36
|
+
readonly status: number,
|
|
37
|
+
) {
|
|
38
|
+
super(message);
|
|
39
|
+
this.name = "S3HttpError";
|
|
40
|
+
}
|
|
43
41
|
}
|
|
44
42
|
|
|
45
43
|
export class S3Client {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
44
|
+
private config: ResolvedS3Backend;
|
|
45
|
+
private endpoint: URL;
|
|
46
|
+
private signal?: AbortSignal;
|
|
47
|
+
private requestTimeoutMs: number;
|
|
48
|
+
private omitSessionTokenAfterRejection = false;
|
|
51
49
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
this.endpoint = new URL(config.profile.endpoint);
|
|
59
|
-
this.signal = signal;
|
|
60
|
-
this.requestTimeoutMs = requestTimeoutMs;
|
|
61
|
-
}
|
|
50
|
+
constructor(config: ResolvedS3Backend, signal?: AbortSignal, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
|
|
51
|
+
this.config = config;
|
|
52
|
+
this.endpoint = new URL(config.profile.endpoint);
|
|
53
|
+
this.signal = signal;
|
|
54
|
+
this.requestTimeoutMs = requestTimeoutMs;
|
|
55
|
+
}
|
|
62
56
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
throw new S3HttpError(`S3 remote read failed (HTTP ${response.status}).`, response.status);
|
|
83
|
-
}
|
|
57
|
+
/** Read-only access probe. A bare 404 cannot distinguish a missing key from a missing bucket. */
|
|
58
|
+
async diagnoseRead(key: string): Promise<"readable" | "missing-key" | "missing-bucket" | "unknown-404"> {
|
|
59
|
+
this.signal?.throwIfAborted();
|
|
60
|
+
const response = await this.request("GET", key);
|
|
61
|
+
if (response.ok) {
|
|
62
|
+
await response.body?.cancel();
|
|
63
|
+
this.signal?.throwIfAborted();
|
|
64
|
+
return "readable";
|
|
65
|
+
}
|
|
66
|
+
const detail = await this.readErrorText(response);
|
|
67
|
+
this.signal?.throwIfAborted();
|
|
68
|
+
if (response.status === 404) {
|
|
69
|
+
const code = /<Code>\s*([A-Za-z]+)\s*<\/Code>/u.exec(detail)?.[1];
|
|
70
|
+
if (code === "NoSuchBucket") return "missing-bucket";
|
|
71
|
+
if (code === "NoSuchKey") return "missing-key";
|
|
72
|
+
return "unknown-404";
|
|
73
|
+
}
|
|
74
|
+
throw new S3HttpError(`S3 remote read failed (HTTP ${response.status}).`, response.status);
|
|
75
|
+
}
|
|
84
76
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
}
|
|
115
|
-
throw lastError;
|
|
116
|
-
}
|
|
77
|
+
async getJson<T>(key: string): Promise<RemoteObject<T>> {
|
|
78
|
+
const maxAttempts = 3;
|
|
79
|
+
let lastError: unknown;
|
|
80
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
81
|
+
const object = await this.request("GET", key);
|
|
82
|
+
if (object.status === 404) return { missing: true };
|
|
83
|
+
if (!object.ok) {
|
|
84
|
+
throw new S3HttpError(`S3 GET failed (${object.status}): ${await this.readErrorText(object)}`, object.status);
|
|
85
|
+
}
|
|
86
|
+
const body = await readBoundedText(object, MAX_JSON_RESPONSE_BYTES, "S3 JSON response");
|
|
87
|
+
// R2 can intermittently return an empty 200 body (read-after-write
|
|
88
|
+
// inconsistency on a long-lived keep-alive connection), which makes
|
|
89
|
+
// response.json() throw "JSON Parse error: Unexpected EOF" under Bun.
|
|
90
|
+
// Retry so the transient blip is absorbed instead of surfacing as a
|
|
91
|
+
// "pi-sync auto sync skipped" warning on every session start.
|
|
92
|
+
if (body.length > 0) {
|
|
93
|
+
return {
|
|
94
|
+
value: JSON.parse(body) as T,
|
|
95
|
+
etag: normalizeEtag(object.headers.get("etag")),
|
|
96
|
+
missing: false,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
lastError = new Error(`S3 GET returned an empty body for ${key}`);
|
|
100
|
+
if (attempt < maxAttempts) {
|
|
101
|
+
await sleep(250 * attempt, undefined, { signal: this.signal });
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
throw lastError;
|
|
105
|
+
}
|
|
117
106
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
lastError = new Error(`S3 GET returned an empty body for ${key}`);
|
|
146
|
-
if (attempt < maxAttempts) {
|
|
147
|
-
await sleep(250 * attempt, undefined, { signal: this.signal });
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
throw lastError;
|
|
151
|
-
}
|
|
107
|
+
async getBuffer(key: string): Promise<RemoteObject<Buffer>> {
|
|
108
|
+
const maxAttempts = 3;
|
|
109
|
+
let lastError: unknown;
|
|
110
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
111
|
+
const object = await this.request("GET", key);
|
|
112
|
+
if (object.status === 404) return { missing: true };
|
|
113
|
+
if (!object.ok) {
|
|
114
|
+
throw new S3HttpError(`S3 GET failed (${object.status}): ${await this.readErrorText(object)}`, object.status);
|
|
115
|
+
}
|
|
116
|
+
const buffer = await readBoundedBuffer(object, MAX_SNAPSHOT_RESPONSE_BYTES, "S3 snapshot response");
|
|
117
|
+
// R2 can intermittently return an empty 200 body on a long-lived
|
|
118
|
+
// keep-alive connection (same root cause as the getJson retry
|
|
119
|
+
// above). getBuffer is only used for snapshot .json.gz payloads,
|
|
120
|
+
// which are always non-empty, so an empty body is a transient
|
|
121
|
+
// blip, not a legitimate response. Retry so the checksum guard
|
|
122
|
+
// downstream doesn't surface it as "Remote snapshot checksum
|
|
123
|
+
// mismatch".
|
|
124
|
+
if (buffer.length > 0) {
|
|
125
|
+
return { value: buffer, etag: normalizeEtag(object.headers.get("etag")), missing: false };
|
|
126
|
+
}
|
|
127
|
+
lastError = new Error(`S3 GET returned an empty body for ${key}`);
|
|
128
|
+
if (attempt < maxAttempts) {
|
|
129
|
+
await sleep(250 * attempt, undefined, { signal: this.signal });
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
throw lastError;
|
|
133
|
+
}
|
|
152
134
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
135
|
+
async putJson(key: string, value: unknown) {
|
|
136
|
+
const body = Buffer.from(JSON.stringify(value, null, "\t"), "utf8");
|
|
137
|
+
await this.putBuffer(key, body, "application/json");
|
|
138
|
+
}
|
|
157
139
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
`S3 PUT failed (${response.status}): ${await this.readErrorText(response)}`,
|
|
173
|
-
response.status,
|
|
174
|
-
);
|
|
175
|
-
}
|
|
176
|
-
}
|
|
140
|
+
async putBuffer(key: string, body: Buffer, contentType: string, options: { ifAbsent?: boolean } = {}) {
|
|
141
|
+
const headers: Record<string, string> = { "content-type": contentType };
|
|
142
|
+
if (options.ifAbsent) headers["if-none-match"] = "*";
|
|
143
|
+
const response = await this.request("PUT", key, body, headers);
|
|
144
|
+
if (options.ifAbsent && response.status === 412) {
|
|
145
|
+
throw new S3ObjectAlreadyExistsError(key);
|
|
146
|
+
}
|
|
147
|
+
if (!response.ok) {
|
|
148
|
+
throw new S3HttpError(
|
|
149
|
+
`S3 PUT failed (${response.status}): ${await this.readErrorText(response)}`,
|
|
150
|
+
response.status,
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
177
154
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
throw error;
|
|
222
|
-
}
|
|
223
|
-
await response.body?.cancel();
|
|
224
|
-
const retry = await send(undefined);
|
|
225
|
-
if (retry.ok || retry.status === 404) this.omitSessionTokenAfterRejection = true;
|
|
226
|
-
return retry;
|
|
227
|
-
}
|
|
155
|
+
private async request(method: "GET" | "PUT", key: string, body?: Buffer, extraHeaders: Record<string, string> = {}) {
|
|
156
|
+
const url = new URL(this.endpoint.toString());
|
|
157
|
+
url.pathname = posixJoin(url.pathname, this.config.destination.bucket, encodeKey(key));
|
|
158
|
+
const send = async (sessionToken: string | undefined) => {
|
|
159
|
+
const transportSignal = this.transportSignal();
|
|
160
|
+
const headers = await signedHeaders({
|
|
161
|
+
method,
|
|
162
|
+
url,
|
|
163
|
+
body,
|
|
164
|
+
extraHeaders,
|
|
165
|
+
accessKeyId: this.config.profile.accessKeyId,
|
|
166
|
+
secretAccessKey: this.config.profile.secretAccessKey,
|
|
167
|
+
sessionToken,
|
|
168
|
+
region: this.config.profile.region,
|
|
169
|
+
});
|
|
170
|
+
try {
|
|
171
|
+
return await fetch(url, {
|
|
172
|
+
method,
|
|
173
|
+
headers,
|
|
174
|
+
body: body ? new Uint8Array(body) : undefined,
|
|
175
|
+
signal: transportSignal,
|
|
176
|
+
});
|
|
177
|
+
} catch (error) {
|
|
178
|
+
if (error instanceof Error && error.name === "AbortError") throw error;
|
|
179
|
+
if (error instanceof Error && error.name === "TimeoutError") {
|
|
180
|
+
throw new Error("S3 request timed out.", { cause: error });
|
|
181
|
+
}
|
|
182
|
+
throw new Error(`S3 request failed: ${this.redact(errorMessage(error))}`, { cause: error });
|
|
183
|
+
}
|
|
184
|
+
};
|
|
185
|
+
const sessionToken = this.omitSessionTokenAfterRejection ? undefined : this.config.profile.sessionToken;
|
|
186
|
+
const response = await send(sessionToken);
|
|
187
|
+
try {
|
|
188
|
+
if (!(await this.shouldRetryWithoutSessionToken(response, sessionToken))) return response;
|
|
189
|
+
} catch (error) {
|
|
190
|
+
await response.body?.cancel().catch(() => undefined);
|
|
191
|
+
throw error;
|
|
192
|
+
}
|
|
193
|
+
await response.body?.cancel();
|
|
194
|
+
const retry = await send(undefined);
|
|
195
|
+
if (retry.ok || retry.status === 404) this.omitSessionTokenAfterRejection = true;
|
|
196
|
+
return retry;
|
|
197
|
+
}
|
|
228
198
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
199
|
+
private transportSignal() {
|
|
200
|
+
const timeout = AbortSignal.timeout(this.requestTimeoutMs);
|
|
201
|
+
return this.signal ? AbortSignal.any([this.signal, timeout]) : timeout;
|
|
202
|
+
}
|
|
233
203
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
await readBoundedText(response.clone(), MAX_ERROR_RESPONSE_BYTES, "S3 error response"),
|
|
248
|
-
);
|
|
249
|
-
}
|
|
204
|
+
private async shouldRetryWithoutSessionToken(response: Response, sessionToken: string | undefined) {
|
|
205
|
+
if (
|
|
206
|
+
!sessionToken ||
|
|
207
|
+
!isCloudflareR2Endpoint(this.config.profile.endpoint) ||
|
|
208
|
+
response.ok ||
|
|
209
|
+
response.status !== 400
|
|
210
|
+
) {
|
|
211
|
+
return false;
|
|
212
|
+
}
|
|
213
|
+
return isSecurityTokenInvalidArgument(
|
|
214
|
+
await readBoundedText(response.clone(), MAX_ERROR_RESPONSE_BYTES, "S3 error response"),
|
|
215
|
+
);
|
|
216
|
+
}
|
|
250
217
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
}
|
|
259
|
-
}
|
|
218
|
+
private async readErrorText(response: Response) {
|
|
219
|
+
try {
|
|
220
|
+
return this.redact(await readBoundedText(response, MAX_ERROR_RESPONSE_BYTES, "S3 error response"));
|
|
221
|
+
} catch (error) {
|
|
222
|
+
return this.redact(errorMessage(error));
|
|
223
|
+
}
|
|
224
|
+
}
|
|
260
225
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
226
|
+
private redact(value: string) {
|
|
227
|
+
let redacted = value;
|
|
228
|
+
let endpointUsername: string | undefined;
|
|
229
|
+
let endpointPassword: string | undefined;
|
|
230
|
+
let endpointQueryValues: string[] = [];
|
|
231
|
+
try {
|
|
232
|
+
const endpoint = new URL(this.config.profile.endpoint);
|
|
233
|
+
endpointUsername = endpoint.username;
|
|
234
|
+
endpointPassword = endpoint.password;
|
|
235
|
+
endpointQueryValues = [...endpoint.searchParams.values()];
|
|
236
|
+
} catch {
|
|
237
|
+
// The constructor already validates normal runtime endpoints.
|
|
238
|
+
}
|
|
239
|
+
for (const secret of [
|
|
240
|
+
this.config.profile.accessKeyId,
|
|
241
|
+
this.config.profile.secretAccessKey,
|
|
242
|
+
this.config.profile.sessionToken,
|
|
243
|
+
endpointUsername,
|
|
244
|
+
endpointPassword,
|
|
245
|
+
...endpointQueryValues,
|
|
246
|
+
]) {
|
|
247
|
+
if (secret) redacted = redacted.replaceAll(secret, "[REDACTED]");
|
|
248
|
+
}
|
|
249
|
+
return redacted;
|
|
250
|
+
}
|
|
286
251
|
}
|
|
287
252
|
|
|
288
253
|
async function readBoundedText(response: Response, limit: number, label: string) {
|
|
289
|
-
|
|
254
|
+
return (await readBoundedBuffer(response, limit, label)).toString("utf8");
|
|
290
255
|
}
|
|
291
256
|
|
|
292
257
|
async function readBoundedBuffer(response: Response, limit: number, label: string) {
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
258
|
+
const contentLength = Number(response.headers.get("content-length"));
|
|
259
|
+
if (Number.isFinite(contentLength) && contentLength > limit) {
|
|
260
|
+
// Do not await cancellation of a cloned stream before its sibling can be released.
|
|
261
|
+
void response.body?.cancel().catch(() => undefined);
|
|
262
|
+
throw new Error(`${label} exceeds the ${limit}-byte limit.`);
|
|
263
|
+
}
|
|
264
|
+
if (!response.body) return Buffer.alloc(0);
|
|
265
|
+
const reader = response.body.getReader();
|
|
266
|
+
const chunks: Buffer[] = [];
|
|
267
|
+
let total = 0;
|
|
268
|
+
try {
|
|
269
|
+
while (true) {
|
|
270
|
+
const { done, value } = await reader.read();
|
|
271
|
+
if (done) break;
|
|
272
|
+
total += value.byteLength;
|
|
273
|
+
if (total > limit) {
|
|
274
|
+
void reader.cancel().catch(() => undefined);
|
|
275
|
+
throw new Error(`${label} exceeds the ${limit}-byte limit.`);
|
|
276
|
+
}
|
|
277
|
+
chunks.push(Buffer.from(value));
|
|
278
|
+
}
|
|
279
|
+
} finally {
|
|
280
|
+
reader.releaseLock();
|
|
281
|
+
}
|
|
282
|
+
return Buffer.concat(chunks, total);
|
|
318
283
|
}
|
|
319
284
|
|
|
320
285
|
function errorMessage(error: unknown) {
|
|
321
|
-
|
|
286
|
+
return error instanceof Error ? error.message : String(error);
|
|
322
287
|
}
|
|
323
288
|
|
|
324
289
|
async function signedHeaders(input: {
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
290
|
+
method: string;
|
|
291
|
+
url: URL;
|
|
292
|
+
body?: Buffer;
|
|
293
|
+
extraHeaders: Record<string, string>;
|
|
294
|
+
accessKeyId: string;
|
|
295
|
+
secretAccessKey: string;
|
|
296
|
+
sessionToken?: string;
|
|
297
|
+
region: string;
|
|
333
298
|
}) {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
"aws4_request",
|
|
367
|
-
);
|
|
368
|
-
const signature = createHmac("sha256", signingKey).update(stringToSign).digest("hex");
|
|
369
|
-
return {
|
|
370
|
-
...headers,
|
|
371
|
-
authorization: `AWS4-HMAC-SHA256 Credential=${input.accessKeyId}/${scope}, SignedHeaders=${signedHeaderNames.join(";")}, Signature=${signature}`,
|
|
372
|
-
};
|
|
299
|
+
const now = new Date();
|
|
300
|
+
const amzDate = iso8601Basic(now);
|
|
301
|
+
const dateStamp = amzDate.slice(0, 8);
|
|
302
|
+
const payloadHash = sha256(input.body ?? Buffer.alloc(0));
|
|
303
|
+
const headers: Record<string, string> = {
|
|
304
|
+
...lowercaseKeys(input.extraHeaders),
|
|
305
|
+
host: input.url.host,
|
|
306
|
+
"x-amz-content-sha256": payloadHash,
|
|
307
|
+
"x-amz-date": amzDate,
|
|
308
|
+
};
|
|
309
|
+
if (input.sessionToken) headers["x-amz-security-token"] = input.sessionToken;
|
|
310
|
+
const signedHeaderNames = Object.keys(headers).sort();
|
|
311
|
+
const canonicalHeaders = signedHeaderNames.map((name) => `${name}:${headers[name]?.trim()}\n`).join("");
|
|
312
|
+
const canonicalRequest = [
|
|
313
|
+
input.method,
|
|
314
|
+
input.url.pathname,
|
|
315
|
+
input.url.searchParams.toString(),
|
|
316
|
+
canonicalHeaders,
|
|
317
|
+
signedHeaderNames.join(";"),
|
|
318
|
+
payloadHash,
|
|
319
|
+
].join("\n");
|
|
320
|
+
const scope = `${dateStamp}/${input.region}/s3/aws4_request`;
|
|
321
|
+
const stringToSign = ["AWS4-HMAC-SHA256", amzDate, scope, sha256(Buffer.from(canonicalRequest))].join("\n");
|
|
322
|
+
const signingKey = hmac(
|
|
323
|
+
hmac(hmac(hmac(Buffer.from(`AWS4${input.secretAccessKey}`), dateStamp), input.region), "s3"),
|
|
324
|
+
"aws4_request",
|
|
325
|
+
);
|
|
326
|
+
const signature = createHmac("sha256", signingKey).update(stringToSign).digest("hex");
|
|
327
|
+
return {
|
|
328
|
+
...headers,
|
|
329
|
+
authorization: `AWS4-HMAC-SHA256 Credential=${input.accessKeyId}/${scope}, SignedHeaders=${signedHeaderNames.join(";")}, Signature=${signature}`,
|
|
330
|
+
};
|
|
373
331
|
}
|
|
374
332
|
|
|
375
333
|
function hmac(key: Buffer, value: string) {
|
|
376
|
-
|
|
334
|
+
return createHmac("sha256", key).update(value).digest();
|
|
377
335
|
}
|
|
378
336
|
|
|
379
337
|
function sha256(value: Buffer) {
|
|
380
|
-
|
|
338
|
+
return createHash("sha256").update(value).digest("hex");
|
|
381
339
|
}
|
|
382
340
|
|
|
383
341
|
function lowercaseKeys(value: Record<string, string>) {
|
|
384
|
-
|
|
342
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key.toLowerCase(), item]));
|
|
385
343
|
}
|
|
386
344
|
|
|
387
345
|
function normalizeEtag(value: string | null) {
|
|
388
|
-
|
|
346
|
+
return value ?? undefined;
|
|
389
347
|
}
|
|
390
348
|
|
|
391
349
|
function isSecurityTokenInvalidArgument(text: string) {
|
|
392
|
-
|
|
393
|
-
text.includes("<Code>InvalidArgument</Code>") &&
|
|
394
|
-
text.includes("<Message>X-Amz-Security-Token</Message>")
|
|
395
|
-
);
|
|
350
|
+
return text.includes("<Code>InvalidArgument</Code>") && text.includes("<Message>X-Amz-Security-Token</Message>");
|
|
396
351
|
}
|