@vincentt-xr/cli 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/MIGRATION.md +31 -0
- package/README.md +94 -1
- package/dist/cli/index.js +170 -84
- package/dist/cli/usage-text.js +32 -10
- package/dist/domain/api.d.ts +10 -8
- package/dist/domain/api.js +14 -9
- package/dist/domain/command.js +25 -16
- package/dist/domain/copy.d.ts +6 -3
- package/dist/domain/copy.js +10 -6
- package/dist/dx/command.d.ts +24 -4
- package/dist/dx/command.js +80 -25
- package/dist/dx/consent.js +6 -5
- package/dist/dx/journal.d.ts +17 -2
- package/dist/dx/journal.js +19 -3
- package/dist/dx/report.js +4 -3
- package/dist/dx/reportIo.js +6 -5
- package/dist/dx/send.js +6 -5
- package/dist/lib/backend.d.ts +0 -16
- package/dist/lib/backend.js +2 -26
- package/dist/lib/detect.d.ts +87 -0
- package/dist/lib/detect.js +867 -0
- package/dist/lib/diagnostics.js +2 -2
- package/dist/lib/feedback.js +1 -1
- package/dist/lib/feedbackCopy.js +1 -1
- package/dist/lib/initFrom.js +4 -4
- package/dist/lib/lifecycle.d.ts +2 -0
- package/dist/lib/lifecycle.js +9 -3
- package/dist/lib/listProjects.js +2 -1
- package/dist/lib/projects.d.ts +37 -2
- package/dist/lib/projects.js +86 -21
- package/dist/lib/settings.d.ts +113 -0
- package/dist/lib/settings.js +284 -0
- package/dist/lib/settingsCopy.d.ts +6 -0
- package/dist/lib/settingsCopy.js +99 -0
- package/dist/lib/settle.d.ts +61 -0
- package/dist/lib/settle.js +338 -0
- package/dist/lib/template.d.ts +19 -0
- package/dist/lib/template.js +73 -0
- package/dist/lib/terminalText.d.ts +3 -0
- package/dist/lib/terminalText.js +10 -0
- package/dist/lib/version.js +5 -11
- package/dist/lib/whoami.js +10 -3
- package/dist/library/assetsCommand.js +1 -1
- package/dist/library/client.js +8 -3
- package/dist/library/render.js +8 -6
- package/dist/login/authBackend.js +3 -2
- package/dist/login/ensure.d.ts +8 -2
- package/dist/login/ensure.js +21 -10
- package/dist/login/guard.js +5 -4
- package/dist/login/loginCommand.d.ts +2 -2
- package/dist/login/loginCommand.js +29 -5
- package/dist/login/pkce.js +1 -1
- package/dist/login/pkceLogin.js +1 -1
- package/dist/platform/outdated.js +2 -2
- package/dist/platform/upgradeReport.js +16 -12
- package/dist/preview/api.d.ts +5 -11
- package/dist/preview/api.js +13 -15
- package/dist/preview/command.d.ts +29 -0
- package/dist/preview/command.js +61 -1
- package/dist/preview/copy.d.ts +35 -3
- package/dist/preview/copy.js +102 -10
- package/dist/preview/proxy.d.ts +38 -1
- package/dist/preview/proxy.js +151 -2
- package/dist/preview/qrPage.js +1 -1
- package/dist/preview/runner.d.ts +25 -13
- package/dist/preview/runner.js +127 -26
- package/dist/preview/viewers.js +1 -1
- package/dist/publish/api.d.ts +22 -5
- package/dist/publish/api.js +80 -9
- package/dist/publish/command.d.ts +27 -0
- package/dist/publish/command.js +261 -50
- package/dist/publish/copy.d.ts +78 -5
- package/dist/publish/copy.js +271 -37
- package/dist/publish/fit.d.ts +15 -0
- package/dist/publish/fit.js +103 -0
- package/dist/publish/manifest.d.ts +44 -3
- package/dist/publish/manifest.js +172 -3
- package/dist/publish/prompt.d.ts +4 -0
- package/dist/publish/prompt.js +15 -1
- package/dist/publish/upload.d.ts +59 -5
- package/dist/publish/upload.js +138 -30
- package/dist/publish/visibility.d.ts +21 -0
- package/dist/publish/visibility.js +68 -0
- package/dist/relay/annotations.js +9 -4
- package/dist/relay/server.d.ts +21 -0
- package/dist/relay/server.js +35 -1
- package/dist/shared/authError.d.ts +27 -0
- package/dist/shared/authError.js +48 -4
- package/dist/shared/config.d.ts +21 -1
- package/dist/shared/config.js +39 -8
- package/dist/shared/countLimit.d.ts +7 -2
- package/dist/shared/countLimit.js +12 -5
- package/dist/shared/http.d.ts +23 -0
- package/dist/shared/http.js +58 -0
- package/package.json +7 -3
package/dist/publish/manifest.js
CHANGED
|
@@ -11,13 +11,178 @@ import { normalizePath, validatePathSet } from "./paths.js";
|
|
|
11
11
|
export class ManifestError extends Error {
|
|
12
12
|
reason;
|
|
13
13
|
offendingPath;
|
|
14
|
-
|
|
14
|
+
paths;
|
|
15
|
+
stats;
|
|
16
|
+
constructor(reason, offendingPath, message, detail = {}) {
|
|
15
17
|
super(message);
|
|
16
18
|
this.reason = reason;
|
|
17
19
|
this.offendingPath = offendingPath;
|
|
18
20
|
this.name = "ManifestError";
|
|
21
|
+
this.paths = detail.paths ?? [];
|
|
22
|
+
this.stats = detail.stats;
|
|
19
23
|
}
|
|
20
24
|
}
|
|
25
|
+
/** The only dotted names allowed, and only at the output root. Case-insensitive. */
|
|
26
|
+
const ALLOWED_ROOT_DIR = ".well-known";
|
|
27
|
+
const ALLOWED_ROOT_FILE = ".nojekyll";
|
|
28
|
+
const LFS_POINTER_MAX_BYTES = 1024;
|
|
29
|
+
const LFS_FIRST_LINE = "version https://git-lfs.github.com/spec/v1";
|
|
30
|
+
function isInside(child, parent) {
|
|
31
|
+
const rel = path.relative(parent, child);
|
|
32
|
+
return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel);
|
|
33
|
+
}
|
|
34
|
+
function hasDottedSegment(rel) {
|
|
35
|
+
return rel.split(/[\\/]/).some((seg) => seg.startsWith("."));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The leak walk. Unlike `walkDist` it records what it does not upload: a dotted
|
|
39
|
+
* name is an offender (a folder once, with a trailing slash, never descended),
|
|
40
|
+
* and every symlink is resolved and judged against the real repo root.
|
|
41
|
+
*/
|
|
42
|
+
async function walkOutput(outReal, rootReal) {
|
|
43
|
+
const files = [];
|
|
44
|
+
const offenders = [];
|
|
45
|
+
const visit = async (dir) => {
|
|
46
|
+
const entries = await fs.readdir(dir, { withFileTypes: true });
|
|
47
|
+
entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
48
|
+
for (const entry of entries) {
|
|
49
|
+
const full = path.join(dir, entry.name);
|
|
50
|
+
const rel = path.relative(outReal, full).split(path.sep).join("/");
|
|
51
|
+
const atRoot = dir === outReal;
|
|
52
|
+
const lower = entry.name.toLowerCase();
|
|
53
|
+
if (entry.name.startsWith(".")) {
|
|
54
|
+
const allowed = atRoot &&
|
|
55
|
+
((lower === ALLOWED_ROOT_DIR && entry.isDirectory()) ||
|
|
56
|
+
(lower === ALLOWED_ROOT_FILE && entry.isFile()));
|
|
57
|
+
if (!allowed) {
|
|
58
|
+
offenders.push({ path: entry.isDirectory() ? `${rel}/` : rel });
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (entry.isSymbolicLink()) {
|
|
63
|
+
const target = await symlinkTarget(full);
|
|
64
|
+
if (!isInside(target, rootReal)) {
|
|
65
|
+
offenders.push({ path: rel, target });
|
|
66
|
+
}
|
|
67
|
+
else if (hasDottedSegment(path.relative(rootReal, target))) {
|
|
68
|
+
offenders.push({
|
|
69
|
+
path: rel,
|
|
70
|
+
target: path.relative(rootReal, target).split(path.sep).join("/"),
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
// A symlink that resolves cleanly inside is not uploaded, as before.
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (entry.isDirectory()) {
|
|
77
|
+
await visit(full);
|
|
78
|
+
}
|
|
79
|
+
else if (entry.isFile()) {
|
|
80
|
+
const stat = await fs.stat(full);
|
|
81
|
+
files.push({ rel, size: stat.size });
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
await visit(outReal);
|
|
86
|
+
return { files, offenders };
|
|
87
|
+
}
|
|
88
|
+
/** Where a symlink lands: its realpath, or its lexical target when it dangles. */
|
|
89
|
+
async function symlinkTarget(link) {
|
|
90
|
+
try {
|
|
91
|
+
return await fs.realpath(link);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
const raw = await fs.readlink(link).catch(() => link);
|
|
95
|
+
return path.resolve(path.dirname(link), raw);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** A Git LFS pointer file: at most 1 KB, the spec's exact first line, and an oid line. */
|
|
99
|
+
export async function isLfsPointer(absPath, size) {
|
|
100
|
+
if (size > LFS_POINTER_MAX_BYTES)
|
|
101
|
+
return false;
|
|
102
|
+
const head = await fs.readFile(absPath, "utf8");
|
|
103
|
+
const lines = head.split("\n");
|
|
104
|
+
if (lines[0] !== LFS_FIRST_LINE)
|
|
105
|
+
return false;
|
|
106
|
+
return lines.some((l) => l.startsWith("oid sha256:"));
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* The leak and LFS checks, in that order, before anything is hashed or sent.
|
|
110
|
+
* Throws ManifestError; returns nothing on a clean output.
|
|
111
|
+
*/
|
|
112
|
+
/**
|
|
113
|
+
* Folders an output must never be, or sit inside: the repository's history and
|
|
114
|
+
* the Vincentt binding. Compared case-insensitively, as a default macOS volume
|
|
115
|
+
* resolves `.GIT` to `.git`. Other dotted parents stay allowed (`.output/public`).
|
|
116
|
+
*/
|
|
117
|
+
const FORBIDDEN_OUTPUT_SEGMENTS = new Set([".git", ".vincentt"]);
|
|
118
|
+
function forbiddenSegmentIn(rel) {
|
|
119
|
+
return rel
|
|
120
|
+
.split(/[\\/]/)
|
|
121
|
+
.find((seg) => FORBIDDEN_OUTPUT_SEGMENTS.has(seg.toLowerCase()));
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* The leak and LFS checks, in that order, before anything is hashed or sent.
|
|
125
|
+
* Returns the output-relative file list the checks passed, which is exactly the
|
|
126
|
+
* list the manifest is built from: what is checked is what is uploaded.
|
|
127
|
+
*/
|
|
128
|
+
export async function checkOutput(distDir, repoRoot) {
|
|
129
|
+
const rootReal = await fs.realpath(repoRoot).catch(() => path.resolve(repoRoot));
|
|
130
|
+
const lexical = path.resolve(distDir);
|
|
131
|
+
const lexicalRoot = path.resolve(repoRoot);
|
|
132
|
+
if (lexical === lexicalRoot) {
|
|
133
|
+
throw new ManifestError("output_is_repo_root", distDir, `${distDir} is the repo root`);
|
|
134
|
+
}
|
|
135
|
+
if (!isInside(lexical, lexicalRoot) && !isInside(lexical, rootReal)) {
|
|
136
|
+
throw new ManifestError("output_outside_repo", distDir, `${distDir} is outside the repo`);
|
|
137
|
+
}
|
|
138
|
+
let outReal;
|
|
139
|
+
try {
|
|
140
|
+
outReal = await fs.realpath(distDir);
|
|
141
|
+
}
|
|
142
|
+
catch (err) {
|
|
143
|
+
if (err.code === "ENOENT") {
|
|
144
|
+
throw new ManifestError("no_dist", distDir, `no dist at ${distDir}`);
|
|
145
|
+
}
|
|
146
|
+
throw err;
|
|
147
|
+
}
|
|
148
|
+
if (outReal === rootReal) {
|
|
149
|
+
throw new ManifestError("output_is_repo_root", distDir, `${distDir} resolves to the repo root`);
|
|
150
|
+
}
|
|
151
|
+
if (!isInside(outReal, rootReal)) {
|
|
152
|
+
throw new ManifestError("output_outside_repo", distDir, `${distDir} resolves outside the repo`);
|
|
153
|
+
}
|
|
154
|
+
const hiddenParent = forbiddenSegmentIn(path.relative(lexicalRoot, lexical)) ??
|
|
155
|
+
forbiddenSegmentIn(path.relative(rootReal, outReal));
|
|
156
|
+
if (hiddenParent !== undefined) {
|
|
157
|
+
throw new ManifestError("output_in_hidden", hiddenParent, `${distDir} is inside ${hiddenParent}`);
|
|
158
|
+
}
|
|
159
|
+
if (!(await fs.stat(outReal)).isDirectory()) {
|
|
160
|
+
throw new ManifestError("no_dist", distDir, `no dist at ${distDir}`);
|
|
161
|
+
}
|
|
162
|
+
const walk = await walkOutput(outReal, rootReal);
|
|
163
|
+
const stats = {
|
|
164
|
+
files: walk.files.length,
|
|
165
|
+
bytes: walk.files.reduce((n, f) => n + f.size, 0),
|
|
166
|
+
};
|
|
167
|
+
if (walk.offenders.length > 0) {
|
|
168
|
+
throw new ManifestError("secret_path", walk.offenders[0].path, "hidden paths in the output", {
|
|
169
|
+
paths: walk.offenders,
|
|
170
|
+
stats,
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
const pointers = [];
|
|
174
|
+
for (const f of walk.files) {
|
|
175
|
+
if (await isLfsPointer(path.join(outReal, f.rel), f.size))
|
|
176
|
+
pointers.push({ path: f.rel });
|
|
177
|
+
}
|
|
178
|
+
if (pointers.length > 0) {
|
|
179
|
+
throw new ManifestError("lfs_pointer", pointers[0].path, "Git LFS pointers in the output", {
|
|
180
|
+
paths: pointers,
|
|
181
|
+
stats,
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
return walk.files;
|
|
185
|
+
}
|
|
21
186
|
/** Recursively list files under `dir` as dir-relative POSIX paths. Symlinks are not followed. */
|
|
22
187
|
export async function walkDist(dir, base = dir) {
|
|
23
188
|
let entries;
|
|
@@ -54,8 +219,12 @@ export async function hashFile(absPath) {
|
|
|
54
219
|
* hashed, so the string this returns is the string the API validates, signs, and
|
|
55
220
|
* later lists — there is no second form anywhere.
|
|
56
221
|
*/
|
|
57
|
-
export async function buildManifest(distDir) {
|
|
58
|
-
|
|
222
|
+
export async function buildManifest(distDir, opts = {}) {
|
|
223
|
+
// With a repo root, the list the leak and LFS checks passed IS the upload
|
|
224
|
+
// list. A second walk could see a file created after the checks ran.
|
|
225
|
+
const raw = opts.repoRoot !== undefined
|
|
226
|
+
? (await checkOutput(distDir, opts.repoRoot)).map((f) => f.rel)
|
|
227
|
+
: await walkDist(distDir);
|
|
59
228
|
const normalized = [];
|
|
60
229
|
for (const rel of raw) {
|
|
61
230
|
const result = normalizePath(rel);
|
package/dist/publish/prompt.d.ts
CHANGED
|
@@ -3,5 +3,9 @@ export type Confirm = (prompt: string, defaultYes: boolean) => Promise<boolean>;
|
|
|
3
3
|
export declare function stdinConfirm(input?: NodeJS.ReadableStream, output?: NodeJS.WritableStream): Confirm;
|
|
4
4
|
/** Reads one free-text line. An empty answer takes the offered default. */
|
|
5
5
|
export type Ask = (prompt: string, fallback: string) => Promise<string>;
|
|
6
|
+
/** Input ended (Ctrl-D, a closed pipe) before a free-text question was answered. */
|
|
7
|
+
export declare class PromptClosedError extends Error {
|
|
8
|
+
constructor();
|
|
9
|
+
}
|
|
6
10
|
export declare function stdinAsk(input?: NodeJS.ReadableStream, output?: NodeJS.WritableStream): Ask;
|
|
7
11
|
export declare function interpretAnswer(answer: string, defaultYes: boolean): boolean;
|
package/dist/publish/prompt.js
CHANGED
|
@@ -32,11 +32,25 @@ export function stdinConfirm(input = process.stdin, output = process.stdout) {
|
|
|
32
32
|
}
|
|
33
33
|
};
|
|
34
34
|
}
|
|
35
|
+
/** Input ended (Ctrl-D, a closed pipe) before a free-text question was answered. */
|
|
36
|
+
export class PromptClosedError extends Error {
|
|
37
|
+
constructor() {
|
|
38
|
+
super("Input ended before the question was answered. Nothing more was done.");
|
|
39
|
+
this.name = "PromptClosedError";
|
|
40
|
+
}
|
|
41
|
+
}
|
|
35
42
|
export function stdinAsk(input = process.stdin, output = process.stdout) {
|
|
36
43
|
return async (prompt, fallback) => {
|
|
37
44
|
const rl = readline.createInterface({ input, output });
|
|
38
45
|
try {
|
|
39
|
-
|
|
46
|
+
// EOF is a refusal, not an empty answer: taking the fallback here would
|
|
47
|
+
// register a name or address the creator never agreed to.
|
|
48
|
+
const answer = await new Promise((resolve) => {
|
|
49
|
+
rl.question(prompt, resolve);
|
|
50
|
+
rl.once("close", () => resolve(undefined));
|
|
51
|
+
});
|
|
52
|
+
if (answer === undefined)
|
|
53
|
+
throw new PromptClosedError();
|
|
40
54
|
return answer.trim() === "" ? fallback : answer.trim();
|
|
41
55
|
}
|
|
42
56
|
finally {
|
package/dist/publish/upload.d.ts
CHANGED
|
@@ -4,7 +4,23 @@ export declare class UploadFailed extends Error {
|
|
|
4
4
|
readonly path: string;
|
|
5
5
|
readonly status: number;
|
|
6
6
|
readonly badDigest: boolean;
|
|
7
|
-
|
|
7
|
+
/** S3's own error code, when the response carried one. */
|
|
8
|
+
readonly s3Code?: string | undefined;
|
|
9
|
+
/** A transport error or 5xx outlasted every retry: re-running is the remedy. */
|
|
10
|
+
readonly retryable: boolean;
|
|
11
|
+
constructor(path: string, status: number, badDigest: boolean,
|
|
12
|
+
/** S3's own error code, when the response carried one. */
|
|
13
|
+
s3Code?: string | undefined,
|
|
14
|
+
/** A transport error or 5xx outlasted every retry: re-running is the remedy. */
|
|
15
|
+
retryable?: boolean);
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A presigned URL was refused because it expired. The upload has to be
|
|
19
|
+
* re-opened for fresh URLs; nothing about the file is wrong.
|
|
20
|
+
*/
|
|
21
|
+
export declare class UploadExpired extends Error {
|
|
22
|
+
readonly path: string;
|
|
23
|
+
constructor(path: string);
|
|
8
24
|
}
|
|
9
25
|
export interface UploadProgress {
|
|
10
26
|
filesDone: number;
|
|
@@ -19,12 +35,50 @@ export interface UploadOptions {
|
|
|
19
35
|
concurrency?: number;
|
|
20
36
|
onProgress?: (p: UploadProgress) => void;
|
|
21
37
|
fetchImpl?: typeof fetch;
|
|
22
|
-
/**
|
|
38
|
+
/** Attempts per file for a transport error or 5xx. A BadDigest is retried once. */
|
|
23
39
|
maxAttempts?: number;
|
|
40
|
+
/** Test seams for the backoff. */
|
|
41
|
+
sleep?: (ms: number) => Promise<void>;
|
|
42
|
+
random?: () => number;
|
|
24
43
|
}
|
|
25
44
|
/**
|
|
26
|
-
* PUT
|
|
27
|
-
*
|
|
28
|
-
*
|
|
45
|
+
* One PUT's deadline: a minute, plus the time the file takes at 64 KB/s. A fixed
|
|
46
|
+
* deadline would fail every large file on a slow uplink; none at all lets a
|
|
47
|
+
* stalled connection hang the publish forever. The api sizes the upload window
|
|
48
|
+
* at the same rate, so a PUT this deadline still waits for is never refused by
|
|
49
|
+
* an expired link.
|
|
50
|
+
*/
|
|
51
|
+
export declare function putDeadlineMs(bytes: number): number;
|
|
52
|
+
/** Exponential backoff with full jitter: uniform in [0, min(cap, base * 2^(n-1))). */
|
|
53
|
+
export declare function backoffMs(attempt: number, random?: () => number): number;
|
|
54
|
+
export type S3Outcome = {
|
|
55
|
+
kind: "ok";
|
|
56
|
+
} | {
|
|
57
|
+
kind: "expired";
|
|
58
|
+
} | {
|
|
59
|
+
kind: "bad_digest";
|
|
60
|
+
code: string;
|
|
61
|
+
} | {
|
|
62
|
+
kind: "retry";
|
|
63
|
+
code?: string;
|
|
64
|
+
} | {
|
|
65
|
+
kind: "fatal";
|
|
66
|
+
code?: string;
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* What a PUT's response means, read from S3's own error code rather than the
|
|
70
|
+
* status alone. A 400 is many things — S3 answers a stalled body with 400
|
|
71
|
+
* RequestTimeout — and only BadDigest means the bytes on disk are not the bytes
|
|
72
|
+
* that were hashed.
|
|
73
|
+
*/
|
|
74
|
+
export declare function classifyS3Response(status: number, body: string): S3Outcome;
|
|
75
|
+
/**
|
|
76
|
+
* PUT every target. The api returns targets only for files not yet landed when
|
|
77
|
+
* it resumes an open upload, so a re-run simply uploads fewer — resumption is
|
|
78
|
+
* silent, because the creator did not ask for a resume, they asked to publish.
|
|
79
|
+
*
|
|
80
|
+
* On the first file that cannot land, no new PUT starts; the ones in flight
|
|
81
|
+
* finish, and the first failure is thrown (UploadExpired only when nothing
|
|
82
|
+
* worse happened, since re-opening cannot fix a changed file).
|
|
29
83
|
*/
|
|
30
84
|
export declare function uploadAll(opts: UploadOptions): Promise<void>;
|
package/dist/publish/upload.js
CHANGED
|
@@ -7,35 +7,109 @@
|
|
|
7
7
|
// makes every published index.html download instead of render in the other
|
|
8
8
|
// (§2.7's contract traps 3 and 4).
|
|
9
9
|
//
|
|
10
|
-
// The CLI never synthesizes a MIME type and never parses the presigned URL.
|
|
11
|
-
|
|
10
|
+
// The CLI never synthesizes a MIME type and never parses the presigned URL. The
|
|
11
|
+
// one header it adds is Content-Length, which S3 requires on a PUT and which a
|
|
12
|
+
// streamed body does not carry on its own.
|
|
13
|
+
import { createReadStream, promises as fs } from "node:fs";
|
|
12
14
|
import path from "node:path";
|
|
15
|
+
import { Readable } from "node:stream";
|
|
16
|
+
import { withDeadline } from "../shared/http.js";
|
|
13
17
|
export class UploadFailed extends Error {
|
|
14
18
|
path;
|
|
15
19
|
status;
|
|
16
20
|
badDigest;
|
|
17
|
-
|
|
18
|
-
|
|
21
|
+
s3Code;
|
|
22
|
+
retryable;
|
|
23
|
+
constructor(path, status, badDigest,
|
|
24
|
+
/** S3's own error code, when the response carried one. */
|
|
25
|
+
s3Code,
|
|
26
|
+
/** A transport error or 5xx outlasted every retry: re-running is the remedy. */
|
|
27
|
+
retryable = false) {
|
|
28
|
+
super(`upload failed for ${path} (${status}${s3Code ? ` ${s3Code}` : ""})`);
|
|
19
29
|
this.path = path;
|
|
20
30
|
this.status = status;
|
|
21
31
|
this.badDigest = badDigest;
|
|
32
|
+
this.s3Code = s3Code;
|
|
33
|
+
this.retryable = retryable;
|
|
22
34
|
this.name = "UploadFailed";
|
|
23
35
|
}
|
|
24
36
|
}
|
|
37
|
+
/**
|
|
38
|
+
* A presigned URL was refused because it expired. The upload has to be
|
|
39
|
+
* re-opened for fresh URLs; nothing about the file is wrong.
|
|
40
|
+
*/
|
|
41
|
+
export class UploadExpired extends Error {
|
|
42
|
+
path;
|
|
43
|
+
constructor(path) {
|
|
44
|
+
super(`the upload link for ${path} expired`);
|
|
45
|
+
this.path = path;
|
|
46
|
+
this.name = "UploadExpired";
|
|
47
|
+
}
|
|
48
|
+
}
|
|
25
49
|
const DEFAULT_CONCURRENCY = 6;
|
|
50
|
+
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
51
|
+
const BACKOFF_BASE_MS = 500;
|
|
52
|
+
const BACKOFF_CAP_MS = 15_000;
|
|
53
|
+
/**
|
|
54
|
+
* One PUT's deadline: a minute, plus the time the file takes at 64 KB/s. A fixed
|
|
55
|
+
* deadline would fail every large file on a slow uplink; none at all lets a
|
|
56
|
+
* stalled connection hang the publish forever. The api sizes the upload window
|
|
57
|
+
* at the same rate, so a PUT this deadline still waits for is never refused by
|
|
58
|
+
* an expired link.
|
|
59
|
+
*/
|
|
60
|
+
export function putDeadlineMs(bytes) {
|
|
61
|
+
return 60_000 + Math.ceil(bytes / 64_000) * 1000;
|
|
62
|
+
}
|
|
63
|
+
/** Exponential backoff with full jitter: uniform in [0, min(cap, base * 2^(n-1))). */
|
|
64
|
+
export function backoffMs(attempt, random = Math.random) {
|
|
65
|
+
const ceiling = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * 2 ** Math.max(0, attempt - 1));
|
|
66
|
+
return Math.floor(random() * ceiling);
|
|
67
|
+
}
|
|
26
68
|
/**
|
|
27
|
-
* PUT
|
|
28
|
-
*
|
|
29
|
-
*
|
|
69
|
+
* What a PUT's response means, read from S3's own error code rather than the
|
|
70
|
+
* status alone. A 400 is many things — S3 answers a stalled body with 400
|
|
71
|
+
* RequestTimeout — and only BadDigest means the bytes on disk are not the bytes
|
|
72
|
+
* that were hashed.
|
|
73
|
+
*/
|
|
74
|
+
export function classifyS3Response(status, body) {
|
|
75
|
+
if (status >= 200 && status < 300)
|
|
76
|
+
return { kind: "ok" };
|
|
77
|
+
const code = /<Code>([^<]+)<\/Code>/.exec(body)?.[1];
|
|
78
|
+
const message = /<Message>([^<]*)<\/Message>/.exec(body)?.[1] ?? "";
|
|
79
|
+
if (status === 403 && code === "AccessDenied" && /request has expired/i.test(message)) {
|
|
80
|
+
return { kind: "expired" };
|
|
81
|
+
}
|
|
82
|
+
if (code === "BadDigest")
|
|
83
|
+
return { kind: "bad_digest", code };
|
|
84
|
+
if (code === "RequestTimeout" || code === "IncompleteBody")
|
|
85
|
+
return { kind: "retry", code };
|
|
86
|
+
if (status >= 500 || status === 408 || status === 429)
|
|
87
|
+
return { kind: "retry", code };
|
|
88
|
+
return { kind: "fatal", code };
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* PUT every target. The api returns targets only for files not yet landed when
|
|
92
|
+
* it resumes an open upload, so a re-run simply uploads fewer — resumption is
|
|
93
|
+
* silent, because the creator did not ask for a resume, they asked to publish.
|
|
94
|
+
*
|
|
95
|
+
* On the first file that cannot land, no new PUT starts; the ones in flight
|
|
96
|
+
* finish, and the first failure is thrown (UploadExpired only when nothing
|
|
97
|
+
* worse happened, since re-opening cannot fix a changed file).
|
|
30
98
|
*/
|
|
31
99
|
export async function uploadAll(opts) {
|
|
32
100
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
33
101
|
const sizes = new Map(opts.manifest.map((f) => [f.path, f.size]));
|
|
34
|
-
const bytesTotal = opts.
|
|
102
|
+
const bytesTotal = opts.targets.reduce((n, t) => n + (sizes.get(t.path) ?? 0), 0);
|
|
35
103
|
const queue = [...opts.targets];
|
|
36
|
-
const
|
|
104
|
+
const retry = {
|
|
105
|
+
maxAttempts: opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS,
|
|
106
|
+
sleep: opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms))),
|
|
107
|
+
random: opts.random ?? Math.random,
|
|
108
|
+
};
|
|
37
109
|
let filesDone = 0;
|
|
38
110
|
let bytesDone = 0;
|
|
111
|
+
let failed = false;
|
|
112
|
+
let failure;
|
|
39
113
|
const report = () => opts.onProgress?.({
|
|
40
114
|
filesDone,
|
|
41
115
|
filesTotal: opts.targets.length,
|
|
@@ -44,11 +118,19 @@ export async function uploadAll(opts) {
|
|
|
44
118
|
});
|
|
45
119
|
report();
|
|
46
120
|
const worker = async () => {
|
|
47
|
-
|
|
121
|
+
while (!failed) {
|
|
48
122
|
const target = queue.shift();
|
|
49
123
|
if (!target)
|
|
50
124
|
return;
|
|
51
|
-
|
|
125
|
+
try {
|
|
126
|
+
await putOne(target, opts.distDir, fetchImpl, retry);
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
if (!failed || failure instanceof UploadExpired)
|
|
130
|
+
failure = err;
|
|
131
|
+
failed = true;
|
|
132
|
+
return;
|
|
133
|
+
}
|
|
52
134
|
filesDone += 1;
|
|
53
135
|
bytesDone += sizes.get(target.path) ?? 0;
|
|
54
136
|
report();
|
|
@@ -56,38 +138,64 @@ export async function uploadAll(opts) {
|
|
|
56
138
|
};
|
|
57
139
|
const workers = Array.from({ length: Math.min(opts.concurrency ?? DEFAULT_CONCURRENCY, queue.length || 1) }, () => worker());
|
|
58
140
|
await Promise.all(workers);
|
|
141
|
+
if (failed)
|
|
142
|
+
throw failure;
|
|
59
143
|
}
|
|
60
|
-
async function putOne(target, distDir, fetchImpl,
|
|
144
|
+
async function putOne(target, distDir, fetchImpl, retry) {
|
|
61
145
|
const abs = path.join(distDir, ...target.path.split("/"));
|
|
62
|
-
const
|
|
146
|
+
const { size } = await fs.stat(abs);
|
|
147
|
+
const put = withDeadline(fetchImpl, putDeadlineMs(size));
|
|
63
148
|
let lastStatus = 0;
|
|
149
|
+
let lastCode;
|
|
64
150
|
let sawBadDigest = false;
|
|
65
|
-
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
151
|
+
for (let attempt = 1; attempt <= retry.maxAttempts; attempt++) {
|
|
152
|
+
if (attempt > 1)
|
|
153
|
+
await retry.sleep(backoffMs(attempt - 1, retry.random));
|
|
154
|
+
// A fresh stream per attempt: a body cannot be replayed once read.
|
|
155
|
+
const stream = createReadStream(abs);
|
|
66
156
|
let res;
|
|
157
|
+
let text;
|
|
67
158
|
try {
|
|
68
|
-
res = await
|
|
159
|
+
res = await put(target.url, {
|
|
69
160
|
method: "PUT",
|
|
70
|
-
// VERBATIM. No key dropped, no key
|
|
71
|
-
|
|
72
|
-
|
|
161
|
+
// VERBATIM. No key dropped, no key renamed; Content-Length is the one
|
|
162
|
+
// addition, required by S3 and absent from a streamed body.
|
|
163
|
+
headers: { ...target.headers, "content-length": String(size) },
|
|
164
|
+
body: Readable.toWeb(stream),
|
|
165
|
+
duplex: "half",
|
|
73
166
|
});
|
|
167
|
+
text = res.ok ? "" : await res.text().catch(() => "");
|
|
74
168
|
}
|
|
75
169
|
catch {
|
|
76
170
|
lastStatus = 0;
|
|
171
|
+
lastCode = undefined;
|
|
77
172
|
continue;
|
|
78
173
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
const
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
throw new
|
|
88
|
-
|
|
89
|
-
|
|
174
|
+
finally {
|
|
175
|
+
stream.destroy();
|
|
176
|
+
}
|
|
177
|
+
const outcome = classifyS3Response(res.status, text);
|
|
178
|
+
switch (outcome.kind) {
|
|
179
|
+
case "ok":
|
|
180
|
+
return;
|
|
181
|
+
case "expired":
|
|
182
|
+
throw new UploadExpired(target.path);
|
|
183
|
+
case "bad_digest":
|
|
184
|
+
// The bytes on disk differ from what was hashed. One re-PUT covers a
|
|
185
|
+
// torn read; a repeat means the file genuinely changed under us.
|
|
186
|
+
if (sawBadDigest)
|
|
187
|
+
throw new UploadFailed(target.path, res.status, true, outcome.code);
|
|
188
|
+
sawBadDigest = true;
|
|
189
|
+
lastStatus = res.status;
|
|
190
|
+
lastCode = outcome.code;
|
|
191
|
+
continue;
|
|
192
|
+
case "retry":
|
|
193
|
+
lastStatus = res.status;
|
|
194
|
+
lastCode = outcome.code;
|
|
195
|
+
continue;
|
|
196
|
+
case "fatal":
|
|
197
|
+
throw new UploadFailed(target.path, res.status, false, outcome.code);
|
|
90
198
|
}
|
|
91
199
|
}
|
|
92
|
-
throw new UploadFailed(target.path, lastStatus, sawBadDigest);
|
|
200
|
+
throw new UploadFailed(target.path, lastStatus, sawBadDigest, lastCode, !sawBadDigest);
|
|
93
201
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/** The longest the CLI waits for the address to serve the new version. */
|
|
2
|
+
export declare const VISIBILITY_WAIT_MS = 90000;
|
|
3
|
+
export interface VisibilityDeps {
|
|
4
|
+
fetchImpl: typeof fetch;
|
|
5
|
+
now: () => number;
|
|
6
|
+
sleep: (ms: number) => Promise<void>;
|
|
7
|
+
/** Called once, before the first wait, and only if the first probe missed. */
|
|
8
|
+
onWaiting?: () => void;
|
|
9
|
+
}
|
|
10
|
+
export interface VisibilityResult {
|
|
11
|
+
visible: boolean;
|
|
12
|
+
waitedMs: number;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* Probe `liveUrl` until its body hashes to `indexSha256` (std base64, as in the
|
|
16
|
+
* manifest) or `VISIBILITY_WAIT_MS` passes. Never throws: a failed probe is a miss.
|
|
17
|
+
*
|
|
18
|
+
* Elapsed time is the larger of the clock and the summed sleeps, so the bound
|
|
19
|
+
* holds under a frozen test clock as well as against slow probes.
|
|
20
|
+
*/
|
|
21
|
+
export declare function waitUntilServing(liveUrl: string, indexSha256: string, deps: VisibilityDeps): Promise<VisibilityResult>;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
// After commit, wait until the public address serves the version just committed.
|
|
2
|
+
//
|
|
3
|
+
// The api flips the slug pointer in CloudFront KeyValueStore, which is eventually
|
|
4
|
+
// consistent: staging measured 13 to 36 s between "committed" and the edge serving
|
|
5
|
+
// the new bytes. The marker is index.html itself. The CLI already holds its sha256
|
|
6
|
+
// from the manifest, so a probe hashes the served body and compares; nothing new
|
|
7
|
+
// has to be added to the bundle, the router or the api.
|
|
8
|
+
//
|
|
9
|
+
// Used by publish/command.ts. IO (fetch, clock, sleep) is injected for tests.
|
|
10
|
+
import { createHash } from "node:crypto";
|
|
11
|
+
import { withDeadline } from "../shared/http.js";
|
|
12
|
+
/** The longest the CLI waits for the address to serve the new version. */
|
|
13
|
+
export const VISIBILITY_WAIT_MS = 90_000;
|
|
14
|
+
const FIRST_DELAY_MS = 1_000;
|
|
15
|
+
const MAX_DELAY_MS = 5_000;
|
|
16
|
+
const PROBE_DEADLINE_MS = 10_000;
|
|
17
|
+
/**
|
|
18
|
+
* Probe `liveUrl` until its body hashes to `indexSha256` (std base64, as in the
|
|
19
|
+
* manifest) or `VISIBILITY_WAIT_MS` passes. Never throws: a failed probe is a miss.
|
|
20
|
+
*
|
|
21
|
+
* Elapsed time is the larger of the clock and the summed sleeps, so the bound
|
|
22
|
+
* holds under a frozen test clock as well as against slow probes.
|
|
23
|
+
*/
|
|
24
|
+
export async function waitUntilServing(liveUrl, indexSha256, deps) {
|
|
25
|
+
const probe = withDeadline(deps.fetchImpl, PROBE_DEADLINE_MS);
|
|
26
|
+
const startedAt = deps.now();
|
|
27
|
+
let slept = 0;
|
|
28
|
+
let delay = FIRST_DELAY_MS;
|
|
29
|
+
const elapsed = () => Math.max(deps.now() - startedAt, slept);
|
|
30
|
+
for (let attempt = 1;; attempt++) {
|
|
31
|
+
if (await serves(probe, liveUrl, indexSha256, attempt)) {
|
|
32
|
+
return { visible: true, waitedMs: elapsed() };
|
|
33
|
+
}
|
|
34
|
+
const left = VISIBILITY_WAIT_MS - elapsed();
|
|
35
|
+
if (left <= 0)
|
|
36
|
+
return { visible: false, waitedMs: elapsed() };
|
|
37
|
+
if (attempt === 1)
|
|
38
|
+
deps.onWaiting?.();
|
|
39
|
+
const step = Math.min(delay, left);
|
|
40
|
+
await deps.sleep(step);
|
|
41
|
+
slept += step;
|
|
42
|
+
delay = Math.min(delay * 2, MAX_DELAY_MS);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
async function serves(probe, liveUrl, indexSha256, attempt) {
|
|
46
|
+
try {
|
|
47
|
+
const res = await probe(bustedUrl(liveUrl, attempt), {
|
|
48
|
+
headers: { "cache-control": "no-cache", pragma: "no-cache" },
|
|
49
|
+
});
|
|
50
|
+
if (!res.ok)
|
|
51
|
+
return false;
|
|
52
|
+
const body = Buffer.from(await res.arrayBuffer());
|
|
53
|
+
return createHash("sha256").update(body).digest("base64") === indexSha256;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function bustedUrl(liveUrl, attempt) {
|
|
60
|
+
try {
|
|
61
|
+
const u = new URL(liveUrl);
|
|
62
|
+
u.searchParams.set("vincentt-check", `${Date.now().toString(36)}-${attempt}`);
|
|
63
|
+
return u.toString();
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return liveUrl;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -74,7 +74,7 @@ export class AnnotationStore {
|
|
|
74
74
|
};
|
|
75
75
|
this.items.push(annotation);
|
|
76
76
|
await fs.appendFile(path.join(this.dir, STORE_FILE), JSON.stringify(annotation) + "\n");
|
|
77
|
-
this.wake(
|
|
77
|
+
this.wake();
|
|
78
78
|
return annotation;
|
|
79
79
|
}
|
|
80
80
|
/** The first annotation with seq greater than `after`, if one is already held. */
|
|
@@ -113,12 +113,17 @@ export class AnnotationStore {
|
|
|
113
113
|
this.waiters.push(waiter);
|
|
114
114
|
});
|
|
115
115
|
}
|
|
116
|
-
|
|
116
|
+
// A waiter gets the LOWEST held seq past its cursor, not the annotation whose
|
|
117
|
+
// write just finished: concurrent adds can finish persisting out of seq order,
|
|
118
|
+
// and handing over the later one first moves the consumer's cursor past the
|
|
119
|
+
// earlier one for good.
|
|
120
|
+
wake() {
|
|
117
121
|
const remaining = [];
|
|
118
122
|
for (const w of this.waiters) {
|
|
119
|
-
|
|
123
|
+
const next = this.peekAfter(w.after);
|
|
124
|
+
if (next) {
|
|
120
125
|
clearTimeout(w.timer);
|
|
121
|
-
w.resolve({ kind: "annotation", annotation:
|
|
126
|
+
w.resolve({ kind: "annotation", annotation: next });
|
|
122
127
|
}
|
|
123
128
|
else {
|
|
124
129
|
remaining.push(w);
|