@vincentt-xr/cli 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/MIGRATION.md +15 -0
  2. package/dist/cli/index.js +63 -6
  3. package/dist/domain/api.js +16 -12
  4. package/dist/domain/command.js +9 -0
  5. package/dist/dx/command.js +3 -0
  6. package/dist/dx/consent.js +10 -14
  7. package/dist/dx/reportIo.js +6 -4
  8. package/dist/dx/send.js +6 -4
  9. package/dist/lib/backend.js +5 -10
  10. package/dist/lib/exitCodes.d.ts +6 -0
  11. package/dist/lib/exitCodes.js +6 -0
  12. package/dist/lib/terminalText.d.ts +6 -0
  13. package/dist/lib/terminalText.js +10 -0
  14. package/dist/lib/whoami.js +3 -0
  15. package/dist/library/assetsCommand.js +4 -0
  16. package/dist/library/client.js +6 -8
  17. package/dist/login/authBackend.js +18 -15
  18. package/dist/login/loginCommand.js +31 -4
  19. package/dist/login/pkceLogin.d.ts +10 -0
  20. package/dist/login/pkceLogin.js +24 -1
  21. package/dist/platform/releaseFetch.js +36 -69
  22. package/dist/preview/api.js +9 -12
  23. package/dist/preview/command.d.ts +3 -2
  24. package/dist/preview/command.js +8 -5
  25. package/dist/preview/copy.d.ts +8 -1
  26. package/dist/preview/copy.js +16 -3
  27. package/dist/preview/proxy.js +5 -39
  28. package/dist/preview/runner.d.ts +25 -9
  29. package/dist/preview/runner.js +42 -15
  30. package/dist/publish/api.js +14 -14
  31. package/dist/publish/command.d.ts +3 -0
  32. package/dist/publish/command.js +71 -7
  33. package/dist/publish/copy.d.ts +9 -0
  34. package/dist/publish/copy.js +83 -14
  35. package/dist/publish/keyScan.d.ts +25 -0
  36. package/dist/publish/keyScan.js +88 -0
  37. package/dist/publish/manifest.d.ts +51 -19
  38. package/dist/publish/manifest.js +246 -63
  39. package/dist/publish/upload.d.ts +8 -0
  40. package/dist/publish/upload.js +15 -0
  41. package/dist/publish/weight.d.ts +23 -0
  42. package/dist/publish/weight.js +113 -0
  43. package/dist/shared/credentialNames.d.ts +17 -0
  44. package/dist/shared/credentialNames.js +72 -0
  45. package/dist/shared/http.d.ts +34 -0
  46. package/dist/shared/http.js +98 -2
  47. package/dist/shared/upgradeError.d.ts +62 -0
  48. package/dist/shared/upgradeError.js +215 -0
  49. package/dist/tunnel/caps.d.ts +19 -0
  50. package/dist/tunnel/caps.js +52 -0
  51. package/dist/tunnel/cbor.d.ts +39 -12
  52. package/dist/tunnel/cbor.js +84 -10
  53. package/dist/tunnel/client.d.ts +48 -2
  54. package/dist/tunnel/client.js +157 -44
  55. package/dist/tunnel/frame.d.ts +1 -1
  56. package/dist/tunnel/frame.js +8 -4
  57. package/dist/tunnel/index.d.ts +3 -2
  58. package/dist/tunnel/index.js +3 -2
  59. package/package.json +4 -3
@@ -0,0 +1,88 @@
1
+ // Content rules for credential files: a private-key header (PEM, OpenSSH, PGP)
2
+ // and a Google service-account or user credential JSON. Bytes are read as
3
+ // latin1, one character per byte, so the ASCII header matches inside UTF-8 and
4
+ // no decode can throw.
5
+ /** The label is bounded to 40 characters so the carried tail stays finite. */
6
+ export const KEY_HEADER = /-----BEGIN [A-Z0-9 ]{0,40}PRIVATE KEY( BLOCK)?-----/;
7
+ /**
8
+ * A newline (real, or the two-character `\n` or four-character `\r\n` of a
9
+ * string literal), then a base64 line.
10
+ */
11
+ export const KEY_BODY = /(?:\r?\n|(?:\\r)?\\n)[A-Za-z0-9+/=]{64}/;
12
+ /** Below this offset a header alone is a match. */
13
+ export const HEAD_BYTES = 4096;
14
+ /** How far past a header its key body may sit. */
15
+ export const BODY_WINDOW = 256;
16
+ /** Covers the longest header plus the body window. */
17
+ const CARRY = 512;
18
+ export function createKeyScanner() {
19
+ let buf = "";
20
+ let bufStart = 0;
21
+ // Absolute offset below which every header has been judged.
22
+ let judgedTo = 0;
23
+ let matched = false;
24
+ const scan = (final) => {
25
+ const re = new RegExp(KEY_HEADER.source, "g");
26
+ re.lastIndex = Math.max(0, judgedTo - bufStart);
27
+ let m;
28
+ while ((m = re.exec(buf)) !== null) {
29
+ const at = bufStart + m.index;
30
+ if (at < HEAD_BYTES) {
31
+ matched = true;
32
+ return;
33
+ }
34
+ const end = m.index + m[0].length;
35
+ if (!final && end + BODY_WINDOW > buf.length) {
36
+ judgedTo = at;
37
+ return;
38
+ }
39
+ if (KEY_BODY.test(buf.slice(end, end + BODY_WINDOW))) {
40
+ matched = true;
41
+ return;
42
+ }
43
+ judgedTo = at + 1;
44
+ re.lastIndex = m.index + 1;
45
+ }
46
+ };
47
+ return {
48
+ push(chunk) {
49
+ if (matched)
50
+ return;
51
+ buf += chunk;
52
+ scan(false);
53
+ if (buf.length > CARRY) {
54
+ const drop = buf.length - CARRY;
55
+ buf = buf.slice(drop);
56
+ bufStart += drop;
57
+ }
58
+ },
59
+ end() {
60
+ if (!matched)
61
+ scan(true);
62
+ return matched;
63
+ },
64
+ get matched() {
65
+ return matched;
66
+ },
67
+ };
68
+ }
69
+ /**
70
+ * The top-level `type` of a Google credential JSON, or undefined. A parse
71
+ * failure is "not a match" and its message is discarded: it echoes the input.
72
+ */
73
+ export function credentialJson(bytes) {
74
+ const start = bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf
75
+ ? 3
76
+ : 0;
77
+ let value;
78
+ try {
79
+ value = JSON.parse(Buffer.from(bytes.buffer, bytes.byteOffset + start, bytes.length - start).toString("utf8"));
80
+ }
81
+ catch {
82
+ return undefined;
83
+ }
84
+ if (value === null || typeof value !== "object" || Array.isArray(value))
85
+ return undefined;
86
+ const type = value.type;
87
+ return type === "service_account" || type === "authorized_user" ? type : undefined;
88
+ }
@@ -1,3 +1,4 @@
1
+ import { type CredentialJsonType } from "./keyScan.js";
1
2
  import { type PathRejection } from "./paths.js";
2
3
  export interface ManifestEntry {
3
4
  /** dist-relative, forward slashes, no leading `/`, normalized. */
@@ -10,13 +11,16 @@ export interface Manifest {
10
11
  files: ManifestEntry[];
11
12
  totalBytes: number;
12
13
  }
13
- export type ManifestReason = PathRejection | "no_dist" | "output_is_repo_root" | "output_outside_repo" | "output_in_hidden" | "secret_path" | "lfs_pointer";
14
- /** An output-relative path the leak or LFS check refused. */
14
+ export type ManifestReason = PathRejection | "no_dist" | "output_is_repo_root" | "output_outside_repo" | "output_in_hidden" | "secret_path" | "credential_file" | "lfs_pointer" | "file_unreadable";
15
+ export type CredentialWhy = "name" | "private_key" | CredentialJsonType;
16
+ /** An output-relative path the leak, credential or LFS check refused. */
15
17
  export interface Offender {
16
18
  /** Output-relative, POSIX. A refused folder carries a trailing `/`. */
17
19
  path: string;
18
20
  /** For a symlink: where it resolves, repo-relative inside and absolute outside. */
19
21
  target?: string;
22
+ /** For a credential file: what made it one. */
23
+ why?: CredentialWhy;
20
24
  }
21
25
  export interface ManifestErrorDetail {
22
26
  paths?: Offender[];
@@ -36,33 +40,61 @@ export declare class ManifestError extends Error {
36
40
  };
37
41
  constructor(reason: ManifestReason, offendingPath: string, message: string, detail?: ManifestErrorDetail);
38
42
  }
43
+ /** A file the walk passed: what the hash step must find again at the same path. */
44
+ export interface WalkedFile {
45
+ /** Output-relative, POSIX. */
46
+ rel: string;
47
+ /** Absolute, under the real output folder. */
48
+ abs: string;
49
+ size: number;
50
+ dev: number;
51
+ ino: number;
52
+ }
39
53
  export interface BuildManifestOptions {
40
54
  /**
41
- * The directory holding the binding. When given, the output is refused before
42
- * any hashing if it is the repo root, lies outside it, holds a dotted name
43
- * outside the two allowed ones, holds a symlink that resolves outside the root
44
- * or onto a dotted path, or holds a Git LFS pointer. All of it is local.
55
+ * The directory holding the binding. Required: there is no unchecked
56
+ * manifest. The output is refused before any hashing if it is the repo root,
57
+ * lies outside it, holds a dotted name outside the two allowed ones, holds a
58
+ * symlink that resolves outside the root or onto a dotted path, holds a
59
+ * credential file, or holds a Git LFS pointer. All of it is local.
45
60
  */
46
- repoRoot?: string;
61
+ repoRoot: string;
62
+ /**
63
+ * Test-only: runs after the checks and before the hash step, so a test can
64
+ * change a checked file. Never set by the command.
65
+ */
66
+ beforeHash?: (files: readonly WalkedFile[]) => Promise<void> | void;
47
67
  }
48
68
  /** A Git LFS pointer file: at most 1 KB, the spec's exact first line, and an oid line. */
49
- export declare function isLfsPointer(absPath: string, size: number): Promise<boolean>;
69
+ export declare function isLfsPointer(head: Uint8Array, size: number): boolean;
50
70
  /**
51
- * The leak and LFS checks, in that order, before anything is hashed or sent.
52
- * Returns the output-relative file list the checks passed, which is exactly the
53
- * list the manifest is built from: what is checked is what is uploaded.
71
+ * The early pass's read: the first `min(size, 4096)` bytes, or the whole file
72
+ * for a `.json` of at most 64 KB. Opens without following links.
54
73
  */
55
- export declare function checkOutput(distDir: string, repoRoot: string): Promise<{
56
- rel: string;
74
+ export declare function readHead(file: WalkedFile): Promise<Buffer>;
75
+ /**
76
+ * The leak, credential and LFS checks, in that order, before anything is hashed
77
+ * or sent. Returns the file list the checks passed, which is exactly the list
78
+ * the manifest is built from: what is checked is what is uploaded.
79
+ */
80
+ export declare function checkOutput(distDir: string, repoRoot: string): Promise<WalkedFile[]>;
81
+ export interface HashResult {
82
+ /** base64 std, the exact form `x-amz-checksum-sha256` wants. */
83
+ sha256: string;
57
84
  size: number;
58
- }[]>;
59
- /** Recursively list files under `dir` as dir-relative POSIX paths. Symlinks are not followed. */
60
- export declare function walkDist(dir: string, base?: string): Promise<string[]>;
61
- /** sha256 of a file, base64 std — the exact form `x-amz-checksum-sha256` wants. */
62
- export declare function hashFile(absPath: string): Promise<string>;
85
+ /** Set when the hashed bytes hold a private key or a Google credential. */
86
+ why?: CredentialWhy;
87
+ }
88
+ /**
89
+ * Hash a walked file and re-run the content rules over the exact bytes hashed.
90
+ * The file is opened without following links and must still be the regular
91
+ * file the walk saw. Every read error throws `file_unreadable`; only a JSON
92
+ * parse failure is "not a match".
93
+ */
94
+ export declare function hashAndScan(absPath: string, walked: WalkedFile): Promise<HashResult>;
63
95
  /**
64
96
  * Build the manifest for a dist directory. Every path is normalized before it is
65
97
  * hashed, so the string this returns is the string the API validates, signs, and
66
98
  * later lists — there is no second form anywhere.
67
99
  */
68
- export declare function buildManifest(distDir: string, opts?: BuildManifestOptions): Promise<Manifest>;
100
+ export declare function buildManifest(distDir: string, opts: BuildManifestOptions): Promise<Manifest>;
@@ -5,8 +5,10 @@
5
5
  // every single PUT while both repos' unit suites stay green (§2.7's first
6
6
  // contract trap).
7
7
  import { createHash } from "node:crypto";
8
- import { createReadStream, promises as fs } from "node:fs";
8
+ import { constants as fsConstants, promises as fs } from "node:fs";
9
9
  import path from "node:path";
10
+ import { PUBLISH_CREDENTIAL_NAMES, PUBLISH_STRIP, foldName, isCredentialName, } from "../shared/credentialNames.js";
11
+ import { createKeyScanner, credentialJson } from "./keyScan.js";
10
12
  import { normalizePath, validatePathSet } from "./paths.js";
11
13
  export class ManifestError extends Error {
12
14
  reason;
@@ -27,6 +29,13 @@ const ALLOWED_ROOT_DIR = ".well-known";
27
29
  const ALLOWED_ROOT_FILE = ".nojekyll";
28
30
  const LFS_POINTER_MAX_BYTES = 1024;
29
31
  const LFS_FIRST_LINE = "version https://git-lfs.github.com/spec/v1";
32
+ /** The early pass reads this much of every file. */
33
+ const HEAD_READ_BYTES = 4096;
34
+ /** A `.json` at most this size is read whole and parsed. */
35
+ const CREDENTIAL_JSON_MAX_BYTES = 65_536;
36
+ const READ_CONCURRENCY = 16;
37
+ const HASH_CHUNK_BYTES = 64 * 1024;
38
+ const OPEN_NOFOLLOW = fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0);
30
39
  function isInside(child, parent) {
31
40
  const rel = path.relative(parent, child);
32
41
  return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel);
@@ -34,14 +43,21 @@ function isInside(child, parent) {
34
43
  function hasDottedSegment(rel) {
35
44
  return rel.split(/[\\/]/).some((seg) => seg.startsWith("."));
36
45
  }
46
+ function isPublishCredentialName(name) {
47
+ return isCredentialName(foldName(name), PUBLISH_CREDENTIAL_NAMES, PUBLISH_STRIP);
48
+ }
49
+ function isJsonName(rel) {
50
+ return rel.toLowerCase().endsWith(".json");
51
+ }
37
52
  /**
38
- * The leak walk. Unlike `walkDist` it records what it does not upload: a dotted
39
- * name is an offender (a folder once, with a trailing slash, never descended),
40
- * and every symlink is resolved and judged against the real repo root.
53
+ * The one walk. It records what it does not upload: a dotted name is an
54
+ * offender (a folder once, with a trailing slash, never descended), every
55
+ * symlink is resolved and judged against the real repo root, and a name on the
56
+ * credential list is recorded apart (a folder once, never descended).
41
57
  */
42
58
  async function walkOutput(outReal, rootReal) {
43
- const files = [];
44
- const offenders = [];
59
+ const walk = { files: [], offenders: [], credentialNames: [] };
60
+ let seq = 0;
45
61
  const visit = async (dir) => {
46
62
  const entries = await fs.readdir(dir, { withFileTypes: true });
47
63
  entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
@@ -55,17 +71,17 @@ async function walkOutput(outReal, rootReal) {
55
71
  ((lower === ALLOWED_ROOT_DIR && entry.isDirectory()) ||
56
72
  (lower === ALLOWED_ROOT_FILE && entry.isFile()));
57
73
  if (!allowed) {
58
- offenders.push({ path: entry.isDirectory() ? `${rel}/` : rel });
74
+ walk.offenders.push({ path: entry.isDirectory() ? `${rel}/` : rel });
59
75
  continue;
60
76
  }
61
77
  }
62
78
  if (entry.isSymbolicLink()) {
63
79
  const target = await symlinkTarget(full);
64
80
  if (!isInside(target, rootReal)) {
65
- offenders.push({ path: rel, target });
81
+ walk.offenders.push({ path: rel, target });
66
82
  }
67
83
  else if (hasDottedSegment(path.relative(rootReal, target))) {
68
- offenders.push({
84
+ walk.offenders.push({
69
85
  path: rel,
70
86
  target: path.relative(rootReal, target).split(path.sep).join("/"),
71
87
  });
@@ -73,17 +89,37 @@ async function walkOutput(outReal, rootReal) {
73
89
  // A symlink that resolves cleanly inside is not uploaded, as before.
74
90
  continue;
75
91
  }
92
+ const nameMatched = isPublishCredentialName(entry.name);
76
93
  if (entry.isDirectory()) {
94
+ if (nameMatched) {
95
+ walk.credentialNames.push({
96
+ seq: seq++,
97
+ offender: { path: `${rel}/`, why: "name" },
98
+ });
99
+ continue;
100
+ }
77
101
  await visit(full);
78
102
  }
79
103
  else if (entry.isFile()) {
80
- const stat = await fs.stat(full);
81
- files.push({ rel, size: stat.size });
104
+ const stat = await fs.lstat(full);
105
+ const at = seq++;
106
+ if (nameMatched) {
107
+ walk.credentialNames.push({ seq: at, offender: { path: rel, why: "name" } });
108
+ }
109
+ walk.files.push({
110
+ rel,
111
+ abs: full,
112
+ size: stat.size,
113
+ dev: stat.dev,
114
+ ino: stat.ino,
115
+ seq: at,
116
+ nameMatched,
117
+ });
82
118
  }
83
119
  }
84
120
  };
85
121
  await visit(outReal);
86
- return { files, offenders };
122
+ return walk;
87
123
  }
88
124
  /** Where a symlink lands: its realpath, or its lexical target when it dangles. */
89
125
  async function symlinkTarget(link) {
@@ -96,19 +132,93 @@ async function symlinkTarget(link) {
96
132
  }
97
133
  }
98
134
  /** A Git LFS pointer file: at most 1 KB, the spec's exact first line, and an oid line. */
99
- export async function isLfsPointer(absPath, size) {
135
+ export function isLfsPointer(head, size) {
100
136
  if (size > LFS_POINTER_MAX_BYTES)
101
137
  return false;
102
- const head = await fs.readFile(absPath, "utf8");
103
- const lines = head.split("\n");
138
+ const lines = Buffer.from(head.buffer, head.byteOffset, head.byteLength)
139
+ .toString("utf8")
140
+ .split("\n");
104
141
  if (lines[0] !== LFS_FIRST_LINE)
105
142
  return false;
106
143
  return lines.some((l) => l.startsWith("oid sha256:"));
107
144
  }
145
+ class UnreadableFile extends Error {
146
+ }
108
147
  /**
109
- * The leak and LFS checks, in that order, before anything is hashed or sent.
110
- * Throws ManifestError; returns nothing on a clean output.
148
+ * Open a walked file without following a link, and confirm it is still the
149
+ * regular file the walk saw: the same device and inode before and after the
150
+ * open. Anything else throws UnreadableFile.
111
151
  */
152
+ async function openWalked(file) {
153
+ let handle;
154
+ try {
155
+ const before = await fs.lstat(file.abs);
156
+ if (!before.isFile() || before.dev !== file.dev || before.ino !== file.ino) {
157
+ throw new UnreadableFile();
158
+ }
159
+ handle = await fs.open(file.abs, OPEN_NOFOLLOW);
160
+ const after = await handle.stat();
161
+ if (!after.isFile() || after.dev !== file.dev || after.ino !== file.ino) {
162
+ throw new UnreadableFile();
163
+ }
164
+ return handle;
165
+ }
166
+ catch {
167
+ await handle?.close().catch(() => undefined);
168
+ throw new UnreadableFile();
169
+ }
170
+ }
171
+ /** Exactly `length` bytes from offset 0; a short read is unreadable. */
172
+ async function readExactly(handle, length) {
173
+ const buf = Buffer.alloc(length);
174
+ let got = 0;
175
+ while (got < length) {
176
+ const { bytesRead } = await handle.read(buf, got, length - got, got);
177
+ if (bytesRead === 0)
178
+ throw new UnreadableFile();
179
+ got += bytesRead;
180
+ }
181
+ return buf;
182
+ }
183
+ /**
184
+ * The early pass's read: the first `min(size, 4096)` bytes, or the whole file
185
+ * for a `.json` of at most 64 KB. Opens without following links.
186
+ */
187
+ export async function readHead(file) {
188
+ const whole = isJsonName(file.rel) && file.size <= CREDENTIAL_JSON_MAX_BYTES;
189
+ const handle = await openWalked(file);
190
+ try {
191
+ return await readExactly(handle, whole ? file.size : Math.min(file.size, HEAD_READ_BYTES));
192
+ }
193
+ catch {
194
+ throw new UnreadableFile();
195
+ }
196
+ finally {
197
+ await handle.close().catch(() => undefined);
198
+ }
199
+ }
200
+ /** The content rules over a head (or whole small JSON). */
201
+ function contentWhy(rel, size, bytes) {
202
+ const scanner = createKeyScanner();
203
+ scanner.push(bytes.subarray(0, HEAD_READ_BYTES).toString("latin1"));
204
+ if (scanner.end())
205
+ return "private_key";
206
+ if (isJsonName(rel) && size <= CREDENTIAL_JSON_MAX_BYTES)
207
+ return credentialJson(bytes);
208
+ return undefined;
209
+ }
210
+ async function mapLimit(items, limit, fn) {
211
+ const out = new Array(items.length);
212
+ let next = 0;
213
+ const worker = async () => {
214
+ while (next < items.length) {
215
+ const i = next++;
216
+ out[i] = await fn(items[i]);
217
+ }
218
+ };
219
+ await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
220
+ return out;
221
+ }
112
222
  /**
113
223
  * Folders an output must never be, or sit inside: the repository's history and
114
224
  * the Vincentt binding. Compared case-insensitively, as a default macOS volume
@@ -121,9 +231,9 @@ function forbiddenSegmentIn(rel) {
121
231
  .find((seg) => FORBIDDEN_OUTPUT_SEGMENTS.has(seg.toLowerCase()));
122
232
  }
123
233
  /**
124
- * The leak and LFS checks, in that order, before anything is hashed or sent.
125
- * Returns the output-relative file list the checks passed, which is exactly the
126
- * list the manifest is built from: what is checked is what is uploaded.
234
+ * The leak, credential and LFS checks, in that order, before anything is hashed
235
+ * or sent. Returns the file list the checks passed, which is exactly the list
236
+ * the manifest is built from: what is checked is what is uploaded.
127
237
  */
128
238
  export async function checkOutput(distDir, repoRoot) {
129
239
  const rootReal = await fs.realpath(repoRoot).catch(() => path.resolve(repoRoot));
@@ -170,10 +280,42 @@ export async function checkOutput(distDir, repoRoot) {
170
280
  stats,
171
281
  });
172
282
  }
283
+ const heads = await mapLimit(walk.files, READ_CONCURRENCY, async (f) => {
284
+ try {
285
+ const head = await readHead(f);
286
+ return {
287
+ why: f.nameMatched ? undefined : contentWhy(f.rel, f.size, head),
288
+ lfs: isLfsPointer(head, f.size),
289
+ };
290
+ }
291
+ catch {
292
+ return { unreadable: true };
293
+ }
294
+ });
295
+ const credentials = [...walk.credentialNames];
173
296
  const pointers = [];
174
- for (const f of walk.files) {
175
- if (await isLfsPointer(path.join(outReal, f.rel), f.size))
297
+ walk.files.forEach((f, i) => {
298
+ const h = heads[i];
299
+ if ("unreadable" in h)
300
+ return;
301
+ if (h.why !== undefined)
302
+ credentials.push({ seq: f.seq, offender: { path: f.rel, why: h.why } });
303
+ if (h.lfs)
176
304
  pointers.push({ path: f.rel });
305
+ });
306
+ if (credentials.length > 0) {
307
+ const paths = credentials.sort((a, b) => a.seq - b.seq).map((c) => c.offender);
308
+ throw new ManifestError("credential_file", paths[0].path, "credential files in the output", {
309
+ paths,
310
+ stats,
311
+ });
312
+ }
313
+ // A file the checks could not read fails closed; it is never assumed clean.
314
+ const unreadable = walk.files.find((_, i) => "unreadable" in heads[i]);
315
+ if (unreadable !== undefined) {
316
+ throw new ManifestError("file_unreadable", unreadable.rel, "a file could not be read", {
317
+ stats,
318
+ });
177
319
  }
178
320
  if (pointers.length > 0) {
179
321
  throw new ManifestError("lfs_pointer", pointers[0].path, "Git LFS pointers in the output", {
@@ -181,69 +323,110 @@ export async function checkOutput(distDir, repoRoot) {
181
323
  stats,
182
324
  });
183
325
  }
184
- return walk.files;
326
+ return walk.files.map(({ rel, abs, size, dev, ino }) => ({ rel, abs, size, dev, ino }));
185
327
  }
186
- /** Recursively list files under `dir` as dir-relative POSIX paths. Symlinks are not followed. */
187
- export async function walkDist(dir, base = dir) {
188
- let entries;
328
+ /**
329
+ * Hash a walked file and re-run the content rules over the exact bytes hashed.
330
+ * The file is opened without following links and must still be the regular
331
+ * file the walk saw. Every read error throws `file_unreadable`; only a JSON
332
+ * parse failure is "not a match".
333
+ */
334
+ export async function hashAndScan(absPath, walked) {
335
+ const unreadable = () => new ManifestError("file_unreadable", walked.rel, "a file changed while it was read");
336
+ let handle;
189
337
  try {
190
- entries = await fs.readdir(dir, { withFileTypes: true });
338
+ handle = await openWalked({ ...walked, abs: absPath });
191
339
  }
192
- catch (err) {
193
- if (err.code === "ENOENT") {
194
- throw new ManifestError("no_dist", dir, `no dist at ${dir}`);
195
- }
196
- throw err;
340
+ catch {
341
+ throw unreadable();
197
342
  }
198
- const out = [];
199
- for (const entry of entries) {
200
- const full = path.join(dir, entry.name);
201
- if (entry.isDirectory())
202
- out.push(...(await walkDist(full, base)));
203
- else if (entry.isFile()) {
204
- out.push(path.relative(base, full).split(path.sep).join("/"));
343
+ try {
344
+ const { size } = await handle.stat();
345
+ const hash = createHash("sha256");
346
+ const scanner = createKeyScanner();
347
+ const keepJson = isJsonName(walked.rel) && size <= CREDENTIAL_JSON_MAX_BYTES;
348
+ const kept = [];
349
+ const chunk = Buffer.alloc(HASH_CHUNK_BYTES);
350
+ let read = 0;
351
+ for (;;) {
352
+ const { bytesRead } = await handle.read(chunk, 0, chunk.length, read);
353
+ if (bytesRead === 0)
354
+ break;
355
+ const piece = chunk.subarray(0, bytesRead);
356
+ hash.update(piece);
357
+ scanner.push(piece.toString("latin1"));
358
+ if (keepJson)
359
+ kept.push(Buffer.from(piece));
360
+ read += bytesRead;
361
+ if (read > size)
362
+ throw unreadable();
205
363
  }
364
+ if (read !== size)
365
+ throw unreadable();
366
+ let why = scanner.end() ? "private_key" : undefined;
367
+ if (why === undefined && keepJson)
368
+ why = credentialJson(Buffer.concat(kept));
369
+ return { sha256: hash.digest("base64"), size, why };
370
+ }
371
+ catch (err) {
372
+ if (err instanceof ManifestError)
373
+ throw err;
374
+ throw unreadable();
375
+ }
376
+ finally {
377
+ await handle.close().catch(() => undefined);
206
378
  }
207
- return out;
208
- }
209
- /** sha256 of a file, base64 std — the exact form `x-amz-checksum-sha256` wants. */
210
- export async function hashFile(absPath) {
211
- const hash = createHash("sha256");
212
- const stream = createReadStream(absPath);
213
- for await (const chunk of stream)
214
- hash.update(chunk);
215
- return hash.digest("base64");
216
379
  }
217
380
  /**
218
381
  * Build the manifest for a dist directory. Every path is normalized before it is
219
382
  * hashed, so the string this returns is the string the API validates, signs, and
220
383
  * later lists — there is no second form anywhere.
221
384
  */
222
- export async function buildManifest(distDir, opts = {}) {
223
- // With a repo root, the list the leak and LFS checks passed IS the upload
224
- // list. A second walk could see a file created after the checks ran.
225
- const raw = opts.repoRoot !== undefined
226
- ? (await checkOutput(distDir, opts.repoRoot)).map((f) => f.rel)
227
- : await walkDist(distDir);
385
+ export async function buildManifest(distDir, opts) {
386
+ // The list the checks passed IS the upload list. A second walk could see a
387
+ // file created after the checks ran.
388
+ const walked = await checkOutput(distDir, opts.repoRoot);
228
389
  const normalized = [];
229
- for (const rel of raw) {
230
- const result = normalizePath(rel);
390
+ for (const file of walked) {
391
+ const result = normalizePath(file.rel);
231
392
  if (!result.ok) {
232
- throw new ManifestError(result.reason, rel, `${rel}: ${result.reason}`);
393
+ throw new ManifestError(result.reason, file.rel, `${file.rel}: ${result.reason}`);
233
394
  }
234
- normalized.push({ rel, norm: result.path });
395
+ normalized.push({ file, norm: result.path });
235
396
  }
236
397
  const pathError = validatePathSet(normalized.map((n) => n.norm));
237
398
  if (pathError) {
238
399
  throw new ManifestError(pathError.reason, pathError.path, `${pathError.path}: ${pathError.reason}`);
239
400
  }
401
+ await opts.beforeHash?.(walked);
402
+ const stats = {
403
+ files: walked.length,
404
+ bytes: walked.reduce((n, f) => n + f.size, 0),
405
+ };
240
406
  const files = [];
407
+ const credentials = [];
241
408
  let totalBytes = 0;
242
- for (const { rel, norm } of normalized) {
243
- const abs = path.join(distDir, rel);
244
- const stat = await fs.stat(abs);
245
- files.push({ path: norm, size: stat.size, sha256: await hashFile(abs) });
246
- totalBytes += stat.size;
409
+ for (const { file, norm } of normalized) {
410
+ let hashed;
411
+ try {
412
+ hashed = await hashAndScan(file.abs, file);
413
+ }
414
+ catch (err) {
415
+ if (err instanceof ManifestError) {
416
+ throw new ManifestError(err.reason, err.offendingPath, err.message, { stats });
417
+ }
418
+ throw err;
419
+ }
420
+ if (hashed.why !== undefined)
421
+ credentials.push({ path: file.rel, why: hashed.why });
422
+ files.push({ path: norm, size: hashed.size, sha256: hashed.sha256 });
423
+ totalBytes += hashed.size;
424
+ }
425
+ if (credentials.length > 0) {
426
+ throw new ManifestError("credential_file", credentials[0].path, "credential files in the output", {
427
+ paths: credentials,
428
+ stats,
429
+ });
247
430
  }
248
431
  files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
249
432
  return { files, totalBytes };
@@ -22,6 +22,14 @@ export declare class UploadExpired extends Error {
22
22
  readonly path: string;
23
23
  constructor(path: string);
24
24
  }
25
+ /**
26
+ * The api returned an upload target for a path this run's manifest does not
27
+ * hold. Nothing is sent: only files that passed this run's checks are uploaded.
28
+ */
29
+ export declare class UploadTargetRejected extends Error {
30
+ readonly path: string;
31
+ constructor(path: string);
32
+ }
25
33
  export interface UploadProgress {
26
34
  filesDone: number;
27
35
  filesTotal: number;
@@ -46,6 +46,18 @@ export class UploadExpired extends Error {
46
46
  this.name = "UploadExpired";
47
47
  }
48
48
  }
49
+ /**
50
+ * The api returned an upload target for a path this run's manifest does not
51
+ * hold. Nothing is sent: only files that passed this run's checks are uploaded.
52
+ */
53
+ export class UploadTargetRejected extends Error {
54
+ path;
55
+ constructor(path) {
56
+ super(`the upload target ${path} is not in this build`);
57
+ this.path = path;
58
+ this.name = "UploadTargetRejected";
59
+ }
60
+ }
49
61
  const DEFAULT_CONCURRENCY = 6;
50
62
  const DEFAULT_MAX_ATTEMPTS = 5;
51
63
  const BACKOFF_BASE_MS = 500;
@@ -99,6 +111,9 @@ export function classifyS3Response(status, body) {
99
111
  export async function uploadAll(opts) {
100
112
  const fetchImpl = opts.fetchImpl ?? fetch;
101
113
  const sizes = new Map(opts.manifest.map((f) => [f.path, f.size]));
114
+ const foreign = opts.targets.find((t) => !sizes.has(t.path));
115
+ if (foreign !== undefined)
116
+ throw new UploadTargetRejected(foreign.path);
102
117
  const bytesTotal = opts.targets.reduce((n, t) => n + (sizes.get(t.path) ?? 0), 0);
103
118
  const queue = [...opts.targets];
104
119
  const retry = {