@vincentt-xr/cli 0.20.0 → 0.22.0

package/dist/publish/manifest.js

@@ -5,8 +5,10 @@
 // every single PUT while both repos' unit suites stay green (§2.7's first
 // contract trap).
 import { createHash } from "node:crypto";
-import { createReadStream, promises as fs } from "node:fs";
+import { constants as fsConstants, promises as fs } from "node:fs";
 import path from "node:path";
+import { PUBLISH_CREDENTIAL_NAMES, PUBLISH_STRIP, foldName, isCredentialName, } from "../shared/credentialNames.js";
+import { createKeyScanner, credentialJson } from "./keyScan.js";
 import { normalizePath, validatePathSet } from "./paths.js";
 export class ManifestError extends Error {
     reason;
@@ -27,6 +29,13 @@ const ALLOWED_ROOT_DIR = ".well-known";
 const ALLOWED_ROOT_FILE = ".nojekyll";
 const LFS_POINTER_MAX_BYTES = 1024;
 const LFS_FIRST_LINE = "version https://git-lfs.github.com/spec/v1";
+/** The early pass reads this much of every file. */
+const HEAD_READ_BYTES = 4096;
+/** A `.json` at most this size is read whole and parsed. */
+const CREDENTIAL_JSON_MAX_BYTES = 65_536;
+const READ_CONCURRENCY = 16;
+const HASH_CHUNK_BYTES = 64 * 1024;
+const OPEN_NOFOLLOW = fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0);
 function isInside(child, parent) {
     const rel = path.relative(parent, child);
     return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel);
@@ -34,20 +43,35 @@ function isInside(child, parent) {
 function hasDottedSegment(rel) {
     return rel.split(/[\\/]/).some((seg) => seg.startsWith("."));
 }
+function isPublishCredentialName(name) {
+    return isCredentialName(foldName(name), PUBLISH_CREDENTIAL_NAMES, PUBLISH_STRIP);
+}
+function isJsonName(rel) {
+    return rel.toLowerCase().endsWith(".json");
+}
 /**
- * The leak walk. Unlike `walkDist` it records what it does not upload: a dotted
- * name is an offender (a folder once, with a trailing slash, never descended),
- * and every symlink is resolved and judged against the real repo root.
+ * The one walk. It records what it does not upload: a dotted name is an
+ * offender (a folder once, with a trailing slash, never descended), every
+ * symlink is resolved and judged against the real repo root, and a name on the
+ * credential list is recorded apart (a folder once, never descended).
  */
-async function walkOutput(outReal, rootReal) {
-    const files = [];
-    const offenders = [];
+async function walkOutput(outReal, rootReal, hooks) {
+    const walk = { files: [], offenders: [], credentialNames: [] };
+    let seq = 0;
+    const relOf = (full) => path.relative(outReal, full).split(path.sep).join("/");
+    // A read error's message carries the raw absolute path. It becomes a refusal
+    // naming the output-relative path, which the copy escapes.
+    const unreadable = (rel) => new ManifestError("file_unreadable", rel, "a path could not be read");
     const visit = async (dir) => {
-        const entries = await fs.readdir(dir, { withFileTypes: true });
+        await hooks.beforeReadDir?.(dir);
+        const dirRel = dir === outReal ? "" : `${relOf(dir)}/`;
+        const entries = await fs
+            .readdir(dir, { withFileTypes: true })
+            .catch(() => Promise.reject(unreadable(dirRel)));
         entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
         for (const entry of entries) {
             const full = path.join(dir, entry.name);
-            const rel = path.relative(outReal, full).split(path.sep).join("/");
+            const rel = relOf(full);
             const atRoot = dir === outReal;
             const lower = entry.name.toLowerCase();
             if (entry.name.startsWith(".")) {
@@ -55,17 +79,17 @@ async function walkOutput(outReal, rootReal) {
                     ((lower === ALLOWED_ROOT_DIR && entry.isDirectory()) ||
                         (lower === ALLOWED_ROOT_FILE && entry.isFile()));
                 if (!allowed) {
-                    offenders.push({ path: entry.isDirectory() ? `${rel}/` : rel });
+                    walk.offenders.push({ path: entry.isDirectory() ? `${rel}/` : rel });
                     continue;
                 }
             }
             if (entry.isSymbolicLink()) {
                 const target = await symlinkTarget(full);
                 if (!isInside(target, rootReal)) {
-                    offenders.push({ path: rel, target });
+                    walk.offenders.push({ path: rel, target });
                 }
                 else if (hasDottedSegment(path.relative(rootReal, target))) {
-                    offenders.push({
+                    walk.offenders.push({
                         path: rel,
                         target: path.relative(rootReal, target).split(path.sep).join("/"),
                     });
@@ -73,17 +97,56 @@ async function walkOutput(outReal, rootReal) {
                 // A symlink that resolves cleanly inside is not uploaded, as before.
                 continue;
             }
+            const nameMatched = isPublishCredentialName(entry.name);
             if (entry.isDirectory()) {
+                if (nameMatched) {
+                    walk.credentialNames.push({
+                        seq: seq++,
+                        offender: { path: `${rel}/`, why: "name" },
+                    });
+                    continue;
+                }
                 await visit(full);
             }
             else if (entry.isFile()) {
-                const stat = await fs.stat(full);
-                files.push({ rel, size: stat.size });
+                const stat = await fs.lstat(full).catch(() => Promise.reject(unreadable(rel)));
+                const at = seq++;
+                if (nameMatched) {
+                    walk.credentialNames.push({ seq: at, offender: { path: rel, why: "name" } });
+                }
+                walk.files.push({
+                    rel,
+                    abs: full,
+                    size: stat.size,
+                    dev: stat.dev,
+                    ino: stat.ino,
+                    seq: at,
+                    nameMatched,
+                });
             }
         }
     };
     await visit(outReal);
-    return { files, offenders };
+    return walk;
+}
+/**
+ * Files whose real parent folder is not inside the real output. The walk reads
+ * a folder by path, so a folder swapped for a symlink mid-walk is walked through
+ * the link; this catches it after the fact. A parent that cannot be resolved
+ * counts as escaped: it is never assumed inside.
+ */
+async function escapedFiles(files, outReal) {
+    const parents = new Map();
+    const inside = (dir) => {
+        let p = parents.get(dir);
+        if (p === undefined) {
+            p = fs.realpath(dir).then((real) => real === outReal || isInside(real, outReal), () => false);
+            parents.set(dir, p);
+        }
+        return p;
+    };
+    const verdicts = await mapLimit(files, READ_CONCURRENCY, (f) => inside(path.dirname(f.abs)));
+    return files.filter((_, i) => !verdicts[i]).map((f) => ({ path: f.rel }));
 }
 /** Where a symlink lands: its realpath, or its lexical target when it dangles. */
 async function symlinkTarget(link) {
@@ -96,19 +159,93 @@ async function symlinkTarget(link) {
     }
 }
 /** A Git LFS pointer file: at most 1 KB, the spec's exact first line, and an oid line. */
-export async function isLfsPointer(absPath, size) {
+export function isLfsPointer(head, size) {
     if (size > LFS_POINTER_MAX_BYTES)
         return false;
-    const head = await fs.readFile(absPath, "utf8");
-    const lines = head.split("\n");
+    const lines = Buffer.from(head.buffer, head.byteOffset, head.byteLength)
+        .toString("utf8")
+        .split("\n");
     if (lines[0] !== LFS_FIRST_LINE)
         return false;
     return lines.some((l) => l.startsWith("oid sha256:"));
 }
+class UnreadableFile extends Error {
+}
+/**
+ * Open a walked file without following a link, and confirm it is still the
+ * regular file the walk saw: the same device and inode before and after the
+ * open. Anything else throws UnreadableFile.
+ */
+async function openWalked(file) {
+    let handle;
+    try {
+        const before = await fs.lstat(file.abs);
+        if (!before.isFile() || before.dev !== file.dev || before.ino !== file.ino) {
+            throw new UnreadableFile();
+        }
+        handle = await fs.open(file.abs, OPEN_NOFOLLOW);
+        const after = await handle.stat();
+        if (!after.isFile() || after.dev !== file.dev || after.ino !== file.ino) {
+            throw new UnreadableFile();
+        }
+        return handle;
+    }
+    catch {
+        await handle?.close().catch(() => undefined);
+        throw new UnreadableFile();
+    }
+}
+/** Exactly `length` bytes from offset 0; a short read is unreadable. */
+async function readExactly(handle, length) {
+    const buf = Buffer.alloc(length);
+    let got = 0;
+    while (got < length) {
+        const { bytesRead } = await handle.read(buf, got, length - got, got);
+        if (bytesRead === 0)
+            throw new UnreadableFile();
+        got += bytesRead;
+    }
+    return buf;
+}
 /**
- * The leak and LFS checks, in that order, before anything is hashed or sent.
- * Throws ManifestError; returns nothing on a clean output.
+ * The early pass's read: the first `min(size, 4096)` bytes, or the whole file
+ * for a `.json` of at most 64 KB. Opens without following links.
  */
+export async function readHead(file) {
+    const whole = isJsonName(file.rel) && file.size <= CREDENTIAL_JSON_MAX_BYTES;
+    const handle = await openWalked(file);
+    try {
+        return await readExactly(handle, whole ? file.size : Math.min(file.size, HEAD_READ_BYTES));
+    }
+    catch {
+        throw new UnreadableFile();
+    }
+    finally {
+        await handle.close().catch(() => undefined);
+    }
+}
+/** The content rules over a head (or whole small JSON). */
+function contentWhy(rel, size, bytes) {
+    const scanner = createKeyScanner();
+    scanner.push(bytes.subarray(0, HEAD_READ_BYTES).toString("latin1"));
+    if (scanner.end())
+        return "private_key";
+    if (isJsonName(rel) && size <= CREDENTIAL_JSON_MAX_BYTES)
+        return credentialJson(bytes);
+    return undefined;
+}
+async function mapLimit(items, limit, fn) {
+    const out = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < items.length) {
+            const i = next++;
+            out[i] = await fn(items[i]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return out;
+}
 /**
  * Folders an output must never be, or sit inside: the repository's history and
  * the Vincentt binding. Compared case-insensitively, as a default macOS volume
@@ -121,11 +258,11 @@ function forbiddenSegmentIn(rel) {
         .find((seg) => FORBIDDEN_OUTPUT_SEGMENTS.has(seg.toLowerCase()));
 }
 /**
- * The leak and LFS checks, in that order, before anything is hashed or sent.
- * Returns the output-relative file list the checks passed, which is exactly the
- * list the manifest is built from: what is checked is what is uploaded.
+ * The leak, credential and LFS checks, in that order, before anything is hashed
+ * or sent. Returns the file list the checks passed, which is exactly the list
+ * the manifest is built from: what is checked is what is uploaded.
  */
-export async function checkOutput(distDir, repoRoot) {
+export async function checkOutput(distDir, repoRoot, hooks = {}) {
     const rootReal = await fs.realpath(repoRoot).catch(() => path.resolve(repoRoot));
     const lexical = path.resolve(distDir);
     const lexicalRoot = path.resolve(repoRoot);
@@ -143,7 +280,7 @@ export async function checkOutput(distDir, repoRoot) {
         if (err.code === "ENOENT") {
             throw new ManifestError("no_dist", distDir, `no dist at ${distDir}`);
         }
-        throw err;
+        throw new ManifestError("file_unreadable", "", "the output could not be read");
     }
     if (outReal === rootReal) {
         throw new ManifestError("output_is_repo_root", distDir, `${distDir} resolves to the repo root`);
@@ -156,24 +293,63 @@ export async function checkOutput(distDir, repoRoot) {
     if (hiddenParent !== undefined) {
         throw new ManifestError("output_in_hidden", hiddenParent, `${distDir} is inside ${hiddenParent}`);
     }
-    if (!(await fs.stat(outReal)).isDirectory()) {
+    const outStat = await fs.stat(outReal).catch(() => {
+        throw new ManifestError("file_unreadable", "", "the output could not be read");
+    });
+    if (!outStat.isDirectory()) {
         throw new ManifestError("no_dist", distDir, `no dist at ${distDir}`);
     }
-    const walk = await walkOutput(outReal, rootReal);
+    const walk = await walkOutput(outReal, rootReal, hooks);
     const stats = {
         files: walk.files.length,
         bytes: walk.files.reduce((n, f) => n + f.size, 0),
     };
+    const escaped = await escapedFiles(walk.files, outReal);
+    if (escaped.length > 0) {
+        throw new ManifestError("path_escaped", escaped[0].path, "files resolved outside the output", { paths: escaped, stats });
+    }
     if (walk.offenders.length > 0) {
         throw new ManifestError("secret_path", walk.offenders[0].path, "hidden paths in the output", {
             paths: walk.offenders,
             stats,
         });
     }
+    const heads = await mapLimit(walk.files, READ_CONCURRENCY, async (f) => {
+        try {
+            const head = await readHead(f);
+            return {
+                why: f.nameMatched ? undefined : contentWhy(f.rel, f.size, head),
+                lfs: isLfsPointer(head, f.size),
+            };
+        }
+        catch {
+            return { unreadable: true };
+        }
+    });
+    const credentials = [...walk.credentialNames];
     const pointers = [];
-    for (const f of walk.files) {
-        if (await isLfsPointer(path.join(outReal, f.rel), f.size))
+    walk.files.forEach((f, i) => {
+        const h = heads[i];
+        if ("unreadable" in h)
+            return;
+        if (h.why !== undefined)
+            credentials.push({ seq: f.seq, offender: { path: f.rel, why: h.why } });
+        if (h.lfs)
             pointers.push({ path: f.rel });
+    });
+    if (credentials.length > 0) {
+        const paths = credentials.sort((a, b) => a.seq - b.seq).map((c) => c.offender);
+        throw new ManifestError("credential_file", paths[0].path, "credential files in the output", {
+            paths,
+            stats,
+        });
+    }
+    // A file the checks could not read fails closed; it is never assumed clean.
+    const unreadable = walk.files.find((_, i) => "unreadable" in heads[i]);
+    if (unreadable !== undefined) {
+        throw new ManifestError("file_unreadable", unreadable.rel, "a file could not be read", {
+            stats,
+        });
     }
     if (pointers.length > 0) {
         throw new ManifestError("lfs_pointer", pointers[0].path, "Git LFS pointers in the output", {
@@ -181,69 +357,112 @@ export async function checkOutput(distDir, repoRoot) {
             stats,
         });
     }
-    return walk.files;
+    return walk.files.map(({ rel, abs, size, dev, ino }) => ({ rel, abs, size, dev, ino }));
 }
-/** Recursively list files under `dir` as dir-relative POSIX paths. Symlinks are not followed. */
-export async function walkDist(dir, base = dir) {
-    let entries;
+/**
+ * Hash a walked file and re-run the content rules over the exact bytes hashed.
+ * The file is opened without following links and must still be the regular
+ * file the walk saw. Every read error throws `file_unreadable`; only a JSON
+ * parse failure is "not a match".
+ */
+export async function hashAndScan(absPath, walked) {
+    const unreadable = () => new ManifestError("file_unreadable", walked.rel, "a file changed while it was read");
+    let handle;
     try {
-        entries = await fs.readdir(dir, { withFileTypes: true });
+        handle = await openWalked({ ...walked, abs: absPath });
     }
-    catch (err) {
-        if (err.code === "ENOENT") {
-            throw new ManifestError("no_dist", dir, `no dist at ${dir}`);
-        }
-        throw err;
-    }
-    const out = [];
-    for (const entry of entries) {
-        const full = path.join(dir, entry.name);
-        if (entry.isDirectory())
-            out.push(...(await walkDist(full, base)));
-        else if (entry.isFile()) {
-            out.push(path.relative(base, full).split(path.sep).join("/"));
+    catch {
+        throw unreadable();
+    }
+    try {
+        const { size } = await handle.stat();
+        const hash = createHash("sha256");
+        const scanner = createKeyScanner();
+        const keepJson = isJsonName(walked.rel) && size <= CREDENTIAL_JSON_MAX_BYTES;
+        const kept = [];
+        const chunk = Buffer.alloc(HASH_CHUNK_BYTES);
+        let read = 0;
+        for (;;) {
+            const { bytesRead } = await handle.read(chunk, 0, chunk.length, read);
+            if (bytesRead === 0)
+                break;
+            const piece = chunk.subarray(0, bytesRead);
+            hash.update(piece);
+            scanner.push(piece.toString("latin1"));
+            if (keepJson)
+                kept.push(Buffer.from(piece));
+            read += bytesRead;
+            if (read > size)
+                throw unreadable();
         }
+        if (read !== size)
+            throw unreadable();
+        let why = scanner.end() ? "private_key" : undefined;
+        if (why === undefined && keepJson)
+            why = credentialJson(Buffer.concat(kept));
+        return { sha256: hash.digest("base64"), size, why };
+    }
+    catch (err) {
+        if (err instanceof ManifestError)
+            throw err;
+        throw unreadable();
+    }
+    finally {
+        await handle.close().catch(() => undefined);
     }
-    return out;
-}
-/** sha256 of a file, base64 std — the exact form `x-amz-checksum-sha256` wants. */
-export async function hashFile(absPath) {
-    const hash = createHash("sha256");
-    const stream = createReadStream(absPath);
-    for await (const chunk of stream)
-        hash.update(chunk);
-    return hash.digest("base64");
 }
 /**
  * Build the manifest for a dist directory. Every path is normalized before it is
  * hashed, so the string this returns is the string the API validates, signs, and
  * later lists — there is no second form anywhere.
  */
-export async function buildManifest(distDir, opts = {}) {
-    // With a repo root, the list the leak and LFS checks passed IS the upload
-    // list. A second walk could see a file created after the checks ran.
-    const raw = opts.repoRoot !== undefined
-        ? (await checkOutput(distDir, opts.repoRoot)).map((f) => f.rel)
-        : await walkDist(distDir);
+export async function buildManifest(distDir, opts) {
+    // The list the checks passed IS the upload list. A second walk could see a
+    // file created after the checks ran.
+    const walked = await checkOutput(distDir, opts.repoRoot, {
+        beforeReadDir: opts.beforeReadDir,
+    });
     const normalized = [];
-    for (const rel of raw) {
-        const result = normalizePath(rel);
+    for (const file of walked) {
+        const result = normalizePath(file.rel);
         if (!result.ok) {
-            throw new ManifestError(result.reason, rel, `${rel}: ${result.reason}`);
+            throw new ManifestError(result.reason, file.rel, `${file.rel}: ${result.reason}`);
         }
-        normalized.push({ rel, norm: result.path });
+        normalized.push({ file, norm: result.path });
     }
     const pathError = validatePathSet(normalized.map((n) => n.norm));
     if (pathError) {
         throw new ManifestError(pathError.reason, pathError.path, `${pathError.path}: ${pathError.reason}`);
     }
+    await opts.beforeHash?.(walked);
+    const stats = {
+        files: walked.length,
+        bytes: walked.reduce((n, f) => n + f.size, 0),
+    };
     const files = [];
+    const credentials = [];
     let totalBytes = 0;
-    for (const { rel, norm } of normalized) {
-        const abs = path.join(distDir, rel);
-        const stat = await fs.stat(abs);
-        files.push({ path: norm, size: stat.size, sha256: await hashFile(abs) });
-        totalBytes += stat.size;
+    for (const { file, norm } of normalized) {
+        let hashed;
+        try {
+            hashed = await hashAndScan(file.abs, file);
+        }
+        catch (err) {
+            if (err instanceof ManifestError) {
+                throw new ManifestError(err.reason, err.offendingPath, err.message, { stats });
+            }
+            throw err;
+        }
+        if (hashed.why !== undefined)
+            credentials.push({ path: file.rel, why: hashed.why });
+        files.push({ path: norm, size: hashed.size, sha256: hashed.sha256 });
+        totalBytes += hashed.size;
+    }
+    if (credentials.length > 0) {
+        throw new ManifestError("credential_file", credentials[0].path, "credential files in the output", {
+            paths: credentials,
+            stats,
+        });
     }
     files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
     return { files, totalBytes };

package/dist/publish/upload.d.ts

@@ -22,6 +22,24 @@ export declare class UploadExpired extends Error {
     readonly path: string;
     constructor(path: string);
 }
+/**
+ * The api returned an upload target for a path this run's manifest does not
+ * hold. Nothing is sent: only files that passed this run's checks are uploaded.
+ */
+export declare class UploadTargetRejected extends Error {
+    readonly path: string;
+    constructor(path: string);
+}
+/**
+ * A target's signed checksum is not the one this run's manifest holds for that
+ * path. The PUT reopens the file by path, so the signed checksum is what makes
+ * "checked is uploaded" hold; a target that would sign anything else is refused
+ * before any byte is sent.
+ */
+export declare class UploadChecksumMismatch extends Error {
+    readonly path: string;
+    constructor(path: string);
+}
 export interface UploadProgress {
     filesDone: number;
     filesTotal: number;

package/dist/publish/upload.js

@@ -46,6 +46,37 @@ export class UploadExpired extends Error {
         this.name = "UploadExpired";
     }
 }
+/**
+ * The api returned an upload target for a path this run's manifest does not
+ * hold. Nothing is sent: only files that passed this run's checks are uploaded.
+ */
+export class UploadTargetRejected extends Error {
+    path;
+    constructor(path) {
+        super(`the upload target ${path} is not in this build`);
+        this.path = path;
+        this.name = "UploadTargetRejected";
+    }
+}
+/**
+ * A target's signed checksum is not the one this run's manifest holds for that
+ * path. The PUT reopens the file by path, so the signed checksum is what makes
+ * "checked is uploaded" hold; a target that would sign anything else is refused
+ * before any byte is sent.
+ */
+export class UploadChecksumMismatch extends Error {
+    path;
+    constructor(path) {
+        super(`the upload target ${path} does not carry this build's checksum`);
+        this.path = path;
+        this.name = "UploadChecksumMismatch";
+    }
+}
+const CHECKSUM_HEADER = "x-amz-checksum-sha256";
+function signedChecksum(headers) {
+    const key = Object.keys(headers).find((k) => k.toLowerCase() === CHECKSUM_HEADER);
+    return key === undefined ? undefined : headers[key];
+}
 const DEFAULT_CONCURRENCY = 6;
 const DEFAULT_MAX_ATTEMPTS = 5;
 const BACKOFF_BASE_MS = 500;
@@ -99,6 +130,13 @@ export function classifyS3Response(status, body) {
 export async function uploadAll(opts) {
     const fetchImpl = opts.fetchImpl ?? fetch;
     const sizes = new Map(opts.manifest.map((f) => [f.path, f.size]));
+    const foreign = opts.targets.find((t) => !sizes.has(t.path));
+    if (foreign !== undefined)
+        throw new UploadTargetRejected(foreign.path);
+    const sums = new Map(opts.manifest.map((f) => [f.path, f.sha256]));
+    const mismatched = opts.targets.find((t) => signedChecksum(t.headers) !== sums.get(t.path));
+    if (mismatched !== undefined)
+        throw new UploadChecksumMismatch(mismatched.path);
     const bytesTotal = opts.targets.reduce((n, t) => n + (sizes.get(t.path) ?? 0), 0);
     const queue = [...opts.targets];
     const retry = {

package/dist/publish/weight.d.ts

@@ -0,0 +1,23 @@
+import type { ManifestEntry } from "./manifest.js";
+export declare const HEAVY_MODEL_BYTES = 10000000;
+export declare const HEAVY_IMAGE_PX = 2048;
+export declare const IMAGE_HEAD_BYTES = 524288;
+export type HeavyFinding = {
+    path: string;
+    kind: "model";
+    bytes: number;
+} | {
+    path: string;
+    kind: "image";
+    width: number;
+    height: number;
+};
+export type ReadHead = (path: string, n: number) => Promise<Uint8Array>;
+export declare function heavyFileFindings(files: readonly ManifestEntry[], readHead: ReadHead): Promise<HeavyFinding[]>;
+type Size = {
+    width: number;
+    height: number;
+};
+/** Pixel size from a file header, or undefined for an unknown or truncated one. */
+export declare function imageSize(head: Uint8Array): Size | undefined;
+export {};