@vincentt-xr/cli 0.19.0 → 0.21.0

package/dist/publish/manifest.d.ts

@@ -1,3 +1,4 @@
+import { type CredentialJsonType } from "./keyScan.js";
 import { type PathRejection } from "./paths.js";
 export interface ManifestEntry {
     /** dist-relative, forward slashes, no leading `/`, normalized. */
@@ -10,13 +11,16 @@ export interface Manifest {
     files: ManifestEntry[];
     totalBytes: number;
 }
-export type ManifestReason = PathRejection | "no_dist" | "output_is_repo_root" | "output_outside_repo" | "output_in_hidden" | "secret_path" | "lfs_pointer";
-/** An output-relative path the leak or LFS check refused. */
+export type ManifestReason = PathRejection | "no_dist" | "output_is_repo_root" | "output_outside_repo" | "output_in_hidden" | "secret_path" | "credential_file" | "lfs_pointer" | "file_unreadable";
+export type CredentialWhy = "name" | "private_key" | CredentialJsonType;
+/** An output-relative path the leak, credential or LFS check refused. */
 export interface Offender {
     /** Output-relative, POSIX. A refused folder carries a trailing `/`. */
     path: string;
     /** For a symlink: where it resolves, repo-relative inside and absolute outside. */
     target?: string;
+    /** For a credential file: what made it one. */
+    why?: CredentialWhy;
 }
 export interface ManifestErrorDetail {
     paths?: Offender[];
@@ -36,33 +40,61 @@ export declare class ManifestError extends Error {
     };
     constructor(reason: ManifestReason, offendingPath: string, message: string, detail?: ManifestErrorDetail);
 }
+/** A file the walk passed: what the hash step must find again at the same path. */
+export interface WalkedFile {
+    /** Output-relative, POSIX. */
+    rel: string;
+    /** Absolute, under the real output folder. */
+    abs: string;
+    size: number;
+    dev: number;
+    ino: number;
+}
 export interface BuildManifestOptions {
     /**
-     * The directory holding the binding. When given, the output is refused before
-     * any hashing if it is the repo root, lies outside it, holds a dotted name
-     * outside the two allowed ones, holds a symlink that resolves outside the root
-     * or onto a dotted path, or holds a Git LFS pointer. All of it is local.
+     * The directory holding the binding. Required: there is no unchecked
+     * manifest. The output is refused before any hashing if it is the repo root,
+     * lies outside it, holds a dotted name outside the two allowed ones, holds a
+     * symlink that resolves outside the root or onto a dotted path, holds a
+     * credential file, or holds a Git LFS pointer. All of it is local.
      */
-    repoRoot?: string;
+    repoRoot: string;
+    /**
+     * Test-only: runs after the checks and before the hash step, so a test can
+     * change a checked file. Never set by the command.
+     */
+    beforeHash?: (files: readonly WalkedFile[]) => Promise<void> | void;
 }
 /** A Git LFS pointer file: at most 1 KB, the spec's exact first line, and an oid line. */
-export declare function isLfsPointer(absPath: string, size: number): Promise<boolean>;
+export declare function isLfsPointer(head: Uint8Array, size: number): boolean;
 /**
- * The leak and LFS checks, in that order, before anything is hashed or sent.
- * Returns the output-relative file list the checks passed, which is exactly the
- * list the manifest is built from: what is checked is what is uploaded.
+ * The early pass's read: the first `min(size, 4096)` bytes, or the whole file
+ * for a `.json` of at most 64 KB. Opens without following links.
  */
-export declare function checkOutput(distDir: string, repoRoot: string): Promise<{
-    rel: string;
+export declare function readHead(file: WalkedFile): Promise<Buffer>;
+/**
+ * The leak, credential and LFS checks, in that order, before anything is hashed
+ * or sent. Returns the file list the checks passed, which is exactly the list
+ * the manifest is built from: what is checked is what is uploaded.
+ */
+export declare function checkOutput(distDir: string, repoRoot: string): Promise<WalkedFile[]>;
+export interface HashResult {
+    /** base64 std, the exact form `x-amz-checksum-sha256` wants. */
+    sha256: string;
     size: number;
-}[]>;
-/** Recursively list files under `dir` as dir-relative POSIX paths. Symlinks are not followed. */
-export declare function walkDist(dir: string, base?: string): Promise<string[]>;
-/** sha256 of a file, base64 std — the exact form `x-amz-checksum-sha256` wants. */
-export declare function hashFile(absPath: string): Promise<string>;
+    /** Set when the hashed bytes hold a private key or a Google credential. */
+    why?: CredentialWhy;
+}
+/**
+ * Hash a walked file and re-run the content rules over the exact bytes hashed.
+ * The file is opened without following links and must still be the regular
+ * file the walk saw. Every read error throws `file_unreadable`; only a JSON
+ * parse failure is "not a match".
+ */
+export declare function hashAndScan(absPath: string, walked: WalkedFile): Promise<HashResult>;
 /**
  * Build the manifest for a dist directory. Every path is normalized before it is
  * hashed, so the string this returns is the string the API validates, signs, and
  * later lists — there is no second form anywhere.
  */
-export declare function buildManifest(distDir: string, opts?: BuildManifestOptions): Promise<Manifest>;
+export declare function buildManifest(distDir: string, opts: BuildManifestOptions): Promise<Manifest>;

package/dist/publish/manifest.js

@@ -5,8 +5,10 @@
 // every single PUT while both repos' unit suites stay green (§2.7's first
 // contract trap).
 import { createHash } from "node:crypto";
-import { createReadStream, promises as fs } from "node:fs";
+import { constants as fsConstants, promises as fs } from "node:fs";
 import path from "node:path";
+import { PUBLISH_CREDENTIAL_NAMES, PUBLISH_STRIP, foldName, isCredentialName, } from "../shared/credentialNames.js";
+import { createKeyScanner, credentialJson } from "./keyScan.js";
 import { normalizePath, validatePathSet } from "./paths.js";
 export class ManifestError extends Error {
     reason;
@@ -27,6 +29,13 @@ const ALLOWED_ROOT_DIR = ".well-known";
 const ALLOWED_ROOT_FILE = ".nojekyll";
 const LFS_POINTER_MAX_BYTES = 1024;
 const LFS_FIRST_LINE = "version https://git-lfs.github.com/spec/v1";
+/** The early pass reads this much of every file. */
+const HEAD_READ_BYTES = 4096;
+/** A `.json` at most this size is read whole and parsed. */
+const CREDENTIAL_JSON_MAX_BYTES = 65_536;
+const READ_CONCURRENCY = 16;
+const HASH_CHUNK_BYTES = 64 * 1024;
+const OPEN_NOFOLLOW = fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0);
 function isInside(child, parent) {
     const rel = path.relative(parent, child);
     return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel);
@@ -34,14 +43,21 @@ function isInside(child, parent) {
 function hasDottedSegment(rel) {
     return rel.split(/[\\/]/).some((seg) => seg.startsWith("."));
 }
+function isPublishCredentialName(name) {
+    return isCredentialName(foldName(name), PUBLISH_CREDENTIAL_NAMES, PUBLISH_STRIP);
+}
+function isJsonName(rel) {
+    return rel.toLowerCase().endsWith(".json");
+}
 /**
- * The leak walk. Unlike `walkDist` it records what it does not upload: a dotted
- * name is an offender (a folder once, with a trailing slash, never descended),
- * and every symlink is resolved and judged against the real repo root.
+ * The one walk. It records what it does not upload: a dotted name is an
+ * offender (a folder once, with a trailing slash, never descended), every
+ * symlink is resolved and judged against the real repo root, and a name on the
+ * credential list is recorded apart (a folder once, never descended).
  */
 async function walkOutput(outReal, rootReal) {
-    const files = [];
-    const offenders = [];
+    const walk = { files: [], offenders: [], credentialNames: [] };
+    let seq = 0;
     const visit = async (dir) => {
         const entries = await fs.readdir(dir, { withFileTypes: true });
         entries.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
@@ -55,17 +71,17 @@ async function walkOutput(outReal, rootReal) {
                     ((lower === ALLOWED_ROOT_DIR && entry.isDirectory()) ||
                         (lower === ALLOWED_ROOT_FILE && entry.isFile()));
                 if (!allowed) {
-                    offenders.push({ path: entry.isDirectory() ? `${rel}/` : rel });
+                    walk.offenders.push({ path: entry.isDirectory() ? `${rel}/` : rel });
                     continue;
                 }
             }
             if (entry.isSymbolicLink()) {
                 const target = await symlinkTarget(full);
                 if (!isInside(target, rootReal)) {
-                    offenders.push({ path: rel, target });
+                    walk.offenders.push({ path: rel, target });
                 }
                 else if (hasDottedSegment(path.relative(rootReal, target))) {
-                    offenders.push({
+                    walk.offenders.push({
                         path: rel,
                         target: path.relative(rootReal, target).split(path.sep).join("/"),
                     });
@@ -73,17 +89,37 @@ async function walkOutput(outReal, rootReal) {
                 // A symlink that resolves cleanly inside is not uploaded, as before.
                 continue;
             }
+            const nameMatched = isPublishCredentialName(entry.name);
             if (entry.isDirectory()) {
+                if (nameMatched) {
+                    walk.credentialNames.push({
+                        seq: seq++,
+                        offender: { path: `${rel}/`, why: "name" },
+                    });
+                    continue;
+                }
                 await visit(full);
             }
             else if (entry.isFile()) {
-                const stat = await fs.stat(full);
-                files.push({ rel, size: stat.size });
+                const stat = await fs.lstat(full);
+                const at = seq++;
+                if (nameMatched) {
+                    walk.credentialNames.push({ seq: at, offender: { path: rel, why: "name" } });
+                }
+                walk.files.push({
+                    rel,
+                    abs: full,
+                    size: stat.size,
+                    dev: stat.dev,
+                    ino: stat.ino,
+                    seq: at,
+                    nameMatched,
+                });
             }
         }
     };
     await visit(outReal);
-    return { files, offenders };
+    return walk;
 }
 /** Where a symlink lands: its realpath, or its lexical target when it dangles. */
 async function symlinkTarget(link) {
@@ -96,19 +132,93 @@ async function symlinkTarget(link) {
     }
 }
 /** A Git LFS pointer file: at most 1 KB, the spec's exact first line, and an oid line. */
-export async function isLfsPointer(absPath, size) {
+export function isLfsPointer(head, size) {
     if (size > LFS_POINTER_MAX_BYTES)
         return false;
-    const head = await fs.readFile(absPath, "utf8");
-    const lines = head.split("\n");
+    const lines = Buffer.from(head.buffer, head.byteOffset, head.byteLength)
+        .toString("utf8")
+        .split("\n");
     if (lines[0] !== LFS_FIRST_LINE)
         return false;
     return lines.some((l) => l.startsWith("oid sha256:"));
 }
+class UnreadableFile extends Error {
+}
 /**
- * The leak and LFS checks, in that order, before anything is hashed or sent.
- * Throws ManifestError; returns nothing on a clean output.
+ * Open a walked file without following a link, and confirm it is still the
+ * regular file the walk saw: the same device and inode before and after the
+ * open. Anything else throws UnreadableFile.
  */
+async function openWalked(file) {
+    let handle;
+    try {
+        const before = await fs.lstat(file.abs);
+        if (!before.isFile() || before.dev !== file.dev || before.ino !== file.ino) {
+            throw new UnreadableFile();
+        }
+        handle = await fs.open(file.abs, OPEN_NOFOLLOW);
+        const after = await handle.stat();
+        if (!after.isFile() || after.dev !== file.dev || after.ino !== file.ino) {
+            throw new UnreadableFile();
+        }
+        return handle;
+    }
+    catch {
+        await handle?.close().catch(() => undefined);
+        throw new UnreadableFile();
+    }
+}
+/** Exactly `length` bytes from offset 0; a short read is unreadable. */
+async function readExactly(handle, length) {
+    const buf = Buffer.alloc(length);
+    let got = 0;
+    while (got < length) {
+        const { bytesRead } = await handle.read(buf, got, length - got, got);
+        if (bytesRead === 0)
+            throw new UnreadableFile();
+        got += bytesRead;
+    }
+    return buf;
+}
+/**
+ * The early pass's read: the first `min(size, 4096)` bytes, or the whole file
+ * for a `.json` of at most 64 KB. Opens without following links.
+ */
+export async function readHead(file) {
+    const whole = isJsonName(file.rel) && file.size <= CREDENTIAL_JSON_MAX_BYTES;
+    const handle = await openWalked(file);
+    try {
+        return await readExactly(handle, whole ? file.size : Math.min(file.size, HEAD_READ_BYTES));
+    }
+    catch {
+        throw new UnreadableFile();
+    }
+    finally {
+        await handle.close().catch(() => undefined);
+    }
+}
+/** The content rules over a head (or whole small JSON). */
+function contentWhy(rel, size, bytes) {
+    const scanner = createKeyScanner();
+    scanner.push(bytes.subarray(0, HEAD_READ_BYTES).toString("latin1"));
+    if (scanner.end())
+        return "private_key";
+    if (isJsonName(rel) && size <= CREDENTIAL_JSON_MAX_BYTES)
+        return credentialJson(bytes);
+    return undefined;
+}
+async function mapLimit(items, limit, fn) {
+    const out = new Array(items.length);
+    let next = 0;
+    const worker = async () => {
+        while (next < items.length) {
+            const i = next++;
+            out[i] = await fn(items[i]);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return out;
+}
 /**
  * Folders an output must never be, or sit inside: the repository's history and
  * the Vincentt binding. Compared case-insensitively, as a default macOS volume
@@ -121,9 +231,9 @@ function forbiddenSegmentIn(rel) {
         .find((seg) => FORBIDDEN_OUTPUT_SEGMENTS.has(seg.toLowerCase()));
 }
 /**
- * The leak and LFS checks, in that order, before anything is hashed or sent.
- * Returns the output-relative file list the checks passed, which is exactly the
- * list the manifest is built from: what is checked is what is uploaded.
+ * The leak, credential and LFS checks, in that order, before anything is hashed
+ * or sent. Returns the file list the checks passed, which is exactly the list
+ * the manifest is built from: what is checked is what is uploaded.
  */
 export async function checkOutput(distDir, repoRoot) {
     const rootReal = await fs.realpath(repoRoot).catch(() => path.resolve(repoRoot));
@@ -170,10 +280,42 @@ export async function checkOutput(distDir, repoRoot) {
             stats,
         });
     }
+    const heads = await mapLimit(walk.files, READ_CONCURRENCY, async (f) => {
+        try {
+            const head = await readHead(f);
+            return {
+                why: f.nameMatched ? undefined : contentWhy(f.rel, f.size, head),
+                lfs: isLfsPointer(head, f.size),
+            };
+        }
+        catch {
+            return { unreadable: true };
+        }
+    });
+    const credentials = [...walk.credentialNames];
     const pointers = [];
-    for (const f of walk.files) {
-        if (await isLfsPointer(path.join(outReal, f.rel), f.size))
+    walk.files.forEach((f, i) => {
+        const h = heads[i];
+        if ("unreadable" in h)
+            return;
+        if (h.why !== undefined)
+            credentials.push({ seq: f.seq, offender: { path: f.rel, why: h.why } });
+        if (h.lfs)
             pointers.push({ path: f.rel });
+    });
+    if (credentials.length > 0) {
+        const paths = credentials.sort((a, b) => a.seq - b.seq).map((c) => c.offender);
+        throw new ManifestError("credential_file", paths[0].path, "credential files in the output", {
+            paths,
+            stats,
+        });
+    }
+    // A file the checks could not read fails closed; it is never assumed clean.
+    const unreadable = walk.files.find((_, i) => "unreadable" in heads[i]);
+    if (unreadable !== undefined) {
+        throw new ManifestError("file_unreadable", unreadable.rel, "a file could not be read", {
+            stats,
+        });
     }
     if (pointers.length > 0) {
         throw new ManifestError("lfs_pointer", pointers[0].path, "Git LFS pointers in the output", {
@@ -181,69 +323,110 @@ export async function checkOutput(distDir, repoRoot) {
             stats,
         });
     }
-    return walk.files;
+    return walk.files.map(({ rel, abs, size, dev, ino }) => ({ rel, abs, size, dev, ino }));
 }
-/** Recursively list files under `dir` as dir-relative POSIX paths. Symlinks are not followed. */
-export async function walkDist(dir, base = dir) {
-    let entries;
+/**
+ * Hash a walked file and re-run the content rules over the exact bytes hashed.
+ * The file is opened without following links and must still be the regular
+ * file the walk saw. Every read error throws `file_unreadable`; only a JSON
+ * parse failure is "not a match".
+ */
+export async function hashAndScan(absPath, walked) {
+    const unreadable = () => new ManifestError("file_unreadable", walked.rel, "a file changed while it was read");
+    let handle;
     try {
-        entries = await fs.readdir(dir, { withFileTypes: true });
+        handle = await openWalked({ ...walked, abs: absPath });
     }
-    catch (err) {
-        if (err.code === "ENOENT") {
-            throw new ManifestError("no_dist", dir, `no dist at ${dir}`);
-        }
-        throw err;
+    catch {
+        throw unreadable();
     }
-    const out = [];
-    for (const entry of entries) {
-        const full = path.join(dir, entry.name);
-        if (entry.isDirectory())
-            out.push(...(await walkDist(full, base)));
-        else if (entry.isFile()) {
-            out.push(path.relative(base, full).split(path.sep).join("/"));
+    try {
+        const { size } = await handle.stat();
+        const hash = createHash("sha256");
+        const scanner = createKeyScanner();
+        const keepJson = isJsonName(walked.rel) && size <= CREDENTIAL_JSON_MAX_BYTES;
+        const kept = [];
+        const chunk = Buffer.alloc(HASH_CHUNK_BYTES);
+        let read = 0;
+        for (;;) {
+            const { bytesRead } = await handle.read(chunk, 0, chunk.length, read);
+            if (bytesRead === 0)
+                break;
+            const piece = chunk.subarray(0, bytesRead);
+            hash.update(piece);
+            scanner.push(piece.toString("latin1"));
+            if (keepJson)
+                kept.push(Buffer.from(piece));
+            read += bytesRead;
+            if (read > size)
+                throw unreadable();
         }
+        if (read !== size)
+            throw unreadable();
+        let why = scanner.end() ? "private_key" : undefined;
+        if (why === undefined && keepJson)
+            why = credentialJson(Buffer.concat(kept));
+        return { sha256: hash.digest("base64"), size, why };
+    }
+    catch (err) {
+        if (err instanceof ManifestError)
+            throw err;
+        throw unreadable();
+    }
+    finally {
+        await handle.close().catch(() => undefined);
     }
-    return out;
-}
-/** sha256 of a file, base64 std — the exact form `x-amz-checksum-sha256` wants. */
-export async function hashFile(absPath) {
-    const hash = createHash("sha256");
-    const stream = createReadStream(absPath);
-    for await (const chunk of stream)
-        hash.update(chunk);
-    return hash.digest("base64");
 }
 /**
  * Build the manifest for a dist directory. Every path is normalized before it is
  * hashed, so the string this returns is the string the API validates, signs, and
  * later lists — there is no second form anywhere.
  */
-export async function buildManifest(distDir, opts = {}) {
-    // With a repo root, the list the leak and LFS checks passed IS the upload
-    // list. A second walk could see a file created after the checks ran.
-    const raw = opts.repoRoot !== undefined
-        ? (await checkOutput(distDir, opts.repoRoot)).map((f) => f.rel)
-        : await walkDist(distDir);
+export async function buildManifest(distDir, opts) {
+    // The list the checks passed IS the upload list. A second walk could see a
+    // file created after the checks ran.
+    const walked = await checkOutput(distDir, opts.repoRoot);
     const normalized = [];
-    for (const rel of raw) {
-        const result = normalizePath(rel);
+    for (const file of walked) {
+        const result = normalizePath(file.rel);
         if (!result.ok) {
-            throw new ManifestError(result.reason, rel, `${rel}: ${result.reason}`);
+            throw new ManifestError(result.reason, file.rel, `${file.rel}: ${result.reason}`);
         }
-        normalized.push({ rel, norm: result.path });
+        normalized.push({ file, norm: result.path });
     }
     const pathError = validatePathSet(normalized.map((n) => n.norm));
     if (pathError) {
         throw new ManifestError(pathError.reason, pathError.path, `${pathError.path}: ${pathError.reason}`);
     }
+    await opts.beforeHash?.(walked);
+    const stats = {
+        files: walked.length,
+        bytes: walked.reduce((n, f) => n + f.size, 0),
+    };
     const files = [];
+    const credentials = [];
     let totalBytes = 0;
-    for (const { rel, norm } of normalized) {
-        const abs = path.join(distDir, rel);
-        const stat = await fs.stat(abs);
-        files.push({ path: norm, size: stat.size, sha256: await hashFile(abs) });
-        totalBytes += stat.size;
+    for (const { file, norm } of normalized) {
+        let hashed;
+        try {
+            hashed = await hashAndScan(file.abs, file);
+        }
+        catch (err) {
+            if (err instanceof ManifestError) {
+                throw new ManifestError(err.reason, err.offendingPath, err.message, { stats });
+            }
+            throw err;
+        }
+        if (hashed.why !== undefined)
+            credentials.push({ path: file.rel, why: hashed.why });
+        files.push({ path: norm, size: hashed.size, sha256: hashed.sha256 });
+        totalBytes += hashed.size;
+    }
+    if (credentials.length > 0) {
+        throw new ManifestError("credential_file", credentials[0].path, "credential files in the output", {
+            paths: credentials,
+            stats,
+        });
     }
     files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
     return { files, totalBytes };

package/dist/publish/upload.d.ts

@@ -22,6 +22,14 @@ export declare class UploadExpired extends Error {
     readonly path: string;
     constructor(path: string);
 }
+/**
+ * The api returned an upload target for a path this run's manifest does not
+ * hold. Nothing is sent: only files that passed this run's checks are uploaded.
+ */
+export declare class UploadTargetRejected extends Error {
+    readonly path: string;
+    constructor(path: string);
+}
 export interface UploadProgress {
     filesDone: number;
     filesTotal: number;

package/dist/publish/upload.js

@@ -46,6 +46,18 @@ export class UploadExpired extends Error {
         this.name = "UploadExpired";
     }
 }
+/**
+ * The api returned an upload target for a path this run's manifest does not
+ * hold. Nothing is sent: only files that passed this run's checks are uploaded.
+ */
+export class UploadTargetRejected extends Error {
+    path;
+    constructor(path) {
+        super(`the upload target ${path} is not in this build`);
+        this.path = path;
+        this.name = "UploadTargetRejected";
+    }
+}
 const DEFAULT_CONCURRENCY = 6;
 const DEFAULT_MAX_ATTEMPTS = 5;
 const BACKOFF_BASE_MS = 500;
@@ -99,6 +111,9 @@ export function classifyS3Response(status, body) {
 export async function uploadAll(opts) {
     const fetchImpl = opts.fetchImpl ?? fetch;
     const sizes = new Map(opts.manifest.map((f) => [f.path, f.size]));
+    const foreign = opts.targets.find((t) => !sizes.has(t.path));
+    if (foreign !== undefined)
+        throw new UploadTargetRejected(foreign.path);
     const bytesTotal = opts.targets.reduce((n, t) => n + (sizes.get(t.path) ?? 0), 0);
     const queue = [...opts.targets];
     const retry = {

package/dist/publish/weight.d.ts

@@ -0,0 +1,23 @@
+import type { ManifestEntry } from "./manifest.js";
+export declare const HEAVY_MODEL_BYTES = 10000000;
+export declare const HEAVY_IMAGE_PX = 2048;
+export declare const IMAGE_HEAD_BYTES = 524288;
+export type HeavyFinding = {
+    path: string;
+    kind: "model";
+    bytes: number;
+} | {
+    path: string;
+    kind: "image";
+    width: number;
+    height: number;
+};
+export type ReadHead = (path: string, n: number) => Promise<Uint8Array>;
+export declare function heavyFileFindings(files: readonly ManifestEntry[], readHead: ReadHead): Promise<HeavyFinding[]>;
+type Size = {
+    width: number;
+    height: number;
+};
+/** Pixel size from a file header, or undefined for an unknown or truncated one. */
+export declare function imageSize(head: Uint8Array): Size | undefined;
+export {};