@indigoai-us/hq-cli 5.345.170 → 5.345.172

package/dist/lib/secret-scan/range.js

@@ -0,0 +1,266 @@
+/**
+ * Range scan: the lines a branch added between <base> and <head>, as
+ * `commit-secret-scan.sh pr-check` needs them.
+ *
+ * What is scanned:
+ *   - The added lines of every non-merge commit in base..head, each against
+ *     its own parent. A secret committed in one commit and deleted in a later
+ *     one is still reported: it stays in the branch history.
+ *   - For every merge commit in base..head, the lines that differ from all of
+ *     its parents (git's combined diff, "++" lines): content the merge itself
+ *     introduced, such as a conflict resolution. Lines a merge only carries
+ *     over from one parent are either base content or scanned in that
+ *     parent's own commits.
+ *   - The added lines of the net diff merge-base(base, head)..head.
+ *   - Binary files those diffs add or change: path rules, plus their bytes
+ *     scanned as text up to MAX_BINARY_BYTES (larger ones are warned about).
+ * Lines that already existed on the base, and earlier versions of lines the
+ * branch did not touch, are not scanned.
+ *
+ * Config comes from the repo's .gitleaks.toml:
+ *   - Allowlists: the exact-value exemptions committed on the base (the same
+ *     contract as `scan secrets --repo --allowlist-rev`). A branch cannot
+ *     exempt its own commits by adding an allowlist.
+ *   - Rules: [[rules]] committed on the base, added to the built-in rules.
+ *     Rules the branch adds or changes are not compiled or run: a branch-added
+ *     regex could be written to backtrack for minutes (ReDoS), and the base
+ *     copy has been reviewed.
+ * A rejected allowlist or rules section is warned about and dropped; the scan
+ * still runs with the built-in rules (stricter, never looser).
+ *
+ * Range errors: an unknown rev, a reversed range (head behind base) and a
+ * merge-base failure other than "no common history" exit 2. base == head is an
+ * empty range: a warning and no findings.
+ */
+import { execFileSync, spawnSync } from "node:child_process";
+import { scanFragment, scanPath } from "./detect.js";
+import { InvalidAllowlistError, InvalidRepoRulesError, ScanConfigError, loadCommittedAllowlists, loadCommittedRules } from "./repo-config.js";
+import { compiledRules } from "./rules.js";
+import { ALLOWLIST_IGNORED_WARNING, parseUnifiedDiff, repoRoot } from "./staged.js";
+export const RULES_IGNORED_WARNING = "HQ secret scan: warning: the repo .gitleaks.toml [[rules]] were rejected; scanning with the built-in rules only";
+export const EMPTY_RANGE_WARNING = "HQ secret scan: warning: the range is empty (base and head are the same commit); nothing to scan";
+/** Binary blobs larger than this are not read; the scan warns instead. */
+export const MAX_BINARY_BYTES = 10 * 1024 * 1024;
+const PATCH_ARGS = [
+    "--no-color",
+    "--no-ext-diff",
+    "--no-textconv",
+    "--unified=0",
+    "--diff-filter=ACMR",
+    "--src-prefix=a/",
+    "--dst-prefix=b/",
+];
+function git(root, args) {
+    return execFileSync("git", ["-c", "core.quotePath=false", ...args], {
+        cwd: root,
+        encoding: "utf8",
+        maxBuffer: 1024 * 1024 * 1024,
+        stdio: ["ignore", "pipe", "pipe"],
+    });
+}
+function commit(root, rev, label) {
+    if (!rev || rev.startsWith("-"))
+        throw new ScanConfigError(`${label} ${JSON.stringify(rev)} is not a commit`);
+    try {
+        return git(root, ["rev-parse", "--verify", "-q", `${rev}^{commit}`]).trim();
+    }
+    catch {
+        throw new ScanConfigError(`${label} ${JSON.stringify(rev)} is not a commit in ${root}`);
+    }
+}
+/** `<base>..<head>` or `<base>` (head = HEAD). A three-dot range is rejected: the net diff is always taken from the merge base. */
+export function parseRange(range) {
+    if (range.includes("..."))
+        throw new ScanConfigError(`--range takes <base>..<head> or <base>, not ${JSON.stringify(range)}`);
+    const i = range.indexOf("..");
+    if (i < 0)
+        return { base: range, head: "HEAD" };
+    return { base: range.slice(0, i), head: range.slice(i + 2) || "HEAD" };
+}
+function warnAndDrop(load, warning, kind) {
+    try {
+        return load();
+    }
+    catch (err) {
+        if (!(err instanceof kind))
+            throw err;
+        process.stderr.write(`${warning}\n`);
+        return [];
+    }
+}
+function rangeRules(root, base) {
+    const builtIn = compiledRules();
+    const repo = warnAndDrop(() => loadCommittedAllowlists(root, base), ALLOWLIST_IGNORED_WARNING, InvalidAllowlistError);
+    const extra = warnAndDrop(() => loadCommittedRules(root, base), RULES_IGNORED_WARNING, InvalidRepoRulesError);
+    return { ...builtIn, rules: [...builtIn.rules, ...extra], repo };
+}
+/** merge-base, "" when the two have no common history; any other git failure is an error. */
+function mergeBase(root, base, head) {
+    const r = spawnSync("git", ["merge-base", base, head], { cwd: root, encoding: "utf8" });
+    if (r.status === 0)
+        return r.stdout.trim();
+    if (r.status === 1 && !r.stdout.trim() && !r.stderr.trim())
+        return "";
+    throw new ScanConfigError(`git merge-base failed: ${(r.stderr || String(r.error ?? "")).trim() || `exit ${r.status}`}`);
+}
+/**
+ * Lines a merge introduced, from `git diff-tree --cc` output: a line counts
+ * only when every parent column is "+", i.e. it is in none of the parents.
+ */
+export function parseCombinedDiff(diff) {
+    const fragments = [];
+    let file = null;
+    let parents = 0;
+    let nextLine = 0;
+    let cur = null;
+    const flush = () => {
+        if (file && cur && cur.lines.length)
+            fragments.push({ file, text: cur.lines.join("\n"), startLine: cur.startLine });
+        cur = null;
+    };
+    for (const raw of diff.split("\n")) {
+        if (raw.startsWith("diff --cc ") || raw.startsWith("diff --combined ")) {
+            flush();
+            file = raw.replace(/^diff --(?:cc|combined) /, "");
+            parents = 0;
+            continue;
+        }
+        if (parents === 0 && raw.startsWith("+++ ")) {
+            const p = raw.slice(4);
+            file = p === "/dev/null" ? null : p.replace(/^b\//, "");
+            continue;
+        }
+        const h = /^(@{3,}) .*?\+(\d+)(?:,\d+)? \1/.exec(raw);
+        if (h) {
+            flush();
+            parents = h[1].length - 1;
+            nextLine = Number(h[2]);
+            continue;
+        }
+        if (parents === 0)
+            continue;
+        const cols = raw.slice(0, parents);
+        if (cols.length < parents) {
+            flush();
+            continue;
+        }
+        if (cols.includes("-")) {
+            flush();
+            continue;
+        } // a line removed from some parent: not in the result
+        if (/^\++$/.test(cols)) {
+            if (!cur)
+                cur = { lines: [], startLine: nextLine };
+            cur.lines.push(raw.slice(parents));
+        }
+        else {
+            flush();
+        }
+        nextLine++;
+    }
+    flush();
+    return fragments;
+}
+/**
+ * Binary files a merge commit changed relative to every parent. The combined
+ * diff only says "Binary files differ" for them, so their content is read
+ * from the merge's tree instead. Deleted files are left out.
+ */
+function mergeBinaryFiles(root, merge) {
+    const files = git(root, ["diff-tree", "-c", "--name-only", "--no-commit-id", "-r", merge]).split("\n").filter(Boolean);
+    return files.filter((file) => {
+        try {
+            // git's own binary test: a NUL byte in the first 8000 bytes of the blob.
+            const head = execFileSync("git", ["cat-file", "blob", `${merge}:${file}`], {
+                cwd: root, maxBuffer: MAX_BINARY_BYTES + 1024, stdio: ["ignore", "pipe", "ignore"],
+            }).subarray(0, 8000);
+            return head.includes(0);
+        }
+        catch (err) {
+            // Too large to read here: still a binary candidate, so binaryText warns about it.
+            if (err.code === "ENOBUFS")
+                return true;
+            return false; // deleted in the merge, or a submodule
+        }
+    });
+}
+function patches(root, base, head) {
+    const out = [];
+    const mb = mergeBase(root, base, head);
+    // No common history: everything on head is new.
+    const net = mb ? git(root, ["diff", ...PATCH_ARGS, mb, head]) : git(root, ["show", "--format=", ...PATCH_ARGS, "--root", head]);
+    out.push({ rev: head, ...parseUnifiedDiff(net) });
+    const commits = git(root, ["rev-list", "--reverse", "--parents", `${base}..${head}`]).split("\n").filter(Boolean);
+    for (const line of commits) {
+        const [c, ...parents] = line.split(" ");
+        if (parents.length > 1) {
+            const cc = git(root, ["diff-tree", "--cc", "-p", "--no-commit-id", "--no-color", "--no-ext-diff", "--no-textconv", "--unified=0", "--src-prefix=a/", "--dst-prefix=b/", c]);
+            out.push({ commit: c, rev: c, fragments: parseCombinedDiff(cc), binaryFiles: mergeBinaryFiles(root, c) });
+        }
+        else {
+            out.push({ commit: c, rev: c, ...parseUnifiedDiff(git(root, ["diff-tree", "-p", "--root", "--no-commit-id", ...PATCH_ARGS, c])) });
+        }
+    }
+    return out;
+}
+/** A binary blob's bytes as text (latin1 keeps every byte), or null when it is too large to read. */
+function binaryText(root, rev, file) {
+    let size;
+    try {
+        size = Number(git(root, ["cat-file", "-s", `${rev}:${file}`]).trim());
+    }
+    catch {
+        return "";
+    }
+    if (!(size <= MAX_BINARY_BYTES))
+        return null;
+    const buf = execFileSync("git", ["cat-file", "blob", `${rev}:${file}`], { cwd: root, maxBuffer: MAX_BINARY_BYTES + 1024, stdio: ["ignore", "pipe", "pipe"] });
+    return buf.toString("latin1");
+}
+/**
+ * Findings for the lines added on <head> since <base>. A finding from a
+ * commit that the net diff no longer shows is named `<file>@<commit12>`, so
+ * the reader can find the version that held it.
+ */
+export function scanRange(cwd, range) {
+    const root = repoRoot(cwd);
+    const parsed = parseRange(range);
+    const base = commit(root, parsed.base, "range base");
+    const head = commit(root, parsed.head, "range head");
+    if (base === head) {
+        process.stderr.write(`${EMPTY_RANGE_WARNING}\n`);
+        return [];
+    }
+    if (!git(root, ["rev-list", "-n", "1", `${base}..${head}`]).trim()) {
+        throw new ScanConfigError(`range ${JSON.stringify(range)} has no commits: head is behind base (reversed range?)`);
+    }
+    const rules = rangeRules(root, base);
+    const out = [];
+    const seen = new Set();
+    const tooLarge = new Set();
+    for (const patch of patches(root, base, head)) {
+        const found = [...patch.fragments.flatMap((f) => scanFragment(f, rules))];
+        for (const file of patch.binaryFiles) {
+            found.push(...scanPath(file, rules));
+            const text = binaryText(root, patch.rev, file);
+            if (text === null)
+                tooLarge.add(file);
+            else if (text)
+                found.push(...scanFragment({ file, text, startLine: 1 }, rules));
+        }
+        for (const f of found) {
+            // The same rule on the same file at the same line is one report. A
+            // repeat never hides the only finding, so a dirty range always fails.
+            const key = `${f.ruleId}\u0000${f.file}\u0000${f.line}`;
+            if (seen.has(key))
+                continue;
+            seen.add(key);
+            out.push(patch.commit ? { ...f, file: `${f.file}@${patch.commit.slice(0, 12)}` } : f);
+        }
+    }
+    for (const file of tooLarge) {
+        process.stderr.write(`HQ secret scan: warning: binary ${file} is larger than ${MAX_BINARY_BYTES} bytes; its content was not scanned\n`);
+    }
+    return out;
+}
+//# sourceMappingURL=range.js.map

package/dist/lib/secret-scan/repo-config.d.ts

@@ -3,8 +3,10 @@
  * the scanned repo's own `.gitleaks.toml`, with gitleaks semantics for
  * `paths`, `regexes`, `regexTarget`, `stopwords`, `commits`, `targetRules` and
  * `condition`. They apply on top of the built-in rules and allowlists, to that
- * repo only. Repo `[[rules]]`, `[extend]` and the rest of the file are ignored:
- * a repo can allow known false positives, not change what is detected.
+ * repo only. `[extend]` and the rest of the file are ignored: a repo can allow
+ * known false positives, not switch built-in detection off. Repo `[[rules]]`
+ * are used by range scans only (`--range`, pr-check), and only add detection
+ * on top of the built-in rules (parseRepoRules below).
  *
  * The file is read from git, never the working tree:
  *   - The staged (index) copy applies only when it is identical to the copy
@@ -33,7 +35,7 @@
  * scan exits 2.
  */
 import { type Probe } from "./probes.js";
-import { type CompiledAllowlist } from "./rules.js";
+import { type CompiledAllowlist, type CompiledRule } from "./rules.js";
 export declare class ScanConfigError extends Error {
 }
 /** The repo's .gitleaks.toml cannot be used; the scan ignores it (see the top of the file). */
@@ -68,4 +70,17 @@ export declare function loadRepoAllowlists(repoRoot: string): CompiledAllowlist[
  * its own commits by adding an allowlist. An unknown rev is an error.
  */
 export declare function loadCommittedAllowlists(repoRoot: string, rev?: string): CompiledAllowlist[];
+/** The repo's [[rules]] cannot be used; a range scan warns and keeps the built-in rules only. */
+export declare class InvalidRepoRulesError extends ScanConfigError {
+}
+/**
+ * Extra detection rules from the `[[rules]]` of a .gitleaks.toml. They are
+ * added to the built-in rules and can only report more: a repo rule cannot
+ * replace or disable a built-in one (a built-in id is refused, `[extend]` and
+ * `useDefault` are ignored), and a rule's own allowlists apply to that rule
+ * only. [] when the file has no rules.
+ */
+export declare function parseRepoRules(text: string, file?: string): CompiledRule[];
+/** Repo rules from the copy of .gitleaks.toml committed at `rev`. [] when there is none. */
+export declare function loadCommittedRules(repoRoot: string, rev: string): CompiledRule[];
 //# sourceMappingURL=repo-config.d.ts.map

package/dist/lib/secret-scan/repo-config.js

@@ -3,8 +3,10 @@
  * the scanned repo's own `.gitleaks.toml`, with gitleaks semantics for
  * `paths`, `regexes`, `regexTarget`, `stopwords`, `commits`, `targetRules` and
  * `condition`. They apply on top of the built-in rules and allowlists, to that
- * repo only. Repo `[[rules]]`, `[extend]` and the rest of the file are ignored:
- * a repo can allow known false positives, not change what is detected.
+ * repo only. `[extend]` and the rest of the file are ignored: a repo can allow
+ * known false positives, not switch built-in detection off. Repo `[[rules]]`
+ * are used by range scans only (`--range`, pr-check), and only add detection
+ * on top of the built-in rules (parseRepoRules below).
  *
  * The file is read from git, never the working tree:
  *   - The staged (index) copy applies only when it is identical to the copy
@@ -361,6 +363,137 @@ export function loadCommittedAllowlists(repoRoot, rev = "HEAD") {
     }
     return revAllowlists(repoRoot, rev, parseExactAllowlists);
 }
+/** The repo's [[rules]] cannot be used; a range scan warns and keeps the built-in rules only. */
+export class InvalidRepoRulesError extends ScanConfigError {
+}
+const RULE_KEYS = new Set(["id", "description", "regex", "path", "entropy", "secretGroup", "keywords", "allowlists", "tags"]);
+const RULE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+/** Detection regexes are longer than allowlist ones; gitleaks' own rules run to a few hundred characters. */
+const MAX_RULE_REGEX_LENGTH = 1024;
+function failRules(file, where, msg) {
+    throw new InvalidRepoRulesError(`${file}: ${where}: ${msg}`);
+}
+function ruleRegex(file, where, key, src, flags = "") {
+    if (typeof src !== "string" || !src)
+        failRules(file, where, `${key} must be a non-empty string`);
+    if (src.length > MAX_RULE_REGEX_LENGTH)
+        failRules(file, where, `${key} is longer than ${MAX_RULE_REGEX_LENGTH} characters`);
+    if (NESTED_QUANTIFIER.test(src))
+        failRules(file, where, `${key} has a nested quantifier; simplify it`);
+    try {
+        return compileGoRegex(src, flags);
+    }
+    catch {
+        failRules(file, where, `${key} is not a supported regex`);
+    }
+}
+function ruleAllowlist(file, where, raw) {
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        failRules(file, where, "must be a table");
+    const t = raw;
+    for (const k of Object.keys(t))
+        if (!KEYS.has(k) || k === "targetRules")
+            failRules(file, where, `unknown key "${k}"`);
+    const list = (key) => {
+        const v = t[key];
+        if (v === undefined)
+            return undefined;
+        if (!Array.isArray(v) || v.some((x) => typeof x !== "string"))
+            failRules(file, where, `${key} must be a list of strings`);
+        return v;
+    };
+    const regexes = list("regexes");
+    const paths = list("paths");
+    regexes?.forEach((src, i) => ruleRegex(file, where, `regexes[${i}]`, src));
+    paths?.forEach((src, i) => ruleRegex(file, where, `paths[${i}]`, src));
+    const condition = typeof t.condition === "string" && t.condition.trim().toUpperCase() === "AND" ? "AND" : "OR";
+    const target = t.regexTarget;
+    if (target !== undefined && target !== "secret" && target !== "match" && target !== "line") {
+        failRules(file, where, 'regexTarget must be "secret", "match" or "line"');
+    }
+    return compileAllowlist({
+        condition,
+        regexTarget: target,
+        paths,
+        regexes,
+        stopwords: list("stopwords"),
+        commits: list("commits"),
+    });
+}
+/**
+ * Extra detection rules from the `[[rules]]` of a .gitleaks.toml. They are
+ * added to the built-in rules and can only report more: a repo rule cannot
+ * replace or disable a built-in one (a built-in id is refused, `[extend]` and
+ * `useDefault` are ignored), and a rule's own allowlists apply to that rule
+ * only. [] when the file has no rules.
+ */
+export function parseRepoRules(text, file = REPO_CONFIG_FILE) {
+    let doc;
+    try {
+        doc = parse(text);
+    }
+    catch (err) {
+        throw new InvalidRepoRulesError(`${file}: could not be parsed: ${err instanceof Error ? err.message.split("\n")[0] : String(err)}`);
+    }
+    if (doc.rules === undefined)
+        return [];
+    if (!Array.isArray(doc.rules))
+        failRules(file, "rules", "must be an array of tables ([[rules]])");
+    const builtIn = new Set(compiledRules().rules.map((r) => r.id));
+    const seen = new Set();
+    return doc.rules.map((raw, i) => {
+        const where = `[[rules]] #${i + 1}`;
+        if (!raw || typeof raw !== "object" || Array.isArray(raw))
+            failRules(file, where, "must be a table");
+        const t = raw;
+        for (const k of Object.keys(t))
+            if (!RULE_KEYS.has(k))
+                failRules(file, where, `unknown key "${k}"`);
+        if (typeof t.id !== "string" || !RULE_ID.test(t.id))
+            failRules(file, where, "id must be 1-64 letters, digits, dots, dashes or underscores");
+        if (builtIn.has(t.id))
+            failRules(file, where, `id "${t.id}" is a built-in rule; a repo rule can add detection, not replace it`);
+        if (seen.has(t.id))
+            failRules(file, where, `id "${t.id}" is used twice`);
+        seen.add(t.id);
+        if (t.regex === undefined && t.path === undefined)
+            failRules(file, where, "needs regex or path");
+        if (t.description !== undefined && typeof t.description !== "string")
+            failRules(file, where, "description must be a string");
+        if (t.entropy !== undefined && (typeof t.entropy !== "number" || !(t.entropy >= 0)))
+            failRules(file, where, "entropy must be a number >= 0");
+        if (t.secretGroup !== undefined && (!Number.isInteger(t.secretGroup) || t.secretGroup < 0)) {
+            failRules(file, where, "secretGroup must be an integer >= 0");
+        }
+        if (t.keywords !== undefined && (!Array.isArray(t.keywords) || t.keywords.some((k) => typeof k !== "string" || !k))) {
+            failRules(file, where, "keywords must be a list of non-empty strings");
+        }
+        if (t.allowlists !== undefined && !Array.isArray(t.allowlists))
+            failRules(file, where, "allowlists must be an array of tables");
+        return {
+            id: t.id,
+            description: typeof t.description === "string" && t.description ? t.description : `Repo rule ${t.id}`,
+            regex: t.regex === undefined ? undefined : ruleRegex(file, where, "regex", t.regex, "g"),
+            path: t.path === undefined ? undefined : ruleRegex(file, where, "path", t.path),
+            entropy: t.entropy,
+            secretGroup: t.secretGroup,
+            keywords: (t.keywords ?? []).map((k) => k.toLowerCase()),
+            allowlists: (t.allowlists ?? []).map((a, j) => ruleAllowlist(file, `${where} allowlists #${j + 1}`, a)),
+        };
+    });
+}
+/** Repo rules from the copy of .gitleaks.toml committed at `rev`. [] when there is none. */
+export function loadCommittedRules(repoRoot, rev) {
+    const e = entry(git(repoRoot, ["ls-tree", rev, "--", REPO_CONFIG_FILE]), 2);
+    if (!e)
+        return [];
+    if (e.mode === "120000")
+        throw new InvalidRepoRulesError(`${REPO_CONFIG_FILE} (${rev}) is a symlink; commit a regular file`);
+    const text = git(repoRoot, ["cat-file", "blob", e.blob]);
+    if (text === null)
+        throw new InvalidRepoRulesError(`${REPO_CONFIG_FILE} (${rev}) could not be read from git`);
+    return parseRepoRules(text, `${REPO_CONFIG_FILE} (${rev})`);
+}
 function headAllowlists(repoRoot, parser = parseRepoAllowlists) {
     if (git(repoRoot, ["rev-parse", "--verify", "-q", "HEAD^{commit}"]) === null)
         return [];

package/dist/lib/sync-core/engine-home.d.ts

@@ -0,0 +1,44 @@
+export type SyncCoreVaultKind = "personal" | "company";
+export type SyncCoreVaultPhase = "warming" | "full";
+export interface SyncCoreVault {
+    uid: string;
+    kind: SyncCoreVaultKind;
+    slug?: string;
+}
+export interface SyncCoreVaultPlan extends SyncCoreVault {
+    phase: SyncCoreVaultPhase;
+    engineHome: string;
+    binaryPath: string;
+    release: string;
+    legacyOffSince?: string;
+    blockedReason?: string;
+}
+export declare const SYNC_CORE_REQUESTS_PER_DAY = 20000;
+export declare function syncCoreEngineHome(home: string, vaultUid: string): string;
+export declare function syncCoreUnitId(vaultUid: string): string;
+export declare function isSyncCoreVaultUid(value: unknown): value is string;
+export declare function engineConfigToml(vault: SyncCoreVault, hqRoot: string, phase: SyncCoreVaultPhase): string;
+export interface WriteEngineHomeOptions {
+    home: string;
+    hqRoot: string;
+    vault: SyncCoreVault;
+    phase: SyncCoreVaultPhase;
+    tokenFile: string;
+}
+export declare function writeEngineHome(opts: WriteEngineHomeOptions): {
+    engineHome: string;
+    changed: boolean;
+};
+export interface SyncCorePlanFigures {
+    at: string;
+    uploads: number;
+    downloads: number;
+    deletes: number;
+    remoteRows: number;
+    localFiles: number;
+}
+export declare function engineReadOnlyPlan(engineHome: string, vaultUid: string): SyncCorePlanFigures | null;
+export declare function clearEngineStatus(home: string, vaultUid: string): void;
+export declare function removeEngineOwnership(home: string, vaultUid: string): void;
+export declare function inspectEngineHome(home: string, hqRoot: string | undefined, plan: SyncCoreVaultPlan, platform: NodeJS.Platform): SyncCoreVaultPlan;
+//# sourceMappingURL=engine-home.d.ts.map