@wxxb789/anc 0.0.1

package/scripts/scan-secrets.js

@@ -0,0 +1,268 @@
+/**
+ * Gitleaks over the exact bytes readers receive, including inflated gzip members.
+ *
+ * The external scanner owns the credential rule set and false-positive policy.
+ * This wrapper owns reproducibility, fail-closed process handling, redaction, and
+ * the public-count/private-path split used by every other publication gate.
+ */
+import { spawnSync } from 'node:child_process';
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync, } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { gunzipSync } from 'node:zlib';
+import { SNAPSHOT_DIRECTORY, SNAPSHOT_FILE_PATTERN } from "../src/lib/snapshot.js";
+import { BuildFailure } from "./write-report.js";
+import { databaseText, isGzip, isSqlite } from "./snapshot-rows.js";
+export const GITLEAKS_VERSION = '8.30.1';
+const DIST = fileURLToPath(new URL('../dist', import.meta.url));
+function processText(value) {
+    return value ?? '';
+}
+function scanProcessDetail(result, findings = []) {
+    return JSON.stringify({
+        status: result.status,
+        error: result.error?.message,
+        stdoutBytes: Buffer.byteLength(processText(result.stdout), 'utf8'),
+        stderrBytes: Buffer.byteLength(processText(result.stderr), 'utf8'),
+        findings,
+    });
+}
+function filesUnder(root) {
+    const files = [];
+    const pending = [root];
+    while (pending.length > 0) {
+        const directory = pending.pop();
+        for (const entry of readdirSync(directory, { withFileTypes: true })) {
+            const path = join(directory, entry.name);
+            if (entry.isDirectory())
+                pending.push(path);
+            else if (entry.isFile())
+                files.push(path);
+            else {
+                throw new BuildFailure('secret-scan-input-invalid', 'secret scan found a non-file output member', relative(root, path));
+            }
+        }
+    }
+    return files.sort();
+}
+function projection(source, target) {
+    const files = filesUnder(source);
+    if (files.length === 0) {
+        throw new BuildFailure('secret-scan-input-empty', 'secret scan refused an empty output', source);
+    }
+    for (const path of files) {
+        const relativePath = relative(source, path);
+        const bytes = readFileSync(path);
+        const gzip = isGzip(bytes);
+        const destination = join(target, relativePath);
+        mkdirSync(dirname(destination), { recursive: true });
+        // Raw-byte coverage is unchanged: the member is copied verbatim, so a value
+        // that survives only in a free page is still scanned as bytes.
+        copyFileSync(path, destination);
+        let payload = bytes;
+        if (gzip) {
+            try {
+                payload = gunzipSync(bytes);
+                writeFileSync(destination + '.inflated', payload);
+            }
+            catch (error) {
+                throw new BuildFailure('secret-scan-input-unreadable', 'secret scan could not inflate one output member', relativePath + ': ' + (error instanceof Error ? error.stack ?? error.message : String(error)));
+            }
+        }
+        // A database is recognised by its magic, on the inflated payload, so gzip and
+        // any filename are the same case. The scanner copies raw bytes, so a row a
+        // `SELECT` cannot reach (a deleted payload in a free page) is still scanned;
+        // the reconstructed rows are a separate member, so a value no byte read can
+        // spell — a BLOB stored as gzip, or one split across an overflow-page pointer
+        // — reaches Gitleaks as text.
+        if (isSqlite(payload)) {
+            let read;
+            try {
+                read = databaseText(payload);
+            }
+            catch (error) {
+                throw new BuildFailure('secret-scan-input-unreadable', 'secret scan could not reconstruct one database member', relativePath + ': ' + (error instanceof Error ? error.message : String(error)));
+            }
+            if (read.unreadable.length > 0) {
+                throw new BuildFailure('secret-scan-input-unreadable', 'secret scan could not read every table in one database member', relativePath + ': ' + read.unreadable.join(', '));
+            }
+            writeFileSync(destination + '.rows', read.values.length === 0 ? '' : `${read.values.join('\n')}\n`);
+        }
+    }
+    return files.length;
+}
+function cleanString(value) {
+    return typeof value === 'string' ? value : undefined;
+}
+function cleanNumber(value) {
+    return typeof value === 'number' && Number.isSafeInteger(value) ? value : undefined;
+}
+function projectedPath(file, projectionRoot) {
+    if (file === undefined)
+        return undefined;
+    const absolute = isAbsolute(file) ? resolve(file) : resolve(process.cwd(), file);
+    const path = relative(projectionRoot, absolute);
+    if (path === '..' || path.startsWith('..' + sep) || isAbsolute(path))
+        return '<outside projection>';
+    return path.split(sep).join('/');
+}
+/** A sidecar this scan adds to a projected member. */
+const PROJECTED_SIDECAR = /\.(inflated|rows)$/;
+/**
+ * The public half of one projected path.
+ *
+ * The snapshot's name is `site.<sha256>.sqlite` and the digest tracks the corpus,
+ * so it may not reach a stream. It is replaced by one stable literal, keeping the
+ * representation as a suffix because the operator's next step differs: raw bytes
+ * can be a deleted payload in a free page; rows are a live stored value.
+ */
+function publicFindingFile(projected) {
+    if (projected === undefined || projected === '<outside projection>')
+        return projected;
+    const segments = projected.split('/');
+    if (segments.length !== 2 || segments[0] !== SNAPSHOT_DIRECTORY)
+        return projected;
+    const name = segments[1];
+    if (!SNAPSHOT_FILE_PATTERN.test(name.replace(PROJECTED_SIDECAR, '')))
+        return projected;
+    const which = name.endsWith('.rows') ? ', reconstructed rows' : name.endsWith('.inflated') ? ', inflated' : '';
+    return `the site snapshot (${SNAPSHOT_DIRECTORY}/)${which}`;
+}
+function sanitize(findings, projectionRoot) {
+    if (!Array.isArray(findings)) {
+        throw new BuildFailure('secret-scanner-failed', 'secret scan failed to produce a valid report', 'Gitleaks JSON report is not an array');
+    }
+    return findings.map((value, index) => {
+        if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+            throw new BuildFailure('secret-scanner-failed', 'secret scan failed to produce a valid report', 'Gitleaks finding ' + index + ' is not an object');
+        }
+        const finding = value;
+        if (finding.Secret !== 'REDACTED' ||
+            typeof finding.Match !== 'string' ||
+            !finding.Match.includes('REDACTED')) {
+            throw new BuildFailure('secret-scanner-failed', 'secret scan report was not fully redacted', 'Gitleaks finding ' + index + ' did not redact Match and Secret');
+        }
+        const projected = projectedPath(cleanString(finding.File), projectionRoot);
+        return {
+            rule: cleanString(finding.RuleID),
+            description: cleanString(finding.Description),
+            file: publicFindingFile(projected),
+            startLine: cleanNumber(finding.StartLine),
+            endLine: cleanNumber(finding.EndLine),
+            startColumn: cleanNumber(finding.StartColumn),
+            endColumn: cleanNumber(finding.EndColumn),
+        };
+    });
+}
+function scannerEnvironment() {
+    const environment = { ...process.env };
+    delete environment['GITLEAKS_CONFIG'];
+    delete environment['GITLEAKS_CONFIG_TOML'];
+    return environment;
+}
+/** Run the pinned scanner without forwarding its path-bearing process streams. */
+export function scanSecrets(root, options = {}) {
+    const executable = options.executable ?? 'gitleaks';
+    const expectedVersion = options.expectedVersion ?? GITLEAKS_VERSION;
+    const environment = scannerEnvironment();
+    const version = spawnSync(executable, ['version'], {
+        encoding: 'utf8',
+        env: environment,
+        timeout: 30_000,
+        windowsHide: true,
+    });
+    if (version.status !== 0 ||
+        processText(version.stdout).trim() !== expectedVersion ||
+        processText(version.stderr).trim() !== '') {
+        throw new BuildFailure('secret-scanner-unavailable', 'secret scan requires Gitleaks ' + expectedVersion, scanProcessDetail(version));
+    }
+    const temporary = mkdtempSync(join(tmpdir(), 'anc-secret-scan-'));
+    const scanRoot = join(temporary, 'artifact');
+    const report = join(temporary, 'report.json');
+    const ignore = join(temporary, 'empty.gitleaksignore');
+    const config = join(temporary, 'gitleaks.toml');
+    const run = () => {
+        mkdirSync(scanRoot);
+        writeFileSync(ignore, '', 'utf8');
+        writeFileSync(config, '[extend]\nuseDefault = true\n', 'utf8');
+        const scanned = projection(root, scanRoot);
+        const result = spawnSync(executable, [
+            'dir',
+            '--no-banner',
+            '--no-color',
+            '--redact=100',
+            '--log-level', 'error',
+            '--ignore-gitleaks-allow',
+            '--gitleaks-ignore-path', ignore,
+            '--config', config,
+            // Gzip is projected explicitly above. One encoding layer still catches
+            // wrapped credentials; depth five spent ~80 seconds on Mermaid bundles.
+            '--max-archive-depth', '0',
+            '--max-decode-depth', '1',
+            // Zero disables size-based skipping: a large output member must be read.
+            '--max-target-megabytes', '0',
+            '--timeout', '120',
+            '--exit-code', '7',
+            '--report-format', 'json',
+            '--report-path', report,
+            scanRoot,
+        ], { encoding: 'utf8', env: environment, timeout: 130_000, windowsHide: true });
+        let rawReport;
+        try {
+            rawReport = JSON.parse(readFileSync(report, 'utf8'));
+        }
+        catch (error) {
+            throw new BuildFailure('secret-scanner-failed', 'secret scan failed to produce a readable report', scanProcessDetail(result) + '\n' + String(error));
+        }
+        const findings = sanitize(rawReport, scanRoot);
+        // At error log level, any stderr means Gitleaks could not read something.
+        if (processText(result.stderr).trim() !== '' || (result.status !== 0 && result.status !== 7)) {
+            throw new BuildFailure('secret-scanner-failed', 'secret scan did not inspect the complete output', scanProcessDetail(result, findings));
+        }
+        const hasFindings = findings.length > 0;
+        if ((result.status === 7) !== hasFindings) {
+            throw new BuildFailure('secret-scanner-failed', 'secret scan status disagreed with its redacted report', scanProcessDetail(result, findings));
+        }
+        if (hasFindings) {
+            throw new BuildFailure('secret-scan-findings', 'secret scan found ' + findings.length + ' finding' + (findings.length === 1 ? '' : 's'), JSON.stringify({ findings }));
+        }
+        return scanned;
+    };
+    let outcome = 0;
+    let failure;
+    try {
+        outcome = run();
+    }
+    catch (error) {
+        failure = error;
+    }
+    try {
+        rmSync(temporary, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 });
+    }
+    catch (cleanupError) {
+        const detail = cleanupError instanceof Error ? cleanupError.stack ?? cleanupError.message : String(cleanupError);
+        failure = failure instanceof BuildFailure
+            ? new BuildFailure(failure.code, failure.message, failure.detail + '\n' + detail)
+            : new BuildFailure('secret-scan-cleanup-failed', 'secret scan could not remove its temporary projection', detail);
+    }
+    if (failure instanceof BuildFailure)
+        throw failure;
+    if (failure !== undefined) {
+        throw new BuildFailure('secret-scanner-failed', 'secret scan failed before inspecting the complete output', failure instanceof Error ? failure.stack ?? failure.message : String(failure));
+    }
+    return outcome;
+}
+/** Own-repository command; the packaged binary imports scanSecrets directly. */
+export function runSecretScanCommand() {
+    try {
+        console.log('secret scan ok: ' + scanSecrets(DIST) + ' files, 0 findings');
+        return 0;
+    }
+    catch (error) {
+        console.error(error instanceof BuildFailure ? error.message : 'secret scan failed');
+        return 1;
+    }
+}
+if (process.argv[1] === fileURLToPath(import.meta.url))
+    process.exit(runSecretScanCommand());