@indigoai-us/hq-cli 5.345.164 → 5.345.166
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/assets/scaffold/core/scripts/rebuild-all-indexes.sh +11 -0
- package/assets/scaffold/core/scripts/rebuild-orchestrator-index.sh +11 -0
- package/assets/scaffold/core/scripts/rebuild-public-knowledge-index.sh +11 -0
- package/assets/scaffold/core/scripts/rebuild-reports-index.sh +11 -0
- package/assets/scaffold/core/scripts/rebuild-social-drafts-index.sh +11 -0
- package/assets/scaffold/core/scripts/rebuild-threads-index.sh +11 -0
- package/assets/scaffold/core/scripts/rebuild-workers-index.sh +11 -0
- package/dist/command-catalog.generated.d.ts +66 -0
- package/dist/command-catalog.generated.js +82 -0
- package/dist/commands/billing-sponsor.d.ts +20 -0
- package/dist/commands/billing-sponsor.js +116 -0
- package/dist/commands/integrations-api.d.ts +15 -0
- package/dist/commands/integrations-api.js +27 -0
- package/dist/commands/integrations-manage.js +4 -2
- package/dist/commands/integrations.js +9 -5
- package/dist/commands/scan.d.ts +6 -0
- package/dist/commands/scan.js +11 -1
- package/dist/hq-cloud-sync-runner/bin/sync-runner.js +210 -197
- package/dist/hq-cloud-sync-runner/package.json +1 -1
- package/dist/lib/doctor/checks/sync-runtime.d.ts +7 -2
- package/dist/lib/doctor/checks/sync-runtime.js +1 -3
- package/dist/lib/flag-registry-worker-source.generated.d.ts +1 -1
- package/dist/lib/flag-registry-worker-source.generated.js +1 -1
- package/dist/lib/index-render/index.d.ts +5 -0
- package/dist/lib/index-render/index.js +22 -1
- package/dist/lib/secret-scan/detect.js +2 -0
- package/dist/lib/secret-scan/probes.d.ts +13 -0
- package/dist/lib/secret-scan/probes.js +69 -0
- package/dist/lib/secret-scan/repo-config.d.ts +24 -0
- package/dist/lib/secret-scan/repo-config.js +159 -29
- package/dist/lib/secret-scan/rules.d.ts +2 -0
- package/dist/lib/secret-scan/staged.d.ts +15 -1
- package/dist/lib/secret-scan/staged.js +46 -4
- package/package.json +8 -8
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { RenderContext, RenderResult } from "./shared.js";
|
|
2
2
|
export type IndexTarget = "all" | "companies" | "company-knowledge" | "orchestrator" | "projects" | "public-knowledge" | "reports" | "social-drafts" | "threads" | "workers";
|
|
3
|
+
/**
|
|
4
|
+
* One-line summaries for targets without their own option grammar. The CLI,
|
|
5
|
+
* the renderer and the bundled shell fallbacks all print the same help text.
|
|
6
|
+
*/
|
|
7
|
+
export declare const GENERIC_INDEX_HELP_SUMMARIES: Readonly<Record<Exclude<IndexTarget, "companies" | "company-knowledge" | "projects">, string>>;
|
|
3
8
|
export declare function renderIndexTarget(target: IndexTarget, context: RenderContext, args?: string[]): RenderResult;
|
|
4
9
|
export { atomicWrite, comparePath, immediateEntries, type RenderContext, type RenderResult } from "./shared.js";
|
|
5
10
|
export { CompanyIndexUserError } from "./companies.js";
|
|
@@ -43,15 +43,36 @@ function companyScope(target, context, args) {
|
|
|
43
43
|
}
|
|
44
44
|
return company;
|
|
45
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* One-line summaries for targets without their own option grammar. The CLI,
|
|
48
|
+
* the renderer and the bundled shell fallbacks all print the same help text.
|
|
49
|
+
*/
|
|
50
|
+
export const GENERIC_INDEX_HELP_SUMMARIES = {
|
|
51
|
+
all: "Regenerate every INDEX.md across the HQ tree",
|
|
52
|
+
orchestrator: "Regenerate the orchestrator workspace INDEX.md",
|
|
53
|
+
"public-knowledge": "Regenerate core/knowledge/public/INDEX.md",
|
|
54
|
+
reports: "Regenerate the reports INDEX.md",
|
|
55
|
+
"social-drafts": "Regenerate workspace/social-drafts/INDEX.md",
|
|
56
|
+
threads: "Regenerate the session-threads INDEX.md",
|
|
57
|
+
workers: "Regenerate the workers INDEX.md",
|
|
58
|
+
};
|
|
59
|
+
function isHelp(args) {
|
|
60
|
+
return args.some((arg) => arg === "--help" || arg === "-h");
|
|
61
|
+
}
|
|
46
62
|
export function renderIndexTarget(target, context, args = []) {
|
|
47
63
|
if (target === "company-knowledge" || target === "projects") {
|
|
48
|
-
if (args
|
|
64
|
+
if (isHelp(args)) {
|
|
49
65
|
(context.output ?? ((message) => process.stdout.write(message)))(`Usage: hq core rebuild-index ${target} [<company> | --company <company>] [--accept]\n\nWith a company, rebuild only that company's index. Without a company, retain the all-company rebuild. Help performs no writes.\n`);
|
|
50
66
|
return { written: [] };
|
|
51
67
|
}
|
|
52
68
|
const company = companyScope(target, context, args);
|
|
53
69
|
return target === "projects" ? renderProjects(context, company) : renderCompanyKnowledge(context, company);
|
|
54
70
|
}
|
|
71
|
+
if (target !== "companies" && isHelp(args)) {
|
|
72
|
+
// Help must never write, matching the CLI and the bundled shell fallback.
|
|
73
|
+
(context.output ?? ((message) => process.stdout.write(message)))(`Usage: hq core rebuild-index ${target} [--accept]\n\n${GENERIC_INDEX_HELP_SUMMARIES[target]}. Help performs no writes.\n`);
|
|
74
|
+
return { written: [] };
|
|
75
|
+
}
|
|
55
76
|
switch (target) {
|
|
56
77
|
case "companies": return renderCompanies(context, args);
|
|
57
78
|
case "orchestrator": return renderOrchestrator(context);
|
|
@@ -38,6 +38,8 @@ function allowed(al, c) {
|
|
|
38
38
|
}
|
|
39
39
|
if (al.commits.length)
|
|
40
40
|
checks.push(c.commit !== undefined && al.commits.includes(c.commit.toLowerCase()));
|
|
41
|
+
if (al.exact?.length)
|
|
42
|
+
checks.push(al.exact.includes(c.secret));
|
|
41
43
|
if (!checks.length)
|
|
42
44
|
return false;
|
|
43
45
|
return al.condition === "AND" ? checks.every(Boolean) : checks.some(Boolean);
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A fixed corpus of realistic credential lines, used to judge a repo
|
|
3
|
+
* allowlist by what it does: each line is placed in a file at the allowlist's
|
|
4
|
+
* paths and run through the real detector. Values are generated from a fixed
|
|
5
|
+
* seed at runtime, so no secret-shaped literal sits in this file and the
|
|
6
|
+
* corpus is the same on every run.
|
|
7
|
+
*/
|
|
8
|
+
export interface Probe {
|
|
9
|
+
kind: string;
|
|
10
|
+
line: string;
|
|
11
|
+
}
|
|
12
|
+
export declare function credentialProbes(): Probe[];
|
|
13
|
+
//# sourceMappingURL=probes.d.ts.map
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A fixed corpus of realistic credential lines, used to judge a repo
|
|
3
|
+
* allowlist by what it does: each line is placed in a file at the allowlist's
|
|
4
|
+
* paths and run through the real detector. Values are generated from a fixed
|
|
5
|
+
* seed at runtime, so no secret-shaped literal sits in this file and the
|
|
6
|
+
* corpus is the same on every run.
|
|
7
|
+
*/
|
|
8
|
+
const UPPER = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
|
9
|
+
const LOWER = "abcdefghijklmnopqrstuvwxyz";
|
|
10
|
+
const DIGITS = "0123456789";
|
|
11
|
+
const ALNUM = UPPER + LOWER + DIGITS;
|
|
12
|
+
const HEX = "0123456789abcdef";
|
|
13
|
+
const B64URL = ALNUM + "-_";
|
|
14
|
+
/** Deterministic xorshift32 generator. */
|
|
15
|
+
function generator(seed) {
|
|
16
|
+
let s = seed >>> 0 || 1;
|
|
17
|
+
return (alphabet, n) => {
|
|
18
|
+
let out = "";
|
|
19
|
+
for (let i = 0; i < n; i++) {
|
|
20
|
+
s ^= s << 13;
|
|
21
|
+
s >>>= 0;
|
|
22
|
+
s ^= s >>> 17;
|
|
23
|
+
s ^= s << 5;
|
|
24
|
+
s >>>= 0;
|
|
25
|
+
out += alphabet[s % alphabet.length];
|
|
26
|
+
}
|
|
27
|
+
return out;
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
const b64url = (s) => Buffer.from(s).toString("base64url");
|
|
31
|
+
const j = (...parts) => parts.join("");
|
|
32
|
+
let cached;
|
|
33
|
+
export function credentialProbes() {
|
|
34
|
+
if (cached)
|
|
35
|
+
return cached;
|
|
36
|
+
const r = generator(0x9e3779b9);
|
|
37
|
+
const probes = [];
|
|
38
|
+
const add = (kind, line) => probes.push({ kind, line });
|
|
39
|
+
// GitHub tokens: classic PAT, OAuth, app and fine-grained PAT.
|
|
40
|
+
add("GitHub PAT", `token = "${j("gh", "p_", r(ALNUM, 36))}"`);
|
|
41
|
+
add("GitHub PAT", `export GH_TOKEN=${j("gh", "o_", r(ALNUM, 36))}`);
|
|
42
|
+
add("GitHub PAT", ` "github": "${j("gh", "s_", r(ALNUM, 36))}",`);
|
|
43
|
+
add("GitHub PAT", j("gith", "ub_pat_", r(ALNUM, 22), "_", r(ALNUM, 59)));
|
|
44
|
+
// AWS access key ids.
|
|
45
|
+
add("AWS key", `aws_access_key_id = ${j("AK", "IA", r(UPPER + "234567", 16))}`);
|
|
46
|
+
add("AWS key", `AWS_ACCESS_KEY_ID="${j("AS", "IA", r(UPPER + "234567", 16))}"`);
|
|
47
|
+
add("AWS key", j("AK", "IA", r(UPPER + "234567", 16)));
|
|
48
|
+
// Slack bot and user tokens.
|
|
49
|
+
add("Slack token", `SLACK_BOT_TOKEN=${j("xo", "xb-", r(DIGITS, 12), "-", r(DIGITS, 13), "-", r(ALNUM, 24))}`);
|
|
50
|
+
add("Slack token", `slack: "${j("xo", "xb-", r(DIGITS, 11), "-", r(DIGITS, 12), "-", r(ALNUM, 24))}"`);
|
|
51
|
+
add("Slack token", `token = "${j("xo", "xp-", r(DIGITS, 12), "-", r(DIGITS, 12), "-", r(DIGITS, 13), "-", r(HEX, 32))}"`);
|
|
52
|
+
// 64-character hex secrets under credential-looking keys.
|
|
53
|
+
add("hex-64", `api_secret = "${r(HEX, 64)}"`);
|
|
54
|
+
add("hex-64", ` "auth_token": "${r(HEX, 64)}",`);
|
|
55
|
+
add("hex-64", `SECRET_KEY=${r(HEX, 64)}`);
|
|
56
|
+
// JWTs of different lengths.
|
|
57
|
+
for (const n of [24, 48, 96]) {
|
|
58
|
+
const header = b64url('{"alg":"HS256","typ":"JWT"}');
|
|
59
|
+
const payload = b64url(JSON.stringify({ sub: r(DIGITS, 10), name: r(LOWER, n / 4), iat: 1700000000 }));
|
|
60
|
+
add("JWT", `const jwt = "${header}.${payload}.${r(B64URL, 43)}";`);
|
|
61
|
+
}
|
|
62
|
+
// Generic key = value credentials of different lengths.
|
|
63
|
+
add("generic api_key", `api_key = "${r(ALNUM, 20)}"`);
|
|
64
|
+
add("generic api_key", `client_secret: '${r(ALNUM, 32)}'`);
|
|
65
|
+
add("generic api_key", `ACCESS_TOKEN=${r(ALNUM, 48)}`);
|
|
66
|
+
cached = probes;
|
|
67
|
+
return probes;
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=probes.js.map
|
|
@@ -18,13 +18,28 @@
|
|
|
18
18
|
* literal prefix (`^golden/...`); `regexes` and `stopwords` need
|
|
19
19
|
* `condition = "AND"`, so they never reach files outside those paths.
|
|
20
20
|
*
|
|
21
|
+
* File scans (`hq scan secrets <files> --repo`, used by pr-check) honor only
|
|
22
|
+
* exact-value exemptions: each allowlist names its paths and lists, in
|
|
23
|
+
* `stopwords`, the exact secret values it allows (e.g. known digests). A
|
|
24
|
+
* detected secret is allowed only when it equals one of them. Regexes,
|
|
25
|
+
* targetRules, commits, regexTarget and path-only allowlists are rejected
|
|
26
|
+
* there, because a pattern can always be widened, alone or combined with other
|
|
27
|
+
* entries, to hide a real credential.
|
|
28
|
+
*
|
|
21
29
|
* Fails closed: anything not acceptable is a ScanConfigError, and the scan
|
|
22
30
|
* exits 2 rather than running with a guess.
|
|
23
31
|
*/
|
|
32
|
+
import { type Probe } from "./probes.js";
|
|
24
33
|
import { type CompiledAllowlist } from "./rules.js";
|
|
25
34
|
export declare class ScanConfigError extends Error {
|
|
26
35
|
}
|
|
27
36
|
export declare const REPO_CONFIG_FILE = ".gitleaks.toml";
|
|
37
|
+
/**
|
|
38
|
+
* The first probe credential the allowlist would hide, judged as the detector
|
|
39
|
+
* judges it (every regex ORed, every check combined by its condition, its
|
|
40
|
+
* targetRules applied), for a file at one of its paths. undefined: none.
|
|
41
|
+
*/
|
|
42
|
+
export declare function suppressedProbe(al: CompiledAllowlist): Probe | undefined;
|
|
28
43
|
/**
|
|
29
44
|
* Length of the literal text right after a leading `^`: letters, digits,
|
|
30
45
|
* `/ _ - @ ~` and backslash-escaped punctuation. A quantifier on the last
|
|
@@ -33,9 +48,18 @@ export declare const REPO_CONFIG_FILE = ".gitleaks.toml";
|
|
|
33
48
|
export declare function literalPathPrefix(src: string): number;
|
|
34
49
|
/** Validated allowlists from the text of a .gitleaks.toml. */
|
|
35
50
|
export declare function parseRepoAllowlists(text: string, file?: string): CompiledAllowlist[];
|
|
51
|
+
/** Validated exact-value allowlists for a file scan, from the text of a .gitleaks.toml. */
|
|
52
|
+
export declare function parseExactAllowlists(text: string, file?: string): CompiledAllowlist[];
|
|
36
53
|
/**
|
|
37
54
|
* Compiled allowlists for a staged scan of the repo at `repoRoot`, read from
|
|
38
55
|
* git as described at the top of this file. [] when the repo has none.
|
|
39
56
|
*/
|
|
40
57
|
export declare function loadRepoAllowlists(repoRoot: string): CompiledAllowlist[];
|
|
58
|
+
/**
|
|
59
|
+
* Exact-value allowlists from the copy committed at `rev` (default HEAD), for a
|
|
60
|
+
* file scan (`hq scan secrets --repo [--allowlist-rev]`). The index and working
|
|
61
|
+
* tree are ignored. pr-check passes the base branch, so a branch cannot exempt
|
|
62
|
+
* its own commits by adding an allowlist. An unknown rev is an error.
|
|
63
|
+
*/
|
|
64
|
+
export declare function loadCommittedAllowlists(repoRoot: string, rev?: string): CompiledAllowlist[];
|
|
41
65
|
//# sourceMappingURL=repo-config.d.ts.map
|
|
@@ -18,13 +18,24 @@
|
|
|
18
18
|
* literal prefix (`^golden/...`); `regexes` and `stopwords` need
|
|
19
19
|
* `condition = "AND"`, so they never reach files outside those paths.
|
|
20
20
|
*
|
|
21
|
+
* File scans (`hq scan secrets <files> --repo`, used by pr-check) honor only
|
|
22
|
+
* exact-value exemptions: each allowlist names its paths and lists, in
|
|
23
|
+
* `stopwords`, the exact secret values it allows (e.g. known digests). A
|
|
24
|
+
* detected secret is allowed only when it equals one of them. Regexes,
|
|
25
|
+
* targetRules, commits, regexTarget and path-only allowlists are rejected
|
|
26
|
+
* there, because a pattern can always be widened, alone or combined with other
|
|
27
|
+
* entries, to hide a real credential.
|
|
28
|
+
*
|
|
21
29
|
* Fails closed: anything not acceptable is a ScanConfigError, and the scan
|
|
22
30
|
* exits 2 rather than running with a guess.
|
|
23
31
|
*/
|
|
24
32
|
import { execFileSync } from "node:child_process";
|
|
33
|
+
import { createHash } from "node:crypto";
|
|
25
34
|
import { parse } from "smol-toml";
|
|
26
35
|
import { compileGoRegex } from "./go-regex.js";
|
|
27
|
-
import {
|
|
36
|
+
import { scanFragment } from "./detect.js";
|
|
37
|
+
import { credentialProbes } from "./probes.js";
|
|
38
|
+
import { compileAllowlist, compiledRules } from "./rules.js";
|
|
28
39
|
export class ScanConfigError extends Error {
|
|
29
40
|
}
|
|
30
41
|
export const REPO_CONFIG_FILE = ".gitleaks.toml";
|
|
@@ -37,6 +48,25 @@ const MAX_REGEX_LENGTH = 256;
|
|
|
37
48
|
const NESTED_QUANTIFIER = /\((?:[^()\\]|\\.)*[*+}](?:[^()\\]|\\.)*\)[*+{]/;
|
|
38
49
|
/** Paths no real allowlist path should match all of. */
|
|
39
50
|
const PROBE_PATHS = ["a", ".env", "src/index.ts", "config/credentials.yml", "deep/nested/dir/file.json"];
|
|
51
|
+
/** File the probes are placed in; the allowlist's own paths are treated as matching it. */
|
|
52
|
+
const PROBE_FILE = "probe/settings.txt";
|
|
53
|
+
let probeBaseline;
|
|
54
|
+
function probeFragment(line) {
|
|
55
|
+
return { file: PROBE_FILE, text: `${line}\n`, startLine: 1 };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The first probe credential the allowlist would hide, judged as the detector
|
|
59
|
+
* judges it (every regex ORed, every check combined by its condition, its
|
|
60
|
+
* targetRules applied), for a file at one of its paths. undefined: none.
|
|
61
|
+
*/
|
|
62
|
+
export function suppressedProbe(al) {
|
|
63
|
+
const base = compiledRules();
|
|
64
|
+
const probes = credentialProbes();
|
|
65
|
+
probeBaseline ??= probes.map((p) => scanFragment(probeFragment(p.line), base).length > 0);
|
|
66
|
+
const atPath = { ...al, paths: al.paths.length ? [/(?:)/] : [] };
|
|
67
|
+
const rules = { ...base, repo: [atPath] };
|
|
68
|
+
return probes.find((p, i) => probeBaseline[i] && scanFragment(probeFragment(p.line), rules).length === 0);
|
|
69
|
+
}
|
|
40
70
|
function fail(file, where, msg) {
|
|
41
71
|
throw new ScanConfigError(`${file}: ${where}: ${msg}`);
|
|
42
72
|
}
|
|
@@ -96,16 +126,38 @@ function topLevelAlternation(src) {
|
|
|
96
126
|
}
|
|
97
127
|
return false;
|
|
98
128
|
}
|
|
99
|
-
|
|
129
|
+
/**
|
|
130
|
+
* How an error names an exemption value: its length and a short hash, never
|
|
131
|
+
* the value itself, which may be a real credential (e.g. one with a typo).
|
|
132
|
+
*/
|
|
133
|
+
function redact(value) {
|
|
134
|
+
return `<${value.length} chars, sha256:${createHash("sha256").update(value).digest("hex").slice(0, 12)}>`;
|
|
135
|
+
}
|
|
136
|
+
/** Names a regexes entry by index and redacted value; paths are shown as written. */
|
|
137
|
+
function entryName(key, src, i) {
|
|
138
|
+
return key === "paths" ? `paths entry ${JSON.stringify(src)}` : `${key}[${i}] ${redact(src)}`;
|
|
139
|
+
}
|
|
140
|
+
function compileChecked(file, where, key, src, i = 0) {
|
|
100
141
|
if (src.length > MAX_REGEX_LENGTH)
|
|
101
142
|
fail(file, where, `${key} entry is longer than ${MAX_REGEX_LENGTH} characters`);
|
|
102
143
|
if (NESTED_QUANTIFIER.test(src))
|
|
103
|
-
fail(file, where, `${key
|
|
144
|
+
fail(file, where, `${entryName(key, src, i)} has a nested quantifier; simplify it`);
|
|
104
145
|
try {
|
|
105
146
|
return compileGoRegex(src);
|
|
106
147
|
}
|
|
107
148
|
catch (err) {
|
|
108
|
-
fail(file, where, `${key
|
|
149
|
+
fail(file, where, `${entryName(key, src, i)} is not a supported regex${key === "paths" ? ` (${err instanceof Error ? err.message : String(err)})` : ""}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
function checkPaths(file, where, paths) {
|
|
153
|
+
for (const src of paths) {
|
|
154
|
+
const re = compileChecked(file, where, "paths", src);
|
|
155
|
+
if (literalPathPrefix(src) < MIN_PATH_PREFIX || topLevelAlternation(src)) {
|
|
156
|
+
fail(file, where, `paths entry ${JSON.stringify(src)} must start with ^ and at least ${MIN_PATH_PREFIX} literal characters (e.g. '^fixtures/')`);
|
|
157
|
+
}
|
|
158
|
+
if (re.test("") || PROBE_PATHS.every((p) => re.test(p))) {
|
|
159
|
+
fail(file, where, `paths entry ${JSON.stringify(src)} matches everything; allowlist specific files`);
|
|
160
|
+
}
|
|
109
161
|
}
|
|
110
162
|
}
|
|
111
163
|
function validate(file, where, raw) {
|
|
@@ -141,37 +193,29 @@ function validate(file, where, raw) {
|
|
|
141
193
|
if (!al.paths?.length) {
|
|
142
194
|
fail(file, where, "needs paths, each starting with ^ and a literal directory or file name (e.g. '^fixtures/')");
|
|
143
195
|
}
|
|
144
|
-
|
|
145
|
-
const re = compileChecked(file, where, "paths", src);
|
|
146
|
-
if (literalPathPrefix(src) < MIN_PATH_PREFIX || topLevelAlternation(src)) {
|
|
147
|
-
fail(file, where, `paths entry ${JSON.stringify(src)} must start with ^ and at least ${MIN_PATH_PREFIX} literal characters (e.g. '^fixtures/')`);
|
|
148
|
-
}
|
|
149
|
-
if (re.test("") || PROBE_PATHS.every((p) => re.test(p))) {
|
|
150
|
-
fail(file, where, `paths entry ${JSON.stringify(src)} matches everything; allowlist specific files`);
|
|
151
|
-
}
|
|
152
|
-
}
|
|
196
|
+
checkPaths(file, where, al.paths);
|
|
153
197
|
if ((al.regexes?.length || al.stopwords?.length) && al.condition !== "AND") {
|
|
154
198
|
fail(file, where, 'regexes and stopwords need condition = "AND", so they only apply inside paths');
|
|
155
199
|
}
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
fail(file, where, `regexes entry ${JSON.stringify(src)} matches everything`);
|
|
159
|
-
}
|
|
160
|
-
for (const s of al.stopwords ?? []) {
|
|
200
|
+
al.regexes?.forEach((src, i) => compileChecked(file, where, "regexes", src, i));
|
|
201
|
+
al.stopwords?.forEach((s, i) => {
|
|
161
202
|
if (s.trim().length < MIN_STOPWORD)
|
|
162
|
-
fail(file, where, `
|
|
203
|
+
fail(file, where, `stopwords[${i}] ${redact(s)} is shorter than ${MIN_STOPWORD} characters`);
|
|
204
|
+
});
|
|
205
|
+
// A path-only allowlist exempts the files it names. One with content checks
|
|
206
|
+
// must still report a real credential in those files, so run the detector on
|
|
207
|
+
// the probe corpus with this allowlist, exactly as a scan would apply it.
|
|
208
|
+
if (al.regexes?.length || al.stopwords?.length) {
|
|
209
|
+
const hidden = suppressedProbe(compileAllowlist(al));
|
|
210
|
+
if (hidden) {
|
|
211
|
+
fail(file, where, `matches everything: it would hide a real ${hidden.kind} credential in its paths; narrow its regexes or stopwords to the known false positive`);
|
|
212
|
+
}
|
|
163
213
|
}
|
|
164
214
|
return al;
|
|
165
215
|
}
|
|
166
216
|
/** Validated allowlists from the text of a .gitleaks.toml. */
|
|
167
217
|
export function parseRepoAllowlists(text, file = REPO_CONFIG_FILE) {
|
|
168
|
-
|
|
169
|
-
try {
|
|
170
|
-
doc = parse(text);
|
|
171
|
-
}
|
|
172
|
-
catch (err) {
|
|
173
|
-
throw new ScanConfigError(`${file}: could not be parsed: ${err instanceof Error ? err.message.split("\n")[0] : String(err)}`);
|
|
174
|
-
}
|
|
218
|
+
const doc = parseToml(text, file);
|
|
175
219
|
const out = [];
|
|
176
220
|
if (doc.allowlist !== undefined)
|
|
177
221
|
out.push(validate(file, "[allowlist]", doc.allowlist));
|
|
@@ -182,6 +226,75 @@ export function parseRepoAllowlists(text, file = REPO_CONFIG_FILE) {
|
|
|
182
226
|
}
|
|
183
227
|
return out.map(compileAllowlist);
|
|
184
228
|
}
|
|
229
|
+
/** Shortest exact value a file-scan allowlist may list; real secrets and digests are longer. */
|
|
230
|
+
const MIN_EXACT_VALUE = 16;
|
|
231
|
+
const EXACT_KEYS = new Set(["description", "condition", "paths", "stopwords"]);
|
|
232
|
+
/**
|
|
233
|
+
* A file-scan allowlist: paths plus the exact secret values to allow, listed
|
|
234
|
+
* in `stopwords`. Anything pattern-based is rejected (see the top of the file).
|
|
235
|
+
*/
|
|
236
|
+
function validateExact(file, where, raw) {
|
|
237
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
238
|
+
fail(file, where, "must be a table");
|
|
239
|
+
const t = raw;
|
|
240
|
+
for (const k of Object.keys(t)) {
|
|
241
|
+
if (!KEYS.has(k))
|
|
242
|
+
fail(file, where, `unknown key "${k}"`);
|
|
243
|
+
if (!EXACT_KEYS.has(k)) {
|
|
244
|
+
fail(file, where, `file scans honor only exact-value exemptions: remove "${k}" and list the exact secret values in stopwords`);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
if (t.description !== undefined && typeof t.description !== "string")
|
|
248
|
+
fail(file, where, "description must be a string");
|
|
249
|
+
if (typeof t.condition !== "string" || t.condition.trim().toUpperCase() !== "AND") {
|
|
250
|
+
fail(file, where, 'file scans need condition = "AND", so the exact values only apply inside paths');
|
|
251
|
+
}
|
|
252
|
+
const paths = stringList(file, where, "paths", t.paths);
|
|
253
|
+
if (!paths?.length) {
|
|
254
|
+
fail(file, where, "needs paths, each starting with ^ and a literal directory or file name (e.g. '^fixtures/')");
|
|
255
|
+
}
|
|
256
|
+
checkPaths(file, where, paths);
|
|
257
|
+
const values = stringList(file, where, "stopwords", t.stopwords);
|
|
258
|
+
if (!values?.length) {
|
|
259
|
+
fail(file, where, "file scans honor only exact-value exemptions: list the exact secret values to allow in stopwords");
|
|
260
|
+
}
|
|
261
|
+
values.forEach((v, i) => {
|
|
262
|
+
if (v.length < MIN_EXACT_VALUE || /\s/.test(v)) {
|
|
263
|
+
fail(file, where, `stopwords[${i}] ${redact(v)} must be an exact secret value of at least ${MIN_EXACT_VALUE} characters with no spaces`);
|
|
264
|
+
}
|
|
265
|
+
});
|
|
266
|
+
return {
|
|
267
|
+
condition: "AND",
|
|
268
|
+
target: "secret",
|
|
269
|
+
paths: paths.map((p) => compileGoRegex(p)),
|
|
270
|
+
regexes: [],
|
|
271
|
+
stopwords: [],
|
|
272
|
+
commits: [],
|
|
273
|
+
targetRules: [],
|
|
274
|
+
exact: [...values],
|
|
275
|
+
};
|
|
276
|
+
}
|
|
277
|
+
/** Validated exact-value allowlists for a file scan, from the text of a .gitleaks.toml. */
|
|
278
|
+
export function parseExactAllowlists(text, file = REPO_CONFIG_FILE) {
|
|
279
|
+
const doc = parseToml(text, file);
|
|
280
|
+
const out = [];
|
|
281
|
+
if (doc.allowlist !== undefined)
|
|
282
|
+
out.push(validateExact(file, "[allowlist]", doc.allowlist));
|
|
283
|
+
if (doc.allowlists !== undefined) {
|
|
284
|
+
if (!Array.isArray(doc.allowlists))
|
|
285
|
+
fail(file, "allowlists", "must be an array of tables ([[allowlists]])");
|
|
286
|
+
doc.allowlists.forEach((a, i) => out.push(validateExact(file, `[[allowlists]] #${i + 1}`, a)));
|
|
287
|
+
}
|
|
288
|
+
return out;
|
|
289
|
+
}
|
|
290
|
+
function parseToml(text, file) {
|
|
291
|
+
try {
|
|
292
|
+
return parse(text);
|
|
293
|
+
}
|
|
294
|
+
catch (err) {
|
|
295
|
+
throw new ScanConfigError(`${file}: could not be parsed: ${err instanceof Error ? err.message.split("\n")[0] : String(err)}`);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
185
298
|
function git(repoRoot, args) {
|
|
186
299
|
try {
|
|
187
300
|
return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
|
@@ -228,10 +341,27 @@ export function loadRepoAllowlists(repoRoot) {
|
|
|
228
341
|
// Changed in this commit: scan with the version already committed at HEAD.
|
|
229
342
|
return headAllowlists(repoRoot);
|
|
230
343
|
}
|
|
231
|
-
|
|
344
|
+
/**
|
|
345
|
+
* Exact-value allowlists from the copy committed at `rev` (default HEAD), for a
|
|
346
|
+
* file scan (`hq scan secrets --repo [--allowlist-rev]`). The index and working
|
|
347
|
+
* tree are ignored. pr-check passes the base branch, so a branch cannot exempt
|
|
348
|
+
* its own commits by adding an allowlist. An unknown rev is an error.
|
|
349
|
+
*/
|
|
350
|
+
export function loadCommittedAllowlists(repoRoot, rev = "HEAD") {
|
|
351
|
+
if (rev === "HEAD")
|
|
352
|
+
return headAllowlists(repoRoot, parseExactAllowlists);
|
|
353
|
+
if (rev.startsWith("-") || git(repoRoot, ["rev-parse", "--verify", "-q", `${rev}^{commit}`]) === null) {
|
|
354
|
+
throw new ScanConfigError(`allowlist rev ${JSON.stringify(rev)} is not a commit in ${repoRoot}`);
|
|
355
|
+
}
|
|
356
|
+
return revAllowlists(repoRoot, rev, parseExactAllowlists);
|
|
357
|
+
}
|
|
358
|
+
function headAllowlists(repoRoot, parser = parseRepoAllowlists) {
|
|
232
359
|
if (git(repoRoot, ["rev-parse", "--verify", "-q", "HEAD^{commit}"]) === null)
|
|
233
360
|
return [];
|
|
234
|
-
|
|
235
|
-
|
|
361
|
+
return revAllowlists(repoRoot, "HEAD", parser);
|
|
362
|
+
}
|
|
363
|
+
function revAllowlists(repoRoot, rev, parser) {
|
|
364
|
+
const e = entry(git(repoRoot, ["ls-tree", rev, "--", REPO_CONFIG_FILE]), 2);
|
|
365
|
+
return e ? parser(readEntry(repoRoot, e, rev), `${REPO_CONFIG_FILE} (${rev})`) : [];
|
|
236
366
|
}
|
|
237
367
|
//# sourceMappingURL=repo-config.js.map
|
|
@@ -12,6 +12,8 @@ export interface CompiledAllowlist {
|
|
|
12
12
|
commits: string[];
|
|
13
13
|
/** Empty: applies to every rule. */
|
|
14
14
|
targetRules: string[];
|
|
15
|
+
/** Exact secret values: allows only a detected secret equal to one of them (file scans with a repo). */
|
|
16
|
+
exact?: string[];
|
|
15
17
|
}
|
|
16
18
|
export interface CompiledRule {
|
|
17
19
|
id: string;
|
|
@@ -18,5 +18,19 @@ export declare function repoRoot(cwd: string): string;
|
|
|
18
18
|
* relative to the repo root, as gitleaks' allowlist paths are.
|
|
19
19
|
*/
|
|
20
20
|
export declare function scanStaged(cwd: string): SecretFinding[];
|
|
21
|
-
export
|
|
21
|
+
export interface ScanFilesOptions {
|
|
22
|
+
/**
|
|
23
|
+
* Apply this repo's committed .gitleaks.toml allowlists. Files under the repo
|
|
24
|
+
* are matched by their repo-relative path, as in a staged scan.
|
|
25
|
+
*/
|
|
26
|
+
repo?: string;
|
|
27
|
+
/**
|
|
28
|
+
* Files laid out as <snapshotRoot>/<commit>/<repo path> (historical copies,
|
|
29
|
+
* as `commit-secret-scan.sh pr-check` writes them) are matched as <repo path>.
|
|
30
|
+
*/
|
|
31
|
+
snapshotRoot?: string;
|
|
32
|
+
/** Read the allowlists from .gitleaks.toml at this commit instead of HEAD (pr-check passes its base). */
|
|
33
|
+
allowlistRev?: string;
|
|
34
|
+
}
|
|
35
|
+
export declare function scanFiles(paths: string[], opts?: ScanFilesOptions): SecretFinding[];
|
|
22
36
|
//# sourceMappingURL=staged.d.ts.map
|
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
* Scan inputs: the staged diff of a git repo (pre-commit), or whole files.
|
|
3
3
|
*/
|
|
4
4
|
import { execFileSync } from "node:child_process";
|
|
5
|
-
import { readFileSync, statSync } from "node:fs";
|
|
5
|
+
import { readFileSync, realpathSync, statSync } from "node:fs";
|
|
6
|
+
import { isAbsolute, relative, resolve, sep } from "node:path";
|
|
6
7
|
import { scanFragment, scanPath } from "./detect.js";
|
|
7
|
-
import { loadRepoAllowlists } from "./repo-config.js";
|
|
8
|
+
import { loadCommittedAllowlists, loadRepoAllowlists } from "./repo-config.js";
|
|
8
9
|
import { compiledRules } from "./rules.js";
|
|
9
10
|
export class ScanInputError extends Error {
|
|
10
11
|
}
|
|
@@ -110,7 +111,43 @@ export function scanStaged(cwd) {
|
|
|
110
111
|
const { fragments, binaryFiles } = parseUnifiedDiff(diff);
|
|
111
112
|
return [...fragments.flatMap((f) => scanFragment(f, rules)), ...binaryFiles.flatMap((f) => scanPath(f, rules))];
|
|
112
113
|
}
|
|
113
|
-
|
|
114
|
+
/** `p` relative to `root`, or null when `p` is not inside `root`. */
|
|
115
|
+
function inside(root, p) {
|
|
116
|
+
const rel = relative(root, p);
|
|
117
|
+
if (!rel || rel.startsWith("..") || isAbsolute(rel))
|
|
118
|
+
return null;
|
|
119
|
+
return rel.split(sep).join("/");
|
|
120
|
+
}
|
|
121
|
+
function real(p) {
|
|
122
|
+
try {
|
|
123
|
+
return realpathSync(p);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return resolve(p);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* The path allowlists see for `p`: repo-relative, or `<path>` for a snapshot at
|
|
131
|
+
* `<snapshotRoot>/<commit>/<path>`. null when `p` (after symlinks and `..`) is
|
|
132
|
+
* outside both roots: such a file gets no repo allowlist at all.
|
|
133
|
+
*/
|
|
134
|
+
function matchPath(p, repo, snapshotRoot) {
|
|
135
|
+
const abs = real(p);
|
|
136
|
+
if (snapshotRoot) {
|
|
137
|
+
const rel = inside(snapshotRoot, abs);
|
|
138
|
+
if (rel !== null) {
|
|
139
|
+
const slash = rel.indexOf("/");
|
|
140
|
+
if (slash > 0)
|
|
141
|
+
return rel.slice(slash + 1);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return inside(repo, abs);
|
|
145
|
+
}
|
|
146
|
+
export function scanFiles(paths, opts = {}) {
|
|
147
|
+
const root = opts.repo !== undefined ? repoRoot(opts.repo) : undefined;
|
|
148
|
+
const rules = root !== undefined ? { ...compiledRules(), repo: loadCommittedAllowlists(root, opts.allowlistRev) } : compiledRules();
|
|
149
|
+
const repo = root !== undefined ? real(root) : undefined;
|
|
150
|
+
const snapshotRoot = opts.snapshotRoot !== undefined ? real(opts.snapshotRoot) : undefined;
|
|
114
151
|
const out = [];
|
|
115
152
|
for (const p of paths) {
|
|
116
153
|
let text;
|
|
@@ -122,7 +159,12 @@ export function scanFiles(paths) {
|
|
|
122
159
|
catch (err) {
|
|
123
160
|
throw new ScanInputError(`could not read ${p}: ${err instanceof Error ? err.message : String(err)}`);
|
|
124
161
|
}
|
|
125
|
-
|
|
162
|
+
const mapped = repo !== undefined ? matchPath(p, repo, snapshotRoot) : p;
|
|
163
|
+
const file = mapped ?? p;
|
|
164
|
+
const use = mapped === null ? compiledRules() : rules;
|
|
165
|
+
// Findings still name the file as given, so a history copy keeps its commit folder.
|
|
166
|
+
const found = [...scanPath(file, use), ...scanFragment({ file, text, startLine: 1 }, use)];
|
|
167
|
+
out.push(...found.map((f) => ({ ...f, file: p })));
|
|
126
168
|
}
|
|
127
169
|
return out;
|
|
128
170
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@indigoai-us/hq-cli",
|
|
3
|
-
"version": "5.345.
|
|
3
|
+
"version": "5.345.166",
|
|
4
4
|
"description": "HQ by Indigo management CLI — modules and cloud sync",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"bin": {
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
"@aws-sdk/client-iot-data-plane": "^3.1096.0",
|
|
53
53
|
"@aws-sdk/client-s3": "^3.1049.0",
|
|
54
54
|
"@aws-sdk/dsql-signer": "^3.1146.0",
|
|
55
|
-
"@indigoai-us/hq-cloud": "6.18.
|
|
55
|
+
"@indigoai-us/hq-cloud": "6.18.101",
|
|
56
56
|
"@indigoai-us/hq-flags-client": "^0.1.2",
|
|
57
57
|
"@modelcontextprotocol/sdk": "1.29.0",
|
|
58
58
|
"@sentry/node": "^10.49.0",
|
|
@@ -114,11 +114,11 @@
|
|
|
114
114
|
"*.{ts,tsx,mts,cts,js,jsx,mjs,cjs}": "eslint --cache --no-error-on-unmatched-pattern"
|
|
115
115
|
},
|
|
116
116
|
"optionalDependencies": {
|
|
117
|
-
"@indigoai-us/hq-hook-win32-x64": "5.345.
|
|
118
|
-
"@indigoai-us/hq-hook-win32-arm64": "5.345.
|
|
119
|
-
"@indigoai-us/hq-hook-darwin-x64": "5.345.
|
|
120
|
-
"@indigoai-us/hq-hook-darwin-arm64": "5.345.
|
|
121
|
-
"@indigoai-us/hq-hook-linux-x64": "5.345.
|
|
122
|
-
"@indigoai-us/hq-hook-linux-arm64": "5.345.
|
|
117
|
+
"@indigoai-us/hq-hook-win32-x64": "5.345.166",
|
|
118
|
+
"@indigoai-us/hq-hook-win32-arm64": "5.345.166",
|
|
119
|
+
"@indigoai-us/hq-hook-darwin-x64": "5.345.166",
|
|
120
|
+
"@indigoai-us/hq-hook-darwin-arm64": "5.345.166",
|
|
121
|
+
"@indigoai-us/hq-hook-linux-x64": "5.345.166",
|
|
122
|
+
"@indigoai-us/hq-hook-linux-arm64": "5.345.166"
|
|
123
123
|
}
|
|
124
124
|
}
|