gitsieve 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +362 -0
- package/dist/cli.d.ts +19 -0
- package/dist/cli.js +147 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +12 -0
- package/dist/config.js +161 -0
- package/dist/config.js.map +1 -0
- package/dist/controls.d.ts +16 -0
- package/dist/controls.js +161 -0
- package/dist/controls.js.map +1 -0
- package/dist/corpus.d.ts +34 -0
- package/dist/corpus.js +121 -0
- package/dist/corpus.js.map +1 -0
- package/dist/git.d.ts +66 -0
- package/dist/git.js +214 -0
- package/dist/git.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/matcher.d.ts +34 -0
- package/dist/matcher.js +75 -0
- package/dist/matcher.js.map +1 -0
- package/dist/preflight.d.ts +21 -0
- package/dist/preflight.js +52 -0
- package/dist/preflight.js.map +1 -0
- package/dist/report.d.ts +8 -0
- package/dist/report.js +0 -0
- package/dist/report.js.map +1 -0
- package/dist/scan.d.ts +17 -0
- package/dist/scan.js +134 -0
- package/dist/scan.js.map +1 -0
- package/dist/secrets.d.ts +13 -0
- package/dist/secrets.js +48 -0
- package/dist/secrets.js.map +1 -0
- package/dist/types.d.ts +104 -0
- package/dist/types.js +11 -0
- package/dist/types.js.map +1 -0
- package/package.json +63 -0
package/dist/config.js
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { isAbsolute, relative, resolve } from "node:path";
|
|
3
|
+
import { CHANNELS, isChannel } from "./types.js";
|
|
4
|
+
export class ConfigError extends Error {
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* The terms file lists the exact strings you are trying to keep out of public
|
|
8
|
+
* view. Committing it into the repository being published would publish them.
|
|
9
|
+
* Keep it outside the repo, or pass --allow-terms-in-repo and accept that.
|
|
10
|
+
*/
|
|
11
|
+
export function assertTermsFileOutsideRepo(configPath, repoRoot) {
|
|
12
|
+
const rel = relative(resolve(repoRoot), resolve(configPath));
|
|
13
|
+
const inside = rel.length > 0 && !rel.startsWith("..") && !isAbsolute(rel);
|
|
14
|
+
if (inside) {
|
|
15
|
+
throw new ConfigError(`terms file ${configPath} is inside the repository being scanned.\n` +
|
|
16
|
+
`It contains the strings you are trying not to publish. Move it outside the repo, ` +
|
|
17
|
+
`or pass --allow-terms-in-repo if the terms are genuinely not sensitive.`);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function asArray(value, field) {
|
|
21
|
+
if (value === undefined)
|
|
22
|
+
return [];
|
|
23
|
+
if (!Array.isArray(value))
|
|
24
|
+
throw new ConfigError(`${field} must be an array`);
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
function parseTerm(raw, index) {
|
|
28
|
+
if (typeof raw === "string") {
|
|
29
|
+
if (raw.length === 0)
|
|
30
|
+
throw new ConfigError(`terms[${index}] is an empty string`);
|
|
31
|
+
return { id: raw, value: raw };
|
|
32
|
+
}
|
|
33
|
+
if (typeof raw !== "object" || raw === null) {
|
|
34
|
+
throw new ConfigError(`terms[${index}] must be a string or an object`);
|
|
35
|
+
}
|
|
36
|
+
const term = raw;
|
|
37
|
+
const value = term["value"];
|
|
38
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
39
|
+
throw new ConfigError(`terms[${index}].value must be a non-empty string`);
|
|
40
|
+
}
|
|
41
|
+
const id = typeof term["id"] === "string" && term["id"] ? term["id"] : value;
|
|
42
|
+
const wholeWord = term["wholeWord"];
|
|
43
|
+
if (wholeWord !== undefined &&
|
|
44
|
+
!["auto", "always", "never"].includes(String(wholeWord))) {
|
|
45
|
+
throw new ConfigError(`terms[${index}].wholeWord must be auto, always or never`);
|
|
46
|
+
}
|
|
47
|
+
const channels = asArray(term["channels"], `terms[${index}].channels`);
|
|
48
|
+
for (const channel of channels) {
|
|
49
|
+
if (!isChannel(channel)) {
|
|
50
|
+
throw new ConfigError(`terms[${index}].channels contains "${String(channel)}"; valid channels are ${CHANNELS.join(", ")}`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return {
|
|
54
|
+
id,
|
|
55
|
+
value,
|
|
56
|
+
...(wholeWord ? { wholeWord: wholeWord } : {}),
|
|
57
|
+
...(term["caseSensitive"] !== undefined
|
|
58
|
+
? { caseSensitive: Boolean(term["caseSensitive"]) }
|
|
59
|
+
: {}),
|
|
60
|
+
...(channels.length > 0
|
|
61
|
+
? { channels: channels }
|
|
62
|
+
: {}),
|
|
63
|
+
...(typeof term["note"] === "string" ? { note: term["note"] } : {}),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
function parseSuppression(raw, index) {
|
|
67
|
+
if (typeof raw !== "object" || raw === null) {
|
|
68
|
+
throw new ConfigError(`suppress[${index}] must be an object`);
|
|
69
|
+
}
|
|
70
|
+
const entry = raw;
|
|
71
|
+
const termId = entry["termId"];
|
|
72
|
+
const reason = entry["reason"];
|
|
73
|
+
if (typeof termId !== "string" || !termId) {
|
|
74
|
+
throw new ConfigError(`suppress[${index}].termId is required`);
|
|
75
|
+
}
|
|
76
|
+
// A suppression without a reason is an unexplained hole in the audit.
|
|
77
|
+
if (typeof reason !== "string" || !reason) {
|
|
78
|
+
throw new ConfigError(`suppress[${index}].reason is required; say why this is not a leak`);
|
|
79
|
+
}
|
|
80
|
+
if (entry["value"] !== undefined || entry["term"] !== undefined) {
|
|
81
|
+
throw new ConfigError(`suppress[${index}] must reference a term by id, never by value; a suppression file that repeats the sensitive string is itself a leak`);
|
|
82
|
+
}
|
|
83
|
+
const channel = entry["channel"];
|
|
84
|
+
if (channel !== undefined && !isChannel(channel)) {
|
|
85
|
+
throw new ConfigError(`suppress[${index}].channel is not a valid channel`);
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
termId,
|
|
89
|
+
reason,
|
|
90
|
+
...(channel ? { channel } : {}),
|
|
91
|
+
...(typeof entry["path"] === "string" ? { path: entry["path"] } : {}),
|
|
92
|
+
...(typeof entry["pathPrefix"] === "string"
|
|
93
|
+
? { pathPrefix: entry["pathPrefix"] }
|
|
94
|
+
: {}),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
function parseControl(raw, index) {
|
|
98
|
+
if (typeof raw !== "object" || raw === null) {
|
|
99
|
+
throw new ConfigError(`controls[${index}] must be an object`);
|
|
100
|
+
}
|
|
101
|
+
const entry = raw;
|
|
102
|
+
const value = entry["value"];
|
|
103
|
+
const channel = entry["channel"];
|
|
104
|
+
if (typeof value !== "string" || !value) {
|
|
105
|
+
throw new ConfigError(`controls[${index}].value must be a non-empty string`);
|
|
106
|
+
}
|
|
107
|
+
if (!isChannel(channel)) {
|
|
108
|
+
throw new ConfigError(`controls[${index}].channel must be one of ${CHANNELS.join(", ")}`);
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
value,
|
|
112
|
+
channel,
|
|
113
|
+
...(typeof entry["note"] === "string" ? { note: entry["note"] } : {}),
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
export function parseConfig(json) {
|
|
117
|
+
if (typeof json !== "object" || json === null) {
|
|
118
|
+
throw new ConfigError("config must be a JSON object");
|
|
119
|
+
}
|
|
120
|
+
const raw = json;
|
|
121
|
+
const terms = asArray(raw["terms"], "terms").map(parseTerm);
|
|
122
|
+
const seen = new Set();
|
|
123
|
+
for (const term of terms) {
|
|
124
|
+
if (seen.has(term.id))
|
|
125
|
+
throw new ConfigError(`duplicate term id "${term.id}"`);
|
|
126
|
+
seen.add(term.id);
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
terms,
|
|
130
|
+
suppress: asArray(raw["suppress"], "suppress").map(parseSuppression),
|
|
131
|
+
controls: asArray(raw["controls"], "controls").map(parseControl),
|
|
132
|
+
secrets: raw["secrets"] === undefined ? true : Boolean(raw["secrets"]),
|
|
133
|
+
...(typeof raw["maxBlobBytes"] === "number"
|
|
134
|
+
? { maxBlobBytes: raw["maxBlobBytes"] }
|
|
135
|
+
: {}),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
export function loadConfig(path) {
|
|
139
|
+
let text;
|
|
140
|
+
try {
|
|
141
|
+
text = readFileSync(path, "utf8");
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
throw new ConfigError(`cannot read terms file ${path}: ${error.message}`);
|
|
145
|
+
}
|
|
146
|
+
try {
|
|
147
|
+
return parseConfig(JSON.parse(text));
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
if (error instanceof ConfigError)
|
|
151
|
+
throw error;
|
|
152
|
+
throw new ConfigError(`${path} is not valid JSON: ${error.message}`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
export const EMPTY_CONFIG = {
|
|
156
|
+
terms: [],
|
|
157
|
+
suppress: [],
|
|
158
|
+
controls: [],
|
|
159
|
+
secrets: true,
|
|
160
|
+
};
|
|
161
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAQjD,MAAM,OAAO,WAAY,SAAQ,KAAK;CAAG;AAEzC;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CACxC,UAAkB,EAClB,QAAgB;IAEhB,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;IAC7D,MAAM,MAAM,GACV,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9D,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,IAAI,WAAW,CACnB,cAAc,UAAU,4CAA4C;YAClE,mFAAmF;YACnF,yEAAyE,CAC5E,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,OAAO,CAAC,KAAc,EAAE,KAAa;IAC5C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,WAAW,CAAC,GAAG,KAAK,mBAAmB,CAAC,CAAC;IAC9E,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,SAAS,CAAC,GAAY,EAAE,KAAa;IAC5C,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAClB,MAAM,IAAI,WAAW,CAAC,SAAS,KAAK,sBAAsB,CAAC,CAAC;QAC9D,OAAO,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,IAAI,WAAW,CAAC,SAAS,KAAK,iCAAiC,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,IAAI,GAAG,GAA8B,CAAC;IAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;IAC5B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,WAAW,CAAC,SAAS,KAAK,oCAAoC,CAAC,CAAC;IAC5E,CAAC;IACD,MAAM,EAAE,GAAG,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IAC7E,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;IACpC,IACE,SAAS,KAAK,SAAS;QACvB,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,EACxD,CAAC;QACD,MAAM,IAAI,WAAW,CACnB,SAAS,KAAK,2CAA2C,CAC1D,CAAC;IACJ,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,SAAS,KAAK,YAAY,CAAC,CAAC;IACvE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,WAAW,CACnB,SAAS,KAAK,wBAAwB,MAAM,CAAC,OAAO,CAAC,yBAAyB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpG,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO;QACL,EAAE;QACF,KAAK;QACL,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,SAAkC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,SAAS;YACrC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,EAAE;YACnD,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAgC,EAAE;YAChD,CAAC,CAAC,EAAE,CAAC;QACP,GAAG,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpE,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAY,EAAE,KAAa;IACnD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,IAAI,WAAW,CAAC,YAAY,KAAK,qBAAqB,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,KAAK,GAAG,GAA8B,CAAC;IAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;QAC1C,MAAM,IAAI,WAAW,CAAC,YAAY,KAAK,sBAAsB,CAAC,CAAC;IACjE,CAAC;IACD,sEAAsE;IACtE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;QAC1C,MAAM,IAAI,WAAW,CACnB,YAAY,KAAK,kDAAkD,CACpE,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,SAAS,EAAE,CAAC;QAChE,MAAM,IAAI,WAAW,CACnB,YAAY,KAAK,sHAAsH,CACxI,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IACjC,IAAI,OAAO,KAAK,SAAS,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,WAAW,CAAC,YAAY,KAAK,kCAAkC,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO;QACL,MAAM;QACN,MAAM;QACN,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,GAAG,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,GAAG,CAAC,OAAO,KAAK,CAAC,YAAY,CAAC,KAAK,QAAQ;YACzC,CAAC,CAAC,EAAE,UAAU,EAAE,KAAK,CAAC,YAAY,CAAC,EAAE;YACrC,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,GAAY,EAAE,KAAa;IAC/C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,IAAI,WAAW,CAAC,YAAY,KAAK,qBAAqB,CAAC,CAAC;IAChE,CAAC;IACD,MAAM,KAAK,GAAG,GAA8B,CAAC;IAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;IACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,IAAI,WAAW,CACnB,YAAY,KAAK,oCAAoC,CACtD,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,WAAW,CACnB,YAAY,KAAK,4BAA4B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnE,CAAC;IACJ,CAAC;IACD,OAAO;QACL,KAAK;QACL,OAAO;QACP,GAAG,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAa;IACvC,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAC9C,MAAM,IAAI,WAAW,CAAC,8BAA8B,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,GAAG,GAAG,IAA+B,CAAC;IAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,WAAW,CAAC,sBAAsB,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1D,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACpB,CAAC;IACD,OAAO;QACL,KAAK;QACL,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,gBAAgB,CAAC;QACpE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,YAAY,CAAC;QAChE,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACtE,GAAG,CAAC,OAAO,GAAG,CAAC,cAAc,CAAC,KAAK,QAAQ;YACzC,CAAC,CAAC,EAAE,YAAY,EAAE,GAAG,CAAC,cAAc,CAAC,EAAE;YACvC,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,IAAY,CAAC;IACjB,IAAI,CAAC;QACH,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,WAAW,CACnB,0BAA0B,IAAI,KAAM,KAAe,CAAC,OAAO,EAAE,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,WAAW;YAAE,MAAM,KAAK,CAAC;QAC9C,MAAM,IAAI,WAAW,CACnB,GAAG,IAAI,uBAAwB,KAAe,CAAC,OAAO,EAAE,CACzD,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,YAAY,GAAW;IAClC,KAAK,EAAE,EAAE;IACT,QAAQ,EAAE,EAAE;IACZ,QAAQ,EAAE,EAAE;IACZ,OAAO,EAAE,IAAI;CACd,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Corpus } from "./corpus.js";
|
|
2
|
+
import type { Channel, ControlSpec, ControlResult, Warning } from "./types.js";
|
|
3
|
+
export interface DerivedControls {
|
|
4
|
+
controls: ControlSpec[];
|
|
5
|
+
/** Channels that hold data but yielded no control: a hard problem. */
|
|
6
|
+
underivable: Channel[];
|
|
7
|
+
/** Channels that are simply empty in this repository. */
|
|
8
|
+
empty: Channel[];
|
|
9
|
+
}
|
|
10
|
+
export declare function deriveControls(corpus: Corpus): DerivedControls;
|
|
11
|
+
export interface ControlVerification {
|
|
12
|
+
results: ControlResult[];
|
|
13
|
+
failures: ControlResult[];
|
|
14
|
+
warnings: Warning[];
|
|
15
|
+
}
|
|
16
|
+
export declare function verifyControls(corpus: Corpus, derived: ControlSpec[], supplied?: ControlSpec[]): ControlVerification;
|
package/dist/controls.js
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { compileTerm, findMatches } from "./matcher.js";
|
|
2
|
+
/**
|
|
3
|
+
* The rule this tool is built around: never report a zero from a scanner you
|
|
4
|
+
* have not just watched find something.
|
|
5
|
+
*
|
|
6
|
+
* For each channel we take a string that provably exists in that channel --
|
|
7
|
+
* a word from a real file, a real path segment, a word from a real commit
|
|
8
|
+
* message, a real author address -- and push it through the same term pipeline
|
|
9
|
+
* the audit uses. If any of those comes back with zero matches, the scanner is
|
|
10
|
+
* not reading that channel and its "no findings" means nothing.
|
|
11
|
+
*/
|
|
12
|
+
const CONTROL_CHANNELS = [
|
|
13
|
+
"contents",
|
|
14
|
+
"paths",
|
|
15
|
+
"messages",
|
|
16
|
+
"identities",
|
|
17
|
+
"refs",
|
|
18
|
+
];
|
|
19
|
+
function unitsFor(corpus, channel) {
|
|
20
|
+
return corpus.units.filter((unit) => unit.channel === channel);
|
|
21
|
+
}
|
|
22
|
+
/** Longest ASCII word in a string, subject to a length window. */
|
|
23
|
+
function longestWord(text, min, max) {
|
|
24
|
+
const words = text.match(/[A-Za-z0-9_]{2,}/g);
|
|
25
|
+
if (!words)
|
|
26
|
+
return null;
|
|
27
|
+
let best = null;
|
|
28
|
+
for (const word of words) {
|
|
29
|
+
if (word.length < min || word.length > max)
|
|
30
|
+
continue;
|
|
31
|
+
if (!best || word.length > best.length)
|
|
32
|
+
best = word;
|
|
33
|
+
}
|
|
34
|
+
return best;
|
|
35
|
+
}
|
|
36
|
+
function deriveForContents(units) {
|
|
37
|
+
// Deterministic: sort by location so the same repo derives the same control.
|
|
38
|
+
const sorted = [...units].sort((a, b) => a.location.localeCompare(b.location));
|
|
39
|
+
for (const unit of sorted) {
|
|
40
|
+
const word = longestWord(unit.text.slice(0, 200_000), 8, 40);
|
|
41
|
+
if (word)
|
|
42
|
+
return word;
|
|
43
|
+
}
|
|
44
|
+
return null;
|
|
45
|
+
}
|
|
46
|
+
function deriveForPaths(units) {
|
|
47
|
+
let best = null;
|
|
48
|
+
for (const unit of units) {
|
|
49
|
+
for (const segment of unit.text.split("/")) {
|
|
50
|
+
const word = longestWord(segment, 3, 40);
|
|
51
|
+
if (word && (!best || word.length > best.length))
|
|
52
|
+
best = word;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return best;
|
|
56
|
+
}
|
|
57
|
+
function deriveForMessages(units) {
|
|
58
|
+
let best = null;
|
|
59
|
+
for (const unit of units) {
|
|
60
|
+
const word = longestWord(unit.text, 4, 40);
|
|
61
|
+
if (word && (!best || word.length > best.length))
|
|
62
|
+
best = word;
|
|
63
|
+
}
|
|
64
|
+
return best;
|
|
65
|
+
}
|
|
66
|
+
function deriveForIdentities(units) {
|
|
67
|
+
for (const unit of units) {
|
|
68
|
+
const email = unit.text.match(/<([^>\s]+)>/);
|
|
69
|
+
if (email?.[1])
|
|
70
|
+
return email[1];
|
|
71
|
+
const word = longestWord(unit.text, 3, 40);
|
|
72
|
+
if (word)
|
|
73
|
+
return word;
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
function deriveForRefs(units) {
|
|
78
|
+
let best = null;
|
|
79
|
+
for (const unit of units) {
|
|
80
|
+
for (const segment of unit.text.split("/")) {
|
|
81
|
+
const word = longestWord(segment, 3, 40);
|
|
82
|
+
if (word && (!best || word.length > best.length))
|
|
83
|
+
best = word;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return best;
|
|
87
|
+
}
|
|
88
|
+
const DERIVERS = {
|
|
89
|
+
contents: deriveForContents,
|
|
90
|
+
paths: deriveForPaths,
|
|
91
|
+
messages: deriveForMessages,
|
|
92
|
+
identities: deriveForIdentities,
|
|
93
|
+
refs: deriveForRefs,
|
|
94
|
+
};
|
|
95
|
+
export function deriveControls(corpus) {
|
|
96
|
+
const controls = [];
|
|
97
|
+
const underivable = [];
|
|
98
|
+
const empty = [];
|
|
99
|
+
for (const channel of CONTROL_CHANNELS) {
|
|
100
|
+
const units = unitsFor(corpus, channel);
|
|
101
|
+
if (units.length === 0) {
|
|
102
|
+
empty.push(channel);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
const value = DERIVERS[channel](units);
|
|
106
|
+
if (!value) {
|
|
107
|
+
underivable.push(channel);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
controls.push({
|
|
111
|
+
value,
|
|
112
|
+
channel,
|
|
113
|
+
note: `derived from this repository's own ${channel}`,
|
|
114
|
+
});
|
|
115
|
+
}
|
|
116
|
+
return { controls, underivable, empty };
|
|
117
|
+
}
|
|
118
|
+
export function verifyControls(corpus, derived, supplied = []) {
|
|
119
|
+
const results = [];
|
|
120
|
+
const failures = [];
|
|
121
|
+
const warnings = [];
|
|
122
|
+
const all = [
|
|
123
|
+
...derived.map((spec) => ({ spec, derived: true })),
|
|
124
|
+
...supplied.map((spec) => ({ spec, derived: false })),
|
|
125
|
+
];
|
|
126
|
+
for (const { spec, derived: isDerived } of all) {
|
|
127
|
+
// Same compile path, same match path as a real term. A control that took a
|
|
128
|
+
// shortcut around the matcher would verify nothing.
|
|
129
|
+
const compiled = compileTerm({
|
|
130
|
+
id: `control:${spec.channel}`,
|
|
131
|
+
value: spec.value,
|
|
132
|
+
caseSensitive: true,
|
|
133
|
+
});
|
|
134
|
+
let matches = 0;
|
|
135
|
+
for (const unit of corpus.units) {
|
|
136
|
+
if (unit.channel !== spec.channel)
|
|
137
|
+
continue;
|
|
138
|
+
matches += findMatches(compiled.regex, unit.text).length;
|
|
139
|
+
if (matches > 0 && isDerived)
|
|
140
|
+
break; // one hit is proof enough
|
|
141
|
+
}
|
|
142
|
+
const result = {
|
|
143
|
+
value: spec.value,
|
|
144
|
+
channel: spec.channel,
|
|
145
|
+
derived: isDerived,
|
|
146
|
+
matches,
|
|
147
|
+
...(spec.note ? { note: spec.note } : {}),
|
|
148
|
+
};
|
|
149
|
+
results.push(result);
|
|
150
|
+
if (matches === 0)
|
|
151
|
+
failures.push(result);
|
|
152
|
+
}
|
|
153
|
+
if (results.length === 0) {
|
|
154
|
+
warnings.push({
|
|
155
|
+
code: "no-controls",
|
|
156
|
+
message: "no control strings could be established; nothing verified the scanner",
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
return { results, failures, warnings };
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=controls.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"controls.js","sourceRoot":"","sources":["../src/controls.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAGxD;;;;;;;;;GASG;AAEH,MAAM,gBAAgB,GAAc;IAClC,UAAU;IACV,OAAO;IACP,UAAU;IACV,YAAY;IACZ,MAAM;CACP,CAAC;AAEF,SAAS,QAAQ,CAAC,MAAc,EAAE,OAAgB;IAChD,OAAO,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;AACjE,CAAC;AAED,kEAAkE;AAClE,SAAS,WAAW,CAAC,IAAY,EAAE,GAAW,EAAE,GAAW;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC9C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;YAAE,SAAS;QACrD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM;YAAE,IAAI,GAAG,IAAI,CAAC;IACtD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAiB;IAC1C,6EAA6E;IAC7E,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACtC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CACrC,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7D,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,KAAiB;IACvC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,GAAG,IAAI,CAAC;QAChE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAiB;IAC1C,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YAAE,IAAI,GAAG,IAAI,CAAC;IAChE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAiB;IAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,KAAiB;IACtC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACzC,IAAI,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBAAE,IAAI,GAAG,IAAI,CAAC;QAChE,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,QAAQ,GAA0D;IACtE,QAAQ,EAAE,iBAAiB;IAC3B,KAAK,EAAE,cAAc;IACrB,QAAQ,EAAE,iBAAiB;IAC3B,UAAU,EAAE,mBAAmB;IAC/B,IAAI,EAAE,aAAa;CACpB,CAAC;AAUF,MAAM,UAAU,cAAc,CAAC,MAAc;IAC3C,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,MAAM,WAAW,GAAc,EAAE,CAAC;IAClC,MAAM,KAAK,GAAc,EAAE,CAAC;IAE5B,KAAK,MAAM,OAAO,IAAI,gBAAgB,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpB,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK;YACL,OAAO;YACP,IAAI,EAAE,sCAAsC,OAAO,EAAE;SACtD,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;AAC1C,CAAC;AAQD,MAAM,UAAU,cAAc,CAC5B,MAAc,EACd,OAAsB,EACtB,WAA0B,EAAE;IAE5B,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,MAAM,GAAG,GAA8C;QACrD,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;KACtD,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,GAAG,EAAE,CAAC;QAC/C,2EAA2E;QAC3E,oDAAoD;QACpD,MAAM,QAAQ,GAAG,WAAW,CAAC;YAC3B,EAAE,EAAE,WAAW,IAAI,CAAC,OAAO,EAAE;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,aAAa,EAAE,IAAI;SACpB,CAAC,CAAC;QACH,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;gBAAE,SAAS;YAC5C,OAAO,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACzD,IAAI,OAAO,GAAG,CAAC,IAAI,SAAS;gBAAE,MAAM,CAAC,0BAA0B;QACjE,CAAC;QACD,MAAM,MAAM,GAAkB;YAC5B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,SAAS;YAClB,OAAO;YACP,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1C,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,OAAO,KAAK,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,aAAa;YACnB,OAAO,EACL,uEAAuE;SAC1E,CAAC,CAAC;IACL,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;AACzC,CAAC"}
|
package/dist/corpus.d.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { GitRepo } from "./git.js";
|
|
2
|
+
import type { Channel, Warning } from "./types.js";
|
|
3
|
+
export interface TextUnit {
|
|
4
|
+
channel: Channel;
|
|
5
|
+
/** Human-readable place, e.g. "src/a.ts (blob 1a2b3c4)". */
|
|
6
|
+
location: string;
|
|
7
|
+
/** Repository path, when the unit has one. Used by suppressions. */
|
|
8
|
+
path?: string;
|
|
9
|
+
text: string;
|
|
10
|
+
}
|
|
11
|
+
export interface Corpus {
|
|
12
|
+
units: TextUnit[];
|
|
13
|
+
warnings: Warning[];
|
|
14
|
+
/**
|
|
15
|
+
* Reasons the corpus is known to be incomplete. Any entry here makes the
|
|
16
|
+
* whole run untrusted: an incomplete scan that returns zero is exactly the
|
|
17
|
+
* failure this tool exists to prevent.
|
|
18
|
+
*/
|
|
19
|
+
integrityProblems: string[];
|
|
20
|
+
stats: {
|
|
21
|
+
commits: number;
|
|
22
|
+
refs: number;
|
|
23
|
+
paths: number;
|
|
24
|
+
objects: number;
|
|
25
|
+
blobsRead: number;
|
|
26
|
+
bytesScanned: number;
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export interface BuildCorpusOptions {
|
|
30
|
+
maxBlobBytes?: number;
|
|
31
|
+
}
|
|
32
|
+
/** A blob with a NUL byte near its start is not source text. */
|
|
33
|
+
export declare function looksBinary(buffer: Buffer): boolean;
|
|
34
|
+
export declare function buildCorpus(repo: GitRepo, options?: BuildCorpusOptions): Corpus;
|
package/dist/corpus.js
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
const DEFAULT_MAX_BLOB_BYTES = 2 * 1024 * 1024;
|
|
2
|
+
function short(sha) {
|
|
3
|
+
return sha.slice(0, 7);
|
|
4
|
+
}
|
|
5
|
+
/** A blob with a NUL byte near its start is not source text. */
|
|
6
|
+
export function looksBinary(buffer) {
|
|
7
|
+
const window = buffer.subarray(0, Math.min(buffer.length, 8000));
|
|
8
|
+
return window.includes(0);
|
|
9
|
+
}
|
|
10
|
+
export function buildCorpus(repo, options = {}) {
|
|
11
|
+
const maxBlobBytes = options.maxBlobBytes ?? DEFAULT_MAX_BLOB_BYTES;
|
|
12
|
+
const units = [];
|
|
13
|
+
const warnings = [];
|
|
14
|
+
const integrityProblems = [];
|
|
15
|
+
// --- refs -----------------------------------------------------------------
|
|
16
|
+
const refs = repo.refNames();
|
|
17
|
+
for (const ref of refs) {
|
|
18
|
+
units.push({ channel: "refs", location: ref, text: ref });
|
|
19
|
+
}
|
|
20
|
+
// --- paths ----------------------------------------------------------------
|
|
21
|
+
const paths = repo.paths();
|
|
22
|
+
for (const path of paths) {
|
|
23
|
+
units.push({ channel: "paths", location: path, path, text: path });
|
|
24
|
+
}
|
|
25
|
+
// --- commit messages and identities ---------------------------------------
|
|
26
|
+
const commits = repo.commits();
|
|
27
|
+
for (const commit of commits) {
|
|
28
|
+
units.push({
|
|
29
|
+
channel: "messages",
|
|
30
|
+
location: `commit ${short(commit.sha)}`,
|
|
31
|
+
text: commit.message,
|
|
32
|
+
});
|
|
33
|
+
units.push({
|
|
34
|
+
channel: "identities",
|
|
35
|
+
location: `commit ${short(commit.sha)}`,
|
|
36
|
+
text: `author ${commit.authorName} <${commit.authorEmail}>\n` +
|
|
37
|
+
`committer ${commit.committerName} <${commit.committerEmail}>`,
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
// --- object contents ------------------------------------------------------
|
|
41
|
+
const objects = repo.objects();
|
|
42
|
+
const readable = [];
|
|
43
|
+
for (const object of objects) {
|
|
44
|
+
if (object.type !== "blob" && object.type !== "tag")
|
|
45
|
+
continue;
|
|
46
|
+
if (object.size > maxBlobBytes) {
|
|
47
|
+
warnings.push({
|
|
48
|
+
code: "blob-too-large",
|
|
49
|
+
message: `${object.path ?? short(object.sha)} is ${object.size} bytes; contents not scanned (raise --max-blob-bytes)`,
|
|
50
|
+
});
|
|
51
|
+
continue;
|
|
52
|
+
}
|
|
53
|
+
readable.push(object);
|
|
54
|
+
}
|
|
55
|
+
const bytes = repo.readObjects(readable.map((o) => o.sha));
|
|
56
|
+
let blobsRead = 0;
|
|
57
|
+
let bytesScanned = 0;
|
|
58
|
+
let missing = 0;
|
|
59
|
+
let truncated = 0;
|
|
60
|
+
for (const object of readable) {
|
|
61
|
+
const buffer = bytes.get(object.sha);
|
|
62
|
+
if (!buffer) {
|
|
63
|
+
missing += 1;
|
|
64
|
+
continue;
|
|
65
|
+
}
|
|
66
|
+
if (buffer.length !== object.size) {
|
|
67
|
+
truncated += 1;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
if (looksBinary(buffer))
|
|
71
|
+
continue;
|
|
72
|
+
blobsRead += 1;
|
|
73
|
+
bytesScanned += buffer.length;
|
|
74
|
+
const text = buffer.toString("utf8");
|
|
75
|
+
if (object.type === "tag") {
|
|
76
|
+
units.push({
|
|
77
|
+
channel: "messages",
|
|
78
|
+
location: `tag object ${short(object.sha)}`,
|
|
79
|
+
text,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
units.push({
|
|
84
|
+
channel: "contents",
|
|
85
|
+
location: `${object.path ?? "(unnamed)"} (blob ${short(object.sha)})`,
|
|
86
|
+
...(object.path ? { path: object.path } : {}),
|
|
87
|
+
text,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
// Silence is the enemy. If git listed an object and we could not read it,
|
|
92
|
+
// say so and mark the run untrusted rather than scanning what is left.
|
|
93
|
+
if (missing > 0) {
|
|
94
|
+
integrityProblems.push(`${missing} of ${readable.length} objects were listed by git but returned no content`);
|
|
95
|
+
}
|
|
96
|
+
if (truncated > 0) {
|
|
97
|
+
integrityProblems.push(`${truncated} of ${readable.length} objects returned a different byte count than git reported`);
|
|
98
|
+
}
|
|
99
|
+
// Cross-checks between channels. A repository cannot have refs but no
|
|
100
|
+
// commits, or blobs but no paths; if it appears to, a channel was dropped.
|
|
101
|
+
if (refs.length > 0 && commits.length === 0) {
|
|
102
|
+
integrityProblems.push(`git listed ${refs.length} refs but no commits; the messages and identities channels were not read`);
|
|
103
|
+
}
|
|
104
|
+
if (readable.length > 0 && paths.length === 0) {
|
|
105
|
+
integrityProblems.push(`git listed ${readable.length} readable objects but no paths; the paths channel was not read`);
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
units,
|
|
109
|
+
warnings,
|
|
110
|
+
integrityProblems,
|
|
111
|
+
stats: {
|
|
112
|
+
commits: commits.length,
|
|
113
|
+
refs: refs.length,
|
|
114
|
+
paths: paths.length,
|
|
115
|
+
objects: objects.length,
|
|
116
|
+
blobsRead,
|
|
117
|
+
bytesScanned,
|
|
118
|
+
},
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=corpus.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"corpus.js","sourceRoot":"","sources":["../src/corpus.ts"],"names":[],"mappings":"AAmCA,MAAM,sBAAsB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAE/C,SAAS,KAAK,CAAC,GAAW;IACxB,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACzB,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;IACjE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,IAAa,EACb,UAA8B,EAAE;IAEhC,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,sBAAsB,CAAC;IACpE,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,iBAAiB,GAAa,EAAE,CAAC;IAEvC,6EAA6E;IAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IAC5D,CAAC;IAED,6EAA6E;IAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC;IAED,6EAA6E;IAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAC/B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,UAAU;YACnB,QAAQ,EAAE,UAAU,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACvC,IAAI,EAAE,MAAM,CAAC,OAAO;SACrB,CAAC,CAAC;QACH,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,YAAY;YACrB,QAAQ,EAAE,UAAU,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YACvC,IAAI,EACF,UAAU,MAAM,CAAC,UAAU,KAAK,MAAM,CAAC,WAAW,KAAK;gBACvD,aAAa,MAAM,CAAC,aAAa,KAAK,MAAM,CAAC,cAAc,GAAG;SACjE,CAAC,CAAC;IACL,CAAC;IAED,6EAA6E;IAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAiB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;YAAE,SAAS;QAC9D,IAAI,MAAM,CAAC,IAAI,GAAG,YAAY,EAAE,CAAC;YAC/B,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,gBAAgB;gBACtB,OAAO,EAAE,GAAG,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,IAAI,uDAAuD;aACtH,CAAC,CAAC;YACH,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,IAAI,CAAC,CAAC;YACb,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;YAClC,SAAS,IAAI,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,IAAI,WAAW,CAAC,MAAM,CAAC;YAAE,SAAS;QAClC,SAAS,IAAI,CAAC,CAAC;QACf,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;QAC9B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC;gBACT,OAAO,EAAE,UAAU;gBACnB,QAAQ,EAAE,cAAc,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;gBAC3C,IAAI;aACL,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC;gBACT,OAAO,EAAE,UAAU;gBACnB,QAAQ,EAAE,GAAG,MAAM,CAAC,IAAI,IAAI,WAAW,UAAU,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG;gBACrE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,IAAI;aACL,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,0EAA0E;IAC1E,uEAAuE;IACvE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,iBAAiB,CAAC,IAAI,CACpB,GAAG,OAAO,OAAO,QAAQ,CAAC,MAAM,qDAAqD,CACtF,CAAC;IACJ,CAAC;IACD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,iBAAiB,CAAC,IAAI,CACpB,GAAG,SAAS,OAAO,QAAQ,CAAC,MAAM,4DAA4D,CAC/F,CAAC;IACJ,CAAC;IAED,sEAAsE;IACtE,2EAA2E;IAC3E,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,iBAAiB,CAAC,IAAI,CACpB,cAAc,IAAI,CAAC,MAAM,0EAA0E,CACpG,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9C,iBAAiB,CAAC,IAAI,CACpB,cAAc,QAAQ,CAAC,MAAM,gEAAgE,CAC9F,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK;QACL,QAAQ;QACR,iBAAiB;QACjB,KAAK,EAAE;YACL,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,OAAO,EAAE,OAAO,CAAC,MAAM;YACvB,SAAS;YACT,YAAY;SACb;KACF,CAAC;AACJ,CAAC"}
|
package/dist/git.d.ts
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
export declare class GitError extends Error {
|
|
2
|
+
}
|
|
3
|
+
export interface CommitMeta {
|
|
4
|
+
sha: string;
|
|
5
|
+
message: string;
|
|
6
|
+
authorName: string;
|
|
7
|
+
authorEmail: string;
|
|
8
|
+
committerName: string;
|
|
9
|
+
committerEmail: string;
|
|
10
|
+
}
|
|
11
|
+
export interface ObjectMeta {
|
|
12
|
+
sha: string;
|
|
13
|
+
type: "blob" | "tree" | "commit" | "tag";
|
|
14
|
+
size: number;
|
|
15
|
+
/** Path git associated with the object. Absent for commits and tag objects. */
|
|
16
|
+
path?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* The surface the scanner uses. It is an interface so that a test can hand the
|
|
20
|
+
* scanner a deliberately broken git and prove the control check fails loudly
|
|
21
|
+
* instead of reporting a clean repository.
|
|
22
|
+
*/
|
|
23
|
+
export interface GitRepo {
|
|
24
|
+
readonly root: string;
|
|
25
|
+
isShallow(): boolean;
|
|
26
|
+
refNames(): string[];
|
|
27
|
+
commits(): CommitMeta[];
|
|
28
|
+
/** Every path that has existed in any commit on any ref. */
|
|
29
|
+
paths(): string[];
|
|
30
|
+
objects(): ObjectMeta[];
|
|
31
|
+
/** sha -> raw bytes. Only called for objects returned by objects(). */
|
|
32
|
+
readObjects(shas: string[]): Map<string, Buffer>;
|
|
33
|
+
/** Working-tree file, or null when absent. Used only for .gitignore checks. */
|
|
34
|
+
workingTreeFile(relativePath: string): string | null;
|
|
35
|
+
remoteFetchSpecs(): string[];
|
|
36
|
+
}
|
|
37
|
+
export interface RealGitOptions {
|
|
38
|
+
includeReflog?: boolean;
|
|
39
|
+
}
|
|
40
|
+
export declare class RealGitRepo implements GitRepo {
|
|
41
|
+
readonly root: string;
|
|
42
|
+
private readonly includeReflog;
|
|
43
|
+
constructor(root: string, options?: RealGitOptions);
|
|
44
|
+
private run;
|
|
45
|
+
private tryRun;
|
|
46
|
+
/** `--all` plus, by default, `--reflog`: an amended commit is still on disk. */
|
|
47
|
+
private historyArgs;
|
|
48
|
+
isShallow(): boolean;
|
|
49
|
+
refNames(): string[];
|
|
50
|
+
commits(): CommitMeta[];
|
|
51
|
+
/**
|
|
52
|
+
* `-m` is load-bearing: without it, a path introduced only by a merge commit
|
|
53
|
+
* ("evil merge") never appears in the output. See docs/decisions/0001.
|
|
54
|
+
*/
|
|
55
|
+
paths(): string[];
|
|
56
|
+
objects(): ObjectMeta[];
|
|
57
|
+
readObjects(shas: string[]): Map<string, Buffer>;
|
|
58
|
+
workingTreeFile(relativePath: string): string | null;
|
|
59
|
+
remoteFetchSpecs(): string[];
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* `git cat-file --batch` frames each object as
|
|
63
|
+
* "<sha> <type> <size>\n" + <size bytes> + "\n"
|
|
64
|
+
* and a missing object as "<sha> missing\n".
|
|
65
|
+
*/
|
|
66
|
+
export declare function parseBatch(raw: Buffer, into: Map<string, Buffer>): void;
|