circle-ir-ai 2.7.19 → 2.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/secret-scan/history-patterns.d.ts +30 -0
- package/dist/secret-scan/history-patterns.d.ts.map +1 -0
- package/dist/secret-scan/history-patterns.js +162 -0
- package/dist/secret-scan/history-patterns.js.map +1 -0
- package/dist/secret-scan/index.d.ts +14 -7
- package/dist/secret-scan/index.d.ts.map +1 -1
- package/dist/secret-scan/index.js +15 -8
- package/dist/secret-scan/index.js.map +1 -1
- package/dist/secret-scan/patterns.d.ts +14 -1
- package/dist/secret-scan/patterns.d.ts.map +1 -1
- package/dist/secret-scan/patterns.js +14 -1
- package/dist/secret-scan/patterns.js.map +1 -1
- package/dist/secret-scan/scanner.d.ts +29 -44
- package/dist/secret-scan/scanner.d.ts.map +1 -1
- package/dist/secret-scan/scanner.js +231 -210
- package/dist/secret-scan/scanner.js.map +1 -1
- package/dist/trust/index.d.ts +1 -1
- package/dist/trust/index.d.ts.map +1 -1
- package/dist/trust/index.js +1 -1
- package/dist/trust/index.js.map +1 -1
- package/dist/trust/passes/hardcoded-secrets.d.ts +11 -1
- package/dist/trust/passes/hardcoded-secrets.d.ts.map +1 -1
- package/dist/trust/passes/hardcoded-secrets.js +19 -5
- package/dist/trust/passes/hardcoded-secrets.js.map +1 -1
- package/dist/trust/passes/license-compliance.d.ts +60 -1
- package/dist/trust/passes/license-compliance.d.ts.map +1 -1
- package/dist/trust/passes/license-compliance.js +248 -2
- package/dist/trust/passes/license-compliance.js.map +1 -1
- package/package.json +2 -2
|
@@ -1,42 +1,92 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Secret Scanner Module
|
|
2
|
+
* Secret Scanner Module (Refactored)
|
|
3
3
|
*
|
|
4
|
-
*
|
|
4
|
+
* Architecture:
|
|
5
|
+
* - SAST detection: Delegates to circle-ir's ScanSecretsPass (no regex duplication)
|
|
6
|
+
* - Git history: Scans commits for secrets introduced historically (circle-ir-ai domain)
|
|
7
|
+
* - LLM verification: Reduces false positives via context-aware analysis
|
|
8
|
+
*
|
|
9
|
+
* This module consumes CircleIR findings rather than reimplementing SAST logic.
|
|
5
10
|
*/
|
|
6
11
|
import { execFileSync } from 'child_process';
|
|
7
12
|
import * as fs from 'fs';
|
|
8
13
|
import * as path from 'path';
|
|
9
14
|
import { minimatch } from 'minimatch';
|
|
10
|
-
import {
|
|
15
|
+
import { initAnalyzer, analyze, isAnalyzerInitialized, detectLanguage } from 'circle-ir';
|
|
16
|
+
// Minimal patterns for git history scanning only.
|
|
17
|
+
// Working-tree scanning uses circle-ir's ScanSecretsPass.
|
|
18
|
+
import { HISTORY_SCAN_PATTERNS } from './history-patterns.js';
|
|
19
|
+
/**
|
|
20
|
+
* Map circle-ir severity to our severity type
|
|
21
|
+
*/
|
|
22
|
+
function mapSeverity(severity) {
|
|
23
|
+
const map = {
|
|
24
|
+
critical: 'critical',
|
|
25
|
+
high: 'high',
|
|
26
|
+
medium: 'medium',
|
|
27
|
+
low: 'low',
|
|
28
|
+
info: 'low',
|
|
29
|
+
};
|
|
30
|
+
return map[severity.toLowerCase()] ?? 'medium';
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Map circle-ir finding to DetectedSecret
|
|
34
|
+
*/
|
|
35
|
+
function sastFindingToSecret(finding, fileContent) {
|
|
36
|
+
const lines = fileContent.split('\n');
|
|
37
|
+
const lineContent = lines[finding.line - 1] || '';
|
|
38
|
+
// Extract pattern info from evidence or rule_id
|
|
39
|
+
const evidence = finding.evidence || {};
|
|
40
|
+
const patternName = evidence.provider || finding.rule_id.replace('hardcoded-credential', 'Secret').replace(/-/g, ' ');
|
|
41
|
+
const match = evidence.match || '[redacted]';
|
|
42
|
+
// Determine category from evidence or pattern name
|
|
43
|
+
let category = 'generic';
|
|
44
|
+
if (evidence.provider) {
|
|
45
|
+
const provider = evidence.provider.toLowerCase();
|
|
46
|
+
if (provider.includes('aws'))
|
|
47
|
+
category = 'aws';
|
|
48
|
+
else if (provider.includes('github'))
|
|
49
|
+
category = 'github';
|
|
50
|
+
else if (provider.includes('stripe'))
|
|
51
|
+
category = 'stripe';
|
|
52
|
+
else if (provider.includes('openai'))
|
|
53
|
+
category = 'openai';
|
|
54
|
+
else if (provider.includes('anthropic'))
|
|
55
|
+
category = 'anthropic';
|
|
56
|
+
else if (provider.includes('slack'))
|
|
57
|
+
category = 'slack';
|
|
58
|
+
else if (provider.includes('google'))
|
|
59
|
+
category = 'gcp';
|
|
60
|
+
else if (provider.includes('jwt'))
|
|
61
|
+
category = 'jwt';
|
|
62
|
+
else if (provider.includes('pem') || provider.includes('private key'))
|
|
63
|
+
category = 'private-key';
|
|
64
|
+
else if (provider.includes('npm'))
|
|
65
|
+
category = 'npm';
|
|
66
|
+
}
|
|
67
|
+
else if (evidence.kind === 'entropy') {
|
|
68
|
+
category = 'high-entropy';
|
|
69
|
+
}
|
|
70
|
+
return {
|
|
71
|
+
patternId: finding.rule_id,
|
|
72
|
+
patternName,
|
|
73
|
+
file: finding.file,
|
|
74
|
+
line: finding.line,
|
|
75
|
+
column: 1,
|
|
76
|
+
match: redactSecret(match),
|
|
77
|
+
lineContent: truncateLine(lineContent),
|
|
78
|
+
severity: mapSeverity(finding.severity),
|
|
79
|
+
category,
|
|
80
|
+
presentInHead: true,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
11
83
|
/**
|
|
12
84
|
* Secret Scanner class
|
|
13
85
|
*/
|
|
14
86
|
export class SecretScanner {
|
|
15
|
-
patterns;
|
|
16
87
|
options;
|
|
17
88
|
constructor(options = {}) {
|
|
18
89
|
this.options = options;
|
|
19
|
-
this.patterns = this.selectPatterns(options);
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* Select patterns based on options
|
|
23
|
-
*/
|
|
24
|
-
selectPatterns(options) {
|
|
25
|
-
let patterns = options.patterns || SECRET_PATTERNS;
|
|
26
|
-
// Filter by category
|
|
27
|
-
if (options.includeCategories?.length) {
|
|
28
|
-
patterns = patterns.filter((p) => options.includeCategories.includes(p.category));
|
|
29
|
-
}
|
|
30
|
-
if (options.excludeCategories?.length) {
|
|
31
|
-
patterns = patterns.filter((p) => !options.excludeCategories.includes(p.category));
|
|
32
|
-
}
|
|
33
|
-
// Filter by severity
|
|
34
|
-
if (options.minSeverity) {
|
|
35
|
-
const severityOrder = ['low', 'medium', 'high', 'critical'];
|
|
36
|
-
const minIndex = severityOrder.indexOf(options.minSeverity);
|
|
37
|
-
patterns = patterns.filter((p) => severityOrder.indexOf(p.severity) >= minIndex);
|
|
38
|
-
}
|
|
39
|
-
return patterns;
|
|
40
90
|
}
|
|
41
91
|
/**
|
|
42
92
|
* Scan a directory for secrets
|
|
@@ -44,6 +94,10 @@ export class SecretScanner {
|
|
|
44
94
|
async scan(directory) {
|
|
45
95
|
const startTime = Date.now();
|
|
46
96
|
const secrets = [];
|
|
97
|
+
// Ensure analyzer is initialized
|
|
98
|
+
if (!isAnalyzerInitialized()) {
|
|
99
|
+
await initAnalyzer();
|
|
100
|
+
}
|
|
47
101
|
const progress = {
|
|
48
102
|
phase: 'indexing',
|
|
49
103
|
filesScanned: 0,
|
|
@@ -56,11 +110,11 @@ export class SecretScanner {
|
|
|
56
110
|
progress.totalFiles = files.length;
|
|
57
111
|
progress.phase = 'scanning-files';
|
|
58
112
|
this.options.onProgress?.(progress);
|
|
59
|
-
// Scan current files
|
|
113
|
+
// Scan current files using circle-ir SAST
|
|
60
114
|
for (const file of files) {
|
|
61
115
|
progress.currentFile = file;
|
|
62
116
|
this.options.onProgress?.(progress);
|
|
63
|
-
const fileSecrets = await this.
|
|
117
|
+
const fileSecrets = await this.scanFileWithCircleIR(file);
|
|
64
118
|
for (const secret of fileSecrets) {
|
|
65
119
|
secret.presentInHead = true;
|
|
66
120
|
secrets.push(secret);
|
|
@@ -74,19 +128,7 @@ export class SecretScanner {
|
|
|
74
128
|
if (this.options.scanHistory && this.isGitRepo(directory)) {
|
|
75
129
|
progress.phase = 'scanning-history';
|
|
76
130
|
const historySecrets = await this.scanGitHistory(directory, progress);
|
|
77
|
-
// Mark historical secrets as not present in HEAD if not already found
|
|
78
|
-
//
|
|
79
|
-
// #62: dedup compares `s.file === secret.file`. scanFile emits
|
|
80
|
-
// absolute paths (path.resolve, per #13) but scanDiff used to set
|
|
81
|
-
// `currentFile = line.slice(6)` from `+++ b/<relpath>`, leaving the
|
|
82
|
-
// history-side file as a relative path. The strict equality never
|
|
83
|
-
// matched, so every secret that lived in HEAD AND was added in a
|
|
84
|
-
// scanned commit appeared twice: once active (working-tree walk),
|
|
85
|
-
// once historical (dedup miss). `bySeverity` and `secrets.length`
|
|
86
|
-
// inflated 2×; `activeSecrets` stayed correct because
|
|
87
|
-
// `presentInHead = true` was set independently in the working-tree
|
|
88
|
-
// walk. Fixed by resolving currentFile in scanDiff against the
|
|
89
|
-
// repo directory before constructing DetectedSecret.
|
|
131
|
+
// Mark historical secrets as not present in HEAD if not already found
|
|
90
132
|
for (const secret of historySecrets) {
|
|
91
133
|
const existsInHead = secrets.some((s) => s.file === secret.file &&
|
|
92
134
|
s.patternId === secret.patternId &&
|
|
@@ -98,6 +140,26 @@ export class SecretScanner {
|
|
|
98
140
|
}
|
|
99
141
|
commitsScanned = progress.commitsScanned || 0;
|
|
100
142
|
}
|
|
143
|
+
// LLM verification (optional)
|
|
144
|
+
if (this.options.llmVerify && secrets.length > 0) {
|
|
145
|
+
progress.phase = 'verifying';
|
|
146
|
+
this.options.onProgress?.(progress);
|
|
147
|
+
await this.llmVerifySecrets(secrets, directory);
|
|
148
|
+
}
|
|
149
|
+
// Apply severity filter
|
|
150
|
+
let filteredSecrets = secrets;
|
|
151
|
+
if (this.options.minSeverity) {
|
|
152
|
+
const severityOrder = ['low', 'medium', 'high', 'critical'];
|
|
153
|
+
const minIndex = severityOrder.indexOf(this.options.minSeverity);
|
|
154
|
+
filteredSecrets = secrets.filter((s) => severityOrder.indexOf(s.severity) >= minIndex);
|
|
155
|
+
}
|
|
156
|
+
// Apply category filters
|
|
157
|
+
if (this.options.includeCategories?.length) {
|
|
158
|
+
filteredSecrets = filteredSecrets.filter((s) => this.options.includeCategories.includes(s.category));
|
|
159
|
+
}
|
|
160
|
+
if (this.options.excludeCategories?.length) {
|
|
161
|
+
filteredSecrets = filteredSecrets.filter((s) => !this.options.excludeCategories.includes(s.category));
|
|
162
|
+
}
|
|
101
163
|
progress.phase = 'complete';
|
|
102
164
|
this.options.onProgress?.(progress);
|
|
103
165
|
// Calculate statistics
|
|
@@ -108,19 +170,19 @@ export class SecretScanner {
|
|
|
108
170
|
low: 0,
|
|
109
171
|
};
|
|
110
172
|
const byCategory = {};
|
|
111
|
-
for (const secret of
|
|
173
|
+
for (const secret of filteredSecrets) {
|
|
112
174
|
bySeverity[secret.severity]++;
|
|
113
175
|
byCategory[secret.category] = (byCategory[secret.category] || 0) + 1;
|
|
114
176
|
}
|
|
115
|
-
const activeSecrets =
|
|
116
|
-
const historicalSecrets =
|
|
177
|
+
const activeSecrets = filteredSecrets.filter((s) => s.presentInHead).length;
|
|
178
|
+
const historicalSecrets = filteredSecrets.filter((s) => !s.presentInHead).length;
|
|
117
179
|
// Generate .gitignore recommendations
|
|
118
|
-
const gitignoreRecommendations = this.generateGitignoreRecommendations(
|
|
180
|
+
const gitignoreRecommendations = this.generateGitignoreRecommendations(filteredSecrets);
|
|
119
181
|
return {
|
|
120
182
|
directory,
|
|
121
183
|
filesScanned: files.length,
|
|
122
184
|
commitsScanned,
|
|
123
|
-
secrets,
|
|
185
|
+
secrets: filteredSecrets,
|
|
124
186
|
bySeverity,
|
|
125
187
|
byCategory,
|
|
126
188
|
activeSecrets,
|
|
@@ -130,112 +192,33 @@ export class SecretScanner {
|
|
|
130
192
|
};
|
|
131
193
|
}
|
|
132
194
|
/**
|
|
133
|
-
* Scan a single file
|
|
195
|
+
* Scan a single file using circle-ir's ScanSecretsPass
|
|
134
196
|
*/
|
|
135
|
-
async
|
|
197
|
+
async scanFileWithCircleIR(filePath) {
|
|
136
198
|
const secrets = [];
|
|
137
199
|
try {
|
|
138
200
|
const content = fs.readFileSync(filePath, 'utf-8');
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
201
|
+
const language = detectLanguage(filePath);
|
|
202
|
+
if (!language) {
|
|
203
|
+
return secrets; // Unsupported language
|
|
204
|
+
}
|
|
205
|
+
const result = await analyze(content, path.basename(filePath), language);
|
|
206
|
+
// Extract hardcoded-credential findings from circle-ir
|
|
207
|
+
const credentialFindings = (result.findings ?? []).filter((f) => f.rule_id?.includes('hardcoded-credential'));
|
|
208
|
+
for (const finding of credentialFindings) {
|
|
209
|
+
const secret = sastFindingToSecret(finding, content);
|
|
210
|
+
secret.file = path.resolve(filePath); // Absolute path
|
|
211
|
+
secrets.push(secret);
|
|
147
212
|
}
|
|
148
213
|
}
|
|
149
214
|
catch {
|
|
150
|
-
// Skip files that can't be read
|
|
151
|
-
}
|
|
152
|
-
return secrets;
|
|
153
|
-
}
|
|
154
|
-
/**
|
|
155
|
-
* Scan a single line for secrets
|
|
156
|
-
*/
|
|
157
|
-
scanLine(line, file, lineNum, commit, author, commitDate) {
|
|
158
|
-
const secrets = [];
|
|
159
|
-
// Quick keyword pre-filter
|
|
160
|
-
const lineLower = line.toLowerCase();
|
|
161
|
-
for (const pattern of this.patterns) {
|
|
162
|
-
// Skip if no keywords match (optimization)
|
|
163
|
-
if (pattern.keywords?.length) {
|
|
164
|
-
const hasKeyword = pattern.keywords.some((k) => lineLower.includes(k.toLowerCase()));
|
|
165
|
-
if (!hasKeyword)
|
|
166
|
-
continue;
|
|
167
|
-
}
|
|
168
|
-
// Reset regex lastIndex for global patterns
|
|
169
|
-
pattern.pattern.lastIndex = 0;
|
|
170
|
-
let match;
|
|
171
|
-
while ((match = pattern.pattern.exec(line)) !== null) {
|
|
172
|
-
const matchedText = match[0];
|
|
173
|
-
// Check false positive patterns
|
|
174
|
-
if (pattern.falsePositivePatterns?.length) {
|
|
175
|
-
const isFalsePositive = pattern.falsePositivePatterns.some((fp) => fp.test(line));
|
|
176
|
-
if (isFalsePositive)
|
|
177
|
-
continue;
|
|
178
|
-
}
|
|
179
|
-
// Run validator if present
|
|
180
|
-
if (pattern.validator && !pattern.validator(matchedText)) {
|
|
181
|
-
continue;
|
|
182
|
-
}
|
|
183
|
-
// Context-aware false positive detection:
|
|
184
|
-
// 1. Match inside a regex literal (pattern definition, not real secret)
|
|
185
|
-
if (this.isInsideRegexLiteral(line, match.index, match.index + matchedText.length)) {
|
|
186
|
-
continue;
|
|
187
|
-
}
|
|
188
|
-
// 2. Match is a truncated example (e.g. "Bearer eyJ...", "sk-...")
|
|
189
|
-
if (/\.{2,}["'`]?\s*[,;)}\]]?\s*$/.test(line.slice(match.index)) ||
|
|
190
|
-
/\.{2,}["'`]/.test(matchedText) ||
|
|
191
|
-
/\.{3}/.test(line.slice(match.index, match.index + matchedText.length + 5))) {
|
|
192
|
-
continue;
|
|
193
|
-
}
|
|
194
|
-
// 3. Line is in a documentation/example context (comment with "Examples:", "e.g.")
|
|
195
|
-
if (/^\s*(?:\*|\/\/|#)\s*(?:examples?:|e\.g\.|i\.e\.|sample|usage)/i.test(line)) {
|
|
196
|
-
continue;
|
|
197
|
-
}
|
|
198
|
-
secrets.push({
|
|
199
|
-
patternId: pattern.id,
|
|
200
|
-
patternName: pattern.name,
|
|
201
|
-
file,
|
|
202
|
-
line: lineNum,
|
|
203
|
-
column: match.index + 1,
|
|
204
|
-
match: this.redactSecret(matchedText),
|
|
205
|
-
lineContent: this.truncateLine(line),
|
|
206
|
-
severity: pattern.severity,
|
|
207
|
-
category: pattern.category,
|
|
208
|
-
commit,
|
|
209
|
-
author,
|
|
210
|
-
commitDate,
|
|
211
|
-
presentInHead: false, // Will be updated later
|
|
212
|
-
});
|
|
213
|
-
}
|
|
215
|
+
// Skip files that can't be read or analyzed
|
|
214
216
|
}
|
|
215
217
|
return secrets;
|
|
216
218
|
}
|
|
217
|
-
/**
|
|
218
|
-
* Check if a match position falls inside a regex literal (/.../).
|
|
219
|
-
* Looks for unescaped `/` delimiters surrounding the match range.
|
|
220
|
-
*/
|
|
221
|
-
isInsideRegexLiteral(line, matchStart, matchEnd) {
|
|
222
|
-
// Find regex literals in the line: look for /pattern/flags
|
|
223
|
-
// A regex literal starts with / (not preceded by a word char or /) and
|
|
224
|
-
// ends with / followed by optional flags [gimsuy]
|
|
225
|
-
const regexLiteralPattern = /(?:^|[=(:,;!&|?+\-~^%<>[\s])(\/.+?\/[gimsuy]*)/g;
|
|
226
|
-
let m;
|
|
227
|
-
while ((m = regexLiteralPattern.exec(line)) !== null) {
|
|
228
|
-
const regexStart = m.index + m[0].indexOf(m[1]);
|
|
229
|
-
const regexEnd = regexStart + m[1].length;
|
|
230
|
-
// If the secret match falls inside this regex literal, it's a pattern definition
|
|
231
|
-
if (matchStart >= regexStart && matchEnd <= regexEnd) {
|
|
232
|
-
return true;
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
return false;
|
|
236
|
-
}
|
|
237
219
|
/**
|
|
238
220
|
* Scan git history for secrets
|
|
221
|
+
* Uses minimal patterns since we can't run circle-ir on diffs
|
|
239
222
|
*/
|
|
240
223
|
async scanGitHistory(directory, progress) {
|
|
241
224
|
const secrets = [];
|
|
@@ -252,18 +235,8 @@ export class SecretScanner {
|
|
|
252
235
|
// Get commit info
|
|
253
236
|
const commitInfo = execFileSync('git', ['-C', directory, 'log', '-1', '--format=%an|%aI', commit], { encoding: 'utf-8' }).trim();
|
|
254
237
|
const [author, commitDate] = commitInfo.split('|');
|
|
255
|
-
// Get diff for this commit.
|
|
256
|
-
// `--root` is required so the diff against the empty tree is
|
|
257
|
-
// emitted for the repo's root commit — otherwise `git diff-tree`
|
|
258
|
-
// returns empty for the initial commit and any secret added in
|
|
259
|
-
// commit #1 is silently invisible to the scanner. Discovered
|
|
260
|
-
// while writing the #60 history-exclude test.
|
|
261
238
|
try {
|
|
262
239
|
const diff = execFileSync('git', ['-C', directory, 'diff-tree', '--root', '--no-commit-id', '-r', '-p', commit], { encoding: 'utf-8', maxBuffer: 50 * 1024 * 1024 });
|
|
263
|
-
// Parse diff and scan for secrets.
|
|
264
|
-
// #62: pass `directory` so scanDiff can emit absolute file
|
|
265
|
-
// paths matching scanFile's `path.resolve()` output. Without
|
|
266
|
-
// this, dedup against working-tree secrets fails.
|
|
267
240
|
const diffSecrets = this.scanDiff(diff, commit, author, commitDate, directory);
|
|
268
241
|
secrets.push(...diffSecrets);
|
|
269
242
|
}
|
|
@@ -281,37 +254,23 @@ export class SecretScanner {
|
|
|
281
254
|
return secrets;
|
|
282
255
|
}
|
|
283
256
|
/**
|
|
284
|
-
* Scan a git diff for secrets
|
|
285
|
-
*
|
|
286
|
-
* #62: `repoDir` is required so emitted DetectedSecret.file matches
|
|
287
|
-
* scanFile's `path.resolve()` output and dedup at the caller can find
|
|
288
|
-
* HEAD↔history matches. The relative `currentFile` is preserved for
|
|
289
|
-
* exclude-glob matching (`isPathExcluded` expects repo-relative).
|
|
257
|
+
* Scan a git diff for secrets using minimal history patterns
|
|
290
258
|
*/
|
|
291
259
|
scanDiff(diff, commit, author, commitDate, repoDir) {
|
|
292
260
|
const secrets = [];
|
|
293
261
|
let currentFile = '';
|
|
294
262
|
let lineNum = 0;
|
|
295
|
-
// #60: when the current file is excluded by user globs or built-in
|
|
296
|
-
// regex skips, drop every line of that diff section until the next
|
|
297
|
-
// `+++ b/<path>` marker. Working-tree walk applied excludes; history
|
|
298
|
-
// walk did not, so e.g. Cargo.lock secrets would show up as
|
|
299
|
-
// "Status: Historical" even with `--exclude Cargo.lock`.
|
|
300
263
|
let currentFileExcluded = false;
|
|
301
264
|
const lines = diff.split('\n');
|
|
302
265
|
for (const line of lines) {
|
|
303
|
-
// Track current file
|
|
304
266
|
if (line.startsWith('+++ b/')) {
|
|
305
267
|
currentFile = line.slice(6);
|
|
306
268
|
lineNum = 0;
|
|
307
269
|
currentFileExcluded = this.isPathExcluded(currentFile);
|
|
308
270
|
continue;
|
|
309
271
|
}
|
|
310
|
-
// #60: while the active file is excluded, skip every line —
|
|
311
|
-
// including hunk headers — until the next `+++ b/` marker.
|
|
312
272
|
if (currentFileExcluded)
|
|
313
273
|
continue;
|
|
314
|
-
// Track line numbers from hunk headers
|
|
315
274
|
const hunkMatch = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)/);
|
|
316
275
|
if (hunkMatch) {
|
|
317
276
|
lineNum = parseInt(hunkMatch[1], 10) - 1;
|
|
@@ -320,11 +279,9 @@ export class SecretScanner {
|
|
|
320
279
|
// Only scan added lines
|
|
321
280
|
if (line.startsWith('+') && !line.startsWith('+++')) {
|
|
322
281
|
lineNum++;
|
|
323
|
-
const content = line.slice(1);
|
|
324
|
-
// #62: emit absolute path so dedup against the working-tree
|
|
325
|
-
// secrets (which are absolute via path.resolve) matches.
|
|
282
|
+
const content = line.slice(1);
|
|
326
283
|
const absoluteFile = path.resolve(repoDir, currentFile);
|
|
327
|
-
const lineSecrets = this.
|
|
284
|
+
const lineSecrets = this.scanLineWithPatterns(content, absoluteFile, lineNum, commit, author, commitDate);
|
|
328
285
|
secrets.push(...lineSecrets);
|
|
329
286
|
}
|
|
330
287
|
else if (!line.startsWith('-')) {
|
|
@@ -334,12 +291,88 @@ export class SecretScanner {
|
|
|
334
291
|
return secrets;
|
|
335
292
|
}
|
|
336
293
|
/**
|
|
337
|
-
*
|
|
338
|
-
*
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
294
|
+
* Scan a single line using minimal history patterns
|
|
295
|
+
* This is only used for git history scanning where we can't use circle-ir
|
|
296
|
+
*/
|
|
297
|
+
scanLineWithPatterns(line, file, lineNum, commit, author, commitDate) {
|
|
298
|
+
const secrets = [];
|
|
299
|
+
for (const pattern of HISTORY_SCAN_PATTERNS) {
|
|
300
|
+
pattern.pattern.lastIndex = 0;
|
|
301
|
+
let match;
|
|
302
|
+
while ((match = pattern.pattern.exec(line)) !== null) {
|
|
303
|
+
const matchedText = match[0];
|
|
304
|
+
// Skip false positives
|
|
305
|
+
if (pattern.falsePositivePatterns?.length) {
|
|
306
|
+
const isFalsePositive = pattern.falsePositivePatterns.some((fp) => fp.test(line));
|
|
307
|
+
if (isFalsePositive)
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
if (pattern.validator && !pattern.validator(matchedText)) {
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
secrets.push({
|
|
314
|
+
patternId: pattern.id,
|
|
315
|
+
patternName: pattern.name,
|
|
316
|
+
file,
|
|
317
|
+
line: lineNum,
|
|
318
|
+
column: match.index + 1,
|
|
319
|
+
match: redactSecret(matchedText),
|
|
320
|
+
lineContent: truncateLine(line),
|
|
321
|
+
severity: pattern.severity,
|
|
322
|
+
category: pattern.category,
|
|
323
|
+
commit,
|
|
324
|
+
author,
|
|
325
|
+
commitDate,
|
|
326
|
+
presentInHead: false,
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return secrets;
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* LLM verification to reduce false positives
|
|
334
|
+
*/
|
|
335
|
+
async llmVerifySecrets(secrets, _directory) {
|
|
336
|
+
// Import LLM client dynamically to avoid circular deps
|
|
337
|
+
try {
|
|
338
|
+
const { AxLLMClient } = await import('../llm/ax-client.js');
|
|
339
|
+
const { getDefaultLLMConfig } = await import('../llm/config.js');
|
|
340
|
+
const config = getDefaultLLMConfig();
|
|
341
|
+
const client = new AxLLMClient(config);
|
|
342
|
+
// Batch verify entropy-based findings (most likely FPs)
|
|
343
|
+
const entropySecrets = secrets.filter(s => s.category === 'high-entropy');
|
|
344
|
+
const systemPrompt = 'You are a security expert. Analyze code for hardcoded secrets. Respond with JSON only.';
|
|
345
|
+
for (const secret of entropySecrets) {
|
|
346
|
+
try {
|
|
347
|
+
const userPrompt = `Analyze this code line and determine if it contains a real hardcoded secret or credential.
|
|
348
|
+
|
|
349
|
+
Line: ${secret.lineContent}
|
|
350
|
+
Context: File ${path.basename(secret.file)}, line ${secret.line}
|
|
351
|
+
|
|
352
|
+
Respond with JSON: {"isSecret": true/false, "confidence": 0.0-1.0, "reason": "brief explanation"}
|
|
353
|
+
|
|
354
|
+
Consider:
|
|
355
|
+
- Is this a placeholder, example, or test value?
|
|
356
|
+
- Is this a hash, UUID, or non-sensitive identifier?
|
|
357
|
+
- Is this an actual API key, token, or credential?`;
|
|
358
|
+
const response = await client.chatJSON(systemPrompt, userPrompt, 'verification');
|
|
359
|
+
if (response) {
|
|
360
|
+
secret.llmVerified = response.isSecret;
|
|
361
|
+
secret.llmConfidence = response.confidence;
|
|
362
|
+
}
|
|
363
|
+
}
|
|
364
|
+
catch {
|
|
365
|
+
// LLM verification failed, keep the finding
|
|
366
|
+
secret.llmVerified = undefined;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
catch {
|
|
371
|
+
// LLM not available, skip verification
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* Built-in path-skip patterns
|
|
343
376
|
*/
|
|
344
377
|
static BUILTIN_EXCLUDE_PATTERNS = [
|
|
345
378
|
/node_modules/,
|
|
@@ -371,9 +404,7 @@ export class SecretScanner {
|
|
|
371
404
|
/\.wasm$/i,
|
|
372
405
|
];
|
|
373
406
|
/**
|
|
374
|
-
*
|
|
375
|
-
* regex excludes + user-supplied include/exclude minimatch globs.
|
|
376
|
-
* Used by both the working-tree walk and the git-history diff parser.
|
|
407
|
+
* Check if path should be excluded
|
|
377
408
|
*/
|
|
378
409
|
isPathExcluded(relativePath) {
|
|
379
410
|
if (SecretScanner.BUILTIN_EXCLUDE_PATTERNS.some((p) => p.test(relativePath))) {
|
|
@@ -395,10 +426,6 @@ export class SecretScanner {
|
|
|
395
426
|
getFiles(directory) {
|
|
396
427
|
const files = [];
|
|
397
428
|
const excludePatterns = SecretScanner.BUILTIN_EXCLUDE_PATTERNS;
|
|
398
|
-
// #18: user-provided include/exclude patterns are globs (matching the
|
|
399
|
-
// shape used by `cognium.config.json` and the CLI's `--include` /
|
|
400
|
-
// `--exclude` flags). Match via minimatch — passing them to RegExp
|
|
401
|
-
// crashed on `**` ("nothing to repeat").
|
|
402
429
|
const userExcludeGlobs = this.options.excludeFiles ?? [];
|
|
403
430
|
const userIncludeGlobs = this.options.includeFiles ?? [];
|
|
404
431
|
const walk = (dir) => {
|
|
@@ -407,11 +434,9 @@ export class SecretScanner {
|
|
|
407
434
|
for (const entry of entries) {
|
|
408
435
|
const fullPath = path.join(dir, entry.name);
|
|
409
436
|
const relativePath = path.relative(directory, fullPath);
|
|
410
|
-
// Skip excluded patterns
|
|
411
437
|
if (excludePatterns.some((p) => p.test(relativePath))) {
|
|
412
438
|
continue;
|
|
413
439
|
}
|
|
414
|
-
// Apply user exclude globs
|
|
415
440
|
if (userExcludeGlobs.length) {
|
|
416
441
|
if (userExcludeGlobs.some((g) => minimatch(relativePath, g, { dot: true })))
|
|
417
442
|
continue;
|
|
@@ -420,7 +445,6 @@ export class SecretScanner {
|
|
|
420
445
|
walk(fullPath);
|
|
421
446
|
}
|
|
422
447
|
else if (entry.isFile()) {
|
|
423
|
-
// Apply user include globs
|
|
424
448
|
if (userIncludeGlobs.length) {
|
|
425
449
|
if (!userIncludeGlobs.some((g) => minimatch(relativePath, g, { dot: true })))
|
|
426
450
|
continue;
|
|
@@ -451,26 +475,6 @@ export class SecretScanner {
|
|
|
451
475
|
return false;
|
|
452
476
|
}
|
|
453
477
|
}
|
|
454
|
-
/**
|
|
455
|
-
* Redact a secret for safe display
|
|
456
|
-
*/
|
|
457
|
-
redactSecret(secret) {
|
|
458
|
-
if (secret.length <= 8) {
|
|
459
|
-
return '*'.repeat(secret.length);
|
|
460
|
-
}
|
|
461
|
-
const visibleChars = Math.min(4, Math.floor(secret.length / 4));
|
|
462
|
-
return (secret.slice(0, visibleChars) +
|
|
463
|
-
'*'.repeat(secret.length - visibleChars * 2) +
|
|
464
|
-
secret.slice(-visibleChars));
|
|
465
|
-
}
|
|
466
|
-
/**
|
|
467
|
-
* Truncate long lines
|
|
468
|
-
*/
|
|
469
|
-
truncateLine(line, maxLength = 200) {
|
|
470
|
-
if (line.length <= maxLength)
|
|
471
|
-
return line;
|
|
472
|
-
return line.slice(0, maxLength) + '...';
|
|
473
|
-
}
|
|
474
478
|
/**
|
|
475
479
|
* Generate .gitignore recommendations
|
|
476
480
|
*/
|
|
@@ -478,7 +482,6 @@ export class SecretScanner {
|
|
|
478
482
|
const recommendations = new Set();
|
|
479
483
|
for (const secret of secrets) {
|
|
480
484
|
const file = secret.file;
|
|
481
|
-
// Common sensitive file patterns
|
|
482
485
|
if (/\.env($|\.)/.test(file)) {
|
|
483
486
|
recommendations.add('.env*');
|
|
484
487
|
recommendations.add('!.env.example');
|
|
@@ -514,7 +517,6 @@ export class SecretScanner {
|
|
|
514
517
|
recommendations.add('*.pfx');
|
|
515
518
|
}
|
|
516
519
|
}
|
|
517
|
-
// Always recommend common patterns
|
|
518
520
|
recommendations.add('.env');
|
|
519
521
|
recommendations.add('.env.local');
|
|
520
522
|
recommendations.add('*.pem');
|
|
@@ -522,6 +524,26 @@ export class SecretScanner {
|
|
|
522
524
|
return [...recommendations].sort();
|
|
523
525
|
}
|
|
524
526
|
}
|
|
527
|
+
/**
|
|
528
|
+
* Redact a secret for safe display
|
|
529
|
+
*/
|
|
530
|
+
function redactSecret(secret) {
|
|
531
|
+
if (secret.length <= 8) {
|
|
532
|
+
return '*'.repeat(secret.length);
|
|
533
|
+
}
|
|
534
|
+
const visibleChars = Math.min(4, Math.floor(secret.length / 4));
|
|
535
|
+
return (secret.slice(0, visibleChars) +
|
|
536
|
+
'*'.repeat(secret.length - visibleChars * 2) +
|
|
537
|
+
secret.slice(-visibleChars));
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Truncate long lines
|
|
541
|
+
*/
|
|
542
|
+
function truncateLine(line, maxLength = 200) {
|
|
543
|
+
if (line.length <= maxLength)
|
|
544
|
+
return line;
|
|
545
|
+
return line.slice(0, maxLength) + '...';
|
|
546
|
+
}
|
|
525
547
|
/**
|
|
526
548
|
* Scan a directory for secrets (convenience function)
|
|
527
549
|
*/
|
|
@@ -552,7 +574,6 @@ export function formatSecretReport(result) {
|
|
|
552
574
|
lines.push(`Commits Scanned: ${result.commitsScanned}`);
|
|
553
575
|
lines.push(`Duration: ${result.durationMs}ms`);
|
|
554
576
|
lines.push('');
|
|
555
|
-
// Summary
|
|
556
577
|
lines.push('-'.repeat(40));
|
|
557
578
|
lines.push('SUMMARY');
|
|
558
579
|
lines.push('-'.repeat(40));
|
|
@@ -571,12 +592,10 @@ export function formatSecretReport(result) {
|
|
|
571
592
|
lines.push(` ${category}: ${count}`);
|
|
572
593
|
}
|
|
573
594
|
lines.push('');
|
|
574
|
-
// Detailed findings
|
|
575
595
|
if (result.secrets.length > 0) {
|
|
576
596
|
lines.push('-'.repeat(40));
|
|
577
597
|
lines.push('FINDINGS');
|
|
578
598
|
lines.push('-'.repeat(40));
|
|
579
|
-
// Group by severity
|
|
580
599
|
const grouped = new Map();
|
|
581
600
|
for (const secret of result.secrets) {
|
|
582
601
|
const list = grouped.get(secret.severity) || [];
|
|
@@ -600,10 +619,12 @@ export function formatSecretReport(result) {
|
|
|
600
619
|
lines.push(` Date: ${secret.commitDate}`);
|
|
601
620
|
}
|
|
602
621
|
lines.push(` Status: ${secret.presentInHead ? 'ACTIVE' : 'Historical'}`);
|
|
622
|
+
if (secret.llmVerified !== undefined) {
|
|
623
|
+
lines.push(` LLM Verified: ${secret.llmVerified} (confidence: ${secret.llmConfidence?.toFixed(2)})`);
|
|
624
|
+
}
|
|
603
625
|
}
|
|
604
626
|
}
|
|
605
627
|
}
|
|
606
|
-
// .gitignore recommendations
|
|
607
628
|
if (result.gitignoreRecommendations.length > 0) {
|
|
608
629
|
lines.push('');
|
|
609
630
|
lines.push('-'.repeat(40));
|