@lornatang/dsh-codex-security 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/docs/ARCHITECTURE.md +59 -59
- package/docs/ARTIFACTS.md +49 -49
- package/docs/CODEX-COMPATIBILITY.md +17 -17
- package/docs/SPEC.md +43 -43
- package/docs/TOOLS.md +364 -364
- package/docs/VALIDATION.md +33 -33
- package/docs/superpowers/plans/2026-11-16-deep-scan.md +71 -71
- package/docs/superpowers/specs/2026-11-16-deep-scan-design.md +31 -31
- package/lib/client.js +6 -1
- package/lib/constants.js +1 -1
- package/lib/deep-coordinator.js +103 -14
- package/lib/deep-worker.js +4 -1
- package/lib/git.js +44 -8
- package/lib/service.js +91 -52
- package/lib/store.js +93 -32
- package/lib/target.js +71 -13
- package/lib/tools.js +47 -1
- package/lib/types/constants.d.ts +1 -1
- package/lib/types/deep-coordinator.d.ts +4 -0
- package/lib/types/git.d.ts +4 -0
- package/lib/types/index.d.ts +1 -1
- package/lib/types/service.d.ts +5 -3
- package/lib/types/store.d.ts +11 -2
- package/lib/types/target.d.ts +15 -1
- package/lib/types/util.d.ts +1 -0
- package/lib/util.js +18 -15
- package/package.json +1 -1
- package/schemas/coverage.schema.json +136 -136
- package/skills/README.md +122 -122
- package/skills/assess-patch-risk/SKILL.md +10 -10
- package/skills/attack-path-analysis/SKILL.md +10 -10
- package/skills/deep-security-scan/SKILL.md +15 -15
- package/skills/define-security-policy/SKILL.md +10 -10
- package/skills/finding-discovery/SKILL.md +10 -10
- package/skills/fix-finding/SKILL.md +14 -14
- package/skills/propose-security-hardening/SKILL.md +10 -10
- package/skills/track-findings/SKILL.md +12 -12
- package/skills/validation/SKILL.md +10 -10
- package/skills/verify-fix/SKILL.md +14 -14
package/lib/git.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { execFile } from 'node:child_process';
|
|
1
|
+
import { execFile, spawn } from 'node:child_process';
|
|
2
2
|
import { promisify } from 'node:util';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
4
|
import { createReadStream } from 'node:fs';
|
|
@@ -62,27 +62,27 @@ export async function trackedFiles(root, signal) {
|
|
|
62
62
|
return result.stdout.split('\u0000').filter(Boolean);
|
|
63
63
|
}
|
|
64
64
|
export async function changedFiles(root, options) {
|
|
65
|
-
const args = ['diff', '--name-status', '-z', '--find-renames'];
|
|
65
|
+
const args = ['--no-replace-objects', 'diff', '--name-status', '-z', '--find-renames'];
|
|
66
66
|
if (options.staged)
|
|
67
67
|
args.push('--cached');
|
|
68
68
|
if (options.base) {
|
|
69
|
-
const resolved = await resolveRevision(root, options.base);
|
|
69
|
+
const resolved = await resolveRevision(root, options.base, options.signal);
|
|
70
70
|
if (!resolved)
|
|
71
71
|
throw new Error('codex-security: invalid diff base: ' + options.base);
|
|
72
72
|
if (options.workingTree) {
|
|
73
|
-
const mergeBase = await gitString(root, ['merge-base', resolved, 'HEAD']);
|
|
73
|
+
const mergeBase = await gitString(root, ['--no-replace-objects', 'merge-base', resolved, options.head ?? 'HEAD'], undefined, options.signal);
|
|
74
74
|
if (!mergeBase)
|
|
75
75
|
throw new Error('codex-security: cannot resolve merge base: ' + options.base);
|
|
76
76
|
args.push(mergeBase);
|
|
77
77
|
}
|
|
78
78
|
else {
|
|
79
|
-
args.push(resolved + '...HEAD');
|
|
79
|
+
args.push(resolved + '...' + (options.head ?? 'HEAD'));
|
|
80
80
|
}
|
|
81
81
|
}
|
|
82
82
|
else if (!options.staged) {
|
|
83
|
-
args.push('HEAD');
|
|
83
|
+
args.push(options.head ?? 'HEAD');
|
|
84
84
|
}
|
|
85
|
-
const result = await git(root, args);
|
|
85
|
+
const result = await git(root, [...args, '--'], undefined, options.signal);
|
|
86
86
|
if (!result.ok)
|
|
87
87
|
throw new Error('codex-security: git diff failed: ' + result.stderr.trim());
|
|
88
88
|
const parts = result.stdout.split('\u0000').filter(Boolean);
|
|
@@ -104,13 +104,49 @@ export async function changedFiles(root, options) {
|
|
|
104
104
|
}
|
|
105
105
|
if (options.workingTree && !options.staged) {
|
|
106
106
|
const seen = new Set(files.map((file) => file.path));
|
|
107
|
-
for (const entry of await statusEntries(root)) {
|
|
107
|
+
for (const entry of await statusEntries(root, options.signal)) {
|
|
108
108
|
if (entry.code === '??' && !seen.has(entry.path))
|
|
109
109
|
files.push({ status: 'A', path: entry.path });
|
|
110
110
|
}
|
|
111
111
|
}
|
|
112
112
|
return files;
|
|
113
113
|
}
|
|
114
|
+
/** Read a bounded prefix of a regular file from the exact commit. */
|
|
115
|
+
export async function revisionFilePrefix(root, revision, path, limit) {
|
|
116
|
+
if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(revision) || path.includes('\0') || !Number.isSafeInteger(limit) || limit <= 0)
|
|
117
|
+
return undefined;
|
|
118
|
+
const entry = await git(root, ['--no-replace-objects', '--literal-pathspecs', 'ls-tree', '-z', '--full-tree', revision, '--', path]);
|
|
119
|
+
const match = /^100(?:644|755) blob ([a-f0-9]{40}|[a-f0-9]{64})\t/.exec(entry.stdout);
|
|
120
|
+
if (!entry.ok || !match || entry.stdout.slice(match[0].length) !== path + '\0')
|
|
121
|
+
return undefined;
|
|
122
|
+
const child = spawn('git', ['--no-replace-objects', 'cat-file', 'blob', match[1]], {
|
|
123
|
+
cwd: root, timeout: 60000, windowsHide: true, stdio: ['ignore', 'pipe', 'ignore'],
|
|
124
|
+
});
|
|
125
|
+
const exited = new Promise((resolve) => {
|
|
126
|
+
child.once('error', () => resolve(null));
|
|
127
|
+
child.once('close', resolve);
|
|
128
|
+
});
|
|
129
|
+
const chunks = [];
|
|
130
|
+
let size = 0;
|
|
131
|
+
try {
|
|
132
|
+
for await (const chunk of child.stdout) {
|
|
133
|
+
const prefix = chunk.subarray(0, limit - size);
|
|
134
|
+
chunks.push(prefix);
|
|
135
|
+
size += prefix.length;
|
|
136
|
+
if (size === limit) {
|
|
137
|
+
child.kill();
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
const code = await exited;
|
|
142
|
+
return size === limit || code === 0 ? Buffer.concat(chunks, size) : undefined;
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
child.kill();
|
|
146
|
+
await exited;
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
114
150
|
export async function fileDigest(path, signal) {
|
|
115
151
|
try {
|
|
116
152
|
const hash = createHash('sha256');
|
package/lib/service.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { lstat, readFile } from 'node:fs/promises';
|
|
1
|
+
import { lstat, mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
2
2
|
import { dirname, join, resolve as resolvePath } from 'node:path';
|
|
3
3
|
import { DEFAULT_CONFIG } from './config.js';
|
|
4
4
|
import { PACKAGE_NAME, PLUGIN_NAME, PRODUCER, SKILL_NAMES, VERSION } from './constants.js';
|
|
@@ -6,9 +6,9 @@ import { buildCoverage, buildFindingsDocument, fingerprintFor, normalizeAnchor,
|
|
|
6
6
|
import { gitString } from './git.js';
|
|
7
7
|
import { FINDING_STATUSES, SEVERITIES } from './types.js';
|
|
8
8
|
import { ScanStore, defaultStateDir } from './store.js';
|
|
9
|
-
import { readEvidenceLine, resolveTarget } from './target.js';
|
|
9
|
+
import { readEvidenceLine, readEvidenceLines, resolveTarget } from './target.js';
|
|
10
10
|
import { assertArtifactContract } from './schema.js';
|
|
11
|
-
import { canonicalProspectivePath, ensureDir, isWithin, joinWithin, newId, normalizeLine, nowIso, readFullText, readJson, sanitizeRelative, sha256, truncate, writeFileAtomic, writeJsonAtomic } from './util.js';
|
|
11
|
+
import { canonicalProspectivePath, ensureDir, isWithin, joinWithin, newId, normalizeLine, nowIso, readFullText, readJson, renameFileAtomic, sanitizeRelative, sha256, truncate, writeFileAtomic, writeJsonAtomic } from './util.js';
|
|
12
12
|
import { createExportValidator, writeSecureExport } from './secure-export.js';
|
|
13
13
|
import { acquireStateLock } from './state-lock.js';
|
|
14
14
|
import { WorkerRegistry } from './deep-worker.js';
|
|
@@ -511,6 +511,7 @@ export class SecurityService {
|
|
|
511
511
|
validateFindingEvidence(deduped);
|
|
512
512
|
assertArtifactContract('findings', buildFindingsDocument(scanId, deduped));
|
|
513
513
|
const stored = {
|
|
514
|
+
...(record.target.kind === 'git_diff' && !record.target.workingTree ? { boundRevisionEvidence: true } : {}),
|
|
514
515
|
scanId,
|
|
515
516
|
recordedAt: nowIso(),
|
|
516
517
|
complete: draft.complete === true,
|
|
@@ -574,18 +575,17 @@ export class SecurityService {
|
|
|
574
575
|
if (keys.size >= 500)
|
|
575
576
|
break;
|
|
576
577
|
}
|
|
578
|
+
if (keys.size === 0)
|
|
579
|
+
return new Map();
|
|
580
|
+
// Group by source path first so a draft citing several lines of one file
|
|
581
|
+
// pays one hardened read instead of one per line.
|
|
582
|
+
const requests = Array.from(keys.values());
|
|
583
|
+
const lines = await readEvidenceLines(record.target, requests, this.config.maxFileBytes);
|
|
577
584
|
const evidence = new Map();
|
|
578
|
-
const
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
const lines = await Promise.all(batch.map(async ([key, value]) => {
|
|
583
|
-
const line = await readEvidenceLine(record.target.rootPath, value.path, value.line, this.config.maxFileBytes);
|
|
584
|
-
return { key, line };
|
|
585
|
-
}));
|
|
586
|
-
for (const entry of lines)
|
|
587
|
-
if (entry.line !== undefined)
|
|
588
|
-
evidence.set(entry.key, entry.line);
|
|
585
|
+
for (const [key, value] of keys) {
|
|
586
|
+
const line = lines.get(value.path + ':' + value.line);
|
|
587
|
+
if (line !== undefined)
|
|
588
|
+
evidence.set(key, line);
|
|
589
589
|
}
|
|
590
590
|
return evidence;
|
|
591
591
|
}
|
|
@@ -595,20 +595,19 @@ export class SecurityService {
|
|
|
595
595
|
const files = new Map();
|
|
596
596
|
const evidence = new Map();
|
|
597
597
|
const read = async (path) => {
|
|
598
|
-
let
|
|
599
|
-
if (
|
|
600
|
-
|
|
601
|
-
files.set(path,
|
|
598
|
+
let lines = files.get(path);
|
|
599
|
+
if (lines === undefined) {
|
|
600
|
+
lines = (await registry.read('verified-draft', path)).split('\n');
|
|
601
|
+
files.set(path, lines);
|
|
602
602
|
}
|
|
603
|
-
return
|
|
603
|
+
return lines;
|
|
604
604
|
};
|
|
605
605
|
try {
|
|
606
606
|
for (const finding of findings) {
|
|
607
607
|
if (!Array.isArray(finding.locations))
|
|
608
608
|
throw new Error('codex-security: finding locations are required');
|
|
609
609
|
for (const location of finding.locations) {
|
|
610
|
-
const
|
|
611
|
-
const lines = content.split('\n');
|
|
610
|
+
const lines = await read(location.path);
|
|
612
611
|
if (!Number.isInteger(location.startLine) || location.startLine < 1 || location.startLine > lines.length || (location.endLine !== undefined && location.endLine > lines.length))
|
|
613
612
|
throw new Error('codex-security: finding location is outside the bound source');
|
|
614
613
|
evidence.set(location.path + ':' + location.startLine, truncate(normalizeLine(lines[location.startLine - 1] ?? ''), 400));
|
|
@@ -617,12 +616,11 @@ export class SecurityService {
|
|
|
617
616
|
const item = asObject(raw);
|
|
618
617
|
if (!item || typeof item['path'] !== 'string' || typeof item['code'] !== 'string' || !item['code'])
|
|
619
618
|
throw new Error('codex-security: worker code evidence requires a bound source path and code');
|
|
620
|
-
const
|
|
619
|
+
const lines = await read(item['path']);
|
|
621
620
|
const start = item['startLine'] ?? item['start_line'];
|
|
622
621
|
const end = item['endLine'] ?? item['end_line'] ?? start;
|
|
623
|
-
let excerpt =
|
|
622
|
+
let excerpt = lines.join('\n');
|
|
624
623
|
if (start !== undefined) {
|
|
625
|
-
const lines = content.split('\n');
|
|
626
624
|
if (typeof start !== 'number' || typeof end !== 'number' || !Number.isInteger(start) || !Number.isInteger(end) || start < 1 || end < start || end > lines.length)
|
|
627
625
|
throw new Error('codex-security: code evidence location is outside the bound source');
|
|
628
626
|
excerpt = lines.slice(start - 1, end).join('\n');
|
|
@@ -694,7 +692,7 @@ export class SecurityService {
|
|
|
694
692
|
const severity = this.severityOf(requiredStr(input, 'severity'));
|
|
695
693
|
const path = sanitizeRelative(requiredStr(input, 'path'));
|
|
696
694
|
const startLine = Math.max(1, Math.floor(num(input['startLine']) ?? 1));
|
|
697
|
-
const anchor = str(input['anchor']) ?? normalizeAnchor(ruleId, path, await readEvidenceLine(record.target
|
|
695
|
+
const anchor = str(input['anchor']) ?? normalizeAnchor(ruleId, path, await readEvidenceLine(record.target, path, startLine, this.config.maxFileBytes), str(input['instance']));
|
|
698
696
|
const instance = str(input['instance']);
|
|
699
697
|
const key = ruleId + '\u0000' + anchor + '\u0000' + (instance ?? '');
|
|
700
698
|
const candidateId = 'cand_' + sha256(key).slice(0, 16);
|
|
@@ -919,19 +917,25 @@ export class SecurityService {
|
|
|
919
917
|
const reviewState = draft.input.reviewState ?? { surfaces: [] };
|
|
920
918
|
const warnings = [...record.warnings];
|
|
921
919
|
let migratedDraftIds;
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
920
|
+
const legacyIds = findings.some((finding) => finding.fingerprints.primary !== fingerprintFor(record.target.targetId, finding.ruleId, finding.identity.anchor, finding.identity.instance));
|
|
921
|
+
const verifyRevisionEvidence = record.target.kind === 'git_diff' && !record.target.workingTree && !draft.boundRevisionEvidence;
|
|
922
|
+
if (legacyIds || verifyRevisionEvidence) {
|
|
923
|
+
// Older checkpoints may have derived revision-only anchors from disk.
|
|
924
|
+
// Match identities before publishing; candidate and review records may
|
|
925
|
+
// still reference the original findings.
|
|
925
926
|
const evidenceLines = await this.collectEvidenceLines(record, draft.input.findings);
|
|
926
927
|
const migrated = draft.input.findings.map((input) => normalizeFinding(scanId, input, { rootPath: record.target.rootPath, targetId: record.target.targetId, evidenceLines }));
|
|
927
928
|
if (new Set(migrated.map((finding) => finding.fingerprints.primary)).size !== migrated.length)
|
|
928
929
|
throw new Error('codex-security: legacy draft has duplicate finding identities; record a corrected draft');
|
|
929
930
|
const byIdentity = new Map(migrated.map((finding) => [JSON.stringify([finding.ruleId, finding.identity.anchor, finding.identity.instance ?? '', finding.locations[0]?.path]), finding]));
|
|
930
|
-
|
|
931
|
-
if (
|
|
931
|
+
const matches = findings.map((finding) => ({ previous: finding.findingId, current: byIdentity.get(JSON.stringify([finding.ruleId, finding.identity.anchor, finding.identity.instance ?? '', finding.locations[0]?.path]))?.findingId ?? '' }));
|
|
932
|
+
if (matches.some((entry) => !entry.current))
|
|
932
933
|
throw new Error('codex-security: legacy draft identities changed; record a corrected draft');
|
|
933
|
-
|
|
934
|
-
|
|
934
|
+
if (legacyIds) {
|
|
935
|
+
migratedDraftIds = matches;
|
|
936
|
+
findings = sortedFindings(migrated);
|
|
937
|
+
warnings.push('unsealed legacy draft finding IDs were migrated to the Codex-compatible identity contract');
|
|
938
|
+
}
|
|
935
939
|
}
|
|
936
940
|
if (reviewState.surfaces.length === 0)
|
|
937
941
|
throw new Error('codex-security: at least one coverage surface is required before sealing');
|
|
@@ -1048,23 +1052,47 @@ export class SecurityService {
|
|
|
1048
1052
|
},
|
|
1049
1053
|
};
|
|
1050
1054
|
assertArtifactContract('scan-manifest', manifest);
|
|
1051
|
-
for (const file of sealedFiles) {
|
|
1052
|
-
options?.signal?.throwIfAborted();
|
|
1053
|
-
await writeFileAtomic(joinWithin(dir, file.path), file.content);
|
|
1054
|
-
}
|
|
1055
|
-
await writeJsonAtomic(join(dir, 'scan-manifest.json'), manifest);
|
|
1056
1055
|
const counts = severityCounts(findings);
|
|
1057
1056
|
options?.signal?.throwIfAborted();
|
|
1058
|
-
const
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1057
|
+
const stageDir = await mkdtemp(join(dir, 'checkpoints', 'seal-'));
|
|
1058
|
+
let updated;
|
|
1059
|
+
try {
|
|
1060
|
+
for (const file of sealedFiles) {
|
|
1061
|
+
options?.signal?.throwIfAborted();
|
|
1062
|
+
await writeFileAtomic(join(stageDir, file.path), file.content);
|
|
1063
|
+
}
|
|
1064
|
+
await writeJsonAtomic(join(stageDir, 'scan-manifest.json'), manifest);
|
|
1065
|
+
updated = await this.store.update(scanId, (current) => {
|
|
1066
|
+
current.status = 'completed';
|
|
1067
|
+
current.completedAt = sealedAt;
|
|
1068
|
+
current.findingsCount = findings.length;
|
|
1069
|
+
current.severityCounts = counts;
|
|
1070
|
+
current.artifacts = record.artifacts.map((entry) => ({ ...entry, createdAt: sealedAt }));
|
|
1071
|
+
current.policy = policy;
|
|
1072
|
+
current.warnings = warnings;
|
|
1073
|
+
current.progress.phase = 'completed';
|
|
1074
|
+
}, {
|
|
1075
|
+
signal: options?.signal,
|
|
1076
|
+
publish: async (commit) => {
|
|
1077
|
+
const publish = async () => {
|
|
1078
|
+
options?.signal?.throwIfAborted();
|
|
1079
|
+
// Readers use the scan record as the publication marker. Once
|
|
1080
|
+
// publication starts, finish it before honoring cancellation.
|
|
1081
|
+
for (const file of sealedFiles)
|
|
1082
|
+
await renameFileAtomic(join(stageDir, file.path), join(dir, file.path));
|
|
1083
|
+
await renameFileAtomic(join(stageDir, 'scan-manifest.json'), join(dir, 'scan-manifest.json'));
|
|
1084
|
+
await commit();
|
|
1085
|
+
};
|
|
1086
|
+
if (options?.withPublication)
|
|
1087
|
+
await options.withPublication(publish);
|
|
1088
|
+
else
|
|
1089
|
+
await publish();
|
|
1090
|
+
},
|
|
1091
|
+
});
|
|
1092
|
+
}
|
|
1093
|
+
finally {
|
|
1094
|
+
await rm(stageDir, { recursive: true, force: true }).catch(() => undefined);
|
|
1095
|
+
}
|
|
1068
1096
|
const absolute = (name) => join(dir, name);
|
|
1069
1097
|
return {
|
|
1070
1098
|
ok: true,
|
|
@@ -1138,11 +1166,22 @@ export class SecurityService {
|
|
|
1138
1166
|
async getScan(args) {
|
|
1139
1167
|
const scanId = requiredStr(args, 'scanId');
|
|
1140
1168
|
const { record, dir } = await this.requireScan(scanId);
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
const
|
|
1145
|
-
const
|
|
1169
|
+
// Read the scan files that all belong to this one response from the resolved
|
|
1170
|
+
// directory instead of re-resolving the index (and re-reading scan.json)
|
|
1171
|
+
// through the store once per document.
|
|
1172
|
+
const triage = (await readJson(join(dir, 'triage.json'))) ?? {};
|
|
1173
|
+
const [document, coverage, manifest] = record.status === 'completed'
|
|
1174
|
+
? await Promise.all([
|
|
1175
|
+
readJson(join(dir, 'findings.json')),
|
|
1176
|
+
readJson(join(dir, 'coverage.json')),
|
|
1177
|
+
readJson(join(dir, 'scan-manifest.json')),
|
|
1178
|
+
])
|
|
1179
|
+
: [];
|
|
1180
|
+
const findings = (document?.findings ?? []).map((finding) => {
|
|
1181
|
+
const entry = triage[finding.findingId];
|
|
1182
|
+
return { ...finding, status: entry?.status ?? 'open', ...(entry ? { triage: entry } : {}) };
|
|
1183
|
+
});
|
|
1184
|
+
const artifacts = await this.store.listArtifacts(scanId, dir);
|
|
1146
1185
|
const reviewItems = (await readJson(join(dir, 'review-items.json')))?.items ?? [];
|
|
1147
1186
|
return {
|
|
1148
1187
|
ok: true,
|
package/lib/store.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { readdir } from 'node:fs/promises';
|
|
1
|
+
import { readdir, rm, stat } from 'node:fs/promises';
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import { ensureDir, joinWithin, nowIso, readJson, readText, sanitizeRelative, writeFileAtomic, writeJsonAtomic, sha256 } from './util.js';
|
|
4
4
|
import { acquireStateLock } from './state-lock.js';
|
|
5
5
|
/** Files owned by the seal step; agents cannot overwrite them through save_artifact. */
|
|
6
|
-
const
|
|
6
|
+
const FINAL_ARTIFACTS = new Set(['findings.json', 'coverage.json', 'scan-manifest.json', 'sarif.json', 'report.md', 'threat-model.md']);
|
|
7
|
+
const SEALED_FILES = new Set([...FINAL_ARTIFACTS, 'scan.json', 'threatmodel.md']);
|
|
7
8
|
const INTERNAL_FILES = new Set([...SEALED_FILES, 'candidates.json', 'validations.json', 'attack-paths.json', 'review-items.json', 'triage.json']);
|
|
8
9
|
/**
|
|
9
10
|
* Fold an artifact path into the form Win32 uses when resolving it. Windows
|
|
@@ -57,10 +58,13 @@ export function summarize(record) {
|
|
|
57
58
|
export class ScanStore {
|
|
58
59
|
stateDir;
|
|
59
60
|
queue = Promise.resolve();
|
|
61
|
+
indexCache;
|
|
62
|
+
indexStamp;
|
|
60
63
|
constructor(stateDir) {
|
|
61
64
|
this.stateDir = stateDir;
|
|
62
65
|
}
|
|
63
66
|
get indexPath() { return join(this.stateDir, 'index.json'); }
|
|
67
|
+
get indexLockPath() { return join(this.stateDir, '.locks', 'index.sqlite'); }
|
|
64
68
|
scanDir(record) {
|
|
65
69
|
return join(this.stateDir, 'repositories', record.repoKey, 'scans', record.scanId);
|
|
66
70
|
}
|
|
@@ -71,7 +75,7 @@ export class ScanStore {
|
|
|
71
75
|
serialize(operation, signal) {
|
|
72
76
|
const guarded = async () => {
|
|
73
77
|
signal?.throwIfAborted();
|
|
74
|
-
const release = await acquireStateLock(
|
|
78
|
+
const release = await acquireStateLock(this.indexLockPath, { signal });
|
|
75
79
|
try {
|
|
76
80
|
signal?.throwIfAborted();
|
|
77
81
|
return await operation();
|
|
@@ -84,9 +88,26 @@ export class ScanStore {
|
|
|
84
88
|
this.queue = next.catch(() => undefined);
|
|
85
89
|
return next;
|
|
86
90
|
}
|
|
87
|
-
|
|
91
|
+
/** Cheap identity of index.json so a changed on-disk index invalidates the cache. */
|
|
92
|
+
async indexFileStamp() {
|
|
93
|
+
const info = await stat(this.indexPath).catch(() => undefined);
|
|
94
|
+
return info ? info.mtimeMs + ':' + info.size + ':' + info.ino : undefined;
|
|
95
|
+
}
|
|
96
|
+
async readIndex() {
|
|
97
|
+
const stamp = await this.indexFileStamp();
|
|
98
|
+
if (stamp === undefined)
|
|
99
|
+
return undefined;
|
|
100
|
+
if (this.indexCache && this.indexStamp === stamp)
|
|
101
|
+
return this.indexCache;
|
|
88
102
|
const stored = await readJson(this.indexPath);
|
|
89
|
-
|
|
103
|
+
if (stored?.version !== 1)
|
|
104
|
+
return undefined;
|
|
105
|
+
this.indexCache = stored;
|
|
106
|
+
this.indexStamp = stamp;
|
|
107
|
+
return stored;
|
|
108
|
+
}
|
|
109
|
+
async loadIndex() {
|
|
110
|
+
return (await this.readIndex()) ?? this.rebuildIndex();
|
|
90
111
|
}
|
|
91
112
|
async rebuildIndex() {
|
|
92
113
|
const index = { version: 1, scans: {} };
|
|
@@ -106,6 +127,8 @@ export class ScanStore {
|
|
|
106
127
|
}
|
|
107
128
|
async saveIndex(index) {
|
|
108
129
|
await writeJsonAtomic(this.indexPath, index);
|
|
130
|
+
this.indexCache = index;
|
|
131
|
+
this.indexStamp = await this.indexFileStamp();
|
|
109
132
|
}
|
|
110
133
|
async create(input, options = {}) {
|
|
111
134
|
return this.serialize(async () => {
|
|
@@ -180,21 +203,40 @@ export class ScanStore {
|
|
|
180
203
|
}
|
|
181
204
|
return undefined;
|
|
182
205
|
}
|
|
183
|
-
async update(scanId, mutate) {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
206
|
+
async update(scanId, mutate, options = {}) {
|
|
207
|
+
let published;
|
|
208
|
+
try {
|
|
209
|
+
return await this.serialize(async () => {
|
|
210
|
+
const found = await this.get(scanId);
|
|
211
|
+
if (!found)
|
|
212
|
+
throw new Error('codex-security: scan not found: ' + scanId);
|
|
213
|
+
const record = found.record;
|
|
214
|
+
mutate(record);
|
|
215
|
+
record.updatedAt = nowIso();
|
|
216
|
+
record.progress.updatedAt = record.updatedAt;
|
|
217
|
+
const index = await this.loadIndex();
|
|
218
|
+
const nextIndex = { ...index, scans: { ...index.scans, [scanId]: summarize(record) } };
|
|
219
|
+
if (options.publish) {
|
|
220
|
+
await options.publish(async () => {
|
|
221
|
+
// A missing index is rebuilt from scan records. Invalidate it before
|
|
222
|
+
// commit so a crash cannot leave a stale running summary on disk.
|
|
223
|
+
await rm(this.indexPath, { force: true });
|
|
224
|
+
await this.writeRecord(record);
|
|
225
|
+
published = record;
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
await this.writeRecord(record);
|
|
230
|
+
}
|
|
231
|
+
await this.saveIndex(nextIndex);
|
|
232
|
+
return record;
|
|
233
|
+
}, options.signal);
|
|
234
|
+
}
|
|
235
|
+
catch (error) {
|
|
236
|
+
if (published)
|
|
237
|
+
return published;
|
|
238
|
+
throw error;
|
|
239
|
+
}
|
|
198
240
|
}
|
|
199
241
|
async writeScanFile(scanId, name, value) {
|
|
200
242
|
const found = await this.get(scanId);
|
|
@@ -208,14 +250,31 @@ export class ScanStore {
|
|
|
208
250
|
const found = await this.get(scanId);
|
|
209
251
|
if (!found)
|
|
210
252
|
throw new Error('codex-security: scan not found: ' + scanId);
|
|
211
|
-
|
|
253
|
+
const clean = sanitizeRelative(name);
|
|
254
|
+
if (found.record.status !== 'completed' && FINAL_ARTIFACTS.has(guardName(clean)))
|
|
255
|
+
return undefined;
|
|
256
|
+
return readJson(joinWithin(found.dir, clean));
|
|
212
257
|
}
|
|
213
258
|
async list() {
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
259
|
+
let index = await this.readIndex();
|
|
260
|
+
if (!index) {
|
|
261
|
+
// Rebuild while publication owns the lock without waiting. If the lock
|
|
262
|
+
// is free, persist the repair to restore the cached path for later reads.
|
|
263
|
+
const release = await acquireStateLock(this.indexLockPath, { timeoutMs: 0 }).catch(() => undefined);
|
|
264
|
+
try {
|
|
265
|
+
if (release)
|
|
266
|
+
index = await this.readIndex();
|
|
267
|
+
if (!index) {
|
|
268
|
+
index = await this.rebuildIndex();
|
|
269
|
+
if (release)
|
|
270
|
+
await this.saveIndex(index).catch(() => undefined);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
finally {
|
|
274
|
+
release?.();
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
return Object.values(index.scans).sort((left, right) => right.createdAt.localeCompare(left.createdAt));
|
|
219
278
|
}
|
|
220
279
|
async saveArtifact(scanId, relativePath, content, mediaType) {
|
|
221
280
|
const found = await this.get(scanId);
|
|
@@ -239,17 +298,20 @@ export class ScanStore {
|
|
|
239
298
|
if (!found)
|
|
240
299
|
throw new Error('codex-security: scan not found: ' + scanId);
|
|
241
300
|
const clean = sanitizeRelative(relativePath);
|
|
301
|
+
const guarded = guardName(clean);
|
|
302
|
+
if (guarded.startsWith('checkpoints/seal-') || (found.record.status !== 'completed' && FINAL_ARTIFACTS.has(guarded))) {
|
|
303
|
+
throw new Error('codex-security: artifact not found: ' + relativePath);
|
|
304
|
+
}
|
|
242
305
|
const target = joinWithin(found.dir, clean);
|
|
243
306
|
const content = await readText(target, limit);
|
|
244
307
|
if (content === undefined)
|
|
245
308
|
throw new Error('codex-security: artifact not found: ' + relativePath);
|
|
246
|
-
const { stat } = await import('node:fs/promises');
|
|
247
309
|
const info = await stat(target);
|
|
248
310
|
return { path: clean, content, bytes: info.size, truncated: info.size > limit };
|
|
249
311
|
}
|
|
250
|
-
async listArtifacts(scanId) {
|
|
251
|
-
const
|
|
252
|
-
if (!
|
|
312
|
+
async listArtifacts(scanId, knownDir) {
|
|
313
|
+
const scanDir = knownDir ?? (await this.get(scanId))?.dir;
|
|
314
|
+
if (!scanDir)
|
|
253
315
|
return [];
|
|
254
316
|
const results = [];
|
|
255
317
|
const walk = async (dir, relBase) => {
|
|
@@ -274,7 +336,6 @@ export class ScanStore {
|
|
|
274
336
|
continue;
|
|
275
337
|
if (relBase.length === 0 && SEALED_FILES.has(guardName(entry.name)))
|
|
276
338
|
continue;
|
|
277
|
-
const { stat } = await import('node:fs/promises');
|
|
278
339
|
const info = await stat(join(dir, entry.name)).catch((error) => {
|
|
279
340
|
if (error.code === 'ENOENT')
|
|
280
341
|
return undefined;
|
|
@@ -284,7 +345,7 @@ export class ScanStore {
|
|
|
284
345
|
results.push({ path: rel, bytes: info.size });
|
|
285
346
|
}
|
|
286
347
|
};
|
|
287
|
-
await walk(
|
|
348
|
+
await walk(scanDir, '');
|
|
288
349
|
return results.sort((left, right) => left.path.localeCompare(right.path));
|
|
289
350
|
}
|
|
290
351
|
async readTriage(scanId) {
|
package/lib/target.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { lstat, realpath, stat } from 'node:fs/promises';
|
|
2
2
|
import { basename, dirname, join, relative, resolve, sep } from 'node:path';
|
|
3
3
|
import { createHash } from 'node:crypto';
|
|
4
|
-
import { isProbablyBinary, isWithin, joinWithin, languageOf, listFiles, matchesAny, matchesScopePath, normalizeLine, stableTargetId, truncate } from './util.js';
|
|
5
|
-
import { changedFiles, currentBranch, fileDigest, headRevision, remoteUrl, repoRoot, resolveRevision, statusEntries, trackedFiles } from './git.js';
|
|
4
|
+
import { isProbablyBinary, isWithin, joinWithin, languageOf, listFiles, matchesAny, matchesScopePath, normalizeLine, sanitizeRelative, stableTargetId, truncate } from './util.js';
|
|
5
|
+
import { changedFiles, currentBranch, fileDigest, headRevision, remoteUrl, repoRoot, resolveRevision, revisionFilePrefix, statusEntries, trackedFiles } from './git.js';
|
|
6
6
|
const DEFAULT_EXCLUDES = ['.git/**', 'node_modules/**', 'dist/**', 'build/**', 'out/**', 'coverage/**', '.next/**', '.nuxt/**', 'vendor/**', 'target/**', '.venv/**', '__pycache__/**', '*.min.js', '*.min.css', '*.map', '*.lock', 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'Cargo.lock', 'poetry.lock', 'uv.lock', 'composer.lock', 'go.sum'];
|
|
7
7
|
export async function resolveTarget(options) {
|
|
8
8
|
options.signal?.throwIfAborted();
|
|
@@ -57,7 +57,7 @@ export async function resolveTarget(options) {
|
|
|
57
57
|
let kind = 'directory_snapshot';
|
|
58
58
|
if (gitRepo)
|
|
59
59
|
kind = options.diffTarget || options.workingTree ? 'git_diff' : 'git_worktree';
|
|
60
|
-
const diff = kind === 'git_diff' ? await resolveDiff(rootPath, { diffTarget: baseRevision, workingTree: options.workingTree === true }) : undefined;
|
|
60
|
+
const diff = kind === 'git_diff' ? await resolveDiff(rootPath, { diffTarget: baseRevision, workingTree: options.workingTree === true, head: revision, signal: options.signal }) : undefined;
|
|
61
61
|
const diffFiles = diff?.files.filter((file) => (matchesScopePath(file.path, scope.includePaths) || (file.oldPath !== undefined && matchesScopePath(file.oldPath, scope.includePaths))) &&
|
|
62
62
|
!matchesAny(file.path, excludePaths));
|
|
63
63
|
const changedPaths = diffFiles ? new Set(diffFiles.map((file) => file.path)) : undefined;
|
|
@@ -172,16 +172,77 @@ function displayNameFor(rootPath, remote) {
|
|
|
172
172
|
return name.length > 0 ? name : rootPath;
|
|
173
173
|
}
|
|
174
174
|
export async function resolveDiff(rootPath, options) {
|
|
175
|
-
const head = await headRevision(rootPath);
|
|
176
|
-
const files = await changedFiles(rootPath, { base: options.diffTarget, staged: options.staged, workingTree: options.workingTree });
|
|
175
|
+
const head = options.head ?? await headRevision(rootPath, options.signal);
|
|
176
|
+
const files = await changedFiles(rootPath, { base: options.diffTarget, head, staged: options.staged, workingTree: options.workingTree, signal: options.signal });
|
|
177
177
|
return { files, ...(options.diffTarget ? { base: options.diffTarget } : {}), ...(head ? { head } : {}), workingTree: options.workingTree === true || (!options.diffTarget && !options.staged) };
|
|
178
178
|
}
|
|
179
179
|
/** Read one source line for anchor derivation without trusting it as instructions. */
|
|
180
|
-
export async function readEvidenceLine(
|
|
181
|
-
const { open } = await import('node:fs/promises');
|
|
182
|
-
const { constants } = await import('node:fs');
|
|
180
|
+
export async function readEvidenceLine(source, path, line, maxBytes) {
|
|
183
181
|
if (!Number.isFinite(line) || line < 1)
|
|
184
182
|
return undefined;
|
|
183
|
+
const lines = await readEvidenceFileLines(source, path, maxBytes);
|
|
184
|
+
if (!lines)
|
|
185
|
+
return undefined;
|
|
186
|
+
const value = lines[Math.trunc(line) - 1];
|
|
187
|
+
return value === undefined ? undefined : truncate(normalizeLine(value), 400);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* Read many evidence lines with one hardened open per source file. The old
|
|
191
|
+
* per-line reader repeated the lstat/open/realpath verification and re-read the
|
|
192
|
+
* file prefix for every requested line, so a draft citing several lines of one
|
|
193
|
+
* file paid that cost repeatedly.
|
|
194
|
+
*/
|
|
195
|
+
export async function readEvidenceLines(source, requests, maxBytes) {
|
|
196
|
+
const byPath = new Map();
|
|
197
|
+
for (const request of requests) {
|
|
198
|
+
if (!Number.isFinite(request.line) || request.line < 1)
|
|
199
|
+
continue;
|
|
200
|
+
const line = Math.trunc(request.line);
|
|
201
|
+
const bucket = byPath.get(request.path);
|
|
202
|
+
if (bucket) {
|
|
203
|
+
if (!bucket.includes(line))
|
|
204
|
+
bucket.push(line);
|
|
205
|
+
}
|
|
206
|
+
else
|
|
207
|
+
byPath.set(request.path, [line]);
|
|
208
|
+
}
|
|
209
|
+
const result = new Map();
|
|
210
|
+
const groups = [...byPath.entries()];
|
|
211
|
+
const concurrency = 8;
|
|
212
|
+
for (let index = 0; index < groups.length; index += concurrency) {
|
|
213
|
+
await Promise.all(groups.slice(index, index + concurrency).map(async ([path, lineNumbers]) => {
|
|
214
|
+
const lines = await readEvidenceFileLines(source, path, maxBytes);
|
|
215
|
+
if (!lines)
|
|
216
|
+
return;
|
|
217
|
+
for (const line of lineNumbers) {
|
|
218
|
+
const value = lines[line - 1];
|
|
219
|
+
if (value !== undefined)
|
|
220
|
+
result.set(path + ':' + line, truncate(normalizeLine(value), 400));
|
|
221
|
+
}
|
|
222
|
+
}));
|
|
223
|
+
}
|
|
224
|
+
return result;
|
|
225
|
+
}
|
|
226
|
+
/** Hardened single-file read shared by the line readers. */
|
|
227
|
+
async function readEvidenceFileLines(source, path, maxBytes) {
|
|
228
|
+
const maxPrefixBytes = Math.trunc(Math.min(maxBytes, 256 * 1024));
|
|
229
|
+
if (typeof source !== 'string' && source.kind === 'git_diff' && !source.workingTree) {
|
|
230
|
+
const revision = source.headRevision ?? source.revision;
|
|
231
|
+
if (!revision)
|
|
232
|
+
return undefined;
|
|
233
|
+
let clean;
|
|
234
|
+
try {
|
|
235
|
+
clean = sanitizeRelative(path);
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
return undefined;
|
|
239
|
+
}
|
|
240
|
+
const prefix = await revisionFilePrefix(source.rootPath, revision, clean, maxPrefixBytes);
|
|
241
|
+
return prefix?.toString('utf8').split('\n');
|
|
242
|
+
}
|
|
243
|
+
const rootPath = typeof source === 'string' ? source : source.rootPath;
|
|
244
|
+
const { open } = await import('node:fs/promises');
|
|
245
|
+
const { constants } = await import('node:fs');
|
|
185
246
|
if (typeof path !== 'string' || path.length === 0)
|
|
186
247
|
return undefined;
|
|
187
248
|
let absolute;
|
|
@@ -214,12 +275,9 @@ export async function readEvidenceLine(rootPath, path, line, maxBytes) {
|
|
|
214
275
|
const current = await lstat(canonical).catch(() => undefined);
|
|
215
276
|
if (!current || !current.isFile() || current.dev !== opened.dev || current.ino !== opened.ino || current.size !== opened.size)
|
|
216
277
|
return undefined;
|
|
217
|
-
const buffer = Buffer.alloc(
|
|
278
|
+
const buffer = Buffer.alloc(maxPrefixBytes);
|
|
218
279
|
const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0);
|
|
219
|
-
|
|
220
|
-
const lines = text.split('\n');
|
|
221
|
-
const value = lines[line - 1];
|
|
222
|
-
return value === undefined ? undefined : truncate(normalizeLine(value), 400);
|
|
280
|
+
return buffer.subarray(0, bytesRead).toString('utf8').split('\n');
|
|
223
281
|
}
|
|
224
282
|
catch {
|
|
225
283
|
return undefined;
|