@mnemonik/shared 7.154.2 → 7.155.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ast/grammars.d.ts +4 -2
- package/dist/ast/grammars.d.ts.map +1 -1
- package/dist/ast/grammars.js +45 -3
- package/dist/ast/grammars.js.map +1 -1
- package/dist/codeScanner.d.ts +26 -329
- package/dist/codeScanner.d.ts.map +1 -1
- package/dist/codeScanner.js +65 -1141
- package/dist/codeScanner.js.map +1 -1
- package/dist/fileSelection.d.ts +451 -0
- package/dist/fileSelection.d.ts.map +1 -0
- package/dist/fileSelection.js +1256 -0
- package/dist/fileSelection.js.map +1 -0
- package/dist/hookTrace.d.ts +40 -0
- package/dist/hookTrace.d.ts.map +1 -1
- package/dist/hookTrace.js +126 -0
- package/dist/hookTrace.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/runtimeAssets.d.ts +12 -0
- package/dist/runtimeAssets.d.ts.map +1 -0
- package/dist/runtimeAssets.js +43 -0
- package/dist/runtimeAssets.js.map +1 -0
- package/dist/runtimeReader.d.ts +15 -2
- package/dist/runtimeReader.d.ts.map +1 -1
- package/dist/runtimeReader.js +110 -21
- package/dist/runtimeReader.js.map +1 -1
- package/dist/sourceRedaction.d.ts +45 -1
- package/dist/sourceRedaction.d.ts.map +1 -1
- package/dist/sourceRedaction.js +62 -22
- package/dist/sourceRedaction.js.map +1 -1
- package/dist/sourceRedactionPool.d.ts +198 -0
- package/dist/sourceRedactionPool.d.ts.map +1 -0
- package/dist/sourceRedactionPool.js +756 -0
- package/dist/sourceRedactionPool.js.map +1 -0
- package/dist/vueSource.d.ts +0 -2
- package/dist/vueSource.d.ts.map +1 -1
- package/dist/vueSource.js +8 -19
- package/dist/vueSource.js.map +1 -1
- package/package.json +6 -2
- package/src/ast/grammars.ts +52 -7
- package/src/codeScanner.ts +121 -1321
- package/src/fileSelection.ts +1440 -0
- package/src/hookTrace.ts +143 -0
- package/src/index.ts +31 -2
- package/src/runtimeAssets.ts +49 -0
- package/src/runtimeReader.ts +120 -28
- package/src/sourceRedaction.ts +85 -20
- package/src/sourceRedactionPool.ts +877 -0
- package/src/vueSource.ts +8 -23
- package/tests/runtimeVerify.test.ts +166 -0
package/src/hookTrace.ts
CHANGED
|
@@ -38,6 +38,7 @@ import {
|
|
|
38
38
|
writeSync,
|
|
39
39
|
} from 'node:fs';
|
|
40
40
|
import { dirname, join } from 'node:path';
|
|
41
|
+
import { open as openAsync, stat as statAsync } from 'node:fs/promises';
|
|
41
42
|
import { mnemonikStateDirectory } from './protectedPaths.js';
|
|
42
43
|
import { ensureWindowsPrivateDirectorySync } from './runtimeSigners.js';
|
|
43
44
|
import { scrubSecrets } from './secretPatterns.js';
|
|
@@ -586,3 +587,145 @@ export function startHookTrace(options: HookTraceOptions): HookTrace {
|
|
|
586
587
|
trace.open();
|
|
587
588
|
return trace;
|
|
588
589
|
}
|
|
590
|
+
|
|
591
|
+
/** The trace window a hook reports its recent misses over. */
|
|
592
|
+
export const HOOK_MISS_WINDOW_MS = 30 * 60 * 1000;
|
|
593
|
+
/**
|
|
594
|
+
* How much of the trace file's end is read for that summary, asynchronously
|
|
595
|
+
* and once per hook process (it used to read 256 KB synchronously on every
|
|
596
|
+
* call). About a hundred hook runs: the recent ones are what the count is for.
|
|
597
|
+
*/
|
|
598
|
+
const HOOK_MISS_TAIL_BYTES = 64 * 1024;
|
|
599
|
+
/** Sent on the main hook call: `<missed>/<total>` server calls over the window. */
|
|
600
|
+
export const HOOK_MISSES_HEADER = 'x-mnemonik-hook-misses';
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* How many of this host's recent calls to the server did not get an answer
|
|
604
|
+
* in time: a timeout, a connection error, a gateway or server error, a call
|
|
605
|
+
* still pending at exit, or a whole run the editor killed at its own hook
|
|
606
|
+
* timeout. Each of these is silent where it happens (the hook fails open, by
|
|
607
|
+
* design), so the main hook call carries the count to the server, and
|
|
608
|
+
* machine health tells the agent when a machine is losing memory this way
|
|
609
|
+
* (limits audit #2-#3: the hook budget chain fails silently).
|
|
610
|
+
*
|
|
611
|
+
* Reads only the end of the trace file, and never throws.
|
|
612
|
+
*/
|
|
613
|
+
export async function summarizeRecentHookCalls(
|
|
614
|
+
host: HookTraceHost,
|
|
615
|
+
options: { stateDir?: string; now?: number; windowMs?: number } = {}
|
|
616
|
+
): Promise<{ missed: number; total: number }> {
|
|
617
|
+
// With the recorder off there is no file, which reads as nothing to report.
|
|
618
|
+
const empty = { missed: 0, total: 0 };
|
|
619
|
+
const now = options.now ?? Date.now();
|
|
620
|
+
const since = now - (options.windowMs ?? HOOK_MISS_WINDOW_MS);
|
|
621
|
+
let text: string;
|
|
622
|
+
try {
|
|
623
|
+
const file = hookTraceFile(host, options.stateDir);
|
|
624
|
+
const size = (await statAsync(file)).size;
|
|
625
|
+
const handle = await openAsync(file, 'r');
|
|
626
|
+
try {
|
|
627
|
+
const length = Math.min(size, HOOK_MISS_TAIL_BYTES);
|
|
628
|
+
const buffer = Buffer.alloc(length);
|
|
629
|
+
await handle.read(buffer, 0, length, size - length);
|
|
630
|
+
text = buffer.toString('utf8');
|
|
631
|
+
} finally {
|
|
632
|
+
await handle.close();
|
|
633
|
+
}
|
|
634
|
+
} catch {
|
|
635
|
+
return empty;
|
|
636
|
+
}
|
|
637
|
+
const started = new Map<string, number>();
|
|
638
|
+
const ended = new Set<string>();
|
|
639
|
+
let missed = 0;
|
|
640
|
+
let total = 0;
|
|
641
|
+
for (const line of text.split('\n')) {
|
|
642
|
+
let record: HookTraceRecord;
|
|
643
|
+
try {
|
|
644
|
+
record = JSON.parse(line) as HookTraceRecord;
|
|
645
|
+
} catch {
|
|
646
|
+
continue; // the first line of the tail is usually cut
|
|
647
|
+
}
|
|
648
|
+
const at = Date.parse(record.ts);
|
|
649
|
+
if (!(at >= since)) continue;
|
|
650
|
+
if (record.phase === 'start') started.set(record.id, at);
|
|
651
|
+
else if (record.phase === 'end') {
|
|
652
|
+
ended.add(record.id);
|
|
653
|
+
// One logical call per path in a run: a hedged request's aborted
|
|
654
|
+
// loser, or a first attempt whose retry was answered, is not a miss.
|
|
655
|
+
// Only a call none of whose requests got an answer is.
|
|
656
|
+
const answeredByPath = new Map<string, boolean>();
|
|
657
|
+
for (const call of record.calls ?? []) {
|
|
658
|
+
const status = call.status ?? 0;
|
|
659
|
+
const answered =
|
|
660
|
+
call.error === undefined &&
|
|
661
|
+
call.pending !== true &&
|
|
662
|
+
status > 0 &&
|
|
663
|
+
status < 500 &&
|
|
664
|
+
status !== 408 &&
|
|
665
|
+
status !== 429;
|
|
666
|
+
answeredByPath.set(call.path, (answeredByPath.get(call.path) ?? false) || answered);
|
|
667
|
+
}
|
|
668
|
+
for (const answered of answeredByPath.values()) {
|
|
669
|
+
total++;
|
|
670
|
+
if (!answered) missed++;
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
// A run with a start and no end was ended by the editor at its timeout.
|
|
675
|
+
for (const [id, at] of started) {
|
|
676
|
+
if (ended.has(id) || now - at < 60_000) continue;
|
|
677
|
+
total++;
|
|
678
|
+
missed++;
|
|
679
|
+
}
|
|
680
|
+
return { missed, total };
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
const summaries = new Map<
|
|
684
|
+
string,
|
|
685
|
+
{ read: Promise<Record<string, string>>; value?: Record<string, string> }
|
|
686
|
+
>();
|
|
687
|
+
|
|
688
|
+
function summaryFor(
|
|
689
|
+
host: HookTraceHost,
|
|
690
|
+
options: { stateDir?: string; now?: number }
|
|
691
|
+
): { read: Promise<Record<string, string>>; value?: Record<string, string> } {
|
|
692
|
+
const key = `${host}\0${options.stateDir ?? ''}\0${options.now ?? ''}`;
|
|
693
|
+
const cached = summaries.get(key);
|
|
694
|
+
if (cached) return cached;
|
|
695
|
+
const entry: { read: Promise<Record<string, string>>; value?: Record<string, string> } = {
|
|
696
|
+
read: Promise.resolve({}),
|
|
697
|
+
};
|
|
698
|
+
entry.read = summarizeRecentHookCalls(host, options).then(
|
|
699
|
+
({ missed, total }): Record<string, string> => {
|
|
700
|
+
const value: Record<string, string> =
|
|
701
|
+
total > 0 ? { [HOOK_MISSES_HEADER]: `${missed}/${total}` } : {};
|
|
702
|
+
entry.value = value;
|
|
703
|
+
return value;
|
|
704
|
+
}
|
|
705
|
+
);
|
|
706
|
+
summaries.set(key, entry);
|
|
707
|
+
return entry;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
/**
|
|
711
|
+
* The header the main hook call carries, or none when nothing was called
|
|
712
|
+
* recently. Read once per hook process, without blocking it.
|
|
713
|
+
*/
|
|
714
|
+
export function hookMissesHeaders(
|
|
715
|
+
host: HookTraceHost,
|
|
716
|
+
options: { stateDir?: string; now?: number } = {}
|
|
717
|
+
): Promise<Record<string, string>> {
|
|
718
|
+
return summaryFor(host, options).read;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
/**
|
|
722
|
+
* The header if the read has finished, else none, and never a wait: the
|
|
723
|
+
* count only feeds machine health, so it must never delay a hook request
|
|
724
|
+
* (the hook starts the read when it loads, so it is normally ready).
|
|
725
|
+
*/
|
|
726
|
+
export function hookMissesHeadersIfReady(
|
|
727
|
+
host: HookTraceHost,
|
|
728
|
+
options: { stateDir?: string; now?: number } = {}
|
|
729
|
+
): Record<string, string> {
|
|
730
|
+
return summaryFor(host, options).value ?? {};
|
|
731
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -20,7 +20,10 @@ export {
|
|
|
20
20
|
isGitBoundary,
|
|
21
21
|
isProjectBoundary,
|
|
22
22
|
rootProjectId,
|
|
23
|
-
|
|
23
|
+
SCRUB_VERSION,
|
|
24
|
+
fileOp,
|
|
25
|
+
statTuple,
|
|
26
|
+
type FileStatTuple,
|
|
24
27
|
isLinkedWorktreeCheckout,
|
|
25
28
|
FIXTURE_PATH_RE,
|
|
26
29
|
isFixturePath,
|
|
@@ -97,7 +100,33 @@ export {
|
|
|
97
100
|
scrubSecrets,
|
|
98
101
|
redactHighEntropyTokens,
|
|
99
102
|
} from './secretPatterns.js';
|
|
100
|
-
export {
|
|
103
|
+
export {
|
|
104
|
+
scrubSourceSecrets,
|
|
105
|
+
scrubSourceSecretsInProcess,
|
|
106
|
+
installSourceRedactor,
|
|
107
|
+
MAX_SOURCE_REDACTION_BYTES,
|
|
108
|
+
type SourceRedactor,
|
|
109
|
+
type SourceRedactionOptions,
|
|
110
|
+
} from './sourceRedaction.js';
|
|
111
|
+
export {
|
|
112
|
+
SourceRedactionPool,
|
|
113
|
+
redactionWorkerThread,
|
|
114
|
+
STRIKE_REASONS,
|
|
115
|
+
type RedactionChild,
|
|
116
|
+
runSourceRedactionWorker,
|
|
117
|
+
redactionHeapMb,
|
|
118
|
+
cgroupMemory,
|
|
119
|
+
containerMemoryProbe,
|
|
120
|
+
scannerMemoryProbe,
|
|
121
|
+
processCpuSeconds,
|
|
122
|
+
CPU_CEILING_SECONDS,
|
|
123
|
+
SCANNER_TOTAL_MEMORY_MB,
|
|
124
|
+
type MemoryProbe,
|
|
125
|
+
type MemoryReading,
|
|
126
|
+
type RedactionPriority,
|
|
127
|
+
LARGE_FILE_BYTES as LARGE_SOURCE_REDACTION_BYTES,
|
|
128
|
+
type SourceRedactionPoolOptions,
|
|
129
|
+
} from './sourceRedactionPool.js';
|
|
101
130
|
export {
|
|
102
131
|
redactCredentials,
|
|
103
132
|
credentialValues,
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Never use bytes from the packaged scanner's extracted runtime unchecked
|
|
3
|
+
* (review of 26 Oct 2026). JavaScript is checked as it loads
|
|
4
|
+
* (packages/scanner/src/verifiedLoader.cts); this covers what is read with fs
|
|
5
|
+
* and then run or parsed as code: the grammar WebAssembly, web-tree-sitter's
|
|
6
|
+
* own runtime WebAssembly and the tag queries. The caller passes the buffer it
|
|
7
|
+
* is about to use, so the bytes checked are the bytes used.
|
|
8
|
+
*
|
|
9
|
+
* The packaged executable's entry point publishes the embedded assets' hashes
|
|
10
|
+
* on globalThis (and runtimeWorker.ts in each worker). Anywhere else, in
|
|
11
|
+
* development, tests and on the server, there is nothing to check against and
|
|
12
|
+
* this does nothing.
|
|
13
|
+
*/
|
|
14
|
+
import { createHash } from 'node:crypto';
|
|
15
|
+
import { relative, sep } from 'node:path';
|
|
16
|
+
|
|
17
|
+
interface VerifiedRuntime {
|
|
18
|
+
root: string;
|
|
19
|
+
hashes: ReadonlyMap<string, string>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const RUNTIME = Symbol.for('mnemonik.sea.verifiedLoader');
|
|
23
|
+
|
|
24
|
+
/** Bytes that are not the embedded asset they must be. */
|
|
25
|
+
export class RuntimeAssetError extends Error {}
|
|
26
|
+
|
|
27
|
+
function runtime(): VerifiedRuntime | undefined {
|
|
28
|
+
return (globalThis as { [RUNTIME]?: VerifiedRuntime })[RUNTIME];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Whether this process runs from the packaged executable's verified runtime. */
|
|
32
|
+
export function inVerifiedRuntime(): boolean {
|
|
33
|
+
return runtime() !== undefined;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Check bytes read from `path` against the embedded asset they must equal.
|
|
38
|
+
* Throws `mnemonik_asset_integrity` when they differ and
|
|
39
|
+
* `mnemonik_unverified_asset` for a path that is not an embedded asset.
|
|
40
|
+
*/
|
|
41
|
+
export function verifyRuntimeAsset(path: string, bytes: Uint8Array): void {
|
|
42
|
+
const verified = runtime();
|
|
43
|
+
if (!verified) return;
|
|
44
|
+
const key = relative(verified.root, path).split(sep).join('/');
|
|
45
|
+
const expected = key.startsWith('../') || key === '..' ? undefined : verified.hashes.get(key);
|
|
46
|
+
if (!expected) throw new RuntimeAssetError(`mnemonik_unverified_asset: ${path}`);
|
|
47
|
+
if (createHash('sha256').update(bytes).digest('hex') !== expected)
|
|
48
|
+
throw new RuntimeAssetError(`mnemonik_asset_integrity: ${key}`);
|
|
49
|
+
}
|
package/src/runtimeReader.ts
CHANGED
|
@@ -90,6 +90,63 @@ export const versionName = (v: string): string => {
|
|
|
90
90
|
return v;
|
|
91
91
|
};
|
|
92
92
|
|
|
93
|
+
/**
|
|
94
|
+
* Filesystem operations one runtime verification keeps in flight. Each file
|
|
95
|
+
* costs a few round trips (lstat, open, read, close); awaited one at a time, a
|
|
96
|
+
* 2,000-file hook runtime took 6.6 s to verify (26 Oct 2026). Every check is
|
|
97
|
+
* still made, for every file.
|
|
98
|
+
*/
|
|
99
|
+
export const VERIFY_CONCURRENCY = 16;
|
|
100
|
+
|
|
101
|
+
/** Runs `work` with at most `limit` calls in flight; nested calls must not share a limiter. */
|
|
102
|
+
function limiter(limit: number) {
|
|
103
|
+
let active = 0;
|
|
104
|
+
const waiting: Array<() => void> = [];
|
|
105
|
+
return async <T>(work: () => Promise<T>): Promise<T> => {
|
|
106
|
+
if (active >= limit) await new Promise<void>((release) => waiting.push(release));
|
|
107
|
+
active += 1;
|
|
108
|
+
try {
|
|
109
|
+
return await work();
|
|
110
|
+
} finally {
|
|
111
|
+
active -= 1;
|
|
112
|
+
waiting.shift()?.();
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* A runtime file's bytes, read from the file that was opened: it must be a
|
|
119
|
+
* regular file (not followed through a link), this user's, with exactly the
|
|
120
|
+
* mode its manifest entry implies. POSIX only.
|
|
121
|
+
*/
|
|
122
|
+
async function ownedFileBytes(path: string, mode: number): Promise<Buffer> {
|
|
123
|
+
const fd = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW).catch((error: unknown) => {
|
|
124
|
+
// O_NOFOLLOW refuses a link with ELOOP: a link is not an owned file.
|
|
125
|
+
throw (error as NodeJS.ErrnoException).code === 'ELOOP'
|
|
126
|
+
? new RuntimeError('permission')
|
|
127
|
+
: error;
|
|
128
|
+
});
|
|
129
|
+
try {
|
|
130
|
+
const stat = await fd.stat();
|
|
131
|
+
if (!stat.isFile() || stat.uid !== process.getuid?.() || (stat.mode & 0o777) !== mode)
|
|
132
|
+
throw new RuntimeError('permission');
|
|
133
|
+
return await fd.readFile();
|
|
134
|
+
} finally {
|
|
135
|
+
await fd.close();
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Waits for every promise, then throws the first rejection in order: nothing a
|
|
141
|
+
* failed verification started is still running when it reports.
|
|
142
|
+
*/
|
|
143
|
+
async function settleAll(work: Promise<unknown>[]): Promise<void> {
|
|
144
|
+
const failed = (await Promise.allSettled(work)).find(
|
|
145
|
+
(result): result is PromiseRejectedResult => result.status === 'rejected'
|
|
146
|
+
);
|
|
147
|
+
if (failed) throw failed.reason;
|
|
148
|
+
}
|
|
149
|
+
|
|
93
150
|
const statIdentity = (stat: Stats): string =>
|
|
94
151
|
[stat.dev, stat.ino, stat.mode, stat.uid, stat.size, stat.mtimeMs, stat.ctimeMs].join(':');
|
|
95
152
|
|
|
@@ -237,17 +294,30 @@ export class RuntimeReader {
|
|
|
237
294
|
);
|
|
238
295
|
}
|
|
239
296
|
}
|
|
297
|
+
/**
|
|
298
|
+
* `proven` (one verification's own set, POSIX only) holds directories this
|
|
299
|
+
* verification has already inspected in full: their components are not
|
|
300
|
+
* inspected again for every file beneath them, and directories proven here
|
|
301
|
+
* are added to it.
|
|
302
|
+
*/
|
|
240
303
|
async inspect(
|
|
241
304
|
path: string,
|
|
242
305
|
directory = false,
|
|
243
306
|
allowMissing = false,
|
|
244
|
-
replacing = false
|
|
307
|
+
replacing = false,
|
|
308
|
+
proven?: Set<string>
|
|
245
309
|
): Promise<void> {
|
|
246
310
|
const absolute = this.guardedPath(path);
|
|
247
311
|
const windowsPaths: string[] = [];
|
|
248
|
-
|
|
249
|
-
|
|
312
|
+
const remembered = process.platform === 'win32' ? undefined : proven;
|
|
313
|
+
const checkedDirectories: string[] = [];
|
|
314
|
+
// Directories proven by this verification are skipped; when the parent is
|
|
315
|
+
// proven, so is every directory above it.
|
|
316
|
+
const parent = dirname(absolute);
|
|
317
|
+
let part = remembered?.has(parent) ? parent : parse(absolute).root;
|
|
318
|
+
for (const name of absolute.slice(part.length).split(sep).filter(Boolean)) {
|
|
250
319
|
part = join(part, name);
|
|
320
|
+
if (part !== absolute && remembered?.has(part)) continue;
|
|
251
321
|
const stat = await lstat(part).catch((e) => {
|
|
252
322
|
if (allowMissing && missing(e)) return null;
|
|
253
323
|
throw e;
|
|
@@ -295,7 +365,9 @@ export class RuntimeReader {
|
|
|
295
365
|
windowsPaths.push(part);
|
|
296
366
|
} else if (stat.uid !== process.getuid?.() || (stat.mode & 0o077) !== 0)
|
|
297
367
|
throw new RuntimeError('permission');
|
|
368
|
+
if (stat.isDirectory()) checkedDirectories.push(part);
|
|
298
369
|
}
|
|
370
|
+
if (remembered) for (const checked of checkedDirectories) remembered.add(checked);
|
|
299
371
|
await this.audit(windowsPaths);
|
|
300
372
|
for (const part of windowsPaths) {
|
|
301
373
|
const verdict = this.windowsAudit.get(part)?.verdict;
|
|
@@ -321,8 +393,8 @@ export class RuntimeReader {
|
|
|
321
393
|
this.windowsAudit.delete(absolute);
|
|
322
394
|
this.windowsStats.delete(absolute);
|
|
323
395
|
}
|
|
324
|
-
async bytes(path: string): Promise<Buffer> {
|
|
325
|
-
await this.inspect(path);
|
|
396
|
+
async bytes(path: string, proven?: Set<string>): Promise<Buffer> {
|
|
397
|
+
await this.inspect(path, false, false, false, proven);
|
|
326
398
|
const fd = await open(
|
|
327
399
|
path,
|
|
328
400
|
constants.O_RDONLY | (process.platform === 'win32' ? 0 : constants.O_NOFOLLOW)
|
|
@@ -417,25 +489,41 @@ export class RuntimeReader {
|
|
|
417
489
|
throw new RuntimeError('unsigned');
|
|
418
490
|
if (process.platform === 'win32')
|
|
419
491
|
await this.audit(Object.keys(m.files).map((name) => join(directory, safePath(name))));
|
|
492
|
+
// Every directory and file is inspected (type, owner, private mode) and
|
|
493
|
+
// every file is read and hashed, as before; up to VERIFY_CONCURRENCY at a
|
|
494
|
+
// time, and each directory's own components only once.
|
|
420
495
|
const actual: string[] = [];
|
|
496
|
+
const proven = new Set<string>();
|
|
497
|
+
// Windows keeps one at a time: its ACL audit shares caches across calls.
|
|
498
|
+
const limit = limiter(process.platform === 'win32' ? 1 : VERIFY_CONCURRENCY);
|
|
421
499
|
const walk = async (dir: string, prefix = ''): Promise<void> => {
|
|
422
|
-
await
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
500
|
+
const entries = await limit(async () => {
|
|
501
|
+
await this.inspect(dir, true, false, false, proven);
|
|
502
|
+
return readdir(dir, { withFileTypes: true });
|
|
503
|
+
});
|
|
504
|
+
await settleAll(
|
|
505
|
+
entries.map(async (entry) => {
|
|
506
|
+
const key = prefix + entry.name;
|
|
507
|
+
if (!prefix && entry.name === 'manifest.json') return;
|
|
508
|
+
const path = join(dir, entry.name);
|
|
509
|
+
if (entry.isDirectory()) await walk(path, key + '/');
|
|
510
|
+
else {
|
|
511
|
+
// POSIX: a link or anything else that is not a regular file is
|
|
512
|
+
// refused here, and every listed file's type, owner and exact mode
|
|
513
|
+
// are checked on the file itself when it is read below. Windows
|
|
514
|
+
// inspects each file for its ACL audit.
|
|
515
|
+
if (process.platform === 'win32')
|
|
516
|
+
await limit(() => this.inspect(path, false, false, false, proven));
|
|
517
|
+
else if (!entry.isFile()) throw new RuntimeError('permission');
|
|
518
|
+
actual.push(key);
|
|
519
|
+
}
|
|
520
|
+
})
|
|
521
|
+
);
|
|
433
522
|
};
|
|
434
523
|
await walk(directory);
|
|
435
524
|
if (actual.sort().join('\n') !== Object.keys(m.files).sort().join('\n'))
|
|
436
525
|
throw new RuntimeError('digest_mismatch');
|
|
437
|
-
|
|
438
|
-
for (const [name, expected] of Object.entries(m.files)) {
|
|
526
|
+
for (const expected of Object.values(m.files))
|
|
439
527
|
if (
|
|
440
528
|
!expected ||
|
|
441
529
|
!/^[a-f0-9]{64}$/.test(expected.sha256) ||
|
|
@@ -444,17 +532,21 @@ export class RuntimeReader {
|
|
|
444
532
|
typeof expected.executable !== 'boolean'
|
|
445
533
|
)
|
|
446
534
|
throw new RuntimeError('digest_mismatch');
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
535
|
+
let total = 0;
|
|
536
|
+
await settleAll(
|
|
537
|
+
Object.entries(m.files).map(([name, expected]) =>
|
|
538
|
+
limit(async () => {
|
|
539
|
+
const path = join(directory, safePath(name));
|
|
540
|
+
const contents =
|
|
541
|
+
process.platform === 'win32'
|
|
542
|
+
? await this.bytes(path, proven)
|
|
543
|
+
: await ownedFileBytes(path, expected.executable ? 0o700 : 0o600);
|
|
544
|
+
if (hash(contents) !== expected.sha256 || contents.length !== expected.size)
|
|
545
|
+
throw new RuntimeError('digest_mismatch');
|
|
546
|
+
total += contents.length;
|
|
547
|
+
})
|
|
454
548
|
)
|
|
455
|
-
|
|
456
|
-
total += contents.length;
|
|
457
|
-
}
|
|
549
|
+
);
|
|
458
550
|
if (total !== m.totalSize) throw new RuntimeError('digest_mismatch');
|
|
459
551
|
if (m.signer) {
|
|
460
552
|
const signer =
|
package/src/sourceRedaction.ts
CHANGED
|
@@ -2,7 +2,6 @@ import { parseVueSource, walkVueTemplate, VUE_FIDELITY_MARKER } from './vueSourc
|
|
|
2
2
|
import { basename, extname } from 'node:path';
|
|
3
3
|
import { Parser, type Node, type Tree } from 'web-tree-sitter';
|
|
4
4
|
import { loadGrammar, objcppSyntaxIssue, type AstLanguageId } from './ast/grammars.js';
|
|
5
|
-
import { MAX_AST_PARSE_BYTES } from './ast/astChunker.js';
|
|
6
5
|
import { commandLineCredentialRanges, redactCredentials } from './redactCredentials.js';
|
|
7
6
|
import {
|
|
8
7
|
redactHighEntropyTokens,
|
|
@@ -15,8 +14,52 @@ function scrubPayload(text: string): string {
|
|
|
15
14
|
return redactCredentials(scrubSecrets(text), { commandLineOnly: true }).text;
|
|
16
15
|
}
|
|
17
16
|
|
|
17
|
+
/**
|
|
18
|
+
* The largest source redacted with its syntax tree: the scanner's own file
|
|
19
|
+
* ceiling (MAX_SCANNED_FILE_BYTES, 10 MB; repeated here because codeScanner.ts
|
|
20
|
+
* imports this module). It was 2 MiB, sized from "the largest file in this
|
|
21
|
+
* repository", and above it every file took the text fallback: PowerShell,
|
|
22
|
+
* CMake, Groovy, Nix, Terraform and the bundled languages were stamped
|
|
23
|
+
* "source-fidelity-unavailable", so connected code refused every such file
|
|
24
|
+
* over 2 MiB, and other languages could keep structural [REDACTED] the
|
|
25
|
+
* extractors refuse (streams F1/F2, 25 October 2026). Memory for a large
|
|
26
|
+
* parse is bounded by running it in a child process sized from the file
|
|
27
|
+
* (installSourceRedactor below; sourceRedactionPool.ts), not by refusing it.
|
|
28
|
+
*/
|
|
29
|
+
export const MAX_SOURCE_REDACTION_BYTES = 10 * 1024 * 1024;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Where a process sends source it should not parse in its own memory: the
|
|
33
|
+
* server sends Swift and every file of a megabyte or more to a Liftoff-only
|
|
34
|
+
* child sized from the file; the scanner sends large files to its own
|
|
35
|
+
* child. Unset (tests, small tools), everything is redacted in process.
|
|
36
|
+
*/
|
|
37
|
+
export interface SourceRedactionOptions {
|
|
38
|
+
/** 'interactive' when an agent is waiting on this file (file_push). */
|
|
39
|
+
priority?: 'interactive' | 'background';
|
|
40
|
+
}
|
|
41
|
+
export interface SourceRedactor {
|
|
42
|
+
routes(filePath: string, bytes: number): boolean;
|
|
43
|
+
redact(text: string, filePath: string, options?: SourceRedactionOptions): Promise<string>;
|
|
44
|
+
}
|
|
45
|
+
let outOfProcess: SourceRedactor | null = null;
|
|
46
|
+
export function installSourceRedactor(redactor: SourceRedactor | null): void {
|
|
47
|
+
outOfProcess = redactor;
|
|
48
|
+
}
|
|
49
|
+
|
|
18
50
|
/** Source syntax stays executable; unsupported or incomplete input uses the conservative scrubber. */
|
|
19
|
-
export async function scrubSourceSecrets(
|
|
51
|
+
export async function scrubSourceSecrets(
|
|
52
|
+
text: string,
|
|
53
|
+
filePath: string,
|
|
54
|
+
options?: SourceRedactionOptions
|
|
55
|
+
): Promise<string> {
|
|
56
|
+
if (outOfProcess && text && outOfProcess.routes(filePath, Buffer.byteLength(text, 'utf8')))
|
|
57
|
+
return outOfProcess.redact(text, filePath, options);
|
|
58
|
+
return scrubSourceSecretsInProcess(text, filePath);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The redaction itself, in this process. The out-of-process worker calls this. */
|
|
62
|
+
export async function scrubSourceSecretsInProcess(text: string, filePath: string): Promise<string> {
|
|
20
63
|
const extension = extname(filePath).toLowerCase();
|
|
21
64
|
if (extension === '.vue') return scrubVueSource(text);
|
|
22
65
|
if (extension === '.svelte') return scrubSvelteSource(text);
|
|
@@ -136,7 +179,7 @@ export async function scrubSourceSecrets(text: string, filePath: string): Promis
|
|
|
136
179
|
? marker + clean
|
|
137
180
|
: clean;
|
|
138
181
|
};
|
|
139
|
-
if (!text || !language || Buffer.byteLength(text, 'utf8') >
|
|
182
|
+
if (!text || !language || Buffer.byteLength(text, 'utf8') > MAX_SOURCE_REDACTION_BYTES) {
|
|
140
183
|
return fallback();
|
|
141
184
|
}
|
|
142
185
|
let parser: Parser | undefined;
|
|
@@ -1833,6 +1876,32 @@ function redactNode(node: Node, sensitive = false): string {
|
|
|
1833
1876
|
}
|
|
1834
1877
|
|
|
1835
1878
|
/** Sanitize parser-owned payload spans; no SFC boundary is inferred from text matching. */
|
|
1879
|
+
/**
|
|
1880
|
+
* Apply replacement spans to `text` in one pass, or null when two overlap
|
|
1881
|
+
* (or a span is malformed).
|
|
1882
|
+
* The result equals applying them one at a time from the last start to the
|
|
1883
|
+
* first (spans that start together: the one added later lands first), which
|
|
1884
|
+
* is what the Vue scrubber did with a slice-and-concatenate per edit; that
|
|
1885
|
+
* copied the whole file for every edit, about 40 s for a 1 MB component.
|
|
1886
|
+
*/
|
|
1887
|
+
export function applySpanEdits(
|
|
1888
|
+
text: string,
|
|
1889
|
+
edits: ReadonlyArray<{ start: number; end: number; clean: string }>
|
|
1890
|
+
): string | null {
|
|
1891
|
+
// Stable sort, then reversed: ascending start, and among equal starts the
|
|
1892
|
+
// later-added edit first, exactly as the one-at-a-time order placed them.
|
|
1893
|
+
const ordered = [...edits].sort((a, b) => b.start - a.start).reverse();
|
|
1894
|
+
const pieces: string[] = [];
|
|
1895
|
+
let offset = 0;
|
|
1896
|
+
for (const e of ordered) {
|
|
1897
|
+
if (e.start < offset || e.end < e.start || e.end > text.length) return null;
|
|
1898
|
+
pieces.push(text.slice(offset, e.start), e.clean);
|
|
1899
|
+
offset = e.end;
|
|
1900
|
+
}
|
|
1901
|
+
pieces.push(text.slice(offset));
|
|
1902
|
+
return pieces.join('');
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1836
1905
|
async function scrubVueSource(text: string): Promise<string> {
|
|
1837
1906
|
if (!text) return text;
|
|
1838
1907
|
const parsed = await parseVueSource(text, true);
|
|
@@ -1973,14 +2042,8 @@ async function scrubVueSource(text: string): Promise<string> {
|
|
|
1973
2042
|
const cleanTail = scrubPayload(tail);
|
|
1974
2043
|
if (cleanTail !== tail) faithful = false;
|
|
1975
2044
|
edit(gapStart, text.length, cleanTail);
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
let boundary = text.length;
|
|
1979
|
-
for (const e of edits) {
|
|
1980
|
-
if (e.end > boundary) return fallback();
|
|
1981
|
-
clean = clean.slice(0, e.start) + e.clean + clean.slice(e.end);
|
|
1982
|
-
boundary = e.start;
|
|
1983
|
-
}
|
|
2045
|
+
const clean = applySpanEdits(text, edits);
|
|
2046
|
+
if (clean === null) return fallback();
|
|
1984
2047
|
if ('reason' in (await parseVueSource(clean))) faithful = false;
|
|
1985
2048
|
return faithful || clean.startsWith(VUE_FIDELITY_MARKER) ? clean : VUE_FIDELITY_MARKER + clean;
|
|
1986
2049
|
} catch {
|
|
@@ -1996,29 +2059,33 @@ async function scrubSvelteSource(text: string): Promise<string> {
|
|
|
1996
2059
|
const fallback = (): string => (text.startsWith(marker) ? '' : marker) + scrubPayload(text);
|
|
1997
2060
|
if (!text) return text;
|
|
1998
2061
|
if (text.includes('mnemonik:svelte-source-fidelity-unavailable')) return fallback();
|
|
1999
|
-
if (Buffer.byteLength(text, 'utf8') >
|
|
2062
|
+
if (Buffer.byteLength(text, 'utf8') > MAX_SOURCE_REDACTION_BYTES) return fallback();
|
|
2000
2063
|
let parser: Parser | undefined;
|
|
2001
2064
|
try {
|
|
2002
|
-
|
|
2065
|
+
// No clock and no node count decide this file: both refused large
|
|
2066
|
+
// components outright (a 4 s deadline and 200,000 nodes). Memory is
|
|
2067
|
+
// bounded by the process the parse runs in (installSourceRedactor).
|
|
2003
2068
|
const { parse } = await import('svelte/compiler');
|
|
2004
2069
|
const ast = parse(text, { modern: true });
|
|
2005
2070
|
const edits: Array<{ start: number; end: number; clean: string }> = [];
|
|
2006
2071
|
let faithful = true;
|
|
2007
|
-
|
|
2072
|
+
// Grammars first: loading one initializes tree-sitter, and a Parser
|
|
2073
|
+
// constructed before that throws. In a fresh process (a redaction child,
|
|
2074
|
+
// or a scanner whose first file is a component) that made every Svelte
|
|
2075
|
+
// file take the unfaithful fallback.
|
|
2008
2076
|
const js = await loadGrammar('javascript');
|
|
2009
2077
|
const ts = await loadGrammar('typescript');
|
|
2010
2078
|
if ('unavailable' in js || 'unavailable' in ts) return fallback();
|
|
2079
|
+
parser = new Parser();
|
|
2011
2080
|
parser.setLanguage(js.ok.language);
|
|
2012
2081
|
const activeParser = parser;
|
|
2013
2082
|
const syntax = (source: string, expression = false, sensitive = false): string => {
|
|
2014
2083
|
const prefix = expression ? `const ${sensitive ? 'password' : '__mnemonik_svelte'} = (` : '';
|
|
2015
2084
|
const suffix = expression ? '\n);' : '';
|
|
2016
2085
|
const full = prefix + source + suffix;
|
|
2017
|
-
const tree = activeParser.parse(full
|
|
2018
|
-
progressCallback: () => performance.now() > deadline,
|
|
2019
|
-
});
|
|
2086
|
+
const tree = activeParser.parse(full);
|
|
2020
2087
|
try {
|
|
2021
|
-
if (!tree || tree.rootNode.hasError
|
|
2088
|
+
if (!tree || tree.rootNode.hasError) {
|
|
2022
2089
|
faithful = false;
|
|
2023
2090
|
return sensitive ? literalPayload(source, true) : scrubPayload(source);
|
|
2024
2091
|
}
|
|
@@ -2062,9 +2129,7 @@ async function scrubSvelteSource(text: string): Promise<string> {
|
|
|
2062
2129
|
const pending: Array<{ value: unknown; sensitive: boolean }> = [
|
|
2063
2130
|
{ value: ast.fragment, sensitive: false },
|
|
2064
2131
|
];
|
|
2065
|
-
let count = 0;
|
|
2066
2132
|
while (pending.length) {
|
|
2067
|
-
if (++count > 200_000 || performance.now() > deadline) return fallback();
|
|
2068
2133
|
const entry = pending.pop();
|
|
2069
2134
|
if (!entry || !entry.value || typeof entry.value !== 'object') continue;
|
|
2070
2135
|
if (Array.isArray(entry.value)) {
|