mtech-ai-local-runner 0.1.43 → 0.1.44

package/README.md

@@ -1,3 +1,13 @@
+## 0.1.44: stop repeating whole-checkout checks in P1 verification
+
+The P1 verifier rereads the whole model checkout for every source check and every candidate check. On the AI
+Gateway repository (3,781 files) the live pilot spent about 12.5 minutes in `p1_verify`, and part of that was the same
+check run twice with nothing in between. Each command is now bracketed by one fresh source check and one fresh
+candidate check: before the first command, between commands and after the last. A source re-check compares the Git
+listings and every file's mode and sha256 with the values recorded at the seal instead of rereading every base blob,
+and an object already present in the private checkout is verified without being compressed again. Every file is
+still read and hashed at each check, so the same changes are rejected as before.
+
 ## 0.1.43: give whole-checkout snapshots whole-checkout bounds
 
 The P1 verifier snapshots the whole model checkout before it runs the reviewed commands. It passed the candidate limits

package/dist/src/publication/git-snapshot.js

@@ -3,11 +3,10 @@ import { promisify } from 'node:util';
 import { constants } from 'node:fs';
 import { lstat, mkdir, mkdtemp, open, realpath, readdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { createHash } from 'node:crypto';
-import { deflateSync } from 'node:zlib';
 import path from 'node:path';
 import { measureCommandFile } from '../safety/registered-command-integrity.js';
 import { protocolHash } from './candidate.js';
-import { RawGitObjects } from './raw-git-objects.js';
+import { deflateObject, RawGitObjects } from './raw-git-objects.js';
 const execute = promisify(execFile), denied = () => Error('SNAPSHOT_REJECTED');
 const digest = (data) => createHash('sha256').update(data).digest('hex');
 const safe = (p) => p.length > 0 && !path.isAbsolute(p) && !p.includes('\\') && !p.includes('\0') && p.split('/').every(s => s !== '' && s !== '.' && s !== '..' && !/^(?:\.git|\.ssh|\.aws|\.codex|\.claude|\.env|\.netrc|\.npmrc|credentials(?:\..*)?)$/i.test(s));
@@ -21,6 +20,7 @@ export const isP1WorkingBranch = (branch) => typeof branch === 'string' && branc
 export class GitSnapshotHost {
     options;
     seals = new WeakSet();
+    listings = new WeakMap();
     ownedRoots = new Set();
     constructor(options) { this.options = structuredClone(options); }
     async git(args) {
@@ -135,15 +135,21 @@ export class GitSnapshotHost {
             await fd.close();
         }
     }
-    async capture() {
+    /** Checks the options every read relies on, then that HEAD is the base commit on the working branch. */
+    async head() {
         const o = this.options, branchAllowed = o.p1WorkingBranch === undefined ? /^ai-work\/local\/[A-Za-z0-9-]+$/.test(o.workingBranch) : o.p1WorkingBranch === o.workingBranch && isP1WorkingBranch(o.workingBranch);
         if (!/^[a-f0-9]{40}$/.test(o.baseCommitSha) || !branchAllowed || await realpath(o.root) !== o.root || !Number.isSafeInteger(o.maxFileBytes) || o.maxFileBytes < 1 || !Number.isSafeInteger(o.maxTotalBytes) || o.maxTotalBytes < o.maxFileBytes)
             throw denied();
         const head = (await this.git(['rev-parse', '--verify', 'HEAD'])).toString().trim(), branch = (await this.git(['symbolic-ref', '--short', 'HEAD'])).toString().trim();
         if (head !== o.baseCommitSha || branch !== o.workingBranch)
             throw denied();
-        const base = new Map(), gitlinks = new Map();
-        for (const line of (await this.git(['ls-tree', '-r', '-z', '--full-tree', o.baseCommitSha])).toString().split('\0').filter(Boolean)) {
+    }
+    async capture() {
+        const o = this.options;
+        await this.head();
+        const head = o.baseCommitSha, branch = o.workingBranch;
+        const base = new Map(), gitlinks = new Map(), baseListing = await this.git(['ls-tree', '-r', '-z', '--full-tree', o.baseCommitSha]);
+        for (const line of baseListing.toString().split('\0').filter(Boolean)) {
             const link = /^160000 commit ([a-f0-9]{40})\t(.+)$/s.exec(line);
             if (link) {
                 if (!safe(link[2]))
@@ -158,8 +164,8 @@ export class GitSnapshotHost {
         }
         // Gitlinks remain exact base-tree references. Never traverse or copy their source.
         // Staged replacement/deletion is refused rather than publishing hidden changes.
-        const staged = new Map();
-        for (const line of (await this.git(['ls-files', '--stage', '-z'])).toString().split('\0').filter(Boolean)) {
+        const staged = new Map(), stagedListing = await this.git(['ls-files', '--stage', '-z']);
+        for (const line of stagedListing.toString().split('\0').filter(Boolean)) {
             const entry = /^(100644|100755|160000) ([a-f0-9]{40}) 0\t(.+)$/s.exec(line);
             if (!entry || !safe(entry[3]))
                 throw denied();
@@ -170,11 +176,12 @@ export class GitSnapshotHost {
             expected.set(name, '160000 ' + oid);
         if (protocolHash([...expected].sort()) !== protocolHash([...staged].sort()))
             throw denied();
-        const names = new Set([...base.keys(), ...(await this.git(['ls-files', '--others', '--exclude-standard', '-z', '--'])).toString().split('\0').filter(Boolean)]);
+        const untrackedListing = await this.git(['ls-files', '--others', '--exclude-standard', '-z', '--']);
+        const names = [...new Set([...base.keys(), ...untrackedListing.toString().split('\0').filter(Boolean)])].sort();
         const blobs = await this.batchObjects([...base.values()].map(v => v.blob));
         const files = [], changes = [];
         let total = 0;
-        for (const name of [...names].sort()) {
+        for (const name of names) {
             if ([...gitlinks.keys()].some(p => name === p || name.startsWith(p + '/') || p.startsWith(name + '/')))
                 throw denied();
             const current = await this.currentFile(name), prior = base.get(name);
@@ -196,7 +203,30 @@ export class GitSnapshotHost {
                 changes.push({ path: name, status: !prior ? 'added' : !current ? 'deleted' : 'modified', before: before?.toString('base64') ?? null, after: current?.bytes.toString('base64') ?? null, before_mode: prior?.mode ?? null, after_mode: current?.mode ?? null });
         }
         const observation = { base_commit_sha: o.baseCommitSha, head_commit_sha: head, working_branch: branch, diff: JSON.stringify({ format: 'local-candidate-binary-diff-v1', changes }), changed_files: changes.map(({ path, status }) => ({ path, status })) };
-        return { observation, files, fingerprint: protocolHash({ observation, files: files.map(f => ({ path: f.path, mode: f.mode, sha256: digest(f.bytes) })) }) };
+        const listing = { base: digest(baseListing), staged: digest(stagedListing), untracked: digest(untrackedListing), names };
+        return { observation, files, listing, fingerprint: protocolHash({ observation, files: files.map(f => ({ path: f.path, mode: f.mode, sha256: digest(f.bytes) })) }) };
+    }
+    /** The checkout still equals a capture when HEAD, the branch, the three Git listings and every
+     * file's mode and sha256 are unchanged. The capture's observation follows from those alone: the
+     * base blobs that an unchanged listing names are immutable objects, so they are not read again. */
+    async matches(listing, treeFingerprint) {
+        const o = this.options;
+        await this.head();
+        if (digest(await this.git(['ls-tree', '-r', '-z', '--full-tree', o.baseCommitSha])) !== listing.base || digest(await this.git(['ls-files', '--stage', '-z'])) !== listing.staged || digest(await this.git(['ls-files', '--others', '--exclude-standard', '-z', '--'])) !== listing.untracked)
+            throw denied();
+        const files = [];
+        let total = 0;
+        for (const name of listing.names) {
+            const current = await this.currentFile(name);
+            if (!current)
+                continue;
+            total += current.bytes.length;
+            if (total > o.maxTotalBytes)
+                throw denied();
+            files.push({ path: current.path, mode: current.mode, sha256: digest(current.bytes) });
+        }
+        if (protocolHash(files) !== treeFingerprint)
+            throw denied();
     }
     /** Creates a separate local-only Git checkout with exact base parent objects.
      * Raw object copying and raw file writes invoke no filters, templates or hooks;
@@ -241,7 +271,7 @@ export class GitSnapshotHost {
                     throw denied();
                 const dir = path.join(gitdir, 'objects', oid.slice(0, 2));
                 await mkdir(dir, { recursive: true, mode: 0o700 });
-                await writeFile(path.join(dir, oid.slice(2)), deflateSync(raw), { flag: 'wx', mode: 0o400 });
+                await writeFile(path.join(dir, oid.slice(2)), deflateObject(raw), { flag: 'wx', mode: 0o400 });
             }
             const copy = async (from, to) => { for (const name of await readdir(from)) {
                 const source = path.join(from, name), target = path.join(to, name), stat = await lstat(source);
@@ -272,11 +302,10 @@ export class GitSnapshotHost {
     async publicationCheckout(sealed) { return this.privateCheckout(sealed); }
     async assertSourceUnchanged(sealed) {
         try {
-            if (!this.seals.has(sealed))
-                throw denied();
-            const capture = await this.capture();
-            if (protocolHash(capture.observation) !== protocolHash(sealed.observation) || protocolHash(capture.files.map(f => ({ path: f.path, mode: f.mode, sha256: digest(f.bytes) }))) !== sealed.treeFingerprint)
+            const listing = this.listings.get(sealed);
+            if (!this.seals.has(sealed) || !listing)
                 throw denied();
+            await this.matches(listing, sealed.treeFingerprint);
         }
         catch {
             throw denied();
@@ -371,9 +400,9 @@ export class GitSnapshotHost {
                 await mkdir(path.dirname(file), { recursive: true, mode: 0o700 });
                 await writeFile(file, f.bytes, { flag: 'wx', mode: (f.mode & 0o111) ? 0o500 : 0o400 });
             }
-            if ((await this.capture()).fingerprint !== capture.fingerprint)
-                throw denied();
+            // The source must not have changed while it was copied.
             const expected = protocolHash(capture.files.map(f => ({ path: f.path, mode: f.mode, sha256: digest(f.bytes) })));
+            await this.matches(capture.listing, expected);
             const assertUnchanged = async () => {
                 const entries = [];
                 const walk = async (dir, prefix = '') => { for (const item of (await readdir(dir)).sort()) {
@@ -396,6 +425,7 @@ export class GitSnapshotHost {
             await assertUnchanged();
             const seal = { root, observation: structuredClone(capture.observation), treeFingerprint: expected, assertUnchanged };
             this.seals.add(seal);
+            this.listings.set(seal, capture.listing);
             return Object.freeze(seal);
         }
         catch {

package/dist/src/publication/raw-git-objects.js

@@ -2,6 +2,8 @@ import { createHash } from 'node:crypto';
 import { deflateSync, inflateSync } from 'node:zlib';
 import { mkdir, readFile, writeFile } from 'node:fs/promises';
 import path from 'node:path';
+/** Git reads any zlib level, and these objects live only as long as one private checkout. */
+export const deflateObject = (raw) => deflateSync(raw, { level: 1 });
 /** Local raw objects only: no filters, hooks, Git identity/config or transport. */
 export class RawGitObjects {
     checkout;
@@ -13,13 +15,21 @@ export class RawGitObjects {
         if (write) {
             const dir = path.join(this.checkout, '.git', 'objects', oid.slice(0, 2)), file = path.join(dir, oid.slice(2));
             await mkdir(dir, { recursive: true, mode: 0o700 });
-            try {
-                await writeFile(file, deflateSync(raw), { flag: 'wx', mode: 0o400 });
-            }
-            catch (e) {
-                if (e.code !== 'EEXIST' || !inflateSync(await readFile(file)).equals(raw))
-                    throw Error('GIT_OBJECT_REJECTED');
-            }
+            // An object already present is verified, not compressed again only to be discarded.
+            let existing = await readFile(file).catch((e) => { if (e.code !== 'ENOENT')
+                throw Error('GIT_OBJECT_REJECTED'); return undefined; });
+            if (existing === undefined)
+                try {
+                    await writeFile(file, deflateObject(raw), { flag: 'wx', mode: 0o400 });
+                    return oid;
+                }
+                catch (e) {
+                    if (e.code !== 'EEXIST')
+                        throw Error('GIT_OBJECT_REJECTED');
+                    existing = await readFile(file);
+                }
+            if (!inflateSync(existing).equals(raw))
+                throw Error('GIT_OBJECT_REJECTED');
         }
         return oid;
     }

package/dist/src/verification/runner-observed-verifier.js

@@ -108,9 +108,7 @@ export class RunnerObservedVerifier {
             if (observation.base_commit_sha !== f.base_commit_sha || observation.head_commit_sha !== f.base_commit_sha || observation.working_branch !== f.working_branch || !observation.changed_files.length || observation.changed_files.length > o.candidateLimits.max_changed_files || Buffer.byteLength(observation.diff) > o.candidateLimits.max_diff_bytes || observation.changed_files.some(e => !o.allowedPaths.includes(e.path) || !exactPath(e.path) || isBlocked(e.path, o.blockedPaths)) || changedFileBytes(observation.diff).some(n => n > o.candidateLimits.max_file_bytes))
                 throw denied();
             await this.active();
-            await sealed.assertUnchanged();
             const candidate = await host.verificationCandidate(sealed);
-            await candidate.assertUnchanged();
             if (!hex(candidate.commitSha, 40))
                 throw denied();
             const changed_paths = buildP1ChangedPaths({ candidate_commit_sha: candidate.commitSha, base_commit_sha: f.base_commit_sha, entries: observation.changed_files }, v.bounds);
@@ -118,15 +116,15 @@ export class RunnerObservedVerifier {
             const registered = await buildReviewedProjectRegistration({ root: candidate.root, manifest: o.registration.manifest, review: o.registration.review, allowedPaths: o.allowedPaths, blockedPaths: o.blockedPaths });
             const policy = await createIsolatedCommandPolicy({ root: candidate.root, commands: registered.commands, allowedPaths: o.allowedPaths, blockedPaths: o.blockedPaths }, true);
             const receipts = [];
+            // One fresh source and candidate check before the first command, between commands and after the
+            // last. Each check rereads the whole checkout, and repeating one with nothing run in between proves nothing.
+            const snapshots = host, check = async () => { await this.active(); await snapshots.assertSourceUnchanged(sealed); await candidate.assertUnchanged(); };
             try {
-                for (const command of registered.commands) {
-                    await this.active();
-                    await host.assertSourceUnchanged(sealed);
-                    await candidate.assertUnchanged();
+                await check();
+                for (const [index, command] of registered.commands.entries()) {
                     const measurement = await policy.measureTest(command.id, { timeoutMs: v.bounds.max_command_runtime_ms, maxOutputBytes: v.bounds.max_output_bytes }, o.signal);
-                    await this.active();
-                    await host.assertSourceUnchanged(sealed);
-                    await candidate.assertUnchanged();
+                    if (index < registered.commands.length - 1)
+                        await check();
                     const identity = { binding_id: f.binding_id, job_id: f.job_id, generation: f.generation, contract_fingerprint: f.contract_fingerprint, profile_fingerprint: v.profile_fingerprint, frozen_criteria_fingerprint: v.frozen_criteria_fingerprint, candidate_commit_sha: candidate.commitSha, snapshot_fingerprint, command_id: command.id, command_fingerprint: protocolHash(command) };
                     receipts.push({ schema_version: 'runner-command-receipt-v1', receipt_id: protocolHash({ digest_domain: 'p1-command-receipt-id-v1', ...identity }), ...identity, observed_at: Math.floor(Date.now() / 1000), ...measurement });
                 }
@@ -134,9 +132,7 @@ export class RunnerObservedVerifier {
             finally {
                 policy.revoke();
             }
-            await this.active();
-            await host.assertSourceUnchanged(sealed);
-            await candidate.assertUnchanged();
+            await check();
             return { candidate_commit_sha: candidate.commitSha, snapshot_fingerprint, changed_paths, receipts };
         }
         catch {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mtech-ai-local-runner",
-  "version": "0.1.43",
+  "version": "0.1.44",
   "private": false,
   "license": "UNLICENSED",
   "bin": {