@xuda.io/ai_module 1.1.5658 → 1.1.5660

package/xucode_proxy.mjs

@@ -0,0 +1,426 @@
+// The Xucode provider proxy: the region server holds the credential, the machine never does.
+//
+// docs/plans/xucode.md 9.2 built the git mirror so the CUSTOMER'S repository token never reaches
+// a machine that is running code an AI just wrote. This file is the same argument pointed at the
+// other credential. Once the engine moved onto the machine (the clone fix), the only way codex
+// could authenticate was for OUR platform OpenAI key to be sitting there, on a box whose whole
+// purpose is executing untrusted code. A shared key with our billing behind it is a worse thing
+// to leak than one customer's repo token, so it stays here and the machine gets a token that is
+// worth nothing anywhere else.
+//
+// What the machine receives is a per-run bearer that only this endpoint understands:
+//
+//   - it is signed, so it cannot be minted on the machine
+//   - it expires with the run, so a copied token dies on its own
+//   - it names a path allow-list, so it cannot be pointed at unrelated provider APIs
+//   - it is counted, so a machine that decides to mine tokens hits a wall
+//
+// A machine that is fully compromised can still spend the run's own budget through this endpoint
+// while the run is open. That is a much smaller blast radius than handing over the key itself,
+// and it is bounded in time, in path and in request count. It is not zero, and section 9.3's
+// other layers are what carry the rest.
+//
+// ── Why a signed token and not a database row ─────────────────────────────────────────────────
+// The run executes in the ai_module worker; this endpoint is served by http_module. They are
+// different processes, so an in-memory handle cannot be shared, and a Couch lookup on every
+// single API call would put a database round trip in front of every model request. An HMAC needs
+// no shared state and no lookup: expiry does the revoking.
+
+import crypto from 'node:crypto';
+import https from 'node:https';
+import http from 'node:http';
+import path from 'node:path';
+
+if (!global._conf) {
+  global._conf = (await import(path.join(process.env.XUDA_HOME, 'common', 'load_conf.mjs'))).loadConf();
+}
+
+const proxy_conf = () => global._conf.xucode?.proxy || {};
+
+// ── Upstreams ─────────────────────────────────────────────────────────────────────────────────
+// Only providers we actually front. An unknown provider is refused rather than defaulted, because
+// defaulting here would mean forwarding a credential somewhere nobody chose.
+export const UPSTREAMS = {
+  openai: { origin: 'https://api.openai.com', auth: 'bearer' },
+  anthropic: { origin: 'https://api.anthropic.com', auth: 'x-api-key' },
+};
+
+// The paths an engine legitimately needs. Anything else is refused: this endpoint exists to let
+// codex do its job, not to be a general purpose OpenAI account for whoever holds the token.
+// `metered` marks the ones that actually spend money, which is what the per-run cap counts.
+const ALLOWED = {
+  openai: [
+    { re: /^\/v1\/responses(\/[A-Za-z0-9_-]+)?$/, methods: ['POST', 'GET', 'DELETE'], metered: true },
+    { re: /^\/v1\/responses\/[A-Za-z0-9_-]+\/cancel$/, methods: ['POST'], metered: false },
+    { re: /^\/v1\/chat\/completions$/, methods: ['POST'], metered: true },
+    { re: /^\/v1\/models$/, methods: ['GET'], metered: false },
+  ],
+  anthropic: [
+    { re: /^\/v1\/messages$/, methods: ['POST'], metered: true },
+    { re: /^\/v1\/messages\/count_tokens$/, methods: ['POST'], metered: false },
+    { re: /^\/v1\/models$/, methods: ['GET'], metered: false },
+  ],
+};
+
+export const match_route = function (provider, pathname, method) {
+  for (const rule of ALLOWED[provider] || []) {
+    if (rule.re.test(pathname)) {
+      return rule.methods.includes(String(method || '').toUpperCase()) ? { ok: true, metered: rule.metered } : { ok: false, reason: 'method not allowed on this path' };
+    }
+  }
+  return { ok: false, reason: 'this path is not available through Xucode' };
+};
+
+// ── The signing secret ────────────────────────────────────────────────────────────────────────
+// Derived from the provider key rather than added to the secrets file, for two reasons: it needs
+// no new secret to distribute to every region, and rotating the provider key invalidates every
+// outstanding token for free, which is exactly what rotating a key should do. An explicit secret
+// in config wins if anyone ever wants to separate the two.
+export const proxy_secret = function (conf = global._conf) {
+  const explicit = conf?.xucode?.proxy?.secret;
+  if (explicit) return String(explicit);
+  const key = conf?.OPENAI_API_KEY || '';
+  if (!key) return null;
+  return crypto.createHash('sha256').update(`xucode-proxy/v1|${key}`).digest('hex');
+};
+
+const b64url = (buf) => Buffer.from(buf).toString('base64').replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, '');
+const b64url_decode = (s) => Buffer.from(String(s).replace(/-/g, '+').replace(/_/g, '/'), 'base64');
+
+const sign = (payload_b64, secret) => b64url(crypto.createHmac('sha256', secret).update(payload_b64).digest());
+
+// ── Tokens ────────────────────────────────────────────────────────────────────────────────────
+// Deliberately short field names: this value is passed through a shell environment and a TOML
+// file, and a long token is one more thing that can get truncated somewhere unhelpful.
+// `key_app_id` names the account database holding the customer's OWN key for this provider, and
+// its absence means "use the platform key". It is an identifier, not a credential: a machine that
+// steals a token can already spend that run, and the id buys nothing without database access. The
+// alternative was resolving accounts inside http_module, or a broker method that returns a raw
+// customer key, and that second one is exactly what xucode_resolve_api_key refuses to be.
+export const mint_token = function ({ run_id, uid, project_id, engine, provider, key_app_id = null, ttl_ms, secret = proxy_secret(), now = Date.now() } = {}) {
+  if (!secret) return { error: 'the proxy has no signing secret (no provider key on this box)' };
+  if (!run_id || !provider) return { error: 'a run token needs a run id and a provider' };
+  if (!UPSTREAMS[provider]) return { error: `unknown provider ${provider}` };
+  const payload = {
+    r: String(run_id),
+    u: uid ? String(uid) : '',
+    p: project_id ? String(project_id) : '',
+    e: engine ? String(engine) : '',
+    v: provider,
+    x: now + (Number(ttl_ms) || 30 * 60 * 1000),
+  };
+  if (key_app_id) payload.k = String(key_app_id);
+  const body = b64url(JSON.stringify(payload));
+  return { token: `xc1.${body}.${sign(body, secret)}`, expires_at: payload.x };
+};
+
+export const verify_token = function (token, { secret = proxy_secret(), now = Date.now() } = {}) {
+  if (!secret) return { ok: false, reason: 'proxy not configured' };
+  const parts = String(token || '').split('.');
+  if (parts.length !== 3 || parts[0] !== 'xc1') return { ok: false, reason: 'malformed token' };
+  const expected = sign(parts[1], secret);
+  // Compare through timingSafeEqual, and only after a length check: it throws on a length
+  // mismatch, and an exception here would read as a server fault rather than a bad token.
+  const a = Buffer.from(expected);
+  const b = Buffer.from(parts[2]);
+  if (a.length !== b.length || !crypto.timingSafeEqual(a, b)) return { ok: false, reason: 'bad signature' };
+  let claims;
+  try {
+    claims = JSON.parse(b64url_decode(parts[1]).toString('utf8'));
+  } catch {
+    return { ok: false, reason: 'malformed token' };
+  }
+  if (!claims || typeof claims !== 'object') return { ok: false, reason: 'malformed token' };
+  if (!UPSTREAMS[claims.v]) return { ok: false, reason: 'unknown provider' };
+  if (!(Number(claims.x) > now)) return { ok: false, reason: 'this run has finished' };
+  return { ok: true, claims };
+};
+
+// ── Whose key answers a run ───────────────────────────────────────────────────────────────────
+// Two cases, decided by the token rather than by anything the machine says:
+//
+//   the token names an account database  ->  that customer's own key (BYOK)
+//   it does not                          ->  our platform key, for the engines we front
+//
+// A machine cannot move itself between those cases: the claim is signed. It cannot reach another
+// account's key either, because the only account it can name is the one the token was minted for.
+//
+// Cached per RUN, expiring with the token, so a run costs one database read rather than one per
+// model request. Caching a miss is deliberate: a key deleted mid-run should stay refused for that
+// run rather than becoming a lookup on every retry.
+export const create_key_resolver = function ({ db_query, platform_key = () => global._conf?.OPENAI_API_KEY || null, now = () => Date.now() } = {}) {
+  if (typeof db_query !== 'function') throw new Error('create_key_resolver needs db_query');
+  const cache = new Map();
+
+  return async function resolve_key(claims) {
+    if (!claims?.k) {
+      // No customer key for this provider. We only front OpenAI; an anthropic run that gets here
+      // has lost its key between mint and use, and inventing one of ours would put Anthropic
+      // charges on our account for a customer's run.
+      return claims?.v === 'openai' ? platform_key() : null;
+    }
+
+    const hit = cache.get(claims.r);
+    if (hit && hit.exp > now()) return hit.key;
+
+    let key = null;
+    try {
+      const q = await db_query(claims.k, { selector: { docType: 'xucode_key', provider: claims.v, stat: 3 }, limit: 1 });
+      key = q?.docs?.[0]?.api_key || null;
+    } catch (err) {
+      console.error(`[xucode-proxy] could not read the account key: ${err?.message || err}`);
+      return null;
+    }
+
+    cache.set(claims.r, { key, exp: Number(claims.x) || now() });
+    for (const [k, v] of cache) if (v.exp <= now()) cache.delete(k);
+    return key;
+  };
+};
+
+// ── Usage, read off the wire ──────────────────────────────────────────────────────────────────
+// The engine already reports its own usage, but the engine runs on a machine we are choosing not
+// to trust with a key, so its self-report is not evidence. What passes through here is, and it
+// costs one scan of the bytes we are already forwarding.
+//
+// Streaming responses arrive as SSE, where only `data:` lines carry JSON; non-streaming ones are
+// a single JSON body. Both are handled, and neither is allowed to throw: a parse failure means we
+// lose a usage number, which must never mean the customer loses their run.
+export const create_usage_scanner = function (on_usage) {
+  let buffer = '';
+  let plain = '';
+  let reported = null;
+
+  const take = (obj) => {
+    const u = obj?.usage || obj?.response?.usage || obj?.message?.usage;
+    if (!u || typeof u !== 'object') return;
+    const input = Number(u.input_tokens ?? u.prompt_tokens ?? 0) || 0;
+    const output = Number(u.output_tokens ?? u.completion_tokens ?? 0) || 0;
+    if (!input && !output) return;
+    // Providers report cumulatively within a stream, so the last complete report wins rather
+    // than a sum, which would multiply a long conversation by the number of events in it.
+    reported = { input_tokens: input, output_tokens: output };
+  };
+
+  return {
+    write(chunk) {
+      const text = chunk.toString('utf8');
+      buffer += text;
+      const lines = buffer.split('\n');
+      buffer = lines.pop() || '';
+      let sse = false;
+      for (const line of lines) {
+        if (!line.startsWith('data:')) continue;
+        sse = true;
+        const body = line.slice(5).trim();
+        if (!body || body === '[DONE]') continue;
+        try {
+          take(JSON.parse(body));
+        } catch {
+          // A split JSON event, or a keep-alive comment. Not worth reconstructing.
+        }
+      }
+      // A non-streaming reply has no `data:` lines at all, so keep a bounded copy to parse at
+      // the end. Bounded because a large embeddings-style body must not be buffered whole.
+      if (!sse && plain.length < 262144) plain += text;
+    },
+    end() {
+      if (!reported && plain) {
+        try {
+          take(JSON.parse(plain));
+        } catch {
+          // Not JSON, or truncated by the cap. Nothing to report.
+        }
+      }
+      if (reported && typeof on_usage === 'function') on_usage(reported);
+      return reported;
+    },
+  };
+};
+
+// Headers that belong to one hop and must not be copied onto the next, plus the ones we replace
+// ourselves. Passing `host` through would send api.openai.com a Host of dev.xuda.ai and get a
+// certificate error; passing `authorization` through would forward the run token upstream, where
+// it means nothing.
+// `accept-encoding` is dropped so the upstream answers in plain bytes, and that is a deliberate
+// choice rather than an oversight. Cloudflare adds `Accept-Encoding: gzip` to everything it sends
+// an origin, so without this the provider compresses, and a compressed body is unreadable to the
+// usage scanner below: metering would silently report nothing on every real call. It also removes
+// any chance of an event stream arriving gzipped, which is a slow, confusing way for streaming to
+// break. The extra bytes are between two servers and cost nothing worth having.
+const DROP_REQUEST_HEADERS = new Set(['host', 'authorization', 'x-api-key', 'connection', 'keep-alive', 'proxy-authorization', 'proxy-connection', 'transfer-encoding', 'upgrade', 'te', 'trailer', 'cookie', 'content-length', 'accept-encoding', 'cf-connecting-ip', 'cf-ipcountry', 'cf-ray', 'cf-visitor', 'x-forwarded-for', 'x-forwarded-proto', 'x-forwarded-host', 'x-real-ip']);
+// `content-encoding` is NOT dropped. Stripping it while forwarding the bytes verbatim tells the
+// caller a compressed body is plain text, and everything downstream reads gibberish.
+const DROP_RESPONSE_HEADERS = new Set(['connection', 'keep-alive', 'transfer-encoding', 'upgrade', 'trailer', 'content-length']);
+
+const provider_error = (res, status, message, type = 'invalid_request_error') => {
+  if (res.headersSent) return res.end();
+  res.writeHead(status, { 'Content-Type': 'application/json', 'Cache-Control': 'no-transform' });
+  res.end(JSON.stringify({ error: { message, type } }));
+};
+
+const bearer_of = (req) => {
+  const auth = req.headers?.authorization || '';
+  const m = /^Bearer\s+(.+)$/i.exec(String(auth).trim());
+  if (m) return m[1].trim();
+  // Anthropic's SDK sends the credential as x-api-key rather than a bearer, so an engine
+  // pointed here through ANTHROPIC_BASE_URL arrives that way.
+  return String(req.headers?.['x-api-key'] || '').trim() || null;
+};
+
+// ── The endpoint ──────────────────────────────────────────────────────────────────────────────
+// `resolve_key` is injected because deciding WHOSE key answers a run is a policy question that
+// belongs to the caller, not to a forwarder. Today it answers "ours, for codex"; when BYOK keys
+// move behind this endpoint too, that is a change there and nothing here moves.
+export const create_proxy = function ({ resolve_key, secret = null, request_impl = null, now = () => Date.now(), log = (msg) => console.log(msg), on_usage = null } = {}) {
+  if (typeof resolve_key !== 'function') throw new Error('create_proxy needs resolve_key');
+
+  // Per-run request counts, in memory. This is a blunt instrument on purpose: it is a ceiling
+  // that stops a runaway or a hijacked machine, not an accounting system. It lives per process,
+  // so with several http workers the effective ceiling is the cap times the worker count, which
+  // is still a ceiling and still bounded.
+  const counts = new Map();
+  const bump = (run_id, exp) => {
+    const cur = counts.get(run_id) || { n: 0, exp };
+    cur.n += 1;
+    cur.exp = exp;
+    counts.set(run_id, cur);
+    return cur.n;
+  };
+  const prune = () => {
+    const t = now();
+    for (const [k, v] of counts) if (v.exp <= t) counts.delete(k);
+  };
+
+  const secret_for = () => secret || proxy_secret();
+
+  return {
+    stats: () => ({ tracked_runs: counts.size }),
+
+    async handle(req, res, rel_path) {
+      try {
+        prune();
+
+        const conf = proxy_conf();
+        if (conf.enabled === false) return provider_error(res, 503, 'Xucode is not accepting engine traffic on this server.');
+
+        const token = bearer_of(req);
+        if (!token) return provider_error(res, 401, 'Missing bearer token.', 'authentication_error');
+
+        const v = verify_token(token, { secret: secret_for(), now: now() });
+        if (!v.ok) return provider_error(res, 401, `Xucode rejected this run token: ${v.reason}.`, 'authentication_error');
+
+        const provider = v.claims.v;
+        // Query strings are forwarded, but the allow-list is matched on the path alone so a
+        // query cannot be used to reach a different endpoint.
+        const q = rel_path.indexOf('?');
+        const pathname = q === -1 ? rel_path : rel_path.slice(0, q);
+        const route = match_route(provider, pathname, req.method);
+        if (!route.ok) {
+          log(`[xucode-proxy] refused ${req.method} ${pathname} for run ${v.claims.r}: ${route.reason}`);
+          return provider_error(res, 403, `Xucode does not allow ${req.method} ${pathname}.`);
+        }
+
+        if (route.metered) {
+          const cap = Number(conf.max_requests_per_run) || 400;
+          const n = bump(v.claims.r, Number(v.claims.x) || now());
+          if (n > cap) {
+            log(`[xucode-proxy] run ${v.claims.r} hit the request cap (${cap})`);
+            return provider_error(res, 429, 'This run has made too many model requests and has been stopped.', 'rate_limit_error');
+          }
+        }
+
+        const key = await resolve_key(v.claims);
+        if (!key) return provider_error(res, 502, 'Xucode has no credential for this engine on this server.', 'api_error');
+
+        const upstream = UPSTREAMS[provider];
+        const target = new URL(upstream.origin + rel_path);
+        const headers = {};
+        for (const [k, val] of Object.entries(req.headers || {})) {
+          if (!DROP_REQUEST_HEADERS.has(k.toLowerCase())) headers[k] = val;
+        }
+        headers.host = target.host;
+        if (upstream.auth === 'x-api-key') {
+          headers['x-api-key'] = key;
+          if (!headers['anthropic-version']) headers['anthropic-version'] = '2023-06-01';
+        } else {
+          headers.authorization = `Bearer ${key}`;
+        }
+
+        const client = target.protocol === 'http:' ? http : https;
+        const doRequest = request_impl || client.request.bind(client);
+
+        const scanner = create_usage_scanner((usage) => {
+          log(`[xucode-proxy] run ${v.claims.r} ${v.claims.e || provider} usage in=${usage.input_tokens} out=${usage.output_tokens}`);
+          if (typeof on_usage === 'function') {
+            try {
+              on_usage({ ...v.claims, ...usage });
+            } catch (err) {
+              console.error(`[xucode-proxy] usage sink failed: ${err?.message || err}`);
+            }
+          }
+        });
+
+        const upreq = doRequest(
+          {
+            protocol: target.protocol,
+            hostname: target.hostname,
+            port: target.port || (target.protocol === 'http:' ? 80 : 443),
+            path: target.pathname + target.search,
+            method: req.method,
+            headers,
+          },
+          (upres) => {
+            const out = {};
+            for (const [k, val] of Object.entries(upres.headers || {})) {
+              if (!DROP_RESPONSE_HEADERS.has(k.toLowerCase())) out[k] = val;
+            }
+            // http_module runs compression() in front of every route. It respects a
+            // no-transform directive, and without one it buffers a text/event-stream into
+            // silence: the customer would watch a run produce nothing until it finished.
+            out['cache-control'] = 'no-transform';
+            res.writeHead(upres.statusCode || 502, out);
+            if (typeof res.flushHeaders === 'function') res.flushHeaders();
+
+            upres.on('data', (chunk) => {
+              scanner.write(chunk);
+              res.write(chunk);
+            });
+            upres.on('end', () => {
+              scanner.end();
+              res.end();
+            });
+            upres.on('error', () => {
+              scanner.end();
+              res.end();
+            });
+          },
+        );
+
+        // A long turn is normal here, so the socket timeout has to outlast one, and the run's
+        // own timeout is the real bound. Too short a value shows up as a run that dies partway
+        // through for no visible reason.
+        upreq.setTimeout(Number(conf.upstream_timeout_ms) || 20 * 60 * 1000, () => {
+          upreq.destroy(new Error('upstream timed out'));
+        });
+
+        upreq.on('error', (err) => {
+          log(`[xucode-proxy] upstream error for run ${v.claims.r}: ${err?.message || err}`);
+          provider_error(res, 502, 'Xucode could not reach the model provider.', 'api_error');
+        });
+
+        // If the engine gives up, stop paying for the answer.
+        const abort = () => upreq.destroy();
+        req.on('aborted', abort);
+        res.on('close', () => {
+          if (!res.writableEnded) abort();
+        });
+
+        req.pipe(upreq);
+      } catch (err) {
+        console.error(`[xucode-proxy] ${err?.message || err}`);
+        provider_error(res, 500, 'Xucode could not process that request.', 'api_error');
+      }
+    },
+  };
+};

package/{xudex_run.mjs → xucode_run.mjs}

@@ -1,6 +1,6 @@
-// A xudex run: the piece that makes the other five worth having.
+// A xucode run: the piece that makes the other five worth having.
 //
-// docs/plans/xudex.md. Everything before this was a component with a test. This is the loop
+// docs/plans/xucode.md. Everything before this was a component with a test. This is the loop
 // the customer actually experiences, and it is the whole product in one function:
 //
 //   mirror  ->  workspace  ->  engine  ->  capture  ->  verify  ->  repair  ->  answer
@@ -33,6 +33,34 @@ export const run_branch = function (conversation_id) {
   return `xuda/${String(conversation_id || '').replace(/^cov_/, '').slice(0, 12)}`;
 };
 
+// Single-quote for the machine's shell. Same rule as the VM substrate: everything crossing this
+// boundary is attacker-influenced, and the one value here is a signed token, so a quoting slip
+// would leak it into a log or a command line.
+const shq = (value) => `'${String(value).replace(/'/g, `'\\''`)}'`;
+
+// Files an engine needs before it starts, written wherever it asks for them.
+//
+// The runner does not know or care what these are: today it is a codex provider config carrying
+// a proxy token, tomorrow it is whatever the next engine wants. They are written OUTSIDE the
+// project directory on purpose, because anything inside it is either a change the customer gets
+// blamed for or a file `git add -A` would commit on their behalf.
+//
+// The content crosses as base64 read from STDIN. Not decoration: `-w0` is GNU-only and the macOS
+// build refuses a positional filename outright while still creating the empty output, which is
+// how an earlier version of this shipped a file that read as corrupt on one platform and worked
+// on the other.
+export const write_launch_files = async function (workspace, files) {
+  if (!files?.length) return { ok: true };
+  const steps = files.map((f) => {
+    const dir = path.posix.dirname(f.path);
+    const b64 = Buffer.from(String(f.content), 'utf8').toString('base64');
+    return `mkdir -p ${shq(dir)} && printf %s ${shq(b64)} | base64 -d > ${shq(f.path)} && chmod 600 ${shq(f.path)}`;
+  });
+  const ret = await workspace.exec(['sh', '-c', steps.join(' && ')], { timeout_ms: 60000 });
+  if (ret.exit_code !== 0) return { ok: false, error: (ret.stderr || '').trim() || `could not write engine files (exit ${ret.exit_code})` };
+  return { ok: true };
+};
+
 // git status porcelain, minus whatever was already dirty when the run started. This is UI-226's
 // decision and it is load-bearing: the change set is what the REPOSITORY says changed, never what
 // the agent claims it changed. An agent that forgets to mention a file, or invents one, cannot
@@ -53,17 +81,17 @@ export const parse_status = function (stdout) {
   return out;
 };
 
-export const create_runner = function ({ runtime, mirror, engines, verify, tracker_factory, run_process, git_exec, conf = () => global._conf.xudex || {}, now = () => Date.now() }) {
+export const create_runner = function ({ runtime, mirror, engines, verify, tracker_factory, run_process, git_exec, conf = () => global._conf.xucode || {}, now = () => Date.now() }) {
   for (const [name, dep] of Object.entries({ runtime, mirror, engines, verify, tracker_factory, run_process, git_exec })) {
     if (!dep) throw new Error(`create_runner needs ${name}`);
   }
 
-  const projects_root = () => conf().projects_root || '/srv/xudex';
+  const projects_root = () => conf().projects_root || '/srv/xucode';
   const sample_interval_ms = () => Number(conf().tracker?.sample_interval_ms) || 15000;
 
   // One engine turn. Everything about spawning a CLI lives here, so the run below reads as a
   // sequence of decisions rather than a pile of process handling.
-  const one_turn = async function ({ workspace, launch, prompt, emit }) {
+  const one_turn = async function ({ workspace, launch, prompt, emit, should_abort = null }) {
     const state = engines.create_run_state();
     const feed = engines.create_stream(launch.engine, (event) => {
       state.accept(event);
@@ -74,8 +102,13 @@ export const create_runner = function ({ runtime, mirror, engines, verify, track
       env: launch.env,
       timeout_ms: Number(conf().run_timeout_ms) || 30 * 60 * 1000,
       onStdout: feed,
+      should_abort,
     });
 
+    if (ret.aborted) {
+      return { ...state.state, exit_code: ret.exit_code, timed_out: false, aborted: true, error: ret.abort_reason || 'This run was stopped.' };
+    }
+
     // A CLI that dies without emitting a usable error still has to produce one, or the customer
     // sees a run that simply stopped. stderr is the only thing left to say why.
     if (ret.exit_code !== 0 && !state.state.error) {
@@ -88,7 +121,7 @@ export const create_runner = function ({ runtime, mirror, engines, verify, track
     run_branch,
     parse_status,
 
-    async run({ uid, repo, project_id, conversation_id, prompt, launch, workspace_dir, emit = null, verify_ctx = {}, max_repair_attempts } = {}) {
+    async run({ uid, repo, project_id, conversation_id, prompt, launch, workspace_dir, emit = null, verify_ctx = {}, verify_ctx_factory = null, max_repair_attempts } = {}) {
       const fail = (message, extra = {}) => ({ code: -1, data: message, ...extra });
       if (!prompt) return fail('There is nothing to do: the request was empty.');
       if (!launch || launch.error) return fail(launch?.error || 'That engine is not available.', { needs_key: launch?.needs_key || null });
@@ -103,24 +136,40 @@ export const create_runner = function ({ runtime, mirror, engines, verify, track
       const mirrored = await mirror.ensure({ uid, repo });
       if (mirrored.error) return fail(mirrored.error);
 
+      // ── 2. A workspace, THEN the repository inside it ────────────────────────────────
+      // Order matters and getting it wrong was a real bug: cloning on the region server and then
+      // acquiring a VM workspace put the working copy and the engine on two different computers,
+      // so the engine ran in an empty directory. The workspace is obtained first and hydrates
+      // ITSELF, which is the only version that holds for every substrate.
       const dir = workspace_dir || path.join(projects_root(), String(project_id));
-      const prepared = await mirror.sync_workspace({ mirror_dir: mirrored.dir, dest: dir, branch, default_branch: repo.default_branch });
-      if (prepared.error) return fail(prepared.error);
-
-      // ── 2. A workspace to run in ─────────────────────────────────────────────────────
       let workspace;
       try {
-        workspace = await runtime.acquire({ project_id, dir: prepared.dir });
+        workspace = await runtime.acquire({ uid, project_id, app_id: project_id, dir });
       } catch (err) {
         return fail('Xuda could not prepare a machine for this project. Please try again.');
       }
 
+      if (emit) emit({ type: 'phase', text: 'Putting the repository on the machine' });
+      const prepared = await mirror.hydrate_workspace({ workspace, uid, repo, branch, default_branch: repo.default_branch });
+      if (prepared.error) {
+        try {
+          await workspace.release();
+        } catch (e) {}
+        return fail(prepared.error);
+      }
+
       try {
         // What was already dirty before we started. Without this the run gets credited with a
         // change somebody made by hand, and revert would then throw away their work.
         const before = await workspace.exec(['git', 'status', '--porcelain', '--untracked-files=all'], { timeout_ms: 60000 });
         const pre_dirty = new Set(parse_status(before.stdout).map((f) => f.rel));
 
+        // Whatever the engine needs on disk before it can start. For a proxied engine this is
+        // the provider config carrying the run token, and without it the engine authenticates
+        // against nothing and the run dies on a 401 that reads like our fault.
+        const placed = await write_launch_files(workspace, launch.files);
+        if (!placed.ok) return fail('Xuda could not prepare the engine on this machine. Please try again.');
+
         // ── 3. The engine ──────────────────────────────────────────────────────────────
         const tracker = tracker_factory({ workspace });
         await tracker.start();
@@ -128,14 +177,37 @@ export const create_runner = function ({ runtime, mirror, engines, verify, track
           tracker.sample().catch(() => {});
         }, sample_interval_ms());
 
+        // The kill switch (plan 9.3 layer 6). The tracker is already watching this run; this is
+        // what lets it ACT on what it sees instead of writing a verdict after the cycles are
+        // spent. It stays shut unless `tracker.enforce` is on, and even then it waits out a
+        // grace period, so the normal state of this callback is to return null forever.
+        let killed_for = null;
+        const should_abort = async () => {
+          const live = tracker.live_verdict();
+          if (!live.stop) return null;
+          killed_for = live.reasons.join('; ');
+          console.warn(`[xucode] KILLING RUN uid=${uid} app=${project_id} after ${live.elapsed_seconds}s: ${killed_for}`);
+          return 'This run was stopped because it looked like it was doing something other than building your project. If that is wrong, please contact support.';
+        };
+
         let turn;
         try {
           if (emit) emit({ type: 'phase', text: `Running ${launch.label}` });
-          turn = await one_turn({ workspace, launch, prompt, emit });
+          turn = await one_turn({ workspace, launch, prompt, emit, should_abort });
         } finally {
           clearInterval(sampler);
         }
 
+        // A killed run does not continue into capture, verify or the mirror. Everything after
+        // this point exists to hand work back to the customer, and a run we just judged as abuse
+        // has no work to hand back.
+        if (turn.aborted) {
+          const record = tracker.finish({ usage: turn.usage, engine: launch.engine, meta: { uid, project_id, conversation_id, branch } });
+          record.killed = true;
+          record.kill_reasons = killed_for ? killed_for.split('; ') : record.reasons;
+          return { code: -1, data: turn.error, killed: true, record };
+        }
+
         // A provider error is the customer's provider talking, not us, and it is shown verbatim
         // (plan 5.2). With BYOK we cannot see their balance or their rate limits, so dressing a
         // 429 up as "something went wrong" would send them looking in the wrong place.
@@ -170,6 +242,13 @@ export const create_runner = function ({ runtime, mirror, engines, verify, track
         };
 
         if (emit) emit({ type: 'phase', text: 'Checking the change against your build and tests' });
+
+        // What the project looks like, asked of the machine holding it rather than assumed from
+        // this server's disk. Read AFTER the engine ran on purpose: a run that adds a test script
+        // should be checked by it, and a plan built beforehand would miss the gate the change
+        // itself introduced.
+        const ctx = verify_ctx_factory ? await verify_ctx_factory(workspace) : verify_ctx;
+
         const verified = await verify.verify_with_repair({
           workspace,
           // Repair whenever the tree is red, not only when the run has files to its name. Those
@@ -180,14 +259,23 @@ export const create_runner = function ({ runtime, mirror, engines, verify, track
           repair,
           max_attempts: max_repair_attempts,
           on_step: (step) => emit && emit({ type: 'phase', text: `${step.name}: ${step.ran ? (step.ok ? 'passed' : 'failed') : 'not configured'}` }),
-          ...verify_ctx,
+          ...ctx,
         });
 
         // Files are re-read after repair, because a repair changes more of them.
         const final_status = repair_turns ? await workspace.exec(['git', 'status', '--porcelain', '--untracked-files=all'], { timeout_ms: 60000 }) : after;
         const final_files = parse_status(final_status.stdout).filter((f) => !pre_dirty.has(f.rel));
 
-        // ── 6. Close the books ──────────────────────────────────────────────────────────
+        // ── 6. Carry the branch back to the mirror ──────────────────────────────────────
+        // Without this the work exists only on the machine: the mirror would not have it, so Commit
+        // and Push would have nothing to send and a reclaimed machine would take the work with it.
+        // Only attempted when the run actually produced something.
+        if (final_files.length) {
+          const collected = await mirror.collect_from_workspace({ workspace, uid, repo, branch });
+          if (collected.error) console.error(`[xucode] ${collected.error} (${project_id} ${branch})`);
+        }
+
+        // ── 7. Close the books ──────────────────────────────────────────────────────────
         const record = tracker.finish({
           usage: turn.usage,
           engine: launch.engine,
@@ -219,6 +307,15 @@ export const create_runner = function ({ runtime, mirror, engines, verify, track
           record,
         };
       } finally {
+        // The run token dies with the run, but leaving it written on the machine outlives the
+        // point of scoping it. Best effort: a machine that cannot be reached to clean up is
+        // holding a token that has already expired.
+        if (launch.scratch_dir) {
+          try {
+            await workspace.exec(['sh', '-c', `rm -rf ${shq(launch.scratch_dir)}`], { timeout_ms: 30000 });
+          } catch (e) {}
+        }
+
         // Releasing without destroy: on a persistent machine the working copy IS the warmth the
         // customer is paying for, and on an ephemeral one the substrate throws the whole thing
         // away regardless.