@xuda.io/ai_module 1.1.5656 → 1.1.5658

package/xudex_verify.mjs

@@ -0,0 +1,272 @@
+// The verify loop. This is the product.
+//
+// docs/plans/xudex.md section 6. Codex and Claude Code are terminals: they will happily
+// tell you they are done and leave your build broken, because they have no environment to
+// check against. Xudex has one, so after every run and BEFORE the customer is told anything
+// succeeded, the change is put through the project's own gates:
+//
+//   install (only if the manifest moved) -> typecheck -> test -> build -> the running app
+//
+// A failure is fed back to the engine for a bounded number of repair attempts. If it cannot
+// get to green it SAYS SO and shows the failure. It never claims success on a broken tree,
+// and it never hands back a working copy the customer has to untangle by hand.
+//
+// This is also where the honest version of the efficiency claim comes from (plan section
+// 11): fewer retries is efficiency, and it arrives as a byproduct of checking rather than
+// as a head-on token fight we would lose.
+//
+// ── Two rules that sound small and are not ─────────────────────────────────────────────
+//
+// NEVER CLAIM A STEP THAT DID NOT RUN. A project with no test script has not passed its
+// tests; it has no tests. Reporting "tests passed" there is the same lie as reporting it on
+// a broken tree, just harder to notice, and it is the lie that would make the whole
+// verification worthless the first time somebody checked.
+//
+// NEVER LET THE AGENT DELETE THE TEST. Told only "the tests fail", a coding agent will
+// cheerfully make them stop failing by removing them. The repair prompt says so explicitly,
+// because it is the single most common way an automated fix loop produces a green build
+// that means nothing.
+
+import path from 'node:path';
+
+if (!global._conf) {
+  global._conf = (await import(path.join(process.env.XUDA_HOME, 'common', 'load_conf.mjs'))).loadConf();
+}
+
+const verify_conf = () => global._conf.xudex?.verify || {};
+
+const DEFAULTS = {
+  max_repair_attempts: 2,
+  step_timeout_ms: 10 * 60 * 1000,
+  install_timeout_ms: 15 * 60 * 1000,
+  output_tail_lines: 60,
+  output_max_chars: 8000,
+};
+
+const conf_num = (key) => Number(verify_conf()[key]) || DEFAULTS[key];
+
+// The npm placeholder. `npm init` writes a test script that exits 1 with "no test
+// specified", so treating the mere presence of `scripts.test` as "this project has tests"
+// would fail every fresh project forever and teach customers the loop is broken.
+const NPM_TEST_PLACEHOLDER = /no test specified/i;
+
+// Which package manager the project actually uses, decided by its lockfile rather than by
+// preference. Running npm in a pnpm project half-works and then fails strangely later.
+export const detect_package_manager = function (files) {
+  if (files.includes('pnpm-lock.yaml')) return 'pnpm';
+  if (files.includes('yarn.lock')) return 'yarn';
+  return 'npm';
+};
+
+const install_argv = (pm) => (pm === 'pnpm' ? ['pnpm', 'install'] : pm === 'yarn' ? ['yarn', 'install'] : ['npm', 'install']);
+const run_argv = (pm, script) => (pm === 'npm' ? ['npm', 'run', script] : [pm, 'run', script]);
+
+// Work out what this project can actually be checked with. Every step carries WHY it was
+// included or skipped, because "skipped" has to be reportable: a step that silently
+// vanishes is indistinguishable from one that passed.
+export const plan_steps = function ({ manifest, files = [], node_modules_present = false, manifest_changed = false } = {}) {
+  const pkg = manifest || {};
+  const scripts = pkg.scripts || {};
+  const pm = detect_package_manager(files);
+  const has_deps = Object.keys(pkg.dependencies || {}).length > 0 || Object.keys(pkg.devDependencies || {}).length > 0;
+  const steps = [];
+
+  // Install, only when it would change something. Reinstalling on every run would add
+  // minutes to a loop whose whole promise is that the second run is fast.
+  if (has_deps && (!node_modules_present || manifest_changed)) {
+    steps.push({
+      name: 'install',
+      argv: install_argv(pm),
+      timeout_ms: conf_num('install_timeout_ms'),
+      why: !node_modules_present ? 'dependencies are not installed yet' : 'the manifest changed',
+    });
+  } else if (has_deps) {
+    steps.push({ name: 'install', skipped: true, why: 'dependencies already installed and the manifest did not change' });
+  }
+
+  if (scripts.typecheck) {
+    steps.push({ name: 'typecheck', argv: run_argv(pm, 'typecheck'), timeout_ms: conf_num('step_timeout_ms'), why: 'the project has a typecheck script' });
+  } else if (files.includes('tsconfig.json')) {
+    steps.push({ name: 'typecheck', argv: ['npx', '--no-install', 'tsc', '--noEmit'], timeout_ms: conf_num('step_timeout_ms'), why: 'the project has a tsconfig' });
+  } else {
+    steps.push({ name: 'typecheck', skipped: true, why: 'this project is not typechecked' });
+  }
+
+  if (scripts.test && !NPM_TEST_PLACEHOLDER.test(scripts.test)) {
+    steps.push({ name: 'test', argv: run_argv(pm, 'test'), timeout_ms: conf_num('step_timeout_ms'), why: 'the project has a test script' });
+  } else {
+    steps.push({ name: 'test', skipped: true, why: scripts.test ? 'the test script is the npm placeholder' : 'this project has no tests' });
+  }
+
+  if (scripts.build) {
+    steps.push({ name: 'build', argv: run_argv(pm, 'build'), timeout_ms: conf_num('step_timeout_ms'), why: 'the project has a build script' });
+  } else {
+    steps.push({ name: 'build', skipped: true, why: 'this project has no build step' });
+  }
+
+  return { package_manager: pm, steps };
+};
+
+// Keep the END of the output. Compilers and test runners print the summary last, and a
+// head-truncated log is the half without the error in it.
+export const tail_output = function (text, lines = conf_num('output_tail_lines'), max_chars = conf_num('output_max_chars')) {
+  const all = String(text || '').split('\n');
+  let out = all.slice(-lines).join('\n');
+  if (out.length > max_chars) out = out.slice(-max_chars);
+  return out.trim();
+};
+
+// What the engine is told when a step fails. Deliberately narrow: one failing step, its
+// command, its exit code and the tail of what it printed. Handing over every step's output
+// buries the actual error in noise and costs tokens for the privilege.
+export const repair_prompt = function (result) {
+  const failed = result?.failed;
+  if (!failed) return null;
+  return [
+    'Your change did not pass verification. Fix it.',
+    '',
+    `Step: ${failed.name}`,
+    `Command: ${(failed.argv || []).join(' ')}`,
+    `Exit code: ${failed.exit_code}`,
+    '',
+    'Output:',
+    failed.output,
+    '',
+    // Without this line an agent will reliably "fix" a failing test by deleting it, or by
+    // loosening the assertion until it passes. That produces a green build that means
+    // nothing, which is worse than a red one because it is believed.
+    'Fix the underlying cause. Do NOT delete, skip or weaken tests to make this pass, and do not disable typechecking. If you believe the test itself is wrong, say so and explain why instead of changing it.',
+  ].join('\n');
+};
+
+// Run the gates. `workspace` is the runtime interface's handle (xudex_runtime.mjs), so this
+// is identical on a VM, an ephemeral runner or a container.
+//
+// `runtime_check` is the seam for the preview leg (build order step 11): once a preview
+// exists it gets passed in here and its runtime errors become just another failing step.
+// Absent, that stage is reported as not run rather than quietly assumed fine.
+export const run_verify = async function ({ workspace, manifest, files = [], node_modules_present = false, manifest_changed = false, runtime_check = null, on_step = null } = {}) {
+  const planned = plan_steps({ manifest, files, node_modules_present, manifest_changed });
+  const results = [];
+  let failed = null;
+
+  for (const step of planned.steps) {
+    if (step.skipped) {
+      results.push({ name: step.name, ran: false, ok: null, why: step.why });
+      if (on_step) on_step(results[results.length - 1]);
+      continue;
+    }
+
+    const started = Date.now();
+    let ret;
+    try {
+      ret = await workspace.exec(step.argv, { timeout_ms: step.timeout_ms });
+    } catch (err) {
+      ret = { exit_code: -1, stdout: '', stderr: `could not run ${step.argv.join(' ')}: ${err.message}`, timed_out: false };
+    }
+    const output = tail_output(`${ret.stdout || ''}\n${ret.stderr || ''}`);
+    const entry = {
+      name: step.name,
+      ran: true,
+      ok: ret.exit_code === 0,
+      exit_code: ret.exit_code,
+      argv: step.argv,
+      ms: Date.now() - started,
+      timed_out: ret.timed_out === true,
+      output,
+      why: step.why,
+    };
+    results.push(entry);
+    if (on_step) on_step(entry);
+
+    // Stop at the first failure. Later gates run on the state the failing one left behind,
+    // so their output is noise at best and misleading at worst: a typecheck error makes the
+    // test suite fail too, and reporting both invites fixing the wrong one.
+    if (!entry.ok) {
+      failed = entry;
+      break;
+    }
+  }
+
+  // The running application, when there is one to check.
+  if (!failed && typeof runtime_check === 'function') {
+    const started = Date.now();
+    let rc;
+    try {
+      rc = await runtime_check(workspace);
+    } catch (err) {
+      rc = { ok: false, output: `the preview check failed: ${err.message}` };
+    }
+    const entry = { name: 'runtime', ran: true, ok: rc.ok !== false, ms: Date.now() - started, output: tail_output(rc.output || ''), argv: ['preview'], exit_code: rc.ok === false ? 1 : 0, why: 'the project has a running preview' };
+    results.push(entry);
+    if (on_step) on_step(entry);
+    if (!entry.ok) failed = entry;
+  } else if (!failed) {
+    results.push({ name: 'runtime', ran: false, ok: null, why: 'no preview is running for this project' });
+  }
+
+  const ran = results.filter((r) => r.ran);
+  return {
+    ok: !failed,
+    failed,
+    steps: results,
+    package_manager: planned.package_manager,
+    // The honest headline. "Green" is only claimed for gates that actually ran, and a
+    // project with nothing to check gets told that rather than congratulated.
+    summary: summarize(results),
+    checked: ran.length,
+  };
+};
+
+export const summarize = function (results) {
+  const ran = results.filter((r) => r.ran);
+  const failed = ran.find((r) => r.ok === false);
+  if (failed) return `${failed.name} failed`;
+  if (!ran.length) return 'nothing to verify: this project has no install, typecheck, test or build step';
+  const names = ran.map((r) => r.name).join(', ');
+  const skipped = results.filter((r) => !r.ran && r.name !== 'runtime').map((r) => r.name);
+  return skipped.length ? `${names} passed (${skipped.join(' and ')} not configured)` : `${names} passed`;
+};
+
+// The loop itself: verify, and if it failed, hand the failure back to the engine and verify
+// again, up to a bounded number of attempts.
+//
+// `repair` is injected so this file never learns how to launch an engine, which is what
+// keeps the two seams (engines, runtime) independent of each other.
+export const verify_with_repair = async function ({ workspace, repair, max_attempts = conf_num('max_repair_attempts'), on_step = null, on_attempt = null, ...ctx } = {}) {
+  const attempts = [];
+  let result = await run_verify({ workspace, on_step, ...ctx });
+  attempts.push({ attempt: 0, ok: result.ok, summary: result.summary });
+
+  let n = 0;
+  while (!result.ok && typeof repair === 'function' && n < max_attempts) {
+    n++;
+    if (on_attempt) on_attempt({ attempt: n, of: max_attempts, fixing: result.failed?.name });
+    let repaired = false;
+    try {
+      repaired = await repair(repair_prompt(result), { attempt: n, failed: result.failed });
+    } catch (err) {
+      console.warn(`[xudex] repair attempt ${n} threw: ${err.message}`);
+      break;
+    }
+    // A repair that declined to change anything will fail identically next time, so
+    // spending another engine turn on it is pure cost.
+    if (repaired === false) break;
+    // After a repair, dependencies may have moved, so the manifest is treated as changed.
+    result = await run_verify({ workspace, on_step, ...ctx, manifest_changed: true });
+    attempts.push({ attempt: n, ok: result.ok, summary: result.summary });
+  }
+
+  return {
+    ...result,
+    attempts,
+    repair_attempts: n,
+    // What the customer is told. On failure this is the whole point of the feature: it did
+    // not work, here is which gate, here is what it printed.
+    verdict: result.ok
+      ? n > 0
+        ? `${result.summary} after ${n} automatic ${n === 1 ? 'fix' : 'fixes'}`
+        : result.summary
+      : `${result.summary}${n ? ` after ${n} automatic ${n === 1 ? 'fix attempt' : 'fix attempts'}` : ''}`,
+  };
+};

package/xudex_vm.mjs

@@ -0,0 +1,218 @@
+// The VM substrate: a xudex workspace on the account's own machine.
+//
+// docs/plans/xudex.md 3.2 and 3.8. This is the second implementation of the runtime
+// interface, and building it is what proves that seam was real rather than a comment: the
+// engine, the verify loop, the tracker and preview are not touched by anything in this file.
+//
+// ── It does NOT use SSH, and that is not a preference ──────────────────────────────────
+// A xudex machine is `app_type: 'vps'` (the flavor rides on `is_xudex`, see deploy_xudex),
+// and customer VPS on this platform are KEYLESS by design: since the 2026-07-15 consent
+// split no standing platform SSH key lives in those guests, and they correctly refuse one.
+// Reaching them over SSH would mean pushing the control-plane key back in, which regresses
+// the keyless model on purpose-built grounds.
+//
+// So this runs over the Proxmox qemu-guest-agent channel, exactly as the Files feature
+// already does (`_farm_fs_via_agent` in app_module). That happens to line up perfectly with
+// the product rule in plan 3.7: the customer gets no shell, and neither do we.
+//
+// ── What the channel cannot do, stated up front ────────────────────────────────────────
+// `agent/exec` dispatches a command and returns a pid; output is only readable once the
+// process EXITS. There is no stream. A build takes minutes and an engine run takes longer,
+// so two things follow and both are handled below:
+//
+//   1. Polling needs a far bigger budget than the Files layer's ~25 seconds.
+//   2. Live progress has to come from somewhere else. Long commands are redirected to a log
+//      file in the guest and that file is tailed incrementally between status polls, which
+//      is how `onStdout` keeps working for callers that stream.
+
+import path from 'node:path';
+
+if (!global._conf) {
+  global._conf = (await import(path.join(process.env.XUDA_HOME, 'common', 'load_conf.mjs'))).loadConf();
+}
+
+// Single-quote for the guest's shell. Every path and value crossing this boundary is
+// attacker-influenced: the agent writes the file names and the customer writes the prompt.
+export const shq = (value) => `'${String(value).replace(/'/g, `'\\''`)}'`;
+
+const vm_conf = () => global._conf.xudex?.vm || {};
+
+export const create_vm_substrate = function ({ pve_request, resolve_machine, delay = (ms) => new Promise((r) => setTimeout(r, ms)), now = () => Date.now() }) {
+  if (typeof pve_request !== 'function') throw new Error('create_vm_substrate needs pve_request');
+  if (typeof resolve_machine !== 'function') throw new Error('create_vm_substrate needs resolve_machine');
+
+  const poll_interval = () => Number(vm_conf().poll_interval_ms) || 700;
+
+  // Dispatch a script in the guest and wait for it, with a deadline rather than a poll count.
+  // `on_tail` is called with new bytes from the log file as they appear, which is the only way
+  // to see progress on a channel that otherwise speaks once, at the end.
+  const guest_exec = async function ({ node_doc, vmid, script, timeout_ms, on_tail = null, log_file = null }) {
+    const body = [
+      ['command', '/bin/bash'],
+      ['command', '-c'],
+      ['command', script],
+    ];
+    const disp = await pve_request(node_doc, 'POST', `/nodes/${node_doc.node}/qemu/${vmid}/agent/exec`, body);
+    const pid = disp && (disp.pid ?? disp.PID);
+    if (pid == null) throw new Error('guest exec returned no pid (is qemu-guest-agent running?)');
+
+    const deadline = now() + (Number(timeout_ms) || 30 * 60 * 1000);
+    let offset = 0;
+
+    while (now() < deadline) {
+      const st = await pve_request(node_doc, 'GET', `/nodes/${node_doc.node}/qemu/${vmid}/agent/exec-status?pid=${pid}`);
+      if (st && st.exited) {
+        return {
+          stdout: st['out-data'] != null ? String(st['out-data']) : '',
+          stderr: st['err-data'] != null ? String(st['err-data']) : '',
+          exit_code: st.exitcode ?? 0,
+          truncated: !!(st['out-truncated'] || st['err-truncated']),
+          timed_out: false,
+        };
+      }
+
+      // Still running. If the caller wants progress and we redirected to a log, read what is
+      // new since last time. `tail -c +N` is one byte-offset read rather than re-sending the
+      // whole file every second, which matters when a build prints megabytes.
+      if (on_tail && log_file) {
+        try {
+          const chunk = await pve_request(node_doc, 'POST', `/nodes/${node_doc.node}/qemu/${vmid}/agent/exec`, [
+            ['command', '/bin/bash'],
+            ['command', '-c'],
+            ['command', `tail -c +${offset + 1} ${shq(log_file)} 2>/dev/null | head -c 200000`],
+          ]);
+          const cpid = chunk && (chunk.pid ?? chunk.PID);
+          if (cpid != null) {
+            const cst = await pve_request(node_doc, 'GET', `/nodes/${node_doc.node}/qemu/${vmid}/agent/exec-status?pid=${cpid}`);
+            const text = cst && cst.exited && cst['out-data'] != null ? String(cst['out-data']) : '';
+            if (text) {
+              offset += Buffer.byteLength(text);
+              on_tail(text);
+            }
+          }
+        } catch (e) {
+          // Progress is a courtesy. Losing it must never fail the command it is watching.
+        }
+      }
+
+      await delay(poll_interval());
+    }
+
+    // Out of time. Kill the process group in the guest rather than leaving a build running on
+    // a machine the customer is paying for, then say so.
+    try {
+      await pve_request(node_doc, 'POST', `/nodes/${node_doc.node}/qemu/${vmid}/agent/exec`, [
+        ['command', '/bin/bash'],
+        ['command', '-c'],
+        ['command', `kill -TERM -${pid} 2>/dev/null; kill -KILL -${pid} 2>/dev/null; true`],
+      ]);
+    } catch (e) {}
+    return { stdout: '', stderr: `the command did not finish within ${Math.round((Number(timeout_ms) || 0) / 1000)}s and was stopped`, exit_code: 124, truncated: false, timed_out: true };
+  };
+
+  return {
+    kind: 'vm',
+    guest_exec,
+
+    // Available when this box can reach Proxmox at all. Whether a PARTICULAR account has a
+    // machine is answered by acquire, because that is a per-request question.
+    available() {
+      return global._conf.xudex?.enabled === true && !!global._conf.proxmox && global._conf.proxmox.fs_agent !== false;
+    },
+
+    async acquire({ uid, project_id, app_id }) {
+      const machine = await resolve_machine({ uid, app_id: app_id || project_id });
+      if (!machine || machine.error) throw new Error(machine?.error || 'this account has no xudex machine');
+      const { node_doc, vmid } = machine;
+      const root = global._conf.xudex?.projects_root || '/srv/xudex';
+      const dir = path.posix.join(root, String(project_id));
+
+      await guest_exec({ node_doc, vmid, script: `mkdir -p ${shq(dir)}`, timeout_ms: 60000 });
+
+      const safe = (rel) => {
+        const resolved = path.posix.resolve(dir, String(rel || ''));
+        if (resolved !== dir && !resolved.startsWith(dir + '/')) throw new Error(`path escapes the workspace: ${rel}`);
+        return resolved;
+      };
+
+      return {
+        id: `vm:${vmid}:${project_id}`,
+        kind: 'vm',
+        project_id,
+        dir,
+        vmid,
+
+        async exec(argv, opts = {}) {
+          if (!Array.isArray(argv) || !argv.length) throw new Error('exec needs an argv array');
+          const cwd = opts.cwd ? safe(opts.cwd) : dir;
+          const env = Object.entries(opts.env || {})
+            .map(([k, v]) => `export ${k}=${shq(v)};`)
+            .join(' ');
+          const command = argv.map(shq).join(' ');
+          const started = now();
+
+          // A caller that wants progress gets a log file and an incremental tail; one that does
+          // not gets the plain form, which is cheaper by one exec per poll.
+          const log_file = opts.onStdout ? `/tmp/xudex-exec-${vmid}-${started}.log` : null;
+          const script = log_file
+            ? `cd ${shq(cwd)}; ${env} { ${command}; } > ${shq(log_file)} 2>&1; rc=$?; cat ${shq(log_file)}; rm -f ${shq(log_file)}; exit $rc`
+            : `cd ${shq(cwd)}; ${env} ${command}`;
+
+          const ret = await guest_exec({ node_doc, vmid, script, timeout_ms: opts.timeout_ms, on_tail: opts.onStdout || null, log_file });
+          return { exit_code: ret.exit_code, stdout: ret.stdout || '', stderr: ret.stderr || '', timed_out: ret.timed_out === true, ms: now() - started, truncated: ret.truncated };
+        },
+
+        // Base64 across the channel in both directions. Proxmox hands back out-data already
+        // decoded as TEXT, so anything binary (or merely non-UTF8) would be mangled in transit;
+        // encoding inside the guest is the same fix the Files layer arrived at.
+        async read_file(rel) {
+          const p = safe(rel);
+          const ret = await guest_exec({ node_doc, vmid, script: `base64 -w0 ${shq(p)}`, timeout_ms: 120000 });
+          if (ret.exit_code !== 0) throw new Error(`could not read ${rel}: ${(ret.stderr || '').trim()}`);
+          return Buffer.from((ret.stdout || '').trim(), 'base64').toString('utf8');
+        },
+
+        async write_file(rel, data) {
+          const p = safe(rel);
+          const b64 = Buffer.from(String(data), 'utf8').toString('base64');
+          // Chunked because the whole script is one argv element and the guest has a hard argv
+          // limit (~128 KB); a single big write silently fails at the API boundary otherwise.
+          const CHUNK = 60 * 1024;
+          await guest_exec({ node_doc, vmid, script: `mkdir -p ${shq(path.posix.dirname(p))} && : > ${shq(p)}.b64`, timeout_ms: 60000 });
+          for (let i = 0; i < b64.length; i += CHUNK) {
+            const part = b64.slice(i, i + CHUNK);
+            const ret = await guest_exec({ node_doc, vmid, script: `printf %s ${shq(part)} >> ${shq(p)}.b64`, timeout_ms: 120000 });
+            if (ret.exit_code !== 0) throw new Error(`could not write ${rel}: ${(ret.stderr || '').trim()}`);
+          }
+          const fin = await guest_exec({ node_doc, vmid, script: `base64 -d ${shq(p)}.b64 > ${shq(p)} && rm -f ${shq(p)}.b64`, timeout_ms: 120000 });
+          if (fin.exit_code !== 0) throw new Error(`could not write ${rel}: ${(fin.stderr || '').trim()}`);
+          return true;
+        },
+
+        async list_dir(rel = '.') {
+          const ret = await guest_exec({ node_doc, vmid, script: `ls -1A ${shq(safe(rel))} 2>/dev/null || true`, timeout_ms: 60000 });
+          return (ret.stdout || '')
+            .split('\n')
+            .map((s) => s.trim())
+            .filter(Boolean);
+        },
+
+        // The preview URL. A xudex machine publishes nothing but previews (plan 3.7), so this is
+        // the one port that is ever reachable, and it goes through the platform's existing web
+        // route for the app rather than a second proxy invented here.
+        async expose_port(port) {
+          const base = global._conf.xudex?.preview_domain;
+          if (!base) return null;
+          return { url: `https://${String(project_id).replace(/[^a-zA-Z0-9]/g, '')}.${base}`, port };
+        },
+
+        // The machine persists between runs, which is the whole point of the paid tiers, so
+        // release is a no-op unless the caller explicitly wants the working copy gone.
+        async release(opts = {}) {
+          if (opts.destroy === true) await guest_exec({ node_doc, vmid, script: `rm -rf ${shq(dir)}`, timeout_ms: 120000 });
+          return true;
+        },
+      };
+    },
+  };
+};