@agent-link/agent 0.1.298 → 0.1.300

package/launcher/launcher.cjs

@@ -4,7 +4,43 @@
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
-const { spawn } = require('child_process');
+const crypto = require('crypto');
+const childProcess = require('child_process');
+const { spawn } = childProcess;
+let forceLastKnownGood = false;
+let repairResumeJobId = null;
+let workerOwnedJobId = null;
+const JOB_ID = /^[a-zA-Z0-9-]{1,128}$/;
+const LAUNCH_HOST_TYPES = new Set([
+  'standalone',
+  'systemd',
+  'launchd',
+  'windows-startup',
+  'winsw',
+]);
+const UPGRADE_PHASES = new Set([
+  'created',
+  'downloading',
+  'verifying',
+  'staging',
+  'self-checking',
+  'worker-starting',
+  'worker-ready',
+  'shutdown-requested',
+  'old-agent-stopped',
+  'switching',
+  'target-starting',
+  'target-local-health',
+  'relay-pending',
+  'rolling-back',
+  'rollback-starting',
+  'rollback-local-health',
+  'repair-starting',
+  'completed',
+  'rolled-back',
+  'repair-required',
+  'failed',
+]);
 
 function fail(message) {
   process.stderr.write(`[AgentLink Launcher] ${message}\n`);
@@ -63,11 +99,472 @@ function loadPointer(file, versionsRoot) {
   return { ...value, entry: canonicalEntry };
 }
 
+function loadJson(file) {
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+function isRecord(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function validateLatestReference(value) {
+  if (
+    !isRecord(value)
+    || value.schemaVersion !== 2
+    || typeof value.jobId !== 'string'
+    || !JOB_ID.test(value.jobId)
+    || typeof value.phase !== 'string'
+    || !UPGRADE_PHASES.has(value.phase)
+  ) {
+    throw new Error('invalid latest upgrade state');
+  }
+  return value;
+}
+
+function loadRecoveryLaunchSpec(launchSpecPath, agentHome) {
+  if (typeof launchSpecPath !== 'string' || launchSpecPath.length === 0) {
+    throw new Error('recovery LaunchSpec path is invalid');
+  }
+  const canonicalHome = fs.realpathSync(agentHome);
+  const resolvedLaunchSpec = path.resolve(launchSpecPath);
+  const canonicalLaunchSpec = fs.realpathSync(resolvedLaunchSpec);
+  if (
+    path.resolve(agentHome) !== canonicalHome
+    || resolvedLaunchSpec !== canonicalLaunchSpec
+    || !isInside(canonicalHome, canonicalLaunchSpec)
+    || !fs.statSync(canonicalLaunchSpec).isFile()
+  ) {
+    throw new Error('recovery LaunchSpec is relocated or escapes the trusted data root');
+  }
+  const launchSpec = loadJson(canonicalLaunchSpec);
+  if (
+    !isRecord(launchSpec)
+    || !isRecord(launchSpec.host)
+    || !LAUNCH_HOST_TYPES.has(launchSpec.host.type)
+  ) {
+    throw new Error('recovery LaunchSpec has an invalid shape');
+  }
+  return launchSpec;
+}
+
+function validateRecoveryJob(value, expectedJobId, agentHome) {
+  if (
+    !isRecord(value)
+    || value.schemaVersion !== 2
+    || value.jobId !== expectedJobId
+    || value.restartOwner !== 'worker'
+    || typeof value.phase !== 'string'
+    || !UPGRADE_PHASES.has(value.phase)
+    || typeof value.lockNonce !== 'string'
+    || value.lockNonce.length === 0
+    || path.resolve(value.dataRoot || '') !== agentHome
+  ) {
+    throw new Error('upgrade job identity, schema, or data root is invalid');
+  }
+  loadRecoveryLaunchSpec(value.launchSpecPath, agentHome);
+  return value;
+}
+
+function atomicJson(file, value) {
+  fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+  const temporary = `${file}.${process.pid}.${Date.now()}.tmp`;
+  const fd = fs.openSync(temporary, 'wx', 0o600);
+  try {
+    fs.writeFileSync(fd, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
+    fs.fsyncSync(fd);
+  } finally {
+    fs.closeSync(fd);
+  }
+  fs.renameSync(temporary, file);
+  if (process.platform !== 'win32') fs.chmodSync(file, 0o600);
+}
+
+function processIdentity(pid) {
+  if (!Number.isSafeInteger(pid) || pid <= 0) return { status: 'error' };
+  if (process.platform === 'linux') {
+    try {
+      const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf8').trim();
+      const end = stat.lastIndexOf(')');
+      const fields = stat.slice(end + 2).split(/\s+/);
+      const boot = fs.readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim();
+      return end >= 0 && fields[19] && boot
+        ? { status: 'found', identity: `linux:${boot}:${fields[19]}` }
+        : { status: 'error' };
+    } catch (error) {
+      return error.code === 'ENOENT' || error.code === 'ESRCH'
+        ? { status: 'absent' }
+        : { status: 'error' };
+    }
+  }
+  if (process.platform === 'darwin') {
+    try {
+      const started = childProcess.execFileSync('/bin/ps', ['-o', 'lstart=', '-p', String(pid)], {
+        encoding: 'utf8',
+        timeout: 3000,
+      }).trim();
+      return started ? { status: 'found', identity: `darwin:${started}` } : { status: 'absent' };
+    } catch (error) {
+      return error.status === 1 ? { status: 'absent' } : { status: 'error' };
+    }
+  }
+  if (process.platform === 'win32') {
+    try {
+      const script = '& { param([int]$ProcessId) $p=Get-CimInstance Win32_Process -Filter ("ProcessId="+$ProcessId);if($p){$p.CreationDate.ToUniversalTime().ToString("O",[Globalization.CultureInfo]::InvariantCulture)} }';
+      const started = childProcess.execFileSync('powershell.exe', [
+        '-NoLogo', '-NoProfile', '-NonInteractive', '-Command', script, '-ProcessId', String(pid),
+      ], { encoding: 'utf8', windowsHide: true, timeout: 3000 }).trim();
+      return started ? { status: 'found', identity: `win32:${started}` } : { status: 'absent' };
+    } catch {
+      return { status: 'error' };
+    }
+  }
+  return { status: 'error' };
+}
+
+function identityState(identity) {
+  if (!identity) return { status: 'absent' };
+  const actual = processIdentity(identity.pid);
+  if (actual.status !== 'found') return actual;
+  return actual.identity === identity.processStartIdentity
+    ? { status: 'match' }
+    : { status: 'mismatch' };
+}
+
+function sameIdentity(left, right) {
+  return isRecord(left)
+    && isRecord(right)
+    && Number.isSafeInteger(left.pid)
+    && left.pid > 0
+    && left.pid === right.pid
+    && typeof left.processStartIdentity === 'string'
+    && left.processStartIdentity === right.processStartIdentity;
+}
+
+function hasValidWorkerRecoveryOwner(agentHome, job) {
+  if (
+    process.env.AGENTLINK_RECOVERY_OWNER !== 'worker'
+    || process.env.AGENTLINK_UPGRADE_JOB !== job.jobId
+    || !['target-starting', 'rollback-starting'].includes(job.phase)
+    || !isRecord(job.worker)
+    || job.worker.protocolVersion !== 1
+    || identityState(job.worker).status !== 'match'
+  ) {
+    return false;
+  }
+  try {
+    const lockDirectory = path.join(agentHome, 'upgrades', 'active.lock');
+    const owner = loadJson(path.join(lockDirectory, 'owner.json'));
+    const claim = loadJson(path.join(lockDirectory, `handoff-${job.lockNonce}.claim`));
+    return isRecord(owner)
+      && owner.jobId === job.jobId
+      && owner.nonce === job.lockNonce
+      && path.resolve(owner.dataRoot || '') === agentHome
+      && sameIdentity(owner.worker, job.worker)
+      && isRecord(claim)
+      && claim.role === 'worker'
+      && claim.jobId === job.jobId
+      && claim.nonce === job.lockNonce
+      && sameIdentity(claim.worker, job.worker);
+  } catch {
+    return false;
+  }
+}
+
+function verifyLockReclaimable(agentHome, job) {
+  const lock = path.join(agentHome, 'upgrades', 'active.lock');
+  if (!fs.existsSync(lock)) return null;
+  const owner = loadJson(path.join(lock, 'owner.json'));
+  if (owner.jobId !== job.jobId || owner.nonce !== job.lockNonce) {
+    throw new Error('active upgrade lock does not match interrupted job');
+  }
+  for (const identity of [owner.controller, owner.worker]) {
+    const state = identityState(identity);
+    if (state.status === 'match') throw new Error('active upgrade lock still has a live owner');
+    if (state.status === 'error') {
+      throw new Error('cannot safely reclaim active upgrade lock after an identity lookup failure');
+    }
+  }
+  return { lock, owner };
+}
+
+function releaseStaleLock(agentHome, job) {
+  const reclaimable = verifyLockReclaimable(agentHome, job);
+  if (!reclaimable) return;
+  const { lock } = reclaimable;
+  const generation = crypto.createHash('sha256')
+    .update(`${reclaimable.owner.jobId}\0${reclaimable.owner.nonce}\0${reclaimable.owner.controller.pid}\0${reclaimable.owner.controller.processStartIdentity}`)
+    .digest('hex')
+    .slice(0, 16);
+  const stale = `${lock}.stale-${job.lockNonce}-${generation}`;
+  try {
+    fs.renameSync(lock, stale);
+  } catch (error) {
+    if (
+      ['EEXIST', 'ENOTEMPTY', 'ENOENT', 'EPERM'].includes(error.code)
+      && fs.existsSync(stale)
+    ) {
+      return;
+    }
+    throw error;
+  }
+}
+
+function markInterruptedPreShutdown(agentHome, jobPath, job, reason) {
+  verifyLockReclaimable(agentHome, job);
+  const updated = {
+    ...job,
+    phase: 'failed',
+    updatedAt: new Date().toISOString(),
+    error: reason,
+  };
+  atomicJson(jobPath, updated);
+  atomicJson(path.join(agentHome, 'upgrades', 'latest.json'), updated);
+  releaseStaleLock(agentHome, updated);
+}
+
+function reconcileTerminalLock(agentHome, job) {
+  const lock = path.join(agentHome, 'upgrades', 'active.lock');
+  if (!fs.existsSync(lock)) return false;
+  const owner = loadJson(path.join(lock, 'owner.json'));
+  if (owner.jobId !== job.jobId || owner.nonce !== job.lockNonce) {
+    throw new Error('terminal upgrade lock does not match the durable job');
+  }
+  let ownerAlive = false;
+  for (const identity of [owner.controller, owner.worker]) {
+    const state = identityState(identity);
+    if (state.status === 'match') ownerAlive = true;
+    if (state.status === 'error') {
+      throw new Error('cannot reconcile terminal upgrade lock after an identity lookup failure');
+    }
+  }
+  let replacementAlive = false;
+  for (const identity of [job.repairRuntime, job.rollbackRuntime, job.targetRuntime]) {
+    const state = identityState(identity);
+    if (state.status === 'match') replacementAlive = true;
+    if (state.status === 'error') {
+      throw new Error('cannot reconcile terminal runtime after an identity lookup failure');
+    }
+  }
+  if (ownerAlive) return true;
+  releaseStaleLock(agentHome, job);
+  return replacementAlive;
+}
+
+function escapeXml(value) {
+  return value
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;');
+}
+
+function startRecoveryWorker(agentHome, jobPath, job) {
+  const workerPath = path.join(path.dirname(jobPath), 'worker.cjs');
+  if (!fs.statSync(workerPath).isFile()) throw new Error('recovery worker is missing');
+  const logPath = path.join(path.dirname(jobPath), 'worker.log');
+  const hostType = loadRecoveryLaunchSpec(job.launchSpecPath, agentHome).host.type;
+  const environment = { ...process.env, AGENTLINK_HOME: agentHome };
+  if (hostType === 'systemd' && process.platform !== 'win32') {
+    childProcess.execFileSync('systemd-run', [
+      '--user',
+      `--unit=agentlink-upgrade-${job.jobId}`,
+      '--collect',
+      '--property=Type=exec',
+      '--property=KillMode=process',
+      `--property=StandardOutput=append:${logPath}`,
+      `--property=StandardError=append:${logPath}`,
+      `--setenv=AGENTLINK_HOME=${agentHome}`,
+      process.execPath,
+      workerPath,
+      '--job',
+      jobPath,
+    ], {
+      cwd: path.dirname(jobPath),
+      env: environment,
+      stdio: 'ignore',
+      windowsHide: true,
+      timeout: 10_000,
+    });
+    return;
+  }
+  if (hostType === 'launchd' && process.platform === 'darwin') {
+    const label = `com.agentlink.upgrade.${job.jobId}`;
+    const plistPath = path.join(path.dirname(jobPath), 'worker.plist');
+    const plist = `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0"><dict>
+<key>Label</key><string>${escapeXml(label)}</string>
+<key>ProgramArguments</key><array>
+<string>${escapeXml(process.execPath)}</string><string>${escapeXml(workerPath)}</string>
+<string>--job</string><string>${escapeXml(jobPath)}</string>
+</array>
+<key>WorkingDirectory</key><string>${escapeXml(path.dirname(jobPath))}</string>
+<key>EnvironmentVariables</key><dict>
+<key>AGENTLINK_HOME</key><string>${escapeXml(agentHome)}</string>
+</dict>
+<key>RunAtLoad</key><true/>
+<key>StandardOutPath</key><string>${escapeXml(logPath)}</string>
+<key>StandardErrorPath</key><string>${escapeXml(logPath)}</string>
+</dict></plist>
+`;
+    fs.writeFileSync(plistPath, plist, { encoding: 'utf8', mode: 0o600 });
+    fs.chmodSync(plistPath, 0o600);
+    const domain = `gui/${process.getuid()}`;
+    try {
+      childProcess.execFileSync('launchctl', ['bootout', `${domain}/${label}`], {
+        cwd: path.dirname(jobPath),
+        env: environment,
+        stdio: 'ignore',
+        windowsHide: true,
+        timeout: 5000,
+      });
+    } catch {
+      // A crashed worker may already have been unloaded.
+    }
+    childProcess.execFileSync('launchctl', ['bootstrap', domain, plistPath], {
+      cwd: path.dirname(jobPath),
+      env: environment,
+      stdio: 'ignore',
+      windowsHide: true,
+      timeout: 10_000,
+    });
+    return;
+  }
+  const output = fs.openSync(logPath, 'a', 0o600);
+  try {
+    const child = spawn(process.execPath, [workerPath, '--job', jobPath], {
+      cwd: path.dirname(jobPath),
+      env: environment,
+      detached: true,
+      stdio: ['ignore', output, output],
+      windowsHide: true,
+    });
+    child.unref();
+  } finally {
+    fs.closeSync(output);
+  }
+}
+
+function recoverUpgrade(agentHome) {
+  const latestPath = path.join(agentHome, 'upgrades', 'latest.json');
+  if (!fs.existsSync(latestPath)) return false;
+  let latest;
+  try {
+    latest = validateLatestReference(loadJson(latestPath));
+  } catch (error) {
+    process.stderr.write(`[AgentLink Launcher] Upgrade state is corrupt; using last-known-good and repair is required: ${error.message}\n`);
+    atomicJson(path.join(agentHome, 'upgrades', 'recovery-error.json'), {
+      phase: 'repair-required',
+      updatedAt: new Date().toISOString(),
+      error: `corrupt latest upgrade state: ${error.message}`,
+    });
+    forceLastKnownGood = true;
+    return false;
+  }
+  const referencedJobId = latest.jobId;
+  const jobPath = path.join(agentHome, 'upgrades', referencedJobId, 'job.json');
+  let job;
+  try {
+    job = validateRecoveryJob(loadJson(jobPath), referencedJobId, agentHome);
+  } catch (error) {
+    process.stderr.write(`[AgentLink Launcher] Upgrade job is corrupt; using last-known-good and repair is required: ${error.message}\n`);
+    atomicJson(path.join(agentHome, 'upgrades', 'recovery-error.json'), {
+      phase: 'repair-required',
+      jobId: referencedJobId,
+      updatedAt: new Date().toISOString(),
+      error: `corrupt upgrade job: ${error.message}`,
+    });
+    forceLastKnownGood = true;
+    return false;
+  }
+  if (hasValidWorkerRecoveryOwner(agentHome, job)) {
+    workerOwnedJobId = job.jobId;
+    return false;
+  }
+  if (
+    process.env.AGENTLINK_RECOVERY_OWNER === 'repair'
+    && process.env.AGENTLINK_UPGRADE_JOB === job.jobId
+    && job.phase === 'repair-starting'
+  ) {
+    repairResumeJobId = job.jobId;
+    forceLastKnownGood = true;
+    return false;
+  }
+  if (job.phase === 'repair-required') {
+    atomicJson(latestPath, job);
+    reconcileTerminalLock(agentHome, job);
+    process.stderr.write('[AgentLink Launcher] Upgrade repair is required; run agentlink-client upgrade-repair.\n');
+    return true;
+  }
+  if (['completed', 'rolled-back', 'failed'].includes(job.phase)) {
+    atomicJson(latestPath, job);
+    return reconcileTerminalLock(agentHome, job);
+  }
+  if (job.phase === 'repair-starting') {
+    for (const identity of [job.repairRuntime, job.repairLauncher]) {
+      const state = identityState(identity);
+      if (state.status === 'match') return true;
+      if (state.status === 'error') {
+        throw new Error('cannot determine interrupted repair process identity');
+      }
+    }
+    const launcherIdentity = processIdentity(process.pid);
+    if (launcherIdentity.status !== 'found') {
+      throw new Error('cannot determine exact repair resume launcher identity');
+    }
+    job = {
+      ...job,
+      repairLauncher: {
+        pid: process.pid,
+        processStartIdentity: launcherIdentity.identity,
+      },
+      updatedAt: new Date().toISOString(),
+    };
+    atomicJson(jobPath, job);
+    atomicJson(latestPath, job);
+    repairResumeJobId = job.jobId;
+    forceLastKnownGood = true;
+    return false;
+  }
+  if (['created', 'downloading', 'verifying', 'staging', 'self-checking'].includes(job.phase)) {
+    try {
+      markInterruptedPreShutdown(agentHome, jobPath, job, `launcher recovered interrupted ${job.phase}`);
+    } catch (error) {
+      if (error.message === 'active upgrade lock still has a live owner') return true;
+      throw error;
+    }
+    return false;
+  }
+  if (job.phase === 'worker-starting') {
+    const state = identityState(job.worker);
+    if (state.status === 'match') return true;
+    if (state.status === 'error') throw new Error('cannot determine interrupted worker identity');
+    markInterruptedPreShutdown(agentHome, jobPath, job, 'launcher recovered a worker that never became ready');
+    return false;
+  }
+  const workerState = identityState(job.worker);
+  if (workerState.status === 'match') return true;
+  if (workerState.status === 'error') throw new Error('cannot determine recovery worker identity');
+  startRecoveryWorker(agentHome, jobPath, job);
+  return true;
+}
+
 const agentHome = path.resolve(process.env.AGENTLINK_HOME || path.join(os.homedir(), '.agentlink'));
 const versionsRoot = path.join(agentHome, 'versions');
+try {
+  if (recoverUpgrade(agentHome)) return;
+} catch (error) {
+  process.stderr.write(`[AgentLink Launcher] Upgrade recovery could not start: ${error.message}\n`);
+  process.exitCode = 1;
+  return;
+}
 let pointer;
 try {
-  pointer = loadPointer(path.join(agentHome, 'current-version.json'), versionsRoot);
+  pointer = loadPointer(
+    path.join(agentHome, forceLastKnownGood ? 'last-known-good.json' : 'current-version.json'),
+    versionsRoot,
+  );
 } catch (currentError) {
   try {
     pointer = loadPointer(path.join(agentHome, 'last-known-good.json'), versionsRoot);
@@ -78,14 +575,31 @@ try {
   }
 }
 
-const child = spawn(process.execPath, [pointer.entry, ...process.argv.slice(2)], {
+const {
+  AGENTLINK_RECOVERY_OWNER: _recoveryOwner,
+  AGENTLINK_UPGRADE_JOB: _upgradeJob,
+  ...agentEnvironment
+} = process.env;
+const forwardedArgs = process.argv.slice(2);
+const activeUpgradeJobId = repairResumeJobId || workerOwnedJobId;
+if (activeUpgradeJobId && !forwardedArgs.includes('--upgrade-job')) {
+  forwardedArgs.push('--upgrade-job', activeUpgradeJobId);
+}
+const child = spawn(process.execPath, [pointer.entry, ...forwardedArgs], {
   stdio: 'inherit',
   windowsHide: true,
   env: {
-    ...process.env,
+    ...agentEnvironment,
     AGENTLINK_HOME: agentHome,
     AGENTLINK_VERSION_STORE_ACTIVE: '1',
     AGENTLINK_SELECTED_VERSION: pointer.version,
+    AGENTLINK_SELECTED_INTEGRITY: pointer.integrity,
+    ...(activeUpgradeJobId ? {
+      AGENTLINK_UPGRADE_JOB: activeUpgradeJobId,
+    } : {}),
+    ...(repairResumeJobId ? {
+      AGENTLINK_RECOVERY_OWNER: 'repair',
+    } : {}),
   },
 });
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agent-link/agent",
-  "version": "0.1.298",
+  "version": "0.1.300",
   "description": "AgentLink local agent CLI",
   "license": "MIT",
   "repository": {