@agentpulselabs/cli-plugin-pulse 0.2.0 → 1.0.0

package/src/lib/processOwnership.js

@@ -0,0 +1,252 @@
+import { readFile } from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { HubError } from './errors.js';
+import { ensureOwnerOnlyDirectory, readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
+import { execFileBounded as execFile } from './subprocess.js';
+
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+const PROCESS_FIELDS = ['pid', 'pgid', 'bootId', 'startToken'];
+const NO_CHANGE = Symbol('no-change');
+
+function validProcessIdentity(identity, expectedPgid = null) {
+  return (
+    Number.isInteger(identity?.pid) &&
+    identity.pid > 0 &&
+    Number.isInteger(identity.pgid) &&
+    identity.pgid > 0 &&
+    (expectedPgid === null || identity.pgid === expectedPgid) &&
+    typeof identity.bootId === 'string' &&
+    identity.bootId.length > 0 &&
+    typeof identity.startToken === 'string' &&
+    identity.startToken.length > 0
+  );
+}
+
+function processIdentity(identity) {
+  return Object.fromEntries(PROCESS_FIELDS.map((field) => [field, identity[field]]));
+}
+
+export class ProcessOwnerStore {
+  #queue = Promise.resolve();
+
+  static async open(directory) {
+    await ensureOwnerOnlyDirectory(directory);
+    const file = path.join(directory, 'process-owners.json');
+    const data = await readOwnerOnlyJson(file, { schemaVersion: 1, owners: {} });
+    if (data.schemaVersion !== 1 || !data.owners || typeof data.owners !== 'object') {
+      throw new HubError('HUB_PROCESS_STORE_INVALID', 'Process owner store is invalid');
+    }
+    if (
+      Object.keys(data.owners).length > 1_000 ||
+      Buffer.byteLength(JSON.stringify(data), 'utf8') > 1024 * 1024
+    ) {
+      throw new HubError('HUB_PROCESS_STORE_INVALID', 'Process owner store exceeds safety bounds');
+    }
+    for (const [processRef, owner] of Object.entries(data.owners)) {
+      if (
+        owner.processRef !== processRef ||
+        !Number.isInteger(owner.pid) ||
+        owner.pid < 1 ||
+        owner.pgid !== owner.pid ||
+        typeof owner.bootId !== 'string' ||
+        typeof owner.startToken !== 'string' ||
+        !validProcessIdentity(owner.sentinel, owner.pgid) ||
+        !Array.isArray(owner.members) ||
+        owner.members.length < 1 ||
+        owner.members.length > 256 ||
+        owner.members.some((member) => !validProcessIdentity(member, owner.pgid)) ||
+        !['running', 'stopped'].includes(owner.state)
+      ) {
+        throw new HubError('HUB_PROCESS_STORE_INVALID', 'Persisted process identity is invalid');
+      }
+    }
+    return new ProcessOwnerStore(file, data);
+  }
+
+  constructor(file, data) {
+    this.file = file;
+    this.data = data;
+  }
+
+  async #mutate(operation) {
+    const pending = this.#queue.then(async () => {
+      const draft = structuredClone(this.data);
+      const result = operation(draft);
+      if (result === NO_CHANGE) return null;
+      await writeOwnerOnlyJson(this.file, draft);
+      this.data = draft;
+      return structuredClone(result);
+    });
+    this.#queue = pending.catch(() => {});
+    return pending;
+  }
+
+  get(processRef) {
+    const record = this.data.owners[processRef];
+    return record ? structuredClone(record) : null;
+  }
+
+  async record(identity) {
+    if (
+      typeof identity.processRef !== 'string' ||
+      identity.processRef.length > 128 ||
+      !Number.isInteger(identity.pid) ||
+      identity.pid < 1 ||
+      identity.pgid !== identity.pid ||
+      !UUID.test(String(identity.jobId || '')) ||
+      !Number.isInteger(identity.attemptNo) ||
+      identity.attemptNo < 1 ||
+      !Number.isSafeInteger(identity.fenceToken) ||
+      identity.fenceToken < 1 ||
+      !['cursor', 'claude'].includes(identity.provider)
+    ) {
+      throw new HubError('HUB_PROCESS_IDENTITY_INVALID', 'Process ownership record is invalid');
+    }
+    return this.#mutate((draft) => {
+      const entries = Object.values(draft.owners);
+      if (entries.length >= 800) {
+        const stopped = entries
+          .filter((owner) => owner.state === 'stopped')
+          .sort((left, right) => String(left.stoppedAt).localeCompare(String(right.stoppedAt)));
+        for (const owner of stopped.slice(0, Math.max(0, entries.length - 700))) {
+          delete draft.owners[owner.processRef];
+        }
+      }
+      if (Object.keys(draft.owners).length >= 1_000) {
+        throw new HubError('HUB_PROCESS_STORE_FULL', 'Process owner store reached its safety limit');
+      }
+      const record = {
+        ...structuredClone(identity),
+        sentinel: processIdentity(identity),
+        members: [processIdentity(identity)],
+        state: 'running',
+        recordedAt: new Date().toISOString(),
+      };
+      draft.owners[identity.processRef] = record;
+      return record;
+    });
+  }
+
+  async updateMembers(processRef, expectedSentinel, members) {
+    if (
+      !Array.isArray(members) ||
+      members.length < 1 ||
+      members.length > 256 ||
+      members.some((member) => !validProcessIdentity(member, expectedSentinel?.pgid))
+    ) {
+      throw new HubError('HUB_PROCESS_IDENTITY_INVALID', 'Process member identity is invalid');
+    }
+    return this.#mutate((draft) => {
+      const record = draft.owners[processRef];
+      if (
+        !record ||
+        record.state !== 'running' ||
+        PROCESS_FIELDS.some((field) => record.sentinel[field] !== expectedSentinel[field])
+      ) {
+        return NO_CHANGE;
+      }
+      const nextMembers = members.map(processIdentity).sort((left, right) => left.pid - right.pid);
+      if (JSON.stringify(record.members) === JSON.stringify(nextMembers)) return NO_CHANGE;
+      record.members = nextMembers;
+      record.membersObservedAt = new Date().toISOString();
+      return record;
+    });
+  }
+
+  async markStopped(processRef) {
+    return this.#mutate((draft) => {
+      const record = draft.owners[processRef];
+      if (!record) return null;
+      record.state = 'stopped';
+      record.stoppedAt = new Date().toISOString();
+      return record;
+    });
+  }
+}
+
+export class ProcessIdentityProvider {
+  constructor({ platform = process.platform, execute = execFile } = {}) {
+    this.platform = platform;
+    this.execute = execute;
+  }
+
+  async capture(pid) {
+    if (!Number.isInteger(pid) || pid < 1) {
+      throw new HubError('HUB_PROCESS_IDENTITY_INVALID', 'Process pid is invalid');
+    }
+    switch (this.platform) {
+      case 'linux': {
+        const [bootId, stat] = await Promise.all([
+          readFile('/proc/sys/kernel/random/boot_id', 'utf8'),
+          readFile(`/proc/${pid}/stat`, 'utf8'),
+        ]);
+        const close = stat.lastIndexOf(') ');
+        const fields = stat.slice(close + 2).trim().split(/\s+/);
+        const startToken = fields[19];
+        const pgid = Number(fields[2]);
+        if (!startToken) throw new HubError('HUB_PROCESS_IDENTITY_INVALID', 'Linux start token is unavailable');
+        if (!Number.isInteger(pgid) || pgid < 1) {
+          throw new HubError('HUB_PROCESS_IDENTITY_INVALID', 'Linux process group is unavailable');
+        }
+        return { pid, pgid, bootId: bootId.trim(), startToken };
+      }
+      case 'darwin': {
+        const started = await this.execute('/bin/ps', ['-o', 'pgid=,lstart=', '-p', String(pid)]);
+        const bootMinute = Math.round((Date.now() - os.uptime() * 1_000) / 60_000);
+        const bootId = `darwin:${bootMinute}`;
+        const match = String(started.stdout).trim().match(/^(\d+)\s+(.+)$/);
+        const pgid = Number(match?.[1]);
+        const startToken = match?.[2];
+        if (!bootId || !startToken || !Number.isInteger(pgid) || pgid < 1) {
+          throw new HubError('HUB_PROCESS_IDENTITY_INVALID', 'macOS process identity is unavailable');
+        }
+        return { pid, pgid, bootId, startToken };
+      }
+      default:
+        throw new HubError('HUB_UNSUPPORTED_PLATFORM', `Unsupported process platform: ${this.platform}`);
+    }
+  }
+
+  async matches(identity) {
+    try {
+      const current = await this.capture(identity.pid);
+      return (
+        current.bootId === identity.bootId &&
+        current.startToken === identity.startToken &&
+        current.pgid === identity.pgid
+      );
+    } catch {
+      return false;
+    }
+  }
+
+  async groupAlive(pgid) {
+    try {
+      process.kill(-pgid, 0);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async groupMembers(pgid) {
+    const listed = await this.execute('/bin/ps', ['-axo', 'pid=,pgid=']);
+    const pids = String(listed.stdout)
+      .split('\n')
+      .map((line) => line.trim().match(/^(\d+)\s+(\d+)$/))
+      .filter((match) => match && Number(match[2]) === pgid)
+      .map((match) => Number(match[1]))
+      .slice(0, 256);
+    const captured = await Promise.all(
+      pids.map(async (pid) => {
+        try {
+          return await this.capture(pid);
+        } catch {
+          return null;
+        }
+      })
+    );
+    return captured.filter((identity) => identity && identity.pgid === pgid);
+  }
+}

package/src/lib/processSupervisor.js

@@ -0,0 +1,376 @@
+import { spawn } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
+import { HubError } from './errors.js';
+import { ProcessIdentityProvider } from './processOwnership.js';
+import { redactText } from './redaction.js';
+
+function signalLiveChild(child, signal) {
+  if (!child.pid || child.exitCode !== null || child.signalCode !== null) return false;
+  try {
+    return child.kill(signal);
+  } catch {
+    return false;
+  }
+}
+
+export class ProcessSupervisor {
+  #processes = new Map();
+
+  constructor({
+    spawnProcess = spawn,
+    roots = [],
+    graceMs = 5_000,
+    ownerStore = null,
+    identityProvider = new ProcessIdentityProvider(),
+    memberRefreshMs = 50,
+    signalGroup = (pgid, signal) => process.kill(-pgid, signal),
+  } = {}) {
+    this.spawnProcess = spawnProcess;
+    this.roots = roots;
+    this.graceMs = graceMs;
+    this.ownerStore = ownerStore;
+    this.identityProvider = identityProvider;
+    this.memberRefreshMs = memberRefreshMs;
+    this.signalGroup = signalGroup;
+  }
+
+  async probe(bin, args = ['--version']) {
+    try {
+      const result = await this.run({
+        bin,
+        args,
+        cwd: process.cwd(),
+        timeoutMs: 10_000,
+        maxOutputBytes: 64 * 1024,
+        owned: false,
+      });
+      return {
+        available: result.exitCode === 0,
+        version: result.stdout.trim().split('\n')[0] || null,
+      };
+    } catch (error) {
+      return { available: false, code: error.code || 'HUB_PROVIDER_MISSING' };
+    }
+  }
+
+  run({
+    bin,
+    args,
+    cwd,
+    env = {},
+    timeoutMs,
+    maxOutputBytes,
+    signal,
+    onStart,
+    input = '',
+    owned = true,
+    provider = null,
+    attempt = null,
+  }) {
+    if (typeof bin !== 'string' || !Array.isArray(args) || args.some((arg) => typeof arg !== 'string')) {
+      throw new HubError('HUB_PROCESS_INVALID', 'Provider process requires a binary and string argv');
+    }
+    const processRef = randomUUID();
+    return new Promise((resolve, reject) => {
+      const child = this.spawnProcess(bin, args, {
+        cwd,
+        env,
+        detached: true,
+        shell: false,
+        stdio: ['pipe', 'pipe', 'pipe'],
+        windowsHide: true,
+      });
+      const record = { processRef, child, state: 'running', startedAt: Date.now() };
+      this.#processes.set(processRef, record);
+      const ownership = (async () => {
+        if (!owned) return { processRef, pid: child.pid, pgid: child.pid };
+        if (!this.ownerStore || !attempt || !provider) {
+          throw new HubError(
+            'HUB_PROCESS_OWNERSHIP_REQUIRED',
+            'Owned provider execution requires a durable owner store, provider, and attempt tuple'
+          );
+        }
+        const osIdentity = await this.identityProvider.capture(child.pid);
+        const identity = await this.ownerStore.record({
+          processRef,
+          ...osIdentity,
+          provider,
+          jobId: attempt.jobId,
+          attemptNo: attempt.attemptNo,
+          fenceToken: attempt.fenceToken,
+        });
+        record.identity = identity;
+        await onStart?.(identity);
+        return identity;
+      })();
+      const stdout = [];
+      const stderr = [];
+      let outputBytes = 0;
+      let settled = false;
+      let timer;
+      let memberTimer;
+
+      const finish = (callback) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timer);
+        clearInterval(memberTimer);
+        signal?.removeEventListener('abort', onAbort);
+        callback();
+      };
+      const escalate = async (code, message) => {
+        try {
+          if (owned) {
+            const identity = await ownership;
+            const outcome = await this.cancel(identity);
+            if (!outcome.verifiedStopped) {
+              finish(() =>
+                reject(new HubError('HUB_CANCELLATION_UNPROVEN', 'Owned process did not stop'))
+              );
+              return;
+            }
+          } else {
+            signalLiveChild(child, 'SIGTERM');
+            await delay(this.graceMs);
+            signalLiveChild(child, 'SIGKILL');
+          }
+          finish(() => reject(new HubError(code, message)));
+        } catch (error) {
+          finish(() => reject(error));
+        }
+      };
+      const append = (target, chunk) => {
+        outputBytes += chunk.length;
+        if (outputBytes > maxOutputBytes) {
+          record.state = 'output_exceeded';
+          void escalate('HUB_BUDGET_OUTPUT', 'Provider output budget exceeded');
+          return;
+        }
+        target.push(chunk);
+      };
+      child.stdout?.on('data', (chunk) => append(stdout, chunk));
+      child.stderr?.on('data', (chunk) => append(stderr, chunk));
+      child.on('error', (error) => {
+        record.state = 'failed';
+        finish(() =>
+          reject(new HubError('HUB_PROVIDER_MISSING', `Could not start ${bin}`, { cause: error }))
+        );
+      });
+      child.on('close', (code, childSignal) => {
+        const priorState = record.state;
+        record.state = 'exited';
+        record.endedAt = Date.now();
+        finish(() => void ownership.then(async (identity) => {
+          if (owned && !await this.#groupAlive(identity)) {
+            await this.ownerStore.markStopped(identity.processRef);
+          }
+          if (priorState === 'output_exceeded') {
+            reject(new HubError('HUB_BUDGET_OUTPUT', 'Provider output budget exceeded'));
+            return;
+          }
+          if (priorState === 'timed_out') {
+            reject(new HubError('HUB_BUDGET_TIME', 'Provider wall-time budget exceeded'));
+            return;
+          }
+          if (priorState === 'cancelling') {
+            reject(new HubError('HUB_PROVIDER_CANCELLED', 'Provider process was cancelled'));
+            return;
+          }
+          const cleanStdout = redactText(Buffer.concat(stdout).toString('utf8'), {
+            roots: this.roots,
+            maxBytes: maxOutputBytes,
+          }).text;
+          const cleanStderr = redactText(Buffer.concat(stderr).toString('utf8'), {
+            roots: this.roots,
+            maxBytes: maxOutputBytes,
+          }).text;
+          resolve({
+            processRef,
+            processIdentity: identity,
+            stdout: cleanStdout,
+            stderr: cleanStderr,
+            exitCode: code,
+            signal: childSignal,
+          });
+        }).catch((error) => reject(error)));
+      });
+      ownership.catch((error) => {
+        record.state = 'failed';
+        void (async () => {
+          if (record.identity) {
+            await this.cancel(record.identity);
+          } else {
+            signalLiveChild(child, 'SIGKILL');
+          }
+          finish(() => reject(error));
+        })().catch((cancelError) => finish(() => reject(cancelError)));
+      });
+      ownership.then((identity) => {
+        if (!owned || typeof this.identityProvider.groupMembers !== 'function') return;
+        let refreshInFlight = false;
+        const refresh = async () => {
+          if (refreshInFlight) return;
+          refreshInFlight = true;
+          try {
+            if (!await this.identityProvider.matches(identity.sentinel || identity)) return;
+            const members = await this.identityProvider.groupMembers(identity.pgid);
+            if (members.length) {
+              await this.ownerStore.updateMembers(
+                identity.processRef,
+                identity.sentinel || identity,
+                members
+              );
+            }
+          } finally {
+            refreshInFlight = false;
+          }
+        };
+        void refresh();
+        memberTimer = setInterval(() => void refresh().catch(() => {}), this.memberRefreshMs);
+        memberTimer.unref?.();
+      }).catch(() => {});
+      ownership.then(() => child.stdin?.end(input)).catch(() => {});
+      const onAbort = () => {
+        record.state = 'cancelling';
+        void escalate('HUB_PROVIDER_CANCELLED', 'Provider process was cancelled');
+      };
+      signal?.addEventListener('abort', onAbort, { once: true });
+      timer = setTimeout(() => {
+        record.state = 'timed_out';
+        void escalate('HUB_BUDGET_TIME', 'Provider wall-time budget exceeded');
+      }, timeoutMs);
+      timer.unref?.();
+    });
+  }
+
+  async #waitForStop(identity, milliseconds) {
+    const deadline = Date.now() + milliseconds;
+    while (Date.now() < deadline) {
+      const [verified, groupAlive] = await Promise.all([
+        this.#verifiedMembers(identity),
+        this.#groupAlive(identity),
+      ]);
+      if (verified.length === 0 && !groupAlive) return true;
+      await delay(Math.min(100, Math.max(1, deadline - Date.now())));
+    }
+    const [verified, groupAlive] = await Promise.all([
+      this.#verifiedMembers(identity),
+      this.#groupAlive(identity),
+    ]);
+    return verified.length === 0 && !groupAlive;
+  }
+
+  #groupAlive(identity) {
+    if (typeof this.identityProvider.groupAlive === 'function') {
+      return this.identityProvider.groupAlive(identity.pgid);
+    }
+    return this.identityProvider.matches(identity);
+  }
+
+  async #verifiedMembers(identity) {
+    const members = Array.isArray(identity.members) && identity.members.length
+      ? identity.members
+      : [identity.sentinel || identity];
+    const matches = await Promise.all(
+      members.map(async (member) => await this.identityProvider.matches(member))
+    );
+    return members.filter((_member, index) => matches[index]);
+  }
+
+  async #refreshMembers(identity) {
+    if (typeof this.identityProvider.groupMembers !== 'function') return identity;
+    const verifiedBefore = await this.#verifiedMembers(identity);
+    if (verifiedBefore.length === 0) return identity;
+    const members = await this.identityProvider.groupMembers(identity.pgid);
+    if (!members.length) return identity;
+    const anchored = members.filter((member) =>
+      verifiedBefore.some((verified) =>
+        ['pid', 'pgid', 'bootId', 'startToken'].every(
+          (field) => member[field] === verified[field]
+        )
+      )
+    );
+    if (anchored.length === 0 || (await this.#verifiedMembers({ members: anchored })).length === 0) {
+      return identity;
+    }
+    await this.ownerStore.updateMembers(
+      identity.processRef,
+      identity.sentinel || identity,
+      members
+    );
+    return this.ownerStore.get(identity.processRef) || identity;
+  }
+
+  async cancel(identity) {
+    if (!identity || !this.ownerStore) return { cancelled: false, verifiedStopped: false, reason: 'not_owned' };
+    const stored = this.ownerStore.get(identity.processRef);
+    const fields = ['pid', 'pgid', 'bootId', 'startToken', 'jobId', 'attemptNo', 'fenceToken'];
+    if (!stored || fields.some((field) => stored[field] !== identity[field])) {
+      return { cancelled: false, verifiedStopped: false, reason: 'not_owned' };
+    }
+    let [verifiedMembers, groupAlive] = await Promise.all([
+      this.#verifiedMembers(stored),
+      this.#groupAlive(stored),
+    ]);
+    if (verifiedMembers.length === 0) {
+      if (!groupAlive) {
+        await this.ownerStore.markStopped(stored.processRef);
+        return { cancelled: true, verifiedStopped: true, reason: 'already_stopped' };
+      }
+      return {
+        cancelled: false,
+        verifiedStopped: false,
+        reason: 'leader_identity_lost',
+      };
+    }
+    const refreshed = await this.#refreshMembers(stored);
+    [verifiedMembers, groupAlive] = await Promise.all([
+      this.#verifiedMembers(refreshed),
+      this.#groupAlive(refreshed),
+    ]);
+    if (verifiedMembers.length === 0 || !groupAlive) {
+      if (verifiedMembers.length === 0 && !groupAlive) {
+        await this.ownerStore.markStopped(refreshed.processRef);
+        return { cancelled: true, verifiedStopped: true, reason: 'already_stopped' };
+      }
+      return { cancelled: false, verifiedStopped: false, reason: 'ownership_changed' };
+    }
+    try {
+      this.signalGroup(refreshed.pgid, 'SIGTERM');
+    } catch {
+      // Verification below determines whether the owned identity is gone.
+    }
+    if (await this.#waitForStop(refreshed, this.graceMs)) {
+      await this.ownerStore.markStopped(refreshed.processRef);
+      return { cancelled: true, verifiedStopped: true, signal: 'SIGTERM' };
+    }
+    if ((await this.#verifiedMembers(refreshed)).length === 0) {
+      return {
+        cancelled: true,
+        verifiedStopped: false,
+        signal: 'SIGTERM',
+        reason: 'group_alive_without_verified_member',
+      };
+    }
+    try {
+      this.signalGroup(refreshed.pgid, 'SIGKILL');
+    } catch {
+      // Verification below determines whether the owned identity is gone.
+    }
+    const verifiedStopped = await this.#waitForStop(refreshed, this.graceMs);
+    if (verifiedStopped) await this.ownerStore.markStopped(refreshed.processRef);
+    return { cancelled: true, verifiedStopped, signal: 'SIGKILL' };
+  }
+
+  status(processRef) {
+    const record = this.#processes.get(processRef);
+    if (!record) return { processRef, state: 'unknown' };
+    return {
+      processRef,
+      state: record.state,
+      startedAt: record.startedAt,
+      endedAt: record.endedAt ?? null,
+    };
+  }
+}