@runeditrun/rer 0.0.0-stage → 0.1.1

package/modules/job-queue/command.mjs

@@ -0,0 +1,64 @@
+import { spawn } from 'node:child_process';
+import { constants } from 'node:os';
+import { randomUUID } from 'node:crypto';
+import { unlink } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import { stopGroup } from './processes.mjs';
+
+export function signalExit(signal) { return 128 + (constants.signals[signal] ?? 1); }
+
+export async function execute(command, job, config, adopt, signal, onStarted, options = {}) {
+  const child = (options.spawn ?? spawn)(process.execPath, [fileURLToPath(new URL('./supervisor.mjs', import.meta.url)), randomUUID(), String(config.killGraceMs)],
+    { cwd: job.repository, env: options.env ?? process.env, detached: true, stdio: ['inherit', 'inherit', 'inherit', 'ipc'] });
+  let owner, finish, outcome, reported = false;
+  const done = new Promise(resolve => { finish = resolve; });
+  const exited = new Promise(resolve => child.once('close', (code, signame) => {
+    finish({ code: code ?? signalExit(signame), signal: signame }); resolve();
+  }));
+  child.once('error', error => finish({ code: 127, error: error.message }));
+  child.on('message', message => {
+    if (message.done) { reported = true; finish({ ...message.done,
+      code: message.done.code ?? signalExit(message.done.signal) }); }
+  });
+  // The supervisor is this process's own child, so its exit always arrives. An
+  // exit without its done report means the job lost its supervisor; close alone
+  // can wait forever, because descendants still hold the inherited output pipes.
+  child.once('exit', (code, signame) => {
+    if (owner && !reported) finish({ code: code ?? signalExit(signame), signal: signame, supervisorEnded: true,
+      error: `${job.label}: the job's supervisor ended before the job finished; what remained of the job was stopped and its slot released.` });
+  });
+  const ended = () => child.exitCode !== null || child.signalCode !== null;
+  const abort = () => finish({ code: signalExit(signal.reason || 'SIGTERM'), signal: signal.reason });
+  signal.addEventListener('abort', abort, { once: true });
+  try {
+    const ready = await new Promise((resolve, reject) => {
+      const timer = setTimeout(() => reject(Error('Job supervisor did not become ready')), 10000);
+      child.once('message', message => { clearTimeout(timer); resolve(message.ready); });
+      child.once('error', error => { clearTimeout(timer); reject(error); });
+      child.once('exit', () => { clearTimeout(timer); reject(Error('Job supervisor exited before readiness')); });
+    });
+    owner = ready;
+    if (!owner || owner.pid !== child.pid) throw Error('Invalid job supervisor identity');
+    // Commit the group lease before allowing the supervisor to spawn any work.
+    await adopt(owner);
+    if (signal.aborted) abort();
+    else if (!ended()) {
+      await onStarted?.(job);
+      if (signal.aborted) abort();
+      else if (!ended()) child.send({ command, cwd: job.repository, nice: config.nice, membership: JSON.stringify({ directory: config.directory, id: job.id, token: job.membership }) });
+    }
+    outcome=await done;return outcome;
+  } finally {
+    signal.removeEventListener('abort', abort);
+    try {
+      if (owner) await stopGroup(owner, config.killGraceMs);
+      else child.kill('SIGKILL'); // The supervisor has not been given GO.
+      // Its group is stopped; output still arriving on a dead supervisor's pipes
+      // comes from escaped descendants, which the caller's cleanup stops.
+      if (outcome?.supervisorEnded) { child.stdout?.destroy(); child.stderr?.destroy(); }
+      await exited;
+      await options.cleanup?.(child,outcome);
+      if (owner?.socket) await unlink(owner.socket).catch(error => { if (error.code !== 'ENOENT') throw error; });
+    } catch (error) { error.keepLease = true; throw error; }
+  }
+}

package/modules/job-queue/config.mjs

@@ -0,0 +1,48 @@
+import { cpus, totalmem, userInfo } from 'node:os';
+import { readFile } from 'node:fs/promises';
+import { resolve } from 'node:path';
+
+export const GIB = 1024 ** 3;
+export const TIMEOUT_EXIT = 75;
+export const priorities = { owner: 0, normal: 1, background: 2 };
+const names = { slots: 'SLOTS', loadThreshold: 'LOAD_THRESHOLD',
+  minFreeMemoryBytes: 'MIN_FREE_MEMORY_BYTES', pollMs: 'POLL_MS',
+  nice: 'NICE', killGraceMs: 'KILL_GRACE_MS' };
+
+export async function configuration({ env = process.env, ...overrides } = {}) {
+  const directory = resolve(overrides.directory ?? env.RER_JOBS_DIR ?? `${userInfo().homedir}/.rer/jobs`);
+  let disk = {};
+  try { disk = JSON.parse(await readFile(resolve(directory, 'config.json'), 'utf8')); }
+  catch (error) { if (error.code !== 'ENOENT') throw error; }
+  const config = { slots: Math.max(1, Math.min(3, Math.floor(cpus().length / 4),
+    Math.floor(totalmem() / (8 * GIB)))), loadThreshold: Infinity,
+  minFreeMemoryBytes: 2 * GIB, pollMs: 3000, nice: 15, killGraceMs: 1000, ...disk };
+  for (const [key, suffix] of Object.entries(names)) {
+    if (env[`RER_JOBS_${suffix}`] !== undefined) config[key] = Number(env[`RER_JOBS_${suffix}`]);
+    if (overrides[key] !== undefined) config[key] = overrides[key];
+    if (!(Number.isFinite(config[key]) || key === 'loadThreshold' && config[key] === Infinity) || config[key] < 0) throw Error(`Invalid job queue ${key}`);
+  }
+  if (!Number.isInteger(config.slots) || config.slots < 1 || config.pollMs < 10
+    || !Number.isInteger(config.nice) || config.nice > 19) throw Error('Invalid slots, pollMs or nice');
+  return { ...config, directory };
+}
+
+export function ordered(jobs) {
+  return [...jobs].sort((a, b) => priorities[a.priority] - priorities[b.priority] || a.sequence - b.sequence);
+}
+
+// Each reason says what is running, what the limit is and how to change it.
+export function admissionDetails(running, config, sample) {
+  if (running >= config.slots) return { code: 'slots', reason: `all job slots are in use (${running} of ${config.slots}; change with RER_JOBS_SLOTS)` };
+  if (!Number.isFinite(sample.load) || !Number.isFinite(sample.availableMemoryBytes))
+    return { code: 'resources_unavailable', reason: 'this computer\'s load or available memory could not be read' };
+  if (sample.load > config.loadThreshold) return { code: 'load',
+    reason: `the 1-minute load is ${sample.load.toFixed(1)} and jobs start at ${config.loadThreshold} or below (change with RER_JOBS_LOAD_THRESHOLD)` };
+  if (sample.availableMemoryBytes <= config.minFreeMemoryBytes) return { code: 'free_memory',
+    reason: `available memory is at or below the floor (change with RER_JOBS_MIN_FREE_MEMORY_BYTES)` };
+  return null;
+}
+
+export function admission(running, config, sample) {
+  return admissionDetails(running, config, sample)?.reason ?? null;
+}

package/modules/job-queue/identity.mjs

@@ -0,0 +1,48 @@
+import { createServer, createConnection } from 'node:net';
+import { randomUUID } from 'node:crypto';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { chmod, unlink } from 'node:fs/promises';
+import { processInfo } from './processes.mjs';
+
+// A lifetime nonce supplements Darwin's second-resolution ps start time.
+// It also distinguishes two library clients in the same Node process.
+export async function identity() {
+  const token = randomUUID(), socket = join(tmpdir(), `rer-job-${token}.sock`);
+  // Darwin's TMPDIR can be long; use the short system temporary directory then.
+  const path = Buffer.byteLength(socket) < 100 ? socket : `/tmp/rer-job-${token}.sock`;
+  const server = createServer(client => { client.end(token); });
+  await new Promise((resolve, reject) => { server.once('error', reject); server.listen(path, resolve); });
+  await chmod(path, 0o600);
+  const { title, ...processIdentity } = await processInfo(process.pid);
+  const info = { ...processIdentity, token, socket: path };
+  return { info, async close() {
+    await new Promise(resolve => server.close(resolve));
+    await unlink(path).catch(error => { if (error.code !== 'ENOENT') throw error; });
+  } };
+}
+
+export async function alive(owner) {
+  const current = await processInfo(owner.pid);
+  if (!current || current.zombie || current.start !== owner.start) return false;
+  if (!owner.socket || !owner.token) throw Error('Lease lacks lifetime identity');
+  return new Promise((resolve, reject) => {
+    const connection = createConnection(owner.socket);
+    let body = '';
+    connection.setTimeout(3000, () => connection.destroy(Error('Owner identity probe timed out; refusing admission')));
+    connection.on('data', data => { body += data; if (body.length > 128) connection.destroy(Error('Invalid owner identity')); });
+    connection.on('end', () => resolve(body === owner.token));
+    connection.on('error', async error => {
+      if (['ENOENT', 'ECONNREFUSED'].includes(error.code)) resolve(false);
+      else if (error.code === 'ECONNRESET') {
+        // Listener shutdown can reset a probe after the initial PID check.
+        // Reclaim proven death; otherwise retain the owner until the next sample.
+        try {
+          const after = await processInfo(owner.pid);
+          resolve(Boolean(after && !after.zombie && after.start === owner.start));
+        } catch (failure) { reject(failure); }
+      }
+      else reject(error);
+    });
+  });
+}

package/modules/job-queue/index.mjs

@@ -0,0 +1,135 @@
+import { randomUUID } from 'node:crypto';
+import { realpath, unlink } from 'node:fs/promises';
+import { performance } from 'node:perf_hooks';
+import { setTimeout as delay } from 'node:timers/promises';
+import { configuration, admissionDetails, ordered, priorities, TIMEOUT_EXIT } from './config.mjs';
+import { identity, alive } from './identity.mjs';
+import { transaction } from './storage.mjs';
+import { sampleResources } from './resources.mjs';
+import { execute, signalExit } from './command.mjs';
+import { stopGroup } from './processes.mjs';
+
+const WAIT_REPORT_INTERVAL_MS = 15_000;
+
+export async function createQueue(options = {}) {
+  const config = await configuration(options), lifetime = await identity();
+  const sampler = options.sampler ?? sampleResources;
+  let active = 0, closed = false;
+  const mutate = action => {
+    if (closed) throw Error('Job queue is closed');
+    return transaction(config.directory, lifetime.info, action);
+  };
+  async function reclaim(state) {
+    for (const key of ['queued', 'running']) {
+      const kept = [];
+      for (const job of state[key]) {
+        if (await alive(job.owner)) kept.push(job);
+        else {
+          if (key === 'running') await stopGroup(job.owner, config.killGraceMs);
+          if (job.owner.socket) await unlink(job.owner.socket).catch(error => { if (error.code !== 'ENOENT') throw error; });
+        }
+      }
+      state[key] = kept;
+    }
+  }
+  async function status() {
+    return mutate(async state => {
+      await reclaim(state);
+      const resources = await sampler();
+      const describe = ({ command, membership, owner: { token, socket, ...owner }, ...job }) => ({ ...job, owner,
+        elapsedSeconds: Math.floor((Date.now() - (job.startedAt ?? job.enqueuedAt)) / 1000) });
+      return { running: state.running.map(describe), queued: ordered(state.queued).map(describe), resources, config,
+        admissionReason: state.queued.length ? admissionDetails(state.running.length, config, resources)?.reason ?? 'ready' : 'queue empty' };
+    });
+  }
+  async function run(command, opts = {}) {
+    if (!Array.isArray(command) || !command.length || command.some(arg => typeof arg !== 'string')) throw Error('Command must be a nonempty string array');
+    if ((opts.kind ?? 'heavy') !== 'heavy' || !Object.hasOwn(priorities, opts.priority ?? 'normal')) throw Error('Invalid job kind or priority');
+    if (opts.maxWait !== undefined && (!Number.isFinite(opts.maxWait) || opts.maxWait < 0)) throw Error('Invalid maxWait');
+    const repository = await realpath(opts.cwd ?? process.cwd());
+    const controller = new AbortController(), start = performance.now();
+    const interrupt = () => controller.abort('SIGINT'), terminate = () => controller.abort('SIGTERM');
+    const external = () => controller.abort(opts.signal.reason === 'SIGINT' ? 'SIGINT' : 'SIGTERM');
+    process.on('SIGINT', interrupt); process.on('SIGTERM', terminate);
+    opts.signal?.addEventListener('abort', external, { once: true });
+    if (opts.signal?.aborted) external();
+    const job = { id: randomUUID(), membership: randomUUID(), label: opts.label ?? command[0], command,
+      repository, priority: opts.priority ?? 'normal',
+      enqueuedAt: Date.now(), owner: lifetime.info };
+    active++;
+    let keepLease = false, lastWaitReportAt = 0, lastWaitSignature;
+    try {
+      await mutate(state => { job.sequence = ++state.sequence; state.queued.push(job); });
+      await opts.onQueued?.(job);
+      for (;;) {
+        if (controller.signal.aborted) return { code: signalExit(controller.signal.reason), signal: controller.signal.reason };
+        const result = await mutate(async state => {
+          await reclaim(state);
+          const resources = await sampler();
+          const admission = admissionDetails(state.running.length, config, resources);
+          const waiting = ordered(state.queued), position = waiting.findIndex(item => item.id === job.id) + 1;
+          const blockers = [], reasonCodes = [];
+          if (position > 1) {
+            blockers.push(`waiting behind ${position - 1} earlier queued ${position === 2 ? 'job' : 'jobs'}`);
+            reasonCodes.push('queue_order');
+          }
+          if (admission) { blockers.push(admission.reason); reasonCodes.push(admission.code); }
+          const ahead = [...state.running.map(item => ({ state: 'running', label: item.label, repository: item.repository })),
+            ...waiting.slice(0, Math.max(0, position - 1)).map(item => ({ state: 'queued', label: item.label, repository: item.repository }))];
+          if (blockers.length) return { reason: blockers.join('; '), blockers, reasonCodes, position, ahead,
+            queued: waiting.length, running: state.running.length, resources,
+            limits: { slots: config.slots, loadThreshold: config.loadThreshold, minFreeMemoryBytes: config.minFreeMemoryBytes } };
+          if (opts.maxWait > 0 && performance.now() - start >= opts.maxWait * 1000)
+            return { reason: 'max-wait elapsed before admission', blockers: ['max-wait elapsed before admission'],
+              reasonCodes: ['max_wait'], position, queued: waiting.length, running: state.running.length, resources,
+              limits: { slots: config.slots, loadThreshold: config.loadThreshold, minFreeMemoryBytes: config.minFreeMemoryBytes } };
+          state.queued = state.queued.filter(item => item.id !== job.id);
+          job.startedAt = Date.now(); state.running.push(job);
+          return null;
+        });
+        if (!result) break;
+        const elapsed = performance.now() - start;
+        if (opts.maxWait !== undefined && elapsed >= opts.maxWait * 1000)
+          return { code: TIMEOUT_EXIT, timedOut: true, reason: result.reason };
+        const signature = `${result.position}:${result.reasonCodes.join(',')}`;
+        const now = performance.now();
+        if (!lastWaitReportAt || signature !== lastWaitSignature || now - lastWaitReportAt >= WAIT_REPORT_INTERVAL_MS) {
+          await opts.onWaiting?.(result.reason, result);
+          lastWaitReportAt = now;
+          lastWaitSignature = signature;
+        }
+        const remaining = opts.maxWait === undefined ? config.pollMs : Math.min(config.pollMs, Math.max(1, opts.maxWait * 1000 - elapsed));
+        await delay(remaining, undefined, { signal: controller.signal }).catch(error => { if (error.name !== 'AbortError') throw error; });
+      }
+      return await execute(command, job, config, owner => mutate(state => {
+        const lease = state.running.find(item => item.id === job.id);
+        if (!lease) throw Error('Job reservation disappeared');
+        lease.owner = owner;
+      }), controller.signal, opts.onStarted, opts);
+    } catch (error) { keepLease = Boolean(error.keepLease); throw error; }
+    finally {
+      try {
+        await mutate(state => { state.queued = state.queued.filter(item => item.id !== job.id);
+          if (!keepLease) state.running = state.running.filter(item => item.id !== job.id); });
+      } finally {
+        active--;
+        process.off('SIGINT', interrupt); process.off('SIGTERM', terminate);
+        opts.signal?.removeEventListener('abort', external);
+      }
+    }
+  }
+  async function drain({ maxWait } = {}) {
+    if (maxWait !== undefined && (!Number.isFinite(maxWait) || maxWait < 0)) throw Error('Invalid maxWait');
+    const start = performance.now();
+    for (;;) {
+      const view = await status();
+      if (!view.running.length && !view.queued.length) return { code: 0, ...view };
+      if (maxWait !== undefined && performance.now() - start >= maxWait * 1000) return { code: TIMEOUT_EXIT, timedOut: true, ...view };
+      await delay(maxWait === undefined ? config.pollMs : Math.min(config.pollMs, Math.max(1, maxWait * 1000 - (performance.now() - start))));
+    }
+  }
+  return { config, run, status, drain, async close() {
+    if (active) throw Error('Cannot close a queue with active jobs');
+    closed = true; await lifetime.close();
+  } };
+}

package/modules/job-queue/membership.mjs

@@ -0,0 +1,30 @@
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { join } from 'node:path';
+import { readFile } from 'node:fs/promises';
+import { alive } from './identity.mjs';
+const execute = promisify(execFile);
+
+// Environment conveys a capability, never a boolean exemption. Validate both
+// the live lease and the calling process's actual ancestry before sharing it.
+export async function liveMembership(value = process.env.RER_JOB_MEMBERSHIP) {
+ if (!value) return null;
+ try {
+  const claim = JSON.parse(value);
+  if (typeof claim.directory !== 'string' || typeof claim.id !== 'string' || typeof claim.token !== 'string') return null;
+  const state = JSON.parse(await readFile(join(claim.directory, 'state.json'), 'utf8'));
+  const job = state.running.find(job => job.id === claim.id && job.membership === claim.token);
+  if (!job || !await alive(job.owner)) return null;
+  if(await descendantOf(job.owner.pid))return {id:job.id,label:job.label,repository:job.repository};
+ } catch { /* Invalid, stale or unrelated inheritance requires normal admission. */ }
+ return null;
+}
+
+export async function descendantOf(owner) {
+ const {stdout}=await execute('/bin/ps',['-A','-o','pid=','-o','ppid=']);
+ const parents=new Map(stdout.trim().split('\n').map(line=>line.trim().split(/\s+/).map(Number))),visited=new Set();
+ for(let pid=process.pid;pid>1&&!visited.has(pid);pid=parents.get(pid)){
+  if(pid===owner)return true;visited.add(pid);
+ }
+ return false;
+}

package/modules/job-queue/module.json

@@ -0,0 +1,10 @@
+{
+  "schema": 1,
+  "id": "job-queue",
+  "kind": "optional-source",
+  "entry": "index.mjs",
+  "cli": "cli.mjs",
+  "copy": ["config.mjs", "resources.mjs", "processes.mjs", "identity.mjs", "storage.mjs", "command.mjs", "supervisor.mjs", "index.mjs", "cli.mjs"],
+  "requires": ["node-22-or-newer", "macos-or-linux", "local-atomic-filesystem", "shared-owner-state-directory"],
+  "status": "Machine-wide development admission and process-group cleanup; rer verbs can adopt the queue independently"
+}

package/modules/job-queue/policy.mjs

@@ -0,0 +1,29 @@
+import { createServer,createConnection } from 'node:net';
+import { chmod,unlink } from 'node:fs/promises';
+import { randomUUID } from 'node:crypto';
+import { identity,alive } from './identity.mjs';
+import { descendantOf } from './membership.mjs';
+
+async function inherited(value) {
+ if(!value)return null;
+ try {
+  const claim=JSON.parse(value);
+  if(!await alive(claim.owner)||!await descendantOf(claim.owner.pid))return null;
+  return await new Promise((resolve,reject)=>{
+   const socket=createConnection(claim.path);let text='';
+   socket.setTimeout(3000,()=>socket.destroy(Error('Policy probe timed out')));
+   socket.on('data',data=>{text+=data;if(text.length>4096)socket.destroy();});
+   socket.on('error',reject);socket.on('end',()=>{try{const policy=JSON.parse(text);resolve(policy.token===claim.token?policy:null);}catch{resolve(null);}});
+  });
+ }catch{return null;}
+}
+export async function callerPolicy(env=process.env) {
+ const value=env.RER_JOB_POLICY, policy=await inherited(value);
+ if(policy)return {ci:policy.ci,environment:{RER_JOB_POLICY:value},close:async()=>{}};
+ const lifetime=await identity(),token=randomUUID(),path=`/tmp/rer-policy-${token}.sock`;
+ // An invalid inherited claim fails closed even when the child changes CI.
+ const ci=!value&&env.CI==='true';
+ const server=createServer(socket=>socket.end(JSON.stringify({token,ci})));
+ await new Promise((resolve,reject)=>{server.once('error',reject);server.listen(path,resolve);});await chmod(path,0o600);
+ return {ci,environment:{RER_JOB_POLICY:JSON.stringify({path,token,owner:lifetime.info})},async close(){await new Promise(resolve=>server.close(resolve));await unlink(path).catch(error=>{if(error.code!=='ENOENT')throw error;});await lifetime.close();}};
+}

package/modules/job-queue/processes.mjs

@@ -0,0 +1,80 @@
+import { readFile, readdir } from 'node:fs/promises';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { setTimeout as delay } from 'node:timers/promises';
+const execute = promisify(execFile);
+
+export async function processInfo(pid) {
+  if (!Number.isInteger(pid) || pid < 1) throw Error('Invalid process PID');
+  if (process.platform === 'linux') {
+    try {
+      const [stat, boot, command] = await Promise.all([readFile(`/proc/${pid}/stat`, 'utf8'),
+        readFile('/proc/sys/kernel/random/boot_id', 'utf8'), readFile(`/proc/${pid}/cmdline`, 'utf8')]);
+      const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+      return { pid, start: `${boot.trim()}:${fields[19]}`, zombie: fields[0] === 'Z',
+        title: command.replace(/\0/g, ' ').trim() };
+    } catch (error) { if (error.code === 'ENOENT' || error.code === 'ESRCH') return null; throw error; }
+  }
+  if (process.platform !== 'darwin') throw Error('Job queue supports macOS and Linux');
+  try {
+    const { stdout } = await execute('/bin/ps', ['-p', String(pid), '-o', 'lstart=', '-o', 'stat=', '-o', 'command='],
+      { env: { ...process.env, LC_ALL: 'C', TZ: 'UTC' } });
+    const match = stdout.trim().match(/^(.{24})\s+(\S+)\s+(.*)$/);
+    if (!match) throw Error(`Cannot inspect process ${pid}`);
+    return { pid, start: match[1], zombie: match[2].startsWith('Z'), title: match[3] };
+  } catch (error) { if (error.code === 1) return null; throw error; }
+}
+
+export function signalGroup(pgid, signal) {
+  try { process.kill(-pgid, signal); return true; }
+  catch (error) { if (error.code === 'ESRCH') return false; throw error; }
+}
+
+async function groupLive(pgid) {
+  if (process.platform === 'darwin') {
+    const { stdout } = await execute('/bin/ps', ['-A', '-o', 'pgid=', '-o', 'stat=']);
+    // A member already exiting (stat flag E) cannot be signalled and is ending,
+    // like a zombie; killpg answers EPERM for a group of only such members.
+    return stdout.trim().split('\n').some(line => {
+      const [group, state] = line.trim().split(/\s+/);
+      return Number(group) === pgid && !state.startsWith('Z') && !state.includes('E');
+    });
+  }
+  for (const name of (await readdir('/proc')).filter(name => /^\d+$/.test(name))) {
+    try {
+      const stat = await readFile(`/proc/${name}/stat`, 'utf8');
+      const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
+      if (Number(fields[2]) === pgid && fields[0] !== 'Z') return true;
+    } catch (error) { if (!['ENOENT', 'ESRCH'].includes(error.code)) throw error; }
+  }
+  return false;
+}
+
+export async function deliverGroupSignal(pgid, signal, { send = signalGroup, live = groupLive } = {}) {
+  try { return send(pgid, signal); }
+  catch (error) {
+    // Darwin excludes zombies in killpg and can return EPERM for a stopped group.
+    // Completion requires independent evidence; a live/unknown group stays held.
+    if (error.code === 'EPERM' && !await live(pgid)) return false;
+    throw error;
+  }
+}
+
+// A group ID cannot be reused while the old group has surviving members (POSIX).
+// A live replacement leader is never signalled; both platforms check its unique title.
+export async function stopGroup(owner, graceMs) {
+  if (!owner?.pgid) return;
+  if (!Number.isInteger(owner.pgid) || owner.pgid < 2 || owner.pgid !== owner.pid)
+    throw Error('Invalid lease process group');
+  const current = await processInfo(owner.pid);
+  if (current && !current.zombie && (current.start !== owner.start
+    || (owner.marker && current.title && !current.title.includes(owner.marker)))) return;
+  if (await deliverGroupSignal(owner.pgid, 'SIGTERM')) {
+    await delay(graceMs);
+    // Check again immediately before escalation, including the recycled PID case.
+    const after = await processInfo(owner.pid);
+    if (after && !after.zombie && (after.start !== owner.start
+      || (owner.marker && after.title && !after.title.includes(owner.marker)))) return;
+    await deliverGroupSignal(owner.pgid, 'SIGKILL');
+  }
+}

package/modules/job-queue/resources.mjs

@@ -0,0 +1,37 @@
+import { loadavg, freemem } from 'node:os';
+import { readFileSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+
+// macOS keeps little memory "free" on purpose: free pages are the last thing it
+// holds. Inactive, speculative and purgeable pages are reclaimed on demand, so
+// the memory a new job can actually use is their sum with the free pages.
+// (Purgeable pages can already sit in the inactive count; they are a small
+// share, so the overlap only makes the reading slightly generous.)
+export function parseVmStat(text) {
+  const size = Number(/page size of (\d+) bytes/.exec(text)?.[1]);
+  const pages = name => Number(new RegExp(`^Pages ${name}:\\s+(\\d+)\\.`, 'm').exec(text)?.[1]);
+  const total = ['free', 'inactive', 'speculative', 'purgeable'].map(pages);
+  if (!Number.isInteger(size) || total.some(value => !Number.isFinite(value))) return NaN;
+  return total.reduce((sum, value) => sum + value, 0) * size;
+}
+
+// Linux reports the same idea directly as MemAvailable, in kibibytes.
+export function parseMemAvailable(text) {
+  const kib = Number(/^MemAvailable:\s+(\d+) kB/m.exec(text)?.[1]);
+  return Number.isFinite(kib) ? kib * 1024 : NaN;
+}
+
+export function availableMemory() {
+  try {
+    if (process.platform === 'darwin') return parseVmStat(execFileSync('/usr/bin/vm_stat', { encoding: 'utf8', timeout: 3000 }));
+    if (process.platform === 'linux') return parseMemAvailable(readFileSync('/proc/meminfo', 'utf8'));
+    return freemem();
+  } catch { return NaN; }
+}
+
+// Built-in OS counters; injection belongs to library tests, never CLI admission.
+// freeMemoryBytes is the raw OS free counter, kept for display only: admission
+// uses availableMemoryBytes.
+export function sampleResources() {
+  return { load: loadavg()[0], freeMemoryBytes: freemem(), availableMemoryBytes: availableMemory() };
+}

package/modules/job-queue/storage.mjs

@@ -0,0 +1,64 @@
+import { mkdir, readFile, writeFile, rename, readdir, unlink, link } from 'node:fs/promises';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { setTimeout as delay } from 'node:timers/promises';
+import { alive } from './identity.mjs';
+
+export async function atomicJson(path, value) {
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  try {
+    await writeFile(temporary, JSON.stringify(value), { flag: 'wx', mode: 0o600 });
+    await rename(temporary, path);
+  } finally { await unlink(temporary).catch(error => { if (error.code !== 'ENOENT') throw error; }); }
+}
+
+export async function readJson(path, fallback) {
+  try { return JSON.parse(await readFile(path, 'utf8')); }
+  catch (error) { if (error.code === 'ENOENT') return fallback; throw error; }
+}
+
+// Lamport's bakery lock: publish choosing before reading tickets, then wait for
+// every earlier (number,id). Unique records avoid the stale-unlink/new-lock race
+// of a single reclaimable lock file. Every publication is atomic and complete.
+export async function transaction(directory, owner, action) {
+  const locks = join(directory, 'locks');
+  await mkdir(locks, { recursive: true, mode: 0o700 });
+  const id = randomUUID(), path = join(locks, `${id}.json`);
+  const candidate = `${path}.tmp`;
+  await writeFile(candidate, JSON.stringify({ id, owner, choosing: true, number: 0 }), { flag: 'wx', mode: 0o600 });
+  try { await link(candidate, path); } finally { await unlink(candidate); }
+  const proofs = new Map();
+  async function liveOwner(record) {
+    const key = `${record.owner.token}:${record.owner.pid}:${record.owner.start}`;
+    const previous = proofs.get(key);
+    // A briefly cached live proof only delays reclamation; it cannot admit an
+    // extra writer. Unique lifetime tokens cannot become live again after death.
+    if (previous && Date.now() - previous.at < 200) return previous.live;
+    const live = await alive(record.owner);
+    proofs.set(key, { live, at: Date.now() });
+    return live;
+  }
+  async function participants() {
+    const records = await Promise.all((await readdir(locks)).filter(name => name.endsWith('.json'))
+      .map(name => readJson(join(locks, name), null)));
+    const live = [];
+    for (const record of records) {
+      if (!record) continue;
+      if (record.id === id || await liveOwner(record)) live.push(record);
+      else await unlink(join(locks, `${record.id}.json`)).catch(error => { if (error.code !== 'ENOENT') throw error; });
+    }
+    return live;
+  }
+  try {
+    const number = Math.max(0, ...(await participants()).map(record => record.number)) + 1;
+    await atomicJson(path, { id, owner, choosing: false, number });
+    while ((await participants()).some(record => record.id !== id && (record.choosing
+      || record.number < number || (record.number === number && record.id < id)))) await delay(20);
+    const statePath = join(directory, 'state.json');
+    const state = await readJson(statePath, { version: 1, sequence: 0, queued: [], running: [] });
+    if (state.version !== 1 || !Array.isArray(state.queued) || !Array.isArray(state.running)) throw Error('Invalid job queue state');
+    const result = await action(state);
+    await atomicJson(statePath, state);
+    return result;
+  } finally { await unlink(path).catch(error => { if (error.code !== 'ENOENT') throw error; }); }
+}

package/modules/job-queue/supervisor.mjs

@@ -0,0 +1,30 @@
+import { spawn } from 'node:child_process';
+import { identity } from './identity.mjs';
+import { signalGroup } from './processes.mjs';
+
+const marker = `rer-job-supervisor-${process.argv[2]}`;
+process.title = marker;
+let stopping = false, started = false;
+function stop() {
+  if (stopping) return;
+  stopping = true;
+  // Remain group leader until escalation: this prevents group-ID reuse while
+  // cleanup is in progress and allows the caller to safely finish cancellation.
+  signalGroup(process.pid, 'SIGTERM');
+  setTimeout(() => signalGroup(process.pid, 'SIGKILL'), Math.max(20, Number(process.argv[3])));
+}
+process.on('SIGINT', stop);
+process.on('SIGTERM', stop);
+process.on('disconnect', stop);
+const lifetime = await identity();
+process.send({ ready: { ...lifetime.info, marker, pgid: process.pid } });
+process.on('message', message => {
+  if (started || stopping || !message.command) return;
+  started = true;
+  const child = spawn('/usr/bin/nice', ['-n', String(message.nice), ...message.command],
+    { cwd: message.cwd, env: { ...process.env, RER_JOB_MEMBERSHIP: message.membership }, stdio: 'inherit' });
+  child.once('error', error => process.send?.({ done: { code: 127, error: error.message } }));
+  child.once('exit', (code, signal) => {
+    if (process.connected) process.send({ done: { code, signal } });
+  });
+});

package/modules/job-queue/waiting.mjs

@@ -0,0 +1,16 @@
+import { GIB } from './config.mjs';
+
+const gib = bytes => `${(bytes / GIB).toFixed(1)} GiB`;
+
+// What a queued job tells the person: its place, who is ahead of it (with their
+// checkout paths), memory against the floor, and how to change that floor.
+export function waitingMessage(wait) {
+  const lines = [`Waiting to start (queue position ${wait.position}/${wait.queued}): ${wait.reason}.`];
+  if (wait.ahead?.length) {
+    lines.push(`Ahead of this job:`);
+    for (const job of wait.ahead) lines.push(`  ${job.state}: ${job.label} in ${job.repository}`);
+  }
+  if (Number.isFinite(wait.resources?.availableMemoryBytes))
+    lines.push(`Memory: ${gib(wait.resources.availableMemoryBytes)} available, floor ${gib(wait.limits.minFreeMemoryBytes)}. Change the floor with RER_JOBS_MIN_FREE_MEMORY_BYTES (bytes; 0 for no floor).`);
+  return lines.join('\n');
+}

package/modules/mail/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Run Edit Run
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.