@assemblyline/al-control 0.0.0-stage → 0.3.0

package/src/transport.js

@@ -0,0 +1,184 @@
+/**
+ * Transport — runs probe queries and pm2 commands on a host.
+ *
+ * - `local`: probe runs in-process; pm2 via execFileSync.
+ * - remote: the system `ssh` binary (so ~/.ssh/config aliases, ProxyJump,
+ *   keys and the agent all apply). Probe queries pipe probe.js on stdin to
+ *   `node -`, so the host needs only node and pm2 — no al-control install,
+ *   no config file, and client and probe are always the same version.
+ *
+ * The remote command string is interpreted by the host's shell, so every
+ * interpolated value goes through shellQuote().
+ */
+
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import path from 'node:path';
+
+import { runQuery } from './probe.js';
+
+const PROBE_SOURCE = readFileSync(new URL('./probe.js', import.meta.url), 'utf8');
+
+/**
+ * POSIX single-quote a string for the remote shell (works in sh, bash, zsh).
+ * @param {string} s
+ * @returns {string}
+ */
+export function shellQuote(s) {
+  return `'${String(s).replace(/'/g, `'\\''`)}'`;
+}
+
+/**
+ * ssh argv up to and including the destination.
+ *
+ * BatchMode: fail instead of hanging on a password prompt. ControlMaster +
+ * ControlPersist: reuse one connection across calls so watch loops don't pay
+ * a handshake per tick. `--` stops option parsing before the alias.
+ *
+ * @param {import('./config.js').HostContext} host
+ * @param {{tty?: boolean}} [opts]
+ * @returns {string[]}
+ */
+export function sshArgs(host, { tty = false } = {}) {
+  // Sockets live under the home dir even when AL_CONTROL_CONFIG_DIR points
+  // elsewhere: unix socket paths cap at 104 bytes, and %C expands to 40 hex
+  // chars. If even that is too long, go without multiplexing.
+  const socket_dir = path.join(homedir(), '.assemblyline', 'al-control');
+  const control_path = path.join(socket_dir, 'ssh-%C');
+  const multiplex = control_path.length - 2 + 40 < 104;
+  if (multiplex) mkdirSync(socket_dir, { recursive: true });
+  return [
+    '-o', 'BatchMode=yes',
+    ...(multiplex ? ['-o', 'ControlMaster=auto', '-o', `ControlPath=${control_path}`, '-o', 'ControlPersist=10m'] : []),
+    ...(tty ? ['-t'] : []),
+    '--', host.ssh,
+  ];
+}
+
+/**
+ * Non-interactive SSH shells skip .bashrc/.zshrc, so nvm-installed node/pm2
+ * are often off PATH. A configured `node:` path fixes both: its directory is
+ * prepended to PATH, and pm2 (a node script) lives alongside it.
+ * @param {import('./config.js').HostContext} host
+ */
+function remoteEnvPrefix(host) {
+  return host.node ? `PATH=${shellQuote(path.posix.dirname(host.node))}:"$PATH" ` : '';
+}
+
+function remoteNode(host) {
+  return host.node ? shellQuote(host.node) : 'node';
+}
+
+/**
+ * Run a probe query on the host.
+ * @param {import('./config.js').HostContext} host
+ * @param {import('./probe.js').ProbeQuery} query
+ * @returns {any} The query's result
+ */
+export function probe(host, query) {
+  if (host.is_local) return runQuery(query);
+
+  // The query rides in the piped script, not in argv: a long dir list would
+  // exceed Linux's 128 KB per-argument limit.
+  const remote_cmd = `${remoteEnvPrefix(host)}${remoteNode(host)} --input-type=module -`;
+  // stderr goes straight to the terminal: ssh (or Tailscale SSH's check mode)
+  // may print an auth prompt / login URL there and wait for the user. Captured,
+  // it would look like a hang. It also means ssh's own errors show verbatim.
+  const res = spawnSync('ssh', [...sshArgs(host), remote_cmd], {
+    input: `${PROBE_SOURCE}\nmain(${JSON.stringify(query)});\n`,
+    encoding: 'utf8',
+    maxBuffer: 64 * 1024 * 1024,
+    stdio: ['pipe', 'pipe', 'inherit'],
+  });
+
+  // EPIPE = ssh exited before reading the probe from stdin; its message is above.
+  if (res.error && res.error.code !== 'EPIPE') throw new Error(`Could not run ssh: ${res.error.message}`);
+  if (res.status === 255) throw new Error(`ssh to ${host.name} (${host.ssh}) failed — see ssh's message above.`);
+  if (res.status === 127) {
+    throw new Error(`node not found on ${host.name}. Set 'node:' for this host in the config to its absolute path (e.g. the output of \`which node\` in an interactive shell there).`);
+  }
+
+  let envelope;
+  try {
+    envelope = JSON.parse(res.stdout);
+  } catch {
+    throw new Error(`Unexpected output from ${host.name} (exit ${res.status}):\n${res.stdout.slice(0, 2000)}`);
+  }
+  if (!envelope.ok) {
+    const err = new Error(`${host.name}: ${envelope.error.message}`);
+    err.code = envelope.error.code;
+    throw err;
+  }
+  return envelope.data;
+}
+
+/**
+ * How to make pm2 reachable on a host.
+ * @param {import('./config.js').HostContext} host
+ */
+export function pm2Hint(host) {
+  return host.is_local
+    ? 'Install it with `npm i -g pm2`.'
+    : `Set 'node:' for host ${host.name} in the config to the node that pm2 was installed with (\`which node\` in an interactive shell there), so its bin dir is on PATH.`;
+}
+
+/**
+ * Probe a host's processes (`procs` or `factories` query). When pm2 couldn't be
+ * queried, pm2-managed factories look like `direct` ones — warn instead of
+ * letting that pass silently.
+ * @param {import('./config.js').HostContext} host
+ * @param {import('./probe.js').ProbeQuery} query
+ */
+export function probeProcesses(host, query) {
+  const result = probe(host, query);
+  if (!result.pm2_found) console.error(`Warning: pm2 not found on ${host.name}; pm2-managed factories will show as "direct". ${pm2Hint(host)}`);
+  return result;
+}
+
+/**
+ * Run pm2 on the host with inherited stdio.
+ *
+ * @param {import('./config.js').HostContext} host
+ * @param {string[]} args - pm2 arguments
+ * @param {object} [opts]
+ * @param {string} [opts.cwd] - absolute dir on the host to run from
+ * @param {boolean} [opts.interactive] - long-running / TUI (logs, monit):
+ *   Ctrl-C is a normal exit, and remote runs get a tty so it reaches pm2
+ */
+export function pm2(host, args, { cwd, interactive = false } = {}) {
+  if (host.is_local) {
+    try {
+      execFileSync('pm2', args, { cwd, stdio: 'inherit' });
+    } catch (err) {
+      if (err.code === 'ENOENT') throw new Error(`pm2 not found. ${pm2Hint(host)}`);
+      if (!interactive) throw new Error(`pm2 ${args[0]} failed (exit ${err.status})`);
+    }
+    return;
+  }
+
+  const cd = cwd ? `cd ${shellQuote(cwd)} && ` : '';
+  const remote_cmd = `${cd}${remoteEnvPrefix(host)}pm2 ${args.map(shellQuote).join(' ')}`;
+  const tty = interactive && Boolean(process.stdin.isTTY);
+  const res = spawnSync('ssh', [...sshArgs(host, { tty }), remote_cmd], { stdio: 'inherit' });
+
+  if (res.error) throw new Error(`Could not run ssh: ${res.error.message}`);
+  if (res.status === 255) throw new Error(`ssh to ${host.name} (${host.ssh}) failed`);
+  if (res.status === 127) {
+    throw new Error(`pm2 not found on ${host.name}. ${pm2Hint(host)}`);
+  }
+  if (res.status !== 0 && !interactive) throw new Error(`pm2 ${args[0]} on ${host.name} failed (exit ${res.status})`);
+}
+
+/**
+ * Interactive login shell on the host, started in `cwd`. Login (`-l`) so the
+ * user's PATH and nvm are set up as in a normal ssh session.
+ * @param {import('./config.js').HostContext} host
+ * @param {string} cwd - absolute dir on the host
+ */
+export function shell(host, cwd) {
+  const remote_cmd = `cd ${shellQuote(cwd)} && exec "\${SHELL:-/bin/sh}" -l`;
+  const res = spawnSync('ssh', [...sshArgs(host, { tty: true }), remote_cmd], { stdio: 'inherit' });
+  if (res.error) throw new Error(`Could not run ssh: ${res.error.message}`);
+  if (res.status === 255) throw new Error(`ssh to ${host.name} (${host.ssh}) failed`);
+}

package/src/utils/conveyors.js

@@ -0,0 +1,50 @@
+// @ts-check
+
+/**
+ * Conveyors: `lib-factory:move_files` steps, which move workpieces from one station's bin to
+ * another's. The step takes either one move (`source_bin`, `target_bin`) or a `moves` array of
+ * them, so every reader goes through here.
+ */
+
+export const CONVEYOR_SLUG = 'lib-factory:move_files';
+
+/**
+ * @typedef {object} ConveyorMove
+ * @property {string} source_bin - `STATION/BIN`, possibly with a sub-bin (`VM3/output/approved`)
+ * @property {string} target_bin
+ * @property {string} [mode]
+ */
+
+/**
+ * Every move a station's conveyor steps make.
+ *
+ * @param {{ steps?: any[] }} station - A station definition
+ * @returns {ConveyorMove[]}
+ */
+export function conveyorMoves(station) {
+  const moves = [];
+  for (const step of station.steps || []) {
+    if (step.slug !== CONVEYOR_SLUG || !step.config) continue;
+    for (const move of step.config.moves ?? [step.config]) {
+      if (move?.source_bin) moves.push(move);
+    }
+  }
+  return moves;
+}
+
+/**
+ * `STATION/BIN` → the bin a conveyor moves it to, for every bin some conveyor drains. A sub-bin
+ * source counts as its station bin. Clearing must skip these bins: another line still reads them.
+ *
+ * @param {{ steps?: any[] }[]} stations - Station definitions
+ * @returns {Map<string, string>}
+ */
+export function drainedBins(stations) {
+  const drained = new Map();
+  for (const station of stations) {
+    for (const move of conveyorMoves(station)) {
+      drained.set(move.source_bin.split('/').slice(0, 2).join('/'), move.target_bin || '?');
+    }
+  }
+  return drained;
+}

package/src/utils/factory-data.js

@@ -0,0 +1,123 @@
+/**
+ * Factory data — what command handlers read, wherever the factory lives.
+ * Each call is one probe round trip (in-process locally, one ssh exec remotely).
+ */
+
+import path from 'node:path';
+
+import { probe } from '../transport.js';
+import { buildLineState } from './line-state.js';
+
+/**
+ * Station + line definitions from the factory's `.orchestrator/`.
+ * @param {import('../config.js').FactoryContext} factory
+ * @returns {{root: string, stations: import('./line-state.js').StationDef[], lines: Object<string, object>}}
+ */
+export function loadDefs(factory) {
+  return probe(factory.host, { type: 'defs', path: factory.path });
+}
+
+/**
+ * Every line's station order, terminal, live bin counts and summary — the
+ * probe's `lines` query. Counts only, so it is cheap to poll over SSH.
+ * @param {import('../config.js').FactoryContext} factory
+ * @returns {{root: string, factory: object|null, lines: Object<string, {code: string, name: string|null, location: string|null, stations: string[], terminal: string, counts: Object<string, Object<string, number|null>>, orphans: Object<string, number>, summary: import('../probe.js').LineSummary}>}}
+ */
+export function loadLineSummaries(factory) {
+  return probe(factory.host, { type: 'lines', path: factory.path });
+}
+
+/**
+ * Live per-line state, plus the station defs it was built from and the
+ * orphaned failed folders (no pointer.json) per station.
+ * @param {import('../config.js').FactoryContext} factory
+ * @returns {{root: string, lines: Object<string, import('./line-state.js').LineState>, stations: import('./line-state.js').StationDef[], orphans: Object<string, Object<string, string>>}}
+ */
+export function loadLines(factory) {
+  const state = probe(factory.host, { type: 'state', path: factory.path });
+  return { root: state.root, lines: buildLineState(state), stations: state.stations, orphans: state.orphans ?? {} };
+}
+
+/**
+ * Parsed log.jsonl per workpiece dir (relative to temp/stations/).
+ * @param {import('../config.js').FactoryContext} factory
+ * @param {string[]} rel_dirs
+ * @returns {Object<string, Array<{ts: string, station: string, event: string}>>}
+ */
+export function loadLogs(factory, rel_dirs) {
+  if (rel_dirs.length === 0) return {};
+  return probe(factory.host, { type: 'logs', path: factory.path, dirs: rel_dirs }).logs;
+}
+
+/**
+ * Live per-line state for each factory. Host-wide (several factories), a factory
+ * whose directory is missing is reported on stderr and skipped; asked for by
+ * name, it's an error.
+ * @param {import('../config.js').FactoryContext[]} factories
+ * @returns {Array<{factory: import('../config.js').FactoryContext, root: string, lines: Object<string, import('./line-state.js').LineState>, orphans: Object<string, Object<string, string>>}>}
+ */
+export function loadLinesEach(factories) {
+  const out = [];
+  for (const factory of factories) {
+    try {
+      out.push({ factory, ...loadLines(factory) });
+    } catch (err) {
+      if (factories.length === 1 || err.code !== 'FACTORY_NOT_FOUND') throw err;
+      console.error(`Skipping ${factory.name}: ${err.message}`);
+    }
+  }
+  return out;
+}
+
+/**
+ * error.json summaries (`failed_at`, `name`, `code`, `message`) per workpiece
+ * dir; `null` where there's none.
+ * @param {import('../config.js').FactoryContext} factory
+ * @param {string[]} rel_dirs
+ */
+export function loadErrors(factory, rel_dirs) {
+  if (rel_dirs.length === 0) return {};
+  return probe(factory.host, { type: 'errors', path: factory.path, dirs: rel_dirs }).errors;
+}
+
+/**
+ * Last activity (ms since epoch) per workpiece dir; `null` if it's gone.
+ * @param {import('../config.js').FactoryContext} factory
+ * @param {string[]} rel_dirs
+ * @returns {Object<string, number|null>}
+ */
+export function loadAges(factory, rel_dirs) {
+  if (rel_dirs.length === 0) return {};
+  return probe(factory.host, { type: 'ages', path: factory.path, dirs: rel_dirs }).ages;
+}
+
+/**
+ * A workpiece folder's files (recursive, with sizes) and its full error.json.
+ * @param {import('../config.js').FactoryContext} factory
+ * @param {string} rel_dir
+ * @returns {{files: Array<{name: string, size: number}>, error: object|null}}
+ */
+export function loadWorkpieceFiles(factory, rel_dir) {
+  return probe(factory.host, { type: 'workpiece', path: factory.path, dir: rel_dir });
+}
+
+/**
+ * Absolute path of a workpiece folder on its host.
+ * @param {string} root - absolute factory root on the host (from the probe)
+ * @param {string} rel_dir - workpiece dir relative to temp/stations/
+ */
+export function workpiecePath(root, rel_dir) {
+  return path.posix.join(root, 'temp', 'stations', rel_dir);
+}
+
+/**
+ * Where a workpiece folder lives: a Cmd-clickable file:// URL locally,
+ * `host:/abs/path` (scp-style) on a remote host.
+ * @param {import('../config.js').FactoryContext} factory
+ * @param {string} root - absolute factory root on the host (from the probe)
+ * @param {string} rel_dir - workpiece dir relative to temp/stations/
+ */
+export function workpieceLocation(factory, root, rel_dir) {
+  const abs = workpiecePath(root, rel_dir);
+  return factory.host.is_local ? `file://${encodeURI(abs)}` : `${factory.host.ssh}:${abs}`;
+}

package/src/utils/failures.js

@@ -0,0 +1,26 @@
+/**
+ * Grouping key for a failed workpiece's error, so failures with the same
+ * cause count together even when their messages differ by an id or path.
+ *
+ * - A leading snake_case code (`gst_state_mismatch: …`) is the key.
+ * - Otherwise the message's first line with paths, id-like tokens and long
+ *   numbers masked, capped at KEY_MAX.
+ */
+
+const KEY_MAX = 100;
+
+/**
+ * @param {{name?: string|null, message?: string|null}|null} err - error.json summary
+ * @returns {string}
+ */
+export function errorKey(err) {
+  if (!err) return '(no error.json)';
+  const first = String(err.message || err.name || '').split('\n')[0].trim();
+  const code = first.match(/^([a-z][a-z0-9]*(?:_[a-z0-9]+)+)\b/);
+  if (code) return code[1];
+  const masked = first
+    .replace(/(?<![\w:/])(?:\/[^\s'"`/<>]+)+\/?/g, '<path>')
+    .replace(/\b(?=[\w-]*\d)(?=[\w-]*[A-Za-z])[\w-]{8,}\b/g, '<id>')
+    .replace(/\b\d{4,}\b/g, '<n>');
+  return masked.length > KEY_MAX ? `${masked.slice(0, KEY_MAX - 1)}…` : masked;
+}

package/src/utils/format.js

@@ -0,0 +1,160 @@
+/**
+ * Shared formatting helpers for CLI output.
+ * Structured text readable by both humans and LLMs.
+ */
+
+/**
+ * Format a header line with right-aligned status.
+ * "Work Record: abc123                    Completed"
+ */
+export function formatHeader(label, id, status) {
+  const left = `${label}: ${id}`;
+  if (!status) return left;
+  const minGap = 4;
+  const width = Math.max(left.length + minGap + status.length, 60);
+  return left + ' '.repeat(width - left.length - status.length) + status;
+}
+
+/**
+ * Format a label: value field with aligned label.
+ * "Name:      Monthly Debt & Equity Monitoring"
+ */
+export function formatField(label, value, labelWidth = 0) {
+  const padded = (label + ':').padEnd(labelWidth || label.length + 1);
+  return `${padded}  ${value ?? '—'}`;
+}
+
+/**
+ * Format a table with dynamic column widths.
+ * @param {string[]} headers - Column header names
+ * @param {string[][]} rows - Array of row arrays (strings)
+ * @returns {string} Formatted table string
+ */
+export function formatTable(headers, rows) {
+  if (rows.length === 0) {
+    return headers.join('  ') + '\n(none)';
+  }
+
+  // Calculate column widths
+  const widths = headers.map((h, i) =>
+    Math.max(h.length, ...rows.map(r => (r[i] || '').length))
+  );
+
+  const headerLine = headers.map((h, i) => h.padEnd(widths[i])).join('  ');
+  const separator = '-'.repeat(headerLine.length);
+  const dataLines = rows.map(row =>
+    row.map((cell, i) => (cell || '—').padEnd(widths[i])).join('  ')
+  );
+
+  return [headerLine, separator, ...dataLines].join('\n');
+}
+
+/**
+ * Format a section divider.
+ * "--- Title ---"
+ */
+export function formatSection(title) {
+  return `\n--- ${title} ---\n`;
+}
+
+/** The zone timestamps render in. Resolved once, so a long listing cannot straddle a DST change mid-table. */
+const LOCAL_TZ = Intl.DateTimeFormat().resolvedOptions().timeZone;
+
+/**
+ * Short zone label, e.g. "PDT" or "Asia/Calcutta". Computed once — it labels a
+ * header, not a value.
+ *
+ * `timeZoneName: 'short'` only yields a real abbreviation for zones ICU has one
+ * for; elsewhere it returns "GMT+5:30", which is correct but noisy in a column
+ * header and tells you less than the zone's own name. So: take the abbreviation
+ * when it is one, otherwise use the IANA name.
+ */
+const LOCAL_TZ_LABEL = (() => {
+  const parts = new Intl.DateTimeFormat('en-US', { timeZoneName: 'short' }).formatToParts(new Date());
+  const short = parts.find((p) => p.type === 'timeZoneName')?.value;
+  if (short && !/^(GMT|UTC)[+-]/.test(short)) return short;
+  return LOCAL_TZ || short || 'local';
+})();
+
+/**
+ * Short name of the local zone, e.g. "PDT" or "Asia/Calcutta".
+ *
+ * Use it to name the zone ONCE per table or detail block — in the column header
+ * (`STARTED (${localTzLabel()})`) or a section heading — never per row.
+ */
+export function localTzLabel() {
+  return LOCAL_TZ_LABEL;
+}
+
+/**
+ * Format an ISO timestamp in the operator's local timezone: "2026-03-15 15:53:01".
+ *
+ * **Local, not UTC.** This used to print `toISOString()` verbatim, so an
+ * operator in IST read every timestamp 5.5 hours early with nothing on screen
+ * saying so — and everything a factory CLI is compared against (`pm2 logs`, file
+ * mtimes, `al-orc` output) is local. Kept identical to al-orc's helper so the
+ * two CLIs never disagree about the same instant.
+ *
+ * Pair with `localTzLabel()` in the surrounding header so the zone is stated
+ * once. sv-SE is used because its locale format is already `YYYY-MM-DD HH:MM:SS`.
+ */
+export function formatDate(iso) {
+  if (!iso) return '—';
+  const d = new Date(iso);
+  if (Number.isNaN(d.getTime())) return '—';
+  return d.toLocaleString('sv-SE', { timeZone: LOCAL_TZ }).replace(',', '');
+}
+
+/**
+ * Wall-clock time only, for append-style feeds where the date is implied by the
+ * session: "15:53:01".
+ *
+ * Accepts an ISO string or a `Date` (`al-control workpieces watch` stamps
+ * `new Date()` as each event arrives).
+ *
+ * @param {string|Date|null|undefined} iso
+ */
+export function formatTime(iso) {
+  if (!iso) return '—';
+  const d = new Date(iso);
+  if (Number.isNaN(d.getTime())) return '—';
+  return d.toLocaleTimeString('sv-SE', { timeZone: LOCAL_TZ });
+}
+
+/**
+ * Format duration between two ISO timestamps.
+ * "1m" / "2h 15m" / "3d 4h"
+ */
+export function formatDuration(start, end) {
+  if (!start || !end) return '—';
+  const ms = new Date(end) - new Date(start);
+  if (ms < 0) return '—';
+
+  const seconds = Math.floor(ms / 1000);
+  if (seconds < 60) return `${seconds}s`;
+
+  const minutes = Math.floor(seconds / 60);
+  if (minutes < 60) return `${minutes}m`;
+
+  const hours = Math.floor(minutes / 60);
+  const remainingMinutes = minutes % 60;
+  if (hours < 24) {
+    return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`;
+  }
+
+  const days = Math.floor(hours / 24);
+  const remainingHours = hours % 24;
+  return remainingHours > 0 ? `${days}d ${remainingHours}h` : `${days}d`;
+}
+
+/**
+ * Format an elapsed duration in ms: "850ms" / "4.2s" / "3m 12s" / "1h 5m".
+ * @param {number} ms
+ */
+export function formatElapsed(ms) {
+  const secs = ms / 1000;
+  if (secs < 1) return `${ms}ms`;
+  if (secs < 60) return `${secs.toFixed(secs < 10 ? 1 : 0)}s`;
+  if (secs < 3600) return `${Math.floor(secs / 60)}m ${Math.round(secs % 60)}s`;
+  return `${Math.floor(secs / 3600)}h ${Math.floor((secs % 3600) / 60)}m`;
+}