@ornncompute/cli 0.2.23 → 1.0.25

package/src/evidence-commands.mjs

@@ -0,0 +1,1340 @@
+import { clusterFactLabel, reservationFactsForCluster } from "./kubernetes-commands.mjs";
+import { listOwnedActiveMachines, resolveMachineReference } from "./machine-reference.mjs";
+import { machineReachability } from "./vm-commands.mjs";
+
+const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+const DEFAULT_HISTORY_MS = 15 * 60 * 1000;
+const DEFAULT_LOG_LIMIT = 200;
+const DEFAULT_MAX_POINTS = 240;
+const DEFAULT_RETRY_WINDOW_MS = 5_000;
+const DEFAULT_FOLLOW_INTERVAL_MS = 5_000;
+const MAX_PAGE_SIZE = 500;
+const MAX_RETRIES = 3;
+const SAFE_METRICS = new Set([
+  "cpu_utilization",
+  "gpu_utilization",
+  "gpu_memory_utilization",
+  "gpu_power_w",
+  "gpu_tflops",
+  "memory_utilization",
+  "storage_utilization",
+]);
+const TERMINAL_MACHINE_STATES = new Set(["deleted", "ended", "released", "terminated"]);
+const UNSAFE_LOG_MESSAGE = [
+  /\b(?:operator(?:name|id|_name|_id)?|provider(?:name|id|_name|_id)?|site_operator|supplier(?:name|id|_name|_id)?)\s*[:=]/i,
+  /\b(?:authorization|api[_ -]?key|access[_ -]?token|refresh[_ -]?token|password|secret|private[_ -]?key)\s*[:=]/i,
+  /-----BEGIN [A-Z ]*PRIVATE KEY-----/i,
+  /\/(?:internal|v1\/(?:orchestrator|observability|cli\/operator))(?:\/|\b)/i,
+  /\b(?:staff_activity|staff activity|internal route|route)\s*[:=]/i,
+];
+const UNSAFE_KEYS = new Set([
+  "authorization",
+  "clientsecret",
+  "operator",
+  "operatorid",
+  "operatorname",
+  "password",
+  "paymentsecret",
+  "privatekey",
+  "provider",
+  "providerid",
+  "providername",
+  "refreshtoken",
+  "siteoperator",
+  "siteoperatorid",
+  "siteoperatorname",
+  "staffactivity",
+  "stripeclientsecret",
+  "supplier",
+  "supplierid",
+  "suppliername",
+  "token",
+]);
+const STABLE_SOURCE_CODES = new Set([
+  "invalid_logs_response",
+  "invalid_metrics_response",
+  "logs_rejected",
+  "logs_unavailable",
+  "metrics_rejected",
+  "metrics_unavailable",
+]);
+
+function record(value) {
+  return value && typeof value === "object" && !Array.isArray(value) ? value : {};
+}
+
+function text(value) {
+  return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+function normalizedKey(value) {
+  return String(value)
+    .replaceAll(/[^a-z0-9]/gi, "")
+    .toLowerCase();
+}
+
+function containsUnsafeFields(value) {
+  if (Array.isArray(value)) return value.some(containsUnsafeFields);
+  if (!value || typeof value !== "object") return false;
+  return Object.entries(value).some(([key, child]) => {
+    const normalized = normalizedKey(key);
+    return (
+      UNSAFE_KEYS.has(normalized) ||
+      normalized.includes("secret") ||
+      normalized.includes("token") ||
+      normalized.includes("password") ||
+      normalized.includes("privatekey") ||
+      normalized.includes("apikey") ||
+      normalized.includes("accesskey") ||
+      normalized.includes("credential") ||
+      containsUnsafeFields(child)
+    );
+  });
+}
+
+function machineId(value) {
+  const source = record(value);
+  return text(source.id) ?? text(source.machine_id) ?? text(source.instance_id);
+}
+
+function reservationId(value) {
+  const source = record(value);
+  return text(source.reservation_id) ?? text(source.reservationId);
+}
+
+function hostname(value) {
+  const source = record(value);
+  return text(source.hostname) ?? text(source.machine_hostname) ?? text(source.site_label);
+}
+
+function stateToken(value) {
+  return (
+    (text(value) ?? "unknown")
+      .toLowerCase()
+      .replaceAll(/[^a-z0-9]+/g, "_")
+      .replaceAll(/^_+|_+$/g, "") || "unknown"
+  );
+}
+
+function integerOption(interaction, value, name, fallback, maximum = MAX_PAGE_SIZE) {
+  if (value === undefined) return fallback;
+  if (!/^\d+$/.test(String(value))) {
+    interaction.usage(`--${name} must be an integer from 1 to ${maximum}.`, {
+      code: "invalid_input",
+      field: name.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()),
+    });
+  }
+  const parsed = Number(value);
+  if (parsed < 1 || parsed > maximum) {
+    interaction.usage(`--${name} must be an integer from 1 to ${maximum}.`, {
+      code: "invalid_input",
+      field: name.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()),
+    });
+  }
+  return parsed;
+}
+
+function durationMilliseconds(interaction, value, name) {
+  if (value === undefined) return null;
+  const match = String(value)
+    .trim()
+    .match(/^(\d+(?:\.\d+)?)(ms|s|m|h)$/i);
+  if (!match) {
+    interaction.usage(`--${name} must be a positive duration such as 500ms, 30s, 5m, or 1h.`, {
+      code: "invalid_input",
+      field: name,
+    });
+  }
+  const unit = match[2].toLowerCase();
+  const multiplier = unit === "ms" ? 1 : unit === "s" ? 1_000 : unit === "m" ? 60_000 : 3_600_000;
+  const milliseconds = Number(match[1]) * multiplier;
+  if (!Number.isFinite(milliseconds) || milliseconds < 1) {
+    interaction.usage(`--${name} must be a positive duration.`, {
+      code: "invalid_input",
+      field: name,
+    });
+  }
+  return Math.round(milliseconds);
+}
+
+function nowMilliseconds(now) {
+  const value = typeof now === "function" ? now() : Date.now();
+  if (value instanceof Date) return value.getTime();
+  return typeof value === "number" && Number.isFinite(value) ? value : Date.now();
+}
+
+function timestamp(interaction, value, field) {
+  const source = String(value);
+  const rfc3339 = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/i;
+  const milliseconds = rfc3339.test(source) ? Date.parse(source) : Number.NaN;
+  if (!Number.isFinite(milliseconds)) {
+    interaction.usage(`--${field} must be an RFC 3339 timestamp.`, {
+      code: "invalid_input",
+      field,
+    });
+  }
+  return milliseconds;
+}
+
+function evidenceWindow(input, interaction, now, { follow = false, logsOnly = false } = {}) {
+  if (input.before !== undefined && !logsOnly) {
+    interaction.usage("--before is available only for a logs command.", {
+      code: "invalid_input",
+      field: "before",
+    });
+  }
+  if (input.before !== undefined && (input.start !== undefined || input.end !== undefined)) {
+    interaction.usage("--before cannot be combined with --start or --end.", {
+      code: "invalid_input",
+      field: "before",
+    });
+  }
+  if ((input.start === undefined) !== (input.end === undefined)) {
+    interaction.usage("--start and --end must be supplied together.", {
+      code: "metric_window_incomplete",
+      field: input.start === undefined ? "start" : "end",
+    });
+  }
+  if (follow && input.end !== undefined) {
+    interaction.usage("--end cannot be combined with --follow.", {
+      code: "invalid_input",
+      field: "end",
+    });
+  }
+
+  const effectiveNow = nowMilliseconds(now);
+  const endMs =
+    input.before !== undefined
+      ? timestamp(interaction, input.before, "before")
+      : input.end !== undefined
+        ? timestamp(interaction, input.end, "end")
+        : effectiveNow;
+  const startMs =
+    input.start !== undefined
+      ? timestamp(interaction, input.start, "start")
+      : endMs - DEFAULT_HISTORY_MS;
+  if (startMs >= endMs) {
+    interaction.usage("The evidence window requires --start before --end.", {
+      code: "invalid_metric_window",
+      field: "start",
+    });
+  }
+  return {
+    start: new Date(startMs).toISOString(),
+    end: new Date(endMs).toISOString(),
+    startMs,
+    endMs,
+  };
+}
+
+function lifecycle(value, kind) {
+  const source = record(value);
+  const state = stateToken(source.state ?? source.status);
+  if (["deleted", "ended", "released", "terminated", "torn_down"].includes(state)) {
+    return { class: "ended", phase: kind === "cluster" ? "torn_down" : "ended" };
+  }
+  if (["failed", "error", "unavailable"].includes(state)) {
+    return { class: "failed", phase: "failed" };
+  }
+  if (["active", "online", "ready", "running", "provisioned"].includes(state)) {
+    return { class: "ready", phase: kind === "cluster" ? "access_ready" : "active" };
+  }
+  if (["creating", "pending", "provisioning", "starting", "switching"].includes(state)) {
+    return { class: "working", phase: "provisioning" };
+  }
+  return { class: "pending", phase: "unknown" };
+}
+
+function reachability(value) {
+  const source = record(value);
+  const explicit = source.reachable;
+  if (explicit === true) return { state: "reachable", observed_at: null };
+  if (explicit === false) return { state: "unreachable", observed_at: null };
+  const candidate = record(source.reachability);
+  const offGrid = record(source.off_grid_reachability);
+  const state = stateToken(candidate.state ?? candidate.status ?? offGrid.state ?? offGrid.status);
+  const phase = stateToken(candidate.phase ?? offGrid.phase);
+  const observedAt =
+    text(candidate.observed_at) ?? text(candidate.checked_at) ?? text(offGrid.last_checked_at);
+  if (["reachable", "live", "online"].includes(state) || phase === "ssh_ready")
+    return { state: "reachable", observed_at: observedAt };
+  if (
+    ["unreachable", "offline", "off_grid"].includes(state) ||
+    ["ssh_unreachable", "unreachable"].includes(phase)
+  )
+    return { state: "unreachable", observed_at: observedAt };
+  const inferred = machineReachability(source);
+  if (inferred.class === "ready") return { state: "reachable", observed_at: observedAt };
+  if (inferred.class === "failed") return { state: "unreachable", observed_at: observedAt };
+  return { state: "unknown", observed_at: observedAt };
+}
+
+function authoritativeHealth(value) {
+  const source = record(value);
+  const assessment = record(source.health_assessment ?? source.authoritative_health);
+  const explicitHealth = record(source.health);
+  const authoritative = Object.keys(assessment).length
+    ? assessment
+    : explicitHealth.authoritative === true
+      ? explicitHealth
+      : {};
+  const state = stateToken(authoritative.state ?? authoritative.status);
+  return {
+    state: ["healthy", "degraded", "unhealthy"].includes(state) ? state : "unknown",
+    assessed_at: text(authoritative.assessed_at) ?? text(authoritative.observed_at),
+  };
+}
+
+function projectMachine(value, fallbackReservationId = null) {
+  const source = record(record(value).machine ?? value);
+  const id = machineId(source);
+  const reservation = reservationId(source) ?? fallbackReservationId;
+  if (!id || !reservation) return null;
+  const mode = stateToken(source.access_mode ?? source.mode);
+  const accessMode =
+    mode === "bare_metal"
+      ? "bare-metal"
+      : ["virtual_machine", "vm"].includes(mode)
+        ? "vm"
+        : "unknown";
+  return {
+    id,
+    hostname: hostname(source),
+    reservation_id: reservation,
+    access_mode: accessMode,
+    lifecycle: lifecycle(source, "machine"),
+    reachability: reachability(source),
+    health: authoritativeHealth(source),
+    source,
+  };
+}
+
+function memberLifecycle(value) {
+  const state = stateToken(value);
+  if (["ready", "active", "running", "joined", "member"].includes(state)) {
+    return { class: "ready", phase: "member" };
+  }
+  if (["failed", "error", "cleanup_failed", "join_failed"].includes(state)) {
+    return { class: "failed", phase: "member_failed" };
+  }
+  if (["deleted", "removed", "detached"].includes(state)) {
+    return { class: "ended", phase: "removed" };
+  }
+  return { class: "working", phase: "membership_pending" };
+}
+
+function clusterLifecycle(source, machines) {
+  const state = stateToken(source.state ?? source.status);
+  const credentialState = stateToken(source.credential_state);
+  if (["deleted", "ended", "released", "terminated", "torn_down"].includes(state)) {
+    return { class: "ended", phase: "torn_down" };
+  }
+  if (state === "deleting") return { class: "working", phase: "tearing_down" };
+  if (["failed", "error", "unavailable"].includes(state)) {
+    return { class: machines.length ? "partial" : "failed", phase: "provisioning_failed" };
+  }
+  if (machines.some((machine) => machine.lifecycle.class === "failed")) {
+    return { class: "partial", phase: "member_failed" };
+  }
+  if (state === "active" && ["ready", "active", "available"].includes(credentialState)) {
+    return { class: "ready", phase: "access_ready" };
+  }
+  if (state === "active") return { class: "working", phase: "credentials_pending" };
+  if (["provisioning", "creating", "pending"].includes(state)) {
+    return { class: "working", phase: "provisioning" };
+  }
+  return { class: "pending", phase: "unknown" };
+}
+
+function projectCluster(value) {
+  const source = record(value);
+  const id = text(source.id) ?? text(source.cluster_id);
+  const reservation = reservationId(source);
+  if (!id || !reservation) return null;
+  const rawMachines = Array.isArray(source.machines)
+    ? source.machines
+    : Array.isArray(source.members)
+      ? source.members
+      : [];
+  const machines = rawMachines.flatMap((member) => {
+    const source = record(member);
+    const id = machineId(source) ?? text(source.node_id);
+    return id
+      ? [
+          {
+            id,
+            hostname: hostname(source),
+            lifecycle: memberLifecycle(source.state ?? source.membership_state),
+          },
+        ]
+      : [];
+  });
+  return {
+    id,
+    reservation_id: reservation,
+    access_mode: ["kubernetes", "slurm"].includes(stateToken(source.access_mode))
+      ? stateToken(source.access_mode)
+      : "unknown",
+    lifecycle: clusterLifecycle(source, machines),
+    gpu_type: text(source.gpu_type ?? source.hardware),
+    reachability: reachability(source),
+    health: authoritativeHealth(source),
+    machines,
+    source,
+  };
+}
+
+function payloadRows(payload, key) {
+  if (Array.isArray(payload)) return payload;
+  const rows = record(payload)[key];
+  return Array.isArray(rows) ? rows : null;
+}
+
+async function activeMachines(fetch, interaction) {
+  const rows = await listOwnedActiveMachines({ fetch }, interaction);
+  return rows.flatMap((row) => {
+    const projected = projectMachine(row);
+    return projected ? [projected] : [];
+  });
+}
+
+async function clusters(fetch, interaction) {
+  const payload = await fetch("/v1/workflows/compute/clusters");
+  const rows = payloadRows(payload, "clusters");
+  if (!rows)
+    interaction.error("The gateway returned an invalid cluster collection.", {
+      code: "invalid_gateway_response",
+    });
+  return rows
+    .flatMap((row) => {
+      const projected = projectCluster(row);
+      return projected ? [projected] : [];
+    })
+    .sort(
+      (left, right) =>
+        left.reservation_id.localeCompare(right.reservation_id) || left.id.localeCompare(right.id)
+    );
+}
+
+function resourceChoice(resource) {
+  if (resource.kind === "machine") {
+    return {
+      label: `VM/Bare Metal · ${resource.machine.hostname ?? "Unnamed machine"} · ${resource.machine.lifecycle.class}`,
+      value: `machine:${resource.machine.id}`,
+    };
+  }
+  return {
+    label: clusterFactLabel(resource.cluster, resource.reservation_facts),
+    value: `cluster:${resource.cluster.id}`,
+  };
+}
+
+async function resolveResource(command, input, fetch, interaction) {
+  const expectedKind = command.startsWith("machines.")
+    ? "machine"
+    : command.startsWith("clusters.")
+      ? "cluster"
+      : null;
+  const positionals = input.$positionals ?? [];
+  if (positionals.length > 1) interaction.usage("Supply exactly one machine or cluster reference.");
+  const reference = text(positionals[0]);
+
+  if (reference && expectedKind !== "cluster") {
+    const source = await resolveMachineReference(reference, { fetch }, interaction, {
+      allowMiss: expectedKind === null,
+    });
+    if (source) {
+      const machine = projectMachine(source);
+      if (!machine) {
+        interaction.error("The gateway returned an invalid machine.", {
+          code: "invalid_gateway_response",
+        });
+      }
+      return { kind: "machine", machine };
+    }
+  }
+  if (reference && UUID.test(reference) && expectedKind !== "machine") {
+    const matches = (await clusters(fetch, interaction)).filter(
+      (cluster) => cluster.id === reference
+    );
+    if (matches.length === 1) return { kind: "cluster", cluster: matches[0] };
+    if (matches.length > 1)
+      interaction.error("The gateway returned duplicate canonical cluster IDs.", {
+        code: "cluster_ambiguous",
+      });
+  }
+  if (reference && expectedKind === "cluster") {
+    interaction.usage(
+      "Clusters are referenced only by canonical ID or an invocation-local guided choice.",
+      {
+        code: "canonical_cluster_id_required",
+        field: "cluster",
+      }
+    );
+  }
+  if (reference) {
+    interaction.error(
+      `${expectedKind === "cluster" ? "Cluster" : "Machine or cluster"} not found: ${reference}.`,
+      {
+        code: expectedKind === "cluster" ? "cluster_not_found" : "resource_not_found",
+      }
+    );
+  }
+
+  if (expectedKind && input.guided !== true) {
+    interaction.usage(`A ${expectedKind} reference is required unless --guided is supplied.`, {
+      code: `${expectedKind}_reference_required`,
+      field: expectedKind,
+    });
+  }
+  if (!interaction.eligibleHuman || interaction.machine) {
+    interaction.usage("A machine or cluster reference is required outside an eligible terminal.", {
+      code: "resource_reference_required",
+      field: "resource",
+    });
+  }
+
+  const resources = [];
+  if (expectedKind !== "cluster") {
+    resources.push(
+      ...(await activeMachines(fetch, interaction)).map((machine) => ({ kind: "machine", machine }))
+    );
+  }
+  if (expectedKind !== "machine") {
+    resources.push(
+      ...(await clusters(fetch, interaction)).map((cluster) => ({ kind: "cluster", cluster }))
+    );
+  }
+  if (!resources.length)
+    interaction.error("No machine or cluster is available in this organization.", {
+      code: "resource_not_found",
+    });
+  if (resources.length === 1) return resources[0];
+  const enriched = await Promise.all(
+    resources.map(async (resource) => {
+      if (resource.kind === "machine") return resource;
+      return {
+        ...resource,
+        reservation_facts: await reservationFactsForCluster(fetch, interaction, resource.cluster),
+      };
+    })
+  );
+  const choices = enriched.map(resourceChoice);
+  const selected = await interaction.ask({ label: "Resource", choices });
+  const match = enriched.find((resource) => resourceChoice(resource).value === selected);
+  if (!match)
+    interaction.usage("Choose one exact machine or cluster.", { code: "resource_ambiguous" });
+  return match;
+}
+
+function resourceBase(resource) {
+  const source = resource.kind === "machine" ? resource.machine : resource.cluster;
+  return {
+    resource: {
+      kind: resource.kind,
+      id: source.id,
+      reservation_id: source.reservation_id,
+      ...(resource.kind === "machine"
+        ? { hostname: source.hostname, access_mode: source.access_mode }
+        : { access_mode: source.access_mode, membership_count: source.machines.length }),
+    },
+    lifecycle: source.lifecycle,
+    reachability: source.reachability,
+    health: source.health,
+    metrics: null,
+    logs: null,
+  };
+}
+
+function sourceMachines(resource) {
+  return resource.kind === "machine"
+    ? [{ id: resource.machine.id, hostname: resource.machine.hostname }]
+    : resource.cluster.machines;
+}
+
+function errorDetail(error) {
+  const body = record(error?.detail);
+  const candidate = body.detail ?? body.error ?? body;
+  if (typeof candidate === "string") return { code: candidate };
+  return record(candidate);
+}
+
+function sourceFailure(error, source, resourceRef) {
+  const status = error?.status;
+  const detail = errorDetail(error);
+  const suppliedCode = text(detail.code);
+  const explicitRetryable = typeof detail.retryable === "boolean" ? detail.retryable : null;
+  const authorization = status === 401 || status === 403;
+  const retryable = authorization
+    ? false
+    : (explicitRetryable ??
+      (status === undefined || status >= 500 || [408, 425, 429].includes(status)));
+  return {
+    code: authorization
+      ? status === 401
+        ? "authentication_required"
+        : "authorization_denied"
+      : suppliedCode && STABLE_SOURCE_CODES.has(suppliedCode)
+        ? suppliedCode
+        : status !== undefined && status < 500
+          ? `${source}_rejected`
+          : `${source}_unavailable`,
+    message: authorization
+      ? status === 401
+        ? "The Ornn session expired while reading evidence."
+        : "The current session cannot read this evidence."
+      : `${source === "metrics" ? "Metrics" : "Logs"} are unavailable for this resource.`,
+    retryable,
+    resource_ref: resourceRef,
+    authorization,
+  };
+}
+
+async function sleepUntil(delay, deps, signal) {
+  if (signal?.aborted) return false;
+  if (!signal) {
+    await deps.sleep(delay);
+    return true;
+  }
+  return new Promise((resolve, reject) => {
+    let settled = false;
+    const finish = (value) => {
+      if (settled) return;
+      settled = true;
+      signal.removeEventListener("abort", aborted);
+      resolve(value);
+    };
+    const aborted = () => finish(false);
+    signal.addEventListener("abort", aborted, { once: true });
+    Promise.resolve(deps.sleep(delay)).then(() => finish(true), reject);
+  });
+}
+
+async function fetchEvidence(request, source, resourceRef, deps, deadline, signal) {
+  let attempt = 0;
+  for (;;) {
+    attempt += 1;
+    const remaining = deadline - nowMilliseconds(deps.now);
+    if (Number.isFinite(deadline) && remaining <= 0) {
+      return {
+        value: null,
+        error: sourceFailure(new Error("deadline elapsed"), source, resourceRef),
+        interrupted: false,
+      };
+    }
+    const timeoutSignal = Number.isFinite(deadline)
+      ? AbortSignal.timeout(Math.max(1, Math.min(remaining, 2_147_483_647)))
+      : null;
+    const requestSignal =
+      signal && timeoutSignal
+        ? AbortSignal.any([signal, timeoutSignal])
+        : (signal ?? timeoutSignal);
+    try {
+      return { value: await request(requestSignal), error: null, interrupted: false };
+    } catch (error) {
+      if (signal?.aborted) return { value: null, error: null, interrupted: true };
+      const failure = sourceFailure(error, source, resourceRef);
+      if (
+        timeoutSignal?.aborted ||
+        failure.authorization ||
+        !failure.retryable ||
+        attempt >= MAX_RETRIES
+      ) {
+        return { value: null, error: failure, interrupted: false };
+      }
+      const delay = Math.min(250 * 2 ** (attempt - 1), 1_000);
+      if (
+        nowMilliseconds(deps.now) + delay >= deadline ||
+        !(await sleepUntil(delay, deps, signal))
+      ) {
+        if (signal?.aborted) return { value: null, error: null, interrupted: true };
+        return { value: null, error: failure, interrupted: false };
+      }
+    }
+  }
+}
+
+function safeMetricName(name) {
+  const suffix = name.match(/_(?:min|max|median|std)$/)?.[0] ?? "";
+  const base = suffix ? name.slice(0, -suffix.length) : name;
+  return SAFE_METRICS.has(base);
+}
+
+function safeMetricValue(value) {
+  if (typeof value === "boolean") return value;
+  if (typeof value === "number") return Number.isFinite(value) ? value : null;
+  if (typeof value !== "string" || !/^-?(?:\d+\.?\d*|\.\d+)(?:e[+-]?\d+)?$/i.test(value.trim())) {
+    return null;
+  }
+  const numeric = Number(value);
+  return Number.isFinite(numeric) ? numeric : null;
+}
+
+function projectMetricPoint(value) {
+  const source = record(value);
+  const observedAt = text(source.observed_at) ?? text(source.timestamp) ?? text(source.date);
+  const rawMetrics = record(source.metrics ?? source.values ?? source.resource_metrics);
+  const metrics = {};
+  for (const [name, raw] of Object.entries(rawMetrics)) {
+    const metric = safeMetricValue(raw);
+    if (safeMetricName(name) && metric !== null) metrics[name] = metric;
+  }
+  if (!observedAt || !Object.keys(metrics).length) return null;
+  return { observed_at: observedAt, metrics };
+}
+
+function withinEvidenceWindow(observedAt, window) {
+  const milliseconds = Date.parse(observedAt);
+  return (
+    Number.isFinite(milliseconds) && milliseconds >= window.startMs && milliseconds <= window.endMs
+  );
+}
+
+function metricPoints(payload, maximum, window) {
+  const source = record(payload);
+  let raw;
+  if (Array.isArray(source.points)) {
+    raw = source.points;
+  } else if (Array.isArray(source.series)) {
+    raw = source.series.flatMap((series) =>
+      Array.isArray(record(series).points) ? series.points : []
+    );
+  } else if (source.machine && typeof source.machine === "object") {
+    raw = [
+      {
+        observed_at:
+          source.machine.observed_at ?? source.machine.resource_metrics?.last_heartbeat_at,
+        resource_metrics: source.machine.resource_metrics,
+      },
+    ];
+  } else {
+    return null;
+  }
+  return raw
+    .flatMap((point) => {
+      const projected = projectMetricPoint(point);
+      return projected && withinEvidenceWindow(projected.observed_at, window) ? [projected] : [];
+    })
+    .sort((left, right) => left.observed_at.localeCompare(right.observed_at))
+    .slice(0, maximum);
+}
+
+function safeLogLine(value) {
+  if (containsUnsafeFields(value)) return null;
+  const source = record(value);
+  const observedAt = text(source.date) ?? text(source.observed_at) ?? text(source.timestamp);
+  const message = text(source.message) ?? text(source.text);
+  if (!observedAt || !message || UNSAFE_LOG_MESSAGE.some((pattern) => pattern.test(message)))
+    return null;
+  const severity = stateToken(source.severity ?? "info");
+  return {
+    observed_at: observedAt,
+    severity: ["error", "warning", "info"].includes(severity) ? severity : "info",
+    message,
+  };
+}
+
+function logLines(payload, maximum, window) {
+  const raw = payloadRows(payload, "lines");
+  if (!raw) return null;
+  return raw
+    .flatMap((line) => {
+      const projected = safeLogLine(line);
+      return projected && withinEvidenceWindow(projected.observed_at, window) ? [projected] : [];
+    })
+    .sort(
+      (left, right) =>
+        left.observed_at.localeCompare(right.observed_at) ||
+        left.message.localeCompare(right.message)
+    )
+    .slice(0, maximum);
+}
+
+function invalidEvidenceResponse(code) {
+  const error = new Error("invalid evidence response");
+  error.status = 502;
+  error.detail = { code, retryable: true };
+  return error;
+}
+
+function query(params) {
+  return new URLSearchParams(
+    Object.entries(params).filter(([, value]) => value !== null && value !== undefined)
+  ).toString();
+}
+
+async function metricSource(machine, reservation, window, maximum, deps, deadline, signal, live) {
+  const path = live ? "live-metrics" : "telemetry-history";
+  const params = live
+    ? { machine_id: machine.id }
+    : { machine_id: machine.id, start: window.start, end: window.end, max_points: String(maximum) };
+  const response = await fetchEvidence(
+    async (requestSignal) => {
+      const payload = await deps.fetch(
+        `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservation)}/${path}?${query(params)}`,
+        { signal: requestSignal }
+      );
+      const points = metricPoints(payload, maximum, window);
+      if (points === null) throw invalidEvidenceResponse("invalid_metrics_response");
+      return points;
+    },
+    "metrics",
+    machine.id,
+    deps,
+    deadline,
+    signal
+  );
+  if (response.interrupted) {
+    return {
+      machine_ref: machine.id,
+      state: "interrupted",
+      points: [],
+      error: null,
+      interrupted: true,
+    };
+  }
+  if (response.error) {
+    return { machine_ref: machine.id, state: "unavailable", points: [], error: response.error };
+  }
+  const points = response.value;
+  return {
+    machine_ref: machine.id,
+    state: points.length ? "available" : "no_data",
+    points,
+    error: null,
+  };
+}
+
+async function logSource(
+  machine,
+  reservation,
+  window,
+  stream,
+  limit,
+  cursor,
+  deps,
+  deadline,
+  signal
+) {
+  const response = await fetchEvidence(
+    async (requestSignal) => {
+      const payload = await deps.fetch(
+        `/v1/workflows/compute/nodes/reservations/${encodeURIComponent(reservation)}/logs?${query({
+          machine_id: machine.id,
+          stream,
+          start: window.start,
+          end: window.end,
+          limit: String(limit),
+          cursor,
+        })}`,
+        { signal: requestSignal }
+      );
+      const entries = logLines(payload, limit, window);
+      if (entries === null) throw invalidEvidenceResponse("invalid_logs_response");
+      return {
+        entries,
+        nextCursor: text(record(payload).next_cursor) ?? text(record(payload).nextCursor),
+      };
+    },
+    "logs",
+    machine.id,
+    deps,
+    deadline,
+    signal
+  );
+  if (response.interrupted) {
+    return {
+      machine_ref: machine.id,
+      state: "interrupted",
+      entries: [],
+      page: { limit, returned: 0, next_cursor: null },
+      error: null,
+      interrupted: true,
+    };
+  }
+  if (response.error) {
+    return {
+      machine_ref: machine.id,
+      state: "unavailable",
+      entries: [],
+      page: { limit, returned: 0, next_cursor: null },
+      error: response.error,
+    };
+  }
+  const { entries, nextCursor } = response.value;
+  return {
+    machine_ref: machine.id,
+    state: entries.length ? "available" : "no_data",
+    entries,
+    page: { limit, returned: entries.length, next_cursor: nextCursor },
+    error: null,
+  };
+}
+
+function aggregateState(sources, itemField) {
+  const unavailable = sources.filter((source) => source.state === "unavailable").length;
+  const useful = sources.some((source) => source[itemField].length > 0);
+  if (unavailable === sources.length && sources.length) return "unavailable";
+  if (unavailable) return "partial";
+  return useful ? "available" : "no_data";
+}
+
+function publicMetricSource(source) {
+  return {
+    machine_ref: source.machine_ref,
+    state: source.state,
+    points: source.points,
+    ...(source.error
+      ? {
+          error: {
+            code: source.error.code,
+            message: source.error.message,
+            retryable: source.error.retryable,
+          },
+        }
+      : {}),
+  };
+}
+
+function publicLogSource(source) {
+  return {
+    machine_ref: source.machine_ref,
+    state: source.state,
+    entries: source.entries,
+    page: source.page,
+    ...(source.error
+      ? {
+          error: {
+            code: source.error.code,
+            message: source.error.message,
+            retryable: source.error.retryable,
+          },
+        }
+      : {}),
+  };
+}
+
+async function collectEvidence(
+  resource,
+  requested,
+  window,
+  options,
+  deps,
+  signal,
+  { live = false } = {}
+) {
+  const machines = sourceMachines(resource);
+  const reservation =
+    resource.kind === "machine" ? resource.machine.reservation_id : resource.cluster.reservation_id;
+  const deadline = options.deadline;
+  const errors = [];
+  let metrics = null;
+  let logs = null;
+  let interrupted = false;
+
+  if (requested.metrics) {
+    const sources = [];
+    for (const machine of machines) {
+      const source = await metricSource(
+        machine,
+        reservation,
+        window,
+        options.maxPoints,
+        deps,
+        deadline,
+        signal,
+        live
+      );
+      sources.push(source);
+      if (source.interrupted) {
+        interrupted = true;
+        break;
+      }
+      if (source.error) errors.push(source.error);
+      if (source.error?.authorization) break;
+    }
+    metrics = {
+      state: aggregateState(sources, "points"),
+      window: { start: window.start, end: window.end, max_points: options.maxPoints },
+      series: sources.map(publicMetricSource),
+    };
+  }
+
+  if (requested.logs && !interrupted && !errors.some((error) => error.authorization)) {
+    const sources = [];
+    for (const machine of machines) {
+      const source = await logSource(
+        machine,
+        reservation,
+        window,
+        options.stream,
+        options.limit,
+        options.cursor,
+        deps,
+        deadline,
+        signal
+      );
+      sources.push(source);
+      if (source.interrupted) {
+        interrupted = true;
+        break;
+      }
+      if (source.error) errors.push(source.error);
+      if (source.error?.authorization) break;
+    }
+    logs = {
+      state: aggregateState(sources, "entries"),
+      stream: options.stream,
+      window: { start: window.start, end: window.end },
+      sources: sources.map(publicLogSource),
+    };
+  }
+
+  return { errors, interrupted, logs, metrics };
+}
+
+function publicErrors(errors) {
+  return errors.map(({ authorization: _authorization, ...error }) => error);
+}
+
+function snapshotExit(errors, requestedCount) {
+  if (!errors.length) return { exitCode: 0, outcome: null };
+  if (errors.some((error) => error.authorization)) {
+    return {
+      exitCode: 5,
+      outcome: { class: "failed", phase: errors.find((error) => error.authorization).code },
+    };
+  }
+  if (errors.length < requestedCount) {
+    return { exitCode: 4, outcome: { class: "partial", phase: "evidence_partial" } };
+  }
+  if (errors.every((error) => error.retryable)) {
+    return { exitCode: 6, outcome: { class: "failed", phase: "evidence_unavailable" } };
+  }
+  return { exitCode: 1, outcome: { class: "failed", phase: "evidence_failed" } };
+}
+
+function evidenceSourceLabel(data, index) {
+  if (data.resource.kind === "machine") return data.resource.hostname ?? "Machine";
+  return `Cluster member ${index + 1}`;
+}
+
+function metricSummary(metrics) {
+  return Object.entries(metrics)
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([name, value]) => `${name.replaceAll("_", " ")}=${value}`)
+    .join(", ");
+}
+
+function evidenceLines(data) {
+  const lines = [
+    data.resource.kind === "machine"
+      ? `Machine: ${data.resource.hostname ?? "selected machine"}`
+      : `${data.resource.access_mode} cluster`,
+    `Lifecycle: ${data.lifecycle.class} (${data.lifecycle.phase})`,
+    `Reachability: ${data.reachability.state}`,
+    `Health: ${data.health.state}`,
+  ];
+  if (data.metrics) {
+    lines.push(`Metrics: ${data.metrics.state}`);
+    for (const [index, source] of data.metrics.series.entries()) {
+      const point = source.points.at(-1);
+      if (point) {
+        lines.push(
+          `  ${evidenceSourceLabel(data, index)} · ${point.observed_at} · ${metricSummary(point.metrics)}`
+        );
+      }
+    }
+  }
+  if (data.logs) {
+    lines.push(`Logs (${data.logs.stream}): ${data.logs.state}`);
+    for (const [index, source] of data.logs.sources.entries()) {
+      for (const entry of source.entries) {
+        lines.push(
+          `  ${evidenceSourceLabel(data, index)} · ${entry.observed_at} · ${entry.severity} · ${entry.message}`
+        );
+      }
+    }
+  }
+  return lines;
+}
+
+function pageFor(resource, requested, logs) {
+  return resource.kind === "machine" && requested.logs && !requested.metrics
+    ? (logs?.sources?.[0]?.page ?? null)
+    : null;
+}
+
+async function snapshot(resource, requested, window, options, deps, interaction) {
+  const collected = await collectEvidence(resource, requested, window, options, deps, null);
+  const data = { ...resourceBase(resource), metrics: collected.metrics, logs: collected.logs };
+  const requestedCount =
+    sourceMachines(resource).length * (Number(requested.metrics) + Number(requested.logs));
+  const status = snapshotExit(collected.errors, requestedCount);
+  return interaction.result({
+    data,
+    errors: publicErrors(collected.errors),
+    exitCode: status.exitCode,
+    outcome: status.outcome ?? data.lifecycle,
+    page: pageFor(resource, requested, data.logs),
+    presentation: { lines: evidenceLines(data) },
+  });
+}
+
+function streamState(data, state) {
+  return {
+    ...data,
+    metrics: data.metrics ? { ...data.metrics, state } : null,
+    logs: data.logs ? { ...data.logs, state } : null,
+  };
+}
+
+function liveState(data) {
+  const metricState = data.metrics?.state;
+  const logState = data.logs?.state;
+  return {
+    ...data,
+    metrics: data.metrics
+      ? {
+          ...data.metrics,
+          state: ["partial", "unavailable"].includes(metricState)
+            ? metricState
+            : data.metrics.series.some((source) => source.points.length)
+              ? "streaming"
+              : "waiting_for_data",
+        }
+      : null,
+    logs: data.logs
+      ? {
+          ...data.logs,
+          state: ["partial", "unavailable"].includes(logState)
+            ? logState
+            : data.logs.sources.some((source) => source.entries.length)
+              ? "streaming"
+              : "waiting_for_data",
+        }
+      : null,
+  };
+}
+
+function writeHumanProgress(stdout, data) {
+  if (data.metrics) {
+    for (const [index, source] of data.metrics.series.entries()) {
+      const point = source.points.at(-1);
+      if (point) {
+        stdout.write(
+          `Metric ${evidenceSourceLabel(data, index)} ${point.observed_at} ${metricSummary(point.metrics)}\n`
+        );
+      }
+    }
+  }
+  if (data.logs) {
+    for (const [index, source] of data.logs.sources.entries()) {
+      for (const line of source.entries) {
+        stdout.write(
+          `${evidenceSourceLabel(data, index)} ${line.observed_at} ${line.severity} ${line.message}\n`
+        );
+      }
+    }
+  }
+}
+
+function streamResult(interaction, data, progress, exitCode, outcome, errors = []) {
+  return interaction.result({
+    data,
+    errors: publicErrors(errors),
+    exitCode,
+    outcome,
+    progress,
+    presentation: { lines: evidenceLines(data) },
+  });
+}
+
+async function follow(resource, requested, initialWindow, options, deps, interaction) {
+  if (interaction.format === "json") {
+    interaction.usage("JSON cannot represent a follow stream. Use --format ndjson.", {
+      code: "json_follow_unsupported",
+      field: "format",
+    });
+  }
+  const count = integerOption(interaction, interaction.options.count, "count", null, MAX_PAGE_SIZE);
+  const timeoutMs = durationMilliseconds(interaction, interaction.options.timeout, "timeout");
+  const intervalMs =
+    durationMilliseconds(interaction, interaction.options.interval, "interval") ??
+    DEFAULT_FOLLOW_INTERVAL_MS;
+  if (interaction.machine && count === null && timeoutMs === null) {
+    interaction.usage("Machine follow requires --count or --timeout so the stream is bounded.", {
+      code: "follow_bound_required",
+      field: "count",
+    });
+  }
+
+  const localController = deps.signal ? null : new AbortController();
+  const signal = deps.signal ?? localController.signal;
+  const onSigint = localController ? () => localController.abort() : null;
+  if (onSigint) process.once("SIGINT", onSigint);
+  const startedAt = nowMilliseconds(deps.now);
+  const deadline = timeoutMs === null ? Number.POSITIVE_INFINITY : startedAt + timeoutMs;
+  let current = resourceBase(resource);
+  const progress = [];
+  let iteration = 0;
+  try {
+    for (;;) {
+      if (signal.aborted) {
+        current = streamState(current, "interrupted");
+        return streamResult(interaction, current, progress, 130, {
+          class: "ended",
+          phase: "interrupted",
+        });
+      }
+      const observedAt = nowMilliseconds(deps.now);
+      if (observedAt >= deadline) {
+        current = streamState(current, "ended");
+        return streamResult(interaction, current, progress, 124, {
+          class: "ended",
+          phase: "observation_timeout",
+        });
+      }
+      iteration += 1;
+      const window = {
+        start: initialWindow.start,
+        end: new Date(observedAt).toISOString(),
+        startMs: initialWindow.startMs,
+        endMs: observedAt,
+      };
+      const collected = await collectEvidence(
+        resource,
+        requested,
+        window,
+        { ...options, deadline },
+        deps,
+        signal,
+        { live: true }
+      );
+      if (collected.interrupted || signal.aborted) {
+        current = streamState(
+          { ...resourceBase(resource), metrics: collected.metrics, logs: collected.logs },
+          "interrupted"
+        );
+        return streamResult(interaction, current, progress, 130, {
+          class: "ended",
+          phase: "interrupted",
+        });
+      }
+      current = liveState({
+        ...resourceBase(resource),
+        metrics: collected.metrics,
+        logs: collected.logs,
+      });
+      if (collected.errors.length) {
+        const requestedCount =
+          sourceMachines(resource).length * (Number(requested.metrics) + Number(requested.logs));
+        const status = snapshotExit(collected.errors, requestedCount);
+        return streamResult(
+          interaction,
+          current,
+          progress,
+          status.exitCode,
+          status.outcome,
+          collected.errors
+        );
+      }
+      const progressEntry = {
+        data: current,
+        message: `Observed ${requested.metrics && requested.logs ? "metrics and logs" : requested.metrics ? "metrics" : "logs"}.`,
+        observedAt: new Date(observedAt).toISOString(),
+        outcome: { class: "working", phase: "streaming" },
+        sequence: iteration,
+      };
+      if (interaction.machine) progress.push(progressEntry);
+      else writeHumanProgress(deps.stdout, current);
+
+      if (count !== null && iteration >= count) {
+        current = streamState(current, "ended");
+        return streamResult(interaction, current, progress, 0, {
+          class: "ended",
+          phase: "observation_complete",
+        });
+      }
+      const remaining = deadline - nowMilliseconds(deps.now);
+      if (remaining <= 0) continue;
+      const continued = await sleepUntil(Math.min(intervalMs, remaining), deps, signal);
+      if (!continued) {
+        current = streamState(current, "interrupted");
+        return streamResult(interaction, current, progress, 130, {
+          class: "ended",
+          phase: "interrupted",
+        });
+      }
+    }
+  } finally {
+    if (onSigint) process.removeListener("SIGINT", onSigint);
+  }
+}
+
+export async function executeEvidenceCommand(command, input, deps) {
+  const interaction = deps.interaction;
+  if (input.follow === true && interaction.format === "json") {
+    interaction.usage("JSON cannot represent a follow stream. Use --format ndjson.", {
+      code: "json_follow_unsupported",
+      field: "format",
+    });
+  }
+  const requested = {
+    metrics: command === "check" || command.endsWith(".metrics"),
+    logs: command === "check" || command.endsWith(".logs"),
+  };
+  const logsOnly = requested.logs && !requested.metrics;
+  const window = evidenceWindow(input, interaction, deps.now, {
+    follow: input.follow === true,
+    logsOnly,
+  });
+  const maxPoints = integerOption(interaction, input.maxPoints, "max-points", DEFAULT_MAX_POINTS);
+  const limit = integerOption(interaction, input.limit, "limit", DEFAULT_LOG_LIMIT);
+  const stream = input.stream ?? "kernel";
+  if (
+    !requested.logs &&
+    (input.stream !== undefined ||
+      input.limit !== undefined ||
+      input.cursor !== undefined ||
+      input.before !== undefined)
+  ) {
+    interaction.usage("Log options are not accepted by a metrics-only command.", {
+      code: "invalid_input",
+      field: "stream",
+    });
+  }
+  if (!requested.metrics && input.maxPoints !== undefined) {
+    interaction.usage("--max-points is not accepted by a logs-only command.", {
+      code: "invalid_input",
+      field: "maxPoints",
+    });
+  }
+  if (
+    input.follow !== true &&
+    (input.count !== undefined || input.timeout !== undefined || input.interval !== undefined)
+  ) {
+    interaction.usage("--count, --timeout, and --interval require --follow.", {
+      code: "invalid_input",
+      field:
+        input.count !== undefined ? "count" : input.timeout !== undefined ? "timeout" : "interval",
+    });
+  }
+  if (input.cursor !== undefined && input.follow === true) {
+    interaction.usage("--cursor cannot be combined with --follow.", {
+      code: "invalid_input",
+      field: "cursor",
+    });
+  }
+
+  const resource = await resolveResource(command, input, deps.fetch, interaction);
+  const options = {
+    cursor: text(input.cursor),
+    deadline: nowMilliseconds(deps.now) + DEFAULT_RETRY_WINDOW_MS,
+    limit,
+    maxPoints,
+    stream,
+  };
+  if (input.follow === true) return follow(resource, requested, window, options, deps, interaction);
+  return snapshot(resource, requested, window, options, deps, interaction);
+}