@ornncompute/cli 0.2.23 → 1.0.25

package/src/admin-mutations.mjs

@@ -0,0 +1,187 @@
+import { createInterface } from "node:readline/promises";
+
+export const ADMIN_MUTATION_SAFETY = Object.freeze({
+  eligibleHumanOnly: "eligible-human-only",
+  reconcile: "reconcile",
+});
+
+export class AdminMutationError extends Error {
+  constructor(message, { code = "admin_mutation_invalid", field } = {}) {
+    super(message);
+    this.name = "AdminMutationError";
+    this.code = code;
+    this.field = field;
+  }
+}
+
+function normalizedReviewValue(value) {
+  const rendered =
+    value === null || value === undefined
+      ? "—"
+      : typeof value === "object"
+        ? JSON.stringify(value)
+        : String(value);
+  return rendered.replaceAll(/[\r\n\t]+/g, " ").trim();
+}
+
+export function writeAdminMutationReview(stdout, review) {
+  if (!review?.title || !Array.isArray(review.fields) || review.fields.length === 0) {
+    throw new AdminMutationError("Admin mutation review is incomplete.", {
+      code: "admin_review_incomplete",
+    });
+  }
+  stdout.write(`Review: ${normalizedReviewValue(review.title)}\n`);
+  for (const field of review.fields) {
+    if (!field?.label) {
+      throw new AdminMutationError("Admin mutation review is incomplete.", {
+        code: "admin_review_incomplete",
+      });
+    }
+    stdout.write(
+      `  ${normalizedReviewValue(field.label)}: ${normalizedReviewValue(field.value)}\n`
+    );
+  }
+  if (review.consequence) {
+    stdout.write(`  Consequence: ${normalizedReviewValue(review.consequence)}\n`);
+  }
+}
+
+async function defaultNoConsent(stdin, stdout) {
+  const readline = createInterface({ input: stdin, output: stdout, terminal: false });
+  try {
+    const answer = await readline.question("Proceed? [y/N] ");
+    return /^(y|yes)$/i.test(answer.trim());
+  } catch (error) {
+    if (
+      error?.code === "ERR_USE_AFTER_CLOSE" ||
+      error?.code === "ABORT_ERR" ||
+      error?.name === "AbortError"
+    ) {
+      return false;
+    }
+    throw error;
+  } finally {
+    readline.close();
+  }
+}
+
+export function assertAdminMutationContext({
+  eligibleHuman,
+  idempotencyKey,
+  machine,
+  safety,
+  yes,
+}) {
+  if (safety === ADMIN_MUTATION_SAFETY.eligibleHumanOnly) {
+    if (!eligibleHuman || machine) {
+      throw new AdminMutationError(
+        "This admin mutation is supported only in an eligible human terminal and cannot run with redirected or machine output.",
+        { code: "unsupported_in_machine_mode" }
+      );
+    }
+    return;
+  }
+  if (safety !== ADMIN_MUTATION_SAFETY.reconcile) {
+    throw new AdminMutationError("Admin mutation safety policy is missing.", {
+      code: "admin_mutation_policy_missing",
+    });
+  }
+  if (!eligibleHuman && yes !== true) {
+    throw new AdminMutationError("Noninteractive admin mutations require --yes.", {
+      code: "consent_required",
+      field: "yes",
+    });
+  }
+  if (!eligibleHuman && !idempotencyKey?.trim()) {
+    throw new AdminMutationError(
+      "Noninteractive admin mutations require --idempotency-key with a caller-stable value.",
+      { code: "idempotency_key_required", field: "idempotency-key" }
+    );
+  }
+}
+
+/**
+ * Shared privileged-mutation gate. All noninteractive prerequisites are checked
+ * before prepare/reconcile/execute can perform I/O. Domain handlers own semantic
+ * validation and exact goal comparison; this function owns context safety,
+ * normalized review, default-no consent, and the apply/satisfied/conflict flow.
+ */
+export async function runAdminMutation({
+  eligibleHuman,
+  execute,
+  idempotencyKey,
+  machine,
+  prepare,
+  reconcile,
+  safety,
+  stdin,
+  stdout,
+  yes = false,
+}) {
+  assertAdminMutationContext({ eligibleHuman, idempotencyKey, machine, safety, yes });
+  if (typeof prepare !== "function" || typeof execute !== "function") {
+    throw new AdminMutationError("Admin mutation handler is incomplete.", {
+      code: "admin_mutation_handler_incomplete",
+    });
+  }
+  if (safety === ADMIN_MUTATION_SAFETY.reconcile && typeof reconcile !== "function") {
+    throw new AdminMutationError("Reconciled admin mutation is missing an exact goal read.", {
+      code: "admin_reconciliation_missing",
+    });
+  }
+
+  let prepared = await prepare();
+  if (!prepared || typeof prepared !== "object") {
+    throw new AdminMutationError("Admin mutation preparation returned no review.", {
+      code: "admin_review_incomplete",
+    });
+  }
+
+  if (safety === ADMIN_MUTATION_SAFETY.reconcile) {
+    const reconciliation = await reconcile(prepared.value);
+    if (reconciliation?.status === "satisfied") {
+      return {
+        data: reconciliation.data,
+        exitCode: 0,
+        outcome: { class: "ready", phase: "reconciled" },
+        reconciled: true,
+        status: "complete",
+      };
+    }
+    if (reconciliation?.status === "conflict") {
+      throw new AdminMutationError(
+        reconciliation.message ?? "Current owner state conflicts with the requested admin goal.",
+        { code: reconciliation.code ?? "admin_goal_conflict" }
+      );
+    }
+    if (reconciliation?.status !== "apply") {
+      throw new AdminMutationError("Admin mutation reconciliation returned an invalid state.", {
+        code: "admin_reconciliation_invalid",
+      });
+    }
+    prepared = {
+      ...prepared,
+      ...(reconciliation.review ? { review: reconciliation.review } : {}),
+      ...(reconciliation.value !== undefined ? { value: reconciliation.value } : {}),
+    };
+  }
+
+  if (!machine) writeAdminMutationReview(stdout, prepared.review);
+  if (eligibleHuman && yes !== true && !(await defaultNoConsent(stdin, stdout))) {
+    return {
+      data: { submitted: false },
+      exitCode: 1,
+      outcome: { class: "ended", phase: "declined" },
+      reconciled: false,
+      status: "declined",
+    };
+  }
+
+  return {
+    data: await execute(prepared.value),
+    exitCode: 0,
+    outcome: { class: "ready", phase: "complete" },
+    reconciled: false,
+    status: "complete",
+  };
+}

package/src/admin-node-diagnostics.mjs

@@ -0,0 +1,412 @@
+import { createHash } from "node:crypto";
+
+import { ADMIN_MUTATION_SAFETY, AdminMutationError, runAdminMutation } from "./admin-mutations.mjs";
+
+const DIAGNOSE_USAGE = "Usage: ornn admin nodes diagnose <node-id>";
+const EXEC_USAGE = "Usage: ornn admin nodes exec <node-id> --command COMMAND --timeout DURATION";
+const OUTPUT_TRUNCATION_MARKER = "\n[ornn] output truncated";
+
+function usageError(message, field) {
+  return new AdminMutationError(message, {
+    code: "usage_error",
+    ...(field ? { field } : {}),
+  });
+}
+
+function optionValue(argv, index, name, usage) {
+  const argument = argv[index];
+  const separator = argument.indexOf("=");
+  if (separator !== -1) return { consumed: 0, value: argument.slice(separator + 1) };
+  if (index + 1 >= argv.length) throw usageError(`${usage}\n--${name} requires a value.`, name);
+  return { consumed: 1, value: argv[index + 1] };
+}
+
+function parseOptions(argv, allowed, usage) {
+  const options = {};
+  const positionals = [];
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (!argument.startsWith("--")) {
+      positionals.push(argument);
+      continue;
+    }
+    const separator = argument.indexOf("=");
+    const name = argument.slice(2, separator === -1 ? undefined : separator);
+    if (!allowed.has(name)) throw usageError(`${usage}\nUnknown option: --${name}`, name);
+    if (options[name] !== undefined) {
+      throw usageError(`${usage}\n--${name} may be supplied only once.`, name);
+    }
+    if (name === "yes") {
+      if (separator !== -1) throw usageError(`${usage}\n--yes does not take a value.`, "yes");
+      options.yes = true;
+      continue;
+    }
+    const parsed = optionValue(argv, index, name, usage);
+    options[name] = parsed.value;
+    index += parsed.consumed;
+  }
+  return { options, positionals };
+}
+
+function timeoutSeconds(value) {
+  if (typeof value !== "string")
+    throw usageError(`${EXEC_USAGE}\n--timeout is required.`, "timeout");
+  const match = value.match(/^([1-9]\d*)(s|m)?$/);
+  if (!match) {
+    throw usageError("--timeout must be whole seconds or minutes from 1s through 15m.", "timeout");
+  }
+  const amount = Number(match[1]);
+  const seconds = amount * (match[2] === "m" ? 60 : 1);
+  if (!Number.isSafeInteger(seconds) || seconds < 1 || seconds > 900) {
+    throw usageError("--timeout must be whole seconds or minutes from 1s through 15m.", "timeout");
+  }
+  return seconds;
+}
+
+export function parseAdminNodeDiagnoseArgs(argv) {
+  const { options, positionals } = parseOptions(
+    argv,
+    new Set(["yes", "idempotency-key"]),
+    DIAGNOSE_USAGE
+  );
+  if (positionals.length !== 1 || !positionals[0]) throw usageError(DIAGNOSE_USAGE, "node-id");
+  return {
+    input: { node_id: positionals[0] },
+    mutation: {
+      idempotencyKey: options["idempotency-key"],
+      safety: ADMIN_MUTATION_SAFETY.eligibleHumanOnly,
+      yes: options.yes === true,
+    },
+  };
+}
+
+export function parseAdminNodeExecArgs(argv) {
+  const { options, positionals } = parseOptions(
+    argv,
+    new Set(["command", "timeout", "yes", "idempotency-key"]),
+    EXEC_USAGE
+  );
+  if (positionals.length !== 1 || !positionals[0]) throw usageError(EXEC_USAGE, "node-id");
+  if (typeof options.command !== "string" || options.command.length === 0) {
+    throw usageError(`${EXEC_USAGE}\n--command is required and must not be empty.`, "command");
+  }
+  return {
+    input: {
+      node_id: positionals[0],
+      command: options.command,
+      timeout_seconds: timeoutSeconds(options.timeout),
+    },
+    mutation: {
+      idempotencyKey: options["idempotency-key"],
+      safety: ADMIN_MUTATION_SAFETY.eligibleHumanOnly,
+      yes: options.yes === true,
+    },
+  };
+}
+
+function presentString(value) {
+  return typeof value === "string" && value.length ? value : null;
+}
+
+function nodeContext(payload, expectedNodeId) {
+  if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
+    throw new AdminMutationError("The node lookup returned an invalid response.", {
+      code: "node_context_invalid",
+    });
+  }
+  const actualNodeId = presentString(payload.id) ?? presentString(payload.node_id);
+  if (actualNodeId !== expectedNodeId) {
+    throw new AdminMutationError("The node lookup did not return the exact requested node.", {
+      code: "exact_node_mismatch",
+      field: "node-id",
+    });
+  }
+  const explicitReachability =
+    presentString(payload.reachability?.status) ??
+    presentString(payload.off_grid_reachability?.status) ??
+    (typeof payload.reachable === "boolean"
+      ? payload.reachable
+        ? "reachable"
+        : "unreachable"
+      : null);
+  return {
+    node_id: actualNodeId,
+    hostname: presentString(payload.hostname),
+    lifecycle:
+      presentString(payload.lifecycle?.phase) ??
+      presentString(payload.lifecycle) ??
+      presentString(payload.status) ??
+      "unknown",
+    reachability: explicitReachability ?? "unknown",
+    observed_at:
+      presentString(payload.observed_at) ??
+      presentString(payload.last_heartbeat_at) ??
+      presentString(payload.off_grid_reachability?.last_checked_at),
+  };
+}
+
+function reviewFields(target) {
+  return [
+    { label: "Node", value: target.node_id },
+    { label: "Hostname", value: target.hostname ?? "unknown" },
+    { label: "Lifecycle", value: target.lifecycle },
+    { label: "Reachability", value: target.reachability },
+  ];
+}
+
+async function prepareTarget(input, runtime, fetch) {
+  const payload = await fetch(`/internal/nodes/${encodeURIComponent(input.node_id)}`, {
+    method: "GET",
+    ...(runtime.signal ? { signal: runtime.signal } : {}),
+  });
+  return nodeContext(payload, input.node_id);
+}
+
+function requestCorrelation(input) {
+  return {
+    command_sha256: createHash("sha256").update(input.command, "utf8").digest("hex"),
+    command_utf8_bytes: Buffer.byteLength(input.command, "utf8"),
+    timeout_seconds: input.timeout_seconds,
+  };
+}
+
+function diagnosisResult(raw, target) {
+  const jobId = presentString(raw?.job_id) ?? presentString(raw?.id);
+  return {
+    data: {
+      node: target,
+      diagnosis: {
+        job_id: jobId,
+        status: presentString(raw?.status) ?? "queued",
+      },
+    },
+    exitCode: 0,
+    outcome: { class: "working", phase: "diagnosis_queued" },
+    action: null,
+    resume: null,
+    errors: [],
+  };
+}
+
+function outputValue(value) {
+  return typeof value === "string" ? value : "";
+}
+
+function executionResult(raw, input, target) {
+  const status = presentString(raw?.status) ?? "failed";
+  const stdout = outputValue(raw?.stdout);
+  const stderr = outputValue(raw?.stderr);
+  const timedOut = raw?.timed_out === true || status === "timed_out";
+  const outputTruncated =
+    stdout.endsWith(OUTPUT_TRUNCATION_MARKER) || stderr.endsWith(OUTPUT_TRUNCATION_MARKER);
+  const partialOutput = timedOut && (stdout.length > 0 || stderr.length > 0);
+  const data = {
+    node: target,
+    request: requestCorrelation(input),
+    result: {
+      status: timedOut ? "timed_out" : status,
+      exit_code: Number.isInteger(raw?.exit_code) ? raw.exit_code : null,
+      signal: presentString(raw?.signal),
+      stdout,
+      stderr,
+      partial_output: partialOutput,
+      output_truncated: outputTruncated,
+    },
+  };
+
+  if (timedOut) {
+    return {
+      data,
+      exitCode: 124,
+      outcome: { class: partialOutput ? "partial" : "failed", phase: "command_timeout" },
+      action: null,
+      resume: null,
+      errors: [
+        {
+          code: "remote_command_timeout",
+          message: "The remote command reached its explicit timeout.",
+          retryable: false,
+        },
+      ],
+    };
+  }
+  if (status === "partial") {
+    return {
+      data,
+      exitCode: 4,
+      outcome: { class: "partial", phase: "partial_output" },
+      action: null,
+      resume: null,
+      errors: [
+        {
+          code: "remote_output_partial",
+          message: "The remote command returned only partial output.",
+          retryable: false,
+        },
+      ],
+    };
+  }
+  if (status !== "succeeded" || raw?.exit_code !== 0) {
+    return {
+      data,
+      exitCode: 1,
+      outcome: { class: "failed", phase: "command_exit" },
+      action: null,
+      resume: null,
+      errors: [
+        {
+          code: "remote_command_failed",
+          message: "The remote command exited without success.",
+          retryable: false,
+        },
+      ],
+    };
+  }
+  return {
+    data,
+    exitCode: 0,
+    outcome: { class: "ready", phase: "command_complete" },
+    action: null,
+    resume: null,
+    errors: [],
+  };
+}
+
+function declinedResult(input) {
+  return {
+    data: {
+      declined: true,
+      node: { node_id: input.node_id },
+      ...(input.command ? { request: requestCorrelation(input) } : {}),
+    },
+    exitCode: 1,
+    outcome: { class: "failed", phase: "consent_declined" },
+    action: null,
+    resume: null,
+    errors: [
+      {
+        code: "consent_declined",
+        message: "The action was declined; no mutation was submitted.",
+        retryable: false,
+      },
+    ],
+  };
+}
+
+function interruptedResult(input) {
+  return {
+    data: {
+      node: { node_id: input.node_id },
+      ...(input.command ? { request: requestCorrelation(input) } : {}),
+      result: { status: "interrupted", remote_outcome: "unknown" },
+    },
+    exitCode: 130,
+    outcome: { class: "partial", phase: "interrupted" },
+    action: null,
+    resume: null,
+    errors: [
+      {
+        code: "local_interruption",
+        message: "Local execution was interrupted; the remote outcome is unknown.",
+        retryable: false,
+      },
+    ],
+  };
+}
+
+function wasInterrupted(error, runtime) {
+  return (
+    runtime.signal?.aborted === true || error?.name === "AbortError" || error?.code === "ABORT_ERR"
+  );
+}
+
+async function runDiagnosis({ input, mutation, runtime, execute, fetch }) {
+  let target;
+  try {
+    const result = await runAdminMutation({
+      ...runtime,
+      ...mutation,
+      stdout: runtime.stderr,
+      prepare: async () => {
+        target = await prepareTarget(input, runtime, fetch);
+        return {
+          review: {
+            title: "Queue node diagnosis",
+            fields: [
+              ...reviewFields(target),
+              { label: "Workflow", value: "CLI diagnosis with zero burn time" },
+            ],
+            consequence: "A staff-authorized diagnosis job will be queued for this exact node.",
+          },
+          value: target,
+        };
+      },
+      execute: async () => execute({ node_id: input.node_id }, { confirmed: true }),
+    });
+    if (result.status === "declined") return declinedResult(input);
+    return diagnosisResult(result.data, target);
+  } catch (error) {
+    if (wasInterrupted(error, runtime)) return interruptedResult(input);
+    throw error;
+  }
+}
+
+async function runExecution({ input, mutation, runtime, execute, fetch }) {
+  let target;
+  try {
+    const result = await runAdminMutation({
+      ...runtime,
+      ...mutation,
+      stdout: runtime.stderr,
+      prepare: async () => {
+        target = await prepareTarget(input, runtime, fetch);
+        return {
+          review: {
+            title: "Execute unrestricted remote command",
+            fields: [
+              ...reviewFields(target),
+              { label: "Exact command", value: JSON.stringify(input.command) },
+              {
+                label: "Command UTF-8 bytes",
+                value: String(Buffer.byteLength(input.command, "utf8")),
+              },
+              { label: "Timeout", value: `${input.timeout_seconds}s` },
+            ],
+            consequence:
+              "This exact command will run with administrative access on the selected node.",
+          },
+          value: target,
+        };
+      },
+      execute: async () =>
+        execute(
+          {
+            node_id: input.node_id,
+            command: input.command,
+            timeout_seconds: input.timeout_seconds,
+          },
+          { confirmed: true }
+        ),
+    });
+    if (result.status === "declined") return declinedResult(input);
+    return executionResult(result.data, input, target);
+  } catch (error) {
+    if (wasInterrupted(error, runtime)) return interruptedResult(input);
+    throw error;
+  }
+}
+
+export const adminNodeDiagnosticsHandlers = [
+  {
+    capabilityId: "node.diagnose",
+    safety: ADMIN_MUTATION_SAFETY.eligibleHumanOnly,
+    parse: (argv) => parseAdminNodeDiagnoseArgs(argv),
+    run: runDiagnosis,
+  },
+  {
+    capabilityId: "node.exec",
+    safety: ADMIN_MUTATION_SAFETY.eligibleHumanOnly,
+    parse: (argv) => parseAdminNodeExecArgs(argv),
+    run: runExecution,
+  },
+];