esoul-sdk 0.17.0 → 0.18.0

package/bin/esoul-device-exec.mjs

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+// The process that OWNS one run. The runtime starts it detached, so it — and
+// the command under it — outlive a runtime restart or upgrade; the runtime
+// reattaches by reading the files it leaves in the run directory.
+//
+//   meta.json  (in)   { argv, cwd, env, timeoutSeconds }
+//   out.log    (out)  stdout and stderr, interleaved, appended as they come
+//   pid.json   (out)  { wrapperPid, childPid, startedAt, bootId }
+//   exit.json  (out)  { exitCode, signal, reason, startedAt, endedAt }
+//   cancel     (in)   present = stop: SIGTERM the command's group, SIGKILL after 10 s
+//
+// Plain JavaScript with no imports beyond node: — it must run from any copy
+// of the package, including the one the service runs, with nothing built.
+import { spawn } from "node:child_process";
+import { existsSync, openSync, readFileSync, writeFileSync, renameSync, watch } from "node:fs";
+import { join } from "node:path";
+
+const dir = process.argv[2];
+if (!dir) {
+  console.error("usage: esoul-device-exec <run-dir>");
+  process.exit(2);
+}
+const meta = JSON.parse(readFileSync(join(dir, "meta.json"), "utf8"));
+const write = (name, value) => {
+  const p = join(dir, name);
+  writeFileSync(`${p}.tmp`, JSON.stringify(value));
+  renameSync(`${p}.tmp`, p);
+};
+const bootId = () => {
+  try {
+    return readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
+  } catch {
+    return null;
+  }
+};
+
+const out = openSync(join(dir, "out.log"), "a");
+const startedAt = Date.now();
+let child;
+try {
+  child = spawn(meta.argv[0], meta.argv.slice(1), {
+    cwd: meta.cwd || undefined,
+    env: meta.env,
+    stdio: ["ignore", out, out],
+    detached: true, // its own process group, so a stop reaches grandchildren too
+  });
+} catch (e) {
+  write("exit.json", { exitCode: null, signal: null, reason: `could not start: ${e.message}`, startedAt, endedAt: Date.now() });
+  process.exit(0);
+}
+
+let reason = null;
+let finished = false;
+child.on("error", (e) => {
+  if (finished) return;
+  finished = true;
+  write("exit.json", { exitCode: null, signal: null, reason: e.code === "ENOENT" ? `program not found: ${meta.argv[0]}` : `could not start: ${e.message}`, startedAt, endedAt: Date.now() });
+  process.exit(0);
+});
+child.on("spawn", () => write("pid.json", { wrapperPid: process.pid, childPid: child.pid, startedAt, bootId: bootId() }));
+
+const stop = (why) => {
+  if (reason || finished) return;
+  reason = why;
+  try {
+    process.kill(-child.pid, "SIGTERM");
+  } catch {}
+  setTimeout(() => {
+    try {
+      process.kill(-child.pid, "SIGKILL");
+    } catch {}
+  }, 10_000).unref();
+};
+
+if (meta.timeoutSeconds > 0) setTimeout(() => stop(`timed out after ${meta.timeoutSeconds} s`), meta.timeoutSeconds * 1000).unref();
+const cancelFile = join(dir, "cancel");
+// The cancel file's text is the reason ("cancelled", or "cancelled: <why>").
+const checkCancel = () => {
+  if (!existsSync(cancelFile)) return;
+  let why = "cancelled";
+  try {
+    why = readFileSync(cancelFile, "utf8").trim() || why;
+  } catch {}
+  stop(why);
+};
+checkCancel();
+try {
+  watch(dir, () => checkCancel()).unref();
+} catch {}
+const poll = setInterval(checkCancel, 1000);
+poll.unref();
+for (const sig of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(sig, () => stop(`runtime sent ${sig}`));
+
+child.on("exit", (code, signal) => {
+  if (finished) return;
+  finished = true;
+  write("exit.json", { exitCode: code, signal, reason, startedAt, endedAt: Date.now() });
+  process.exit(0);
+});

package/dist/device.d.ts

@@ -0,0 +1,136 @@
+/**
+ * Types for `devices(ctx)` — parts of an app that run on the person's own
+ * computer (plugin-device-arm.md §10–§11). The functions live in server.ts
+ * (host-only); the hooks and <ConnectComputer/> in react.ts.
+ *
+ * The flow an app is built around:
+ *   1. its config page renders <ConnectComputer workspaceId nodeId /> — one
+ *      command to paste, a code to check, an Approve button;
+ *   2. its server code runs a DECLARED command (plugin.json `device.commands`):
+ *        const d = devices(ctx);
+ *        const { commandId } = await d.run("train", { epochs: 3 });
+ *   3. its UI shows the run live with useDeviceRun({ workspaceId, nodeId, commandId }).
+ */
+export type DeviceRunStatus = "queued" | "claimed" | "running" | "done" | "expired" | "cancelled" | "lost";
+/** What an app's agents may reach on a computer through the shared command line. */
+export type WorkspaceGrant = {
+    scope: "folder";
+    root: string;
+} | {
+    scope: "computer";
+};
+/**
+ * One line of the shared command line, answered (esoul-sdk/machine shell.ts):
+ * `text` is what a terminal would print (what an agent reads), `data` the same
+ * answer structured, `opId` the journal entry to `undo`.
+ */
+export interface ShellResult {
+    ok: boolean;
+    text: string;
+    data?: unknown;
+    exitCode?: number | null;
+    error?: string;
+    opId?: string;
+    cwd: string;
+}
+/** One computer connected to this app instance. */
+export interface DeviceSummary {
+    linkId: string;
+    status: "active" | "suspended";
+    reason: string | null;
+    /** The declared commands this computer will run (the ones the owner approved). */
+    commands: string[];
+    /** Commands an app update added or changed since approval; they wait for the owner. */
+    pendingApproval: {
+        added: string[];
+        changed: string[];
+    };
+    machine: {
+        machineId: string;
+        hostname: string;
+        os: string;
+        arch: string | null;
+        runtimeVersion: string | null;
+        lastSeenAt: number | null;
+        online: boolean;
+    } | null;
+    /** What this app's agents may reach on that computer (null = declared commands only). */
+    workspace?: WorkspaceGrant | null;
+}
+export interface DeviceRunStarted {
+    commandId: string;
+    status: DeviceRunStatus;
+    /** Whether the computer answered in the last minute. A run for an offline computer waits (until `expiresAt`). */
+    machineOnline: boolean;
+    expiresAt: number;
+}
+export interface DeviceRunView {
+    commandId: string;
+    linkId: string;
+    machineId: string;
+    /** "commands" (a declared command) or "workspace" (the shared command line). */
+    capability?: string;
+    /** The named agent, for workspace jobs. */
+    agent?: string | null;
+    name: string;
+    params: Record<string, unknown>;
+    argv: string[];
+    status: DeviceRunStatus;
+    exitCode: number | null;
+    signal: string | null;
+    /** Why it ended when it did not simply exit: "machine_restarted", "cancelled: …", "timed out after …", a refusal. */
+    reason: string | null;
+    /** The first 1 MB of output (stdout and stderr interleaved). */
+    output?: string;
+    /** When `truncated` (or the command keeps only the tail), the last 64 KB. */
+    tail?: string;
+    /** The command's parsed answer, when it declared `result: "json"` and ended well. */
+    result?: unknown;
+    /** Why `result` is missing: not JSON, or larger than the stored head. */
+    resultError?: string | null;
+    /** How its output travelled: live, batched or final (the command's `output.mode`). */
+    outputMode?: "live" | "batched" | "final";
+    outputBytes: number;
+    outputSeq: number;
+    truncated: boolean;
+    createdAt: number;
+    startedAt: number | null;
+    endedAt: number | null;
+    expiresAt: number;
+}
+export interface DeviceRunOptions {
+    /** Which computer. Optional when exactly one active computer is connected. */
+    linkId?: string;
+    /** How long the run may wait for an offline computer before it expires. Default 600 s. */
+    expiresInSeconds?: number;
+}
+export interface Devices {
+    list(): Promise<DeviceSummary[]>;
+    run(name: string, params?: Record<string, unknown>, opts?: DeviceRunOptions): Promise<DeviceRunStarted>;
+    get(commandId: string): Promise<DeviceRunView>;
+    /** Wait for a run to end, up to `timeoutSeconds` (≤ 55 in an op; use a task for longer). Returns the run as it is then. */
+    wait(commandId: string, opts?: {
+        timeoutSeconds?: number;
+    }): Promise<DeviceRunView>;
+    cancel(commandId: string): Promise<{
+        status: string;
+    }>;
+    /**
+     * One line of the shared command line on the computer, as a NAMED agent —
+     * `ls`, `tree`, `cat -n f --lines 10:40`, `write f` / `edit f` / `patch f`
+     * (content on `stdin`), `run npm test`, `term new dev npm run dev`, `undo <op>`
+     * (`help` lists them). Needs a workspace grant (a folder or the computer).
+     * Waits up to `waitSeconds` (default 30, ≤ 55) and returns the answer.
+     */
+    sh(line: string, opts: {
+        agent: string;
+        stdin?: string;
+        linkId?: string;
+        waitSeconds?: number;
+    }): Promise<ShellResult & {
+        commandId: string;
+        status: DeviceRunStatus;
+    }>;
+}
+/** The statuses a run ends in: done, expired, cancelled, lost. */
+export declare const DEVICE_RUN_TERMINAL: readonly DeviceRunStatus[];

package/dist/device.js

@@ -0,0 +1,15 @@
+/**
+ * Types for `devices(ctx)` — parts of an app that run on the person's own
+ * computer (plugin-device-arm.md §10–§11). The functions live in server.ts
+ * (host-only); the hooks and <ConnectComputer/> in react.ts.
+ *
+ * The flow an app is built around:
+ *   1. its config page renders <ConnectComputer workspaceId nodeId /> — one
+ *      command to paste, a code to check, an Approve button;
+ *   2. its server code runs a DECLARED command (plugin.json `device.commands`):
+ *        const d = devices(ctx);
+ *        const { commandId } = await d.run("train", { epochs: 3 });
+ *   3. its UI shows the run live with useDeviceRun({ workspaceId, nodeId, commandId }).
+ */
+/** The statuses a run ends in: done, expired, cancelled, lost. */
+export const DEVICE_RUN_TERMINAL = ["done", "expired", "cancelled", "lost"];

package/dist/index.d.ts

@@ -14,6 +14,8 @@ export * from "./roles.js";
  * `opTool` in app.tsx — a tool's parameters cannot drift from what its op takes.
  */
 export * from "./ops.js";
+export { deviceBlockProblems, renderCommand, DeviceCommandError, COMMAND_NAME_RE, DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS } from "./machine/commands.js";
+export type { DeviceBlock, CommandSpec, ParamSpec, ConfigKeySpec, RenderedCommand } from "./machine/commands.js";
 /** A chart from data — one SVG for the app's screen and, rendered, for a tool's answer. */
 export * from "./chart.js";
 /**

package/dist/index.js

@@ -14,6 +14,8 @@ export * from "./roles.js";
  * `opTool` in app.tsx — a tool's parameters cannot drift from what its op takes.
  */
 export * from "./ops.js";
+// The device arm's command rules — check a plugin.json `device` block in your own tests (docs/18).
+export { deviceBlockProblems, renderCommand, DeviceCommandError, COMMAND_NAME_RE, DEFAULT_TIMEOUT_SECONDS, MAX_TIMEOUT_SECONDS } from "./machine/commands.js";
 /** A chart from data — one SVG for the app's screen and, rendered, for a tool's answer. */
 export * from "./chart.js";
 /**

package/dist/machine/capabilities.d.ts

@@ -0,0 +1,81 @@
+import type { DoorClient } from "./client.js";
+import { type ResolvedOutput } from "./commands.js";
+import { type WorkspaceGrant } from "./grant.js";
+import { type ClaimedCommand } from "./runner.js";
+import { type LocalLink } from "./state.js";
+export interface Job extends ClaimedCommand {
+    capability: string;
+    agent: string | null;
+}
+export interface JobOutcome {
+    exitCode: number | null;
+    reason?: string | null;
+    signal?: string | null;
+    /** A structured answer (the workspace's ShellResult, …). */
+    result?: unknown;
+}
+export interface JobContext {
+    home: string;
+    link: LocalLink;
+    /** The computer's own ceiling for workspace grants. */
+    ceiling: WorkspaceGrant | null;
+    client: DoorClient;
+    signal: AbortSignal;
+    log: (s: string) => void;
+}
+export interface Capability {
+    name: string;
+    version: number;
+    /** Why this job may NOT run here, or null. Checked before anything starts. */
+    refuse(job: Job, ctx: JobContext): string | null;
+    /** Start the job; resolves when the capability has taken responsibility for reporting it. */
+    start(job: Job, ctx: JobContext): Promise<void>;
+}
+/**
+ * Ordered, batched, retried upload of one job's output (the run directory is
+ * the outbox for detached runs; inline jobs hold theirs here). `finish` drains
+ * the output, then reports the end with the result.
+ */
+export declare class OutputUploader {
+    private readonly client;
+    private readonly commandId;
+    private readonly output;
+    private readonly onCancel;
+    private buf;
+    private seq;
+    private flushing;
+    private timer;
+    constructor(client: DoorClient, commandId: string, output: ResolvedOutput, onCancel: () => void);
+    write(chunk: string): void;
+    private flush;
+    finish(outcome: JobOutcome): Promise<void>;
+}
+declare function markDone(home: string, id: string): void;
+/** Inline jobs a previous runtime started but never reported. */
+export declare function strandedInlineJobs(home: string): string[];
+export declare const markInlineReported: typeof markDone;
+/** The app's declared commands; the detached runner owns and reports each run. */
+export declare function commandsCapability(deps: {
+    startRun: (job: Job, redact: string[], env: Record<string, string>) => Promise<void>;
+}): Capability;
+export interface WorkspaceJobParams {
+    line: string;
+    stdin?: string;
+}
+/** The shared command line, under grant ∩ ceiling, one working directory per named agent. */
+export declare function workspaceCapability(): Capability;
+/** The capability table the runtime dispatches jobs through, by name. */
+export declare function registry(caps: Capability[]): Map<string, Capability>;
+/** What the hello announces: ["commands@1", "workspace@1"]. */
+export declare function announce(reg: Map<string, Capability>): string[];
+/** The capabilities this runtime is built with, as one registry (the supervisor dispatches through it). */
+export declare function builtInCapabilities(startRun: (job: Job, redact: string[], env: Record<string, string>) => Promise<void>): Map<string, Capability>;
+/**
+ * What this runtime can do, said at CONNECT as well as in every hello, so the
+ * platform knows it from the approval on: work asked for before the runtime's
+ * first hello waits in the queue instead of being refused.
+ */
+export declare const RUNTIME_CAPABILITIES: string[];
+/** Read the computer's own ceiling from its state. */
+export declare function localCeiling(home: string): WorkspaceGrant | null;
+export {};

package/dist/machine/capabilities.js

@@ -0,0 +1,211 @@
+/**
+ * CAPABILITIES — the unit the machine side grows by (plugin-device-arm.md §15).
+ *
+ * Everything the platform asks of a computer is a JOB for one capability:
+ *
+ *   commands@1   a declared command from the app's plugin.json (argv, typed params)
+ *   workspace@1  the shared command line in a folder or the whole computer
+ *   (later)      messages, programs, … — each one module registered here
+ *
+ * A capability says whether a job may run on this link (from the runtime's OWN
+ * copy of the grants and the computer's ceiling — never the platform's word
+ * alone), and runs it. Output streams through one ordered uploader; the end is
+ * one report with an optional structured result. The runtime announces the
+ * registry in its hello, so the platform never queues work an older runtime
+ * cannot do.
+ */
+import { mkdirSync, readdirSync, writeFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { renderCommand, resolveOutput, DeviceCommandError } from "./commands.js";
+import { effectiveGrant, parseGrant } from "./grant.js";
+import { commandEnv } from "./runner.js";
+import { execute, openWorkspace, AGENT_NAME_RE } from "./shell.js";
+import { loadConfig, readJson } from "./state.js";
+/**
+ * Ordered, batched, retried upload of one job's output (the run directory is
+ * the outbox for detached runs; inline jobs hold theirs here). `finish` drains
+ * the output, then reports the end with the result.
+ */
+export class OutputUploader {
+    client;
+    commandId;
+    output;
+    onCancel;
+    buf = "";
+    seq = 0;
+    flushing = null;
+    timer = null;
+    constructor(client, commandId, output, onCancel) {
+        this.client = client;
+        this.commandId = commandId;
+        this.output = output;
+        this.onCancel = onCancel;
+    }
+    write(chunk) {
+        if (this.output.mode === "final" || !chunk) {
+            this.buf += chunk;
+            return;
+        }
+        this.buf += chunk;
+        if (this.buf.length >= this.output.maxChunkBytes)
+            void this.flush();
+        else if (!this.timer)
+            this.timer = setTimeout(() => void this.flush(), this.output.mode === "batched" ? this.output.flushMs : 50);
+    }
+    async flush() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        while (this.flushing)
+            await this.flushing;
+        if (!this.buf)
+            return;
+        const run = async () => {
+            while (this.buf) {
+                const data = this.buf.slice(0, this.output.maxChunkBytes);
+                const seq = this.seq + 1;
+                for (let attempt = 0;; attempt++) {
+                    try {
+                        const r = await this.client.call("POST", `/api/device/m/commands/${this.commandId}/output`, { seq, data });
+                        if (r.cancel)
+                            this.onCancel();
+                        break;
+                    }
+                    catch (e) {
+                        const b = e.body;
+                        if (e.status === 409 && typeof b?.expectedSeq === "number") {
+                            this.seq = b.expectedSeq - 1;
+                            if (b.expectedSeq > seq)
+                                break; // the door already has it
+                            continue;
+                        }
+                        if (attempt >= 8)
+                            throw e;
+                        await new Promise((r) => setTimeout(r, Math.min(500 * 2 ** attempt, 15_000)));
+                    }
+                }
+                this.seq = seq;
+                this.buf = this.buf.slice(data.length);
+            }
+        };
+        this.flushing = run().finally(() => (this.flushing = null));
+        await this.flushing;
+    }
+    async finish(outcome) {
+        await this.flush();
+        await this.client.call("POST", `/api/device/m/commands/${this.commandId}/done`, { exitCode: outcome.exitCode, signal: outcome.signal ?? null, reason: outcome.reason ?? null, result: outcome.result ?? undefined });
+    }
+}
+/** Inline jobs leave a marker while they run, so a runtime that died mid-job reports it lost instead of leaving it "running". */
+function markStart(home, id) {
+    const d = join(home, "runs", id);
+    mkdirSync(d, { recursive: true, mode: 0o700 });
+    writeFileSync(join(d, "inline.json"), JSON.stringify({ at: Date.now() }));
+}
+function markDone(home, id) {
+    writeFileSync(join(home, "runs", id, "reported"), String(Date.now()));
+}
+/** Inline jobs a previous runtime started but never reported. */
+export function strandedInlineJobs(home) {
+    const d = join(home, "runs");
+    if (!existsSync(d))
+        return [];
+    return readdirSync(d).filter((id) => existsSync(join(d, id, "inline.json")) && !existsSync(join(d, id, "reported")));
+}
+export const markInlineReported = markDone;
+// ─────────────────────────────── commands@1 ───────────────────────────────
+/** The app's declared commands; the detached runner owns and reports each run. */
+export function commandsCapability(deps) {
+    return {
+        name: "commands",
+        version: 1,
+        refuse(job, ctx) {
+            const block = ctx.link.commands;
+            if (!block)
+                return "this computer holds no approved commands for that app";
+            let argv;
+            try {
+                argv = renderCommand(block, job.name, job.params).argv;
+            }
+            catch (e) {
+                return e instanceof DeviceCommandError ? e.message : String(e);
+            }
+            if (JSON.stringify(argv) !== JSON.stringify(job.argv))
+                return "the platform's command differs from the approved one";
+            return null;
+        },
+        async start(job, ctx) {
+            const block = ctx.link.commands;
+            const spec = block.commands[job.name];
+            const env = commandEnv(process.env, spec.env ?? {}, loadConfig(ctx.home, ctx.link.linkId), spec.config ?? []);
+            await deps.startRun({ ...job, cwd: spec.cwd ?? null, timeoutSeconds: spec.timeoutSeconds ?? 3600, onRestart: spec.onRestart ?? "report", output: resolveOutput(spec.output) }, block.redact ?? [], env);
+        },
+    };
+}
+/** The shared command line, under grant ∩ ceiling, one working directory per named agent. */
+export function workspaceCapability() {
+    return {
+        name: "workspace",
+        version: 1,
+        refuse(job, ctx) {
+            if (!job.agent || !AGENT_NAME_RE.test(job.agent))
+                return "a workspace job needs a named agent";
+            const eff = effectiveGrant(parseGrant(ctx.link.grants?.workspace), ctx.ceiling);
+            return eff.ok ? null : eff.why;
+        },
+        async start(job, ctx) {
+            const eff = effectiveGrant(parseGrant(ctx.link.grants?.workspace), ctx.ceiling);
+            if (!eff.ok)
+                throw new Error(eff.why);
+            const scope = eff.grant.scope === "computer" ? { kind: "computer" } : { kind: "folder", root: eff.grant.root };
+            const ws = openWorkspace(ctx.home, ctx.link.linkId, scope);
+            const params = job.params;
+            markStart(ctx.home, job.commandId);
+            const ac = new AbortController();
+            ctx.signal.addEventListener("abort", () => ac.abort());
+            const up = new OutputUploader(ctx.client, job.commandId, job.output ?? resolveOutput({ mode: "live" }), () => ac.abort());
+            await ctx.client.call("POST", `/api/device/m/commands/${job.commandId}/started`, { startedAt: Date.now() }).catch(() => { });
+            // Run without holding the claim loop: agents work in parallel.
+            void (async () => {
+                let outcome;
+                try {
+                    const r = await execute({ ws, agent: job.agent, line: String(params.line ?? ""), stdin: params.stdin, onOutput: (c) => up.write(c), signal: ac.signal });
+                    outcome = { exitCode: r.exitCode ?? (r.ok ? 0 : 1), reason: r.ok ? null : r.error ?? "failed", result: r };
+                }
+                catch (e) {
+                    outcome = { exitCode: null, reason: `failed on the computer: ${e.message}` };
+                }
+                try {
+                    await up.finish(outcome);
+                    markDone(ctx.home, job.commandId);
+                }
+                catch (e) {
+                    ctx.log(`workspace job ${job.commandId}: could not report (${e.message}); it will be reported lost on restart`);
+                }
+            })();
+        },
+    };
+}
+/** The capability table the runtime dispatches jobs through, by name. */
+export function registry(caps) {
+    return new Map(caps.map((c) => [c.name, c]));
+}
+/** What the hello announces: ["commands@1", "workspace@1"]. */
+export function announce(reg) {
+    return [...reg.values()].map((c) => `${c.name}@${c.version}`);
+}
+/** The capabilities this runtime is built with, as one registry (the supervisor dispatches through it). */
+export function builtInCapabilities(startRun) {
+    return registry([commandsCapability({ startRun }), workspaceCapability()]);
+}
+/**
+ * What this runtime can do, said at CONNECT as well as in every hello, so the
+ * platform knows it from the approval on: work asked for before the runtime's
+ * first hello waits in the queue instead of being refused.
+ */
+export const RUNTIME_CAPABILITIES = announce(builtInCapabilities(async () => { }));
+/** Read the computer's own ceiling from its state. */
+export function localCeiling(home) {
+    return parseGrant(readJson(join(home, "state.json"))?.ceiling ?? null);
+}

package/dist/machine/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};