@openclaw/crabline 0.1.11 → 0.1.12

package/dist/src/cli/program.js

@@ -1,17 +1,28 @@
-import { Command, CommanderError } from "commander";
+import { Command, CommanderError, Option } from "commander";
 import { randomUUID } from "node:crypto";
-import { constants as fsConstants } from "node:fs";
+import { constants as fsConstants, createReadStream } from "node:fs";
 import fs from "node:fs/promises";
 import nodePath from "node:path";
+import { finished } from "node:stream/promises";
 import { lock } from "proper-lockfile";
 import { loadManifest } from "../config/load.js";
+import { EXIT_CODES } from "../core/exit-codes.js";
 import { createRegistry } from "../providers/registry.js";
-import { formatJson, formatRunResultText, sanitizeTerminalText } from "../core/reporters.js";
+import { formatJson, formatJsonResult, formatRunResultText, sanitizeTerminalText, } from "../core/reporters.js";
 import { assertScriptStdinPayloadSize, computeExitCode, runFixtureCommand, runSuite, } from "../core/run.js";
 import { CrablineError, ensureErrorMessage } from "../core/errors.js";
 import { isCrablineServerChannel, startCrablineServer, } from "../servers/index.js";
 const READY_FILE_LEASE_STALE_MS = 60_000;
 const READY_FILE_LEASE_UPDATE_MS = 1_000;
+const MAX_SERVE_CREDENTIALS_BYTES = 64 * 1024;
+const WATCH_SHUTDOWN_GRACE_MS = 250;
+const WATCH_SHUTDOWN_REACHED = Symbol("watch shutdown reached");
+const SERVE_CREDENTIAL_ENV = [
+    ["accessToken", "CRABLINE_ACCESS_TOKEN"],
+    ["adminToken", "CRABLINE_ADMIN_TOKEN"],
+    ["botToken", "CRABLINE_BOT_TOKEN"],
+    ["signingSecret", "CRABLINE_SIGNING_SECRET"],
+];
 const SERVE_PARAM_FACTORIES = {
     mattermost: (shared, commandOptions) => ({
         ...shared,
@@ -126,9 +137,150 @@ async function writeOutput(stream, value) {
 async function print(value) {
     return await writeOutput(process.stdout, `${value}\n`);
 }
+class WatchShutdownDeadlineError extends CrablineError {
+}
+function createWatchShutdownDeadline() {
+    let deadlineAt;
+    const controller = new AbortController();
+    return {
+        async race(operation, operationName) {
+            let timer;
+            let removeStartListener;
+            const deadline = new Promise((_, reject) => {
+                const arm = () => {
+                    timer = setTimeout(() => {
+                        reject(new WatchShutdownDeadlineError(`Provider ${operationName} did not settle within ${WATCH_SHUTDOWN_GRACE_MS}ms during watch shutdown.`, { kind: "timeout" }));
+                    }, Math.max(0, deadlineAt - Date.now()));
+                };
+                if (controller.signal.aborted) {
+                    arm();
+                    return;
+                }
+                controller.signal.addEventListener("abort", arm, { once: true });
+                removeStartListener = () => controller.signal.removeEventListener("abort", arm);
+            });
+            try {
+                return await Promise.race([Promise.resolve(operation), deadline]);
+            }
+            finally {
+                removeStartListener?.();
+                if (timer) {
+                    clearTimeout(timer);
+                }
+            }
+        },
+        start() {
+            if (deadlineAt !== undefined) {
+                return;
+            }
+            deadlineAt = Date.now() + WATCH_SHUTDOWN_GRACE_MS;
+            controller.abort();
+        },
+    };
+}
+function containsWatchShutdownDeadline(error, seen = new Set()) {
+    if (error instanceof WatchShutdownDeadlineError) {
+        return true;
+    }
+    if (!error || typeof error !== "object" || seen.has(error)) {
+        return false;
+    }
+    seen.add(error);
+    if (error instanceof AggregateError) {
+        return error.errors.some((entry) => containsWatchShutdownDeadline(entry, seen));
+    }
+    return "cause" in error && containsWatchShutdownDeadline(error.cause, seen);
+}
 async function withManifest(options, action) {
     return action(await loadManifest(options.config));
 }
+function rejectedSecretOption(flags, optionName, envName) {
+    return new Option(flags).hideHelp().argParser(() => {
+        throw new CrablineError(`${optionName} no longer accepts credential values in command-line arguments. Use ${envName} or --credentials-fd.`, { kind: "config" });
+    });
+}
+function parseCredentialsFd(value) {
+    if (!/^[0-9]+$/u.test(value)) {
+        throw new CrablineError("Serve credentials file descriptor must be 0 or an integer greater than 2.", { kind: "config" });
+    }
+    const fd = Number(value);
+    if (!Number.isSafeInteger(fd) || fd === 1 || fd === 2 || fd > 2_147_483_647) {
+        throw new CrablineError("Serve credentials file descriptor must be 0 or an integer greater than 2.", { kind: "config" });
+    }
+    return fd;
+}
+async function readBoundedFileDescriptor(fd) {
+    const stream = createReadStream("", { autoClose: fd !== 0, fd });
+    const chunks = [];
+    let totalBytes = 0;
+    try {
+        for await (const chunk of stream) {
+            const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+            totalBytes += buffer.byteLength;
+            if (totalBytes > MAX_SERVE_CREDENTIALS_BYTES) {
+                throw new CrablineError(`Serve credentials JSON exceeds ${MAX_SERVE_CREDENTIALS_BYTES} bytes.`, { kind: "config" });
+            }
+            chunks.push(buffer);
+        }
+    }
+    catch (error) {
+        stream.destroy();
+        if (error instanceof CrablineError) {
+            throw error;
+        }
+        throw new CrablineError(`Unable to read serve credentials from file descriptor ${fd}.`, {
+            cause: error,
+            kind: "config",
+        });
+    }
+    finally {
+        if (fd !== 0 && !stream.closed) {
+            stream.destroy();
+            await finished(stream).catch(() => undefined);
+        }
+    }
+    return Buffer.concat(chunks).toString("utf8");
+}
+function parseServeCredentialsJson(input) {
+    let parsed;
+    try {
+        parsed = JSON.parse(input);
+    }
+    catch (error) {
+        throw new CrablineError("Serve credentials input must be valid JSON.", {
+            cause: error,
+            kind: "config",
+        });
+    }
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+        throw new CrablineError("Serve credentials JSON must be an object.", { kind: "config" });
+    }
+    const values = {};
+    for (const [key, value] of Object.entries(parsed)) {
+        if (!SERVE_CREDENTIAL_ENV.some(([name]) => name === key)) {
+            throw new CrablineError("Serve credentials JSON contains unsupported fields.", {
+                kind: "config",
+            });
+        }
+        if (typeof value !== "string") {
+            throw new CrablineError("Serve credentials JSON values must be strings.", {
+                kind: "config",
+            });
+        }
+        values[key] = value;
+    }
+    return values;
+}
+async function resolveServeCredentials(credentialsFd) {
+    const overrides = credentialsFd === undefined
+        ? {}
+        : parseServeCredentialsJson(await readBoundedFileDescriptor(parseCredentialsFd(credentialsFd)));
+    return Object.fromEntries(SERVE_CREDENTIAL_ENV.flatMap(([name, envName]) => {
+        const override = overrides[name];
+        const value = override ?? process.env[envName];
+        return value === undefined ? [] : [[name, value]];
+    }));
+}
 export function createProgram(setExitCode = (code) => {
     process.exitCode = code;
 }, dependencies = {}) {
@@ -138,6 +290,13 @@ export function createProgram(setExitCode = (code) => {
     const publish = dependencies.publishReadyFile ?? publishReadyFileUnlocked;
     const removeReady = dependencies.removeReadyFile ?? removeReadyFileIfOwned;
     const startServer = dependencies.startServer ?? startCrablineServer;
+    const formatCliJson = (value) => {
+        const formatted = formatJsonResult(value);
+        if (!formatted.ok) {
+            setExitCode(EXIT_CODES.FAILURE);
+        }
+        return formatted.output;
+    };
     program
         .exitOverride()
         .name("crabline")
@@ -162,7 +321,7 @@ export function createProgram(setExitCode = (code) => {
             })),
             support: registry.catalog,
         };
-        await print(options.json ? formatJson(payload) : renderProvidersText(payload));
+        await print(options.json ? formatCliJson(payload) : renderProvidersText(payload));
     });
     program
         .command("fixtures")
@@ -171,7 +330,7 @@ export function createProgram(setExitCode = (code) => {
         const options = program.opts();
         const { manifest } = await loadManifest(options.config);
         await print(options.json
-            ? formatJson(manifest.fixtures)
+            ? formatCliJson(manifest.fixtures)
             : manifest.fixtures
                 .map((fixture) => `${sanitizeTerminalText(fixture.id, true)} ${sanitizeTerminalText(fixture.mode, true)} provider=${sanitizeTerminalText(fixture.provider, true)} target=${sanitizeTerminalText(fixture.target.id, true)}`)
                 .join("\n"));
@@ -194,8 +353,8 @@ export function createProgram(setExitCode = (code) => {
             modeOverride: "probe",
             registry,
         });
-        await print(options.json ? formatJson(result) : formatRunResultText(result));
         setExitCode(computeExitCode(result));
+        await print(options.json ? formatCliJson(result) : formatRunResultText(result));
     });
     for (const mode of ["send", "roundtrip", "agent"]) {
         program
@@ -212,8 +371,8 @@ export function createProgram(setExitCode = (code) => {
                 modeOverride: mode,
                 registry,
             });
-            await print(options.json ? formatJson(result) : formatRunResultText(result));
             setExitCode(computeExitCode(result));
+            await print(options.json ? formatCliJson(result) : formatRunResultText(result));
         });
     }
     program
@@ -229,8 +388,8 @@ export function createProgram(setExitCode = (code) => {
             manifestPath: path,
             registry,
         });
-        await print(options.json ? formatJson(result) : formatRunResultText(result));
         setExitCode(computeExitCode(result));
+        await print(options.json ? formatCliJson(result) : formatRunResultText(result));
     });
     program
         .command("watch <fixtureId>")
@@ -250,12 +409,18 @@ export function createProgram(setExitCode = (code) => {
                 });
             }
             const controller = new AbortController();
+            const shutdownDeadline = createWatchShutdownDeadline();
             let iterator;
             const stopWatch = onceAsync(async () => {
+                shutdownDeadline.start();
                 controller.abort();
-                await iterator?.return?.();
+                const returnResult = iterator?.return?.();
+                if (returnResult) {
+                    await shutdownDeadline.race(returnResult, "watch iterator return");
+                }
             });
             const shutdown = installShutdownHandler(stopWatch);
+            const shutdownSettled = shutdown.wait().then(() => WATCH_SHUTDOWN_REACHED);
             const lifecycleErrors = [];
             try {
                 const watchContext = {
@@ -282,14 +447,17 @@ export function createProgram(setExitCode = (code) => {
                 const watch = provider.watch(watchContext);
                 iterator = watch[Symbol.asyncIterator]();
                 while (true) {
-                    const result = await iterator.next();
+                    const result = await Promise.race([iterator.next(), shutdownSettled]);
+                    if (typeof result === "symbol") {
+                        break;
+                    }
                     if (result.done) {
                         break;
                     }
                     const message = result.value;
-                    const written = await print(options.json
-                        ? formatJson(message)
-                        : `${sanitizeTerminalText(message.sentAt, true)} ${sanitizeTerminalText(message.author, true)} ${sanitizeTerminalText(message.text, true)}`);
+                    const written = await shutdownDeadline.race(print(options.json
+                        ? formatCliJson(message)
+                        : `${sanitizeTerminalText(message.sentAt, true)} ${sanitizeTerminalText(message.author, true)} ${sanitizeTerminalText(message.text, true)}`), "watch output write");
                     if (!written) {
                         break;
                     }
@@ -307,7 +475,10 @@ export function createProgram(setExitCode = (code) => {
                 }
             }
             try {
-                await provider.cleanup?.();
+                const cleanup = provider.cleanup?.();
+                if (cleanup) {
+                    await shutdownDeadline.race(cleanup, "cleanup");
+                }
             }
             catch (error) {
                 if (!lifecycleErrors.includes(error)) {
@@ -327,17 +498,24 @@ export function createProgram(setExitCode = (code) => {
         .description("Start a local provider server that OpenClaw live adapters can target")
         .option("--host <host>", "Bind host", "127.0.0.1")
         .option("--port <port>", "Bind port", "0")
-        .option("--admin-token <token>", "Admin token for inbound test messages (or CRABLINE_ADMIN_TOKEN)")
         .option("--account <number>", "Signal account number")
-        .option("--access-token <token>", "Matrix or WhatsApp access token (or CRABLINE_ACCESS_TOKEN)")
-        .option("--bot-token <token>", "Mattermost, Slack, Telegram, or Zalo bot token (or CRABLINE_BOT_TOKEN)")
         .option("--bot-username <username>", "Mattermost, Telegram, or Zalo bot username", "crabline_bot")
+        .option("--credentials-fd <fd>", "Read JSON credentials from fd 0 or an inherited fd; values override CRABLINE_* env")
         .option("--recorder <path>", "JSONL recorder path")
         .option("--ready-file <path>", "Write the server runtime manifest to this path")
         .option("--self-jid <jid>", "WhatsApp self JID")
         .option("--show-secrets", "Print provider credentials in text output", false)
-        .option("--signing-secret <secret>", "Slack signing secret (or CRABLINE_SIGNING_SECRET)")
         .option("--once", "Start, print the runtime manifest, and stop immediately", false)
+        .addOption(rejectedSecretOption("--access-token <token>", "--access-token", "CRABLINE_ACCESS_TOKEN"))
+        .addOption(rejectedSecretOption("--admin-token <token>", "--admin-token", "CRABLINE_ADMIN_TOKEN"))
+        .addOption(rejectedSecretOption("--bot-token <token>", "--bot-token", "CRABLINE_BOT_TOKEN"))
+        .addOption(rejectedSecretOption("--signing-secret <secret>", "--signing-secret", "CRABLINE_SIGNING_SECRET"))
+        .addHelpText("after", [
+        "",
+        "Credential JSON fields: accessToken, adminToken, botToken, signingSecret",
+        "Environment fallbacks: CRABLINE_ACCESS_TOKEN, CRABLINE_ADMIN_TOKEN,",
+        "  CRABLINE_BOT_TOKEN, CRABLINE_SIGNING_SECRET",
+    ].join("\n"))
         .action(async (provider, commandOptions) => {
         const options = program.opts();
         if (!isCrablineServerChannel(provider)) {
@@ -345,10 +523,6 @@ export function createProgram(setExitCode = (code) => {
                 kind: "config",
             });
         }
-        commandOptions.accessToken ??= process.env.CRABLINE_ACCESS_TOKEN;
-        commandOptions.adminToken ??= process.env.CRABLINE_ADMIN_TOKEN;
-        commandOptions.botToken ??= process.env.CRABLINE_BOT_TOKEN;
-        commandOptions.signingSecret ??= process.env.CRABLINE_SIGNING_SECRET;
         if (!/^[0-9]+$/u.test(commandOptions.port)) {
             throw new CrablineError(`Invalid local server port: ${commandOptions.port}`, {
                 kind: "config",
@@ -366,6 +540,11 @@ export function createProgram(setExitCode = (code) => {
                 kind: "config",
             });
         }
+        const credentials = await resolveServeCredentials(commandOptions.credentialsFd);
+        commandOptions.accessToken = credentials.accessToken;
+        commandOptions.adminToken = credentials.adminToken;
+        commandOptions.botToken = credentials.botToken;
+        commandOptions.signingSecret = credentials.signingSecret;
         let server;
         let startupSettled = false;
         let settleStartup;
@@ -389,14 +568,41 @@ export function createProgram(setExitCode = (code) => {
                 settlePublication?.();
             }
         };
+        let releaseReadyLease;
+        let publishedReady;
         const close = onceAsync(async () => {
             await startup;
             await publication;
-            await server?.close();
+            const lifecycleErrors = [];
+            if (commandOptions.readyFile && publishedReady) {
+                try {
+                    await removeReady(commandOptions.readyFile, publishedReady.contents, publishedReady.identity);
+                }
+                catch (error) {
+                    lifecycleErrors.push(error);
+                }
+            }
+            let serverClosed = false;
+            try {
+                await server?.close();
+                serverClosed = true;
+            }
+            catch (error) {
+                lifecycleErrors.push(error);
+            }
+            if (serverClosed && releaseReadyLease) {
+                try {
+                    await releaseReadyLease();
+                }
+                catch (error) {
+                    lifecycleErrors.push(error);
+                }
+            }
+            if (lifecycleErrors.length > 0) {
+                throw combineLifecycleErrors(lifecycleErrors, "Crabline server shutdown failed.");
+            }
         });
         const shutdown = installShutdownHandler(close);
-        let releaseReadyLease;
-        let publishedReady;
         let actionFailed = false;
         let actionError;
         try {
@@ -424,8 +630,13 @@ export function createProgram(setExitCode = (code) => {
                     await shutdown.wait();
                 }
                 else {
+                    let outputWritten = true;
                     try {
-                        const payload = formatJson(server.manifest);
+                        const formatted = formatJsonResult(server.manifest);
+                        if (!formatted.ok) {
+                            throw new Error("Unable to serialize server manifest.");
+                        }
+                        const payload = formatted.output;
                         if (commandOptions.readyFile) {
                             const readyContents = `${payload}\n`;
                             publishedReady = {
@@ -433,14 +644,25 @@ export function createProgram(setExitCode = (code) => {
                                 identity: await publish(commandOptions.readyFile, readyContents),
                             };
                         }
-                        await print(options.json
-                            ? payload
-                            : renderServeText(server.manifest, commandOptions.showSecrets === true));
+                        if (shutdown.requested()) {
+                            outputWritten = false;
+                            finishPublication();
+                            await shutdown.wait();
+                        }
+                        else {
+                            outputWritten = await print(options.json
+                                ? payload
+                                : renderServeText(server.manifest, commandOptions.showSecrets === true));
+                            if (!outputWritten) {
+                                finishPublication();
+                                await close();
+                            }
+                        }
                     }
                     finally {
                         finishPublication();
                     }
-                    if (shutdown.requested() || !commandOptions.once) {
+                    if (outputWritten && (shutdown.requested() || !commandOptions.once)) {
                         await shutdown.wait();
                     }
                 }
@@ -453,23 +675,12 @@ export function createProgram(setExitCode = (code) => {
         finishStartup();
         finishPublication();
         const cleanupErrors = [];
-        const readyToRemove = publishedReady;
-        for (const cleanup of [
-            close,
-            ...(commandOptions.readyFile && readyToRemove
-                ? [
-                    () => removeReady(commandOptions.readyFile, readyToRemove.contents, readyToRemove.identity),
-                ]
-                : []),
-            ...(releaseReadyLease ? [releaseReadyLease] : []),
-        ]) {
-            try {
-                await cleanup();
-            }
-            catch (error) {
-                if (!actionFailed || error !== actionError) {
-                    cleanupErrors.push(error);
-                }
+        try {
+            await close();
+        }
+        catch (error) {
+            if (!actionFailed || error !== actionError) {
+                cleanupErrors.push(error);
             }
         }
         shutdown.dispose();
@@ -486,8 +697,8 @@ export function createProgram(setExitCode = (code) => {
         const findings = diagnose(manifest);
         const ok = findings.length === 0;
         const payload = { findings, ok };
-        await print(options.json ? formatJson(payload) : ok ? "doctor ok" : findings.join("\n"));
         setExitCode(ok ? 0 : 10);
+        await print(options.json ? formatCliJson(payload) : ok ? "doctor ok" : findings.join("\n"));
     });
     return program;
 }
@@ -842,11 +1053,11 @@ function diagnose(manifest) {
     }
     return findings;
 }
-export async function runCli(argv) {
+export async function runCli(argv, options = {}) {
     let exitCode = 0;
     const program = createProgram((code) => {
         exitCode = code;
-    });
+    }, options.dependencies);
     const parserErrors = [];
     const parserOutput = [];
     const bufferParserErrors = (command) => {
@@ -896,6 +1107,9 @@ export async function runCli(argv) {
         else {
             await writeOutput(process.stderr, `${ensureErrorMessage(error)}\n`);
         }
+        if (options.forceExit && containsWatchShutdownDeadline(error)) {
+            options.forceExit(errorExitCode);
+        }
         return errorExitCode;
     }
 }