cuttlely 1.0.2 → 1.0.3

package/README.md

@@ -41,10 +41,39 @@ cuttlely --help
 cuttlely --version
 ```
 
+## Check a draft
+
+`check` runs the Publish check on a flow's draft and waits for it. It reads the API key from `CUTTLELY_API_KEY` and calls the server you name. It does not start Docker.
+
+```bash
+export CUTTLELY_API_KEY=<key>
+npx cuttlely check --flow <flow id> --server http://127.0.0.1:43117 --timeout 300 --json check.json --fail-on-changed
+```
+
+| Flag                | Meaning                                                                                                   |
+| ------------------- | --------------------------------------------------------------------------------------------------------- |
+| `--flow`            | Flow id from the canvas address bar. Required.                                                            |
+| `--server`          | Server origin. Default `http://127.0.0.1:43117`.                                                          |
+| `--timeout`         | Seconds to wait. Default 300.                                                                             |
+| `--json`            | Write the full check JSON to this path.                                                                   |
+| `--fail-on-changed` | Exit 1 when any answer is Changed. Without it, a finished check exits 0 and the summary still lists them. |
+
+`pnpm dev` serves the API on port 3000. `scripts/start-cuttlely.sh` and `npx cuttlely start` listen on port 43117.
+
+| Code | When                                                                                                                                     |
+| ---- | ---------------------------------------------------------------------------------------------------------------------------------------- |
+| 0    | The check finished. No answer was Changed, or `--fail-on-changed` was off.                                                               |
+| 1    | `--fail-on-changed` was set and at least one answer was Changed.                                                                         |
+| 2    | The check timed out, the server returned an error, `CUTTLELY_API_KEY` was missing, the flow has no draft, or the command was used wrong. |
+
+A check the server marks stopped or failed also exits 2. If the draft is saved, published, or discarded while the check runs, the command exits 2, because the answers may be for a draft that is already gone. After the check has started, a timeout, a 403, or any other failure asks the server to stop it. Cancelling the command stops that server-side check too. That stop is best effort: if it fails, the exit code is still 2.
+
+The key needs both permissions for the flow: `chatflows:view` and `chatflows:update`, or `agentflows:view` and `agentflows:update`. Update starts the check. View is what reading the result uses. The guide is [Fail a build when answers change](https://github.com/cuttlely/cuttlely/blob/main/docs/how-to/ci-gate.md).
+
 The container name is `cuttlely`. `start` leaves it running when it is already on the same image and port. `upgrade` always pulls and starts it again. The volume is `cuttlely-data`, mounted at `/var/cuttlely`.
 
 Compose uses the volume name `cuttlely_cuttlely-data`. This launcher uses `cuttlely-data`.
 
 The first account is created from the one-time setup link in `cuttlely logs`.
 
-This package makes no network call of its own. Docker pulls the image, and the command waits on the local health check.
+The start, stop, status, logs and upgrade commands make no network call of their own. Docker pulls the image, and start waits on the local health check. `check` calls the Cuttlely server named with `--server`.

package/bin/cuttlely.js

@@ -1,9 +1,10 @@
 #!/usr/bin/env node
-import { main } from '../lib/cli.js'
+import { commandFailureCode, main } from '../lib/cli.js'
 
+const argv = process.argv.slice(2)
 try {
-    process.exitCode = await main(process.argv.slice(2))
+    process.exitCode = await main(argv)
 } catch (error) {
     console.error(error instanceof Error ? error.message : String(error))
-    process.exitCode = 1
+    process.exitCode = commandFailureCode(argv)
 }

package/lib/check.js

@@ -0,0 +1,340 @@
+// Runs the Publish check (POST /api/v1/flow-drafts/:id/check) and waits for it.
+// Talks only to the server named with --server. The API key stays in the environment.
+import { writeFileSync } from 'node:fs'
+
+export const API_KEY_ENV = 'CUTTLELY_API_KEY'
+export const DEFAULT_SERVER = 'http://127.0.0.1:43117'
+export const DEFAULT_TIMEOUT_SECONDS = 300
+export const DEFAULT_QUESTION_COUNT = 4
+export const POLL_MS = 1000
+
+export const EXIT_PASS = 0
+export const EXIT_CHANGED = 1
+export const EXIT_ERROR = 2
+
+const ANSWER_CLIP = 240
+
+function redact(text, secret) {
+    if (!secret || typeof text !== 'string' || !text.includes(secret)) return text
+    return text.split(secret).join('[redacted]')
+}
+
+function redactValue(value, secret) {
+    if (typeof value === 'string') return redact(value, secret)
+    if (Array.isArray(value)) return value.map((item) => redactValue(item, secret))
+    if (value && typeof value === 'object') {
+        const copy = {}
+        for (const [key, item] of Object.entries(value)) copy[key] = redactValue(item, secret)
+        return copy
+    }
+    return value
+}
+
+function oneLine(text) {
+    const flat = String(text ?? '')
+        .replace(/\s+/g, ' ')
+        .trim()
+    if (!flat) return '(empty)'
+    if (flat.length <= ANSWER_CLIP) return flat
+    return `${flat.slice(0, ANSWER_CLIP - 3)}...`
+}
+
+function formatMs(value) {
+    return value == null ? 'n/a' : `${value} ms`
+}
+
+function formatUsd(value) {
+    const n = Number(value)
+    if (!Number.isFinite(n)) return ''
+    if (n === 0) return '$0'
+    if (Math.abs(n) >= 0.01) return `$${n.toFixed(2)}`
+    return `$${n.toFixed(8).replace(/0+$/, '').replace(/\.$/, '')}`
+}
+
+function estimateLine(label, usd, price) {
+    if (price === 'no price') return `${label}: estimated cost no price`
+    if (price === 'partial') {
+        const money = formatUsd(usd)
+        return money ? `${label}: estimated ${money} (some models have no price)` : `${label}: estimated cost no price`
+    }
+    if (price === 'known') {
+        const money = formatUsd(usd)
+        return money ? `${label}: estimated ${money}` : ''
+    }
+    return ''
+}
+
+export function changedCases(check) {
+    return (check?.cases || []).filter((row) => row && row.same === false && row.live && row.draft && !row.live.error && !row.draft.error)
+}
+
+function errorCases(check) {
+    return (check?.cases || []).filter((row) => row && (row.live?.error || row.draft?.error))
+}
+
+export function summaryText(check) {
+    const summary = check.summary || {}
+    const lines = [
+        `Check ${check.id} ${check.status}`,
+        `Questions: ${summary.ran ?? 0} ran, ${summary.same ?? 0} same, ${summary.changed ?? 0} changed, ${summary.errors ?? 0} errors`,
+        `Live: ${summary.liveTokens ?? 0} tokens, median ${formatMs(summary.liveMedianMs)}`,
+        `Draft: ${summary.draftTokens ?? 0} tokens, median ${formatMs(summary.draftMedianMs)}`
+    ]
+    // The check total, with each model priced once. It is not the sum of the per-answer figures.
+    const liveEstimate = estimateLine('Live', summary.liveEstimatedCostUsd, summary.livePrice)
+    const draftEstimate = estimateLine('Draft', summary.draftEstimatedCostUsd, summary.draftPrice)
+    if (liveEstimate) lines.push(liveEstimate)
+    if (draftEstimate) lines.push(draftEstimate)
+    if (check.liveEmpty) lines.push('Live has no cards, so only the draft was asked.')
+    if (check.stopReason) lines.push(`Stopped: ${check.stopReason}`)
+    if (check.error) lines.push(`Error: ${check.error}`)
+    const changed = changedCases(check)
+    if (changed.length) {
+        lines.push('', 'Changed')
+        for (const row of changed) {
+            lines.push(`  ${oneLine(row.question)}`)
+            lines.push(`    Live: ${oneLine(row.live.text)}`)
+            lines.push(`    Draft: ${oneLine(row.draft.text)}`)
+        }
+    }
+    const errors = errorCases(check)
+    if (errors.length) {
+        lines.push('', 'Errors')
+        for (const row of errors) {
+            const why = [row.live?.error, row.draft?.error].filter(Boolean).join('; ')
+            lines.push(`  ${oneLine(row.question)}`)
+            lines.push(`    ${oneLine(why)}`)
+        }
+    }
+    return lines.join('\n')
+}
+
+// Replay is the only verdict this command runs. A later scored mode (pass or fail
+// from a test set) should return the same { code, label } and be selected in selectVerdict.
+export function replayVerdict(check, { failOnChanged }) {
+    if (!check || typeof check !== 'object') return { code: EXIT_ERROR, label: 'error' }
+    if (check.status === 'failed') return { code: EXIT_ERROR, label: 'failed' }
+    if (check.status === 'stopped') return { code: EXIT_ERROR, label: 'stopped' }
+    if (check.status !== 'done') return { code: EXIT_ERROR, label: 'incomplete' }
+    if (check.draftChanged) return { code: EXIT_ERROR, label: 'draft-changed' }
+    const changed = Number(check.summary?.changed ?? 0)
+    if (!Number.isFinite(changed)) return { code: EXIT_ERROR, label: 'error' }
+    if (failOnChanged && changed > 0) return { code: EXIT_CHANGED, label: 'changed' }
+    return { code: EXIT_PASS, label: 'pass' }
+}
+
+export function selectVerdict(check, options) {
+    return replayVerdict(check, options)
+}
+
+function deadlineSignal(deadline, now) {
+    const remaining = Math.max(1, deadline - now())
+    if (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') return AbortSignal.timeout(remaining)
+    return undefined
+}
+
+async function callApi(fetchImpl, url, { method, headers, body, signal }) {
+    let response
+    try {
+        response = await fetchImpl(url, { method, headers, body, redirect: 'manual', signal })
+    } catch (error) {
+        const name = error && error.name
+        if (name === 'AbortError' || name === 'TimeoutError') return { timeout: true }
+        return { network: true, message: error instanceof Error ? error.message : String(error) }
+    }
+    const status = Number(response.status)
+    if (status >= 300 && status < 400) {
+        return { ok: false, status, message: 'The server sent a redirect, which this command will not follow.' }
+    }
+    let text = ''
+    try {
+        text = typeof response.text === 'function' ? await response.text() : ''
+    } catch (error) {
+        return { network: true, message: error instanceof Error ? error.message : String(error) }
+    }
+    let json = null
+    if (text) {
+        try {
+            json = JSON.parse(text)
+        } catch {
+            json = null
+        }
+    }
+    const raw = json && (json.message || json.error)
+    const message = typeof raw === 'string' && raw ? raw : text ? text.slice(0, 300) : `The server returned HTTP ${status}.`
+    return { ok: status >= 200 && status < 300, status, message, json }
+}
+
+function isNoDraft(result) {
+    return result.status === 409 && typeof result.message === 'string' && /no unpublished changes/i.test(result.message)
+}
+
+// Starting a check needs update. Reading it needs view. A key with only one of those gets 403,
+// and the server message does not say which permission is missing.
+function permissionDeniedMessage() {
+    return 'The API key needs chatflows:view and chatflows:update, or agentflows:view and agentflows:update.'
+}
+
+function writeResult(writeFile, jsonPath, check, secret) {
+    if (!jsonPath || !check) return
+    const safe = redactValue(check, secret)
+    writeFile(jsonPath, `${JSON.stringify(safe, null, 4)}\n`)
+}
+
+async function stopCheck(fetchImpl, url, headers) {
+    try {
+        const signal = typeof AbortSignal !== 'undefined' && AbortSignal.timeout ? AbortSignal.timeout(5000) : undefined
+        await fetchImpl(url, { method: 'DELETE', headers, redirect: 'manual', signal })
+    } catch {
+        // The timeout result stands either way. A stuck stop must not hide it.
+    }
+}
+
+export async function runCheck(options, io = {}) {
+    const out = io.out || ((line) => console.info(line))
+    const err = io.err || ((line) => console.error(line))
+    const env = io.env || process.env
+    const fetchImpl = io.fetch || globalThis.fetch.bind(globalThis)
+    const sleep = io.sleep || ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+    const now = io.now || Date.now
+    const writeFile = io.writeFile || ((file, text) => writeFileSync(file, text))
+    const secret = typeof env[API_KEY_ENV] === 'string' ? env[API_KEY_ENV].trim() : ''
+    if (!secret) {
+        err(`${API_KEY_ENV} is not set.`)
+        return EXIT_ERROR
+    }
+
+    const server = options.server || DEFAULT_SERVER
+    const timeoutSeconds = options.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS
+    const flow = options.flow
+    const failOnChanged = Boolean(options.failOnChanged)
+    const base = `${server}/api/v1/flow-drafts/${encodeURIComponent(flow)}/check`
+    const headers = { Authorization: `Bearer ${secret}`, Accept: 'application/json' }
+    const deadline = now() + timeoutSeconds * 1000
+
+    const say = (text) => redact(text, secret)
+
+    const fail = (text) => {
+        err(say(text))
+        return EXIT_ERROR
+    }
+
+    // Set once the server has accepted the check. Every later failure tries to stop it.
+    let check = null
+    const stopIfStarted = async () => {
+        if (!check?.id) return
+        await stopCheck(fetchImpl, `${base}/${encodeURIComponent(check.id)}`, headers)
+    }
+    // The stop is best effort. A failed DELETE does not replace the exit code.
+    const failRunning = async (text) => {
+        await stopIfStarted()
+        return fail(text)
+    }
+    // Signals are armed only after the id is known, and removed on a normal return so a
+    // finished check is left alone. process.exit skips the finally, so the handler disarms first.
+    const signalTarget = io.signals || process
+    let signalExit = false
+    const onSignal = () => {
+        if (signalExit) return
+        signalExit = true
+        disarmSignals()
+        void (async () => {
+            await stopIfStarted()
+            err('The check was cancelled.')
+            const exit = io.exit || ((code) => process.exit(code))
+            exit(EXIT_ERROR)
+        })()
+    }
+    const disarmSignals = () => {
+        signalTarget.removeListener('SIGTERM', onSignal)
+        signalTarget.removeListener('SIGINT', onSignal)
+    }
+    const armSignals = () => {
+        signalTarget.on('SIGTERM', onSignal)
+        signalTarget.on('SIGINT', onSignal)
+    }
+
+    try {
+        const started = await callApi(fetchImpl, base, {
+            method: 'POST',
+            headers: { ...headers, 'Content-Type': 'application/json' },
+            body: JSON.stringify({ count: DEFAULT_QUESTION_COUNT }),
+            signal: deadlineSignal(deadline, now)
+        })
+        if (started.timeout) return fail(`Timed out after ${timeoutSeconds}s before the check started.`)
+        if (started.network) return fail(`Could not reach ${server}. ${started.message}`)
+        if (!started.ok) {
+            if (started.status === 403) return fail(permissionDeniedMessage())
+            if (isNoDraft(started)) return fail(`No draft to check. ${started.message}`)
+            return fail(`Check failed (${started.status}): ${started.message}`)
+        }
+        check = started.json
+        if (!check || typeof check !== 'object' || !check.id) return fail('The server did not return a check.')
+        armSignals()
+        out(`Started check ${check.id} for flow ${flow} (${check.total ?? 0} questions).`)
+
+        const giveUp = async () => {
+            await stopIfStarted()
+            try {
+                writeResult(writeFile, options.jsonPath, check, secret)
+            } catch (error) {
+                err(say(`Could not write ${options.jsonPath}. ${error instanceof Error ? error.message : String(error)}`))
+            }
+            return fail(`Timed out after ${timeoutSeconds}s waiting for check ${check.id}.`)
+        }
+
+        while (check.status === 'running') {
+            // Read before sleeping, and once more at the deadline, so a check that
+            // finishes in the last poll interval is not reported from a stale body.
+            const polled = await callApi(fetchImpl, `${base}/${encodeURIComponent(check.id)}`, {
+                method: 'GET',
+                headers,
+                signal: deadlineSignal(deadline + POLL_MS, now)
+            })
+            if (polled.timeout) return giveUp()
+            if (polled.network) return failRunning(`Could not reach ${server}. ${polled.message}`)
+            if (polled.status === 403) return failRunning(permissionDeniedMessage())
+            if (!polled.ok) return failRunning(`Check failed (${polled.status}): ${polled.message}`)
+            if (!polled.json || typeof polled.json !== 'object') return failRunning('The server did not return a check.')
+            const previous = check.completed
+            check = polled.json
+            if (check.status === 'running' && check.completed !== previous && check.completed != null) {
+                out(`Asked ${check.completed} of ${check.total}.`)
+            }
+            if (check.status !== 'running') break
+            const remaining = deadline - now()
+            if (remaining <= 0) return giveUp()
+            await sleep(Math.min(POLL_MS, remaining))
+        }
+
+        try {
+            writeResult(writeFile, options.jsonPath, check, secret)
+        } catch (error) {
+            err(say(`Could not write ${options.jsonPath}. ${error instanceof Error ? error.message : String(error)}`))
+            out(say(summaryText(check)))
+            await stopIfStarted()
+            return EXIT_ERROR
+        }
+        out(say(summaryText(check)))
+        const verdict = selectVerdict(check, { failOnChanged })
+        if (verdict.label === 'draft-changed') {
+            err('The draft changed while the check was running, so the result is out of date.')
+        } else if (verdict.label === 'failed') {
+            err(say(check.error ? `The check failed. ${check.error}` : 'The check failed.'))
+        } else if (verdict.label === 'stopped') {
+            err(`The check stopped${check.stopReason ? ` (${check.stopReason})` : ''} before it finished.`)
+        } else if (verdict.label === 'changed') {
+            const count = Number(check.summary?.changed ?? 0)
+            err(count === 1 ? '1 answer changed.' : `${count} answers changed.`)
+        } else if (verdict.code === EXIT_ERROR) {
+            err('The check did not finish.')
+        }
+        if (verdict.code === EXIT_ERROR) await stopIfStarted()
+        return verdict.code
+    } catch (error) {
+        await stopIfStarted()
+        return fail(error instanceof Error ? error.message : String(error))
+    } finally {
+        if (!signalExit) disarmSignals()
+    }
+}

package/lib/cli.js

@@ -4,6 +4,7 @@ import { readFileSync, statSync } from 'node:fs'
 import { spawn } from 'node:child_process'
 import { dirname, join } from 'node:path'
 import { fileURLToPath } from 'node:url'
+import { runCheck } from './check.js'
 import { spawnDocker } from './docker.js'
 
 const PKG = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json')
@@ -20,7 +21,12 @@ export const HEALTH_TIMEOUT_MS = 6 * 60 * 1000
 const TAG_RE = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/
 const INSPECT_FORMAT = '{{.State.Status}}|{{.Config.Image}}|{{.Image}}|{{range $p, $b := .HostConfig.PortBindings}}{{$p}} {{end}}'
 
-export class UsageError extends Error {}
+export class UsageError extends Error {
+    constructor(message, exitCode = 1) {
+        super(message)
+        this.exitCode = exitCode
+    }
+}
 
 export function helpText() {
     return [
@@ -32,6 +38,7 @@ export function helpText() {
         '  cuttlely status',
         '  cuttlely logs [--follow]',
         '  cuttlely upgrade [--port <n>] [--tag <tag>] [--env-file <path>] [--no-open]',
+        '  cuttlely check --flow <id> [--server <url>] [--timeout <seconds>] [--json <path>] [--fail-on-changed]',
         '  cuttlely --help',
         '  cuttlely --version',
         '',
@@ -45,30 +52,96 @@ export function helpText() {
         '  status    Show whether Cuttlely is running.',
         '  logs      Show container logs. --follow keeps streaming.',
         '  upgrade   Pull the image and start it again.',
+        '  check     Run the Publish check on a flow draft. Reads CUTTLELY_API_KEY.',
+        '            The key needs chatflows:view and chatflows:update, or agentflows:view and agentflows:update.',
+        '',
+        'Exit codes for check:',
+        '  0  The check finished. No answer was Changed, or --fail-on-changed was off.',
+        '  1  --fail-on-changed was set and at least one answer was Changed.',
+        '  2  The check timed out, the server returned an error, CUTTLELY_API_KEY was missing, the flow has no draft, or the command was used wrong.',
+        'A check the server marks stopped or failed also exits 2. If the draft is saved, published, or discarded while the check runs, the command exits 2, because the answers may be for a draft that is already gone. After the check has started, a timeout, a 403, or any other failure asks the server to stop it. That stop is best effort: if it fails, the exit code is still 2.',
+        'Cancelling the command stops the server-side check.',
         '',
         `Install guide: ${INSTALL_DOCS}`
     ].join('\n')
 }
 
-function parsePort(value) {
+function parsePort(value, exitCode = 1) {
     if (!value || !/^[1-9]\d{0,4}$/.test(value) || Number(value) > 65535) {
-        throw new UsageError(`Port must be a number from 1 to 65535, got ${value || '(missing)'}.`)
+        throw new UsageError(`Port must be a number from 1 to 65535, got ${value || '(missing)'}.`, exitCode)
     }
     return Number(value)
 }
 
-function parseTag(value) {
+function parseTag(value, exitCode = 1) {
     if (!value || !TAG_RE.test(value)) {
-        throw new UsageError(`Not a valid image tag: ${value || '(missing)'}. Example: 1.0.0 or latest.`)
+        throw new UsageError(`Not a valid image tag: ${value || '(missing)'}. Example: 1.0.0 or latest.`, exitCode)
+    }
+    return value
+}
+
+// These options take the next argv token as their value. A following `check` is that value, not the command.
+// Any other previous token, including an unknown flag, leaves `check` as the command.
+const VALUE_OPTIONS = new Set(['--flow', '--server', '--timeout', '--json', '--port', '-p', '--tag', '--env-file'])
+
+export function isCheckCommand(argv) {
+    return argv.some((item, index) => {
+        if (item !== 'check') return false
+        if (index === 0) return true
+        return !VALUE_OPTIONS.has(argv[index - 1])
+    })
+}
+
+// An unexpected failure while running check is not a Changed answer.
+export function commandFailureCode(argv) {
+    return isCheckCommand(argv) ? 2 : 1
+}
+
+const FLOW_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/
+
+function parseFlow(value) {
+    if (!value || !FLOW_ID_RE.test(value)) {
+        throw new UsageError('--flow needs the flow id from the canvas address bar.', 2)
     }
     return value
 }
 
+function parseServer(value) {
+    let url
+    try {
+        url = new URL(value)
+    } catch {
+        throw new UsageError('--server needs an http or https URL, for example http://127.0.0.1:43117.', 2)
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        throw new UsageError('--server needs an http or https URL, for example http://127.0.0.1:43117.', 2)
+    }
+    if (url.username || url.password) {
+        throw new UsageError(`Put the API key in CUTTLELY_API_KEY, not in the server URL.`, 2)
+    }
+    if (url.pathname !== '/' && url.pathname !== '') {
+        throw new UsageError('--server is the server origin, for example http://127.0.0.1:43117.', 2)
+    }
+    return url.origin
+}
+
+function parseTimeout(value) {
+    if (!value || !/^[1-9]\d{0,4}$/.test(value) || Number(value) > 3600) {
+        throw new UsageError(`Timeout must be a whole number of seconds from 1 to 3600, got ${value || '(missing)'}.`, 2)
+    }
+    return Number(value)
+}
+
+function parseJsonPath(value) {
+    if (!value || value.includes('\0')) throw new UsageError('--json needs a file path.', 2)
+    return value
+}
+
 function takeValue(argv, index, flag) {
     const inline = argv[index].indexOf('=')
     if (inline !== -1) return { value: argv[index].slice(inline + 1), next: index }
     const value = argv[index + 1]
-    if (!value || value.startsWith('-')) throw new UsageError(`${flag} needs a value. Run cuttlely --help.`)
+    if (!value || value.startsWith('-')) throw new UsageError(`${flag} needs a value. Run cuttlely --help.`, commandFailureCode(argv))
     return { value, next: index + 1 }
 }
 
@@ -81,37 +154,99 @@ export function parseArgs(argv) {
         tag: VERSION,
         envFile: null,
         open: true,
-        follow: false
+        follow: false,
+        flow: null,
+        server: null,
+        timeoutSeconds: null,
+        jsonPath: null,
+        failOnChanged: false
     }
     const words = []
+    const dockerFlags = []
+    const checkFlags = []
     for (let i = 0; i < argv.length; i++) {
         const arg = argv[i]
         if (arg === '--help' || arg === '-h') opts.help = true
         else if (arg === '--version' || arg === '-V') opts.version = true
-        else if (arg === '--no-open') opts.open = false
-        else if (arg === '--follow' || arg === '-f') opts.follow = true
-        else if (arg === '--port' || arg === '-p' || arg.startsWith('--port=')) {
+        else if (arg === '--no-open') {
+            opts.open = false
+            dockerFlags.push('--no-open')
+        } else if (arg === '--follow' || arg === '-f') {
+            opts.follow = true
+            dockerFlags.push(arg === '-f' ? '-f' : '--follow')
+        } else if (arg === '--port' || arg === '-p' || arg.startsWith('--port=')) {
             const taken = takeValue(argv, i, '--port')
-            opts.port = parsePort(taken.value)
+            opts.port = parsePort(taken.value, commandFailureCode(argv))
             i = taken.next
+            dockerFlags.push('--port')
         } else if (arg === '--tag' || arg.startsWith('--tag=')) {
             const taken = takeValue(argv, i, '--tag')
-            opts.tag = parseTag(taken.value)
+            opts.tag = parseTag(taken.value, commandFailureCode(argv))
             i = taken.next
+            dockerFlags.push('--tag')
         } else if (arg === '--env-file' || arg.startsWith('--env-file=')) {
             const taken = takeValue(argv, i, '--env-file')
             opts.envFile = taken.value
             i = taken.next
-        } else if (arg.startsWith('-')) throw new UsageError(`Unknown option ${arg}. Run cuttlely --help.`)
-        else words.push(arg)
+            dockerFlags.push('--env-file')
+        } else if (arg === '--flow' || arg.startsWith('--flow=')) {
+            const taken = takeValue(argv, i, '--flow')
+            opts.flow = taken.value
+            i = taken.next
+            checkFlags.push('--flow')
+        } else if (arg === '--server' || arg.startsWith('--server=')) {
+            const taken = takeValue(argv, i, '--server')
+            opts.server = taken.value
+            i = taken.next
+            checkFlags.push('--server')
+        } else if (arg === '--timeout' || arg.startsWith('--timeout=')) {
+            const taken = takeValue(argv, i, '--timeout')
+            opts.timeoutSeconds = taken.value
+            i = taken.next
+            checkFlags.push('--timeout')
+        } else if (arg === '--json' || arg.startsWith('--json=')) {
+            const taken = takeValue(argv, i, '--json')
+            opts.jsonPath = taken.value
+            i = taken.next
+            checkFlags.push('--json')
+        } else if (arg === '--fail-on-changed') {
+            opts.failOnChanged = true
+            checkFlags.push('--fail-on-changed')
+        } else if (arg.startsWith('-')) {
+            throw new UsageError(`Unknown option ${arg}. Run cuttlely --help.`, commandFailureCode(argv))
+        } else words.push(arg)
     }
-    if (words.length > 1) throw new UsageError(`Unknown argument ${words[1]}. Run cuttlely --help.`)
+    if (words.length > 1) throw new UsageError(`Unknown argument ${words[1]}. Run cuttlely --help.`, words[0] === 'check' ? 2 : 1)
     if (words.length === 1) {
-        if (!['start', 'stop', 'status', 'logs', 'upgrade', 'help'].includes(words[0])) {
+        if (!['start', 'stop', 'status', 'logs', 'upgrade', 'help', 'check'].includes(words[0])) {
             throw new UsageError(`Unknown command ${words[0]}. Run cuttlely --help.`)
         }
         opts.command = words[0]
     }
+    if (opts.command === 'check') {
+        const checkExit = 2
+        if (dockerFlags.length) {
+            throw new UsageError(`${dockerFlags[0]} is not a check option. Run cuttlely check --help.`, checkExit)
+        }
+        if (!opts.help && !opts.version) {
+            if (!opts.flow) throw new UsageError('check needs --flow <id>. Run cuttlely check --help.', checkExit)
+            opts.flow = parseFlow(opts.flow)
+            if (checkFlags.includes('--server')) {
+                if (!opts.server)
+                    throw new UsageError('--server needs an http or https URL, for example http://127.0.0.1:43117.', checkExit)
+                opts.server = parseServer(opts.server)
+            }
+            if (checkFlags.includes('--timeout')) {
+                if (!opts.timeoutSeconds)
+                    throw new UsageError('Timeout must be a whole number of seconds from 1 to 3600, got (missing).', checkExit)
+                opts.timeoutSeconds = parseTimeout(opts.timeoutSeconds)
+            }
+            // An empty --json= is still the flag. Skipping it would run the check and write nothing.
+            if (checkFlags.includes('--json')) opts.jsonPath = parseJsonPath(opts.jsonPath)
+        }
+    } else if (checkFlags.length) {
+        throw new UsageError(`Unknown option ${checkFlags[0]}. Run cuttlely --help.`)
+    }
     return opts
 }
 
@@ -407,7 +542,7 @@ export async function main(argv, io = {}) {
     } catch (error) {
         if (error instanceof UsageError) {
             err(error.message)
-            return 1
+            return error.exitCode
         }
         throw error
     }
@@ -424,6 +559,9 @@ export async function main(argv, io = {}) {
         out,
         err,
         docker: io.docker || spawnDocker,
+        env: io.env || process.env,
+        fetch: io.fetch || globalThis.fetch.bind(globalThis),
+        writeFile: io.writeFile,
         fetchImpl: io.fetch || globalThis.fetch.bind(globalThis),
         sleep: io.sleep || ((ms) => new Promise((resolve) => setTimeout(resolve, ms))),
         open: io.open || openBrowser,
@@ -431,6 +569,18 @@ export async function main(argv, io = {}) {
         timeoutMs: io.healthTimeoutMs ?? HEALTH_TIMEOUT_MS
     }
     try {
+        if (opts.command === 'check') {
+            return await runCheck(
+                {
+                    flow: opts.flow,
+                    server: opts.server,
+                    timeoutSeconds: opts.timeoutSeconds,
+                    jsonPath: opts.jsonPath,
+                    failOnChanged: opts.failOnChanged
+                },
+                ctx
+            )
+        }
         if (opts.command === 'start') return await launch(ctx, { restart: false })
         if (opts.command === 'upgrade') return await launch(ctx, { restart: true })
         if (opts.command === 'stop') return await stop(ctx)
@@ -439,7 +589,7 @@ export async function main(argv, io = {}) {
     } catch (error) {
         if (error instanceof UsageError) {
             err(error.message)
-            return 1
+            return error.exitCode
         }
         throw error
     }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "cuttlely",
-    "version": "1.0.2",
+    "version": "1.0.3",
     "description": "Start Cuttlely in Docker. The open-source agent platform for applications.",
     "type": "module",
     "bin": {