@openclaw/crabline 0.1.9 → 0.1.10

package/dist/src/core/run.js

@@ -1,43 +1,186 @@
 import { matchesInbound } from "./matcher.js";
 import { createOutboundText } from "./message-template.js";
 import { createNonce } from "./nonces.js";
+import { inboundRegexSafetyError } from "./safe-regex.js";
 import { CrablineError, ensureErrorMessage } from "./errors.js";
+import { EXIT_CODES } from "./exit-codes.js";
+const INBOUND_DEADLINE_REACHED = Symbol("inbound deadline reached");
+const INBOUND_ABORT_GRACE_MS = 250;
+const MAX_EXCLUDED_INBOUND_IDS = 1_024;
+const MAX_SCRIPT_STDIN_BYTES = 1024 * 1024;
+class UnsettledProviderOperationError extends CrablineError {
+}
 function sleep(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
+async function raceInboundDeadline(operation, timeoutMs) {
+    let timer;
+    const deadline = new Promise((resolve) => {
+        timer = setTimeout(() => resolve(INBOUND_DEADLINE_REACHED), timeoutMs);
+    });
+    try {
+        return await Promise.race([operation, deadline]);
+    }
+    finally {
+        if (timer) {
+            clearTimeout(timer);
+        }
+    }
+}
+async function withFixtureDeadline(operation, timeoutMs, operationName) {
+    const controller = new AbortController();
+    const pending = Promise.resolve().then(async () => await operation(controller.signal));
+    const result = await raceInboundDeadline(pending, timeoutMs);
+    if (result === INBOUND_DEADLINE_REACHED) {
+        const timeoutError = new CrablineError(`Provider ${operationName} timed out after ${timeoutMs}ms.`, {
+            kind: "timeout",
+        });
+        controller.abort(timeoutError);
+        const settled = pending.then(() => true, () => true);
+        if ((await raceInboundDeadline(settled, INBOUND_ABORT_GRACE_MS)) === INBOUND_DEADLINE_REACHED) {
+            throw new UnsettledProviderOperationError(`Provider ${operationName} did not settle within ${INBOUND_ABORT_GRACE_MS}ms after abort.`, {
+                cause: timeoutError,
+                kind: "timeout",
+            });
+        }
+        throw timeoutError;
+    }
+    return result;
+}
+async function withCleanupDeadline(operation, timeoutMs, operationName) {
+    const result = await raceInboundDeadline(operation, timeoutMs);
+    if (result === INBOUND_DEADLINE_REACHED) {
+        throw new CrablineError(`Provider ${operationName} timed out after ${timeoutMs}ms.`, {
+            kind: "timeout",
+        });
+    }
+    return result;
+}
+function jsonStringByteLength(value, limit) {
+    let bytes = 2;
+    for (let index = 0; index < value.length && bytes <= limit; index += 1) {
+        const code = value.charCodeAt(index);
+        if (code === 0x22 || code === 0x5c || code === 0x08 || code === 0x09 || code === 0x0a) {
+            bytes += 2;
+        }
+        else if (code === 0x0c || code === 0x0d) {
+            bytes += 2;
+        }
+        else if (code <= 0x1f) {
+            bytes += 6;
+        }
+        else if (code >= 0xd800 && code <= 0xdbff) {
+            const next = value.charCodeAt(index + 1);
+            if (next >= 0xdc00 && next <= 0xdfff) {
+                bytes += 4;
+                index += 1;
+            }
+            else {
+                bytes += 6;
+            }
+        }
+        else if (code >= 0xdc00 && code <= 0xdfff) {
+            bytes += 6;
+        }
+        else if (code <= 0x7f) {
+            bytes += 1;
+        }
+        else if (code <= 0x7ff) {
+            bytes += 2;
+        }
+        else {
+            bytes += 3;
+        }
+    }
+    return bytes;
+}
+function boundedJsonByteLength(value, limit, ancestors = new Set(), arrayEntry = false) {
+    if (value === null || value === undefined || typeof value === "function") {
+        return value === undefined && !arrayEntry ? 0 : 4;
+    }
+    if (typeof value === "string") {
+        return jsonStringByteLength(value, limit);
+    }
+    if (typeof value === "number") {
+        return Number.isFinite(value) ? String(value).length : 4;
+    }
+    if (typeof value === "boolean") {
+        return value ? 4 : 5;
+    }
+    if (typeof value !== "object") {
+        return 0;
+    }
+    if (ancestors.has(value)) {
+        throw new CrablineError("Script command input must not contain circular values.", {
+            kind: "config",
+        });
+    }
+    ancestors.add(value);
+    let bytes = 2;
+    let entries = 0;
+    if (Array.isArray(value)) {
+        for (const entry of value) {
+            if (entries > 0) {
+                bytes += 1;
+            }
+            bytes += boundedJsonByteLength(entry, limit - bytes, ancestors, true);
+            entries += 1;
+            if (bytes > limit) {
+                break;
+            }
+        }
+    }
+    else {
+        for (const [key, entry] of Object.entries(value)) {
+            if (entry === undefined || typeof entry === "function") {
+                continue;
+            }
+            if (entries > 0) {
+                bytes += 1;
+            }
+            bytes += jsonStringByteLength(key, limit - bytes) + 1;
+            bytes += boundedJsonByteLength(entry, limit - bytes, ancestors);
+            entries += 1;
+            if (bytes > limit) {
+                break;
+            }
+        }
+    }
+    ancestors.delete(value);
+    return bytes;
+}
+export function assertScriptStdinPayloadSize(payload) {
+    if (boundedJsonByteLength(payload, MAX_SCRIPT_STDIN_BYTES) > MAX_SCRIPT_STDIN_BYTES) {
+        throw new CrablineError(`Script command input exceeded ${MAX_SCRIPT_STDIN_BYTES} bytes.`, {
+            kind: "config",
+        });
+    }
+}
+function scriptPayloadBase(context) {
+    return {
+        fixture: context.fixture,
+        provider: {
+            config: context.config,
+            id: context.providerId,
+            manifestPath: context.manifestPath,
+        },
+    };
+}
+async function abortAndDrainInboundWait(operation, controller) {
+    controller.abort(new Error("Inbound wait deadline reached."));
+    const settled = operation.then(() => true, () => true);
+    return (await raceInboundDeadline(settled, INBOUND_ABORT_GRACE_MS)) !== INBOUND_DEADLINE_REACHED;
+}
 export async function runFixtureCommand(params) {
-    const fixture = params.manifest.fixtures.find((entry) => entry.id === params.fixtureId);
-    if (!fixture) {
+    const configuredFixture = params.manifest.fixtures.find((entry) => entry.id === params.fixtureId);
+    if (!configuredFixture) {
         throw new CrablineError(`Unknown fixture: ${params.fixtureId}`, { kind: "config" });
     }
+    const mode = params.modeOverride ?? configuredFixture.mode;
+    const fixture = mode === configuredFixture.mode ? configuredFixture : { ...configuredFixture, mode };
     const provider = params.registry.resolve(fixture.provider, fixture.id);
-    const mode = params.modeOverride ?? fixture.mode;
     const diagnostics = [];
-    if (!provider.supports.includes(mode)) {
-        return {
-            diagnostics: [`provider ${fixture.provider} does not support mode ${mode}`],
-            failureKind: "config",
-            fixtureId: fixture.id,
-            mode,
-            ok: false,
-            providerId: fixture.provider,
-        };
-    }
-    for (const envName of [
-        ...fixture.env,
-        ...(params.manifest.providers[fixture.provider]?.env ?? []),
-    ]) {
-        if (!process.env[envName]) {
-            return {
-                diagnostics: [`missing env: ${envName}`],
-                failureKind: "config",
-                fixtureId: fixture.id,
-                mode,
-                ok: false,
-                providerId: fixture.provider,
-            };
-        }
-    }
+    let abortDrainFailed = false;
     const contextBase = {
         config: params.manifest.providers[fixture.provider],
         fixture,
@@ -47,9 +190,43 @@ export async function runFixtureCommand(params) {
     };
     let result;
     try {
-        if (mode === "probe") {
+        if (!provider.supports.includes(mode)) {
+            result = {
+                diagnostics: [`provider ${fixture.provider} does not support mode ${mode}`],
+                failureKind: "config",
+                fixtureId: fixture.id,
+                mode,
+                ok: false,
+                providerId: fixture.provider,
+            };
+        }
+        if (!result) {
+            for (const envName of [
+                ...fixture.env,
+                ...(params.manifest.providers[fixture.provider]?.env ?? []),
+            ]) {
+                if (!process.env[envName]) {
+                    result = {
+                        diagnostics: [`missing env: ${envName}`],
+                        failureKind: "config",
+                        fixtureId: fixture.id,
+                        mode,
+                        ok: false,
+                        providerId: fixture.provider,
+                    };
+                    break;
+                }
+            }
+        }
+        if (result) {
+            // Preflight failures still flow through provider cleanup before returning.
+        }
+        else if (mode === "probe") {
             try {
-                const probeResult = await provider.probe(contextBase);
+                if (contextBase.config.adapter === "script") {
+                    assertScriptStdinPayloadSize(scriptPayloadBase(contextBase));
+                }
+                const probeResult = await withFixtureDeadline(async (signal) => await provider.probe({ ...contextBase, signal }), fixture.timeoutMs, "probe");
                 return (result = {
                     diagnostics: probeResult.details,
                     failureKind: probeResult.healthy ? undefined : "connectivity",
@@ -60,28 +237,171 @@ export async function runFixtureCommand(params) {
                 });
             }
             catch (error) {
-                return (result = toFailure(fixture.id, fixture.provider, mode, error));
+                if (error instanceof UnsettledProviderOperationError) {
+                    abortDrainFailed = true;
+                }
+                return (result = toFailure(fixture.id, fixture.provider, mode, error, "connectivity"));
             }
         }
-        let attempts = 0;
-        const maxAttempts = fixture.retries + 1;
-        let lastFailure = null;
-        while (attempts < maxAttempts) {
-            attempts += 1;
-            const nonce = createNonce(fixture.id);
-            try {
-                const outboundText = createOutboundText({ ...fixture, mode }, nonce);
-                const since = new Date().toISOString();
-                const accepted = await provider.send({
-                    ...contextBase,
-                    mode,
-                    nonce,
-                    text: outboundText,
-                });
-                diagnostics.push(`accepted message ${accepted.messageId}`);
-                if (mode === "send") {
+        else {
+            if (fixture.inboundMatch.strategy === "regex" && fixture.inboundMatch.pattern) {
+                try {
+                    RegExp(fixture.inboundMatch.pattern, "u");
+                    const safetyError = inboundRegexSafetyError(fixture.inboundMatch.pattern);
+                    if (safetyError) {
+                        throw new Error(safetyError);
+                    }
+                }
+                catch (error) {
+                    return (result = toFailure(fixture.id, fixture.provider, mode, new CrablineError(`Invalid inbound regex: ${ensureErrorMessage(error)}`, {
+                        cause: error,
+                        kind: "config",
+                    }), "config"));
+                }
+            }
+            let attempts = 0;
+            const maxAttempts = fixture.retries + 1;
+            let lastFailure = null;
+            while (attempts < maxAttempts) {
+                attempts += 1;
+                const nonce = createNonce(fixture.id);
+                try {
+                    const outboundText = createOutboundText({ ...fixture, mode }, nonce);
+                    const since = new Date().toISOString();
+                    let accepted;
+                    try {
+                        const sendContext = {
+                            ...contextBase,
+                            mode,
+                            nonce,
+                            text: outboundText,
+                        };
+                        if (contextBase.config.adapter === "script") {
+                            assertScriptStdinPayloadSize({
+                                ...scriptPayloadBase(contextBase),
+                                outbound: {
+                                    mode,
+                                    nonce,
+                                    target: fixture.target,
+                                    text: outboundText,
+                                },
+                            });
+                        }
+                        accepted = await withFixtureDeadline(async (signal) => await provider.send({ ...sendContext, signal }), fixture.timeoutMs, "send");
+                    }
+                    catch (error) {
+                        lastFailure = toFailure(fixture.id, fixture.provider, mode, error, "outbound", nonce);
+                        if (error instanceof UnsettledProviderOperationError) {
+                            abortDrainFailed = true;
+                            break;
+                        }
+                        continue;
+                    }
+                    if (!accepted.accepted) {
+                        lastFailure = toFailure(fixture.id, fixture.provider, mode, new CrablineError(`Provider rejected outbound message ${accepted.messageId}.`, {
+                            kind: "outbound",
+                        }), "outbound", nonce);
+                        continue;
+                    }
+                    diagnostics.push(`accepted message ${accepted.messageId}`);
+                    if (mode === "send") {
+                        return (result = {
+                            diagnostics,
+                            fixtureId: fixture.id,
+                            mode,
+                            nonce,
+                            ok: true,
+                            providerId: fixture.provider,
+                        });
+                    }
+                    const inboundDeadline = Date.now() + fixture.timeoutMs;
+                    const seenInbound = new Set();
+                    const excludedInboundIds = new Set();
+                    let inbound;
+                    try {
+                        while (Date.now() < inboundDeadline) {
+                            const timeoutMs = inboundDeadline - Date.now();
+                            const controller = new AbortController();
+                            const excludeIds = [...excludedInboundIds];
+                            const waitContext = {
+                                ...contextBase,
+                                excludeIds,
+                                nonce,
+                                signal: controller.signal,
+                                since,
+                                threadId: accepted.threadId,
+                                timeoutMs,
+                            };
+                            if (contextBase.config.adapter === "script") {
+                                assertScriptStdinPayloadSize({
+                                    ...scriptPayloadBase(contextBase),
+                                    wait: {
+                                        excludeIds,
+                                        nonce,
+                                        since,
+                                        target: fixture.target,
+                                        threadId: accepted.threadId,
+                                        timeoutMs,
+                                    },
+                                });
+                            }
+                            const wait = provider.waitForInbound(waitContext);
+                            const candidate = await raceInboundDeadline(wait, timeoutMs);
+                            if (candidate === INBOUND_DEADLINE_REACHED) {
+                                if (!(await abortAndDrainInboundWait(wait, controller))) {
+                                    abortDrainFailed = true;
+                                    throw new UnsettledProviderOperationError(`Provider inbound wait did not settle within ${INBOUND_ABORT_GRACE_MS}ms after abort.`, { kind: "inbound" });
+                                }
+                                break;
+                            }
+                            if (!candidate) {
+                                break;
+                            }
+                            const key = JSON.stringify([candidate.provider, candidate.threadId, candidate.id]);
+                            if (seenInbound.has(key)) {
+                                await sleep(Math.min(10, Math.max(0, inboundDeadline - Date.now())));
+                                continue;
+                            }
+                            seenInbound.add(key);
+                            if (matchesInbound(candidate, fixture.inboundMatch, nonce)) {
+                                inbound = candidate;
+                                break;
+                            }
+                            if (!excludedInboundIds.has(candidate.id)) {
+                                if (excludedInboundIds.size >= MAX_EXCLUDED_INBOUND_IDS) {
+                                    throw new CrablineError(`Provider returned more than ${MAX_EXCLUDED_INBOUND_IDS} unmatched inbound message IDs.`, { kind: "inbound" });
+                                }
+                                excludedInboundIds.add(candidate.id);
+                            }
+                        }
+                    }
+                    catch (error) {
+                        lastFailure = toFailure(fixture.id, fixture.provider, mode, error, "inbound", nonce);
+                        if (abortDrainFailed) {
+                            break;
+                        }
+                        continue;
+                    }
+                    if (!inbound) {
+                        lastFailure = {
+                            diagnostics: [
+                                ...diagnostics,
+                                `timed out waiting for inbound after ${fixture.timeoutMs}ms`,
+                            ],
+                            failureKind: "timeout",
+                            fixtureId: fixture.id,
+                            mode,
+                            nonce,
+                            ok: false,
+                            providerId: fixture.provider,
+                        };
+                        if (attempts < maxAttempts) {
+                            await sleep(50);
+                        }
+                        continue;
+                    }
                     return (result = {
-                        diagnostics,
+                        diagnostics: [...diagnostics, `matched inbound ${inbound.id}`],
                         fixtureId: fixture.id,
                         mode,
                         nonce,
@@ -89,76 +409,60 @@ export async function runFixtureCommand(params) {
                         providerId: fixture.provider,
                     });
                 }
-                const inbound = await provider.waitForInbound({
-                    ...contextBase,
-                    nonce,
-                    since,
-                    threadId: accepted.threadId,
-                    timeoutMs: fixture.timeoutMs,
-                });
-                if (!inbound) {
-                    lastFailure = {
-                        diagnostics: [
-                            ...diagnostics,
-                            `timed out waiting for inbound after ${fixture.timeoutMs}ms`,
-                        ],
-                        failureKind: "timeout",
-                        fixtureId: fixture.id,
-                        mode,
-                        nonce,
-                        ok: false,
-                        providerId: fixture.provider,
-                    };
-                    await sleep(50);
-                    continue;
-                }
-                if (!matchesInbound(inbound, fixture.inboundMatch, nonce)) {
-                    lastFailure = {
-                        diagnostics: [...diagnostics, `received non-matching inbound message ${inbound.id}`],
-                        failureKind: "assertion",
-                        fixtureId: fixture.id,
-                        mode,
-                        nonce,
-                        ok: false,
-                        providerId: fixture.provider,
-                    };
-                    await sleep(50);
-                    continue;
+                catch (error) {
+                    lastFailure = toFailure(fixture.id, fixture.provider, mode, error, "assertion", nonce);
                 }
-                return (result = {
-                    diagnostics: [...diagnostics, `matched inbound ${inbound.id}`],
-                    fixtureId: fixture.id,
-                    mode,
-                    nonce,
-                    ok: true,
-                    providerId: fixture.provider,
-                });
-            }
-            catch (error) {
-                lastFailure = toFailure(fixture.id, fixture.provider, mode, error, nonce);
             }
+            return (result = lastFailure ?? {
+                diagnostics: ["unknown failure"],
+                failureKind: "assertion",
+                fixtureId: fixture.id,
+                mode,
+                ok: false,
+                providerId: fixture.provider,
+            });
         }
-        return (result = lastFailure ?? {
-            diagnostics: ["unknown failure"],
-            failureKind: "assertion",
-            fixtureId: fixture.id,
-            mode,
-            ok: false,
-            providerId: fixture.provider,
-        });
     }
     finally {
+        const cleanupErrors = [];
+        try {
+            provider.beginCleanup?.();
+        }
+        catch (error) {
+            cleanupErrors.push(error);
+        }
         try {
-            await provider.cleanup?.();
+            const cleanup = provider.cleanup?.();
+            if (cleanup) {
+                if (abortDrainFailed &&
+                    (await raceInboundDeadline(cleanup, INBOUND_ABORT_GRACE_MS)) === INBOUND_DEADLINE_REACHED) {
+                    cleanupErrors.push(new Error(`Provider cleanup did not settle within ${INBOUND_ABORT_GRACE_MS}ms after an aborted operation.`));
+                }
+                if (!abortDrainFailed) {
+                    await withCleanupDeadline(cleanup, fixture.timeoutMs, "cleanup");
+                }
+            }
         }
         catch (error) {
-            const diagnostic = `cleanup failed: ${ensureErrorMessage(error)}`;
+            cleanupErrors.push(error);
+        }
+        if (cleanupErrors.length > 0) {
+            const cleanupError = cleanupErrors.length === 1
+                ? cleanupErrors[0]
+                : new AggregateError(cleanupErrors, cleanupErrors.map((error) => ensureErrorMessage(error)).join("; "));
+            const diagnostic = `cleanup failed: ${ensureErrorMessage(cleanupError)}`;
             if (result) {
                 result.diagnostics.push(diagnostic);
+                if (result.ok) {
+                    result.exitCode = EXIT_CODES.ASSERTION;
+                    result.failureKind = "assertion";
+                    result.ok = false;
+                }
             }
             else {
                 result = {
                     diagnostics: [...diagnostics, diagnostic],
+                    exitCode: EXIT_CODES.ASSERTION,
                     failureKind: "assertion",
                     fixtureId: fixture.id,
                     mode,
@@ -195,6 +499,9 @@ export function computeExitCode(result) {
     if (result.ok) {
         return 0;
     }
+    if (result.exitCode !== undefined) {
+        return result.exitCode;
+    }
     switch (result.failureKind) {
         case "auth":
             return 11;
@@ -214,11 +521,12 @@ export function computeExitCode(result) {
             return 1;
     }
 }
-function toFailure(fixtureId, providerId, mode, error, nonce) {
+function toFailure(fixtureId, providerId, mode, error, fallbackKind, nonce) {
     const diagnostics = [ensureErrorMessage(error)];
     if (error instanceof CrablineError) {
         return {
             diagnostics,
+            exitCode: error.exitCode,
             failureKind: error.kind,
             fixtureId,
             mode,
@@ -229,7 +537,7 @@ function toFailure(fixtureId, providerId, mode, error, nonce) {
     }
     return {
         diagnostics,
-        failureKind: "assertion",
+        failureKind: fallbackKind,
         fixtureId,
         mode,
         nonce,