@workbench-ai/workbench-core 0.0.46

package/dist/generic-spec.js

@@ -0,0 +1,656 @@
+import { createHash } from "node:crypto";
+import YAML from "yaml";
+export const BENCHMARK_SPEC_FILE = "benchmark.yaml";
+export const DEFAULT_EXECUTION_RESOURCES = {
+    cpu: 2,
+    memoryGb: 4,
+    diskGb: 10,
+    timeoutMinutes: 20,
+};
+export function validateWorkbenchResolvedSourceYaml(source) {
+    const errors = [];
+    const warnings = [];
+    const trimmed = source.trim();
+    if (!trimmed) {
+        errors.push("Resolved Workbench source cannot be empty.");
+    }
+    if (trimmed) {
+        try {
+            resolveWorkbenchResolvedSourceYaml(source);
+        }
+        catch (error) {
+            errors.push(...splitErrorMessage(error));
+        }
+    }
+    return {
+        ok: errors.length === 0,
+        errors,
+        warnings,
+    };
+}
+export function resolveWorkbenchResolvedSourceYaml(source) {
+    const parsed = parseYamlRecord(source, "resolved Workbench source");
+    const errors = [];
+    rejectUnknownKeys(parsed, "resolved Workbench source", [
+        "version",
+        "benchmark",
+        "subject",
+        "optimizer",
+    ], errors);
+    if (parsed.version !== 3) {
+        throw new Error("Resolved Workbench source version must be 3.");
+    }
+    const benchmark = normalizeBenchmarkRecord(readRequiredRecord(parsed.benchmark, "resolved Workbench source.benchmark", errors), "benchmark.yaml", errors);
+    const subject = normalizeSubjectRecord(readRequiredRecord(parsed.subject, "resolved Workbench source.subject", errors), "resolved Workbench source.subject", errors);
+    const optimizer = parsed.optimizer === undefined
+        ? undefined
+        : normalizeOptimizerRecord(readRequiredRecord(parsed.optimizer, "resolved Workbench source.optimizer", errors), "optimizer YAML", errors);
+    if (errors.length > 0) {
+        throw new Error(errors.join("\n"));
+    }
+    return genericSpecFromAuthoredBundle({
+        version: 3,
+        benchmark: benchmark,
+        subject: subject,
+        ...(optimizer ? { optimizer } : {}),
+    });
+}
+export function engineResolveBindingForSourceYaml(source) {
+    return engineResolveBindingForSpec(resolveWorkbenchResolvedSourceYaml(source));
+}
+export function engineResolveBindingForSpec(spec) {
+    const resolver = engineResolveInvocationForSpec(spec);
+    return {
+        engine: spec.benchmark.engine.use,
+        resolver: {
+            use: resolver.use,
+            withFingerprint: fingerprintJson(resolver.with ?? {}),
+        },
+    };
+}
+export function resolveWorkbenchSourceFiles(args) {
+    return genericSpecFromAuthoredBundle(parseWorkbenchSourceFiles({
+        benchmarkSource: args.benchmarkSource,
+        subjectSource: args.subjectSource,
+        optimizerSource: args.optimizerSource,
+    }));
+}
+export function parseWorkbenchSourceFiles(args) {
+    const errors = [];
+    const benchmark = normalizeBenchmarkRecord(parseYamlRecord(args.benchmarkSource, BENCHMARK_SPEC_FILE), BENCHMARK_SPEC_FILE, errors);
+    const subject = normalizeSubjectRecord(parseYamlRecord(args.subjectSource ?? "", "subject YAML"), "subject YAML", errors);
+    const optimizer = args.optimizerSource?.trim()
+        ? normalizeOptimizerRecord(parseYamlRecord(args.optimizerSource, "optimizer YAML"), "optimizer YAML", errors)
+        : undefined;
+    if (errors.length > 0) {
+        throw new Error(errors.join("\n"));
+    }
+    return {
+        version: 3,
+        benchmark: benchmark,
+        subject: subject,
+        ...(optimizer ? { optimizer } : {}),
+    };
+}
+export function serializeWorkbenchResolvedSourceYaml(source) {
+    return YAML.stringify(source).trimEnd() + "\n";
+}
+export function isWorkbenchSubjectManifestPath(filePath) {
+    return /^subjects\/[^/]+\/subject\.ya?ml$/iu.test(filePath.replace(/\\/gu, "/").replace(/^\/+/u, "").replace(/^(?:\.\/)+/u, ""));
+}
+export function resolveEngineCaseExecutionConfig(args) {
+    return {
+        prompt: args.engineCase.prompt,
+        environment: mergeRuntime(args.spec.environment, args.engineCase.environment),
+        run: args.spec.run,
+    };
+}
+export function engineResolveInvocationForSpec(spec) {
+    return spec.engineResolve;
+}
+export function engineCaseFilesForRuntimeInput(args) {
+    void args.spec;
+    return engineCaseSubjectVisibleFiles(args.engineCase);
+}
+export function engineCaseSubjectVisibleFiles(engineCase) {
+    return (engineCase.files.subjectVisible ?? [])
+        .map((file) => ({ ...file }))
+        .sort((left, right) => left.path.localeCompare(right.path));
+}
+export function engineCaseEnginePrivateFiles(engineCase) {
+    return (engineCase.files.enginePrivate ?? [])
+        .map((file) => ({ ...file }))
+        .sort((left, right) => left.path.localeCompare(right.path));
+}
+export function runtimeResources(runtime) {
+    const resources = runtime.resources ?? {};
+    return {
+        cpu: readPositiveNumber(resources.cpu, DEFAULT_EXECUTION_RESOURCES.cpu),
+        memoryGb: readPositiveNumber(resources.memoryGb, DEFAULT_EXECUTION_RESOURCES.memoryGb),
+        diskGb: readPositiveNumber(resources.diskGb, DEFAULT_EXECUTION_RESOURCES.diskGb),
+        timeoutMinutes: readPositiveNumber(resources.timeoutMinutes, DEFAULT_EXECUTION_RESOURCES.timeoutMinutes),
+    };
+}
+export function runtimeNetwork(runtime) {
+    return runtime.network ?? { egress: "open" };
+}
+export function runtimeSandboxRef(runtime) {
+    return `dockerfile://${runtime.dockerfile}`;
+}
+function genericSpecFromAuthoredBundle(source) {
+    const engineRuntime = engineRuntimeFromConfig(source.benchmark.engine);
+    const engineRun = cloneEngineInvocation(source.benchmark.engine);
+    const engineResolve = cloneEngineInvocation(source.benchmark.engine);
+    return {
+        version: 3,
+        name: source.benchmark.name,
+        description: source.benchmark.description,
+        benchmark: {
+            name: source.benchmark.name,
+            description: source.benchmark.description,
+            engine: cloneJson(source.benchmark.engine),
+        },
+        subject: {
+            name: source.subject.name,
+            ...(source.subject.description ? { description: source.subject.description } : {}),
+            files: cloneJson(source.subject.files),
+        },
+        ...(source.optimizer
+            ? {
+                optimizer: {
+                    name: source.optimizer.name,
+                    ...(source.optimizer.description ? { description: source.optimizer.description } : {}),
+                    edits: [...source.optimizer.edits],
+                },
+            }
+            : {}),
+        environment: cloneJson(engineRuntime),
+        adapters: [
+            ...new Set([
+                ...source.benchmark.adapters,
+                ...source.subject.adapters,
+                ...(source.optimizer?.adapters ?? []),
+            ]),
+        ],
+        engine: cloneJson(source.benchmark.engine),
+        engineResolve: cloneJson(engineResolve),
+        ...(source.optimizer ? { improve: cloneJson(source.optimizer.improve) } : {}),
+        run: cloneJson(source.subject.run),
+        engineRun: cloneJson(engineRun),
+    };
+}
+function normalizeBenchmarkRecord(record, label, errors) {
+    if (!record) {
+        return null;
+    }
+    rejectUnknownKeys(record, label, [
+        "version",
+        "name",
+        "description",
+        "adapters",
+        "engine",
+    ], errors);
+    requireVersionThree(record.version, label, errors);
+    const name = readRequiredString(record.name, `${label}.name`, errors);
+    const description = readRequiredString(record.description, `${label}.description`, errors);
+    const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
+    const engine = normalizePhaseAdapter(record.engine, `${label}.engine`, errors);
+    if (engine) {
+        normalizeEngineRuntimeConfig(engine, `${label}.engine.with`, errors);
+    }
+    return name && description && engine
+        ? {
+            version: 3,
+            name,
+            description,
+            adapters,
+            engine,
+        }
+        : null;
+}
+function normalizeEngineRuntimeConfig(engine, label, errors) {
+    const config = engine.with && typeof engine.with === "object" && !Array.isArray(engine.with)
+        ? engine.with
+        : {};
+    if (config.environment !== undefined) {
+        const environment = normalizeRuntime(config.environment, `${label}.environment`, errors);
+        if (environment) {
+            config.environment = environment;
+            engine.with = config;
+        }
+    }
+}
+function normalizeSubjectRecord(record, label, errors) {
+    if (!record) {
+        return null;
+    }
+    rejectUnknownKeys(record, label, [
+        "version",
+        "name",
+        "description",
+        "files",
+        "adapters",
+        "run",
+    ], errors);
+    requireVersionThree(record.version, label, errors);
+    const name = readRequiredString(record.name, `${label}.name`, errors);
+    const description = readOptionalString(record.description, `${label}.description`, errors);
+    const files = normalizePathRef(record.files, `${label}.files`, errors);
+    const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
+    const run = normalizePhaseAdapter(record.run, `${label}.run`, errors);
+    return name && files && run
+        ? {
+            version: 3,
+            name,
+            ...(description ? { description } : {}),
+            files,
+            adapters,
+            run,
+        }
+        : null;
+}
+function normalizeOptimizerRecord(record, label, errors) {
+    if (!record) {
+        return null;
+    }
+    rejectUnknownKeys(record, label, [
+        "version",
+        "name",
+        "description",
+        "edits",
+        "adapters",
+        "improve",
+    ], errors);
+    requireVersionThree(record.version, label, errors);
+    const name = readRequiredString(record.name, `${label}.name`, errors);
+    const description = readOptionalString(record.description, `${label}.description`, errors);
+    const edits = normalizeRelativePathList(record.edits, `${label}.edits`, errors);
+    const adapters = normalizeAdapterSources(record.adapters, `${label}.adapters`, errors);
+    const improve = normalizePhaseAdapter(record.improve, `${label}.improve`, errors);
+    return name && edits.length > 0 && improve
+        ? {
+            version: 3,
+            name,
+            ...(description ? { description } : {}),
+            edits,
+            adapters,
+            improve,
+        }
+        : null;
+}
+function requireVersionThree(value, label, errors) {
+    if (value !== 3) {
+        errors.push(`${label}.version must be 3.`);
+    }
+}
+function normalizeRuntime(value, label, errors) {
+    const record = readRequiredRecord(value, label, errors);
+    if (!record) {
+        return null;
+    }
+    rejectUnknownKeys(record, label, ["dockerfile", "workdir", "resources", "network"], errors);
+    const dockerfile = normalizeWorkspaceLiteralPath(record.dockerfile, `${label}.dockerfile`, errors);
+    const runtime = {
+        dockerfile: dockerfile ?? "environment/Dockerfile",
+    };
+    const workdir = readOptionalString(record.workdir, `${label}.workdir`, errors);
+    if (workdir) {
+        runtime.workdir = workdir;
+    }
+    if (record.resources !== undefined) {
+        const resources = readRequiredRecord(record.resources, `${label}.resources`, errors);
+        if (resources) {
+            rejectUnknownKeys(resources, `${label}.resources`, [
+                "cpu",
+                "memoryGb",
+                "diskGb",
+                "timeoutMinutes",
+            ], errors);
+            readOptionalPositiveNumber(resources.cpu, `${label}.resources.cpu`, errors);
+            readOptionalPositiveNumber(resources.memoryGb, `${label}.resources.memoryGb`, errors);
+            readOptionalPositiveNumber(resources.diskGb, `${label}.resources.diskGb`, errors);
+            readOptionalPositiveNumber(resources.timeoutMinutes, `${label}.resources.timeoutMinutes`, errors);
+            runtime.resources = resources;
+        }
+    }
+    if (record.network !== undefined) {
+        const network = readRequiredRecord(record.network, `${label}.network`, errors);
+        if (network) {
+            const normalized = normalizeNetworkConfig(network, `${label}.network`, errors);
+            if (normalized) {
+                runtime.network = normalized;
+            }
+        }
+    }
+    return runtime;
+}
+function normalizeRuntimeOverride(value, label, errors) {
+    const record = readRequiredRecord(value, label, errors);
+    if (!record) {
+        return null;
+    }
+    rejectUnknownKeys(record, label, ["dockerfile", "workdir", "resources", "network"], errors);
+    const runtime = {};
+    if (record.dockerfile !== undefined) {
+        const dockerfile = normalizeWorkspaceLiteralPath(record.dockerfile, `${label}.dockerfile`, errors);
+        if (dockerfile) {
+            runtime.dockerfile = dockerfile;
+        }
+    }
+    const workdir = readOptionalString(record.workdir, `${label}.workdir`, errors);
+    if (workdir) {
+        runtime.workdir = workdir;
+    }
+    if (record.resources !== undefined) {
+        const resources = readRequiredRecord(record.resources, `${label}.resources`, errors);
+        if (resources) {
+            rejectUnknownKeys(resources, `${label}.resources`, [
+                "cpu",
+                "memoryGb",
+                "diskGb",
+                "timeoutMinutes",
+            ], errors);
+            readOptionalPositiveNumber(resources.cpu, `${label}.resources.cpu`, errors);
+            readOptionalPositiveNumber(resources.memoryGb, `${label}.resources.memoryGb`, errors);
+            readOptionalPositiveNumber(resources.diskGb, `${label}.resources.diskGb`, errors);
+            readOptionalPositiveNumber(resources.timeoutMinutes, `${label}.resources.timeoutMinutes`, errors);
+            runtime.resources = resources;
+        }
+    }
+    if (record.network !== undefined) {
+        const network = readRequiredRecord(record.network, `${label}.network`, errors);
+        if (network) {
+            const normalized = normalizeNetworkConfig(network, `${label}.network`, errors);
+            if (normalized) {
+                runtime.network = normalized;
+            }
+        }
+    }
+    return Object.keys(runtime).length > 0 ? runtime : null;
+}
+function engineRuntimeFromConfig(engine) {
+    const config = engine.with && typeof engine.with === "object" && !Array.isArray(engine.with)
+        ? engine.with
+        : {};
+    const environment = config.environment;
+    if (environment && typeof environment === "object" && !Array.isArray(environment)) {
+        const record = environment;
+        return {
+            dockerfile: typeof record.dockerfile === "string" && record.dockerfile.trim()
+                ? record.dockerfile
+                : "environment/Dockerfile",
+            ...(typeof record.workdir === "string" && record.workdir.trim() ? { workdir: record.workdir } : {}),
+            ...(record.resources && typeof record.resources === "object" && !Array.isArray(record.resources)
+                ? { resources: record.resources }
+                : {}),
+            ...(record.network && typeof record.network === "object" && !Array.isArray(record.network)
+                ? { network: record.network }
+                : {}),
+        };
+    }
+    return {
+        dockerfile: "environment/Dockerfile",
+        network: { egress: "open" },
+        resources: {
+            cpu: DEFAULT_EXECUTION_RESOURCES.cpu,
+            memoryGb: DEFAULT_EXECUTION_RESOURCES.memoryGb,
+            diskGb: DEFAULT_EXECUTION_RESOURCES.diskGb,
+            timeoutMinutes: DEFAULT_EXECUTION_RESOURCES.timeoutMinutes,
+        },
+    };
+}
+function cloneEngineInvocation(engine) {
+    return {
+        use: engine.use,
+        with: cloneJson(engine.with ?? {}),
+        ...(engine.auth !== undefined ? { auth: cloneJson(engine.auth) } : {}),
+    };
+}
+function mergeRuntime(base, override) {
+    if (!override) {
+        return cloneJson(base);
+    }
+    return {
+        ...cloneJson(base),
+        ...cloneJson(override),
+        resources: {
+            ...(base.resources ?? {}),
+            ...(override.resources ?? {}),
+        },
+        network: override.network ?? base.network,
+    };
+}
+function normalizeAdapterSources(value, label, errors) {
+    if (value === undefined) {
+        return [];
+    }
+    if (!Array.isArray(value)) {
+        errors.push(`${label} must be an array of adapter sources.`);
+        return [];
+    }
+    const sources = value.flatMap((entry, index) => {
+        if (typeof entry !== "string" || entry.trim().length === 0) {
+            errors.push(`${label}[${index}] must be a non-empty string.`);
+            return [];
+        }
+        return [entry.trim()];
+    });
+    return [...new Set(sources)];
+}
+function normalizeNetworkConfig(network, label, errors) {
+    rejectUnknownKeys(network, label, ["egress", "allow"], errors);
+    const egress = readOptionalString(network.egress, `${label}.egress`, errors) ?? "open";
+    if (egress !== "none" && egress !== "open" && egress !== "allowlist") {
+        errors.push(`${label}.egress must be none, open, or allowlist.`);
+        return null;
+    }
+    const allow = network.allow === undefined
+        ? undefined
+        : normalizeNetworkAllowList(network.allow, `${label}.allow`, errors);
+    if (egress !== "allowlist") {
+        if (network.allow !== undefined) {
+            errors.push(`${label}.allow is only supported when ${label}.egress is allowlist.`);
+        }
+        return { egress };
+    }
+    if (!allow || allow.length === 0) {
+        errors.push(`${label}.allow must contain at least one host when ${label}.egress is allowlist.`);
+    }
+    return {
+        egress,
+        ...(allow && allow.length > 0 ? { allow } : {}),
+    };
+}
+function normalizeNetworkAllowList(value, label, errors) {
+    if (!Array.isArray(value)) {
+        errors.push(`${label} must be an array of hosts.`);
+        return [];
+    }
+    return value.flatMap((entry, index) => {
+        if (typeof entry !== "string" || entry.trim().length === 0) {
+            errors.push(`${label}[${index}] must be a non-empty string.`);
+            return [];
+        }
+        return [entry.trim()];
+    });
+}
+function normalizePhaseAdapter(value, label, errors) {
+    const spec = readAdapterRecord(value, label, errors);
+    if (!spec) {
+        return null;
+    }
+    return {
+        use: spec.use,
+        with: readJsonRecord(spec.with ?? {}),
+        ...(spec.auth !== undefined ? { auth: spec.auth } : {}),
+    };
+}
+function normalizeWorkspaceLiteralPath(value, label, errors) {
+    const raw = readRequiredString(value, label, errors);
+    if (!raw) {
+        return null;
+    }
+    return normalizeLiteralPathString(raw, label, errors);
+}
+function normalizePathRef(value, label, errors) {
+    const record = readRequiredRecord(value, label, errors);
+    if (!record) {
+        return null;
+    }
+    rejectUnknownKeys(record, label, ["path"], errors);
+    const refPath = normalizeWorkspaceLiteralPath(record.path, `${label}.path`, errors);
+    return refPath ? { path: refPath } : null;
+}
+function normalizeRelativePathList(value, label, errors) {
+    if (!Array.isArray(value) || value.length === 0) {
+        errors.push(`${label} must include at least one path.`);
+        return [];
+    }
+    const paths = value.flatMap((entry, index) => {
+        if (typeof entry !== "string" || entry.trim().length === 0) {
+            errors.push(`${label}[${index}] must be a non-empty string.`);
+            return [];
+        }
+        const normalized = normalizeLiteralPathString(entry, `${label}[${index}]`, errors);
+        return normalized ? [normalized] : [];
+    });
+    return [...new Set(paths)];
+}
+function normalizeLiteralPathString(value, label, errors) {
+    const trimmed = value.trim();
+    if (/^(?:\/|[A-Za-z]:[\\/])/u.test(trimmed)) {
+        errors.push(`${label} must be a relative path, not an absolute path.`);
+        return null;
+    }
+    const normalized = trimmed.replace(/\\/gu, "/");
+    if (!normalized || normalized.includes("\0")) {
+        errors.push(`${label} must be a non-empty relative path.`);
+        return null;
+    }
+    if (/[*?]/u.test(normalized)) {
+        errors.push(`${label} must be a literal path, not a glob.`);
+        return null;
+    }
+    const parts = normalized.split("/");
+    if (parts.some((part) => part === ".." || part === "." || part === "")) {
+        errors.push(`${label} must not contain empty, '.', or '..' path segments.`);
+        return null;
+    }
+    return normalized;
+}
+function parseYamlRecord(source, label = "YAML") {
+    const document = YAML.parseDocument(source, { prettyErrors: true });
+    if (document.errors.length > 0) {
+        throw new Error(document.errors.map((error) => `${label}: ${error.message}`).join("\n"));
+    }
+    const parsed = document.toJS();
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+        throw new Error(`${label} must be a YAML object.`);
+    }
+    return parsed;
+}
+function readAdapterRecord(value, label, errors) {
+    const record = readRequiredRecord(value, label, errors);
+    if (!record) {
+        return undefined;
+    }
+    rejectUnknownKeys(record, label, ["use", "with", "auth"], errors);
+    const use = readRequiredString(record.use, `${label}.use`, errors);
+    const config = record.with === undefined
+        ? {}
+        : readRequiredRecord(record.with, `${label}.with`, errors);
+    return use
+        ? {
+            use,
+            ...(config ? { with: config } : {}),
+            ...(record.auth !== undefined && isJson(record.auth) ? { auth: record.auth } : {}),
+        }
+        : undefined;
+}
+function rejectUnknownKeys(record, label, allowed, errors) {
+    const extras = Object.keys(record).filter((key) => !allowed.includes(key));
+    if (extras.length > 0) {
+        errors.push(`${label} includes unsupported ${extras.length === 1 ? "field" : "fields"}: ${extras.join(", ")}.`);
+    }
+}
+function readRequiredRecord(value, label, errors) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        errors.push(`${label} must be an object.`);
+        return null;
+    }
+    return value;
+}
+function readRequiredString(value, label, errors) {
+    if (typeof value !== "string" || value.trim().length === 0) {
+        errors.push(`${label} must be a non-empty string.`);
+        return null;
+    }
+    return value.trim();
+}
+function readOptionalString(value, label, errors) {
+    if (value === undefined) {
+        return undefined;
+    }
+    if (typeof value !== "string" || value.trim().length === 0) {
+        errors.push(`${label} must be a non-empty string when provided.`);
+        return undefined;
+    }
+    return value.trim();
+}
+function readOptionalPositiveNumber(value, label, errors) {
+    if (value === undefined) {
+        return;
+    }
+    if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
+        errors.push(`${label} must be a positive number.`);
+    }
+}
+function readPositiveNumber(value, fallback) {
+    return typeof value === "number" && Number.isFinite(value) && value > 0
+        ? value
+        : fallback;
+}
+function readJsonRecord(value) {
+    if (!isJson(value)) {
+        return {};
+    }
+    return value;
+}
+function isJson(value) {
+    if (value === null) {
+        return true;
+    }
+    if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+        return Number.isFinite(value) || typeof value !== "number";
+    }
+    if (Array.isArray(value)) {
+        return value.every(isJson);
+    }
+    if (typeof value === "object") {
+        return Object.values(value).every(isJson);
+    }
+    return false;
+}
+function fingerprintJson(value) {
+    return createHash("sha256")
+        .update(JSON.stringify(canonicalJson(value)))
+        .digest("hex");
+}
+function canonicalJson(value) {
+    if (Array.isArray(value)) {
+        return value.map(canonicalJson);
+    }
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value)
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([key, entry]) => [key, canonicalJson(entry)]));
+    }
+    return value;
+}
+function splitErrorMessage(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return message.split(/\n+/u).map((entry) => entry.trim()).filter(Boolean);
+}
+function cloneJson(value) {
+    return JSON.parse(JSON.stringify(value));
+}