@workbench-ai/workbench 0.0.46

package/dist/init-scaffold.d.ts

@@ -0,0 +1,22 @@
+export type InitSubjectKind = "skill" | "command";
+export type InitAgent = string;
+export interface WorkbenchInitScaffoldOptions {
+    kind: InitSubjectKind;
+    name: string;
+    agent?: InitAgent;
+    example: boolean;
+}
+export interface WorkbenchInitScaffoldFile {
+    path: string;
+    content: string;
+}
+export interface WorkbenchInitScaffold {
+    kind: InitSubjectKind;
+    name: string;
+    subjectRoot: string;
+    seedFileTarget: string;
+    seedDirectoryTarget: string;
+    files: WorkbenchInitScaffoldFile[];
+}
+export declare function createWorkbenchInitScaffold(options: WorkbenchInitScaffoldOptions): WorkbenchInitScaffold;
+//# sourceMappingURL=init-scaffold.d.ts.map

package/dist/init-scaffold.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"init-scaffold.d.ts","sourceRoot":"","sources":["../src/init-scaffold.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,eAAe,GAAG,OAAO,GAAG,SAAS,CAAC;AAClD,MAAM,MAAM,SAAS,GAAG,MAAM,CAAC;AAE/B,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,yBAAyB;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,eAAe,CAAC;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,EAAE,yBAAyB,EAAE,CAAC;CACpC;AAED,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,4BAA4B,GAAG,qBAAqB,CAkBxG"}

package/dist/init-scaffold.js

@@ -0,0 +1,30 @@
+import { defaultWorkbenchInitTemplatePack } from "./init-template-pack.js";
+export function createWorkbenchInitScaffold(options) {
+    const template = defaultWorkbenchInitTemplatePack[options.kind];
+    const slug = slugify(options.name);
+    const agent = template.requiresAgent ? requireAgent(options) : undefined;
+    const context = {
+        name: options.name,
+        slug,
+        ...(agent ? { agent } : {}),
+        example: options.example,
+    };
+    return {
+        kind: template.kind,
+        name: options.name,
+        subjectRoot: template.subjectRoot(context),
+        seedFileTarget: template.seedFileTarget(context),
+        seedDirectoryTarget: template.seedDirectoryTarget(context),
+        files: template.files(context),
+    };
+}
+function requireAgent(options) {
+    if (options.agent && /^[a-z][a-z0-9-]*$/u.test(options.agent)) {
+        return options.agent;
+    }
+    throw new Error(`--agent is required for --${options.kind} and must be a lowercase adapter id.`);
+}
+function slugify(value) {
+    const slug = value.trim().toLowerCase().replace(/[^a-z0-9]+/gu, "-").replace(/^-+|-+$/gu, "");
+    return slug || "workbench-subject";
+}

package/dist/init-template-pack.d.ts

@@ -0,0 +1,19 @@
+import type { InitAgent, InitSubjectKind, WorkbenchInitScaffoldFile } from "./init-scaffold.js";
+interface WorkbenchInitTemplateContext {
+    name: string;
+    slug: string;
+    agent?: InitAgent;
+    example: boolean;
+}
+export interface WorkbenchInitTemplate {
+    kind: InitSubjectKind;
+    requiresAgent: boolean;
+    subjectRoot(context: WorkbenchInitTemplateContext): string;
+    seedFileTarget(context: WorkbenchInitTemplateContext): string;
+    seedDirectoryTarget(context: WorkbenchInitTemplateContext): string;
+    files(context: WorkbenchInitTemplateContext): WorkbenchInitScaffoldFile[];
+}
+export type WorkbenchInitTemplatePack = Record<InitSubjectKind, WorkbenchInitTemplate>;
+export declare const defaultWorkbenchInitTemplatePack: WorkbenchInitTemplatePack;
+export {};
+//# sourceMappingURL=init-template-pack.d.ts.map

package/dist/init-template-pack.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"init-template-pack.d.ts","sourceRoot":"","sources":["../src/init-template-pack.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,yBAAyB,EAC1B,MAAM,oBAAoB,CAAC;AAE5B,UAAU,4BAA4B;IACpC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,eAAe,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;IACvB,WAAW,CAAC,OAAO,EAAE,4BAA4B,GAAG,MAAM,CAAC;IAC3D,cAAc,CAAC,OAAO,EAAE,4BAA4B,GAAG,MAAM,CAAC;IAC9D,mBAAmB,CAAC,OAAO,EAAE,4BAA4B,GAAG,MAAM,CAAC;IACnE,KAAK,CAAC,OAAO,EAAE,4BAA4B,GAAG,yBAAyB,EAAE,CAAC;CAC3E;AAED,MAAM,MAAM,yBAAyB,GAAG,MAAM,CAAC,eAAe,EAAE,qBAAqB,CAAC,CAAC;AAEvF,eAAO,MAAM,gCAAgC,EAAE,yBA+C9C,CAAC"}

package/dist/init-template-pack.js

@@ -0,0 +1,250 @@
+export const defaultWorkbenchInitTemplatePack = {
+    skill: {
+        kind: "skill",
+        requiresAgent: true,
+        subjectRoot: ({ agent }) => `subjects/${requiredAgent(agent)}/files`,
+        seedFileTarget: ({ agent }) => `subjects/${requiredAgent(agent)}/files/SKILL.md`,
+        seedDirectoryTarget: ({ agent }) => `subjects/${requiredAgent(agent)}/files`,
+        files: ({ name, slug, agent, example }) => {
+            const adapter = requiredAgent(agent);
+            return [
+                { path: "benchmark.yaml", content: skillBenchmarkSpec(name, adapter) },
+                { path: `subjects/${adapter}/subject.yaml`, content: skillSubjectSpec(name, adapter) },
+                { path: `optimizers/${adapter}.yaml`, content: optimizerSpec(name, "SKILL.md", adapter) },
+                { path: `subjects/${adapter}/files/SKILL.md`, content: skillMarkdown(name, slug, example) },
+                { path: `subjects/${adapter}/files/agents/openai.yaml`, content: skillOpenAiMetadata(name, slug) },
+                { path: "environment/Dockerfile", content: nodeDockerfile() },
+                { path: "tasks/task-001/task.yaml", content: taskYaml(skillCasePrompt(name)) },
+                { path: "tasks/task-001/tests/rubric.md", content: skillExpectedRubric() },
+                ...(example ? [
+                    { path: "tasks/task-002/task.yaml", content: taskYaml(`Use ${name} for a second realistic prompt with different constraints.\n`) },
+                    { path: "tasks/task-002/tests/rubric.md", content: skillExpectedRubric() },
+                ] : []),
+            ];
+        },
+    },
+    command: {
+        kind: "command",
+        requiresAgent: false,
+        subjectRoot: () => "subjects/command/files",
+        seedFileTarget: () => "subjects/command/files/run.js",
+        seedDirectoryTarget: () => "subjects/command/files",
+        files: ({ name, example }) => [
+            { path: "benchmark.yaml", content: commandBenchmarkSpec(name) },
+            { path: "subjects/command/subject.yaml", content: commandSubjectSpec(name) },
+            { path: "optimizers/command.yaml", content: commandOptimizerSpec(name) },
+            { path: "subjects/command/files/run.js", content: commandRunnerSource() },
+            { path: "environment/Dockerfile", content: nodeDockerfile() },
+            { path: "tasks/task-001/task.yaml", content: taskYaml("The command should produce a concise result for this task.\n") },
+            { path: "tasks/task-001/tests/required-output.txt", content: "command subject ran\n" },
+            { path: "tasks/task-001/tests/test.sh", content: commandTestScript() },
+            ...(example ? [
+                { path: "tasks/task-002/task.yaml", content: taskYaml("The command should still produce deterministic output for a second task.\n") },
+                { path: "tasks/task-002/tests/required-output.txt", content: "command subject ran\n" },
+                { path: "tasks/task-002/tests/test.sh", content: commandTestScript() },
+            ] : []),
+        ],
+    },
+};
+function requiredAgent(agent) {
+    if (!agent) {
+        throw new Error("Template requires an agent adapter id.");
+    }
+    return agent;
+}
+function yamlString(value) {
+    return JSON.stringify(value);
+}
+function taskYaml(task) {
+    return [
+        "version: 3",
+        "task: |-",
+        ...task.trimEnd().split("\n").map((line) => `  ${line}`),
+        "tests:",
+        "  path: tests",
+        "",
+    ].join("\n");
+}
+function skillBenchmarkSpec(name, agent) {
+    return [
+        "version: 3",
+        `name: ${yamlString(name)}`,
+        `description: ${yamlString(`Evaluate the ${name} skill across representative tasks.`)}`,
+        "engine:",
+        "  use: workbench",
+        "  with:",
+        "    environment:",
+        "      dockerfile: environment/Dockerfile",
+        "    score:",
+        "      use: rubric",
+        "      with:",
+        "        instructions: Score the completed task from the current working directory and engine-private verifier files. Do not score the subject instructions by keyword matching.",
+        "        parallelism: 2",
+        "        judge:",
+        `          use: ${agent}`,
+        "        criteria:",
+        "          - id: task_fit",
+        "            description: The response follows the task prompt and uses the skill's workflow.",
+        "            weight: 1",
+        "          - id: output_quality",
+        "            description: The produced output is complete, readable, and directly useful.",
+        "            weight: 1",
+        "",
+    ].join("\n");
+}
+function skillSubjectSpec(name, agent) {
+    return [
+        "version: 3",
+        `name: ${yamlString(name)}`,
+        "files:",
+        "  path: files",
+        "run:",
+        `  use: ${agent}`,
+        "  with:",
+        "    instructions: Use paths.workspace for mutable subject files and paths.case for public task files from WORKBENCH_ADAPTER_REQUEST. Mutate the workspace to complete the task.",
+        "",
+    ].join("\n");
+}
+function optimizerSpec(name, editablePath, agent) {
+    return [
+        "version: 3",
+        `name: ${yamlString(`${name} optimizer`)}`,
+        `description: ${yamlString(`Improve subject files for ${name}.`)}`,
+        "edits:",
+        `  - ${editablePath}`,
+        "improve:",
+        `  use: ${agent}`,
+        "",
+    ].join("\n");
+}
+function commandBenchmarkSpec(name) {
+    return [
+        "version: 3",
+        `name: ${yamlString(name)}`,
+        `description: ${yamlString(`Evaluate the ${name} command implementation across representative tasks.`)}`,
+        "engine:",
+        "  use: workbench",
+        "  with:",
+        "    environment:",
+        "      dockerfile: environment/Dockerfile",
+        "    score:",
+        "      use: tests",
+        "",
+    ].join("\n");
+}
+function commandSubjectSpec(name) {
+    const runnerCommand = JSON.stringify("node run.js");
+    return [
+        "version: 3",
+        `name: ${yamlString(name)}`,
+        "files:",
+        "  path: files",
+        "run:",
+        "  use: command",
+        "  with:",
+        `    command: ${runnerCommand}`,
+        "",
+    ].join("\n");
+}
+function commandOptimizerSpec(name) {
+    const optimizerCommand = JSON.stringify("node -e \"const fs=require('fs');const file='run.js';const current=fs.existsSync(file)?fs.readFileSync(file,'utf8'):'';const next=current.replace(/\\s*$/,'')+'\\n// Workbench subject revision.\\n';fs.writeFileSync(file,next);\"");
+    return [
+        "version: 3",
+        `name: ${yamlString(`${name} optimizer`)}`,
+        `description: ${yamlString(`Improve subject command files for ${name}.`)}`,
+        "edits:",
+        "  - run.js",
+        "improve:",
+        "  use: command",
+        "  with:",
+        `    command: ${optimizerCommand}`,
+        "",
+    ].join("\n");
+}
+function commandTestScript() {
+    return [
+        "#!/usr/bin/env sh",
+        "set -eu",
+        "expected=$(node -e \"const fs=require('fs'),path=require('path');const r=JSON.parse(fs.readFileSync(process.env.WORKBENCH_ADAPTER_REQUEST,'utf8'));process.stdout.write(fs.readFileSync(path.join(r.paths.enginePrivate,'required-output.txt'),'utf8'));\")",
+        "verifier_logs=$(node -e \"const fs=require('fs'),path=require('path');const r=JSON.parse(fs.readFileSync(process.env.WORKBENCH_ADAPTER_REQUEST,'utf8'));process.stdout.write(path.join(r.paths.logs,'verifier'));\")",
+        "actual=$(cat command-output.txt 2>/dev/null || true)",
+        "mkdir -p \"$verifier_logs\"",
+        "case \"$actual\" in",
+        "  *\"$expected\"*) printf '{\"reward\":1,\"exact\":1}\\n' > \"$verifier_logs/reward.json\" ;;",
+        "  *) printf '{\"reward\":0,\"exact\":0}\\n' > \"$verifier_logs/reward.json\" ;;",
+        "esac",
+        "",
+    ].join("\n");
+}
+function nodeDockerfile() {
+    return [
+        "FROM node:22-slim",
+        "",
+        ...caCertificatesDockerfileStep(),
+        "",
+    ].join("\n");
+}
+function caCertificatesDockerfileStep() {
+    return [
+        "RUN apt-get update \\",
+        "    && apt-get install -y --no-install-recommends ca-certificates \\",
+        "    && rm -rf /var/lib/apt/lists/*",
+    ];
+}
+function skillMarkdown(name, slug, example) {
+    return [
+        "---",
+        `name: ${slug}`,
+        `description: ${yamlString(`Use this skill whenever the user asks for ${name} work, related deliverable generation, or iterative improvement of this workflow.`)}`,
+        "---",
+        "",
+        `# ${name}`,
+        "",
+        "Use this skill to turn the user's request into a concrete deliverable.",
+        "",
+        "## Workflow",
+        "",
+        "1. Identify the requested deliverable and success criteria.",
+        "2. Gather only the source context needed for this deliverable.",
+        "3. Produce the deliverable in the format the user can use directly.",
+        "4. Validate the output against the success criteria before returning it.",
+        "",
+        ...(example ? [
+            "## Example",
+            "",
+            "When a prompt asks for a concrete deliverable, produce the deliverable first and keep explanation secondary.",
+            "",
+        ] : []),
+    ].join("\n");
+}
+function skillOpenAiMetadata(name, slug) {
+    return [
+        `name: ${slug}`,
+        `description: ${yamlString(`Generate and improve ${name} deliverables.`)}`,
+        "",
+    ].join("\n");
+}
+function skillCasePrompt(name) {
+    return [
+        `Use the ${name} skill to produce a small but complete deliverable for a realistic request.`,
+        "Return the deliverable content and a one-sentence validation note.",
+        "",
+    ].join("\n");
+}
+function skillExpectedRubric() {
+    return [
+        "Reward complete, usable deliverables created from the task input.",
+        "Penalize placeholder output, missing validation, and keyword-only compliance.",
+        "",
+    ].join("\n");
+}
+function commandRunnerSource() {
+    return [
+        "const fs = require('fs');",
+        "const path = require('path');",
+        "",
+        "fs.writeFileSync(path.join(process.cwd(), 'command-output.txt'), 'command subject ran\\n');",
+        "console.log('command subject ran');",
+        "",
+    ].join("\n");
+}

package/dist/local-archive.d.ts

@@ -0,0 +1,23 @@
+import { type SubjectRecord, type EvaluationResultRecord, type HostedWorkbenchJob, type RunSummary, type RuntimeEvent, type SurfaceSnapshotFile } from "@workbench-ai/workbench-core";
+export interface LocalArchiveSnapshot {
+    activeId: string | null;
+    subjects: SubjectRecord[];
+    subjectFiles: Record<string, SurfaceSnapshotFile[]>;
+    evaluations: EvaluationResultRecord[];
+    runs: RunSummary[];
+    events: RuntimeEvent[];
+}
+export declare function localRuntimeDir(workspace: string): string;
+export declare function loadLocalArchive(workspace: string): Promise<LocalArchiveSnapshot>;
+export declare function saveLocalArchive(workspace: string, snapshot: LocalArchiveSnapshot): Promise<void>;
+export declare function saveLocalJobs(workspace: string, jobs: readonly HostedWorkbenchJob[]): Promise<void>;
+export declare function readLocalExecutionFiles(workspace: string, jobId: string): Promise<SurfaceSnapshotFile[]>;
+export declare function upsertLocalSubject(snapshot: LocalArchiveSnapshot, subject: SubjectRecord, files: readonly SurfaceSnapshotFile[]): LocalArchiveSnapshot;
+export declare function upsertLocalEvaluation(snapshot: LocalArchiveSnapshot, evaluation: EvaluationResultRecord): LocalArchiveSnapshot;
+export declare function appendLocalRun(snapshot: LocalArchiveSnapshot, run: RunSummary, events: readonly RuntimeEvent[]): LocalArchiveSnapshot;
+export declare function setLocalActive(snapshot: LocalArchiveSnapshot, activeId: string | null): LocalArchiveSnapshot;
+export declare function readLocalSubject(snapshot: LocalArchiveSnapshot, subjectId: string): SubjectRecord;
+export declare function readLocalSubjectFiles(snapshot: LocalArchiveSnapshot, subjectId: string): SurfaceSnapshotFile[];
+export declare function materializeSubjectRoot(workspace: string, subjectRoot: string, files: readonly SurfaceSnapshotFile[]): Promise<string[]>;
+export declare function findArchivedFile(files: readonly SurfaceSnapshotFile[], filePath: string): SurfaceSnapshotFile | null;
+//# sourceMappingURL=local-archive.d.ts.map

package/dist/local-archive.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-archive.d.ts","sourceRoot":"","sources":["../src/local-archive.ts"],"names":[],"mappings":"AAGA,OAAO,EAGL,KAAK,aAAa,EAClB,KAAK,sBAAsB,EAC3B,KAAK,kBAAkB,EACvB,KAAK,UAAU,EACf,KAAK,YAAY,EAEjB,KAAK,mBAAmB,EAEzB,MAAM,8BAA8B,CAAC;AAOtC,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,EAAE,CAAC,CAAC;IACpD,WAAW,EAAE,sBAAsB,EAAE,CAAC;IACtC,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,MAAM,EAAE,YAAY,EAAE,CAAC;CACxB;AAQD,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEzD;AAED,wBAAsB,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAuBvF;AAED,wBAAsB,gBAAgB,CACpC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,oBAAoB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,aAAa,CACjC,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,SAAS,kBAAkB,EAAE,GAClC,OAAO,CAAC,IAAI,CAAC,CA2Bf;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAIhC;AAED,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,oBAAoB,EAC9B,OAAO,EAAE,aAAa,EACtB,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,oBAAoB,CAYtB;AAED,wBAAgB,qBAAqB,CACnC,QAAQ,EAAE,oBAAoB,EAC9B,UAAU,EAAE,sBAAsB,GACjC,oBAAoB,CAQtB;AAED,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,oBAAoB,EAC9B,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,SAAS,YAAY,EAAE,GAC9B,oBAAoB,CAYtB;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,oBAAoB,CAK5G;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,GAAG,aAAa,CAMjG;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,oBAAoB,EAAE,SAAS,EAAE,MAAM,GAAG,mBAAmB,EAAE,CAG9G;AAkjBD,wBAAsB,sBAAsB,CAC1C,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,SAAS,mBAAmB,EAAE,GACpC,OAAO,CAAC,MAAM,EAAE,CAAC,CAOnB;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,mBAAmB,EAAE,EACrC,QAAQ,EAAE,MAAM,GACf,mBAAmB,GAAG,IAAI,CAG5B"}