@zof-ai/agent-evals 0.0.0-stage → 0.1.0

package/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## 0.1.0 (unreleased)
+
+Initial independently authored local implementation. Candidate package/repository names and Apache-2.0 licensing require owner and legal approval before publication.

package/CONTRIBUTING.md

@@ -0,0 +1,5 @@
+# Contributing
+
+Use Node.js 22 or 24, run `npm ci --ignore-scripts`, `npm run build`, and `npm test`. Add a regression test for functional or security changes. Keep changes independently authored and free of proprietary code, credentials and customer data. Discuss incompatible API changes before implementation. Contributions use the repository license and require maintainer review; dependency updates are never automatically merged.
+
+Maintainers review issue reproductions, security boundaries, tests, compatibility and package contents. Use semantic versioning; announce breaking interfaces in the changelog. Deprecations require a documented replacement and at least one minor release before removal.

package/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -1,3 +1,47 @@
-# Temporary Holding Version
+# Zof Agent Evaluations
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Evaluate a JavaScript candidate snapshot against explicit input/output tests in a constrained **local Docker container**. Candidate output is compared in the parent process; reports include test outcomes and fixed reason codes, never raw output or logs. Three public task definitions come from `@zof-ai/reliability-bench` and results use Evidence 1.0.0.
+
+Version 0.1.0 is prepared under the Apache-2.0 license. Use the GitHub source installation below; npm registry publication is not claimed.
+
+Requires Node.js 22 or newer, a local Docker engine and the pinned Node image documented in [sandbox controls](docs/SANDBOX.md). The evaluator never pulls images, installs candidate dependencies, or uses a remote Docker endpoint.
+
+<!-- ZOF_RELEASE_INSTALL_START -->
+## Install from GitHub source
+
+Requires Node.js 22 or 24, npm, and Git. Filesystem operations require Linux or macOS with Python 3.10 or newer; Windows file operations fail closed. Clone this repository and explicitly prepare its reviewed release dependencies:
+
+```sh
+git clone https://github.com/Zof-AI/zof-agent-evals.git
+cd zof-agent-evals
+node scripts/bootstrap.mjs --allow-network
+npm ci --ignore-scripts --prefer-offline
+npm run build
+npm test
+```
+
+The bootstrap downloads only the exact `v0.1.0` dependency assets from [GitHub Releases](https://github.com/Zof-AI/zof-agent-evals/releases), as listed in `scripts/release-dependencies.json`, verifies SHA256 and byte size, and adds them to the npm cache. It does not extract archives, run package scripts, collect application data, or send credentials. Downloads require the explicit flag and follow only HTTPS GitHub asset redirects. Installation then fetches ordinary third-party dependencies from npm. Use a writable npm cache; set `npm_config_cache` consistently for both commands if needed. An unavailable asset, changed hash, or oversized response stops installation.
+
+GitHub source and release artifacts are the initial distribution channel. These instructions do not claim that `@zof-ai` packages are published on npm or that the VS Code extension is listed in Marketplace.
+<!-- ZOF_RELEASE_INSTALL_END -->
+
+## Three examples
+
+```sh
+npm run build
+node dist/cli.js evaluate correct.mjs --root examples --task deduplicate-payments
+node dist/cli.js evaluate incorrect.mjs --root examples --task deduplicate-payments
+node dist/cli.js evaluate profile.mjs --root examples --task compatible-profile
+npm test
+npm run test:container
+```
+
+Correct and profile exit 0; incorrect exits 1. Missing Docker, rejected input, timeouts or incomplete execution exit 2. Known candidate failures remain failed evidence. The explicit evaluate command authorizes running only the selected bounded `.mjs` file against the selected data-only task. It never executes the source on the host.
+
+Candidates export `solve(input)` returning a JSON-compatible value, synchronously or asynchronously. Each case runs in a fresh container. Only one candidate file, one case input, and a fixed harness are copied into a fresh protected staging directory; the original directory is not mounted. Expected values remain outside the container.
+
+`node dist/cli.js tasks` lists the actual public tasks. `--task-file task.json` loads an explicitly selected custom data-only task under the same root. `--timeout 500..10000` controls total per-case time including container startup. `--agent name` records a bounded, redacted user-provided label. No environment variables are copied into evidence.
+
+Library: `evaluateCandidate({root, candidate, task: idOrTask, timeoutMs?, agent?: {name,version?}})` returns `{taskId,outcomes,evidence}`. `validateTask(value)` validates and copies the data-only task. `SANDBOX_IMAGE` exports the pinned image identifier. `sandboxArguments` exposes the constrained argument builder for auditing, without executing a command.
+
+Read [task format](docs/TASKS.md), [sandbox and limitations](docs/SANDBOX.md), and [security reporting](SECURITY.md).

package/SECURITY.md

@@ -0,0 +1,9 @@
+# Security policy
+
+The 0.1.x line is the initial supported release line. Local analyses are bounded and offline unless an operation is explicitly documented otherwise. Do not provide production secrets or execute untrusted generated code.
+
+Use GitHub private vulnerability reporting when enabled for this repository. If it is unavailable, contact an organization owner through an already verified private channel; do not file credentials, exploit details, or customer data in a public issue. Reports will be triaged, reproduced using synthetic data, fixed and reviewed before coordinated disclosure. No response SLA is promised for this initial release.
+
+Untrusted candidates are never imported or executed on the host. They are copied as a bounded module into a protected temporary staging parent and run in a non-root, network-disabled, read-only Docker container with CPU/memory/PID/time/output limits. Raw logs are discarded. No host credential or source-directory mount is allowed. See docs/SANDBOX.md for cleanup verification, local-only engine enforcement, and kernel/runtime limitations. The tool is not a public multi-tenant execution service. Missing isolation produces missing evidence rather than a passing result.
+
+Filesystem operations use bounded, directory-relative descriptors on Linux/macOS with Python 3.10 or newer. Symlinks, hard-linked inputs, special files and traversal are rejected. Open descriptors remain attached to the original directory when names change. File contents can still change during a read or snapshot; this is not an atomic filesystem snapshot. Windows filesystem operations fail closed; in-memory JSON validation and analysis do not require the file helper.

package/dist/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/dist/cli.js

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+import { parseArgs } from 'node:util';
+import { safeReadJson, serializeEvidence } from '@zof-ai/evidence-spec';
+import { listEvaluationTasks } from '@zof-ai/reliability-bench';
+import { evaluateCandidate, validateTask } from './index.js';
+try {
+    const { values, positionals } = parseArgs({ options: { root: { type: 'string', default: '.' }, task: { type: 'string', default: 'deduplicate-payments' }, 'task-file': { type: 'string' }, timeout: { type: 'string', default: '2000' }, agent: { type: 'string' }, help: { type: 'boolean' }, version: { type: 'boolean' } }, allowPositionals: true });
+    if (values.help)
+        console.log('Usage: zof-agent-evals evaluate <candidate.mjs> --root directory [--task id | --task-file relative.json] [--timeout 500..10000] [--agent name]\nCandidates run only in constrained local Docker; no automatic image pull. Exit0 pass,1 candidate failure,2 incomplete/missing/input.\nUse zof-agent-evals tasks to list public data-only tasks.');
+    else if (values.version)
+        console.log('0.1.0');
+    else if (positionals.length === 1 && positionals[0] === 'tasks')
+        console.log(JSON.stringify(listEvaluationTasks(), null, 2));
+    else if (positionals.length === 2 && positionals[0] === 'evaluate') {
+        const task = values['task-file'] ? validateTask(await safeReadJson(values.root, values['task-file'])) : values.task;
+        const result = await evaluateCandidate({ root: values.root, candidate: positionals[1], task, timeoutMs: Number(values.timeout), ...(values.agent ? { agent: { name: values.agent } } : {}) });
+        process.stdout.write(serializeEvidence(result.evidence));
+        process.exitCode = result.evidence.execution.status === 'completed' ? (result.evidence.execution.result === 'pass' ? 0 : 1) : 2;
+    }
+    else
+        throw new Error('usage');
+}
+catch {
+    console.error(JSON.stringify({ error: 'Invalid candidate, task or evaluation options.' }));
+    process.exitCode = 2;
+}

package/dist/index.d.ts

@@ -0,0 +1,26 @@
+import type { Evidence } from '@zof-ai/evidence-spec';
+import type { EvaluationTask } from '@zof-ai/reliability-bench';
+export declare const SANDBOX_IMAGE = "node@sha256:43ac6c60b8f89723f746e8a92ce91abd5017e627ce1ddfe4238355d3a30b772c";
+export interface EvaluationOptions {
+    root: string;
+    candidate: string;
+    task: string | EvaluationTask;
+    timeoutMs?: number;
+    agent?: {
+        name: string;
+        version?: string;
+    };
+}
+export interface EvaluationResult {
+    taskId: string;
+    outcomes: Array<{
+        name: string;
+        status: 'passed' | 'failed' | 'error';
+        reason: 'MATCH' | 'MISMATCH' | 'CANDIDATE_ERROR' | 'TIMEOUT' | 'OUTPUT_LIMIT' | 'SANDBOX_UNAVAILABLE' | 'CLEANUP_UNCONFIRMED';
+    }>;
+    evidence: Evidence;
+}
+/** Bounded public data-only task format. No shell commands, harness code, imports or endpoint fields. */
+export declare function validateTask(value: unknown): EvaluationTask;
+export declare function sandboxArguments(directory: string, name: string): string[];
+export declare function evaluateCandidate(options: EvaluationOptions): Promise<EvaluationResult>;

package/dist/index.js

@@ -0,0 +1,178 @@
+import { execFile, spawn } from 'node:child_process';
+import { promisify, isDeepStrictEqual } from 'node:util';
+import { mkdtemp, writeFile, rm, mkdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { randomUUID } from 'node:crypto';
+import { createEvidence, safeReadFile, assertBounded, parseJson, redactText } from '@zof-ai/evidence-spec';
+import { getEvaluationTask } from '@zof-ai/reliability-bench';
+export const SANDBOX_IMAGE = 'node@sha256:43ac6c60b8f89723f746e8a92ce91abd5017e627ce1ddfe4238355d3a30b772c';
+const exec = promisify(execFile);
+const MAX_SOURCE_BYTES = 128 * 1024;
+const MAX_OUTPUT_BYTES = 64 * 1024;
+/** Bounded public data-only task format. No shell commands, harness code, imports or endpoint fields. */
+export function validateTask(value) {
+    assertBounded(value);
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new Error('Invalid evaluation task.');
+    const t = value;
+    if (t.version !== '1.0.0' || typeof t.id !== 'string' || !(/^[a-z0-9][a-z0-9-]{0,99}$/).test(t.id) || typeof t.description !== 'string' || t.description.length > 1000 || !Array.isArray(t.cases) || !t.cases.length || t.cases.length > 32 || Object.keys(t).some(k => !['version', 'id', 'description', 'cases'].includes(k)))
+        throw new Error('Invalid evaluation task.');
+    for (const item of t.cases) {
+        if (!item || typeof item !== 'object' || Array.isArray(item))
+            throw new Error('Invalid test case.');
+        const c = item;
+        if (typeof c.name !== 'string' || !c.name || c.name.length > 200 || !Object.hasOwn(c, 'input') || !Object.hasOwn(c, 'expected') || Object.keys(c).some(k => !['name', 'input', 'expected'].includes(k)))
+            throw new Error('Invalid test case.');
+        if (Buffer.byteLength(JSON.stringify(c)) > 32 * 1024)
+            throw new Error('Test case exceeds limit.');
+    }
+    return structuredClone(value);
+}
+export function sandboxArguments(directory, name) {
+    if (!path.isAbsolute(directory) || /[\x00-\x1f,]/.test(directory) || !/^zof-eval-[a-z0-9-]+$/.test(name))
+        throw new Error('Invalid sandbox path or name.');
+    return ['run', '--rm', '--pull=never', '--name', name, '--network=none', '--read-only', '--cap-drop=ALL', '--security-opt=no-new-privileges', '--pids-limit=32', '--memory=128m', '--memory-swap=128m', '--cpus=0.5', '--user=65534:65534', '--tmpfs=/tmp:rw,noexec,nosuid,size=16m', '--workdir=/task', '--mount', `type=bind,src=${directory},dst=/task,readonly`, SANDBOX_IMAGE, 'node', '--max-old-space-size=64', '/task/runner.mjs'];
+}
+const harness = "import fs from 'node:fs'; const write=process.stdout.write.bind(process.stdout);const stringify=JSON.stringify; const input=JSON.parse(fs.readFileSync('/task/input.json','utf8'));try{const module=await import('/task/candidate.mjs');if(typeof module.solve!=='function')process.exit(41);const output=await module.solve(input);const text=stringify(output);if(typeof text!=='string')process.exit(42);write(text);}catch{process.exit(43);}";
+async function ready() {
+    try {
+        const context = await exec('docker', ['context', 'inspect', '--format', '{{(index .Endpoints "docker").Host}}'], { timeout: 5000, maxBuffer: 4096 });
+        // Only local Unix/named-pipe engines are supported. Never redirect evaluation to a remote endpoint.
+        const endpoint = process.env.DOCKER_HOST ?? context.stdout.trim();
+        if (!endpoint.startsWith('unix://') && !endpoint.startsWith('npipe://'))
+            return;
+        await exec('docker', ['--host', endpoint, 'image', 'inspect', SANDBOX_IMAGE, '--format', '{{.Id}}'], { timeout: 5000, maxBuffer: 4096 });
+        return endpoint;
+    }
+    catch {
+        return;
+    }
+}
+async function runOne(directory, timeoutMs, endpoint) {
+    const name = `zof-eval-${randomUUID()}`;
+    let result = { reason: 'CANDIDATE_ERROR' };
+    try {
+        result = await new Promise(resolve => {
+            const child = spawn('docker', ['--host', endpoint, ...sandboxArguments(directory, name)], { stdio: ['ignore', 'pipe', 'pipe'] });
+            let stdout = Buffer.alloc(0), count = 0, stopped = false, reason;
+            const stop = (why) => { if (!stopped) {
+                stopped = true;
+                reason = why;
+                child.kill('SIGKILL');
+            } };
+            const timer = setTimeout(() => stop('TIMEOUT'), timeoutMs);
+            child.stdout.on('data', (chunk) => { count += chunk.length; if (count > MAX_OUTPUT_BYTES)
+                stop('OUTPUT_LIMIT');
+            else
+                stdout = Buffer.concat([stdout, chunk]); });
+            // Raw stderr is counted and discarded; it is never recorded or returned.
+            child.stderr.on('data', (chunk) => { count += chunk.length; if (count > MAX_OUTPUT_BYTES)
+                stop('OUTPUT_LIMIT'); });
+            child.on('error', () => { clearTimeout(timer); resolve({ reason: 'SANDBOX_UNAVAILABLE' }); });
+            child.on('close', code => { clearTimeout(timer); if (reason) {
+                resolve({ reason });
+                return;
+            } if (code !== 0) {
+                resolve({ reason: code === 125 ? 'SANDBOX_UNAVAILABLE' : 'CANDIDATE_ERROR' });
+                return;
+            } try {
+                resolve({ reason: 'MATCH', value: parseJson(stdout.toString('utf8'), MAX_OUTPUT_BYTES) });
+            }
+            catch {
+                resolve({ reason: 'CANDIDATE_ERROR' });
+            } });
+        });
+    }
+    finally {
+        // Killing a Docker client alone does not stop its container. Always remove the unique job container.
+        let confirmed = false;
+        for (let attempt = 0; attempt < 5 && !confirmed; attempt++) {
+            try {
+                await exec('docker', ['--host', endpoint, 'rm', '-f', name], { timeout: 5000, maxBuffer: 4096 });
+                confirmed = true;
+            }
+            catch {
+                try {
+                    const remaining = await exec('docker', ['--host', endpoint, 'ps', '-a', '--filter', `name=^/${name}$`, '--format', '{{.ID}}'], { timeout: 5000, maxBuffer: 4096 });
+                    confirmed = !remaining.stdout.trim();
+                }
+                catch { }
+            }
+            if (!confirmed)
+                await new Promise(resolve => setTimeout(resolve, 100));
+        }
+        if (!confirmed)
+            result = { reason: 'CLEANUP_UNCONFIRMED' };
+    }
+    return result;
+}
+/** Explicit invocation authorizes only this bounded source file and data-only task in local Docker. */
+let activeEvaluations = 0;
+export async function evaluateCandidate(options) {
+    if (activeEvaluations >= 2)
+        throw new Error('Concurrent evaluation limit reached.');
+    activeEvaluations++;
+    try {
+        return await evaluateInternal(options);
+    }
+    finally {
+        activeEvaluations--;
+    }
+}
+async function evaluateInternal(options) {
+    if (typeof options.candidate !== 'string' || !options.candidate.endsWith('.mjs'))
+        throw new Error('Candidate must be an mjs module.');
+    const task = validateTask(typeof options.task === 'string' ? getEvaluationTask(options.task) : options.task);
+    const timeout = options.timeoutMs ?? 2000;
+    if (!Number.isInteger(timeout) || timeout < 500 || timeout > 10000)
+        throw new Error('Timeout must be between 500 and 10000 milliseconds.');
+    const source = await safeReadFile(options.root, options.candidate, MAX_SOURCE_BYTES);
+    if (options.agent !== undefined) {
+        assertBounded(options.agent);
+        if (typeof options.agent.name !== 'string' || options.agent.name.length > 100 || (options.agent.version !== undefined && (typeof options.agent.version !== 'string' || options.agent.version.length > 100)))
+            throw new Error('Invalid agent metadata.');
+    }
+    const outcomes = [];
+    let status = 'completed';
+    const endpoint = await ready();
+    if (!endpoint) {
+        outcomes.push(...task.cases.map(c => ({ name: redactText(c.name), status: 'error', reason: 'SANDBOX_UNAVAILABLE' })));
+        status = 'missing';
+    }
+    else {
+        const stagingRoot = await mkdtemp(path.join(tmpdir(), 'zof-eval-'));
+        const directory = path.join(stagingRoot, 'task');
+        try {
+            await mkdir(directory, { mode: 0o755 });
+            await writeFile(path.join(directory, 'candidate.mjs'), source, { mode: 0o444 });
+            await writeFile(path.join(directory, 'runner.mjs'), harness, { mode: 0o444 });
+            for (const item of task.cases) {
+                await rm(path.join(directory, 'input.json'), { force: true });
+                await writeFile(path.join(directory, 'input.json'), JSON.stringify(item.input), { mode: 0o444 });
+                const result = await runOne(directory, timeout, endpoint);
+                const matched = result.reason === 'MATCH' && isDeepStrictEqual(result.value, item.expected);
+                const reason = result.reason === 'MATCH' ? (matched ? 'MATCH' : 'MISMATCH') : result.reason;
+                outcomes.push({ name: redactText(item.name), status: reason === 'MATCH' ? 'passed' : reason === 'MISMATCH' ? 'failed' : 'error', reason });
+                if (reason === 'TIMEOUT' || reason === 'OUTPUT_LIMIT')
+                    status = 'partial';
+                if (reason === 'SANDBOX_UNAVAILABLE' || reason === 'CLEANUP_UNCONFIRMED') {
+                    status = 'error';
+                    break;
+                }
+            }
+        }
+        finally {
+            await rm(stagingRoot, { recursive: true, force: true });
+        }
+    }
+    const failed = outcomes.some(o => o.status !== 'passed');
+    const agent = {};
+    if (options.agent) {
+        agent.name = redactText(options.agent.name);
+        if (options.agent.version)
+            agent.version = redactText(options.agent.version);
+    }
+    const evidence = createEvidence({ tool: { name: 'zof-agent-evals', version: '0.1.0' }, type: 'agent-evaluation', execution: { status, result: status === 'missing' || status === 'error' ? 'unknown' : failed ? 'fail' : 'pass' }, tests: outcomes.map(o => ({ name: o.name, status: o.status })), findings: outcomes.filter(o => o.reason !== 'MATCH').map((o, i) => ({ id: `case-${i + 1}`, severity: 'info', message: o.reason })), environment: { os: process.platform, architecture: process.arch, runtime: process.version }, extensions: { 'org.zof.evaluation': { taskId: task.id, taskVersion: task.version, engine: 'docker', image: SANDBOX_IMAGE, timeoutMs: timeout, agent } } });
+    return { taskId: task.id, outcomes, evidence };
+}

package/docs/RELEASE.md

@@ -0,0 +1,13 @@
+# Release process
+
+Version 0.1.0 is prepared under [Apache-2.0](../LICENSE). [GitHub Releases](https://github.com/Zof-AI/zof-agent-evals/releases) is the initial distribution channel. Refer to the release's actual assets and commit SHA; these instructions are not evidence that an upload or installation succeeded.
+
+Start with the [source installation](../README.md#install-from-github-source). Run `node scripts/bootstrap.mjs --allow-network`, `npm ci --ignore-scripts --prefer-offline`, `npm run build`, `npm test`, and `npm pack --ignore-scripts`. The bootstrap checks the exact dependency release SHA256 and byte sizes. Do not replace failed checks with an unverified dependency or a `file:` reference in published metadata.
+
+Before releasing a new version, inspect source and Git history, dependency licenses, build outputs and every package entry. Resolve security findings; run independent negative and installation tests. Record artifact SHA256 and release commits. Maintain branch protection, private vulnerability reporting, least-privilege workflows and a protected release environment. The manual release-candidate workflow builds artifacts and does not publish automatically. Material release changes require a new review.
+
+Publish the reviewed package tarball as a GitHub Release asset. npm registry publication requires independently verified `@zof-ai` scope ownership and an authenticated publisher. Registry publication and registry install acceptance are not claimed by a GitHub release.
+
+After publication, retrieve assets anonymously, verify their checksums, install in a clean project, and execute documented examples. Check every intended distribution channel separately. Missing or unrun acceptance is not a pass. Hosted Zof integration remains unavailable until an approved public contract and real service acceptance exist.
+
+CI targets Node 22/24 on Linux/macOS. File operations require Python 3.10 or newer and fail closed on Windows; in-memory JSON operations remain portable. Only completed jobs establish platform verification. Docker acceptance runs on Linux; editor acceptance exercises a real VS Code host.

package/docs/SANDBOX.md

@@ -0,0 +1,17 @@
+# Sandbox controls and privacy
+
+Candidate code runs only inside the locally available pinned image:
+
+`node@sha256:43ac6c60b8f89723f746e8a92ce91abd5017e627ce1ddfe4238355d3a30b772c`
+
+The image is the Node 22 slim reference used for this release candidate. Image availability must be arranged explicitly by an authorized operator. No automatic pull or remote Docker endpoint is accepted.
+
+Controls: network none, read-only root filesystem, UID/GID 65534, all capabilities dropped, no-new-privileges, Docker's default seccomp profile, PID limit 32, memory/swap 128 MiB, CPU limit 0.5, Node heap 64 MiB, and a 16 MiB noexec/nosuid temporary filesystem. There are no published ports, privileged mode, host PID namespaces, production directories, credential mounts or Docker socket mounts. The only bind mount is the copied task staging directory, read-only.
+
+The host staging parent has mode 0700; its mounted child permits the non-root container to read only the copied candidate, current input and fixed harness. The original source directory and expected values are not mounted. At most two evaluations may be active per process; additional calls are rejected. Each case gets a unique container name. A wall-clock timeout kills the Docker client and triggers forced container removal; bounded follow-up checks confirm absence. Cleanup uncertainty is reported as an error and stops further cases. Staging files are removed in finally blocks.
+
+Candidate stdout/stderr share a 64 KiB limit. Stderr is discarded immediately. Stdout is parsed only as a JSON result, compared privately with expected output, and discarded. No raw outputs, logs, source, expected values, arbitrary environment or Docker diagnostics are returned. Only explicit agent metadata, outcomes and safe fixed codes are recorded. Known secret patterns are redacted in supplied labels; do not use production credentials in task data.
+
+Docker shares a kernel with its Linux host and is not a complete defense against container-runtime/kernel vulnerabilities. This release demonstrates bounded local isolation for synthetic tasks, not multi-tenant hostile-code hosting. Do not expose it as a public compute service or run it with privileged options. There is no unsafe execution mode. Resource limits apply per process/container; an external service would additionally need server-side global quota/authentication/isolation controls and is not implemented.
+
+Input file reads require a stable caller-controlled source root, reject symlinks/traversal and never fetch URLs. Local Docker daemon access is a prerequisite and is trusted. Windows engine behavior and hosted runner matrices have not been accepted in this local release candidate; measured container acceptance uses local Docker Desktop on macOS with Linux containers.

package/docs/TASKS.md

@@ -0,0 +1,15 @@
+# Task format 1.0.0
+
+```json
+{"version":"1.0.0","id":"double","description":"Double the supplied number","cases":[{"name":"positive","input":2,"expected":4},{"name":"zero","input":0,"expected":0}]}
+```
+
+Tasks contain only id, version, description and 1–32 cases. Each case contains a bounded name, JSON input and JSON expected value. Unknown properties are rejected. Cases cannot contain commands, imports, harness source, environment mappings or endpoint settings. A case is at most 32 KiB, with the shared evidence JSON depth and node bounds.
+
+Candidate source is a single `.mjs` file of at most 128 KiB. It may use Node built-ins within the container, but has no installed third-party dependencies and cannot fetch them. Multi-file changes, arbitrary repository build scripts, package installation and non-JavaScript candidates are currently unsupported. Save the resulting self-contained module snapshot for evaluation; this tool does not apply patches to a repository.
+
+The parent compares actual JSON output with the expected value using strict deep equality. It does not trust candidate-supplied pass labels. Extra logging causes invalid protocol output and cannot be used to forge a summary report. Public tasks can be overfit and are not hidden security tests. Test passing is a measured observation for these cases, not proof of general correctness.
+
+Reasons are MATCH, MISMATCH, CANDIDATE_ERROR, TIMEOUT, OUTPUT_LIMIT, SANDBOX_UNAVAILABLE, or CLEANUP_UNCONFIRMED. No caught candidate exception text or output is included. Missing sandbox produces missing/unknown evidence. Candidate mismatches/errors produce failed tests. Timeout/output limit yields partial/fail; unconfirmed cleanup or infrastructure loss yields error/unknown and stops remaining cases.
+
+The optional agent name/version is an explicit assertion supplied by the user. No agent identity is authenticated and no model ranking is claimed. Core evidence source repository/commit are unknown because only a selected file snapshot is loaded. Evaluation metadata records task version, pinned image and time limit. The outer environment field identifies the host runner runtime, not the container's untrusted self-report.

package/examples/correct.mjs

@@ -0,0 +1 @@
+export function solve(payments){const seen=new Set();return payments.reduce((sum,p)=>{if(seen.has(p.id))return sum;seen.add(p.id);return sum+p.amount;},0);}

package/examples/incorrect.mjs

@@ -0,0 +1 @@
+export function solve(payments){return payments.reduce((sum,p)=>sum+p.amount,0);}

package/examples/profile.mjs

@@ -0,0 +1 @@
+export function solve(profile){return {name:profile.name,displayName:profile.name};}

package/package.json

@@ -1,6 +1,63 @@
 {
   "name": "@zof-ai/agent-evals",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Constrained local Docker evaluation of untrusted JavaScript candidates",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "zof-agent-evals": "./dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE",
+    "CHANGELOG.md",
+    "docs",
+    "examples",
+    "SECURITY.md",
+    "CONTRIBUTING.md"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Zof-AI/zof-agent-evals.git"
+  },
+  "homepage": "https://github.com/Zof-AI/zof-agent-evals#readme",
+  "bugs": {
+    "url": "https://github.com/Zof-AI/zof-agent-evals/issues"
+  },
+  "keywords": [
+    "verification",
+    "testing",
+    "zof"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "test": "node --test tests/*.test.mjs",
+    "check": "npm run build && npm test",
+    "test:container": "node tests/container.mjs",
+    "test:package": "node tests/package.mjs"
+  },
+  "devDependencies": {
+    "typescript": "5.9.3",
+    "@types/node": "22.19.11"
+  },
+  "dependencies": {
+    "@zof-ai/evidence-spec": "0.1.0",
+    "@zof-ai/reliability-bench": "0.1.0"
+  }
+}