eval-quality 0.1.0

package/dist/testing/probe-conformance.js

@@ -0,0 +1,186 @@
+import { probeParsers } from '../ports/environment-probe-port.js';
+import { buildScenario, countCalls, describeThrown, disposeScenario, faultView, reportOf, runSharedAssertions, settle, titledOutcome, withDispose, } from './conformance.js';
+function authorizationFor(subject, request) {
+    return subject.policy.authorizations.find((each) => each.interfaceId === request.interfaceId);
+}
+const DENIED = 'forbidden-target';
+const CAPPED = 'budget-exhausted';
+const PROBE_ASSERTIONS = [
+    {
+        // Without this one, a subject that denies everything scores twelve of
+        // twelve on default-deny while being useless. AD-35 spends a clause on
+        // why a blanket loopback ban is wrong.
+        id: 'probe/allow-authorized-loopback',
+        title: 'an explicitly authorized target is reached and observed',
+        request: (subject) => subject.authorizedRequest,
+        expectation: { kind: 'resolves' },
+    },
+    {
+        // A subject that throws on 5xx fails here and nowhere else, and it makes
+        // AD-10's seeded-fault check unimplementable.
+        id: 'probe/observe-anomalous-status',
+        title: 'a 500 from an authorized target is an observation, not a fault',
+        request: (subject) => subject.faultingRequest,
+        expectation: {
+            kind: 'resolves',
+            check: (observation) => observation.status === 500
+                ? undefined
+                : `observed status ${observation.status}, expected 500`,
+        },
+    },
+    {
+        id: 'probe/deny-unmapped-interface',
+        title: 'an interface the mapping does not name is refused before a packet leaves',
+        request: (subject) => subject.unmappedRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        id: 'probe/deny-unauthorized-loopback',
+        title: 'a loopback address no authorization names is denied',
+        request: (subject) => subject.deniedAddressRequests.loopback,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        id: 'probe/deny-private',
+        title: 'a private RFC 1918 address is denied',
+        request: (subject) => subject.deniedAddressRequests.private,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        id: 'probe/deny-link-local',
+        title: 'a link-local address is denied',
+        request: (subject) => subject.deniedAddressRequests.linkLocal,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        id: 'probe/deny-metadata',
+        title: 'a cloud metadata address is denied',
+        request: (subject) => subject.deniedAddressRequests.metadata,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        id: 'probe/deny-unauthorized-method',
+        title: 'a method the authorization does not list is denied',
+        request: (subject) => subject.unauthorizedMethodRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        id: 'probe/deny-unauthorized-scheme',
+        title: 'a scheme the authorization does not name is denied',
+        request: (subject) => subject.unauthorizedSchemeRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 0,
+    },
+    {
+        // The first hop is authorized and happens. The redirect target is
+        // revalidated and refused, so the second hop never runs. An adapter that
+        // follows the redirect resolves, and fails here.
+        id: 'probe/deny-on-redirect',
+        title: 'a redirect to a denied target is revalidated and refused',
+        request: (subject) => subject.redirectingRequest,
+        expectation: { kind: 'rejects', code: DENIED },
+        expectedCalls: () => 1,
+    },
+    {
+        // Every hop is authorized, so only the count can refuse it. That is the
+        // half `deny-on-redirect` cannot reach.
+        id: 'probe/cap-redirects',
+        title: 'a chain of authorized redirects past maxRedirects is capped',
+        request: (subject) => subject.overRedirectRequest,
+        expectation: { kind: 'rejects', code: CAPPED },
+        expectedCalls: (subject) => {
+            const authorization = authorizationFor(subject, subject.overRedirectRequest);
+            return authorization === undefined
+                ? "the subject's policy names no authorization for overRedirectRequest.interfaceId, so maxRedirects is unknown"
+                : authorization.maxRedirects + 1;
+        },
+    },
+    {
+        id: 'probe/cap-response-bytes',
+        title: 'a response past maxResponseBytes is a cap, not a policy denial',
+        request: (subject) => subject.oversizeResponseRequest,
+        expectation: { kind: 'rejects', code: CAPPED },
+    },
+    {
+        id: 'probe/cap-elapsed',
+        title: 'a response past maxElapsedMs is a cap, not an abort',
+        request: (subject) => subject.slowRequest,
+        expectation: { kind: 'rejects', code: CAPPED },
+    },
+];
+function checkResolved(assertion, expectation, value) {
+    const parsed = probeParsers.response.safeParse(value);
+    if (!parsed.success) {
+        return titledOutcome(assertion.id, assertion.title, false, 'the resolved value is not a schema-valid ProbeObservation');
+    }
+    const complaint = expectation.check?.(parsed.data);
+    return titledOutcome(assertion.id, assertion.title, complaint === undefined, complaint ?? '');
+}
+function checkRejected(assertion, expectedCode, error) {
+    const view = faultView(error);
+    if (view === undefined) {
+        return titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(error)}, which carries no declared AD-28 code`);
+    }
+    return titledOutcome(assertion.id, assertion.title, view.code === expectedCode, `rejected with code "${view.code}", expected "${expectedCode}"`);
+}
+function checkCalls(assertion, subject, built, base) {
+    if (assertion.expectedCalls === undefined)
+        return base;
+    const expected = assertion.expectedCalls(subject);
+    if (typeof expected === 'string') {
+        return { ...base, passed: false, detail: expected };
+    }
+    const actual = countCalls(built);
+    if (typeof actual === 'string') {
+        return { ...base, passed: false, detail: actual };
+    }
+    if (actual === expected)
+        return base;
+    return {
+        ...base,
+        passed: false,
+        detail: `${base.passed ? '' : `${base.detail}; `}underlyingCalls() was ${actual}, expected ${expected}`,
+    };
+}
+async function runProbeAssertion(assertion, subject) {
+    const run = await buildScenario(subject, 'resolves');
+    if (!run.ok) {
+        return titledOutcome(assertion.id, assertion.title, false, run.detail);
+    }
+    const settled = await settle(run.built.port(assertion.request(subject), new AbortController().signal));
+    let result;
+    if (assertion.expectation.kind === 'resolves') {
+        result =
+            settled.kind === 'resolved'
+                ? checkResolved(assertion, assertion.expectation, settled.value)
+                : titledOutcome(assertion.id, assertion.title, false, `rejected with ${describeThrown(settled.error)} instead of returning an observation`);
+    }
+    else {
+        result =
+            settled.kind === 'rejected'
+                ? checkRejected(assertion, assertion.expectation.code, settled.error)
+                : titledOutcome(assertion.id, assertion.title, false, `resolved instead of rejecting with "${assertion.expectation.code}"`);
+    }
+    result = checkCalls(assertion, subject, run.built, result);
+    return withDispose(result, await disposeScenario(run.built));
+}
+/**
+ * Nineteen outcomes: the six shared assertions plus AD-35's thirteen.
+ * `maxRequestBytes` is the one cap with no assertion; the request shape is the
+ * suite's own, so the suite cannot make a subject emit an oversize request. It
+ * stays declared and adapter-enforced.
+ */
+export async function runEnvironmentProbePortConformance(subject) {
+    const shared = await runSharedAssertions('probe', subject, probeParsers.response);
+    const additional = [];
+    for (const assertion of PROBE_ASSERTIONS) {
+        additional.push(await runProbeAssertion(assertion, subject));
+    }
+    return reportOf(subject.name, 'environment-probe', [...shared, ...additional]);
+}

package/package.json

@@ -0,0 +1,130 @@
+{
+  "name": "eval-quality",
+  "version": "0.1.0",
+  "description": "Compile disciplined Behavioral Evaluation Contracts and score their ability to catch known defects.",
+  "author": "Murat Ozcan",
+  "license": "Apache-2.0",
+  "private": false,
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "eval-quality": "dist/cli/main.js"
+  },
+  "type": "module",
+  "keywords": [
+    "eval",
+    "evals",
+    "evaluation",
+    "agent",
+    "agent-eval",
+    "llm",
+    "behavioral-contract",
+    "oracle",
+    "contract-strength",
+    "testing",
+    "ai"
+  ],
+  "files": [
+    "dist",
+    "schemas",
+    "corpus",
+    "README.md",
+    "LICENSE"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./adapters": {
+      "types": "./dist/adapters/index.d.ts",
+      "default": "./dist/adapters/index.js"
+    },
+    "./conformance": {
+      "types": "./dist/testing/index.d.ts",
+      "default": "./dist/testing/index.js"
+    },
+    "./schemas/*": "./schemas/*",
+    "./corpus/*": "./corpus/*",
+    "./package.json": "./package.json"
+  },
+  "homepage": "https://github.com/bmad-code-org/bmad-eval-quality#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/bmad-code-org/bmad-eval-quality.git"
+  },
+  "bugs": {
+    "url": "https://github.com/bmad-code-org/bmad-eval-quality/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=22.20.0"
+  },
+  "scripts": {
+    "hooks:install": "husky",
+    "build": "tsc -p tsconfig-build.json",
+    "clean": "rm -rf dist",
+    "typecheck": "tsc --noEmit -p tsconfig.json",
+    "lint": "biome check .",
+    "lint:fix": "biome check --write .",
+    "format": "biome format --write .",
+    "check:docs": "node scripts/check-docs.mjs",
+    "check:doc-invocations": "node scripts/check-doc-invocations.mjs",
+    "lint:spine": "python3 scripts/spine-lint/lint_spine.py --registry-ad 5 --workspace-root . --fail-on high",
+    "test:spine-lint": "uv run --with pytest pytest scripts/spine-lint/tests -q",
+    "build:shareable": "node scripts/build-shareable.mjs",
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage",
+    "test:conformance": "npm run build && vitest run tests/adapters tests/testing tests/conformance",
+    "test:watch": "vitest",
+    "check:vectors": "python3 tests/fixtures/derive_vectors.py --check",
+    "generate:schemas": "node scripts/generate-schemas.ts",
+    "check:schemas": "node scripts/check-schemas.ts",
+    "check:ad5-registry": "node scripts/check-ad5-registry.ts",
+    "check:ad28-registry": "node scripts/check-ad28-registry.ts",
+    "generate:ad31-table": "node scripts/generate-ad31-table.ts",
+    "check:ad31-table": "node scripts/check-ad31-table.ts",
+    "check:layers": "node scripts/check-dependency-direction.ts",
+    "check:lineage": "node scripts/check-lineage-ownership.ts",
+    "check:boundary": "node scripts/check-package-boundary.ts",
+    "generate:dev-corpus": "node scripts/generate-dev-corpus.ts",
+    "check:corpus": "node scripts/check-dev-corpus.ts",
+    "check:website-deps": "node scripts/audit-lockfile-age.mjs --lockfile website/package-lock.json && node scripts/check-licenses.mjs --lockfile website/package-lock.json",
+    "check:shareable": "node scripts/check-shareable.mjs",
+    "bench:digest": "node scripts/bench-digest.ts",
+    "docs:dev": "npm --prefix website run dev",
+    "docs:build": "node tools/build-docs.mjs",
+    "docs:preview": "npm --prefix website run preview",
+    "docs:validate-links": "node tools/validate-doc-links.js",
+    "release:prepare": "node scripts/release-prepare.mjs",
+    "release:publish": "gh workflow run publish.yml --ref main",
+    "validate": "npm run build && npm run typecheck && npm run lint && npm run check:docs && npm run check:doc-invocations && npm run check:shareable && npm run lint:spine && npm run check:vectors && npm run check:schemas && npm run check:ad5-registry && npm run check:ad28-registry && npm run check:ad31-table && npm run check:layers && npm run check:lineage && npm run check:boundary && npm run check:corpus && npm run check:website-deps && npm run test:coverage",
+    "prepack": "npm run clean && npm run build",
+    "prepublishOnly": "node scripts/assert-publish-authorized.mjs"
+  },
+  "dependencies": {
+    "zod": "4.4.3"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "2.5.10",
+    "@types/node": "22.20.1",
+    "@vitest/coverage-v8": "4.1.11",
+    "ajv": "8.20.0",
+    "husky": "9.1.7",
+    "lint-staged": "16.4.0",
+    "marked": "18.0.10",
+    "typescript": "7.0.2",
+    "vitest": "4.1.11"
+  },
+  "overrides": {
+    "vite": "7.3.6"
+  },
+  "lint-staged": {
+    "*.{ts,js,json}": [
+      "biome check --write --no-errors-on-unmatched"
+    ]
+  }
+}

package/schemas/artifact-reference.schema.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:eval-quality:schema:artifact-reference",
+  "oneOf": [
+    {
+      "type": "object",
+      "properties": {
+        "storage": {
+          "type": "string",
+          "const": "public"
+        },
+        "path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "privateRef": {
+          "type": "null"
+        },
+        "digest": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+        }
+      },
+      "required": [
+        "storage",
+        "path",
+        "privateRef",
+        "digest"
+      ],
+      "additionalProperties": false
+    },
+    {
+      "type": "object",
+      "properties": {
+        "storage": {
+          "type": "string",
+          "const": "private"
+        },
+        "path": {
+          "type": "null"
+        },
+        "privateRef": {
+          "type": "string",
+          "minLength": 1,
+          "description": "An opaque handle resolved through the corpus port under AD-8, never a filesystem path and never a URL. AD-18 forbids a private path from entering any artifact, and this is the field a caller would otherwise reach for."
+        },
+        "digest": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+        }
+      },
+      "required": [
+        "storage",
+        "path",
+        "privateRef",
+        "digest"
+      ],
+      "additionalProperties": false
+    }
+  ],
+  "description": "A reference to an artifact, public by path or private by opaque handle. Succeeds the prior-art `artifact-ref` schema per AD-24, with its two `if`/`then` branches re-expressed as a discriminated union per AD-13 and its `sha256` field renamed to `digest` against the shared AD-27 primitive. Alone among the twelve interchange artifacts this one carries NO `schemaVersion` and NO AD-29 lineage: it is a reference shape embedded inside other artifacts rather than one that crosses the package boundary alone, so versioning it would add a key to every finding and every manifest entry for no reader. It is in the inventory because it is published, not because it is exchanged; the exemption is asserted by a test against the registry's `carriesLineage` flag."
+}