eval-quality 0.1.0

package/LICENSE

@@ -0,0 +1,190 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1.  Definitions.
+
+    "License" shall mean the terms and conditions for use, reproduction,
+    and distribution as defined by Sections 1 through 9 of this document.
+
+    "Licensor" shall mean the copyright owner or entity authorized by
+    the copyright owner that is granting the License.
+
+    "Legal Entity" shall mean the union of the acting entity and all
+    other entities that control, are controlled by, or are under common
+    control with that entity. For the purposes of this definition,
+    "control" means (i) the power, direct or indirect, to cause the
+    direction or management of such entity, whether by contract or
+    otherwise, or (ii) ownership of fifty percent (50%) or more of the
+    outstanding shares, or (iii) beneficial ownership of such entity.
+
+    "You" (or "Your") shall mean an individual or Legal Entity
+    exercising permissions granted by this License.
+
+    "Source" form shall mean the preferred form for making modifications,
+    including but not limited to software source code, documentation
+    source, and configuration files.
+
+    "Object" form shall mean any form resulting from mechanical
+    transformation or translation of a Source form, including but
+    not limited to compiled object code, generated documentation,
+    and conversions to other media types.
+
+    "Work" shall mean the work of authorship, whether in Source or
+    Object form, made available under the License, as indicated by a
+    copyright notice that is included in or attached to the work
+    (an example is provided in the Appendix below).
+
+    "Derivative Works" shall mean any work, whether in Source or Object
+    form, that is based on (or derived from) the Work and for which the
+    editorial revisions, annotations, elaborations, or other modifications
+    represent, as a whole, an original work of authorship. For the purposes
+    of this License, Derivative Works shall not include works that remain
+    separable from, or merely link (or bind by name) to the interfaces of,
+    the Work and Derivative Works thereof.
+
+    "Contribution" shall mean any work of authorship, including
+    the original version of the Work and any modifications or additions
+    to that Work or Derivative Works thereof, that is intentionally
+    submitted to the Licensor for inclusion in the Work by the copyright owner
+    or by an individual or Legal Entity authorized to submit on behalf of
+    the copyright owner. For the purposes of this definition, "submitted"
+    means any form of electronic, verbal, or written communication sent
+    to the Licensor or its representatives, including but not limited to
+    communication on electronic mailing lists, source code control systems,
+    and issue tracking systems that are managed by, or on behalf of, the
+    Licensor for the purpose of discussing and improving the Work, but
+    excluding communication that is conspicuously marked or otherwise
+    designated in writing by the copyright owner as "Not a Contribution."
+
+    "Contributor" shall mean Licensor and any individual or Legal Entity
+    on behalf of whom a Contribution has been received by the Licensor and
+    subsequently incorporated within the Work.
+
+2.  Grant of Copyright License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    copyright license to reproduce, prepare Derivative Works of,
+    publicly display, publicly perform, sublicense, and distribute the
+    Work and such Derivative Works in Source or Object form.
+
+3.  Grant of Patent License. Subject to the terms and conditions of
+    this License, each Contributor hereby grants to You a perpetual,
+    worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+    (except as stated in this section) patent license to make, have made,
+    use, offer to sell, sell, import, and otherwise transfer the Work,
+    where such license applies only to those patent claims licensable
+    by such Contributor that are necessarily infringed by their
+    Contribution(s) alone or by combination of their Contribution(s)
+    with the Work to which such Contribution(s) was submitted. If You
+    institute patent litigation against any entity (including a
+    cross-claim or counterclaim in a lawsuit) alleging that the Work
+    or a Contribution incorporated within the Work constitutes direct
+    or contributory patent infringement, then any patent licenses
+    granted to You under this License for that Work shall terminate
+    as of the date such litigation is filed.
+
+4.  Redistribution. You may reproduce and distribute copies of the
+    Work or Derivative Works thereof in any medium, with or without
+    modifications, and in Source or Object form, provided that You
+    meet the following conditions:
+
+    (a) You must give any other recipients of the Work or
+    Derivative Works a copy of this License; and
+
+    (b) You must cause any modified files to carry prominent notices
+    stating that You changed the files; and
+
+    (c) You must retain, in the Source form of any Derivative Works
+    that You distribute, all copyright, patent, trademark, and
+    attribution notices from the Source form of the Work,
+    excluding those notices that do not pertain to any part of
+    the Derivative Works; and
+
+    (d) If the Work includes a "NOTICE" text file as part of its
+    distribution, then any Derivative Works that You distribute must
+    include a readable copy of the attribution notices contained
+    within such NOTICE file, excluding any notices that do not
+    pertain to any part of the Derivative Works, in at least one
+    of the following places: within a NOTICE text file distributed
+    as part of the Derivative Works; within the Source form or
+    documentation, if provided along with the Derivative Works; or,
+    within a display generated by the Derivative Works, if and
+    wherever such third-party notices normally appear. The contents
+    of the NOTICE file are for informational purposes only and
+    do not modify the License. You may add Your own attribution
+    notices within Derivative Works that You distribute, alongside
+    or as an addendum to the NOTICE text from the Work, provided
+    that such additional attribution notices cannot be construed
+    as modifying the License.
+
+    You may add Your own copyright statement to Your modifications and
+    may provide additional or different license terms and conditions
+    for use, reproduction, or distribution of Your modifications, or
+    for any such Derivative Works as a whole, provided Your use,
+    reproduction, and distribution of the Work otherwise complies with
+    the conditions stated in this License.
+
+5.  Submission of Contributions. Unless You explicitly state otherwise,
+    any Contribution intentionally submitted for inclusion in the Work
+    by You to the Licensor shall be under the terms and conditions of
+    this License, without any additional terms or conditions.
+    Notwithstanding the above, nothing herein shall supersede or modify
+    the terms of any separate license agreement you may have executed
+    with Licensor regarding such Contributions.
+
+6.  Trademarks. This License does not grant permission to use the trade
+    names, trademarks, service marks, or product names of the Licensor,
+    except as required for reasonable and customary use in describing the
+    origin of the Work and reproducing the content of the NOTICE file.
+
+7.  Disclaimer of Warranty. Unless required by applicable law or
+    agreed to in writing, Licensor provides the Work (and each
+    Contributor provides its Contributions) on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+    implied, including, without limitation, any warranties or conditions
+    of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+    PARTICULAR PURPOSE. You are solely responsible for determining the
+    appropriateness of using or redistributing the Work and assume any
+    risks associated with Your exercise of permissions under this License.
+
+8.  Limitation of Liability. In no event and under no legal theory,
+    whether in tort (including negligence), contract, or otherwise,
+    unless required by applicable law (such as deliberate and grossly
+    negligent acts) or agreed to in writing, shall any Contributor be
+    liable to You for damages, including any direct, indirect, special,
+    incidental, or consequential damages of any character arising as a
+    result of this License or out of the use or inability to use the
+    Work (including but not limited to damages for loss of goodwill,
+    work stoppage, computer failure or malfunction, or any and all
+    other commercial damages or losses), even if such Contributor
+    has been advised of the possibility of such damages.
+
+9.  Accepting Warranty or Additional Liability. While redistributing
+    the Work or Derivative Works thereof, You may choose to offer,
+    and charge a fee for, acceptance of support, warranty, indemnity,
+    or other liability obligations and/or rights consistent with this
+    License. However, in accepting such obligations, You may act only
+    on Your own behalf and on Your sole responsibility, not on behalf
+    of any other Contributor, and only if You agree to indemnify,
+    defend, and hold each Contributor harmless for any liability
+    incurred by, or claims asserted against, such Contributor by reason
+    of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+Copyright 2026 Murat Ozcan
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

package/README.md

@@ -0,0 +1,355 @@
+# `eval-quality`
+
+### `eval-quality` does three things
+
+1. **Compile**: validate and normalize an eval contract into a machine-readable artifact.
+2. **Seal**: render the brief for the independent evaluator while hiding the planted bug and scoring answer.
+3. **Preflight**: verify baseline environment readiness and probe reachability before running an evaluator.
+
+Scoring is the next milestone: comparing the evaluator’s completed findings with the hidden bug signature to determine whether the bug was actually caught.
+
+### What is the eval spec?
+
+It is the test.
+
+More precisely, it is the evaluator’s instructions for how to expose a failure and what evidence counts as finding it.
+
+It defines:
+
+- the behavior being evaluated;
+- the probes the evaluator should perform;
+- the evidence it should inspect;
+- the negative behavior it must rule out;
+- the oracle that determines pass or fail.
+
+For example:
+
+> Send malformed input.
+> Confirm the request fails.
+> Inspect the full response body.
+> Confirm the expected error.
+> Verify that no record was created.
+
+The planted bug might be:
+
+> The API returns the correct error but still creates the record.
+
+A weak eval checks only the response and misses the bug.
+
+A strong eval checks the response **and** persistence, so it catches the bug.
+
+### Caveman summary
+
+Write the eval. Hide the bug. See if the eval catches it.
+
+## Key Concepts
+
+Understanding `eval-quality` requires three core artifacts:
+
+| Concept | What it is | Example |
+| --- | --- | --- |
+| **Contract** (`eval-contract.json`) | The test specification defining expected behaviors, oracles (checks), permitted tools, and evidence rules. | "Verify API rejects invalid JWT and creates zero database records." |
+| **Probe** (`probe.json`) | A diagnostic request sent to the environment to test baseline state, reachability, or fault injection. | A request sending an expired token to `/api/v1/resource`. |
+| **Observation** (`observation.json`) | The empirical response evidence recorded when a probe is executed against the environment. | `{ responseStatus: 401, responseBody: { error: "token_expired" } }` |
+
+### How They Fit Together
+
+```text
+┌────────────────────────┐      ┌────────────────────────┐      ┌────────────────────────┐
+│     Eval Contract      │      │         Probe          │      │      Observation       │
+│  (The Specification)   │ ───► │ (Diagnostic Request)   │ ───► │   (Empirical Result)   │
+│  "What should happen"  │      │  "Send malformed JWT"  │      │   "Got 401, 0 records" │
+└────────────────────────┘      └────────────────────────┘      └────────────────────────┘
+```
+
+## Elaboration
+
+Compile disciplined agent eval contracts, then check whether those contracts can catch known bugs.
+
+An agent can produce an answer that reads as correct and is materially wrong. An eval can make the same mistake.
+
+Weak oracle:
+
+```text
+Check malformed input is handled correctly.
+```
+
+An evaluator given that instruction sends one malformed request, sees an error come back, and reports success. The record that should never have been created was created anyway. Nobody looked.
+
+Strong oracle:
+
+```text
+Send malformed input. Verify the request fails, inspect the full response body,
+confirm the specific error, and confirm no record was created.
+```
+
+A passing eval says little when the contract never asked for the probe that would expose the failure. Testing whether the eval can catch a failure you already know about is the first check worth running.
+
+```text
+product spec
+  → Behavioral Evaluation Contract
+  → known defect or gameability probe
+  → independent evaluator
+  → per-oracle evidence and a gate decision
+```
+
+## What each part provides
+
+`eval-quality` provides:
+
+- the Behavioral Evaluation Contract schema
+- the oracle vocabulary and authoring rules
+- the contract compiler
+- the environment pre-flight
+- Eval Contract strength scoring (next milestone)
+- versioned evidence output and PASS / WAIVED / CONCERNS / FAIL governance (next milestone)
+
+The caller provides:
+
+- execution of its chosen agent, harness, or person
+- repeated trials
+- cost accounting
+- the live system and environment-probe implementation
+- a sealed run record returned for ingestion
+
+`eval-quality` executes nothing: it never spawns a process, calls a model, drives a system under test,
+or invokes a judge. Its pure stages are compile, seal, ingest, pre-flight, score, and emit; compile,
+seal, and pre-flight ship, and ingest, score, and emit are the next milestone. Pre-flight probes the
+fixture through the environment-probe port, so a contract that declares a fixture reset
+needs the caller's probe policy to authorize that operation's method as well as the read methods
+every other pre-flight leg uses. Engine integration is a later adapter behind a port, not a v0
+dependency. See
+[ADR-004](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-004-execution-boundary.md).
+
+## Who it is for
+
+Teams shipping AI agents, coding skills, review bots, MCP-based assistants, or automated test-generation systems, and teams operating human-on-the-loop or dark-factory delivery.
+
+Use `eval-quality` when all three are true:
+
+- An agent, skill, or model judgment is involved.
+- A plausible-looking output can still be materially wrong.
+- Observable evidence or probes can expose the wrong behavior.
+
+Deterministic work does not need it and already has cheaper, stronger evidence from unit, integration, contract, E2E and performance testing.
+
+## Behavioral Evaluation Contracts
+
+A **Behavioral Evaluation Contract** is a versioned specification of the behaviors to probe, the evidence to collect, the negative cases to exercise, and the rules that decide whether the system passes or fails. **Eval Contract** is the shorthand used from here on. The individual checks inside it are **oracles**. The contract carries no prescribed action sequence; the evaluator chooses its own path.
+
+The authoring discipline is a small set of rules that survived the experiments: separate the success indicator from the body, read the whole body, probe malformed and negative inputs, verify per record, and cross-check sibling parameters and sibling tools.
+
+A compiler enforces these rules mechanically against the contract artifact, in three classes. Structural errors fail compilation. Coverage gaps score down without blocking. A waived pattern is allowed when it records the named rule, a rationale, a machine-checkable condition, and the approval.
+
+Rubrics compile under the same discipline: an anchored scale, a bounded length, named failure-mode penalties, rubric identifiers unique across the contract and criterion identifiers unique inside their own rubric, every criterion stating a question, and every criterion's evidence pointer resolving against the declared interfaces. Authored rubric text that asks a judge to grade the subject's own stated reasoning fails a closed-vocabulary check over the wording.
+
+## How Eval Contract strength scoring works
+
+Do not trust a contract because it looks thorough. Put a known defect behind it, run the evaluator, and check whether the contract's oracles caused the defect to be caught.
+
+Two probe classes go behind a contract, and a strong contract rejects both:
+
+- **Defect probes**, where the behavior is simply wrong.
+- **Gameability probes**, where the behavior looks compliant while dodging the oracle's intent. A test that raises coverage while asserting nothing is the familiar version of this.
+
+Probes come from qualified historical defects or verified controlled mutations. The corpus separates a visible development set from an immutable sealed set for each scoring version.
+
+Every required oracle check resolves to exactly one state, and the state travels with the result, so
+"the check reported" is never sufficient on its own: `caught`, `confirmed`, `missed`,
+`passed-clean-control`, `false-positive`, `abstained`, `bypassed`, `unreached`, `oracle-error`,
+`judge-error`, `infrastructure-error`, or `not-applicable`.
+
+A required oracle that missed, abstained, errored, or is absent prevents PASS, and a high overall score never overrides it. An infrastructure error or a failed environment pre-flight is not a behavioral result at all; it invalidates the run and is re-executed rather than scored.
+
+## Using it
+
+`eval-quality` is its own repository and package, not a plugin inside another framework.
+
+The **library** is the primary surface. It exports the contract schema, the oracle vocabulary, the compiler, the pre-flight, and the evidence types. The published typed schema is what lets coding agents author contracts correctly by default, which is how the discipline scales beyond the people who went looking for the tool.
+
+The **CLI** wraps the same library for callers that cannot import TypeScript: CI jobs, GitHub Actions, PR-review and unit-test bots, other frameworks' skills, and any agent permitted to run a shell command.
+
+### What the CLI Commands Do
+
+- **`compile`**: Typechecks an authored `eval-contract.json`. Verifies that all behaviors, oracles, rubrics, and sensitivity witnesses comply with structural and authoring rules.
+- **`seal`**: Generates a `sealed-evaluator-brief.json` by stripping secret defect signatures, planted answers, and author commentary. The brief carries only the directions and safety bounds the evaluator needs.
+- **`preflight`**: Reduces caller-supplied probe observations against the contract to verify environment baseline readiness and probe reachability. Halts early with exit code `3` if the environment is unready.
+
+### Running the CLI
+
+Every command runs through `npx` without installing anything:
+
+```bash
+npx eval-quality compile --in contract.json --out ./eval-out
+
+npx eval-quality seal --in contract.json --out ./eval-out
+
+npx eval-quality preflight --contract contract.json \
+  --probes probes.json --observations observations.json \
+  --run-id 2026-08-28-a --out ./eval-out
+```
+
+Every command is non-interactive: no prompt, no terminal check, and no behaviour that differs when
+stdin is a pipe. Each one is a single call into the library plus artifact serialization.
+
+**Input and output.** An input flag left out reads stdin, and `-` names stdin explicitly; at most one
+input may be `-`. Without `--out` the artifact goes to stdout, so a command composes with a pipe.
+An `--out` ending in `.json` is a file path; anything else is a directory, and the artifact is
+written to `<target>/<kind>.json` where `kind` is `eval-contract`, `sealed-evaluator-brief`, or
+`preflight-verdict`. Diagnostics and errors go to stderr, always, so stdout carries the artifact
+alone.
+
+**Exit codes.**
+
+| Exit Code | Meaning |
+| --- | --- |
+| `0` | success, and every verdict other than FAIL or a promoted CONCERNS |
+| `1` | CONCERNS promoted by `--strict` |
+| `2` | FAIL |
+| `3` | invalid: a pre-flight verdict that did not pass |
+| `4` | structural failure |
+| `5` | runtime fault |
+| `64` | usage error |
+
+Codes 1 and 2 report a scored verdict. Scoring ships in a later release, so no command here reaches
+either yet, and `--strict` changes no code this binary produces. The flag and the two codes are part
+of the published contract, so they are documented now and wired now.
+
+`--strict` is the gate-promotion flag and is accepted on every command. `--strict-inputs` and
+`--no-strict-inputs` are a different switch: they set the compiler's input strictness, which is on
+by default.
+
+**The published JSON Schema.** A consumer that does not read TypeScript validates against the
+twelve generated documents, published at the `eval-quality/schemas/*` subpath:
+
+```ts
+import spec from 'eval-quality/schemas/eval-contract.schema.json' with { type: 'json' }
+```
+
+The import attribute is required: ESM on Node 22 and 24 both throw `ERR_IMPORT_ATTRIBUTE_MISSING`
+without it. The development corpus ships the same way, at `eval-quality/corpus/dev/`, so an adopter
+can read real compiled contracts and one compiled-and-sealed pair without cloning this repository.
+
+## Relationship with BMad and TEA
+
+The dependency runs one way: TEA uses `eval-quality`, and `eval-quality` knows nothing about TEA.
+
+```mermaid
+graph LR
+  TEA["TEA<br/>(reference authoring client)"] -- "drafts a contract, then calls" --> EQ["eval-quality<br/>(this package)"]
+```
+
+TEA is the reference authoring client. It reads BMad planning artifacts, notices eval-relevant work, drafts a contract, and calls this package. It is not co-installed, and `eval-quality` holds no knowledge of TEA, BMad, or any planning-artifact format.
+
+Any human, bot, CI job, skill, or other framework can author a contract and use `eval-quality` directly. The discipline still applies, because the compiler judges the artifact rather than trusting whoever produced it.
+
+Evaluator runs remain isolated to prevent builder-context leakage and preserve traceability. Stronger contract oracles produced the measured detection improvement.
+
+### Real-World Walkthrough: Testing a `bmad-tea` Knowledge Harness
+1. Author an `eval-contract.json` declaring required knowledge step files (e.g. `playwright-utils-mandate.md`).
+2. Run `eval-quality compile --in contract.json` to validate contract structure and discipline rules.
+3. Run `eval-quality seal --in contract.json --out ./run` to generate `sealed-evaluator-brief.json`.
+4. Pass `sealed-evaluator-brief.json` to `bmad-tea` to execute the task without seeing answer keys.
+
+## Evidence and limitations
+
+Holding the model, the budget, the system, and the defects fixed, and changing only how the Eval Contract was authored, sealed-evaluator detection moved from **0.33 to 1.00** across three naturally occurring defects, three repetitions per arm, 19 scored runs.
+
+Both experiment rounds missed at least one preregistered gate. Round 1 recorded `DARK-FACTORY REJECTED`; round 2 block 1 recorded `CONTRACT-DISCIPLINE NOT SUPPORTED`, failing one gate of five on a single unreplicated clean control. The separation comes from two of the three defects, since both arms detected the third in every repetition, and both separating cases carry a recorded measurement-layer confound. The sample covers three defects, one system, and one model. This supports a product-direction decision at narrow scale. Certification would require broader replication.
+
+Read the [product brief](_bmad-output/planning-artifacts/briefs/brief-eval-quality-2026-07-17/brief.md) for the product rationale and the [PRD](_bmad-output/planning-artifacts/prds/prd-eval-quality-2026-07-17/prd.md) for build requirements. The experiment record includes the [round 1 verdict](experiments/hypothesis-validation/DECISION.md), [round 2 results](experiments/hypothesis-validation/PHASE2-RESULTS.md), [metric summary](experiments/hypothesis-validation/results/summary.md), and [protocol](experiments/hypothesis-validation/HYPOTHESIS_VALIDATION_PLAN.md).
+
+## Architecture status
+
+The [architecture spine](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ARCHITECTURE-SPINE.md) is split by pipeline half: the compile-and-seal half is epic-ready, while the score half is not. Gate C closed at zero blocking authoring points and 14 of 14 declaration-only predicates. Gate D's generated-current-fields arm matched the hand-written positive control at 3 of 3 seeded-defect catches, so `seal` joins the stage-one order without adding an evidence-precondition field.
+
+Contract strength scoring has been open since [ADR-007](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-007-compile-score-split.md): three rounds of external review established that the catch rate was 1.00 by construction, because nothing matched a finding to the defect its probe seeded. That input now exists and the mapping that reads it is owed to a reference implementation.
+
+Contract compilation was declared ready in ADR-007 and a fourth review withdrew that claim in [ADR-008](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-008-compile-half-owed-to-calibration.md). The named calibration is now complete. The absent local-only mut2 arm was reconstructed from its recorded base, reproduced its prior black-box behavior, and ran under a pre-registered three-arm, three-repetition design. All three arms composed filters and detected the seeded defect in every valid repetition. This closes the calibration gate narrowly; it does not generalize the historical 0.33-to-1.00 effect beyond one behavior and one controlled mutation.
+
+Both are documented as defects rather than dressed as decisions, because four rounds have shown that a confidently worded revision is the thing that goes wrong here.
+
+The decision record, in order: [ADR-001](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-22/ADR-001-evaluator-isolation-boundary.md) on evaluator isolation, [ADR-002](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-22/ADR-002-contract-authoring-discipline.md) on why authoring discipline is the product, [ADR-003](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-003-measurement-mechanics.md) on measurement mechanics, [ADR-004](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-004-execution-boundary.md) on why this package executes nothing, [ADR-005](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-005-review-round-corrections.md) and [ADR-006](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-006-interaction-plan.md) on what review and hand-authoring corrected, [ADR-007](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-007-compile-score-split.md) on the split, [ADR-008](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-008-compile-half-owed-to-calibration.md) on why the other half stopped claiming to be finished too, and [ADR-009](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/ADR-009-adversarial-gate-corrections.md) on the seventeen places where two conforming implementations still disagreed. Review triage lives in [`reviews/`](_bmad-output/planning-artifacts/architecture/architecture-eval-quality-2026-07-29/reviews/).
+
+## Not building now
+
+Deferred until the contract layer is in real use: claim-to-evidence lineage, semantic checkpoint scoring, process and outcome separation, and first material error attribution.
+
+Out of scope entirely: a new eval engine, a hosted service, a dashboard or GUI, multimodal evaluators, automatic prompt repair, and a generic judge-calibration platform.
+
+## Development
+
+```bash
+npm install
+npm run validate            # typecheck, lint, docs, shareable, spine, vectors, schemas, registries, AD-31 table, layers, lineage, boundary, corpus, tests with coverage
+npm run build               # emit to dist/
+npm run lint:fix            # auto-fix with Biome
+npm run test:coverage       # run the suite and fail below AD-30's 90 percent statement and branch floor on core/
+npm run generate:schemas    # rebuild schemas/*.schema.json from the Zod source
+npm run check:schemas       # fail if the committed schemas differ from the source by one byte
+npm run check:ad5-registry  # fail if the failure-code list drifts from the AD-5 table
+npm run check:lineage       # fail if a module outside the stage table writes an artifact's lineage fields
+npm run check:boundary      # fail if anything the tarball carries references the planning system that produced it
+npm run generate:ad31-table # rebuild docs/ad31-coverage-predicates.generated.md from the predicates
+npm run check:ad31-table    # fail if the committed AD-31 table differs from the builder by one byte
+npm run generate:dev-corpus # rebuild corpus/dev/ from the contract fixtures through the shipped compile and seal
+npm run check:corpus        # fail if the committed corpus differs from the builder by one byte
+npm run build:shareable     # render the planning artifacts to self-contained HTML
+npm run test:conformance    # run the published port conformance suite against every shipped adapter
+```
+
+`schemas/` holds the twelve published JSON Schema documents, generated from the Zod definitions and
+committed. They are the contract for consumers who do not read TypeScript, so they are proven
+equivalent to the source rather than assumed to be: a byte-exact drift check, a rejection suite
+asserting the validator keyword and instance path for every negative fixture, a differential check
+comparing Zod's verdict against a third-party validator's over a generated corpus, and a
+keyword-mutation sweep that deletes each published constraint and requires some fixture to notice.
+Edit the Zod schema and regenerate; never hand-edit a file under `schemas/`.
+
+Every artifact the library hands back is deep-frozen, so it cannot be changed in place. This package
+is ES modules, which are always strict, so an attempt throws a `TypeError` there; a sloppy-mode
+caller sees the write fail silently. A revision is minted as a new artifact carrying its parent's
+digest and a revision count one greater. `check:lineage` fails the build when a lineage field is
+written outside `src/core/schemas/`, `src/core/lineage/`, and the modules the AD-24 stage table
+names as that artifact's producer, which today are `src/core/seal/seal.ts` and
+`src/core/preflight/reduce.ts`.
+
+The `eval-quality/conformance` subpath publishes the port boundary: the four port types, the message
+shapes they carry, and an executable conformance suite. An adapter is conforming when
+`runCorpusPortConformance`, `runClockPortConformance`, `runFileSystemPortConformance`, or
+`runEnvironmentProbePortConformance` returns a report whose `passed` is true, which is the definition
+rather than a paraphrase of one; each returns a report instead of asserting, so the suite carries no
+test framework and runs under whichever one you already use.
+
+```ts
+import { runCorpusPortConformance, type CorpusPort } from 'eval-quality/conformance'
+```
+
+The suite drives a subject through four scenarios and checks six assertions per port method: a
+mechanism failure is a typed fault, exactly one underlying call happens on success and on failure, an
+aborted signal rejects promptly, an in-band error value is thrown rather than returned, and a
+successful call returns a response the published schema accepts. The environment-probe port adds
+thirteen more from AD-35's default-deny target policy. `npm run test:conformance` runs the suite
+against the three adapters this package ships and against an in-repository probe subject that exists
+only as the suite's own subject.
+
+`docs/ad31-coverage-predicates.generated.md` holds AD-31's published predicate table, emitted from
+the seven relevance predicates and their seven satisfaction twins run over a hand-authored contract
+corpus. It is generated by `npm run generate:ad31-table` and guarded by `npm run check:ad31-table`,
+a byte-exact drift check that fails when a predicate changes and the committed document does not, so
+the table is evidence the predicates produce rather than documentation kept beside them. Regenerate
+rather than hand-edit it.
+
+`build:shareable` renders this README, the product brief, the PRD, the architecture spine, all nine ADRs, and every document those pages link to (contributing, code of conduct, security, licence, and the four experiment records) to `_bmad-output/shareable/` as standalone styled HTML for sharing outside the repo. Rendering the linked documents is what lets a recipient without repository access follow the evidence, contribution, security, and licence links instead of hitting a 404; anything that has no page of its own, such as a directory, is marked in the export as needing repository access. Regenerate rather than hand-edit those files: `check:shareable` fails the build when the committed export is stale or carries a repository URL that is not the canonical one. Mermaid diagrams render as code blocks there, which is a known limitation.
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) and our [Code of Conduct](CODE_OF_CONDUCT.md).
+
+## Security
+
+See [SECURITY.md](SECURITY.md). Please do not open a public issue for vulnerabilities.
+
+## License
+
+Apache-2.0 © Murat Ozcan. See [LICENSE](LICENSE).

package/corpus/dev/README.md

@@ -0,0 +1,36 @@
+# Development corpus
+
+Nineteen contracts and one compiled-and-sealed pair, published so an adopter can read real input to
+this package without cloning the repository. Everything here is generated by
+`npm run generate:dev-corpus` and checked byte for byte by `npm run check:corpus`.
+
+## What is here
+
+- `contracts/<contractId>.json`: nineteen contracts, one per AD-20 discipline rule in each
+  declaration state. Sixteen are published only after this package's own compile stage accepts
+  them, so every one of those is a contract the compiler admits. Three fail compilation by design;
+  those ship as authored input, and `index.json` records the failure code each one raises.
+- `compile-seal-example/contract.json` and `compile-seal-example/brief.json`: one contract and
+  the brief this package's compile-then-seal boundary produces from it.
+- `index.json`: every file above, its kind, and the AD-27 digest of the exact bytes on disk.
+
+## These contracts are visible and diagnostic
+
+AD-38 calls a development corpus visible and diagnostic. Nothing here is a holdout: every contract
+is published, readable, and meant to be read while writing your own. A holdout set that measures a
+contract's strength is a separate thing this package does not ship.
+
+## What is absent, and why
+
+**The qualified-probe dimensions are absent.** AD-38 asks for at least one qualified probe per
+probe class and per `expectedClean` state. Qualifying a probe needs a trial reducer, which Owed
+item 1 records as not yet built, and a defect signature for the probe, which Owed item 7 records as
+missing for the only probe this repository names. Both dimensions arrive with the stage that adds
+them.
+
+**Three of the four artifacts in AD-38's end-to-end example are absent.** The example there is a
+sealed brief, a conforming sealed run record, an isolation manifest, and an evaluator
+configuration. The last three are produced by ingest, which does not exist yet, and Owed item 7
+forbids hand-filling downstream values: the chain must be regenerated from the reference reducer
+once that reducer exists. So this directory ships the compile-and-seal pair under a name that does
+not claim AD-38's term.

package/corpus/dev/compile-seal-example/brief.json

@@ -0,0 +1 @@
+{"behaviors":[{"description":"A created thing is readable back in the list of things.","id":"B-001","observableSuccessCriterion":"A list call after a create returns one element per seeded thing, carrying the name the create call sent.","oracles":["O-001","O-002","O-003","O-004","O-005","O-006","O-007"],"requirementLinks":[{"id":"REQ-1","scheme":"local"}],"riskLinks":[{"id":"RISK-1","scheme":"local-risk"}],"severity":"critical"}],"budgets":{"maxCostUsd":"0.25","maxToolCalls":20,"maxWallClockMinutes":5},"contractDigest":"sha256:1f7c657db755e9550ee78c57bdb81ea9f9de9ca227dbeaf823939a0313cbd781","directions":[{"oracleId":"O-001","text":"Its items field from the list things endpoint (with the supplied query limit) is asserted to satisfy the declared \"covers-by-key\" condition. The declared polarity expects this relation to hold. One list call over the seeded set. A list omitting a seeded thing, or repeating one is treated as a defect."},{"oracleId":"O-002","text":"The create thing endpoint (with the supplied body name): its error field, its id field, and its ok field is asserted to satisfy every declared condition together. The declared polarity expects this relation to hold. The whole create response. A create reporting success with no identifier, or with a diagnostic beside it is treated as a defect."},{"oracleId":"O-003","text":"The list things endpoint (with the supplied query limit): its error field and its items field is asserted to satisfy every declared condition together. The declared polarity expects this relation to hold. The list response taken as a whole. A list carrying items alongside a diagnostic field is treated as a defect."},{"oracleId":"O-004","text":"Its error field from the create thing endpoint (with a malformed body name value) and its error field from the list things endpoint (with a malformed query limit value) is asserted to satisfy every declared condition together. The declared polarity expects this relation to hold. Both sibling operations, each given an input that violates its declared type. One sibling rejecting the malformed input while the other accepts it is treated as a defect."},{"oracleId":"O-005","text":"The body name value you sent to the create thing endpoint (with the supplied body name), compared with the query limit value you sent to the list things endpoint (with the supplied query limit) is asserted to satisfy every declared condition together. The declared polarity expects this relation to hold. The two sibling parameters, as sent. One parameter carried and the other dropped is treated as a defect."},{"oracleId":"O-006","text":"The body name value you sent to the create thing endpoint (with the supplied body name), compared with its items field from the list things endpoint (with the supplied query limit) is asserted to contain the declared member. The declared polarity expects this relation to hold. The list read after the create, against the name the create sent. A create reporting success whose thing never appears in a later list is treated as a defect."},{"oracleId":"O-007","text":"Every element reachable through its items field from the list things endpoint (with the supplied query limit) is asserted to meet the declared condition. The declared polarity expects this relation to hold. Every element of the returned list. A list whose first element carries an identifier and whose later elements do not is treated as a defect."}],"parentDigest":null,"permittedInterfaces":[{"kind":"api","logicalId":"thing-api"}],"probeStepBound":8,"revisionCount":0,"safetyLimits":["No request to any host other than the mapped thing-api target."],"schemaVersion":1,"scopedResources":[]}

package/corpus/dev/compile-seal-example/contract.json

@@ -0,0 +1 @@
+{"behaviors":[{"description":"A created thing is readable back in the list of things.","id":"B-001","observableSuccessCriterion":"A list call after a create returns one element per seeded thing, carrying the name the create call sent.","oracles":["O-001","O-002","O-003","O-004","O-005","O-006","O-007"],"requirementLinks":[{"id":"REQ-1","scheme":"local"}],"riskLinks":[{"id":"RISK-1","scheme":"local-risk"}],"severity":"critical"}],"budgets":{"maxCostUsd":"0.25","maxToolCalls":20,"maxWallClockMinutes":5},"contractId":"satisfied-declarations","fixtureReset":null,"forbiddenInputs":["original-spec","source-code","repository","builder-transcript","implementation-logs","comparator-results","human-labels"],"interactionPlan":[{"after":null,"inputBinding":{"body":{"name":{"matcher":"any"}},"header":null,"path":null,"query":null},"operationId":"create-thing","stepId":"create"},{"after":"create","inputBinding":{"body":null,"header":null,"path":null,"query":{"limit":{"literal":10}}},"operationId":"list-things","stepId":"list"},{"after":null,"inputBinding":{"body":{"name":{"matcher":"type-violating"}},"header":null,"path":null,"query":null},"operationId":"create-thing","stepId":"malformed-create"},{"after":null,"inputBinding":{"body":null,"header":null,"path":null,"query":{"limit":{"matcher":"type-violating"}}},"operationId":"list-things","stepId":"malformed-list"}],"oracles":[{"check":{"actualKey":"id","expectedKey":"id","op":"covers-by-key","operands":[{"referenceSet":"expected-things"},{"pointer":"/interactions/list/response-body/items"}]},"commentary":"Reconciles the whole list against the declared set.","direction":{"evidenceTargets":["/interactions/list/response-body/items"],"negativeDomain":"A list omitting a seeded thing, or repeating one.","polarity":"expects-hold","relation":"covers-by-key","scope":"One list call over the seeded set."},"id":"O-001","polarity":"expects-hold"},{"check":{"op":"all","operands":[{"op":"existence","operands":[{"pointer":"/interactions/create/response-body/ok"}]},{"op":"existence","operands":[{"pointer":"/interactions/create/response-body/id"}]},{"op":"absence","operands":[{"pointer":"/interactions/create/response-body/error"}]}]},"commentary":null,"direction":{"evidenceTargets":["/interactions/create/response-body/ok","/interactions/create/response-body/id","/interactions/create/response-body/error"],"negativeDomain":"A create reporting success with no identifier, or with a diagnostic beside it.","polarity":"expects-hold","relation":"all","scope":"The whole create response."},"id":"O-002","polarity":"expects-hold"},{"check":{"op":"all","operands":[{"op":"existence","operands":[{"pointer":"/interactions/list/response-body/items"}]},{"op":"absence","operands":[{"pointer":"/interactions/list/response-body/error"}]}]},"commentary":null,"direction":{"evidenceTargets":["/interactions/list/response-body/items","/interactions/list/response-body/error"],"negativeDomain":"A list carrying items alongside a diagnostic field.","polarity":"expects-hold","relation":"all","scope":"The list response taken as a whole."},"id":"O-003","polarity":"expects-hold"},{"check":{"op":"all","operands":[{"op":"existence","operands":[{"pointer":"/interactions/malformed-create/response-body/error"}]},{"op":"existence","operands":[{"pointer":"/interactions/malformed-list/response-body/error"}]}]},"commentary":null,"direction":{"evidenceTargets":["/interactions/malformed-create/response-body/error","/interactions/malformed-list/response-body/error"],"negativeDomain":"One sibling rejecting the malformed input while the other accepts it.","polarity":"expects-hold","relation":"all","scope":"Both sibling operations, each given an input that violates its declared type."},"id":"O-004","polarity":"expects-hold"},{"check":{"op":"all","operands":[{"op":"existence","operands":[{"pointer":"/interactions/create/call-inputs/body/name"}]},{"op":"existence","operands":[{"pointer":"/interactions/list/call-inputs/query/limit"}]}]},"commentary":null,"direction":{"evidenceTargets":["/interactions/create/call-inputs/body/name","/interactions/list/call-inputs/query/limit"],"negativeDomain":"One parameter carried and the other dropped.","polarity":"expects-hold","relation":"all","scope":"The two sibling parameters, as sent."},"id":"O-005","polarity":"expects-hold"},{"check":{"op":"containment","operands":[{"pointer":"/interactions/list/response-body/items"},{"pointer":"/interactions/create/call-inputs/body/name"}]},"commentary":null,"direction":{"evidenceTargets":["/interactions/list/response-body/items","/interactions/create/call-inputs/body/name"],"negativeDomain":"A create reporting success whose thing never appears in a later list.","polarity":"expects-hold","relation":"containment","scope":"The list read after the create, against the name the create sent."},"id":"O-006","polarity":"expects-hold"},{"check":{"collection":{"pointer":"/interactions/list/response-body/items"},"op":"for-all","predicate":{"op":"existence","operands":[{"pointer":"@/id"}]}},"commentary":null,"direction":{"evidenceTargets":["/interactions/list/response-body/items"],"negativeDomain":"A list whose first element carries an identifier and whose later elements do not.","polarity":"expects-hold","relation":"for-all","scope":"Every element of the returned list."},"id":"O-007","polarity":"expects-hold"}],"parentDigest":null,"permittedInterfaces":[{"kind":"api","logicalId":"thing-api","operations":[{"method":"POST","operationId":"create-thing","pathTemplate":"/things","requestShape":{"body":{"permittedKeys":["name"],"requiredKeys":["name"],"types":{"name":"string"}},"header":{"permittedKeys":[],"requiredKeys":[],"types":{}},"path":{"permittedKeys":[],"requiredKeys":[],"types":{}},"query":{"permittedKeys":[],"requiredKeys":[],"types":{}}},"responseDescriptor":{"channelRoles":{"/error":"diagnostic","/id":"payload","/ok":"success-indicator"},"collectionLocations":[],"permittedKeys":["id","ok","error"],"requiredKeys":["id","ok"],"successIndicator":"/ok","types":{"error":"string","id":"string","ok":"boolean"}},"sensitivityWitness":{"channel":"body","legs":[{"inputs":{"body":{"kind":"json","value":{"name":"alpha"}},"header":{},"path":{},"query":{}},"legId":"create-witness-a"},{"inputs":{"body":{"kind":"json","value":{"name":"beta"}},"header":{},"path":{},"query":{}},"legId":"create-witness-b"}],"relation":{"op":"not","operands":[{"op":"deep-equality","operands":[{"pointer":"/interactions/create-witness-a/response-body"},{"pointer":"/interactions/create-witness-b/response-body"}]}]},"witnessId":"create-thing-sensitivity"},"stateChangeMarker":true,"volatilePointers":["/id"]},{"method":"GET","operationId":"list-things","pathTemplate":"/things","requestShape":{"body":{"permittedKeys":[],"requiredKeys":[],"types":{}},"header":{"permittedKeys":[],"requiredKeys":[],"types":{}},"path":{"permittedKeys":[],"requiredKeys":[],"types":{}},"query":{"permittedKeys":["limit"],"requiredKeys":[],"types":{"limit":"number"}}},"responseDescriptor":{"channelRoles":{"/error":"diagnostic","/items":"collection"},"collectionLocations":[{"expectedCardinality":{"count":3,"mode":"exact"},"pointer":"/items","referenceSet":"expected-things"}],"permittedKeys":["items","error"],"requiredKeys":["items"],"successIndicator":"/items","types":{"error":"string","items":"array"}},"sensitivityWitness":{"channel":"query","legs":[{"inputs":{"body":{"kind":"absent"},"header":{},"path":{},"query":{"limit":1}},"legId":"list-witness-a"},{"inputs":{"body":{"kind":"absent"},"header":{},"path":{},"query":{"limit":2}},"legId":"list-witness-b"}],"relation":{"op":"not","operands":[{"op":"deep-equality","operands":[{"pointer":"/interactions/list-witness-a/response-body"},{"pointer":"/interactions/list-witness-b/response-body"}]}]},"witnessId":"list-things-sensitivity"},"stateChangeMarker":false,"volatilePointers":[]}]}],"probeStepBound":8,"referenceSets":{"expected-things":{"commentary":null,"keys":["id"],"members":[{"id":"t-1"},{"id":"t-2"},{"id":"t-3"}]}},"requiredEvidence":["Request and response pair for every call, in order."],"revisionCount":0,"rubrics":[{"criteria":[{"evidence":"/interactions/list/response-body/items","id":"RC-001","text":"Does the returned list carry every expected identifier?"}],"failureModePenalties":[{"description":"An expected thing is missing.","name":"omission"}],"id":"R-001","maxLength":400,"scaleLevels":[{"anchor":"Every expected thing is present.","level":1}]}],"safetyLimits":["No request to any host other than the mapped thing-api target."],"schemaVersion":1,"scopedResources":null,"siblingGroups":{"operations":[["create-thing","list-things"]],"parameters":[["limit","name"]]},"sourceSpecDigest":null,"testData":{"cleanup":"Delete every thing created during the run.","setup":"Seed exactly three things with identifiers t-1, t-2, t-3."},"waivers":[{"approval":"gate-c-reviewer","condition":null,"expiresAt":"2027-01-01T00:00:00Z","id":"W-001","rationale":"The upstream seed is unavailable in the sandbox environment.","rule":"omission-and-completeness"}]}