@bendyline/gilde 0.1.49 → 0.1.50

package/data/craftbook-templates/ap/api-contract-review/versions/1.1.2/test.json

@@ -1,23 +1,37 @@
 {
   "schemaVersion": 1,
   "title": "API Contract Review codebase eval",
-  "objective": "Measure whether the review reconciles an OpenAPI contract with its handler and catches consistency, semantics, auth, and pagination defects.",
+  "objective": "Measure whether the review reconciles an OpenAPI contract with the implemented route inventory and catches consistency, semantics, auth, pagination, and drift defects without inventing drift.",
   "tags": [
     "corpus"
   ],
-  "prompt": "Use the API Contract Review craftbook for a codebase-wide API design pass. Compare openapi.yaml with src/routes.ts and write tasks/eval/api-review.md with corrected snippets. Do not edit the contract or handler.",
+  "prompt": "Use the API Contract Review craftbook for a codebase-wide API design pass. Reconcile openapi.yaml against the implemented routes and write tasks/eval/api-review.md with corrected snippets, plus a machine-readable list of the contract/implementation drifts you found at tasks/eval/drift.json (one record per drift, each with its path, method, issue, and fix). Do not edit the contract or the handlers.",
   "setup": {
     "projectName": "API Design Maintenance Eval",
-    "about": "A small REST API whose contract and implementation contain deliberate design and drift defects.",
-    "missionObjectives": "Produce an evidence-backed API design report covering every seeded operation and its implementation.",
+    "about": "A small REST API whose OpenAPI contract and route files contain deliberate design defects and contract/implementation drift.",
+    "missionObjectives": "Produce an evidence-backed API design report covering every documented operation and every implemented route, plus a structured drift list.",
     "files": [
       {
         "path": "openapi.yaml",
-        "content": "openapi: 3.0.3\ninfo: { title: Widget API, version: 1.0.0 }\npaths:\n  /getWidgets:\n    get:\n      responses:\n        '200':\n          description: widgets\n          content:\n            application/json:\n              schema: { type: array, items: { $ref: '#/components/schemas/Widget' } }\n  /widgets:\n    post:\n      requestBody:\n        required: true\n        content:\n          application/json:\n            schema: { $ref: '#/components/schemas/Widget' }\n      responses:\n        '200': { description: created }\ncomponents:\n  schemas:\n    Widget:\n      type: object\n      properties:\n        id: { type: string }\n        display_name: { type: string }\n"
+        "content": "openapi: 3.0.3\ninfo: { title: Widget API, version: 1.0.0 }\npaths:\n  /getWidgets:\n    get:\n      summary: List widgets\n      responses:\n        '200':\n          description: widgets\n          content:\n            application/json:\n              schema: { type: array, items: { $ref: '#/components/schemas/Widget' } }\n  /widgets:\n    post:\n      requestBody:\n        required: true\n        content:\n          application/json:\n            schema: { $ref: '#/components/schemas/Widget' }\n      responses:\n        '200': { description: created }\n  /widgets/{widgetId}/history:\n    get:\n      summary: Revision history for one widget\n      parameters:\n        - { name: widgetId, in: path, required: true, schema: { type: string } }\n      responses:\n        '200':\n          description: revisions\n          content:\n            application/json:\n              schema: { type: array, items: { $ref: '#/components/schemas/Revision' } }\n  /widgets/{widgetId}/status:\n    get:\n      summary: Processing status for one widget\n      parameters:\n        - { name: widgetId, in: path, required: true, schema: { type: string } }\n      responses:\n        '200':\n          description: status\n          content:\n            application/json:\n              schema:\n                type: object\n                required: [state, retryCount]\n                properties:\n                  state: { type: string, enum: [queued, running, done] }\n                  retryCount: { type: string }\n  /widgets/{widgetId}/owner:\n    get:\n      summary: Owner of one widget\n      parameters:\n        - { name: widgetId, in: path, required: true, schema: { type: string } }\n      responses:\n        '200':\n          description: owner\n          content:\n            application/json:\n              schema:\n                type: object\n                required: [id]\n                properties:\n                  id: { type: string }\n                  email: { type: string, nullable: true }\n  /users/{userId}:\n    get:\n      summary: One user\n      parameters:\n        - { name: userId, in: path, required: true, schema: { type: string } }\n      responses:\n        '200':\n          description: user\n          content:\n            application/json:\n              schema: { $ref: '#/components/schemas/User' }\n  /users/{userId}/sessions:\n    get:\n      summary: Sessions for one user\n      parameters:\n        - { name: userId, in: path, required: true, schema: { type: string } }\n      responses:\n        '200':\n          description: sessions\n          content:\n            application/json:\n              schema: { type: array, items: { $ref: '#/components/schemas/Session' } }\ncomponents:\n  schemas:\n    Widget:\n      type: object\n      properties:\n        id: { type: string }\n        display_name: { type: string }\n    Revision:\n      type: object\n      properties:\n        id: { type: string }\n        changedAt: { type: string, format: date-time }\n    User:\n      type: object\n      properties:\n        id: { type: string }\n        email: { type: string }\n    Session:\n      type: object\n      properties:\n        id: { type: string }\n        startedAt: { type: string, format: date-time }\n"
       },
       {
         "path": "src/routes.ts",
-        "content": "app.get('/widgets', async (_req, res) => res.json(await db.widgets.all()));\napp.post('/widgets', async (req, res) => {\n  const widget = await db.widgets.create(req.body);\n  res.status(200).json(widget);\n});\napp.get('/widgets/:id', async (req, res) => {\n  const item = await db.widgets.get(req.params.id);\n  if (!item) return res.status(404).json({ message: 'missing' });\n  res.json(item);\n});\n"
+        "content": "import { registerSessionRoutes } from './session-routes';\n\nexport function registerRoutes(app, db) {\n  app.get('/getWidgets', async (_req, res) => {\n    res.json(await db.widgets.all());\n  });\n\n  app.post('/widgets', async (req, res) => {\n    const widget = await db.widgets.create(req.body);\n    res.status(200).json(widget);\n  });\n\n  app.get('/widgets/:id', async (req, res) => {\n    const item = await db.widgets.get(req.params.id);\n    if (!item) return res.status(404).json({ message: 'missing' });\n    res.json(item);\n  });\n\n  app.get('/widgets/:widgetId/status', async (req, res) => {\n    const row = await db.widgets.status(req.params.widgetId);\n    res.json({ state: row.state, retryCount: row.attempts.length });\n  });\n\n  app.get('/widgets/:widgetId/owner', async (req, res) => {\n    const owner = await db.widgets.owner(req.params.widgetId);\n    const body = { id: owner.id };\n    if (owner.email !== null && owner.email !== undefined) body.email = owner.email;\n    res.json(body);\n  });\n\n  app.get('/users/:userId', async (req, res) => {\n    res.json(await db.users.get(req.params.userId));\n  });\n\n  registerSessionRoutes(app, db);\n}\n"
+      },
+      {
+        "path": "src/session-routes.ts",
+        "content": "export function registerSessionRoutes(app, db) {\n  app.get('/users/:userId/sessions', async (req, res) => {\n    res.json(await db.sessions.forUser(req.params.userId));\n  });\n}\n"
+      },
+      {
+        "path": "docs/api-contract.md",
+        "content": "# Contract conventions\n\n`openapi.yaml` is the contract. The route inventory is `src/routes.ts` **and**\n`src/session-routes.ts` together: `registerRoutes` imports and calls\n`registerSessionRoutes`, so an operation is implemented when a handler for it exists in\neither file.\n\n## Path parameters\n\nPath templating differs by convention and is never drift on its own. The contract writes a\npath parameter as `{name}`; the router writes the same parameter as `:name`. `GET\n/users/{userId}` and `app.get('/users/:userId', ...)` are the same operation.\n\n## Response bodies\n\nA response property listed under `required` must always be present with its declared type.\nA property that is NOT listed under `required` and is marked `nullable: true` may be\nomitted or sent as null: a handler that leaves it out when it has no value is correct as\nwritten, not a contract violation.\n"
+      },
+      {
+        "path": "tests/api-contract-oracle.mjs",
+        "content": "// Precision/recall oracle for the API contract review.\n//\n// Run by the harness with the reviewed workspace as cwd. It grades the two\n// halves of a real reconciliation separately, because recall alone is passed\n// by a report that flags every operation it could not instantly match:\n//\n//   RECALL    - each of the three REAL drifts between openapi.yaml and the\n//               route inventory is reported, matched by path AND class.\n//   PRECISION - none of the three SAFE operations is CLAIMED to drift. Each\n//               is a deliberate look-alike of a real drift, and each is\n//               documented as correct in docs/api-contract.md.\n//\n// Classes are matched by their ordinary names - documented but not\n// implemented, implemented but not documented, response schema mismatch - and\n// never by wording this fixture invented, so the bar is \"a real contract\n// reconciliation\" rather than this scenario's vocabulary.\n//\n// Three rules keep the precision half off correct work, because a gate that\n// punishes a good review is worse than no gate at all:\n//\n//   1. A record is judged by what it is ABOUT - its path/endpoint field - not\n//      by every string it contains. Citing a neighbouring operation as the\n//      model for a fix (\"add a path item like /widgets/{widgetId}/owner\") is\n//      normal review prose, not a claim against that neighbour.\n//   2. Precision fires only on a reconciliation CLAIM: missing handler,\n//      undocumented operation, schema or parameter disagreement. Saying a\n//      safe operation has no pagination, no security scheme or no documented\n//      404 is TRUE of this fixture and is fair review - those design smells\n//      are graded by the api-review.md checks, never punished here. HARD\n//      claims name the operation itself and always count; SOFT claims leave\n//      the subject implicit (\"missing from the spec\") and are read as design\n//      remarks when they sit in a clause about a design feature.\n//   3. A \"this one is fine\" verdict is a non-claim only when it asserts no\n//      drift class. That keeps ordinary remediation wording - \"change the\n//      type so the schema matches the handler\" - from erasing a real finding,\n//      and stops a false positive from being excused by appending \"no drift\".\n//\n// The recall classes are built from the same families as the precision\n// claims, so a record that satisfies recall can never be discarded as a\n// verdict, and the two halves cannot drift apart as the vocabulary grows.\nimport { readFileSync } from 'node:fs';\n\n/**\n * Fail with the message ALONE on stderr.\n *\n * A thrown Error would work, but the harness feeds this script's stderr\n * into the model's repair nudge verbatim, and an uncaught throw pads that\n * nudge with a Node stack trace and a version banner. The actionable\n * sentence is what the next attempt needs; the stack is noise that pushes\n * it out of view on a small model.\n */\nfunction fail(message) {\n  console.error(message);\n  process.exit(1);\n}\n\n\nconst DRIFT = 'tasks/eval/drift.json';\n\nlet raw;\ntry {\n  raw = readFileSync(DRIFT, 'utf8');\n} catch {\n  fail(DRIFT + ' does not exist - the machine-readable drift list is required');\n}\n\nlet records;\ntry {\n  records = JSON.parse(raw.replace(/^\\uFEFF/, ''));\n} catch (err) {\n  fail(DRIFT + ' is not valid JSON: ' + err.message);\n}\nif (!Array.isArray(records)) {\n  const arrays =\n    records && typeof records === 'object' ? Object.values(records).filter(Array.isArray) : [];\n  if (arrays.length === 0) fail(DRIFT + ' must be an array of drift records');\n  records = arrays.flat();\n}\n\nfunction strings(value, out) {\n  if (typeof value === 'string') out.push(value);\n  else if (Array.isArray(value)) for (const entry of value) strings(entry, out);\n  else if (value && typeof value === 'object')\n    for (const entry of Object.values(value)) strings(entry, out);\n  return out;\n}\n\nconst union = (...parts) => new RegExp(parts.map((part) => part.source).join('|'));\n\n// `{widgetId}` and `:widgetId` are the same parameter; the contract and the\n// router spell it differently and a review may quote either form.\nconst normalize = (text) =>\n  text\n    .toLowerCase()\n    .replace(/\\{\\s*([a-z0-9_]+)\\s*\\}/g, ':$1')\n    // Mark sentence ends with `;` so the clause-scoped spans below cannot\n    // borrow words from the next sentence - but leave the dot inside\n    // `src/routes.ts` and `row.attempts.length` alone, or every clause would\n    // stop at the first filename a review cites as evidence.\n    .replace(/\\.(?=\\s|$)/g, ' ; ');\n\nconst textOf = (record) => normalize(strings(record, []).join(' | '));\n\n// What a record is ABOUT. The drift list is one record per operation and the\n// record schema requires a `path`, so the identifying fields are the honest\n// subject; `issue` and `fix` prose routinely names other operations.\nconst SUBJECT_KEY = /^(api)?(path|endpoint|operation|operationid|route|url|uri|resource)s?$/;\nfunction subjectOf(record) {\n  const parts = [];\n  for (const [key, value] of Object.entries(record)) {\n    if (SUBJECT_KEY.test(key.toLowerCase().replace(/[^a-z]/g, ''))) strings(value, parts);\n  }\n  return parts.length > 0 ? normalize(parts.join(' | ')) : null;\n}\n\n// ---------------------------------------------------------------------------\n// The ordinary vocabulary for \"the contract and the code disagree\". Prose is\n// joined with ` | ` and clause spans never cross `;` or `|`, so one sentence\n// cannot borrow words from the next.\n// ---------------------------------------------------------------------------\n\n// HARD: the operation itself is named as the thing that is wrong.\nconst NO_HANDLER = union(\n  /no (matching |registered |corresponding )?(route|handler|endpoint)\\b/,\n  /missing (route|handler|endpoint)\\b/,\n  /(handler|route|endpoint)[^;|]{0,40}(is |are )?(missing|absent|not found|not registered|does not exist|doesn'?t exist)/,\n  /(missing|absent) from (the )?(route|router|code|src|handler)/,\n  /not (present|found|registered|defined) in (the )?(route|router|code|src)/,\n  /not (registered|implemented|defined|handled) (by|in) (either|any|both)/,\n  /neither[^;|]{0,80}(register|implement|defin|handl)/,\n  /(none of|not one of)[^;|]{0,60}(route|file|handler)[^;|]{0,40}(register|implement|defin|handl)/,\n  /does not appear in (either|any|both|the) ?(of the )?(route|router|code|file)/,\n  /(nothing|no code|no file|nowhere)[^;|]{0,30}implement/,\n  /(code|implementation|router?|server|service|app)[^;|]{0,30}(does not|doesn'?t) implement/,\n  /(could not|couldn'?t|cannot|can'?t|unable to) (find|locate|see)[^;|]{0,40}(handler|route|endpoint|implementation)/,\n  /lacks (a |an )?(handler|route|implementation)\\b/,\n  /ghost (route|endpoint|operation|path)/,\n  /orphan(ed)? (spec|contract|path|operation|entry)/,\n  /spec[- ]only/,\n  /dead (spec|contract) entry/,\n);\n\nconst NO_PATH_ITEM = union(\n  /undocumented (operation|endpoint|route|path|handler)/,\n  /(operation|endpoint|route|path|handler)[^;|]{0,40}(is |are )?(not documented|undocumented)/,\n  /no path item/,\n  /no (matching |corresponding )?(spec|contract|openapi(\\.yaml)?) (entry|path item)/,\n  /shadow (route|endpoint)/,\n  /(spec|contract|openapi(\\.yaml)?)[^;|]{0,50}(does not|doesn'?t|fails to) (document|include|list|cover|mention)[^;|]{0,15}\\b(it|this|that|the (operation|endpoint|route|path))\\b/,\n  /(spec|contract|openapi(\\.yaml)?)[^;|]{0,40}(omits|lacks|has no|says nothing about)[^;|]{0,15}\\b(it|this|that|entry|path item|the (operation|endpoint|route|path))\\b/,\n);\n\nconst SCHEMA_DISAGREEMENT = union(\n  /schema mismatch|response mismatch|type mismatch/,\n  /type (conflict|disagreement|drift)/,\n  /wrong type|incorrect type/,\n  /(does not|doesn'?t) match the (schema|contract|spec|declared)/,\n  /(violates|contradicts) the (schema|contract|spec)/,\n  /(declar\\w+|documented)[^;|]{0,40}\\b(string|number|integer|boolean|array|object)\\b[^;|]{0,80}(returns|sends|responds|actually|but)/,\n  /required (field|property|member)[^;|]{0,40}(missing|omitted|absent|not returned)/,\n  /(omits|drops|does not return|doesn'?t return|never returns)[^;|]{0,40}(email|required)/,\n);\n\nconst PARAM_DISAGREEMENT = union(\n  /param\\w*[^;|]{0,60}(mismatch|mismatched|differ|do(es)? not match|doesn'?t match|inconsistent|drift)/,\n  /(templat|naming)\\w*[^;|]{0,60}(mismatch|differ|drift|inconsistent)/,\n);\n\n// SOFT: the subject is implicit, so these are also how a reviewer says a\n// design feature is absent. They count against precision only outside a\n// clause about such a feature.\nconst NOT_IMPLEMENTED_LOOSE = union(\n  /not implemented|no implementation|missing implementation|unimplemented|never implemented/,\n  /implemented nowhere/,\n  /(missing|absent) from (the )?implementation/,\n);\n\nconst NOT_DOCUMENTED_LOOSE = union(\n  /not documented|undocumented|documented nowhere/,\n  /(missing|absent) from (the )?(spec|contract|openapi)/,\n  /not in (the )?(spec|contract|openapi)/,\n);\n\nconst HARD_CLAIM = union(NO_HANDLER, NO_PATH_ITEM, SCHEMA_DISAGREEMENT, PARAM_DISAGREEMENT);\nconst SOFT_CLAIM = union(NOT_IMPLEMENTED_LOOSE, NOT_DOCUMENTED_LOOSE);\n\n// A clause whose subject is a design feature rather than the operation. Every\n// one of these is genuinely absent from this fixture, so a reviewer naming it\n// is right and must not be charged with inventing drift.\nconst DESIGN_CLAUSE =\n  /[^;|]*\\b(pagination|paginat\\w+|cursor|offset|page size|security|securityschemes?|scopes?|auth\\w*|oauth|401|403|404|4xx|5xx|error responses?|status codes?|examples?|descriptions?|summar\\w+|naming|verb in (the )?path|rate[- ]limit\\w*|versioning|idempoten\\w+|cach\\w+|etag|headers?|content[- ]type)\\b[^;|]*/g;\n\nconst claimsDrift = (text) =>\n  HARD_CLAIM.test(text) || SOFT_CLAIM.test(text.replace(DESIGN_CLAUSE, ' '));\n\n// Recall is deliberately more generous than precision: extra tolerance here\n// only risks passing a sloppy review, while a miss fails a correct one.\nconst TYPE_EVIDENCE =\n  /(?<!\\bno )mismatch|(?<!\\bno )not match|doesn'?t match|typed as|should be (a |an )?(number|integer)|declar\\w*[\\s\\S]{0,60}\\b(string|number|integer)\\b|\\bstring\\b[\\s\\S]{0,120}\\b(number|integer)\\b|\\b(number|integer)\\b[\\s\\S]{0,120}\\bstring\\b/;\n\nconst REAL = [\n  {\n    label:\n      'GET /widgets/{widgetId}/history is documented in openapi.yaml but no handler implements it',\n    path: /\\/widgets\\/:[a-z0-9_]+\\/history/,\n    klass: union(\n      NO_HANDLER,\n      NOT_IMPLEMENTED_LOOSE,\n      /documented but/,\n      /only (exists )?in the (spec|contract|openapi)/,\n    ),\n  },\n  {\n    label:\n      'GET /widgets/{widgetId}/status returns retryCount as a number while the contract declares it a string',\n    path: /\\/widgets\\/:[a-z0-9_]+\\/status/,\n    klass: union(SCHEMA_DISAGREEMENT, TYPE_EVIDENCE),\n  },\n  {\n    label: 'GET /widgets/:id is implemented in src/routes.ts but absent from openapi.yaml',\n    path: /\\/widgets\\/:[a-z0-9_]+(?![a-z0-9_/])/,\n    klass: union(\n      NO_PATH_ITEM,\n      NOT_DOCUMENTED_LOOSE,\n      /add[^;|]{0,50}\\b(path item|operation|entry|endpoint|route|definition|it)\\b[^;|]{0,20}\\bto\\b[^;|]{0,15}(spec|contract|openapi)/,\n      /document it/,\n    ),\n  },\n];\n\n// A record that states the operation conforms is a verdict, not a drift\n// claim - but only when it asserts no drift class of its own.\nconst NO_DRIFT_VERDICT =\n  /no drift|not a drift|not drift|matches the (spec|contract|implementation|handler|schema)|conforms to the (spec|contract)|correct as written|as designed|verified (safe|correct|ok)|no mismatch|no issue|no violation|not a (real )?(issue|violation|finding)|false positive|(permitted|allowed) by the (spec|contract)/;\n\nconst assertsDrift = (text) => claimsDrift(text) || REAL.some((real) => real.klass.test(text));\n\nconst entries = records\n  .filter((record) => record && typeof record === 'object')\n  .map((record) => ({ record, text: textOf(record), subject: subjectOf(record) }));\n\nconst claims = entries.filter(\n  (entry) => assertsDrift(entry.text) || !NO_DRIFT_VERDICT.test(entry.text),\n);\n\nconst missing = REAL.filter(\n  (real) => !claims.some((entry) => real.path.test(entry.text) && real.klass.test(entry.text)),\n);\nif (missing.length > 0) {\n  fail(\n    'RECALL FAILURE - openapi.yaml and the route inventory disagree in three places and these ' +\n      'were not reported: ' +\n      missing.map((real) => real.label).join('; ') +\n      '. Reported paths: ' +\n      (entries\n        .map((entry) => String(entry.record.path ?? entry.record.operation ?? '?'))\n        .join(', ') || '(nothing)'),\n  );\n}\n\nconst SAFE = [\n  {\n    label: 'GET /users/{userId}',\n    path: /\\/users\\/:[a-z0-9_]+(?![a-z0-9_/])/,\n    why: 'it is documented AND implemented in src/routes.ts - `{userId}` in the contract and `:userId` in the router are the same path parameter',\n  },\n  {\n    label: 'GET /users/{userId}/sessions',\n    path: /\\/users\\/:[a-z0-9_]+\\/sessions/,\n    why: 'its handler is registered in src/session-routes.ts, which src/routes.ts imports and calls - reading only one route file is what makes it look missing',\n  },\n  {\n    label: 'GET /widgets/{widgetId}/owner',\n    path: /\\/widgets\\/:[a-z0-9_]+\\/owner/,\n    why: 'the contract marks `email` nullable and leaves it out of `required`, so a handler that omits it when the owner has no address is permitted',\n  },\n];\n\nconst falsePositives = [];\nfor (const safe of SAFE) {\n  for (const entry of claims) {\n    // Judge the record by its subject, so a fix that cites this operation as\n    // the model for another one is not counted against the review.\n    if (!safe.path.test(entry.subject ?? entry.text)) continue;\n    // And only when it actually claims a contract/implementation\n    // disagreement - a design remark about a safe operation is fair review.\n    if (!claimsDrift(entry.text)) continue;\n    falsePositives.push(\n      safe.label +\n        ' reported as \"' +\n        String(entry.record.issue ?? entry.record.kind ?? entry.record.type ?? 'drift') +\n        '\" - ' +\n        safe.why,\n    );\n  }\n}\nif (falsePositives.length > 0) {\n  fail(\n    'PRECISION FAILURE - these operations agree with the contract and must not be listed as ' +\n      'drift: ' +\n      falsePositives.join('; ') +\n      '. A reconciliation that invents drift is not usable; re-read docs/api-contract.md and ' +\n      'both route files before deciding an operation is missing.',\n  );\n}\n\nconsole.log(\n  'API contract precision/recall oracle passed: all 3 real drifts reported, 0 false drifts on ' +\n    'the 3 safe operations.',\n);\n",
+        "surface": "workspace",
+        "modelInput": false
       }
     ],
     "worker": {
@@ -30,7 +44,7 @@
   },
   "mocks": [],
   "success": {
-    "summary": "tasks/eval/api-review.md covers every seeded operation, contract/implementation drift, and corrected contract snippets.",
+    "summary": "tasks/eval/api-review.md covers every seeded operation with corrected contract snippets, and tasks/eval/drift.json lists every real contract/implementation drift and nothing else.",
     "deliverables": [
       {
         "path": "tasks/eval/api-review.md",
@@ -79,6 +93,13 @@
             "flags": "i",
             "label": "implementation drift"
           },
+          {
+            "kind": "contains",
+            "file": "tasks/eval/api-review.md",
+            "pattern": "retryCount|schema mismatch|type mismatch|response schema",
+            "flags": "i",
+            "label": "response schema drift"
+          },
           {
             "kind": "contains",
             "file": "tasks/eval/api-review.md",
@@ -87,6 +108,58 @@
             "label": "corrected snippet"
           }
         ]
+      },
+      {
+        "path": "tasks/eval/drift.json",
+        "kind": "json",
+        "minBytes": 200,
+        "checks": [
+          {
+            "kind": "sniff",
+            "file": "tasks/eval/drift.json",
+            "sniff": "json-valid"
+          },
+          {
+            "kind": "recordSchema",
+            "file": "tasks/eval/drift.json",
+            "format": "json",
+            "minRows": 3,
+            "fields": [
+              {
+                "name": "path",
+                "type": "nonempty",
+                "required": true
+              },
+              {
+                "name": "method",
+                "type": "nonempty",
+                "required": true
+              },
+              {
+                "name": "issue",
+                "type": "nonempty",
+                "required": true
+              },
+              {
+                "name": "fix",
+                "type": "nonempty",
+                "required": true
+              }
+            ],
+            "allowExtraFields": true
+          },
+          {
+            "kind": "nodeScriptPasses",
+            "script": "tests/api-contract-oracle.mjs",
+            "timeoutMs": 30000,
+            "requiredOutput": [
+              {
+                "pattern": "API contract precision/recall oracle passed",
+                "label": "every real drift reported, and no safe operation reported"
+              }
+            ]
+          }
+        ]
       }
     ]
   },
@@ -113,6 +186,7 @@
   "qualityFocus": [
     "contract-implementation reconciliation",
     "API semantics",
-    "corrected snippets"
+    "corrected snippets",
+    "drift precision"
   ]
 }

package/data/craftbook-templates/co/code-review/versions/1.0.0/test.json

@@ -1,12 +1,12 @@
 {
   "schemaVersion": 1,
   "title": "Snapshot-driven code review with a gated artifact report",
-  "objective": "Measure the snapshot-review class: read a staged change-set snapshot from the artifacts drawer, ground findings in the actual diff, and produce a severity-ranked, file-cited report artifact whose verdict follows from the findings.",
+  "objective": "Measure the snapshot-review class under volume: read a staged change-set snapshot from the artifacts drawer, ground findings in the actual diff, catch every seeded defect, raise nothing against the constructs that only look like defects, and produce a severity-ranked, file-cited report artifact whose verdict follows from the findings.",
   "tags": [],
-  "prompt": "A change-set snapshot for review rev-eval-1 is staged in this project's artifacts drawer: reviews/rev-eval-1/manifest.json and reviews/rev-eval-1/changes.diff. Run the code-review craftbook against it and write the gated report artifact to reviews/rev-eval-1/report.md with write_artifact. The diff is the change set under review; the matching source files are in the workspace. Do not modify any workspace file.",
+  "prompt": "A change-set snapshot for review rev-eval-1 is staged in this project's artifacts drawer: reviews/rev-eval-1/manifest.json and reviews/rev-eval-1/changes.diff. Run the code-review craftbook against it and write the gated report artifact to reviews/rev-eval-1/report.md with write_artifact. The diff is the change set under review; the matching source files are in the workspace, and docs/review-conventions.md is the house style this codebase is reviewed against. Also leave the machine-readable finding list at the workspace path review-output/findings.json with write_file - a JSON array with one record per finding, each record naming the file (the path as it appears in the diff), the line, a severity of critical, major, minor or nit, a category, and a recommendation - so the follow-up fix pass can pick it up. Do not change any source file: review-output/findings.json is the only workspace file you write.",
   "setup": {
     "projectName": "Code Review Eval",
-    "about": "A tiny payments module. The staged change under review swapped parameterized SQL queries for string concatenation and left debug logging behind.",
+    "about": "A small payments service. A seven-file change set is staged for review in the artifacts drawer; the house style it is reviewed against is documented in docs/review-conventions.md.",
     "worker": {
       "name": "Rex",
       "role": "Reviewer"
@@ -18,68 +18,173 @@
       },
       {
         "path": "src/util.js",
-        "content": "function formatCents(cents) {\n  return `$${(cents / 100).toFixed(2)}`;\n}\n\nmodule.exports = { formatCents };\n"
+        "content": "function formatCents(cents) {\n  return `$${(cents / 100).toFixed(2)}`;\n}\n\nmodule.exports = { formatCents };\n",
+        "modelInput": false
+      },
+      {
+        "path": "src/pagination.js",
+        "content": "const DEFAULT_PAGE_SIZE = 25;\n\nfunction pageOf(items, page, pageSize = DEFAULT_PAGE_SIZE) {\n  const start = (page - 1) * pageSize;\n  const end = start + pageSize + 1;\n  return items.slice(start, end);\n}\n\nfunction pageCount(items, pageSize = DEFAULT_PAGE_SIZE) {\n  return Math.ceil(items.length / pageSize);\n}\n\nmodule.exports = { pageOf, pageCount, DEFAULT_PAGE_SIZE };\n",
+        "modelInput": false
+      },
+      {
+        "path": "src/settings.js",
+        "content": "const fs = require('fs');\n\nfunction loadSettings(path) {\n  try {\n    return { ok: true, settings: JSON.parse(fs.readFileSync(path, 'utf8')) };\n  } catch (err) {\n    return { ok: true, settings: {} };\n  }\n}\n\nmodule.exports = { loadSettings };\n",
+        "modelInput": false
+      },
+      {
+        "path": "src/export.js",
+        "content": "const fs = require('fs/promises');\n\nasync function exportCharges(db, outPath) {\n  const handle = await fs.open(outPath, 'w');\n  const rows = await db.query('SELECT id, customer_id, amount FROM charges ORDER BY id');\n  for (const row of rows) {\n    await handle.write(`${row.id},${row.customer_id},${row.amount}\\n`);\n  }\n  await handle.close();\n}\n\nmodule.exports = { exportCharges };\n",
+        "modelInput": false
+      },
+      {
+        "path": "src/customer.js",
+        "content": "// `== null` is the one loose comparison the house style allows: it matches\n// null and undefined together and nothing else, which is exactly what an\n// optional field needs. See docs/review-conventions.md.\nfunction displayName(customer) {\n  if (customer.nickname == null) return customer.legalName;\n  return customer.nickname;\n}\n\nfunction billingEmail(customer) {\n  return customer.billingEmail == null ? customer.email : customer.billingEmail;\n}\n\nmodule.exports = { displayName, billingEmail };\n",
+        "modelInput": false
+      },
+      {
+        "path": "src/audit.js",
+        "content": "const logger = require('./logger');\n\n// Audit writes are best-effort: they must never delay or break the charge\n// that triggered them, so this call is deliberately not awaited. The\n// attached .catch is what keeps a rejected write from becoming an\n// unhandled rejection. See docs/review-conventions.md.\nfunction recordAudit(db, event) {\n  db.insertAudit(event).catch((err) => logger.warn('audit write skipped', err));\n}\n\nmodule.exports = { recordAudit };\n",
+        "modelInput": false
+      },
+      {
+        "path": "src/errors.js",
+        "content": "const logger = require('./logger');\n\n// Express selects error middleware by ARITY: a handler is only registered\n// as an error handler when it declares exactly four parameters. `req` and\n// `next` are required by that signature even though this terminal handler\n// uses neither. See docs/review-conventions.md.\nfunction errorHandler(err, req, res, next) {\n  logger.error('unhandled request error', err);\n  res.status(500).json({ error: 'internal_error' });\n}\n\nfunction register(app) {\n  app.use(errorHandler);\n}\n\nmodule.exports = { errorHandler, register };\n",
+        "modelInput": false
+      },
+      {
+        "path": "src/logger.js",
+        "content": "function warn(message, err) {\n  process.stderr.write(`[warn] ${message}: ${err && err.message}\\n`);\n}\n\nfunction error(message, err) {\n  process.stderr.write(`[error] ${message}: ${err && err.stack}\\n`);\n}\n\nmodule.exports = { warn, error };\n",
+        "modelInput": false
+      },
+      {
+        "path": "docs/review-conventions.md",
+        "content": "# Review conventions\n\nThe house rules this codebase is reviewed against.\n\n- Every SQL statement uses placeholders. A request value is never\n  concatenated or interpolated into a query string.\n- Every acquired resource (file handle, connection, lock) is released on\n  every path out of the function, including the error path.\n- A caught error is either handled or re-thrown. A catch block never\n  reports success.\n- Slice and range bounds are half-open: `slice(start, start + size)`.\n- Debug logging does not ship.\n\n## Deliberate patterns - correct as written\n\nThese three shapes look like common smells and are not. They are house\nstyle, and a review must not raise them as findings.\n\n- `== null` is the one loose comparison the style allows. It matches null\n  and undefined together and nothing else, which is precisely what an\n  optional field needs. It is not to be tightened to `===`.\n- Best-effort background writes (audit, telemetry) are deliberately not\n  awaited, so they cannot delay or break the request that triggered them.\n  A `.catch` handler attached at the call site is the required form; a\n  promise carrying a `.catch` is not a floating promise.\n- Express selects error middleware by arity: a handler is only registered\n  as an error handler when it declares exactly four parameters. The\n  parameters such a handler does not use are required by that signature\n  and stay.\n"
       },
       {
         "path": "reviews/rev-eval-1/manifest.json",
         "surface": "artifact",
-        "content": "{\n  \"version\": 1,\n  \"reviewId\": \"rev-eval-1\",\n  \"kind\": \"commit\",\n  \"createdAt\": \"2026-07-29T09:00:00Z\",\n  \"branch\": \"main\",\n  \"headSha\": \"3d0c1a2\",\n  \"baseRef\": \"HEAD\",\n  \"files\": [\n    { \"path\": \"src/payment.js\", \"kind\": \"modified\", \"additions\": 8, \"deletions\": 2 }\n  ],\n  \"totalFiles\": 1,\n  \"filesTruncated\": false,\n  \"diffFile\": \"changes.diff\",\n  \"diffChars\": 900,\n  \"diffTruncated\": false,\n  \"notes\": []\n}\n"
+        "content": "{\n  \"version\": 1,\n  \"reviewId\": \"rev-eval-1\",\n  \"kind\": \"commit\",\n  \"createdAt\": \"2026-07-29T09:00:00Z\",\n  \"branch\": \"main\",\n  \"headSha\": \"3d0c1a2\",\n  \"baseRef\": \"HEAD\",\n  \"files\": [\n    {\n      \"path\": \"src/payment.js\",\n      \"kind\": \"modified\",\n      \"additions\": 8,\n      \"deletions\": 2\n    },\n    {\n      \"path\": \"src/audit.js\",\n      \"kind\": \"added\",\n      \"additions\": 11,\n      \"deletions\": 0\n    },\n    {\n      \"path\": \"src/customer.js\",\n      \"kind\": \"added\",\n      \"additions\": 13,\n      \"deletions\": 0\n    },\n    {\n      \"path\": \"src/errors.js\",\n      \"kind\": \"added\",\n      \"additions\": 16,\n      \"deletions\": 0\n    },\n    {\n      \"path\": \"src/export.js\",\n      \"kind\": \"added\",\n      \"additions\": 12,\n      \"deletions\": 0\n    },\n    {\n      \"path\": \"src/pagination.js\",\n      \"kind\": \"added\",\n      \"additions\": 13,\n      \"deletions\": 0\n    },\n    {\n      \"path\": \"src/settings.js\",\n      \"kind\": \"added\",\n      \"additions\": 11,\n      \"deletions\": 0\n    }\n  ],\n  \"totalFiles\": 7,\n  \"filesTruncated\": false,\n  \"diffFile\": \"changes.diff\",\n  \"diffChars\": 4396,\n  \"diffTruncated\": false,\n  \"notes\": []\n}\n"
       },
       {
         "path": "reviews/rev-eval-1/changes.diff",
         "surface": "artifact",
-        "content": "diff --git a/src/payment.js b/src/payment.js\nindex 3d0c1a2..9f4b7e1 100644\n--- a/src/payment.js\n+++ b/src/payment.js\n@@ -1,12 +1,16 @@\n const db = require('./db');\n \n async function chargeCustomer(req) {\n   const amount = req.body.amount;\n   const customerId = req.body.customerId;\n-  const rows = await db.query('SELECT * FROM customers WHERE id = $1', [customerId]);\n+  // TODO: validate amount server-side\n+  const rows = await db.query(\"SELECT * FROM customers WHERE id = '\" + customerId + \"'\");\n   const customer = rows[0];\n-  const receipt = await db.query('INSERT INTO charges (customer_id, amount) VALUES ($1, $2) RETURNING id', [customerId, amount]);\n+  console.log('charging', customerId, amount);\n+  const receipt = await db.query(\n+    \"INSERT INTO charges (customer_id, amount) VALUES ('\" + customerId + \"', \" + amount + \") RETURNING id\",\n+  );\n   return { chargeId: receipt[0].id, customer: customer.name };\n }\n \n module.exports = { chargeCustomer };\n"
+        "content": "diff --git a/src/payment.js b/src/payment.js\nindex 3d0c1a2..9f4b7e1 100644\n--- a/src/payment.js\n+++ b/src/payment.js\n@@ -1,12 +1,16 @@\n const db = require('./db');\n \n async function chargeCustomer(req) {\n   const amount = req.body.amount;\n   const customerId = req.body.customerId;\n-  const rows = await db.query('SELECT * FROM customers WHERE id = $1', [customerId]);\n+  // TODO: validate amount server-side\n+  const rows = await db.query(\"SELECT * FROM customers WHERE id = '\" + customerId + \"'\");\n   const customer = rows[0];\n-  const receipt = await db.query('INSERT INTO charges (customer_id, amount) VALUES ($1, $2) RETURNING id', [customerId, amount]);\n+  console.log('charging', customerId, amount);\n+  const receipt = await db.query(\n+    \"INSERT INTO charges (customer_id, amount) VALUES ('\" + customerId + \"', \" + amount + \") RETURNING id\",\n+  );\n   return { chargeId: receipt[0].id, customer: customer.name };\n }\n \n module.exports = { chargeCustomer };\ndiff --git a/src/audit.js b/src/audit.js\nnew file mode 100644\nindex 0000000..297df83\n--- /dev/null\n+++ b/src/audit.js\n@@ -0,0 +1,11 @@\n+const logger = require('./logger');\n+\n+// Audit writes are best-effort: they must never delay or break the charge\n+// that triggered them, so this call is deliberately not awaited. The\n+// attached .catch is what keeps a rejected write from becoming an\n+// unhandled rejection. See docs/review-conventions.md.\n+function recordAudit(db, event) {\n+  db.insertAudit(event).catch((err) => logger.warn('audit write skipped', err));\n+}\n+\n+module.exports = { recordAudit };\ndiff --git a/src/customer.js b/src/customer.js\nnew file mode 100644\nindex 0000000..c722780\n--- /dev/null\n+++ b/src/customer.js\n@@ -0,0 +1,13 @@\n+// `== null` is the one loose comparison the house style allows: it matches\n+// null and undefined together and nothing else, which is exactly what an\n+// optional field needs. See docs/review-conventions.md.\n+function displayName(customer) {\n+  if (customer.nickname == null) return customer.legalName;\n+  return customer.nickname;\n+}\n+\n+function billingEmail(customer) {\n+  return customer.billingEmail == null ? customer.email : customer.billingEmail;\n+}\n+\n+module.exports = { displayName, billingEmail };\ndiff --git a/src/errors.js b/src/errors.js\nnew file mode 100644\nindex 0000000..7107b89\n--- /dev/null\n+++ b/src/errors.js\n@@ -0,0 +1,16 @@\n+const logger = require('./logger');\n+\n+// Express selects error middleware by ARITY: a handler is only registered\n+// as an error handler when it declares exactly four parameters. `req` and\n+// `next` are required by that signature even though this terminal handler\n+// uses neither. See docs/review-conventions.md.\n+function errorHandler(err, req, res, next) {\n+  logger.error('unhandled request error', err);\n+  res.status(500).json({ error: 'internal_error' });\n+}\n+\n+function register(app) {\n+  app.use(errorHandler);\n+}\n+\n+module.exports = { errorHandler, register };\ndiff --git a/src/export.js b/src/export.js\nnew file mode 100644\nindex 0000000..01f5dcf\n--- /dev/null\n+++ b/src/export.js\n@@ -0,0 +1,12 @@\n+const fs = require('fs/promises');\n+\n+async function exportCharges(db, outPath) {\n+  const handle = await fs.open(outPath, 'w');\n+  const rows = await db.query('SELECT id, customer_id, amount FROM charges ORDER BY id');\n+  for (const row of rows) {\n+    await handle.write(`${row.id},${row.customer_id},${row.amount}\\n`);\n+  }\n+  await handle.close();\n+}\n+\n+module.exports = { exportCharges };\ndiff --git a/src/pagination.js b/src/pagination.js\nnew file mode 100644\nindex 0000000..008b78a\n--- /dev/null\n+++ b/src/pagination.js\n@@ -0,0 +1,13 @@\n+const DEFAULT_PAGE_SIZE = 25;\n+\n+function pageOf(items, page, pageSize = DEFAULT_PAGE_SIZE) {\n+  const start = (page - 1) * pageSize;\n+  const end = start + pageSize + 1;\n+  return items.slice(start, end);\n+}\n+\n+function pageCount(items, pageSize = DEFAULT_PAGE_SIZE) {\n+  return Math.ceil(items.length / pageSize);\n+}\n+\n+module.exports = { pageOf, pageCount, DEFAULT_PAGE_SIZE };\ndiff --git a/src/settings.js b/src/settings.js\nnew file mode 100644\nindex 0000000..b7c463b\n--- /dev/null\n+++ b/src/settings.js\n@@ -0,0 +1,11 @@\n+const fs = require('fs');\n+\n+function loadSettings(path) {\n+  try {\n+    return { ok: true, settings: JSON.parse(fs.readFileSync(path, 'utf8')) };\n+  } catch (err) {\n+    return { ok: true, settings: {} };\n+  }\n+}\n+\n+module.exports = { loadSettings };\n"
+      },
+      {
+        "path": "tests/code-review-oracle.mjs",
+        "content": "// Precision/recall oracle for the snapshot code review.\n//\n// Run by the harness with the reviewed workspace as cwd. It grades the two\n// halves of a real review separately, because recall alone is passed by a\n// review that flags everything:\n//\n//   RECALL    - each of the four seeded defects in the change set is\n//               reported, against the right file and as the right class.\n//   PRECISION - no finding repeats one of the three shotgun smells the\n//               change set deliberately baits, and no finding is raised\n//               against a file outside the change set.\n//\n// Two rules keep this from punishing correct work.\n//\n// First, defect classes are matched by the ordinary vocabulary of the\n// class, never by wording this fixture invented: \"26 items per page\",\n// \"the end index is one too high\" and \"half-open range\" all count as the\n// pagination defect. The bar is \"a real review of this class\", not this\n// scenario's phrasing.\n//\n// Second, the precision half is scoped to the SMELL, not to the file.\n// src/customer.js, src/audit.js and src/errors.js are baited with a\n// documented house-style construct that must not be flagged - but they\n// are still ordinary code, and a reviewer may legitimately find\n// something else in them (a missing res.headersSent guard in the error\n// handler, say, or a synchronous throw escaping the best-effort audit\n// call). Failing those would punish the exact deep reading this eval is\n// trying to reward, so only a finding that IS the baited smell counts\n// against precision. Whole-file scope is reserved for the two files that\n// are not in the change set at all, where any finding is out of scope by\n// the book's own rule.\n//\n// Precision is graded on the finding records only: a report that\n// discusses a correct construct and explicitly clears it is good\n// reviewing, and is not penalised here.\nimport { readFileSync } from 'node:fs';\n\n/**\n * Fail with the message ALONE on stderr.\n *\n * A thrown Error would work, but the harness feeds this script's stderr\n * into the model's repair nudge verbatim, and an uncaught throw pads that\n * nudge with a Node stack trace and a version banner. The actionable\n * sentence is what the next attempt needs; the stack is noise that pushes\n * it out of view on a small model.\n */\nfunction fail(message) {\n  console.error(message);\n  process.exit(1);\n}\n\n\nconst FINDINGS = 'review-output/findings.json';\n\nlet raw;\ntry {\n  raw = readFileSync(FINDINGS, 'utf8');\n} catch {\n  fail(FINDINGS + ' does not exist - the machine-readable finding list is required');\n}\n\nlet findings;\ntry {\n  findings = JSON.parse(raw.replace(/^\\uFEFF/, ''));\n} catch (err) {\n  fail(FINDINGS + ' is not valid JSON: ' + err.message);\n}\nif (!Array.isArray(findings)) {\n  const nested =\n    findings && typeof findings === 'object'\n      ? Object.values(findings).find((value) => Array.isArray(value))\n      : null;\n  if (!nested) fail(FINDINGS + ' must be an array of finding records');\n  findings = nested;\n}\nfindings = findings.filter((finding) => finding && typeof finding === 'object');\n\n// Every PROSE string anywhere in the record, nested objects and arrays\n// included - a reviewer who nests the explanation under `detail` or\n// splits it across an array is saying the same thing. Citation and\n// identifier fields are excluded: they are addresses, not description,\n// and letting them through means a line number or a path fragment can\n// satisfy a defect-class matcher by coincidence.\nconst CITATION_KEYS = new Set([\n  'file',\n  'path',\n  'filename',\n  'location',\n  'line',\n  'lines',\n  'endline',\n  'startline',\n  'column',\n  'col',\n  'id',\n  '#',\n]);\nconst collectStrings = (value, out) => {\n  if (typeof value === 'string') out.push(value);\n  else if (Array.isArray(value)) for (const item of value) collectStrings(item, out);\n  else if (value && typeof value === 'object') {\n    for (const [key, item] of Object.entries(value)) {\n      if (CITATION_KEYS.has(key.toLowerCase())) continue;\n      collectStrings(item, out);\n    }\n  }\n  return out;\n};\nconst text = (finding) => collectStrings(finding, []).join(' ').toLowerCase();\n\nconst fileOf = (finding) =>\n  String(finding.file ?? finding.path ?? finding.filename ?? finding.location ?? '')\n    .replace(/\\\\/g, '/')\n    .replace(/^\\.\\//, '')\n    .replace(/^[ab]\\//, '')\n    .replace(/\\s*\\([^()]*\\)\\s*$/, '')\n    .replace(/[:#]\\s*L?\\d+(?:\\s*-\\s*L?\\d+)?\\s*$/, '')\n    .trim();\n\n// Basenames are unique across every seeded file, so a bare `payment.js`\n// citation resolves the same as `src/payment.js`.\nconst onFile = (path) => {\n  const base = path.slice(path.lastIndexOf('/') + 1);\n  const full = new RegExp('(^|/)' + path.replace(/\\./g, '\\\\.') + '$');\n  const bare = new RegExp('^' + base.replace(/\\./g, '\\\\.') + '$');\n  return findings.filter((finding) => {\n    const cited = fileOf(finding);\n    return full.test(cited) || bare.test(cited);\n  });\n};\n\nconst REQUIRED = [\n  {\n    file: 'src/payment.js',\n    label: 'the customer and charge queries build SQL by string concatenation from req.body',\n    match: (t) =>\n      /injection|sqli\\b|parameteri|parametri|placeholder|prepared statement|\\bbind\\b|\\bbinding\\b|bound parameter|concatenat|interpolat|escap|sanitiz|quoting/.test(\n        t,\n      ) ||\n      (/\\bsql\\b|\\bquery\\b|\\bqueries\\b|statement/.test(t) &&\n        /untrusted|user[- ](?:input|supplied|controlled|data)|req\\.body|request (?:value|data|input|body)|unsafe|unsanitiz|unvalidated|string (?:concat|building|built)|built from|splic/.test(\n          t,\n        )),\n  },\n  {\n    file: 'src/pagination.js',\n    label: 'pageOf is off by one on its slice end bound, so pages overlap by a row',\n    match: (t) =>\n      /off.?by.?one|\\bslice\\b|\\bbound(?:ary|aries|s)?\\b|half.?open|exclusive|inclusive|overlap|duplicat|repeat|reappear|one too many|one (?:extra|more)|too many|extra (?:item|row|record|element|entry|result)|\\+\\s?1\\b|pagesize \\+|\\+ pagesize|increment|\\b26\\b|end (?:index|bound|offset|position)|last (?:item|row|record|element)/.test(\n        t,\n      ),\n  },\n  {\n    file: 'src/settings.js',\n    label: 'loadSettings swallows the parse error and still reports ok: true',\n    match: (t) =>\n      /swallow|silent|suppress|\\bmask|\\bhid(?:e|es|den|ing)|\\bignor|empty catch|catch block|\\bok\\b|\\btrue\\b|success|succeed|re-?throw|rethrow|propagat|bubble|surfac|indistinguishable|no indication|cannot tell|unaware|misleading|misreport|false (?:success|positive)|as if|pretend/.test(\n        t,\n      ),\n  },\n  {\n    file: 'src/export.js',\n    label: 'exportCharges leaks the file handle when the query or a write throws',\n    match: (t) =>\n      /leak|\\bclos(?:e|es|ed|ing)\\b|\\bfinally\\b|clean.?up|releas|dispos|\\bhandle\\b|descriptor|\\bfd\\b|resource|left open|remains open|stays open|still open|dangling|exhaust|unclosed/.test(\n        t,\n      ),\n  },\n];\n\nconst missing = REQUIRED.filter(\n  (required) => !onFile(required.file).some((finding) => required.match(text(finding))),\n);\nif (missing.length > 0) {\n  const reported = findings.map((finding) => fileOf(finding) || '?').join(', ') || '(nothing)';\n  fail(\n    'RECALL SHORTFALL - the change set holds four separate defects and these were not reported: ' +\n      missing.map((entry) => entry.file + ' (' + entry.label + ')').join('; ') +\n      '. Files carrying a reported finding: ' +\n      reported +\n      '. Re-walk changes.diff hunk by hunk; every defect above is visible in the added lines.',\n  );\n}\n\n// Baited smells. Each of these three files carries a construct that looks\n// like a common lint smell and is documented house style in\n// docs/review-conventions.md. Only a finding that raises THAT construct\n// is a precision failure; any other observation about the file is fair\n// review work and passes.\nconst BAITED = [\n  {\n    file: 'src/customer.js',\n    smell: 'loose equality',\n    matches: (t) =>\n      /[=!]==?|loose equal|strict equal|\\bequality\\b|double equals|triple equals|coerc|nullish|\\?\\?/.test(\n        t,\n      ),\n    why: '`== null` is the documented house idiom for a null-or-undefined check (docs/review-conventions.md) and is not to be tightened',\n  },\n  {\n    file: 'src/audit.js',\n    smell: 'floating / unawaited promise',\n    matches: (t) =>\n      /floating|un-?awaited|not awaited|missing await|no await|without await|should (?:be )?await|add (?:an )?await|\\bawait\\b|fire.?and.?forget|dangling promise|unhandled (?:promise )?rejection/.test(\n        t,\n      ) &&\n      // A reviewer who spots that a SYNCHRONOUS throw from insertAudit\n      // still escapes into the charge path has found something real, not\n      // the bait. That finding is welcome.\n      !/synchronous|synchronously|thenable|non-?promise|not (?:return )?a promise|does not return|returns undefined|before the promise/.test(\n        t,\n      ),\n    why: 'the audit write is deliberately not awaited and carries a .catch, which is the required form for a best-effort write (docs/review-conventions.md)',\n  },\n  {\n    file: 'src/errors.js',\n    smell: 'unused parameters',\n    matches: (t) =>\n      /unused|not used|never used|un-?referenced|no-unused-vars|underscore|_req|_next|dead (?:parameter|argument)|(?:remove|drop|omit) (?:the )?(?:`?req`?|`?next`?|parameter|argument)|unnecessary (?:parameter|argument)/.test(\n        t,\n      ),\n    why: \"Express registers error middleware by arity, so the four-parameter signature and the parameters it does not use are required (docs/review-conventions.md)\",\n  },\n];\n\n// Not in manifest.json's seven-file list and not in changes.diff. The\n// book says to discuss only files in the change set, so any finding here\n// is out of scope whatever it says.\nconst OUT_OF_SCOPE = [\n  { file: 'src/util.js', why: 'it is not one of the seven files in manifest.json' },\n  { file: 'src/logger.js', why: 'it is not one of the seven files in manifest.json' },\n];\n\nconst nameOf = (finding) =>\n  String(finding.category ?? finding.finding ?? finding.title ?? 'unnamed');\n\nconst wrong = [];\nfor (const bait of BAITED) {\n  for (const finding of onFile(bait.file)) {\n    if (!bait.matches(text(finding))) continue;\n    wrong.push(bait.file + ' (' + nameOf(finding) + ' - ' + bait.smell + ') - ' + bait.why);\n  }\n}\nfor (const out of OUT_OF_SCOPE) {\n  for (const finding of onFile(out.file)) {\n    wrong.push(out.file + ' (' + nameOf(finding) + ') - ' + out.why);\n  }\n}\nif (wrong.length > 0) {\n  fail(\n    'PRECISION SHORTFALL - drop these records from ' +\n      FINDINGS +\n      ': ' +\n      wrong.join('; ') +\n      '. A review that raises everything is not usable. Re-read docs/review-conventions.md, ' +\n      'which names the shapes that are deliberate house style, and re-read manifest.json for ' +\n      'which files are actually under review. Discussing a construct in report.md and clearing ' +\n      'it is fine - listing it as a finding is not. Any OTHER defect you find in these files is ' +\n      'welcome; only the constructs named above are off limits.',\n  );\n}\n\nconsole.log(\n  'CODEREVIEW_ORACLE ok - all 4 seeded defects reported (src/payment.js, src/pagination.js, ' +\n    'src/settings.js, src/export.js) and none of the 3 baited house-style smells or 2 ' +\n    'out-of-scope files carries a finding.',\n);\n",
+        "surface": "harness",
+        "modelInput": false
       }
     ]
   },
   "mocks": [],
   "success": {
-    "summary": "reviews/rev-eval-1/report.md exists in the artifacts drawer with Summary/Findings/Verdict sections, a severity table citing src/payment.js, and a request-changes verdict consistent with the seeded SQL-injection flaw.",
-    "checks": [
-      {
-        "kind": "minBytes",
-        "file": "reviews/rev-eval-1/report.md",
-        "bytes": 400,
-        "artifact": true
-      },
-      {
-        "kind": "contains",
-        "file": "reviews/rev-eval-1/report.md",
-        "pattern": "##\\s+Summary",
-        "label": "Summary section",
-        "artifact": true
-      },
+    "summary": "reviews/rev-eval-1/report.md exists in the artifacts drawer with Summary/Findings/Verdict sections, a severity table citing the flawed files, and a request-changes verdict; review-output/findings.json holds one record per finding carrying at least file, line, severity, category and recommendation (extra context such as a title or a cwe is welcome), lists every seeded defect (unparameterized SQL, the off-by-one page bound, the swallowed settings error, the leaked export file handle), repeats none of the three documented house-style smells, and raises nothing against the two files outside the change set.",
+    "deliverables": [
       {
-        "kind": "contains",
-        "file": "reviews/rev-eval-1/report.md",
-        "pattern": "##\\s+Findings",
-        "label": "Findings section",
-        "artifact": true
+        "path": "reviews/rev-eval-1/report.md",
+        "kind": "markdown-report",
+        "artifact": true,
+        "minBytes": 400,
+        "checks": [
+          {
+            "kind": "contains",
+            "file": "reviews/rev-eval-1/report.md",
+            "pattern": "##\\s+Summary",
+            "label": "Summary section",
+            "artifact": true
+          },
+          {
+            "kind": "contains",
+            "file": "reviews/rev-eval-1/report.md",
+            "pattern": "##\\s+Findings",
+            "label": "Findings section",
+            "artifact": true
+          },
+          {
+            "kind": "contains",
+            "file": "reviews/rev-eval-1/report.md",
+            "pattern": "Verdict:\\s*request-changes",
+            "label": "request-changes verdict (SQL concat is at least major)",
+            "artifact": true
+          },
+          {
+            "kind": "contains",
+            "file": "reviews/rev-eval-1/report.md",
+            "pattern": "src/payment\\.js",
+            "label": "cites the flawed file",
+            "artifact": true
+          },
+          {
+            "kind": "tableShape",
+            "file": "reviews/rev-eval-1/report.md",
+            "requiredColumns": [
+              "Severity",
+              "File",
+              "Finding"
+            ],
+            "minRows": 1,
+            "artifact": true
+          }
+        ]
       },
       {
-        "kind": "contains",
-        "file": "reviews/rev-eval-1/report.md",
-        "pattern": "Verdict:\\s*request-changes",
-        "label": "request-changes verdict (SQL concat is at least major)",
-        "artifact": true
-      },
-      {
-        "kind": "contains",
-        "file": "reviews/rev-eval-1/report.md",
-        "pattern": "src/payment\\.js",
-        "label": "cites the flawed file",
-        "artifact": true
-      },
-      {
-        "kind": "tableShape",
-        "file": "reviews/rev-eval-1/report.md",
-        "requiredColumns": [
-          "Severity",
-          "File",
-          "Finding"
-        ],
-        "minRows": 1,
-        "artifact": true
+        "path": "review-output/findings.json",
+        "kind": "json",
+        "minBytes": 300,
+        "checks": [
+          {
+            "kind": "sniff",
+            "file": "review-output/findings.json",
+            "sniff": "json-valid"
+          },
+          {
+            "kind": "recordSchema",
+            "file": "review-output/findings.json",
+            "format": "json",
+            "minRows": 4,
+            "fields": [
+              {
+                "name": "file",
+                "type": "nonempty",
+                "required": true
+              },
+              {
+                "name": "line",
+                "type": "^\\d+(\\s*-\\s*\\d+)?$",
+                "required": true
+              },
+              {
+                "name": "severity",
+                "type": "^([Cc]ritical|[Mm]ajor|[Mm]inor|[Nn]it|CRITICAL|MAJOR|MINOR|NIT)$",
+                "required": true
+              },
+              {
+                "name": "category",
+                "type": "nonempty",
+                "required": true
+              },
+              {
+                "name": "recommendation",
+                "type": "nonempty",
+                "required": true
+              }
+            ],
+            "allowExtraFields": true
+          },
+          {
+            "kind": "nodeScriptPasses",
+            "script": "tests/code-review-oracle.mjs",
+            "timeoutMs": 30000,
+            "requiredOutput": [
+              {
+                "pattern": "CODEREVIEW_ORACLE ok",
+                "label": "every seeded defect reported, and no correct-as-written construct reported"
+              }
+            ]
+          }
+        ]
       }
     ],
     "taskNotes": {
@@ -92,7 +197,19 @@
           "flags": "i"
         }
       ]
-    }
+    },
+    "unchangedFixtures": [
+      "src/payment.js",
+      "src/util.js",
+      "src/pagination.js",
+      "src/settings.js",
+      "src/export.js",
+      "src/customer.js",
+      "src/audit.js",
+      "src/errors.js",
+      "src/logger.js",
+      "docs/review-conventions.md"
+    ]
   },
   "rubric": {
     "artifact": {
@@ -104,23 +221,28 @@
         "name": "grounding",
         "description": "Findings reference code that actually appears in changes.diff; nothing is invented and cited lines exist."
       },
+      {
+        "name": "precision",
+        "description": "The three documented house-style constructs (the `== null` check, the unawaited best-effort audit write with its .catch, the four-parameter Express error handler) are not raised as findings, and the two files outside the change set (src/util.js, src/logger.js) carry no findings at all; if a house-style construct is discussed it is to clear it. A different, real defect found in one of those three files is good review work and counts in its favour, not against it."
+      },
       {
         "name": "severity-judgment",
-        "description": "The SQL string concatenation is rated critical or major; the console.log and TODO are minor or nit — severities proportionate to impact."
+        "description": "The SQL string concatenation, the off-by-one page bound, the swallowed settings error and the leaked export file handle are rated critical or major; the console.log and TODO are minor or nit - severities proportionate to impact."
       },
       {
         "name": "actionability",
-        "description": "Every finding carries a concrete, minimal recommendation (e.g. return to parameterized queries), not generic advice."
+        "description": "Every finding carries a concrete, minimal recommendation (e.g. return to parameterized queries, close the handle in a finally), not generic advice."
       },
       {
         "name": "verdict-consistency",
         "description": "The verdict follows mechanically from the findings table: any critical or major finding yields request-changes."
       }
     ],
-    "contextNote": "The report is an artifacts-drawer file, not a workspace file — the reviewer must not modify workspace source."
+    "contextNote": "The report is an artifacts-drawer file and review-output/findings.json is the single workspace file the reviewer writes - no source file may change. Some of the reviewed code only looks wrong: docs/review-conventions.md records which shapes are deliberate house style. src/util.js and src/logger.js are in the workspace as context but are not in the change set - manifest.json's seven files are."
   },
   "qualityFocus": [
     "evidence-grounded review",
+    "review precision",
     "file citations",
     "artifact deliverable discipline"
   ]

package/data/craftbook-templates/de/deep-security-review/versions/1.1.2/test.json

@@ -19,9 +19,19 @@
         "path": "src/profile.ts",
         "content": "export function registerProfile(app, db, requireUser) {\n  app.get('/profile', requireUser, async (req, res) => {\n    const row = await db.query('SELECT * FROM users WHERE id = ?', [req.user.id]);\n    res.json(row);\n  });\n}\n"
       },
+      {
+        "path": "src/reports.ts",
+        "content": "import { exec } from 'node:child_process';\n\nconst ALLOWED_SORTS = { name: 'name', created: 'created_at', team: 'team' };\n\nexport function registerReports(app, db, requireUser) {\n  app.get('/health', (_req, res) => res.json({ ok: true }));\n\n  app.get('/reports/rotate', requireUser, (_req, res) => {\n    exec('logrotate -f /etc/logrotate.conf', (_e, out) => res.send(out));\n  });\n\n  app.get('/reports/users', requireUser, async (req, res) => {\n    const column = ALLOWED_SORTS[req.query.sort] ?? 'name';\n    const rows = await db.query(`SELECT id, name FROM users ORDER BY ${column}`);\n    res.json(rows);\n  });\n}\n"
+      },
       {
         "path": "docs/security.md",
-        "content": "# Security model\n\nAll /admin routes must require the requireAdmin middleware before business logic runs. SQL must use placeholders. Shell commands must not include request data.\n"
+        "content": "# Security model\n\nAll /admin routes must require the requireAdmin middleware before business logic runs. SQL must use placeholders. Shell commands must not include request data.\n\n## Deliberate exceptions\n\n- `/health` is unauthenticated by design. It returns a fixed literal and reads nothing.\n- A SQL identifier (a column or table name) cannot be a placeholder. Where one must vary,\n  the only accepted pattern is mapping the request value through a fixed allow-list and\n  interpolating the mapped constant. Code that already does this is correct as written.\n- A shell command whose argument list contains no request data is not command injection.\n"
+      },
+      {
+        "path": "tests/security-oracle.mjs",
+        "content": "// Precision/recall oracle for the deep security review.\n//\n// Run by the harness with the reviewed workspace as cwd. It grades the two\n// halves of a real review separately, because recall alone is passed by a\n// report that flags everything:\n//\n//   RECALL   — each of the three seeded defects in src/admin.ts is reported.\n//   PRECISION — none of the three SAFE constructs is reported. Each is a\n//               deliberate look-alike of a seeded defect and each is\n//               documented as correct in docs/security.md.\n//\n// The classes are matched by their ordinary names, not by any wording this\n// fixture invented, so the bar is \"a real review of this class of defect\"\n// rather than this scenario's vocabulary.\nimport { readFileSync } from 'node:fs';\n\n/**\n * Fail with the message ALONE on stderr.\n *\n * A thrown Error would work, but the harness feeds this script's stderr\n * into the model's repair nudge verbatim, and an uncaught throw pads that\n * nudge with a Node stack trace and a version banner. The actionable\n * sentence is what the next attempt needs; the stack is noise that pushes\n * it out of view on a small model.\n */\nfunction fail(message) {\n  console.error(message);\n  process.exit(1);\n}\n\n\nconst FINDINGS = 'security-review/findings.json';\n\nlet raw;\ntry {\n  raw = readFileSync(FINDINGS, 'utf8');\n} catch {\n  fail(FINDINGS + ' does not exist - the machine-readable findings file is required');\n}\n\nlet findings;\ntry {\n  findings = JSON.parse(raw.replace(/^\\uFEFF/, ''));\n} catch (err) {\n  fail(FINDINGS + ' is not valid JSON: ' + err.message);\n}\nif (!Array.isArray(findings)) {\n  const nested = findings && typeof findings === 'object'\n    ? Object.values(findings).find((value) => Array.isArray(value))\n    : null;\n  if (!nested) fail(FINDINGS + ' must be an array of finding records');\n  findings = nested;\n}\n\nconst text = (finding) =>\n  Object.values(finding)\n    .filter((value) => typeof value === 'string')\n    .join(' ')\n    .toLowerCase();\n\nconst fileOf = (finding) => String(finding.file ?? finding.path ?? '').replace(/\\\\/g, '/');\n\nconst REQUIRED = [\n  {\n    label: 'command injection in the /admin/export shell call',\n    match: /command injection|shell injection|\\bexec\\b|child_process|os command/,\n  },\n  {\n    label: 'SQL injection in the /admin/users query',\n    match: /sql injection|sqli|parameteri[sz]|placeholder|string interpolation into (a )?quer/,\n  },\n  {\n    label: 'missing authorization on the /admin routes',\n    match: /authorization|authentication|requireadmin|access control|missing auth|broken auth/,\n  },\n];\n\nconst adminFindings = findings.filter((finding) => /(^|\\/)src\\/admin\\.ts$/.test(fileOf(finding)));\nconst missing = REQUIRED.filter(\n  (required) => !adminFindings.some((finding) => required.match.test(text(finding))),\n);\nif (missing.length > 0) {\n  fail(\n    'RECALL FAILURE - src/admin.ts holds three separate defects and these were not reported: ' +\n      missing.map((entry) => entry.label).join('; ') +\n      '. Reported for src/admin.ts: ' +\n      (adminFindings.map((finding) => String(finding.category ?? '?')).join(', ') || '(nothing)'),\n  );\n}\n\nconst SAFE_FILES = [\n  {\n    file: 'src/profile.ts',\n    why: 'its query uses a placeholder and its route is behind requireUser',\n  },\n  {\n    file: 'src/reports.ts',\n    why:\n      'its exec call contains no request data, its ORDER BY column comes from a fixed ' +\n      'allow-list, and /health is documented as deliberately unauthenticated',\n  },\n];\n\nconst falsePositives = [];\nfor (const safe of SAFE_FILES) {\n  const pattern = new RegExp('(^|/)' + safe.file.replace(/[.]/g, '\\\\.') + '$');\n  for (const finding of findings) {\n    if (pattern.test(fileOf(finding))) {\n      falsePositives.push(\n        safe.file + ' (' + String(finding.category ?? 'unnamed finding') + ') - ' + safe.why,\n      );\n    }\n  }\n}\nif (falsePositives.length > 0) {\n  fail(\n    'PRECISION FAILURE - these files are correct as written and must not be reported as ' +\n      'vulnerable: ' + falsePositives.join('; ') +\n      '. A review that flags safe code is not usable; re-read docs/security.md, which states ' +\n      'which constructs are deliberate.',\n  );\n}\n\nconsole.log(\n  'Security precision/recall oracle passed: all 3 seeded defects reported on src/admin.ts, ' +\n    '0 false positives on the 2 safe files.',\n);\n",
+        "surface": "workspace",
+        "modelInput": false
       }
     ],
     "worker": {
@@ -125,6 +135,23 @@
                 "name": "theme",
                 "type": "nonempty",
                 "required": true
+              },
+              {
+                "name": "title",
+                "type": "nonempty",
+                "required": true
+              }
+            ],
+            "allowExtraFields": true
+          },
+          {
+            "kind": "nodeScriptPasses",
+            "script": "tests/security-oracle.mjs",
+            "timeoutMs": 30000,
+            "requiredOutput": [
+              {
+                "pattern": "Security precision/recall oracle passed",
+                "label": "every seeded defect reported, and no safe construct reported"
               }
             ]
           }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bendyline/gilde",
-  "version": "0.1.49",
+  "version": "0.1.50",
   "description": "The gilde catalog: model manifests, toolsets, craftbooks, roles, and project types for gezel.",
   "license": "MIT",
   "private": false,

package/schemas/craftbook-doc.schema.json

@@ -1729,6 +1729,9 @@
                                 "auto"
                               ]
                             },
+                            "allowExtraFields": {
+                              "type": "boolean"
+                            },
                             "artifact": {
                               "type": "boolean"
                             }
@@ -2763,6 +2766,9 @@
                                 "auto"
                               ]
                             },
+                            "allowExtraFields": {
+                              "type": "boolean"
+                            },
                             "artifact": {
                               "type": "boolean"
                             }
@@ -4764,6 +4770,9 @@
                                     "auto"
                                   ]
                                 },
+                                "allowExtraFields": {
+                                  "type": "boolean"
+                                },
                                 "artifact": {
                                   "type": "boolean"
                                 }
@@ -5798,6 +5807,9 @@
                                     "auto"
                                   ]
                                 },
+                                "allowExtraFields": {
+                                  "type": "boolean"
+                                },
                                 "artifact": {
                                   "type": "boolean"
                                 }

package/schemas/craftbook-template-version.schema.json

@@ -1325,6 +1325,9 @@
                                 "auto"
                               ]
                             },
+                            "allowExtraFields": {
+                              "type": "boolean"
+                            },
                             "artifact": {
                               "type": "boolean"
                             }
@@ -2359,6 +2362,9 @@
                                 "auto"
                               ]
                             },
+                            "allowExtraFields": {
+                              "type": "boolean"
+                            },
                             "artifact": {
                               "type": "boolean"
                             }
@@ -4354,6 +4360,9 @@
                                     "auto"
                                   ]
                                 },
+                                "allowExtraFields": {
+                                  "type": "boolean"
+                                },
                                 "artifact": {
                                   "type": "boolean"
                                 }
@@ -5388,6 +5397,9 @@
                                     "auto"
                                   ]
                                 },
+                                "allowExtraFields": {
+                                  "type": "boolean"
+                                },
                                 "artifact": {
                                   "type": "boolean"
                                 }

package/schemas/craftbook-test.schema.json

@@ -962,6 +962,9 @@
                                 "auto"
                               ]
                             },
+                            "allowExtraFields": {
+                              "type": "boolean"
+                            },
                             "artifact": {
                               "type": "boolean"
                             }
@@ -2074,6 +2077,9 @@
                           "auto"
                         ]
                       },
+                      "allowExtraFields": {
+                        "type": "boolean"
+                      },
                       "artifact": {
                         "type": "boolean"
                       }
@@ -3186,6 +3192,9 @@
                               "auto"
                             ]
                           },
+                          "allowExtraFields": {
+                            "type": "boolean"
+                          },
                           "artifact": {
                             "type": "boolean"
                           }
@@ -4299,6 +4308,9 @@
                               "auto"
                             ]
                           },
+                          "allowExtraFields": {
+                            "type": "boolean"
+                          },
                           "artifact": {
                             "type": "boolean"
                           }