@bjoernrochel/sevro 0.1.0-rc.2 → 0.1.0-rc.21

package/examples/recipes/runtime/plugins.json

@@ -0,0 +1,4 @@
+{
+  "format": "sevro.runtime.v1",
+  "hooks": { "nativeGoal": true, "plugins": ["startup"] }
+}

package/examples/recipes/runtime/python-uv.json

@@ -0,0 +1,10 @@
+{
+  "format": "sevro.runtime.v1",
+  "environment": {
+    "inherit": ["PATH", "RECIPE_PYTHON"],
+    "set": {
+      "UV_CACHE_DIR": "{{sevro.runtime}}/uv-cache",
+      "UV_PYTHON_DOWNLOADS": "never"
+    }
+  }
+}

package/examples/recipes/semantic-judge.ts

@@ -0,0 +1,29 @@
+/** Protocol control only: use a real judge for qualitative model assessment. */
+const judge = {
+  id: "recipes.protocol-judge",
+  model: "exact-fixture-oracle",
+  effort: "none",
+  run({ prompt }: { prompt: string }) {
+    const match =
+      /<candidate-response-json>\s*([\s\S]*?)\s*<\/candidate-response-json>/.exec(
+        prompt,
+      );
+    if (!match?.[1]) throw new Error("Missing semantic candidate response");
+    const candidate: unknown = JSON.parse(match[1]);
+    const passed = candidate === "Run bun run dev on port 3000.";
+    return Promise.resolve({
+      complete: true,
+      finalMessage: JSON.stringify({
+        checks: [
+          {
+            id: "startup-meaning",
+            verdict: passed ? "pass" : "fail",
+            reason:
+              "Exact deterministic fixture oracle; no model judgment was made.",
+          },
+        ],
+      }),
+    });
+  },
+};
+export default judge;

package/examples/templates/coding/adapter.ts

@@ -0,0 +1,17 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+/** Deterministic file editor; it exercises the evaluator, not model coding ability. */
+export default {
+  id: "template.coding",
+  model: "deterministic-editor",
+  effort: "none",
+  async run({ workspace }: { workspace: string }) {
+    if (process.env.SEVRO_TEMPLATE_FAIL !== "1")
+      await writeFile(
+        join(workspace, "add.sh"),
+        'add() { echo "$(( $1 + $2 ))"; }\n',
+      );
+    return { finalMessage: "Completed the coding task.", complete: true };
+  },
+};

package/examples/templates/coding/case.json

@@ -0,0 +1,15 @@
+{
+  "id": "coding-addition",
+  "prompt": "Fix add.sh so add computes integer addition. Preserve the add shell function interface.",
+  "fixture": { "files": { "add.sh": "add() { echo \"$(( $1 - $2 ))\"; }\n" } },
+  "checks": [
+    {
+      "id": "addition",
+      "grader": "sevro.shell",
+      "configuration": {
+        "run": ". ./add.sh; test \"$(add 2 3)\" = 5 && test \"$(add -2 -3)\" = -5 && test \"$(add 0 0)\" = 0"
+      }
+    }
+  ],
+  "requiredEvidence": []
+}

package/examples/templates/repository-skill/.agents/skills/startup-guide/SKILL.md

@@ -0,0 +1,6 @@
+---
+name: startup-guide
+description: Explain the sample application's start command and port.
+---
+
+Read README.md. State the exact start command and port. Do not change files.

package/examples/templates/repository-skill/extension-data.ts

@@ -0,0 +1,64 @@
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export const description = {
+  extension: { id: "template.startup", version: "1.0.0" },
+  protocols: ["sevro.extension.v1"],
+  requiredCapabilities: ["sevro.codex.repository-invocation"],
+  optionalCapabilities: [],
+  graders: [],
+  taskVerdictPolicies: [],
+};
+
+export const resolved = {
+  cases: [
+    {
+      id: "repository-skill",
+      prompt:
+        "{{sevro.skill_invocation}} Explain the application's start command and port. Do not change files.",
+      fixture: {
+        kind: "generated",
+        commits: [
+          {
+            message: "Initialize application",
+            files: { "README.md": "Run bun run dev on port 3000.\n" },
+          },
+        ],
+      },
+      checks: [
+        {
+          id: "start-command",
+          grader: "sevro.regex",
+          configuration: { pattern: "bun run dev" },
+        },
+        {
+          id: "port",
+          grader: "sevro.regex",
+          configuration: { pattern: "\\b3000\\b" },
+        },
+      ],
+      requiredEvidence: ["sevro.codex.explicit-invocation"],
+      extensionData: {},
+    },
+  ],
+};
+
+export async function prepare() {
+  const relativePath = ".agents/skills/startup-guide/SKILL.md";
+  const body = await readFile(join(import.meta.dir, relativePath));
+  return {
+    artifacts: [
+      {
+        id: "template.startup-skill",
+        relativePath,
+        sha256: createHash("sha256").update(body).digest("hex"),
+        contentBase64: body.toString("base64"),
+        gitExclude: true,
+      },
+    ],
+    requestedInstrumentation: [],
+    extensionData: {},
+    codexRepositorySkillInvocation: { skillName: "startup-guide" },
+  };
+}

package/examples/templates/repository-skill/extension.ts

@@ -0,0 +1,31 @@
+import { description, prepare, resolved } from "./extension-data";
+
+function record(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw new Error("Expected a protocol request object");
+  return value as Record<string, unknown>;
+}
+
+const request = record(JSON.parse(await Bun.stdin.text()) as unknown);
+async function result() {
+  switch (request.method) {
+    case "describe":
+      return description;
+    case "resolve":
+      return resolved;
+    case "prepare":
+      return await prepare();
+    case "evaluate":
+      return { checks: [], metrics: [] };
+    default:
+      throw new Error("Unsupported extension method");
+  }
+}
+process.stdout.write(
+  JSON.stringify({
+    protocol: request.protocol,
+    id: request.id,
+    method: request.method,
+    result: await result(),
+  }),
+);

package/examples/templates/response/adapter.ts

@@ -0,0 +1,12 @@
+/** Deterministic starter; replace with an explicit native route to evaluate a model. */
+export default {
+  id: "template.response",
+  model: "deterministic-example",
+  effort: "none",
+  run() {
+    return Promise.resolve({
+      finalMessage: process.env.SEVRO_TEMPLATE_FAIL === "1" ? "wrong" : "ready",
+      complete: true,
+    });
+  },
+};

package/examples/templates/response/case.json

@@ -0,0 +1,13 @@
+{
+  "id": "response-check",
+  "prompt": "Return exactly ready.",
+  "fixture": { "files": { "README.md": "Response-check fixture.\n" } },
+  "checks": [
+    {
+      "id": "ready",
+      "grader": "sevro.regex",
+      "configuration": { "pattern": "^ready$" }
+    }
+  ],
+  "requiredEvidence": []
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@bjoernrochel/sevro",
-  "version": "0.1.0-rc.2",
-  "license": "BUSL-1.1",
+  "version": "0.1.0-rc.21",
+  "license": "SEE LICENSE IN LICENSE",
   "description": "A standalone evaluation runner for agentic coding capabilities",
   "type": "module",
   "private": false,
@@ -12,11 +12,13 @@
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public",
-    "tag": "next"
+    "tag": "latest"
   },
   "packageManager": "bun@1.3.13",
   "files": [
     "README.md",
+    "CONTRIBUTING.md",
+    "LICENSE",
     "docs/",
     "examples/",
     "schemas/",
@@ -26,12 +28,23 @@
     "sevro": "./src/cli.ts"
   },
   "scripts": {
+    "check:typescript": "bun scripts/check-typescript.ts",
+    "check:typescript:fast": "bun run check:typescript --fast",
+    "lint": "eslint . --max-warnings 0",
     "cli:dev": "bun src/cli.ts",
     "test": "bun test",
+    "schemas:generate": "bun scripts/generate-schemas.ts",
+    "schemas:check": "bun scripts/generate-schemas.ts --check",
     "test:package-install": "bun scripts/verify-package-install.ts",
+    "check:docs": "bun scripts/check-docs.ts",
+    "docs:sync": "bun scripts/sync-release-docs.ts",
+    "check:docs:external": "bun scripts/check-docs.ts --external",
+    "test:docs-examples": "bun scripts/verify-doc-examples.ts",
+    "guide:sync": "bun scripts/sync-guide.ts",
+    "eval:guide": "bun scripts/eval-guide.ts",
     "release:prepare": "bun scripts/prepare-release.ts",
     "typecheck": "tsc --noEmit",
-    "format:check": "prettier --check README.md docs/*.md examples/**/*.json examples/**/*.ts schemas/*.json src/**/*.ts scripts/**/*.ts tests/**/*.ts package.json tsconfig.json .github/workflows/*.yml"
+    "format:check": "prettier --check README.md CONTRIBUTING.md AGENTS.md CLAUDE.md 'docs/**/*.md' '.agents/**/*.md' '.agents/**/*.json' '.claude/**/*.md' 'scripts/**/*.md' 'scripts/**/*.json' examples/**/*.json examples/**/*.ts schemas/*.json src/**/*.ts scripts/**/*.ts scripts/**/*.cjs tests/**/*.ts eslint.config.mjs package.json tsconfig.json typescript-sources.json .github/workflows/*.yml"
   },
   "dependencies": {
     "ajv": "8.20.0",
@@ -39,7 +52,27 @@
     "typescript": "5.9.3"
   },
   "devDependencies": {
-    "@types/bun": "^1.2.0",
-    "prettier": "^3.9.9"
+    "@babel/core": "8.0.6",
+    "@babel/plugin-transform-typescript": "8.0.6",
+    "@eslint/js": "10.0.1",
+    "@types/bun": "1.3.13",
+    "@types/markdown-it": "14.1.2",
+    "@types/shell-quote": "1.7.5",
+    "@types/istanbul-lib-coverage": "2.0.6",
+    "@types/istanbul-lib-instrument": "1.7.8",
+    "@types/istanbul-lib-report": "3.0.3",
+    "@types/istanbul-reports": "3.0.4",
+    "eslint": "10.12.0",
+    "fast-check": "4.10.2",
+    "github-slugger": "2.0.0",
+    "istanbul-lib-coverage": "3.2.2",
+    "istanbul-lib-instrument": "6.0.3",
+    "istanbul-lib-report": "3.0.1",
+    "istanbul-lib-source-maps": "5.0.6",
+    "istanbul-reports": "3.2.0",
+    "markdown-it": "14.2.0",
+    "prettier": "^3.9.9",
+    "shell-quote": "1.12.0",
+    "typescript-eslint": "8.71.1"
   }
 }

package/schemas/report-v1.schema.json

@@ -110,6 +110,15 @@
         "domainOutcomes"
       ],
       "properties": {
+        "nativeTranscripts": {
+          "type": "object",
+          "required": ["retained", "pruned"],
+          "properties": {
+            "retained": { "type": "integer", "minimum": 0 },
+            "pruned": { "type": "integer", "minimum": 0 }
+          },
+          "additionalProperties": false
+        },
         "resultFile": { "$ref": "#/$defs/path" },
         "evidencePath": { "type": ["string", "null"] },
         "runId": { "type": ["string", "null"] },

package/schemas/run-evidence-v1.schema.json

@@ -80,7 +80,8 @@
       "required": ["digest", "redacted"],
       "properties": {
         "digest": { "$ref": "#/$defs/digest" },
-        "redacted": { "type": "object" }
+        "redacted": { "type": "object" },
+        "resolved": { "type": "object" }
       },
       "additionalProperties": false
     },

package/schemas/runtime-v1.schema.json

@@ -0,0 +1,193 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:sevro:schema:runtime:v1",
+  "title": "Sevro runtime policy v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["format"],
+  "properties": {
+    "codexState": {
+      "type": "boolean"
+    },
+    "network": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "localHttp": { "type": "boolean" }
+      }
+    },
+    "nativeTranscripts": {
+      "type": "boolean"
+    },
+    "hooks": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "nativeGoal": {
+          "type": "boolean"
+        },
+        "plugins": {
+          "type": "array",
+          "maxItems": 64,
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "pattern": "^[a-z][a-z0-9-]*$",
+            "maxLength": 128
+          }
+        }
+      }
+    },
+    "runtime": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "artifactDirectories": {
+          "type": "array",
+          "maxItems": 16,
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["name", "path"],
+            "properties": {
+              "name": {
+                "type": "string",
+                "pattern": "^[a-z][a-z0-9_]{0,63}$"
+              },
+              "path": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 4096
+              }
+            }
+          }
+        },
+        "sharedCacheDirectories": {
+          "type": "array",
+          "maxItems": 16,
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["source", "target", "roles"],
+            "properties": {
+              "source": { "type": "string", "minLength": 1, "maxLength": 4096 },
+              "target": {
+                "type": "string",
+                "pattern": "^[a-z][a-z0-9-]{0,63}$",
+                "not": { "enum": ["home", "tmp"] }
+              },
+              "roles": {
+                "type": "array",
+                "minItems": 1,
+                "maxItems": 5,
+                "uniqueItems": true,
+                "items": {
+                  "enum": [
+                    "candidate",
+                    "checks",
+                    "hooks",
+                    "semantic",
+                    "advisory"
+                  ]
+                }
+              }
+            }
+          }
+        },
+        "seedDirectories": {
+          "type": "array",
+          "maxItems": 16,
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["source", "target"],
+            "properties": {
+              "source": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 4096
+              },
+              "target": {
+                "type": "string",
+                "pattern": "^[a-z][a-z0-9-]{0,63}$",
+                "not": {
+                  "enum": ["home", "tmp"]
+                }
+              },
+              "roles": {
+                "type": "array",
+                "minItems": 1,
+                "maxItems": 5,
+                "uniqueItems": true,
+                "items": {
+                  "enum": [
+                    "candidate",
+                    "checks",
+                    "hooks",
+                    "semantic",
+                    "advisory"
+                  ]
+                }
+              }
+            }
+          }
+        }
+      }
+    },
+    "format": {
+      "const": "sevro.runtime.v1"
+    },
+    "filesystem": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "optionalReadOnlyRoots": {
+          "type": "array",
+          "maxItems": 64,
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 4096
+          }
+        },
+        "readOnlyRoots": {
+          "type": "array",
+          "maxItems": 64,
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 4096
+          }
+        }
+      }
+    },
+    "environment": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "inherit": {
+          "type": "array",
+          "maxItems": 64,
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "pattern": "^[A-Za-z_][A-Za-z0-9_]{0,127}$"
+          }
+        },
+        "set": {
+          "type": "object",
+          "maxProperties": 64,
+          "propertyNames": {
+            "pattern": "^[A-Za-z_][A-Za-z0-9_]{0,127}$"
+          },
+          "additionalProperties": {
+            "type": "string",
+            "maxLength": 8192
+          }
+        }
+      }
+    }
+  }
+}

package/src/advisory.ts

@@ -7,14 +7,14 @@ const DIMENSIONS = [
   "scopeDiscipline",
 ] as const;
 
-export interface AdvisoryAssessment {
+export type AdvisoryAssessment = {
   verdict: "pass" | "fail";
   overallScore: number;
   dimensions: Record<(typeof DIMENSIONS)[number], number>;
   strengths: string[];
   weaknesses: string[];
   summary: string;
-}
+};
 
 function object(value: unknown): Record<string, unknown> {
   if (!value || typeof value !== "object" || Array.isArray(value))
@@ -51,13 +51,18 @@ function textList(value: unknown): string[] {
   return value.map(text);
 }
 
-/** Parse a bounded, exact assessment from an independent review host. */
-export function parseAdvisoryAssessment(raw: string): AdvisoryAssessment {
+function advisoryJson(raw: string): Record<string, unknown> {
   if (Buffer.byteLength(raw, "utf8") > 64 * 1024)
     throw new Error("advisory assessment exceeds 64 KiB");
   const trimmed = raw.trim();
   const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/i.exec(trimmed);
-  const value = object(JSON.parse(fenced ? fenced[1]! : trimmed) as unknown);
+  const value: unknown = JSON.parse(fenced?.[1] ?? trimmed);
+  return object(value);
+}
+
+/** Parse a bounded, exact assessment from an independent review host. */
+export function parseAdvisoryAssessment(raw: string): AdvisoryAssessment {
+  const value = advisoryJson(raw);
   exactKeys(value, [
     "verdict",
     "overallScore",