brepjs-verify 0.26.0 → 0.28.0

package/CHANGELOG.md

@@ -1,5 +1,32 @@
 # Changelog
 
+## [0.28.0](https://github.com/andymai/brepjs/compare/brepjs-verify-v0.27.0...brepjs-verify-v0.28.0) (2026-06-19)
+
+
+### Features
+
+* **brepjs-verify:** fan out the eval across sharded CI runners ([#1503](https://github.com/andymai/brepjs/issues/1503)) ([8eb1f69](https://github.com/andymai/brepjs/commit/8eb1f690588e418e4b884e1294ba91b679667d61))
+
+
+### Bug Fixes
+
+* **brepjs-verify:** chamfer-fragility + unwrap/TS2322 SKILL.md rules ([#1506](https://github.com/andymai/brepjs/issues/1506)) ([0f24004](https://github.com/andymai/brepjs/commit/0f24004983effbd85545372b167f472eed35ec96))
+
+## [0.27.0](https://github.com/andymai/brepjs/compare/brepjs-verify-v0.26.0...brepjs-verify-v0.27.0) (2026-06-19)
+
+
+### Features
+
+* **brepjs-verify:** enrich experiment traces with request + code ([#1502](https://github.com/andymai/brepjs/issues/1502)) ([5b9141f](https://github.com/andymai/brepjs/commit/5b9141f8107794683d3ef7945466e326132d4a6d))
+* **brepjs-verify:** Langfuse observability foundation ([#1498](https://github.com/andymai/brepjs/issues/1498)) ([78c81f3](https://github.com/andymai/brepjs/commit/78c81f3b3e4890cbd3abc2474fcc5b7e4926bd3e))
+* **brepjs-verify:** looped skill eval + Langfuse v5 telemetry ([#1494](https://github.com/andymai/brepjs/issues/1494)) ([06a7e7c](https://github.com/andymai/brepjs/commit/06a7e7c6e882fe795122141dce1e4c04933f3fdc))
+
+
+### Bug Fixes
+
+* **brepjs-verify:** SKILL.md import rule — eval-driven self-heal ([#1501](https://github.com/andymai/brepjs/issues/1501)) ([550d2ba](https://github.com/andymai/brepjs/commit/550d2ba08849a1692ff237d1a79a28afd88b9053))
+* **brepjs-verify:** three deferred Langfuse-eval follow-ups ([#1500](https://github.com/andymai/brepjs/issues/1500)) ([90b63bd](https://github.com/andymai/brepjs/commit/90b63bd6a2f277db6ec634f2bbecadf1379f4e4d))
+
 ## [0.26.0](https://github.com/andymai/brepjs/compare/brepjs-verify-v0.25.1...brepjs-verify-v0.26.0) (2026-06-18)
 
 

package/README.md

@@ -110,9 +110,13 @@ Few-shot examples live under `skill/examples/<name>.brep.ts`, each with a `<name
 
 `npm run eval` (`bench/run.ts`) replays every `skill/examples/*.brep.ts` with a sibling `*.expected.json` through the public `runPart` runtime, compares measured volume/area/validity/shape-type against the recorded baseline within each file's tolerance (default 0.5%), prints a PASS/FAIL scorecard, and exits non-zero on any regression. It is deterministic (no LLM or API key) so it runs in CI as the package's regression net. Refresh a baseline by re-recording the example's `*.expected.json` after an intentional geometry change.
 
+### Manual skill eval (`/eval-skill`)
+
+The fast iteration loop — run on your Claude subscription, no API key, no billing. The `/eval-skill` slash command drives the same `bench/prompts.ts` corpus through the current Claude Code session: Claude authors each part from the **deployed `SKILL.md`**, runs `verify --check --snapshot`, judges the rendered snapshots against each prompt's rubric, emits the two-signal scorecard, and proposes SKILL.md fixes from the failures. This is the recommended way to answer "did my SKILL.md edit help?" — see `.claude/commands/eval-skill.md`.
+
 ### Live eval (`npm run eval:live`)
 
-The measurement flywheel: sends ~18 natural-language part prompts (`bench/prompts.ts`) to a real model (using the **deployed `SKILL.md` as the system prompt**, so it measures the actual skill), then verifies each generated part two ways:
+The **automated / isolated** counterpart to `/eval-skill` — opt-in and billed. An SDK harness sends the `bench/prompts.ts` prompts to a model with the **deployed `SKILL.md` as the system prompt** (so it measures the skill _in isolation_, not inside a Claude Code session), then verifies each generated part two ways:
 
 - **Auto (objective):** `runPart --check` → valid solid + any pinned dims within tolerance.
 - **Judge (intent):** a multimodal Claude call looks at the rendered iso/front/top/right snapshots and decides whether the part matches the request + rubric.

package/dist/mcp/server.cjs

@@ -297,6 +297,70 @@ async function appendRunRecord(filePath, record) {
 	await (0, node_fs_promises.appendFile)(filePath, `${JSON.stringify(record)}\n`);
 }
 //#endregion
+//#region src/mcp/telemetry.ts
+/**
+* Optional Langfuse tracing for MCP `run_program` executions — a strictly best-effort side channel,
+* exactly like the `BREPJS_RUN_RECORD_PATH` JSONL record. The @langfuse / OpenTelemetry packages are
+* NOT runtime dependencies of the shipped MCP: they are *dynamically* imported, and only when the
+* `LANGFUSE_*` keys are present. So the server never hard-requires telemetry — no keys (or the deps
+* simply absent in a published install) → {@link traceRun} is a silent no-op. It never throws.
+*/
+var tracerPromise;
+async function initTracer() {
+	if (!process.env["LANGFUSE_PUBLIC_KEY"] || !process.env["LANGFUSE_SECRET_KEY"]) return null;
+	try {
+		const [{ NodeSDK }, otel, { startActiveObservation }, { LangfuseClient }] = await Promise.all([
+			import("@opentelemetry/sdk-node"),
+			import("@langfuse/otel"),
+			import("@langfuse/tracing"),
+			import("@langfuse/client")
+		]);
+		const processor = new otel.LangfuseSpanProcessor();
+		new NodeSDK({ spanProcessors: [processor] }).start();
+		const client = new LangfuseClient();
+		return { trace: async (code, result) => {
+			const rec = buildRunRecord(code, result);
+			startActiveObservation("run_program", (obs) => {
+				obs.update({
+					input: { code },
+					output: {
+						outcome: rec.outcome,
+						ok: rec.ok,
+						measurements: rec.measurements
+					},
+					metadata: {
+						source: "mcp",
+						resultHash: rec.resultHash
+					}
+				});
+				try {
+					client.score.create({
+						traceId: obs.traceId,
+						name: "ok",
+						value: rec.ok ? 1 : 0,
+						dataType: "NUMERIC"
+					});
+				} catch {}
+			});
+			await Promise.allSettled([processor.forceFlush(), client.flush()]);
+		} };
+	} catch (e) {
+		console.warn(`langfuse: MCP tracing unavailable (${e.message.split("\n")[0]})`);
+		return null;
+	}
+}
+/**
+* Trace one `run_program` execution to Langfuse (best-effort; no-op without the `LANGFUSE_*` keys or
+* the optional telemetry deps). Never throws — callers fire-and-forget it as a side channel.
+*/
+async function traceRun(code, result) {
+	try {
+		await (await (tracerPromise ??= initTracer()))?.trace(code, result);
+	} catch (e) {
+		console.warn(`langfuse: traceRun failed (${e.message.split("\n")[0]})`);
+	}
+}
+//#endregion
 //#region src/mcp/tools.ts
 /** JSON Schema for `run_program` input (the low-level Server API takes plain JSON Schema). */
 var RUN_PROGRAM_INPUT_SCHEMA = {
@@ -330,6 +394,7 @@ async function runProgramTool(args) {
 	if (recordPath) appendRunRecord(recordPath, buildRunRecord(args.code, result)).catch((err) => {
 		console.warn(`run-record append failed (BREPJS_RUN_RECORD_PATH=${recordPath}):`, err);
 	});
+	traceRun(args.code, result);
 	if (result.outcome === "completed") {
 		const payload = {
 			outcome: "completed",

package/dist/mcp/server.js

@@ -297,6 +297,70 @@ async function appendRunRecord(filePath, record) {
 	await appendFile(filePath, `${JSON.stringify(record)}\n`);
 }
 //#endregion
+//#region src/mcp/telemetry.ts
+/**
+* Optional Langfuse tracing for MCP `run_program` executions — a strictly best-effort side channel,
+* exactly like the `BREPJS_RUN_RECORD_PATH` JSONL record. The @langfuse / OpenTelemetry packages are
+* NOT runtime dependencies of the shipped MCP: they are *dynamically* imported, and only when the
+* `LANGFUSE_*` keys are present. So the server never hard-requires telemetry — no keys (or the deps
+* simply absent in a published install) → {@link traceRun} is a silent no-op. It never throws.
+*/
+var tracerPromise;
+async function initTracer() {
+	if (!process.env["LANGFUSE_PUBLIC_KEY"] || !process.env["LANGFUSE_SECRET_KEY"]) return null;
+	try {
+		const [{ NodeSDK }, otel, { startActiveObservation }, { LangfuseClient }] = await Promise.all([
+			import("@opentelemetry/sdk-node"),
+			import("@langfuse/otel"),
+			import("@langfuse/tracing"),
+			import("@langfuse/client")
+		]);
+		const processor = new otel.LangfuseSpanProcessor();
+		new NodeSDK({ spanProcessors: [processor] }).start();
+		const client = new LangfuseClient();
+		return { trace: async (code, result) => {
+			const rec = buildRunRecord(code, result);
+			startActiveObservation("run_program", (obs) => {
+				obs.update({
+					input: { code },
+					output: {
+						outcome: rec.outcome,
+						ok: rec.ok,
+						measurements: rec.measurements
+					},
+					metadata: {
+						source: "mcp",
+						resultHash: rec.resultHash
+					}
+				});
+				try {
+					client.score.create({
+						traceId: obs.traceId,
+						name: "ok",
+						value: rec.ok ? 1 : 0,
+						dataType: "NUMERIC"
+					});
+				} catch {}
+			});
+			await Promise.allSettled([processor.forceFlush(), client.flush()]);
+		} };
+	} catch (e) {
+		console.warn(`langfuse: MCP tracing unavailable (${e.message.split("\n")[0]})`);
+		return null;
+	}
+}
+/**
+* Trace one `run_program` execution to Langfuse (best-effort; no-op without the `LANGFUSE_*` keys or
+* the optional telemetry deps). Never throws — callers fire-and-forget it as a side channel.
+*/
+async function traceRun(code, result) {
+	try {
+		await (await (tracerPromise ??= initTracer()))?.trace(code, result);
+	} catch (e) {
+		console.warn(`langfuse: traceRun failed (${e.message.split("\n")[0]})`);
+	}
+}
+//#endregion
 //#region src/mcp/tools.ts
 /** JSON Schema for `run_program` input (the low-level Server API takes plain JSON Schema). */
 var RUN_PROGRAM_INPUT_SCHEMA = {
@@ -330,6 +394,7 @@ async function runProgramTool(args) {
 	if (recordPath) appendRunRecord(recordPath, buildRunRecord(args.code, result)).catch((err) => {
 		console.warn(`run-record append failed (BREPJS_RUN_RECORD_PATH=${recordPath}):`, err);
 	});
+	traceRun(args.code, result);
 	if (result.outcome === "completed") {
 		const payload = {
 			outcome: "completed",

package/dist/mcp/telemetry.d.ts

@@ -0,0 +1,6 @@
+import { RunProgramResult } from '../sandbox/runProgram.js';
+/**
+ * Trace one `run_program` execution to Langfuse (best-effort; no-op without the `LANGFUSE_*` keys or
+ * the optional telemetry deps). Never throws — callers fire-and-forget it as a side channel.
+ */
+export declare function traceRun(code: string, result: RunProgramResult): Promise<void>;

package/dist/sandbox/runProgram.d.ts

@@ -69,6 +69,27 @@ export declare function killActiveSandboxes(signal?: NodeJS.Signals): void;
 export declare function installSandboxShutdownHandlers(): void;
 /** Execute `code` (an agent-authored `.brep.ts` module) in an isolated, resource-bounded child. */
 export declare function runProgram(code: string, opts?: RunProgramOptions): Promise<RunProgramResult>;
+/** Outcome of a sandboxed `--check --step` run. `completed` carries the report; `stepPath` is set
+ * only when a valid solid produced a STEP. */
+export type RunProgramWithStepResult = {
+    outcome: 'completed';
+    report: SerializedReport;
+    stepPath?: string;
+} | {
+    outcome: 'timeout';
+    timeoutMs: number;
+} | {
+    outcome: 'crashed';
+    exitCode: number | null;
+    detail: string;
+};
+/**
+ * Execute `code` with `verify --check --step` in ONE bounded child process: a single spawn yields
+ * both the `auto`-signal report (parsed from stdout) and the STEP the judge renders — one kernel
+ * boot, not the two a `runProgram` + `exportProgram` pair would cost. The caller owns `stepOutPath`;
+ * only the temp program dir is cleaned up.
+ */
+export declare function runProgramWithStep(code: string, stepOutPath: string, opts?: RunProgramOptions): Promise<RunProgramWithStepResult>;
 /**
  * Execute `code` and export artifacts to `outDir`. Artifacts persist in `outDir` (the caller owns
  * it); only the temp program directory is cleaned up. Returns the written paths and any errors.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "brepjs-verify",
-  "version": "0.26.0",
+  "version": "0.28.0",
   "description": "Agent skill + verify/preview tooling for authoring parametric brepjs CAD code",
   "license": "Apache-2.0",
   "type": "module",
@@ -47,6 +47,9 @@
     "test": "vitest run",
     "eval": "tsx bench/run.ts",
     "eval:live": "tsx bench/live.ts",
+    "eval:push": "tsx bench/pushScorecard.ts",
+    "eval:combine": "tsx bench/combineShards.ts",
+    "eval:dataset:sync": "tsx bench/syncDataset.ts",
     "smoke": "node dist/cli/main.js verify tests/fixtures/validBox.brep.ts | node -e \"const r=JSON.parse(require('fs').readFileSync(0,'utf8'));if(r.ok!==true||!(r.measurements.volume>0))process.exit(1)\"",
     "smoke:standalone": "node scripts/smokeStandalone.mjs",
     "prepack": "npm run build"
@@ -63,6 +66,10 @@
   },
   "devDependencies": {
     "@anthropic-ai/sdk": "0.102.0",
+    "@langfuse/client": "5.5.3",
+    "@langfuse/otel": "5.5.3",
+    "@langfuse/tracing": "5.5.3",
+    "@opentelemetry/sdk-node": "0.219.0",
     "@react-three/drei": "^10.7.7",
     "@react-three/fiber": "^9.6.1",
     "@types/node": "25.9.2",

package/reference/llms-full.txt

@@ -736,10 +736,10 @@ import {
   measureDistance
 } from 'brepjs';
 
-measureVolume(solid);                           // number
-measureArea(face);                              // number
-measureLength(edge);                            // number
-measureDistance(shape1, shape2);                 // number
+measureVolume(solid);                           // Result<number> — unwrap()
+measureArea(face);                              // Result<number> — unwrap()
+measureLength(edge);                            // Result<number> — unwrap()
+measureDistance(shape1, shape2);                // Result<number> — unwrap()
 ```
 
 Functional measurement API: