@tryinget/pi-evalset-lab 0.2.0 → 0.3.0

package/CHANGELOG.md

@@ -13,6 +13,20 @@ system4d:
 
 All notable changes to this project should be documented here.
 
+## [0.3.0](https://github.com/tryingET/pi-extensions/compare/pi-evalset-lab-v0.2.0...pi-evalset-lab-v0.3.0) (2026-07-13)
+
+
+### Features
+
+* adopt engineering-core package surfaces ([c4a28c1](https://github.com/tryingET/pi-extensions/commit/c4a28c12c6077ba5b17909bcde3354bb1249e8d0))
+* **extensions:** harden runtime quality across packages ([1ff1eb0](https://github.com/tryingET/pi-extensions/commit/1ff1eb0cf10a3f8f60cf391ccf246b238951a848))
+* migrate pi extensions to pi 0.76 ([93dd0e0](https://github.com/tryingET/pi-extensions/commit/93dd0e0fdc9e23b0fc36661cc0e33972f365bf98))
+
+
+### Bug Fixes
+
+* align extension runtimes with Pi 0.80 ([1702c25](https://github.com/tryingET/pi-extensions/commit/1702c25f9d31bf4f619fdaa6f7a7f898ca5ee48e))
+
 ## [Unreleased]
 
 ### Changed
@@ -20,7 +34,7 @@ All notable changes to this project should be documented here.
 - Canonicalized the legacy standalone `pi-evalset-lab` package into the `pi-extensions` monorepo at `packages/pi-evalset-lab`.
 - Switched npm identity to scoped monorepo package name `@tryinget/pi-evalset-lab` while preserving legacy runtime version `0.2.0`.
 - Moved release automation to root-managed component metadata (`pi-evalset-lab`).
-- Archived the former standalone working copy to `~/programming/pi-extensions/pi-evalset-lab-final-archive.tar.gz` and removed it after validation; no legacy Pi session-history directory existed to relocate.
+- Removed the former standalone `~/programming/pi-extensions/` workspace after canonicalization; no legacy Pi session-history directory existed for `pi-evalset-lab` to relocate.
 
 ## [0.2.0](https://github.com/tryingET/pi-evalset-lab/compare/v0.1.0...v0.2.0) (2026-02-17)
 

package/README.md

@@ -17,8 +17,8 @@ Monorepo package for fixed-task-set eval workflows in Pi (`/evalset run|compare`
 - Workspace path: `packages/pi-evalset-lab`
 - Release component key: `pi-evalset-lab`
 - Former legacy standalone source: `~/programming/pi-extensions/pi-evalset-lab`
-- Canonical package status: canonicalized here; the legacy repo was archived to `~/programming/pi-extensions/pi-evalset-lab-final-archive.tar.gz` and removed after validation.
-- Session-history migration: no legacy Pi session-history directory existed for the old path, so relocation was recorded as `skip-no-history`.
+- Canonical package status: canonicalized here; the former `~/programming/pi-extensions/` workspace was later deleted at operator request.
+- Session-history migration: no legacy Pi session-history directory existed for the old package path, so relocation was recorded as `skip-no-history`.
 
 Primary category fit: **Model & Prompt Management**, **Review & Quality Loops**, **UX & Observability**, **Safety & Governance**.
 
@@ -26,8 +26,8 @@ Primary category fit: **Model & Prompt Management**, **Review & Quality Loops**,
 
 This package expects Pi host runtime APIs and declares them as `peerDependencies`:
 
-- `@mariozechner/pi-coding-agent`
-- `@mariozechner/pi-ai`
+- `@earendil-works/pi-coding-agent`
+- `@earendil-works/pi-ai`
 
 The npm package uses a `files` whitelist so required runtime artifacts are explicitly included:
 

package/extensions/evalset.ts

@@ -1,14 +1,15 @@
 import { createHash, randomUUID } from "node:crypto";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { link, mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { dirname, isAbsolute, resolve } from "node:path";
 import {
+  type Api,
   type AssistantMessage,
   type Context,
   complete,
   type Model,
   type Usage,
-} from "@mariozechner/pi-ai";
-import type { ExtensionAPI, ExtensionCommandContext } from "@mariozechner/pi-coding-agent";
+} from "@earendil-works/pi-ai/compat";
+import type { ExtensionAPI, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
 
 const COMMAND_NAME = "evalset";
 const CUSTOM_MESSAGE_TYPE = "evalset";
@@ -97,6 +98,7 @@ interface EvalCaseResult {
   pass: boolean;
   checks: CaseCheckResult[];
   failedChecks: string[];
+  output: string;
   outputPreview: string;
   latencyMs: number;
   stopReason: string;
@@ -296,11 +298,20 @@ function parseDataset(raw: string): EvalDataset {
     throw new Error("Dataset field 'systemPrompt' must be a string when provided.");
   }
 
-  return {
-    name,
-    systemPrompt,
-    cases: cases.map((entry, index) => parseCase(entry, index)),
-  };
+  const parsedCases = cases.map((entry, index) => parseCase(entry, index));
+  const ids = new Set<string>();
+  for (let index = 0; index < parsedCases.length; index += 1) {
+    const id = parsedCases[index].id;
+    if (id !== undefined && !id.trim()) {
+      throw new Error(`Case at index ${index}: 'id' must not be empty.`);
+    }
+    const normalized = id?.trim() || `case-${index + 1}`;
+    if (ids.has(normalized)) throw new Error(`Duplicate case id: ${normalized}`);
+    ids.add(normalized);
+    parsedCases[index].id = normalized;
+  }
+
+  return { name, systemPrompt, cases: parsedCases };
 }
 
 function toAbsolutePath(cwd: string, inputPath: string): string {
@@ -345,15 +356,17 @@ function requireValue(tokens: string[], index: number, flag: string): string {
 }
 
 function parsePositiveInteger(raw: string, field: string): number {
-  const parsed = Number.parseInt(raw, 10);
-  if (!Number.isFinite(parsed) || parsed <= 0) {
-    throw new Error(`${field} must be a positive integer.`);
-  }
+  if (!/^[1-9]\d*$/.test(raw)) throw new Error(`${field} must be a positive integer.`);
+  const parsed = Number(raw);
+  if (!Number.isSafeInteger(parsed)) throw new Error(`${field} must be a positive integer.`);
   return parsed;
 }
 
 function parseTemperature(raw: string): number {
-  const parsed = Number.parseFloat(raw);
+  if (!/^(?:\d+(?:\.\d*)?|\.\d+)$/.test(raw)) {
+    throw new Error("--temperature must be a number between 0 and 2.");
+  }
+  const parsed = Number(raw);
   if (!Number.isFinite(parsed) || parsed < 0 || parsed > 2) {
     throw new Error("--temperature must be a number between 0 and 2.");
   }
@@ -633,7 +646,7 @@ function defaultRunReportPath(cwd: string, datasetName: string, variantName: str
     cwd,
     ".evalset",
     "reports",
-    `run-${sanitizeSlug(datasetName)}-${sanitizeSlug(variantName)}-${timestampSlug()}.json`,
+    `run-${sanitizeSlug(datasetName)}-${sanitizeSlug(variantName)}-${timestampSlug()}-${randomUUID().slice(0, 8)}.json`,
   );
 }
 
@@ -642,19 +655,28 @@ function defaultCompareReportPath(cwd: string, datasetName: string): string {
     cwd,
     ".evalset",
     "reports",
-    `compare-${sanitizeSlug(datasetName)}-${timestampSlug()}.json`,
+    `compare-${sanitizeSlug(datasetName)}-${timestampSlug()}-${randomUUID().slice(0, 8)}.json`,
   );
 }
 
 async function writeReportFile(path: string, data: unknown): Promise<string> {
   await mkdir(dirname(path), { recursive: true });
-  await writeFile(path, `${JSON.stringify(data, null, 2)}\n`, "utf8");
+  const tempPath = `${path}.tmp-${process.pid}-${randomUUID()}`;
+  try {
+    await writeFile(tempPath, `${JSON.stringify(data, null, 2)}\n`, {
+      encoding: "utf8",
+      flag: "wx",
+    });
+    await link(tempPath, path);
+  } finally {
+    await rm(tempPath, { force: true }).catch(() => undefined);
+  }
   return path;
 }
 
 async function evaluateVariant(args: {
   ctx: ExtensionCommandContext;
-  model: Model;
+  model: Model<Api>;
   datasetPath: string;
   datasetName: string;
   datasetHash: string;
@@ -662,6 +684,7 @@ async function evaluateVariant(args: {
   cases: EvalCaseDefinition[];
   variant: VariantDefinition;
   apiKey?: string;
+  headers?: Record<string, string>;
   temperature?: number;
 }): Promise<EvalRunReport> {
   const {
@@ -674,6 +697,7 @@ async function evaluateVariant(args: {
     cases,
     variant,
     apiKey,
+    headers,
     temperature,
   } = args;
   const runId = randomUUID();
@@ -709,6 +733,7 @@ async function evaluateVariant(args: {
       try {
         const response = await complete(model, context, {
           apiKey,
+          headers,
           temperature,
         });
 
@@ -724,6 +749,7 @@ async function evaluateVariant(args: {
           failedChecks: evaluation.checks
             .filter((check) => !check.pass)
             .map((check) => check.details),
+          output: outputText,
           outputPreview: clip(outputText),
           latencyMs: Date.now() - startedAt,
           stopReason: response.stopReason,
@@ -744,6 +770,7 @@ async function evaluateVariant(args: {
             },
           ],
           failedChecks: [message],
+          output: "",
           outputPreview: "",
           latencyMs: Date.now() - startedAt,
           stopReason: "error",
@@ -826,13 +853,22 @@ function postMessage(pi: ExtensionAPI, message: string, details?: unknown): void
   });
 }
 
-function ensureActiveModel(ctx: ExtensionCommandContext): Model {
+function ensureActiveModel(ctx: ExtensionCommandContext): Model<Api> {
   if (!ctx.model) {
     throw new Error("No active model. Select one first via /model.");
   }
   return ctx.model;
 }
 
+async function resolveRequestAuth(
+  ctx: ExtensionCommandContext,
+  model: Model<Api>,
+): Promise<{ apiKey?: string; headers?: Record<string, string> }> {
+  const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
+  if (!auth.ok) throw new Error(auth.error);
+  return { apiKey: auth.apiKey, headers: auth.headers };
+}
+
 async function handleInit(
   pi: ExtensionAPI,
   ctx: ExtensionCommandContext,
@@ -897,7 +933,7 @@ async function handleRun(
 ): Promise<void> {
   const config = parseRunCommand(tokens);
   const model = ensureActiveModel(ctx);
-  const apiKey = await ctx.modelRegistry.getApiKey(model);
+  const requestAuth = await resolveRequestAuth(ctx, model);
 
   const loaded = await loadDataset(ctx.cwd, config.datasetPath);
   const datasetName = loaded.dataset.name?.trim() || sanitizeSlug(config.datasetPath);
@@ -935,7 +971,7 @@ async function handleRun(
     casesHash,
     cases: datasetCases,
     variant,
-    apiKey,
+    ...requestAuth,
     temperature: config.temperature,
   });
 
@@ -974,7 +1010,7 @@ async function handleCompare(
 ): Promise<void> {
   const config = parseCompareCommand(tokens);
   const model = ensureActiveModel(ctx);
-  const apiKey = await ctx.modelRegistry.getApiKey(model);
+  const requestAuth = await resolveRequestAuth(ctx, model);
 
   const loaded = await loadDataset(ctx.cwd, config.datasetPath);
   const datasetName = loaded.dataset.name?.trim() || sanitizeSlug(config.datasetPath);
@@ -1012,7 +1048,7 @@ async function handleCompare(
     casesHash,
     cases: datasetCases,
     variant: baselineVariant,
-    apiKey,
+    ...requestAuth,
     temperature: config.temperature,
   });
 
@@ -1025,7 +1061,7 @@ async function handleCompare(
     casesHash,
     cases: datasetCases,
     variant: candidateVariant,
-    apiKey,
+    ...requestAuth,
     temperature: config.temperature,
   });
 
@@ -1107,6 +1143,14 @@ function formatError(error: unknown): string {
   return String(error);
 }
 
+export const _test = {
+  parseDataset,
+  parsePositiveInteger,
+  parseTemperature,
+  parseRunCommand,
+  writeReportFile,
+};
+
 export default function (pi: ExtensionAPI): void {
   pi.registerCommand(COMMAND_NAME, {
     description: "Run fixed-task-set evals and compare prompt/system variants",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tryinget/pi-evalset-lab",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "pi extension for fixed-task-set eval runs and prompt/system comparisons",
   "type": "module",
   "license": "SEE LICENSE IN LICENSE",
@@ -31,7 +31,7 @@
     "access": "public"
   },
   "engines": {
-    "node": ">=22"
+    "node": ">=22.19.0"
   },
   "scripts": {
     "fix": "bash ./scripts/quality-gate.sh fix",
@@ -47,7 +47,13 @@
     "docs:list:json": "bash ./scripts/docs-list.sh --json",
     "release:check": "bash ./scripts/release-check.sh",
     "release:check:quick": "SKIP_PI_SMOKE=1 bash ./scripts/release-check.sh",
-    "evalset:export-html": "node ./scripts/export-evalset-report-html.mjs"
+    "evalset:export-html": "node ./scripts/export-evalset-report-html.mjs",
+    "loop-doctor": "bash -lc 'node --version; npm --version; npm pkg get name version >/dev/null; git status --short -- . || true; exit 0'",
+    "loop-verify-fast": "npm run quality:pre-commit",
+    "loop-impact-plan": "bash -lc 'echo \"loop-impact-plan: package-local impact planner is coarse; run npm run loop-impact-run for the full package gate.\"; git status --short -- . || true'",
+    "loop-impact-run": "npm run check",
+    "loop-impact-wide": "npm run check",
+    "loop-landing-check": "npm run check"
   },
   "files": [
     "extensions/evalset.ts",
@@ -56,7 +62,7 @@
     "scripts/export-evalset-report-html.mjs",
     "CHANGELOG.md",
     "policy/security-policy.json",
-    "policy/stack-lane.json"
+    "policy/engineering-lane.json"
   ],
   "pi": {
     "extensions": [
@@ -73,13 +79,17 @@
     "releaseConfigMode": "component"
   },
   "devDependencies": {
-    "@biomejs/biome": "2.3.14"
+    "@biomejs/biome": "2.3.14",
+    "@earendil-works/pi-ai": "^0.80.6",
+    "@earendil-works/pi-coding-agent": "^0.80.6",
+    "@types/node": "^25.9.4",
+    "typescript": "^6.0.3"
   },
   "overrides": {
     "fast-xml-parser": "5.3.6"
   },
   "peerDependencies": {
-    "@mariozechner/pi-coding-agent": "*",
-    "@mariozechner/pi-ai": "*"
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-coding-agent": "*"
   }
 }

package/policy/engineering-lane.json

@@ -0,0 +1,36 @@
+{
+  "lane": "ts",
+  "engineering_core": {
+    "tool": "engineering-core",
+    "lane": "pi-ts",
+    "repository": "https://github.com/lightningralf/engineering-core",
+    "ref": "workspace-local-unpinned",
+    "command": "uv tool -n run --from ~/ai-society/core/engineering-core engineering-core show pi-ts --prefer-repo",
+    "catalog_command": "uv tool -n run --from ~/ai-society/core/engineering-core engineering-core catalog --pretty",
+    "list_disciplines_command": "uv tool -n run --from ~/ai-society/core/engineering-core engineering-core list-disciplines",
+    "list_templates_command": "uv tool -n run --from ~/ai-society/core/engineering-core engineering-core list-templates",
+    "disciplines": [
+      "validation",
+      "testing",
+      "security-privacy",
+      "documentation",
+      "dependency-governance",
+      "local-first-data",
+      "observability",
+      "specification-and-dsls",
+      "engineering-reasoning"
+    ],
+    "loop_validation": {
+      "version": "repo-loop-validation-v1",
+      "contract_doc": "docs/engineering.local.md#repo-loop-validation",
+      "commands": {
+        "loop-doctor": "npm run loop-doctor",
+        "loop-verify-fast": "npm run loop-verify-fast",
+        "loop-impact-plan": "npm run loop-impact-plan",
+        "loop-impact-run": "npm run loop-impact-run",
+        "loop-impact-wide": "npm run loop-impact-wide",
+        "loop-landing-check": "npm run loop-landing-check"
+      }
+    }
+  }
+}

package/prompts/{implementation-planning.md → pi-evalset-lab-implementation-planning.md}

@@ -1,9 +1,9 @@
 ---
-summary: "Draft an implementation plan for a requested change"
+summary: "Draft an implementation plan for a requested pi-evalset-lab change"
 read_when:
   - "Using or maintaining this package prompt template."
   - "Checking prompt metadata for pi package discovery or docs-list validation."
-description: Draft an implementation plan for a requested change
+description: Draft an implementation plan for a requested pi-evalset-lab change
 system4d:
   container: "Prompt template for implementation planning."
   compass: "Turn requests into actionable, risk-aware plans."
@@ -11,7 +11,7 @@ system4d:
   fog: "Hidden constraints unless assumptions are surfaced."
 ---
 
-Create an implementation plan for this request: $@
+Create a pi-evalset-lab implementation plan for this request: $@
 
 Include:
 - Scope and non-goals

package/prompts/{security-review.md → pi-evalset-lab-security-review.md}

@@ -1,9 +1,9 @@
 ---
-summary: "Review a change for security risks and mitigations"
+summary: "Review a pi-evalset-lab change for security risks and mitigations"
 read_when:
   - "Using or maintaining this package prompt template."
   - "Checking prompt metadata for pi package discovery or docs-list validation."
-description: Review a change for security risks and mitigations
+description: Review a pi-evalset-lab change for security risks and mitigations
 system4d:
   container: "Prompt template for security-focused review."
   compass: "Identify practical vulnerabilities before release."
@@ -11,7 +11,7 @@ system4d:
   fog: "Partial context can hide exploit paths."
 ---
 
-Review this change for security concerns: $@
+Review this pi-evalset-lab change for security concerns: $@
 
 Focus on:
 - Input validation and injection risk

package/policy/stack-lane.json

@@ -1,10 +0,0 @@
-{
-  "lane": "ts",
-  "tech_stack_core": {
-    "tool": "tech-stack-core",
-    "lane": "pi-ts",
-    "repository": "https://github.com/lightningralf/tech-stack-core",
-    "ref": "4bd813dbb9ebbd9cb279c7293cd941fd1154d59e",
-    "command": "uv tool run --from ~/ai-society/core/tech-stack-core tech-stack-core show pi-ts --prefer-repo"
-  }
-}