openmerit 0.1.3 → 0.1.5

package/docs/testing.md

@@ -0,0 +1,36 @@
+# Testing
+
+## Standard verification
+
+```sh
+npm install
+npm test
+npm run check
+npm run verify
+npm pack --dry-run
+```
+
+Tests cover protocol contracts, lifecycle decisions, baseline sufficiency, candidate/frontier/swap sequencing, conservative Pareto verification, persistence, audit events, safe evidence identifiers, Pi commands, private intent delivery, evidence requirements, and mutation boundaries.
+
+Automation tests additionally cover task and elapsed cadence, idempotent signal delivery, durable pending work, catalogue changes, metric regression thresholds, native wakeup requests, unsupported-scheduler fallbacks, delayed post-swap verification, and Pi conversion of settled work into a due assessment.
+
+## Pi load smoke test
+
+```sh
+./node_modules/.bin/pi --no-session -e ./packages/pi/src/index.ts --list-models
+```
+
+This proves the real Pi runtime loads the extension and enumerates its available catalogue. It does not prove a provider can complete an end-to-end product intent.
+
+CI also packs `openmerit`, installs the tarball in a temporary directory,
+imports its public core and protocol entry points, and loads the installed Pi
+extension. This checks the published artifact rather than only the workspace.
+
+## Evidence levels
+
+- **Unit verified** — deterministic behavior has automated coverage.
+- **Runtime smoke verified** — the actual Pi runtime loads the extension.
+- **Product trial verified** — Pi creates evals and observability in a real product and completes an intent with evidence.
+- **Production verified** — representative repeated runs support reliability and distribution claims.
+
+The current repository is unit- and runtime-smoke-verified. Product and production claims require target-product evidence.

package/docs/ux-reference.md

@@ -0,0 +1,32 @@
+# Pi UX behavioral reference
+
+This document records user-visible behaviors from the earlier OpenMerit implementation. It is not an implementation specification; the current implementation carries forward only the behaviors listed below, rather than its source code, storage schema, or internal contracts.
+
+## Behaviors worth preserving
+
+- A single `/openmerit` command is the primary in-harness entry point.
+- Running the command without arguments gives a concise operational summary.
+- Long-running work produces short progress notifications instead of appearing silent.
+- Users can pause and resume proactive work without uninstalling the extension.
+- Users can request work explicitly even when proactive behavior is paused.
+- Diagnostics are available from the same command surface.
+- Skipped work explains why it was skipped and how the user can proceed.
+- Failures are visible, actionable, and do not masquerade as successful evidence.
+- The extension behaves sensibly in TUI, RPC, JSON, and print modes; prompts are not attempted when no interactive UI exists.
+
+## Staged behavior
+
+The command shell can carry a proposal through an authorized model change, but only after baseline evidence, budgeted challenger trials, a harness-calculated and OpenMerit-verified frontier, policy approval, and post-swap verification.
+
+## Current direction
+
+The command surface centers on evidence readiness. Its status view should answer:
+
+1. What task profile is active?
+2. Which required metrics are observable?
+3. Which evaluations exist and have been verified?
+4. What intelligent work is Pi currently performing for OpenMerit?
+5. What evidence is still missing or insufficient?
+6. When is the next assessment due?
+
+The approved command surface is documented in [Pi extension](pi-extension.md).

package/package.json

@@ -1,35 +1,32 @@
 {
   "name": "openmerit",
-  "version": "0.1.3",
-  "description": "Find better models for each pi task by comparing quality, cost, and latency.",
+  "version": "0.1.5",
+  "description": "Evidence-driven model decisions for coding harnesses.",
   "type": "module",
-  "keywords": [
-    "pi-package",
-    "pi-extension",
-    "pi-coding-agent",
-    "model-routing",
-    "llm",
-    "openrouter",
-    "evaluation"
-  ],
-  "files": [
-    "dist/",
-    "extension/openmerit.ts",
-    "instructions/",
-    "examples/",
-    "rules.md",
-    "benchmark/invoice_ocr/data/"
-  ],
-  "pi": {
-    "extensions": [
-      "./extension/openmerit.ts"
-    ]
-  },
-  "bin": {
-    "openmerit": "dist/cli.js"
+  "license": "MIT",
+  "keywords": ["pi-package", "pi-extension", "model-routing", "evaluation", "pareto-frontier"],
+  "files": ["dist/", "docs/", "CHANGELOG.md", "LICENSE", "README.md"],
+  "exports": {
+    ".": {
+      "types": "./dist/core/src/index.d.ts",
+      "default": "./dist/core/src/index.js"
+    },
+    "./core": {
+      "types": "./dist/core/src/index.d.ts",
+      "default": "./dist/core/src/index.js"
+    },
+    "./protocol": {
+      "types": "./dist/protocol/src/index.d.ts",
+      "default": "./dist/protocol/src/index.js"
+    },
+    "./pi": {
+      "types": "./dist/pi/src/index.d.ts",
+      "default": "./dist/pi/src/index.js"
+    },
+    "./package.json": "./package.json"
   },
-  "engines": {
-    "node": ">=22.18.0"
+  "pi": {
+    "extensions": ["./dist/pi/src/index.js"]
   },
   "repository": {
     "type": "git",
@@ -42,39 +39,33 @@
   "publishConfig": {
     "access": "public"
   },
+  "workspaces": [
+    "packages/*"
+  ],
+  "engines": {
+    "node": ">=22.19.0"
+  },
   "scripts": {
+    "check": "npm run check --workspaces --if-present",
+    "test": "npm run test --workspaces --if-present",
+    "typecheck": "tsc -p tsconfig.json",
+    "build": "node scripts/build.mjs",
     "prepare": "npm run build",
-    "build": "tsc -p tsconfig.json",
-    "typecheck": "tsc --noEmit && tsc -p benchmark/tsconfig.json",
-    "typecheck:extension": "tsc --noEmit --strict --target ES2022 --module NodeNext --moduleResolution NodeNext --skipLibCheck extension/openmerit.ts",
-    "test": "vitest run",
-    "check": "npm run typecheck && npm run typecheck:extension && npm test && npm run build && npm run benchmark:verify-recorded",
-    "prepublishOnly": "npm run check",
-    "benchmark:invoice:prepare": "node benchmark/invoice_ocr/prepare_data.ts",
-    "benchmark:invoice": "node benchmark/invoice_ocr/benchmark.ts",
-    "benchmark:invoice:audit": "node benchmark/invoice_ocr/audit_results.ts",
-    "benchmark:invoice:round2": "node benchmark/invoice_ocr/benchmark_round2.ts",
-    "benchmark:invoice:round2:audit": "node benchmark/invoice_ocr/audit_round2.ts",
-    "benchmark:text-to-sql": "node benchmark/text_to_sql/benchmark.ts",
-    "benchmark:text-to-sql:audit": "node benchmark/text_to_sql/audit_results.ts",
-    "benchmark:policy": "node benchmark/policy_adjudication/benchmark.ts",
-    "benchmark:policy:audit": "node benchmark/policy_adjudication/audit_results.ts",
-    "benchmark:policy:validate": "node benchmark/policy_adjudication/validate_eval.ts",
-    "benchmark:verify-recorded": "node benchmark/verify_recorded.ts"
+    "verify": "npm test && npm run check && npm run typecheck && npm run build && git diff --check",
+    "prepublishOnly": "npm run verify"
+  },
+  "dependencies": {
+    "@earendil-works/pi-ai": "0.87.0",
+    "typebox": "1.3.27"
   },
-  "license": "MIT",
   "peerDependencies": {
-    "@earendil-works/pi-coding-agent": "*"
+    "@earendil-works/pi-coding-agent": ">=0.87.0 <0.88.0"
   },
   "peerDependenciesMeta": {
-    "@earendil-works/pi-coding-agent": {
-      "optional": true
-    }
+    "@earendil-works/pi-coding-agent": { "optional": true }
   },
   "devDependencies": {
-    "@earendil-works/pi-coding-agent": "^0.85.1",
-    "@types/node": "^22.10.0",
-    "typescript": "^5.7.0",
-    "vitest": "^2.1.0"
+    "@types/node": "^22.18.6",
+    "typescript": "^5.9.3"
   }
 }

package/benchmark/invoice_ocr/data/invoice_01_ground_truth.json

@@ -1,38 +0,0 @@
-{
-  "supplier_name": "PixelWave Solutions Inc.",
-  "customer_name": "Greenfield Marketing Ltd.",
-  "invoice_number": "INV-2023-0098",
-  "po_number": "PO-5678",
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-14",
-  "document_currency": "CAD",
-  "document_total_net": 2220.0,
-  "document_total_tax": 111.0,
-  "document_total_amount": 2331.0,
-  "line_items": [
-    {
-      "description": "Website design services",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "Content writing package",
-      "quantity": 10.0,
-      "unit_price": 50.0,
-      "amount": 500.0
-    },
-    {
-      "description": "SEO optimization",
-      "quantity": 5.0,
-      "unit_price": 80.0,
-      "amount": 400.0
-    },
-    {
-      "description": "Hosting subscription",
-      "quantity": 12.0,
-      "unit_price": 10.0,
-      "amount": 120.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_02_ground_truth.json

@@ -1,32 +0,0 @@
-{
-  "supplier_name": "TechSolutions Ltd",
-  "customer_name": null,
-  "invoice_number": "IE-INV-2024-000123",
-  "po_number": null,
-  "document_date": "2024-03-15",
-  "due_date": null,
-  "document_currency": "EUR",
-  "document_total_net": 6200.0,
-  "document_total_tax": 1143.0,
-  "document_total_amount": 7343.0,
-  "line_items": [
-    {
-      "description": "Consulting services for project X",
-      "quantity": 3.0,
-      "unit_price": 1500.0,
-      "amount": 4500.0
-    },
-    {
-      "description": "High-performance laptop (Model Y)",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "International shipping to Germany",
-      "quantity": 1.0,
-      "unit_price": 500.0,
-      "amount": 500.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_03_ground_truth.json

@@ -1,26 +0,0 @@
-{
-  "supplier_name": "CloudCompute Solutions Inc.",
-  "customer_name": "Acme Corp.",
-  "invoice_number": "INV-20230815-001",
-  "po_number": "PO-987654",
-  "document_date": "2023-08-15",
-  "due_date": "2023-09-14",
-  "document_currency": "USD",
-  "document_total_net": 110.0,
-  "document_total_tax": 9.07,
-  "document_total_amount": 119.07,
-  "line_items": [
-    {
-      "description": "Enterprise Cloud Storage Subscription",
-      "quantity": 1.0,
-      "unit_price": 50.0,
-      "amount": 50.0
-    },
-    {
-      "description": "Premium Support Add-On",
-      "quantity": 2.0,
-      "unit_price": 30.0,
-      "amount": 60.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_04_ground_truth.json

@@ -1,26 +0,0 @@
-{
-  "supplier_name": "Tech Solutions Ltd",
-  "customer_name": "Acme Construction Ltd",
-  "invoice_number": "INV-20230815-001",
-  "po_number": "PO-7654321",
-  "document_date": "2023-08-15",
-  "due_date": "2023-09-14",
-  "document_currency": "GBP",
-  "document_total_net": 1500.0,
-  "document_total_tax": 240.0,
-  "document_total_amount": 1740.0,
-  "line_items": [
-    {
-      "description": "High-performance laptop",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "Office desk chair",
-      "quantity": 2.0,
-      "unit_price": 150.0,
-      "amount": 300.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_05_ground_truth.json

@@ -1,38 +0,0 @@
-{
-  "supplier_name": "Celtic Code Solutions",
-  "customer_name": "North Star Consulting Ltd.",
-  "invoice_number": "INV-2023-0075",
-  "po_number": null,
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-14",
-  "document_currency": "EUR",
-  "document_total_net": 4600.0,
-  "document_total_tax": 1058.0,
-  "document_total_amount": 5658.0,
-  "line_items": [
-    {
-      "description": "UI Development Consulting",
-      "quantity": 20.0,
-      "unit_price": 80.0,
-      "amount": 1600.0
-    },
-    {
-      "description": "UX Research",
-      "quantity": 10.0,
-      "unit_price": 90.0,
-      "amount": 900.0
-    },
-    {
-      "description": "Custom API Integration",
-      "quantity": 15.0,
-      "unit_price": 100.0,
-      "amount": 1500.0
-    },
-    {
-      "description": "Project Management Support",
-      "quantity": 5.0,
-      "unit_price": 120.0,
-      "amount": 600.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_06_ground_truth.json

@@ -1,38 +0,0 @@
-{
-  "supplier_name": "Oak Property Management Ltd",
-  "customer_name": "John Doe",
-  "invoice_number": "RENT-2023-001",
-  "po_number": null,
-  "document_date": "2023-05-01",
-  "due_date": "2023-05-15",
-  "document_currency": "EUR",
-  "document_total_net": 1650.0,
-  "document_total_tax": 103.5,
-  "document_total_amount": 1753.5,
-  "line_items": [
-    {
-      "description": "Monthly Base Rent",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "Utilities (Electricity/Gas)",
-      "quantity": 1.0,
-      "unit_price": 200.0,
-      "amount": 200.0
-    },
-    {
-      "description": "Property Tax",
-      "quantity": 1.0,
-      "unit_price": 150.0,
-      "amount": 150.0
-    },
-    {
-      "description": "Cleaning Service",
-      "quantity": 1.0,
-      "unit_price": 100.0,
-      "amount": 100.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_07_ground_truth.json

@@ -1,20 +0,0 @@
-{
-  "supplier_name": "Acme Office Supplies Inc.",
-  "customer_name": "Global Tech Corp.",
-  "invoice_number": "INV-2023-0001",
-  "po_number": null,
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-15",
-  "document_currency": "USD",
-  "document_total_net": 600.0,
-  "document_total_tax": 42.0,
-  "document_total_amount": 642.0,
-  "line_items": [
-    {
-      "description": "Office chairs",
-      "quantity": 5.0,
-      "unit_price": 120.0,
-      "amount": 600.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_08_ground_truth.json

@@ -1,38 +0,0 @@
-{
-  "supplier_name": "Digital Solutions Pty Ltd",
-  "customer_name": "Global Tech Industries Ltd",
-  "invoice_number": "INV-2024-005",
-  "po_number": null,
-  "document_date": "2024-06-15",
-  "due_date": "2024-07-15",
-  "document_currency": "AUD",
-  "document_total_net": 3840.0,
-  "document_total_tax": 384.0,
-  "document_total_amount": 4224.0,
-  "line_items": [
-    {
-      "description": "Cloud hosting services",
-      "quantity": 12.0,
-      "unit_price": 120.0,
-      "amount": 1440.0
-    },
-    {
-      "description": "Software license renewal",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "Subscription data analytics package",
-      "quantity": 1.0,
-      "unit_price": 800.0,
-      "amount": 800.0
-    },
-    {
-      "description": "Technical support",
-      "quantity": 1.0,
-      "unit_price": 400.0,
-      "amount": 400.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_09_ground_truth.json

@@ -1,26 +0,0 @@
-{
-  "supplier_name": "Acme Consulting Group",
-  "customer_name": null,
-  "invoice_number": "INV-2023-0789",
-  "po_number": "PO-2023-4567",
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-14",
-  "document_currency": "USD",
-  "document_total_net": 3250.0,
-  "document_total_tax": 268.12,
-  "document_total_amount": 3518.12,
-  "line_items": [
-    {
-      "description": "Business Strategy Consultation",
-      "quantity": 10.0,
-      "unit_price": 200.0,
-      "amount": 2000.0
-    },
-    {
-      "description": "Technology Roadmap Development",
-      "quantity": 5.0,
-      "unit_price": 250.0,
-      "amount": 1250.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_10_ground_truth.json

@@ -1,20 +0,0 @@
-{
-  "supplier_name": "VirtualCo Solutions",
-  "customer_name": "John Doe",
-  "invoice_number": "INV-2023-000123",
-  "po_number": null,
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-14",
-  "document_currency": "USD",
-  "document_total_net": 1200.0,
-  "document_total_tax": 96.0,
-  "document_total_amount": 1296.0,
-  "line_items": [
-    {
-      "description": "Enterprise SaaS Subscription for Q3 2023",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/manifest.json

@@ -1,97 +0,0 @@
-{
-  "dataset": "alamgirqazi/invoice-ocr-synthetic",
-  "dataset_revision": "44ddfd3c0c373b69f377910f83c4164a1e52d52c",
-  "license": "CC BY 4.0",
-  "rows": [
-    {
-      "invoice_id": "invoice_01",
-      "row_idx": 2,
-      "file_name": "synth_0002_8f01361b.png",
-      "quality": "clean",
-      "template": "freelancer_invoice",
-      "image": "invoice_01_row_2.jpg",
-      "ground_truth": "invoice_01_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_02",
-      "row_idx": 5,
-      "file_name": "synth_0005_553deef3.png",
-      "quality": "clean",
-      "template": "formal_corporate",
-      "image": "invoice_02_row_5.jpg",
-      "ground_truth": "invoice_02_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_03",
-      "row_idx": 6,
-      "file_name": "synth_0006_474081ca.png",
-      "quality": "clean",
-      "template": "modern_minimal",
-      "image": "invoice_03_row_6.jpg",
-      "ground_truth": "invoice_03_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_04",
-      "row_idx": 7,
-      "file_name": "synth_0007_7292e354.png",
-      "quality": "clean",
-      "template": "utility_bill",
-      "image": "invoice_04_row_7.jpg",
-      "ground_truth": "invoice_04_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_05",
-      "row_idx": 947,
-      "file_name": "synth_0009_8f6b430f.png",
-      "quality": "faded",
-      "template": "freelancer_invoice",
-      "image": "invoice_05_row_947.jpg",
-      "ground_truth": "invoice_05_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_06",
-      "row_idx": 948,
-      "file_name": "synth_0010_73bd8d75.png",
-      "quality": "faded",
-      "template": "formal_corporate",
-      "image": "invoice_06_row_948.jpg",
-      "ground_truth": "invoice_06_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_07",
-      "row_idx": 949,
-      "file_name": "synth_0011_8888d5c6.png",
-      "quality": "faded",
-      "template": "utility_bill",
-      "image": "invoice_07_row_949.jpg",
-      "ground_truth": "invoice_07_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_08",
-      "row_idx": 1888,
-      "file_name": "synth_0012_72cf2460.png",
-      "quality": "bad_scan",
-      "template": "modern_minimal",
-      "image": "invoice_08_row_1888.jpg",
-      "ground_truth": "invoice_08_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_09",
-      "row_idx": 1890,
-      "file_name": "synth_0014_d3972e7f.png",
-      "quality": "bad_scan",
-      "template": "freelancer_invoice",
-      "image": "invoice_09_row_1890.jpg",
-      "ground_truth": "invoice_09_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_10",
-      "row_idx": 1892,
-      "file_name": "synth_0016_223f5434.png",
-      "quality": "bad_scan",
-      "template": "modern_minimal",
-      "image": "invoice_10_row_1892.jpg",
-      "ground_truth": "invoice_10_ground_truth.json"
-    }
-  ]
-}