openmerit 0.1.0

package/instructions/OPENMERIT.md

@@ -0,0 +1,51 @@
+# OpenMerit — Model Merit Harness
+
+You are running under a main agent harness that is observed by **OpenMerit**, an
+external merit harness. OpenMerit's job is to make sure you are always running on
+the best model for the task at hand, at the best price, with a vetted fallback.
+
+## What OpenMerit does
+
+1. **Observes** this harness's session traces (models used, tokens, cost,
+   latency, errors) without intercepting or slowing down your work.
+2. **Evaluates** candidate models for each completed text or image task sequentially
+   through pi and scores their outputs with a judge model.
+3. **Maintains a pareto frontier** per task (quality vs. cost vs. latency) and
+   an aggregate frontier across all of your tasks.
+4. **Watches for new model releases** (provider catalogs + public benchmarks)
+   and queues promising releases for trial against your existing frontier.
+5. **Recommends or applies model swaps**, with reasoning, and keeps your
+   fallback model up to date.
+
+## What you should do
+
+- **Treat model changes as explicit routing decisions.** If you notice your
+  model identity change, it was approved by the user or passed their opt-in
+  auto-apply policy. Continue the task; your instructions and context are
+  unchanged.
+- **Respect the fallback.** If your current model errors, rate-limits, or is
+  retired, OpenMerit can offer the current fallback or apply it under an
+  opt-in automatic policy. Treat an approved fallback as normal operation.
+- **State your task clearly in your first message** of a session when possible.
+  OpenMerit keys its pareto frontier to task signatures; clear task statements
+  produce better model choices for you.
+- **Do not edit OpenMerit state files** (`~/.openmerit/`). If something looks
+  wrong, tell the user.
+
+## What you can ask the user for
+
+- `/openmerit` — show current model, fallback, frontier position, and pending
+  recommendations.
+- `/openmerit apply` / `/openmerit dismiss` — act on a pending recommendation
+  when automatic application is declined by the policy gate.
+- Policy changes (auto-apply thresholds, budgets, provider allow-lists) are
+  made by the user in `~/.openmerit/policy.json`, not by you.
+
+## Guarantees
+
+- Per-task comparisons send the task text, uploaded images (when present), and
+  candidate answers through pi/OpenRouter for model runs and judging. Use
+  non-sensitive examples while evaluating this alpha.
+- The shipped policy is supervised. Swaps happen automatically only after the
+  user opts in and the configured quality/cost guardrails pass; otherwise they
+  remain recommendations for a human to approve.

package/instructions/openmerit.policy.json

@@ -0,0 +1,33 @@
+{
+  "version": 1,
+  "mode": "recommend",
+  "auto_apply": {
+    "enabled": false,
+    "min_score_gain": 0.1,
+    "max_price_ratio": 1.5,
+    "require_frontier": true
+  },
+  "budgets": {
+    "max_usd_per_trial": 0.25,
+    "max_trials_per_day": 20,
+    "max_usd_per_day": 5.0
+  },
+  "providers": {
+    "allow": ["*"],
+    "deny": []
+  },
+  "watch": {
+    "catalog_interval_min": 360,
+    "traces_interval_sec": 5,
+    "trial_interval_min": 30,
+    "models_per_task": 3
+  },
+  "fallback": {
+    "auto_update": true,
+    "min_score": 0.6,
+    "apply_on_error": true
+  },
+  "judge_model": null,
+  "strategist_model": null,
+  "max_usd_per_m": 20.0
+}

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "openmerit",
+  "version": "0.1.0",
+  "description": "Find better models for each pi task by comparing quality, cost, and latency.",
+  "type": "module",
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "pi-coding-agent",
+    "model-routing",
+    "llm",
+    "openrouter",
+    "evaluation"
+  ],
+  "files": [
+    "dist/",
+    "extension/openmerit.ts",
+    "instructions/",
+    "examples/",
+    "benchmark/invoice_ocr/data/"
+  ],
+  "pi": {
+    "extensions": ["./extension/openmerit.ts"]
+  },
+  "bin": {
+    "openmerit": "dist/cli.js"
+  },
+  "engines": {
+    "node": ">=22.18.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/laz-aslam/openmerit.git"
+  },
+  "homepage": "https://github.com/laz-aslam/openmerit#readme",
+  "bugs": {
+    "url": "https://github.com/laz-aslam/openmerit/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "prepare": "npm run build",
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc --noEmit && tsc -p benchmark/tsconfig.json",
+    "typecheck:extension": "tsc --noEmit --strict --target ES2022 --module NodeNext --moduleResolution NodeNext --skipLibCheck extension/openmerit.ts",
+    "test": "vitest run",
+    "check": "npm run typecheck && npm run typecheck:extension && npm test && npm run build && npm run benchmark:verify-recorded",
+    "prepublishOnly": "npm run check",
+    "benchmark:invoice:prepare": "node benchmark/invoice_ocr/prepare_data.ts",
+    "benchmark:invoice": "node benchmark/invoice_ocr/benchmark.ts",
+    "benchmark:invoice:audit": "node benchmark/invoice_ocr/audit_results.ts",
+    "benchmark:invoice:round2": "node benchmark/invoice_ocr/benchmark_round2.ts",
+    "benchmark:invoice:round2:audit": "node benchmark/invoice_ocr/audit_round2.ts",
+    "benchmark:text-to-sql": "node benchmark/text_to_sql/benchmark.ts",
+    "benchmark:text-to-sql:audit": "node benchmark/text_to_sql/audit_results.ts",
+    "benchmark:policy": "node benchmark/policy_adjudication/benchmark.ts",
+    "benchmark:policy:audit": "node benchmark/policy_adjudication/audit_results.ts",
+    "benchmark:policy:validate": "node benchmark/policy_adjudication/validate_eval.ts",
+    "benchmark:verify-recorded": "node benchmark/verify_recorded.ts"
+  },
+  "license": "MIT",
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*"
+  },
+  "peerDependenciesMeta": {
+    "@earendil-works/pi-coding-agent": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@earendil-works/pi-coding-agent": "^0.85.1",
+    "@types/node": "^22.10.0",
+    "typescript": "^5.7.0",
+    "vitest": "^2.1.0"
+  }
+}