openmerit 0.1.4 → 0.1.6-preview.0

package/package.json

@@ -1,36 +1,53 @@
 {
   "name": "openmerit",
-  "version": "0.1.4",
-  "description": "Find better models for each pi task by comparing quality, cost, and latency.",
+  "version": "0.1.6-preview.0",
+  "description": "Evidence-driven model decisions for coding harnesses.",
   "type": "module",
+  "bin": {
+    "openmerit": "./dist/terminal/src/cli.mjs"
+  },
+  "license": "MIT",
   "keywords": [
     "pi-package",
     "pi-extension",
-    "pi-coding-agent",
     "model-routing",
-    "llm",
-    "openrouter",
-    "evaluation"
+    "evaluation",
+    "pareto-frontier"
   ],
   "files": [
-    "dist/",
-    "extension/openmerit.ts",
-    "instructions/",
-    "examples/",
-    "rules.md",
-    "benchmark/invoice_ocr/data/"
+    "dist/core/",
+    "dist/protocol/",
+    "dist/pi/",
+    "dist/terminal/",
+    "docs/",
+    "CHANGELOG.md",
+    "LICENSE",
+    "README.md"
   ],
+  "exports": {
+    ".": {
+      "types": "./dist/core/src/index.d.ts",
+      "default": "./dist/core/src/index.js"
+    },
+    "./core": {
+      "types": "./dist/core/src/index.d.ts",
+      "default": "./dist/core/src/index.js"
+    },
+    "./protocol": {
+      "types": "./dist/protocol/src/index.d.ts",
+      "default": "./dist/protocol/src/index.js"
+    },
+    "./pi": {
+      "types": "./dist/pi/src/index.d.ts",
+      "default": "./dist/pi/src/index.js"
+    },
+    "./package.json": "./package.json"
+  },
   "pi": {
     "extensions": [
-      "./extension/openmerit.ts"
+      "./dist/pi/src/index.js"
     ]
   },
-  "bin": {
-    "openmerit": "dist/cli.js"
-  },
-  "engines": {
-    "node": ">=22.18.0"
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/laz-aslam/openmerit.git"
@@ -42,29 +59,35 @@
   "publishConfig": {
     "access": "public"
   },
+  "workspaces": [
+    "packages/*"
+  ],
+  "engines": {
+    "node": ">=22.19.0"
+  },
   "scripts": {
+    "check": "npm run check --workspaces --if-present",
+    "test": "npm run test --workspaces --if-present",
+    "typecheck": "tsc -p tsconfig.json",
+    "build": "node scripts/build.mjs",
+    "demo": "node scripts/demo.mjs",
+    "predash": "node scripts/build-terminal.mjs",
+    "dash": "node packages/terminal/src/cli.mjs dash",
+    "build:docs": "node scripts/build-docs.mjs",
+    "check:docs": "npm run build:docs && node --test scripts/docs.test.mjs",
+    "check:demo": "node --test demo/test/*.test.mjs",
     "prepare": "npm run build",
-    "build": "tsc -p tsconfig.json",
-    "typecheck": "tsc --noEmit && tsc -p benchmark/tsconfig.json",
-    "typecheck:extension": "tsc --noEmit --strict --target ES2022 --module NodeNext --moduleResolution NodeNext --skipLibCheck extension/openmerit.ts",
-    "test": "vitest run",
-    "check": "npm run typecheck && npm run typecheck:extension && npm test && npm run build && npm run benchmark:verify-recorded",
-    "prepublishOnly": "npm run check",
-    "benchmark:invoice:prepare": "node benchmark/invoice_ocr/prepare_data.ts",
-    "benchmark:invoice": "node benchmark/invoice_ocr/benchmark.ts",
-    "benchmark:invoice:audit": "node benchmark/invoice_ocr/audit_results.ts",
-    "benchmark:invoice:round2": "node benchmark/invoice_ocr/benchmark_round2.ts",
-    "benchmark:invoice:round2:audit": "node benchmark/invoice_ocr/audit_round2.ts",
-    "benchmark:text-to-sql": "node benchmark/text_to_sql/benchmark.ts",
-    "benchmark:text-to-sql:audit": "node benchmark/text_to_sql/audit_results.ts",
-    "benchmark:policy": "node benchmark/policy_adjudication/benchmark.ts",
-    "benchmark:policy:audit": "node benchmark/policy_adjudication/audit_results.ts",
-    "benchmark:policy:validate": "node benchmark/policy_adjudication/validate_eval.ts",
-    "benchmark:verify-recorded": "node benchmark/verify_recorded.ts"
+    "verify": "npm test && npm run check && npm run typecheck && npm run build && npm run check:demo && npm run check:docs && git diff --check",
+    "prepublishOnly": "npm run verify",
+    "build:dash-preview": "node scripts/build-terminal-preview.mjs",
+    "deploy:dash-preview": "npx --yes wrangler@4.136.3 deploy --config wrangler.terminal.jsonc"
+  },
+  "dependencies": {
+    "@earendil-works/pi-ai": "0.87.0",
+    "typebox": "1.3.27"
   },
-  "license": "MIT",
   "peerDependencies": {
-    "@earendil-works/pi-coding-agent": "*"
+    "@earendil-works/pi-coding-agent": ">=0.87.0 <0.88.0"
   },
   "peerDependenciesMeta": {
     "@earendil-works/pi-coding-agent": {
@@ -72,9 +95,18 @@
     }
   },
   "devDependencies": {
-    "@earendil-works/pi-coding-agent": "^0.85.1",
-    "@types/node": "^22.10.0",
-    "typescript": "^5.7.0",
-    "vitest": "^2.1.0"
+    "@resvg/resvg-js": "2.6.2",
+    "@types/node": "^22.18.6",
+    "@xyflow/react": "12.12.0",
+    "elkjs": "0.12.0",
+    "esbuild": "0.28.2",
+    "liveline": "0.0.7",
+    "markdown-it": "15.0.2",
+    "minisearch": "7.2.0",
+    "morphdom": "2.7.8",
+    "react": "19.3.0",
+    "react-dom": "19.3.0",
+    "torph": "0.1.3",
+    "typescript": "^5.9.3"
   }
 }

package/benchmark/invoice_ocr/data/invoice_01_ground_truth.json

@@ -1,38 +0,0 @@
-{
-  "supplier_name": "PixelWave Solutions Inc.",
-  "customer_name": "Greenfield Marketing Ltd.",
-  "invoice_number": "INV-2023-0098",
-  "po_number": "PO-5678",
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-14",
-  "document_currency": "CAD",
-  "document_total_net": 2220.0,
-  "document_total_tax": 111.0,
-  "document_total_amount": 2331.0,
-  "line_items": [
-    {
-      "description": "Website design services",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "Content writing package",
-      "quantity": 10.0,
-      "unit_price": 50.0,
-      "amount": 500.0
-    },
-    {
-      "description": "SEO optimization",
-      "quantity": 5.0,
-      "unit_price": 80.0,
-      "amount": 400.0
-    },
-    {
-      "description": "Hosting subscription",
-      "quantity": 12.0,
-      "unit_price": 10.0,
-      "amount": 120.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_02_ground_truth.json

@@ -1,32 +0,0 @@
-{
-  "supplier_name": "TechSolutions Ltd",
-  "customer_name": null,
-  "invoice_number": "IE-INV-2024-000123",
-  "po_number": null,
-  "document_date": "2024-03-15",
-  "due_date": null,
-  "document_currency": "EUR",
-  "document_total_net": 6200.0,
-  "document_total_tax": 1143.0,
-  "document_total_amount": 7343.0,
-  "line_items": [
-    {
-      "description": "Consulting services for project X",
-      "quantity": 3.0,
-      "unit_price": 1500.0,
-      "amount": 4500.0
-    },
-    {
-      "description": "High-performance laptop (Model Y)",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "International shipping to Germany",
-      "quantity": 1.0,
-      "unit_price": 500.0,
-      "amount": 500.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_03_ground_truth.json

@@ -1,26 +0,0 @@
-{
-  "supplier_name": "CloudCompute Solutions Inc.",
-  "customer_name": "Acme Corp.",
-  "invoice_number": "INV-20230815-001",
-  "po_number": "PO-987654",
-  "document_date": "2023-08-15",
-  "due_date": "2023-09-14",
-  "document_currency": "USD",
-  "document_total_net": 110.0,
-  "document_total_tax": 9.07,
-  "document_total_amount": 119.07,
-  "line_items": [
-    {
-      "description": "Enterprise Cloud Storage Subscription",
-      "quantity": 1.0,
-      "unit_price": 50.0,
-      "amount": 50.0
-    },
-    {
-      "description": "Premium Support Add-On",
-      "quantity": 2.0,
-      "unit_price": 30.0,
-      "amount": 60.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_04_ground_truth.json

@@ -1,26 +0,0 @@
-{
-  "supplier_name": "Tech Solutions Ltd",
-  "customer_name": "Acme Construction Ltd",
-  "invoice_number": "INV-20230815-001",
-  "po_number": "PO-7654321",
-  "document_date": "2023-08-15",
-  "due_date": "2023-09-14",
-  "document_currency": "GBP",
-  "document_total_net": 1500.0,
-  "document_total_tax": 240.0,
-  "document_total_amount": 1740.0,
-  "line_items": [
-    {
-      "description": "High-performance laptop",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "Office desk chair",
-      "quantity": 2.0,
-      "unit_price": 150.0,
-      "amount": 300.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_05_ground_truth.json

@@ -1,38 +0,0 @@
-{
-  "supplier_name": "Celtic Code Solutions",
-  "customer_name": "North Star Consulting Ltd.",
-  "invoice_number": "INV-2023-0075",
-  "po_number": null,
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-14",
-  "document_currency": "EUR",
-  "document_total_net": 4600.0,
-  "document_total_tax": 1058.0,
-  "document_total_amount": 5658.0,
-  "line_items": [
-    {
-      "description": "UI Development Consulting",
-      "quantity": 20.0,
-      "unit_price": 80.0,
-      "amount": 1600.0
-    },
-    {
-      "description": "UX Research",
-      "quantity": 10.0,
-      "unit_price": 90.0,
-      "amount": 900.0
-    },
-    {
-      "description": "Custom API Integration",
-      "quantity": 15.0,
-      "unit_price": 100.0,
-      "amount": 1500.0
-    },
-    {
-      "description": "Project Management Support",
-      "quantity": 5.0,
-      "unit_price": 120.0,
-      "amount": 600.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_06_ground_truth.json

@@ -1,38 +0,0 @@
-{
-  "supplier_name": "Oak Property Management Ltd",
-  "customer_name": "John Doe",
-  "invoice_number": "RENT-2023-001",
-  "po_number": null,
-  "document_date": "2023-05-01",
-  "due_date": "2023-05-15",
-  "document_currency": "EUR",
-  "document_total_net": 1650.0,
-  "document_total_tax": 103.5,
-  "document_total_amount": 1753.5,
-  "line_items": [
-    {
-      "description": "Monthly Base Rent",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "Utilities (Electricity/Gas)",
-      "quantity": 1.0,
-      "unit_price": 200.0,
-      "amount": 200.0
-    },
-    {
-      "description": "Property Tax",
-      "quantity": 1.0,
-      "unit_price": 150.0,
-      "amount": 150.0
-    },
-    {
-      "description": "Cleaning Service",
-      "quantity": 1.0,
-      "unit_price": 100.0,
-      "amount": 100.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_07_ground_truth.json

@@ -1,20 +0,0 @@
-{
-  "supplier_name": "Acme Office Supplies Inc.",
-  "customer_name": "Global Tech Corp.",
-  "invoice_number": "INV-2023-0001",
-  "po_number": null,
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-15",
-  "document_currency": "USD",
-  "document_total_net": 600.0,
-  "document_total_tax": 42.0,
-  "document_total_amount": 642.0,
-  "line_items": [
-    {
-      "description": "Office chairs",
-      "quantity": 5.0,
-      "unit_price": 120.0,
-      "amount": 600.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_08_ground_truth.json

@@ -1,38 +0,0 @@
-{
-  "supplier_name": "Digital Solutions Pty Ltd",
-  "customer_name": "Global Tech Industries Ltd",
-  "invoice_number": "INV-2024-005",
-  "po_number": null,
-  "document_date": "2024-06-15",
-  "due_date": "2024-07-15",
-  "document_currency": "AUD",
-  "document_total_net": 3840.0,
-  "document_total_tax": 384.0,
-  "document_total_amount": 4224.0,
-  "line_items": [
-    {
-      "description": "Cloud hosting services",
-      "quantity": 12.0,
-      "unit_price": 120.0,
-      "amount": 1440.0
-    },
-    {
-      "description": "Software license renewal",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    },
-    {
-      "description": "Subscription data analytics package",
-      "quantity": 1.0,
-      "unit_price": 800.0,
-      "amount": 800.0
-    },
-    {
-      "description": "Technical support",
-      "quantity": 1.0,
-      "unit_price": 400.0,
-      "amount": 400.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_09_ground_truth.json

@@ -1,26 +0,0 @@
-{
-  "supplier_name": "Acme Consulting Group",
-  "customer_name": null,
-  "invoice_number": "INV-2023-0789",
-  "po_number": "PO-2023-4567",
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-14",
-  "document_currency": "USD",
-  "document_total_net": 3250.0,
-  "document_total_tax": 268.12,
-  "document_total_amount": 3518.12,
-  "line_items": [
-    {
-      "description": "Business Strategy Consultation",
-      "quantity": 10.0,
-      "unit_price": 200.0,
-      "amount": 2000.0
-    },
-    {
-      "description": "Technology Roadmap Development",
-      "quantity": 5.0,
-      "unit_price": 250.0,
-      "amount": 1250.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/invoice_10_ground_truth.json

@@ -1,20 +0,0 @@
-{
-  "supplier_name": "VirtualCo Solutions",
-  "customer_name": "John Doe",
-  "invoice_number": "INV-2023-000123",
-  "po_number": null,
-  "document_date": "2023-07-15",
-  "due_date": "2023-08-14",
-  "document_currency": "USD",
-  "document_total_net": 1200.0,
-  "document_total_tax": 96.0,
-  "document_total_amount": 1296.0,
-  "line_items": [
-    {
-      "description": "Enterprise SaaS Subscription for Q3 2023",
-      "quantity": 1.0,
-      "unit_price": 1200.0,
-      "amount": 1200.0
-    }
-  ]
-}

package/benchmark/invoice_ocr/data/manifest.json

@@ -1,97 +0,0 @@
-{
-  "dataset": "alamgirqazi/invoice-ocr-synthetic",
-  "dataset_revision": "44ddfd3c0c373b69f377910f83c4164a1e52d52c",
-  "license": "CC BY 4.0",
-  "rows": [
-    {
-      "invoice_id": "invoice_01",
-      "row_idx": 2,
-      "file_name": "synth_0002_8f01361b.png",
-      "quality": "clean",
-      "template": "freelancer_invoice",
-      "image": "invoice_01_row_2.jpg",
-      "ground_truth": "invoice_01_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_02",
-      "row_idx": 5,
-      "file_name": "synth_0005_553deef3.png",
-      "quality": "clean",
-      "template": "formal_corporate",
-      "image": "invoice_02_row_5.jpg",
-      "ground_truth": "invoice_02_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_03",
-      "row_idx": 6,
-      "file_name": "synth_0006_474081ca.png",
-      "quality": "clean",
-      "template": "modern_minimal",
-      "image": "invoice_03_row_6.jpg",
-      "ground_truth": "invoice_03_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_04",
-      "row_idx": 7,
-      "file_name": "synth_0007_7292e354.png",
-      "quality": "clean",
-      "template": "utility_bill",
-      "image": "invoice_04_row_7.jpg",
-      "ground_truth": "invoice_04_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_05",
-      "row_idx": 947,
-      "file_name": "synth_0009_8f6b430f.png",
-      "quality": "faded",
-      "template": "freelancer_invoice",
-      "image": "invoice_05_row_947.jpg",
-      "ground_truth": "invoice_05_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_06",
-      "row_idx": 948,
-      "file_name": "synth_0010_73bd8d75.png",
-      "quality": "faded",
-      "template": "formal_corporate",
-      "image": "invoice_06_row_948.jpg",
-      "ground_truth": "invoice_06_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_07",
-      "row_idx": 949,
-      "file_name": "synth_0011_8888d5c6.png",
-      "quality": "faded",
-      "template": "utility_bill",
-      "image": "invoice_07_row_949.jpg",
-      "ground_truth": "invoice_07_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_08",
-      "row_idx": 1888,
-      "file_name": "synth_0012_72cf2460.png",
-      "quality": "bad_scan",
-      "template": "modern_minimal",
-      "image": "invoice_08_row_1888.jpg",
-      "ground_truth": "invoice_08_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_09",
-      "row_idx": 1890,
-      "file_name": "synth_0014_d3972e7f.png",
-      "quality": "bad_scan",
-      "template": "freelancer_invoice",
-      "image": "invoice_09_row_1890.jpg",
-      "ground_truth": "invoice_09_ground_truth.json"
-    },
-    {
-      "invoice_id": "invoice_10",
-      "row_idx": 1892,
-      "file_name": "synth_0016_223f5434.png",
-      "quality": "bad_scan",
-      "template": "modern_minimal",
-      "image": "invoice_10_row_1892.jpg",
-      "ground_truth": "invoice_10_ground_truth.json"
-    }
-  ]
-}