create-quorum-router 0.1.7 → 0.1.8

package/README.md

@@ -12,11 +12,12 @@ npx --yes create-quorum-router@latest my-quorum-router-demo
 cd my-quorum-router-demo
 deno --version
 deno task smoke
+deno task calibration:demo
 deno task intake
 deno task supabase:status
 ```
 
-Current package version: `create-quorum-router@0.1.7`. Releases are published
+Current package version: `create-quorum-router@0.1.8`. Releases are published
 from an immutable Git tag through GitHub Actions OIDC Trusted Publishing.
 
 ## What the generated project supports
@@ -24,6 +25,10 @@ from an immutable Git tag through GitHub Actions OIDC Trusted Publishing.
 `deno task smoke` is deterministic fixture-only and does not call a provider
 API.
 
+`deno task calibration:demo` runs the bundled calibration-by-task API against
+deterministic local observations. The report is advisory-only and is not
+connected to routing weights, provider eligibility, or execution.
+
 `deno task intake` is the first real setup command. It detects local provider
 wrappers, checks OAuth/session status, runs safe list-only model inventory where
 possible, writes redacted local health artifacts under `out/`, and recommends
@@ -32,6 +37,7 @@ the next command.
 ```bash
 deno task check
 deno task smoke
+deno task calibration:demo
 deno task intake
 deno task auth:status
 deno task auth:login

package/bin/create-quorum-router.js

@@ -3,7 +3,7 @@ const fs = require("fs");
 const path = require("path");
 const { spawnSync } = require("child_process");
 
-const VERSION = "0.1.7";
+const VERSION = "0.1.8";
 const SUPPORTED_TEMPLATES = new Set(["basic"]);
 
 function usage() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-quorum-router",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "Create a local QuorumRouter project.",
   "license": "MIT",
   "bin": {

package/templates/basic/README.md

@@ -1,7 +1,7 @@
 # QuorumRouter generated workspace
 
 This generated workspace contains the MIT-licensed QuorumRouter current release.
-npm latest targets v0.1.7.
+npm latest targets v0.1.8.
 
 QuorumRouter is **MIT**. It is **open source**. Commercial and production use
 are permitted under the MIT License.
@@ -37,6 +37,7 @@ deno --version
 
 ```bash
 deno task smoke
+deno task calibration:demo
 deno task intake
 deno task auth:status
 deno task models:list
@@ -47,6 +48,11 @@ deno task supabase:status
 `smoke` proves the local scaffold runs with deterministic fixtures only. It does
 **not** call a real provider API.
 
+`calibration:demo` exercises the bundled calibration-by-task API with local
+fixture observations. Calibration reports are advisory-only: the scaffold does
+not use them to change routing weights, ranks, provider eligibility, quorum, or
+execution.
+
 `intake` detects local provider wrappers, checks OAuth/session status, runs safe
 model inventory/list-only probes where possible, writes local health traces
 under `out/`, and recommends the next command.
@@ -218,6 +224,8 @@ not paste them into chat/logs and do not commit `.env`.
   `src/provider_client.ts` — provider discovery and safe invocation.
 - `src/best_route.ts`, `src/agent_chat.ts` — gated dogfood commands.
 - `src/cost_aware.ts` — estimated-cost budget selection for Best Route.
+- `src/calibration.ts`, `src/calibration_demo.ts` — strict advisory calibration
+  aggregation and an offline runnable example.
 - `src/trace.ts`, `src/redact.ts`, `src/schema.ts`, `src/fixture_smoke.ts` —
   trace/redaction/schema/fixture support.
 - `out/.gitkeep` — local output directory placeholder.

package/templates/basic/deno.json

@@ -6,6 +6,7 @@
   "tasks": {
     "check": "deno check main.ts src/*.ts",
     "smoke": "deno run main.ts",
+    "calibration:demo": "deno run src/calibration_demo.ts",
     "intake": "deno run --allow-read --allow-write --allow-env --allow-net --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts intake",
     "auth:status": "deno run --allow-read --allow-env --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts auth:status",
     "auth:login": "deno run src/cli.ts auth:login",

package/templates/basic/src/calibration.ts

@@ -0,0 +1,249 @@
+import { z } from "zod";
+
+export const MAX_CALIBRATION_OBSERVATIONS = 10_000;
+export const MAX_CALIBRATION_IDENTIFIER_LENGTH = 256;
+
+function stableMetric(value: number): number {
+  const rounded = Number(value.toPrecision(15));
+  return rounded === 0 ? 0 : rounded;
+}
+
+function boundedCanonicalText(maxLength: number) {
+  return z.string().trim().transform((value) => value.normalize("NFC")).pipe(
+    z.string().min(1).max(maxLength).refine(
+      (value) => !/[\p{Cc}\p{Cf}\p{Default_Ignorable_Code_Point}]/u.test(value),
+      "control and default-ignorable characters are not allowed",
+    ),
+  );
+}
+
+function hasValidRfc3339Offset(value: string): boolean {
+  if (value.endsWith("Z")) return true;
+  const match = /[+-](\d{2}):(\d{2})$/.exec(value);
+  if (!match) return false;
+  return Number(match[1]) <= 23 && Number(match[2]) <= 59;
+}
+
+const ObservationIdSchema = boundedCanonicalText(
+  MAX_CALIBRATION_IDENTIFIER_LENGTH,
+);
+const TaskTypeSchema = boundedCanonicalText(MAX_CALIBRATION_IDENTIFIER_LENGTH);
+const SourceLabelSchema = boundedCanonicalText(
+  MAX_CALIBRATION_IDENTIFIER_LENGTH,
+);
+const EvaluationTimestampSchema = z.string().datetime({ offset: true }).refine(
+  hasValidRfc3339Offset,
+  "evaluated_at must use a valid RFC 3339 UTC offset",
+);
+
+export const TaskCalibrationSourceSchema = z.object({
+  provider: SourceLabelSchema,
+  model: SourceLabelSchema,
+}).strict();
+
+/**
+ * Structurally validated calibration input supplied by the caller.
+ *
+ * `evaluation_basis` records an unverified provenance assertion; this schema
+ * cannot establish label independence, label quality, example matching, or
+ * freedom from leakage. `confidence` must be the model's probability, recorded
+ * before the correctness label is observed, that this specific answer is
+ * correct. The schema can enforce only the numeric range, not that provenance.
+ */
+export const TaskCalibrationObservationSchema = z.object({
+  observation_id: ObservationIdSchema,
+  task_type: TaskTypeSchema,
+  source: TaskCalibrationSourceSchema,
+  evaluation_basis: z.literal("caller_attested_external_ground_truth"),
+  correct: z.boolean(),
+  confidence: z.number().finite().min(0).max(1),
+  evaluated_at: EvaluationTimestampSchema,
+}).strict();
+
+export const TaskCalibrationObservationListSchema = z.array(
+  TaskCalibrationObservationSchema,
+).max(MAX_CALIBRATION_OBSERVATIONS).superRefine((observations, context) => {
+  const ids = new Set<string>();
+  for (const [index, observation] of observations.entries()) {
+    if (ids.has(observation.observation_id)) {
+      context.addIssue({
+        code: "custom",
+        message: "observation_id must be unique",
+        path: [index, "observation_id"],
+      });
+    }
+    ids.add(observation.observation_id);
+  }
+});
+
+export const TaskCalibrationOptionsSchema = z.object({
+  minimum_sample_count: z.number().finite().int().positive().max(
+    MAX_CALIBRATION_OBSERVATIONS,
+  ).default(20),
+}).strict();
+
+export const TaskCalibrationGroupSchema = z.object({
+  task_type: TaskTypeSchema,
+  source: TaskCalibrationSourceSchema,
+  sample_count: z.number().int().positive().max(MAX_CALIBRATION_OBSERVATIONS),
+  accuracy: z.number().finite().min(0).max(1),
+  mean_confidence: z.number().finite().min(0).max(1),
+  brier_score: z.number().finite().min(0).max(1),
+  mean_calibration_bias: z.number().finite().min(-1).max(1),
+  // "sufficient" means only that the configured sample-count threshold is met.
+  sample_status: z.enum(["insufficient", "sufficient"]),
+}).strict();
+
+export const TaskCalibrationReportSchema = z.object({
+  schema_version: z.literal("quorum-router.calibration-by-task.v1"),
+  advisory_only: z.literal(true),
+  minimum_sample_count: z.number().int().positive().max(
+    MAX_CALIBRATION_OBSERVATIONS,
+  ),
+  groups: z.array(TaskCalibrationGroupSchema).max(MAX_CALIBRATION_OBSERVATIONS),
+}).strict().superRefine((report, context) => {
+  const groupKeys = new Set<string>();
+  for (const [index, group] of report.groups.entries()) {
+    const groupKey = JSON.stringify([
+      group.task_type,
+      group.source.provider,
+      group.source.model,
+    ]);
+    if (groupKeys.has(groupKey)) {
+      context.addIssue({
+        code: "custom",
+        message: "calibration report groups must be unique",
+        path: ["groups", index],
+      });
+    }
+    groupKeys.add(groupKey);
+
+    const expectedStatus = group.sample_count < report.minimum_sample_count
+      ? "insufficient"
+      : "sufficient";
+    if (group.sample_status !== expectedStatus) {
+      context.addIssue({
+        code: "custom",
+        message: "sample_status must match the configured sample threshold",
+        path: ["groups", index, "sample_status"],
+      });
+    }
+
+    const expectedBias = stableMetric(
+      group.mean_confidence - group.accuracy,
+    );
+    if (group.mean_calibration_bias !== expectedBias) {
+      context.addIssue({
+        code: "custom",
+        message: "mean_calibration_bias must equal mean_confidence - accuracy",
+        path: ["groups", index, "mean_calibration_bias"],
+      });
+    }
+  }
+});
+
+export type TaskCalibrationSource = z.infer<
+  typeof TaskCalibrationSourceSchema
+>;
+export type TaskCalibrationObservation = z.infer<
+  typeof TaskCalibrationObservationSchema
+>;
+export type TaskCalibrationOptions = z.input<
+  typeof TaskCalibrationOptionsSchema
+>;
+export type TaskCalibrationGroup = z.infer<
+  typeof TaskCalibrationGroupSchema
+>;
+export type TaskCalibrationReport = z.infer<
+  typeof TaskCalibrationReportSchema
+>;
+
+function stableSum(values: readonly number[]): number {
+  const sorted = [...values].sort((left, right) => left - right);
+  let total = 0;
+  let compensation = 0;
+
+  for (const value of sorted) {
+    const next = total + value;
+    compensation += Math.abs(total) >= Math.abs(value)
+      ? (total - next) + value
+      : (value - next) + total;
+    total = next;
+  }
+
+  return total + compensation;
+}
+
+function compareCodeUnits(left: string, right: string): number {
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}
+
+export function aggregateTaskCalibration(
+  observations: readonly unknown[],
+  options: TaskCalibrationOptions = {},
+): TaskCalibrationReport {
+  const parsedObservations = TaskCalibrationObservationListSchema.parse(
+    observations,
+  );
+  const { minimum_sample_count } = TaskCalibrationOptionsSchema.parse(options);
+  const grouped = new Map<string, TaskCalibrationObservation[]>();
+
+  for (const observation of parsedObservations) {
+    const key = JSON.stringify([
+      observation.task_type,
+      observation.source.provider,
+      observation.source.model,
+    ]);
+    const group = grouped.get(key);
+    if (group) {
+      group.push(observation);
+    } else {
+      grouped.set(key, [observation]);
+    }
+  }
+
+  const groups = [...grouped.values()].map((group) => {
+    const sample_count = group.length;
+    const correctCount = group.reduce(
+      (total, observation) => total + Number(observation.correct),
+      0,
+    );
+    const confidenceTotal = stableSum(
+      group.map((observation) => observation.confidence),
+    );
+    const brierTotal = stableSum(
+      group.map((observation) => {
+        const outcome = Number(observation.correct);
+        return (observation.confidence - outcome) ** 2;
+      }),
+    );
+    const accuracy = stableMetric(correctCount / sample_count);
+    const mean_confidence = stableMetric(confidenceTotal / sample_count);
+
+    return {
+      task_type: group[0].task_type,
+      source: group[0].source,
+      sample_count,
+      accuracy,
+      mean_confidence,
+      brier_score: stableMetric(brierTotal / sample_count),
+      mean_calibration_bias: stableMetric(mean_confidence - accuracy),
+      sample_status: sample_count < minimum_sample_count
+        ? "insufficient" as const
+        : "sufficient" as const,
+    };
+  }).sort((left, right) =>
+    compareCodeUnits(left.task_type, right.task_type) ||
+    compareCodeUnits(left.source.provider, right.source.provider) ||
+    compareCodeUnits(left.source.model, right.source.model)
+  );
+
+  return TaskCalibrationReportSchema.parse({
+    schema_version: "quorum-router.calibration-by-task.v1",
+    advisory_only: true,
+    minimum_sample_count,
+    groups,
+  });
+}

package/templates/basic/src/calibration_demo.ts

@@ -0,0 +1,25 @@
+import { aggregateTaskCalibration } from "./calibration.ts";
+
+const evaluated_at = "2026-07-13T00:00:00Z";
+const report = aggregateTaskCalibration([
+  {
+    observation_id: "demo-1",
+    task_type: "code-review",
+    source: { provider: "OpenAI", model: "example-model" },
+    evaluation_basis: "caller_attested_external_ground_truth",
+    correct: true,
+    confidence: 0.8,
+    evaluated_at,
+  },
+  {
+    observation_id: "demo-2",
+    task_type: "code-review",
+    source: { provider: "OpenAI", model: "example-model" },
+    evaluation_basis: "caller_attested_external_ground_truth",
+    correct: false,
+    confidence: 0.6,
+    evaluated_at,
+  },
+], { minimum_sample_count: 2 });
+
+console.log(JSON.stringify(report, null, 2));