create-quorum-router 0.1.5 → 0.1.8

package/README.md

@@ -12,10 +12,12 @@ npx --yes create-quorum-router@latest my-quorum-router-demo
 cd my-quorum-router-demo
 deno --version
 deno task smoke
+deno task calibration:demo
 deno task intake
+deno task supabase:status
 ```
 
-Current package version: `create-quorum-router@0.1.5`. Releases are published
+Current package version: `create-quorum-router@0.1.8`. Releases are published
 from an immutable Git tag through GitHub Actions OIDC Trusted Publishing.
 
 ## What the generated project supports
@@ -23,6 +25,10 @@ from an immutable Git tag through GitHub Actions OIDC Trusted Publishing.
 `deno task smoke` is deterministic fixture-only and does not call a provider
 API.
 
+`deno task calibration:demo` runs the bundled calibration-by-task API against
+deterministic local observations. The report is advisory-only and is not
+connected to routing weights, provider eligibility, or execution.
+
 `deno task intake` is the first real setup command. It detects local provider
 wrappers, checks OAuth/session status, runs safe list-only model inventory where
 possible, writes redacted local health artifacts under `out/`, and recommends
@@ -31,6 +37,7 @@ the next command.
 ```bash
 deno task check
 deno task smoke
+deno task calibration:demo
 deno task intake
 deno task auth:status
 deno task auth:login
@@ -63,6 +70,19 @@ preferred public dogfood path.
 Do not commit `.env`, `router.config.local.json`, `.quorum-router/`, or `out/`.
 Do not paste tokens into chat/logs.
 
+## Optional user-owned Supabase audit
+
+The generated project works without Supabase. Audit defaults to `disabled`.
+Users who opt in apply the generated migration to a Supabase project they own,
+choose `optional` or `required` in the non-secret feature config, inject a
+project URL, publishable/anon key, and Supabase Auth session JWT at runtime, and
+run `deno task supabase:status` before routing.
+
+The runtime rejects service-role/admin credentials. Audit records exclude
+prompts, model responses, credentials, client-supplied actor/org identity, and
+client timestamps. Agent Bus, Realtime, state sync, and analytics dashboards are
+not part of this integration.
+
 ## Runtime boundaries
 
 - Best Route/direct is production-ready best-answer routing.
@@ -71,7 +91,7 @@ Do not paste tokens into chat/logs.
   slice only when separately configured with signed policy and distinct
   approval.
 - The generated scaffold does not enable mutation by default.
-- No service-role runtime.
+- No service-role/admin runtime credentials.
 - No live Supabase Agent Bus runtime writes.
 - Public launch requires the repository verification, package tarball, registry
   readback, and clean-room NPX scaffold checks to pass.

package/bin/create-quorum-router.js

@@ -3,7 +3,7 @@ const fs = require("fs");
 const path = require("path");
 const { spawnSync } = require("child_process");
 
-const VERSION = "0.1.5";
+const VERSION = "0.1.8";
 const SUPPORTED_TEMPLATES = new Set(["basic"]);
 
 function usage() {
@@ -56,13 +56,33 @@ function parseArgs(argv) {
   return options;
 }
 
-function copyRecursive(from, to) {
+function rejectSymlink(destination) {
+  try {
+    if (fs.lstatSync(destination).isSymbolicLink()) {
+      throw new Error(`refusing to write through symlink: ${destination}`);
+    }
+  } catch (error) {
+    if (error && error.code === "ENOENT") return;
+    throw error;
+  }
+}
+
+function copyRecursive(from, to, targetRoot) {
+  const relative = path.relative(targetRoot, to);
+  if (relative.startsWith("..") || path.isAbsolute(relative)) {
+    throw new Error(`refusing to write outside target directory: ${to}`);
+  }
+  rejectSymlink(to);
   const stat = fs.statSync(from);
   if (stat.isDirectory()) {
     fs.mkdirSync(to, { recursive: true });
     for (const entry of fs.readdirSync(from)) {
       const targetEntry = entry === "gitignore" ? ".gitignore" : entry;
-      copyRecursive(path.join(from, entry), path.join(to, targetEntry));
+      copyRecursive(
+        path.join(from, entry),
+        path.join(to, targetEntry),
+        targetRoot,
+      );
     }
     return;
   }
@@ -97,6 +117,7 @@ function main() {
   }
 
   const targetDir = path.resolve(process.cwd(), parsed.dir);
+  rejectSymlink(targetDir);
   if (isNonEmptyDirectory(targetDir) && !parsed.force) {
     throw new Error(
       `refusing to overwrite non-empty directory: ${targetDir}\n` +
@@ -106,7 +127,7 @@ function main() {
 
   const templateDir = path.join(__dirname, "..", "templates", parsed.template);
   fs.mkdirSync(targetDir, { recursive: true });
-  copyRecursive(templateDir, targetDir);
+  copyRecursive(templateDir, targetDir, targetDir);
   fs.mkdirSync(path.join(targetDir, "out"), { recursive: true });
   fs.writeFileSync(path.join(targetDir, "out", ".gitkeep"), "");
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "create-quorum-router",
-  "version": "0.1.5",
+  "version": "0.1.8",
   "description": "Create a local QuorumRouter project.",
   "license": "MIT",
   "bin": {

package/templates/basic/README.md

@@ -1,7 +1,7 @@
 # QuorumRouter generated workspace
 
 This generated workspace contains the MIT-licensed QuorumRouter current release.
-npm latest targets v0.1.5.
+npm latest targets v0.1.8.
 
 QuorumRouter is **MIT**. It is **open source**. Commercial and production use
 are permitted under the MIT License.
@@ -19,8 +19,10 @@ are permitted under the MIT License.
 - Conversation-only `agent_chat` is explicit opt-in.
 - SafeLoop-backed production repository execution is not enabled by this
   generated scaffold; it requires external signed policy and distinct approval.
-- No service-role runtime.
-- No live Supabase runtime writes.
+- No service-role runtime; service-role/admin credentials are rejected even when
+  audit is disabled.
+- BYO Supabase audit is disabled by default and writes only when explicitly set
+  to `optional` or `required`.
 - No live Supabase Agent Bus runtime writes.
 - Best Route/direct is the production-ready best-answer routing path.
 - `agent-chat` is read-only explicit opt-in only.
@@ -35,15 +37,22 @@ deno --version
 
 ```bash
 deno task smoke
+deno task calibration:demo
 deno task intake
 deno task auth:status
 deno task models:list
 deno task health
+deno task supabase:status
 ```
 
 `smoke` proves the local scaffold runs with deterministic fixtures only. It does
 **not** call a real provider API.
 
+`calibration:demo` exercises the bundled calibration-by-task API with local
+fixture observations. Calibration reports are advisory-only: the scaffold does
+not use them to change routing weights, ranks, provider eligibility, quorum, or
+execution.
+
 `intake` detects local provider wrappers, checks OAuth/session status, runs safe
 model inventory/list-only probes where possible, writes local health traces
 under `out/`, and recommends the next command.
@@ -51,6 +60,37 @@ under `out/`, and recommends the next command.
 In short: intake is the first real setup command before `route:once`,
 `best-route`, or read-only `agent-chat`.
 
+## Optional BYO Supabase audit
+
+Supabase is not required. With no Supabase config or env, every existing task
+keeps its prior behavior and `deno task supabase:status` exits successfully with
+`state: disabled` without making a network request.
+
+To persist route outcomes in a Supabase project you own:
+
+1. Create or choose your Supabase project.
+2. Apply **both** files under `supabase/migrations/` in filename order using an
+   admin context outside the router runtime. The later limits migration is
+   mandatory.
+3. Ensure the active Supabase Auth user session JWT has an authenticated `sub`.
+4. Copy `router.config.example.json` to `router.config.json` and set only
+   `features.supabase.audit.mode` to `optional` or `required`.
+5. Inject `QUORUM_ROUTER_SUPABASE_URL`, `QUORUM_ROUTER_SUPABASE_ANON_KEY`, and
+   `QUORUM_ROUTER_SUPABASE_SESSION_JWT` at runtime.
+6. Run `deno task supabase:status`, then run `route:once` or `best-route`.
+
+`optional` preserves a successful route and prints a warning when audit delivery
+fails. `required` withholds the route result and exits nonzero when audit
+delivery fails. The RPC receives only the route decision and bounded metadata.
+It never receives prompts, model responses, credentials, `org_id`, `actor_id`,
+or `created_at`; the database derives both the actor and this single-user BYO
+audit namespace from `auth.uid()` and owns the timestamp. There is no central or
+shared database and no client tenant claim.
+
+Runtime service-role/admin credentials are forbidden. Agent Bus, Realtime
+wakeup, state sync, and an analytics dashboard are future features and are not
+enabled by this audit integration.
+
 ## Real provider dogfood commands
 
 Run only after `intake` reports a usable OAuth/session/wrapper provider:
@@ -87,6 +127,25 @@ Behavior:
   Only use this with repositories you are allowed to send to the selected
   provider.
 
+### Cost-aware Best Route
+
+Cost-aware routing is disabled unless both budget variables are set. Estimates
+are user-supplied per invocation, not live provider billing data.
+
+```bash
+QUORUM_ROUTER_MAX_BUDGET_USD=0.05 \
+QUORUM_ROUTER_ESTIMATED_COSTS_JSON='{"openai/gpt-5":0.03,"xai/grok":0.02}' \
+RUN_EXTERNAL_MODEL_DOGFOOD=1 \
+  deno task best-route --prompt "Choose the safest launch copy."
+```
+
+Best Route preserves candidate quality/readiness order, admits candidates while
+their configured estimates fit the budget, and records selected and excluded
+model IDs plus reasons in `out/best-route-trace.json`. Models without an
+estimate are excluded when cost-aware routing is enabled. If no candidate fits,
+the command fails before invoking a provider. This is pre-invocation budget
+control; it does not claim exact or live API spend.
+
 ### Forced wrapper provider/model selection
 
 Use these only when you want a specific local wrapper/model. The scaffold fails
@@ -152,17 +211,24 @@ not paste them into chat/logs and do not commit `.env`.
 - `.gitignore` — excludes `.env`, `out/`, `router.config.local.json`,
   `provider_config.json`, and `.quorum-router/`.
 - `router.config.example.json` — non-secret example boundaries.
+- `supabase/migrations/20260701130000_workflow_access_audit.sql` — optional BYO
+  Supabase audit migration
+- `supabase/migrations/20260712211500_workflow_access_audit_limits.sql` —
+  required RPC narrowing and payload-limit migration
 - `src/cli.ts` — command dispatcher.
+- `src/supabase.ts` — offline status and selective audit RPC hook.
 - `src/intake.ts` — first-run onboarding.
 - `src/auth.ts`, `src/auth_oauth.ts`, `src/auth_session.ts`,
   `src/auth_env_fallback.ts` — auth/session/fallback boundaries.
 - `src/provider_registry.ts`, `src/model_inventory.ts`, `src/wrapper_client.ts`,
   `src/provider_client.ts` — provider discovery and safe invocation.
 - `src/best_route.ts`, `src/agent_chat.ts` — gated dogfood commands.
+- `src/cost_aware.ts` — estimated-cost budget selection for Best Route.
+- `src/calibration.ts`, `src/calibration_demo.ts` — strict advisory calibration
+  aggregation and an offline runnable example.
 - `src/trace.ts`, `src/redact.ts`, `src/schema.ts`, `src/fixture_smoke.ts` —
   trace/redaction/schema/fixture support.
 - `out/.gitkeep` — local output directory placeholder.
 
-Public Product Hunt/X launch remains blocked until the user personally runs a
-local pre-release workspace and approves release continuation. This scaffold
-does not publish npm, create a GitHub release, or mutate tags/dist-tags.
+This generated scaffold operates locally. It does not publish npm packages,
+create GitHub releases, or mutate tags or dist-tags.

package/templates/basic/deno.json

@@ -6,12 +6,14 @@
   "tasks": {
     "check": "deno check main.ts src/*.ts",
     "smoke": "deno run main.ts",
+    "calibration:demo": "deno run src/calibration_demo.ts",
     "intake": "deno run --allow-read --allow-write --allow-env --allow-net --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts intake",
     "auth:status": "deno run --allow-read --allow-env --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts auth:status",
     "auth:login": "deno run src/cli.ts auth:login",
     "auth:logout": "deno run --allow-read --allow-write --allow-env src/cli.ts auth:logout",
     "models:list": "deno run --allow-read --allow-write --allow-env --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts models:list",
     "health": "deno run --allow-read --allow-write --allow-env --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts health",
+    "supabase:status": "deno run --allow-read --allow-env src/cli.ts supabase:status",
     "route:once": "deno run --allow-read --allow-write --allow-env --allow-net --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts route:once",
     "best-route": "deno run --allow-read --allow-write --allow-env --allow-net --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts best-route",
     "agent-chat": "deno run --allow-read --allow-write --allow-env --allow-net --allow-run=grok,codex,claude,gemini,qwen,devin src/cli.ts agent-chat"

package/templates/basic/router.config.example.json

@@ -5,6 +5,17 @@
     "enabled_by": "QUORUM_ROUTER_AUTH_MODE=env",
     "usage": "private manual fallback only"
   },
+  "features": {
+    "supabase": {
+      "audit": {
+        "mode": "disabled"
+      }
+    }
+  },
+  "cost_aware": {
+    "enabled_by": "QUORUM_ROUTER_MAX_BUDGET_USD + QUORUM_ROUTER_ESTIMATED_COSTS_JSON",
+    "pricing_source": "user-supplied estimates, not live billing"
+  },
   "runtime_boundaries": {
     "best_route_direct": "production-ready best-answer routing",
     "agent_chat": "experimental explicit opt-in only",

package/templates/basic/src/auth_session.ts

@@ -88,6 +88,7 @@ export function discoverInventory(
       can_invoke: exists,
       command: spec.command,
       args_template: spec.args_template,
+      prompt_transport: spec.prompt_transport,
       notes: exists
         ? [
           ...spec.notes,

package/templates/basic/src/best_route.ts

@@ -21,6 +21,7 @@ import {
 import { buildTrace, score, writeTrace } from "./trace.ts";
 import { callWrapper } from "./wrapper_client.ts";
 import { preparePromptWithContext } from "./context.ts";
+import { selectCandidatesWithinBudget } from "./cost_aware.ts";
 
 function hasExplicitSelection(request: ProviderSelectionRequest): boolean {
   return Boolean(
@@ -261,7 +262,12 @@ export async function runBestRoute(
 > {
   assertOptIn();
   const authMode = parseAuthMode(readRouterEnv("QUORUM_ROUTER_AUTH_MODE"));
-  const { request, candidates } = await discoverCandidates(authMode);
+  const { request, candidates: discoveredCandidates } =
+    await discoverCandidates(
+      authMode,
+    );
+  const costAwareDecision = selectCandidatesWithinBudget(discoveredCandidates);
+  const candidates = costAwareDecision.candidates;
   if (candidates.length === 0) {
     throw new Error(
       "OAuth/session-first provider unavailable. best-route has no usable OAuth/session/wrapper provider yet. Next: deno task auth:login",
@@ -316,6 +322,9 @@ export async function runBestRoute(
       selectionHonored(request, result)
     ),
     fallbackUsed: usedEnvFallback,
+    costAware: costAwareDecision.cost.enabled
+      ? costAwareDecision.cost
+      : undefined,
   });
   const tracePath = await writeTrace("best-route-trace", trace);
   return { results, tracePath, trace };

package/templates/basic/src/calibration.ts

@@ -0,0 +1,249 @@
+import { z } from "zod";
+
+export const MAX_CALIBRATION_OBSERVATIONS = 10_000;
+export const MAX_CALIBRATION_IDENTIFIER_LENGTH = 256;
+
+function stableMetric(value: number): number {
+  const rounded = Number(value.toPrecision(15));
+  return rounded === 0 ? 0 : rounded;
+}
+
+function boundedCanonicalText(maxLength: number) {
+  return z.string().trim().transform((value) => value.normalize("NFC")).pipe(
+    z.string().min(1).max(maxLength).refine(
+      (value) => !/[\p{Cc}\p{Cf}\p{Default_Ignorable_Code_Point}]/u.test(value),
+      "control and default-ignorable characters are not allowed",
+    ),
+  );
+}
+
+function hasValidRfc3339Offset(value: string): boolean {
+  if (value.endsWith("Z")) return true;
+  const match = /[+-](\d{2}):(\d{2})$/.exec(value);
+  if (!match) return false;
+  return Number(match[1]) <= 23 && Number(match[2]) <= 59;
+}
+
+const ObservationIdSchema = boundedCanonicalText(
+  MAX_CALIBRATION_IDENTIFIER_LENGTH,
+);
+const TaskTypeSchema = boundedCanonicalText(MAX_CALIBRATION_IDENTIFIER_LENGTH);
+const SourceLabelSchema = boundedCanonicalText(
+  MAX_CALIBRATION_IDENTIFIER_LENGTH,
+);
+const EvaluationTimestampSchema = z.string().datetime({ offset: true }).refine(
+  hasValidRfc3339Offset,
+  "evaluated_at must use a valid RFC 3339 UTC offset",
+);
+
+export const TaskCalibrationSourceSchema = z.object({
+  provider: SourceLabelSchema,
+  model: SourceLabelSchema,
+}).strict();
+
+/**
+ * Structurally validated calibration input supplied by the caller.
+ *
+ * `evaluation_basis` records an unverified provenance assertion; this schema
+ * cannot establish label independence, label quality, example matching, or
+ * freedom from leakage. `confidence` must be the model's probability, recorded
+ * before the correctness label is observed, that this specific answer is
+ * correct. The schema can enforce only the numeric range, not that provenance.
+ */
+export const TaskCalibrationObservationSchema = z.object({
+  observation_id: ObservationIdSchema,
+  task_type: TaskTypeSchema,
+  source: TaskCalibrationSourceSchema,
+  evaluation_basis: z.literal("caller_attested_external_ground_truth"),
+  correct: z.boolean(),
+  confidence: z.number().finite().min(0).max(1),
+  evaluated_at: EvaluationTimestampSchema,
+}).strict();
+
+export const TaskCalibrationObservationListSchema = z.array(
+  TaskCalibrationObservationSchema,
+).max(MAX_CALIBRATION_OBSERVATIONS).superRefine((observations, context) => {
+  const ids = new Set<string>();
+  for (const [index, observation] of observations.entries()) {
+    if (ids.has(observation.observation_id)) {
+      context.addIssue({
+        code: "custom",
+        message: "observation_id must be unique",
+        path: [index, "observation_id"],
+      });
+    }
+    ids.add(observation.observation_id);
+  }
+});
+
+export const TaskCalibrationOptionsSchema = z.object({
+  minimum_sample_count: z.number().finite().int().positive().max(
+    MAX_CALIBRATION_OBSERVATIONS,
+  ).default(20),
+}).strict();
+
+export const TaskCalibrationGroupSchema = z.object({
+  task_type: TaskTypeSchema,
+  source: TaskCalibrationSourceSchema,
+  sample_count: z.number().int().positive().max(MAX_CALIBRATION_OBSERVATIONS),
+  accuracy: z.number().finite().min(0).max(1),
+  mean_confidence: z.number().finite().min(0).max(1),
+  brier_score: z.number().finite().min(0).max(1),
+  mean_calibration_bias: z.number().finite().min(-1).max(1),
+  // "sufficient" means only that the configured sample-count threshold is met.
+  sample_status: z.enum(["insufficient", "sufficient"]),
+}).strict();
+
+export const TaskCalibrationReportSchema = z.object({
+  schema_version: z.literal("quorum-router.calibration-by-task.v1"),
+  advisory_only: z.literal(true),
+  minimum_sample_count: z.number().int().positive().max(
+    MAX_CALIBRATION_OBSERVATIONS,
+  ),
+  groups: z.array(TaskCalibrationGroupSchema).max(MAX_CALIBRATION_OBSERVATIONS),
+}).strict().superRefine((report, context) => {
+  const groupKeys = new Set<string>();
+  for (const [index, group] of report.groups.entries()) {
+    const groupKey = JSON.stringify([
+      group.task_type,
+      group.source.provider,
+      group.source.model,
+    ]);
+    if (groupKeys.has(groupKey)) {
+      context.addIssue({
+        code: "custom",
+        message: "calibration report groups must be unique",
+        path: ["groups", index],
+      });
+    }
+    groupKeys.add(groupKey);
+
+    const expectedStatus = group.sample_count < report.minimum_sample_count
+      ? "insufficient"
+      : "sufficient";
+    if (group.sample_status !== expectedStatus) {
+      context.addIssue({
+        code: "custom",
+        message: "sample_status must match the configured sample threshold",
+        path: ["groups", index, "sample_status"],
+      });
+    }
+
+    const expectedBias = stableMetric(
+      group.mean_confidence - group.accuracy,
+    );
+    if (group.mean_calibration_bias !== expectedBias) {
+      context.addIssue({
+        code: "custom",
+        message: "mean_calibration_bias must equal mean_confidence - accuracy",
+        path: ["groups", index, "mean_calibration_bias"],
+      });
+    }
+  }
+});
+
+export type TaskCalibrationSource = z.infer<
+  typeof TaskCalibrationSourceSchema
+>;
+export type TaskCalibrationObservation = z.infer<
+  typeof TaskCalibrationObservationSchema
+>;
+export type TaskCalibrationOptions = z.input<
+  typeof TaskCalibrationOptionsSchema
+>;
+export type TaskCalibrationGroup = z.infer<
+  typeof TaskCalibrationGroupSchema
+>;
+export type TaskCalibrationReport = z.infer<
+  typeof TaskCalibrationReportSchema
+>;
+
+function stableSum(values: readonly number[]): number {
+  const sorted = [...values].sort((left, right) => left - right);
+  let total = 0;
+  let compensation = 0;
+
+  for (const value of sorted) {
+    const next = total + value;
+    compensation += Math.abs(total) >= Math.abs(value)
+      ? (total - next) + value
+      : (value - next) + total;
+    total = next;
+  }
+
+  return total + compensation;
+}
+
+function compareCodeUnits(left: string, right: string): number {
+  if (left < right) return -1;
+  if (left > right) return 1;
+  return 0;
+}
+
+export function aggregateTaskCalibration(
+  observations: readonly unknown[],
+  options: TaskCalibrationOptions = {},
+): TaskCalibrationReport {
+  const parsedObservations = TaskCalibrationObservationListSchema.parse(
+    observations,
+  );
+  const { minimum_sample_count } = TaskCalibrationOptionsSchema.parse(options);
+  const grouped = new Map<string, TaskCalibrationObservation[]>();
+
+  for (const observation of parsedObservations) {
+    const key = JSON.stringify([
+      observation.task_type,
+      observation.source.provider,
+      observation.source.model,
+    ]);
+    const group = grouped.get(key);
+    if (group) {
+      group.push(observation);
+    } else {
+      grouped.set(key, [observation]);
+    }
+  }
+
+  const groups = [...grouped.values()].map((group) => {
+    const sample_count = group.length;
+    const correctCount = group.reduce(
+      (total, observation) => total + Number(observation.correct),
+      0,
+    );
+    const confidenceTotal = stableSum(
+      group.map((observation) => observation.confidence),
+    );
+    const brierTotal = stableSum(
+      group.map((observation) => {
+        const outcome = Number(observation.correct);
+        return (observation.confidence - outcome) ** 2;
+      }),
+    );
+    const accuracy = stableMetric(correctCount / sample_count);
+    const mean_confidence = stableMetric(confidenceTotal / sample_count);
+
+    return {
+      task_type: group[0].task_type,
+      source: group[0].source,
+      sample_count,
+      accuracy,
+      mean_confidence,
+      brier_score: stableMetric(brierTotal / sample_count),
+      mean_calibration_bias: stableMetric(mean_confidence - accuracy),
+      sample_status: sample_count < minimum_sample_count
+        ? "insufficient" as const
+        : "sufficient" as const,
+    };
+  }).sort((left, right) =>
+    compareCodeUnits(left.task_type, right.task_type) ||
+    compareCodeUnits(left.source.provider, right.source.provider) ||
+    compareCodeUnits(left.source.model, right.source.model)
+  );
+
+  return TaskCalibrationReportSchema.parse({
+    schema_version: "quorum-router.calibration-by-task.v1",
+    advisory_only: true,
+    minimum_sample_count,
+    groups,
+  });
+}

package/templates/basic/src/calibration_demo.ts

@@ -0,0 +1,25 @@
+import { aggregateTaskCalibration } from "./calibration.ts";
+
+const evaluated_at = "2026-07-13T00:00:00Z";
+const report = aggregateTaskCalibration([
+  {
+    observation_id: "demo-1",
+    task_type: "code-review",
+    source: { provider: "OpenAI", model: "example-model" },
+    evaluation_basis: "caller_attested_external_ground_truth",
+    correct: true,
+    confidence: 0.8,
+    evaluated_at,
+  },
+  {
+    observation_id: "demo-2",
+    task_type: "code-review",
+    source: { provider: "OpenAI", model: "example-model" },
+    evaluation_basis: "caller_attested_external_ground_truth",
+    correct: false,
+    confidence: 0.6,
+    evaluated_at,
+  },
+], { minimum_sample_count: 2 });
+
+console.log(JSON.stringify(report, null, 2));

package/templates/basic/src/cli.ts

@@ -11,6 +11,11 @@ import {
 } from "./model_inventory.ts";
 import { redact, summarize } from "./redact.ts";
 import { parseAuthMode } from "./schema.ts";
+import {
+  auditRouteOutcome,
+  preflightRequiredSupabaseAudit,
+  runSupabaseStatus,
+} from "./supabase.ts";
 import { buildTrace, writeTrace } from "./trace.ts";
 
 const DEFAULT_PROMPT = "Review this README for risky claims.";
@@ -22,6 +27,7 @@ type CommandName =
   | "auth:logout"
   | "models:list"
   | "health"
+  | "supabase:status"
   | "route:once"
   | "best-route"
   | "agent-chat";
@@ -32,11 +38,12 @@ function commandName(): CommandName {
     command === "intake" || command === "auth:status" ||
     command === "auth:login" || command === "auth:logout" ||
     command === "models:list" || command === "health" ||
+    command === "supabase:status" ||
     command === "route:once" || command === "best-route" ||
     command === "agent-chat"
   ) return command;
   throw new Error(
-    "usage: deno task intake|auth:status|auth:login|auth:logout|models:list|health|route:once|best-route|agent-chat",
+    "usage: deno task intake|auth:status|auth:login|auth:logout|models:list|health|supabase:status|route:once|best-route|agent-chat",
   );
 }
 
@@ -101,10 +108,13 @@ try {
   else if (command === "auth:logout") await runAuthLogout();
   else if (command === "models:list") await runModelsList();
   else if (command === "health") await runHealth();
+  else if (command === "supabase:status") await runSupabaseStatus();
   else if (command === "route:once") {
+    await preflightRequiredSupabaseAudit();
     const { results, tracePath, trace } = await invokeSelected(
       promptFromArgs(),
     );
+    await auditRouteOutcome(trace);
     console.log("QuorumRouter route:once");
     console.log(`provider: ${results[0].provider}`);
     console.log(`model: ${results[0].model}`);
@@ -116,7 +126,9 @@ try {
     console.log(`final: ${summarize(results[0].response_summary, 500)}`);
     console.log(`trace: ${tracePath}`);
   } else if (command === "best-route") {
-    const { results, tracePath } = await runBestRoute(promptFromArgs());
+    await preflightRequiredSupabaseAudit();
+    const { results, tracePath, trace } = await runBestRoute(promptFromArgs());
+    await auditRouteOutcome(trace);
     console.log("QuorumRouter best-route");
     console.log(`models_called: ${results.length}`);
     console.log(`trace: ${tracePath}`);