@swissspidy/belay-core 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -1,3 +1,48 @@
-# Temporary Holding Version
+# @swissspidy/belay-core
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Answer AI tasks with a small on-device model, and call a cloud model only for the inputs it is
+unsure about. `@swissspidy/belay-core` has the task, the cascade, confidence handling, calibration files,
+cloud adapters and savings telemetry. It has no runtime dependencies.
+
+Pair it with [`@swissspidy/belay-web`](https://www.npmjs.com/package/@swissspidy/belay-web) for the on-device runners
+(Chrome's Classifier and Prompt APIs), and calibrate the threshold on your own data with
+[`@swissspidy/belay-calibrate`](https://www.npmjs.com/package/@swissspidy/belay-calibrate).
+
+> **Status:** experimental (0.x). The API may change between minor versions.
+
+## Install
+
+```sh
+npm install @swissspidy/belay-core @swissspidy/belay-web
+```
+
+## Usage
+
+```ts
+import { task, fetchAdapter } from '@swissspidy/belay-core';
+import { classifierApi } from '@swissspidy/belay-web';
+
+const triage = task({
+  name: 'ticket-triage',
+  schema: { type: 'categorical', options: ['bug', 'billing', 'feature', 'other'] as const },
+  local: classifierApi(),
+  cloud: fetchAdapter({ url: '/api/belay' }), // your backend calls the LLM
+  threshold: 0.8, // or: calibration: '/belay.calibration.json'
+});
+
+const result = await triage.run(ticketText, { signal });
+// { value: 'billing', confidence: 0.93, source: 'local', escalationReason: null, ... }
+```
+
+The local answer is accepted if and only if `confidence >= threshold`. Otherwise the task
+escalates to the cloud, subject to your privacy policy (`redact`, `consent`, or `never`).
+
+## Documentation
+
+See the [project README](https://github.com/swissspidy/belay#readme) for schemas, confidence,
+privacy, cloud adapters, calibration and telemetry, and the
+[ADRs](https://github.com/swissspidy/belay/tree/main/docs/adr) for the design decisions.
+
+## License
+
+Apache-2.0

package/package.json

@@ -1,6 +1,44 @@
 {
   "name": "@swissspidy/belay-core",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "Confidence-based escalation from on-device to cloud AI: tasks, cascade logic, confidence, calibration types.",
+  "license": "Apache-2.0",
+  "author": "Pascal Birchler",
+  "homepage": "https://github.com/swissspidy/belay/tree/main/packages/core#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/swissspidy/belay.git",
+    "directory": "packages/core"
+  },
+  "bugs": {
+    "url": "https://github.com/swissspidy/belay/issues"
+  },
+  "keywords": [
+    "ai",
+    "on-device",
+    "cascade",
+    "llm",
+    "classification",
+    "calibration",
+    "confidence",
+    "escalation"
+  ],
+  "type": "module",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "scripts": {
+    "build": "tsc -b"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/calibration.ts

@@ -0,0 +1,65 @@
+import { BelayError } from './errors.js';
+import type { CalibrationFile } from './types.js';
+
+export const CALIBRATION_VERSION = 1;
+
+const isNum = (v: unknown): v is number => typeof v === 'number' && Number.isFinite(v);
+const isProb = (v: unknown): v is number => isNum(v) && v >= 0 && v <= 1;
+/** A threshold above 1 keeps nothing local: every run escalates. */
+const isThreshold = (v: unknown): v is number => isNum(v) && v >= 0;
+
+/**
+ * Validates an unknown value as a v1 calibration file. When `task` is given, also checks
+ * that the file belongs to that task. Throws `BelayError('invalid-calibration')`.
+ */
+export function parseCalibration(data: unknown, task?: string): CalibrationFile {
+  const fail = (msg: string): never => {
+    throw new BelayError('invalid-calibration', `Invalid calibration file: ${msg}`);
+  };
+  if (!data || typeof data !== 'object') fail('not an object');
+  const c = data as Partial<CalibrationFile>;
+  if (c.version !== CALIBRATION_VERSION) fail(`unsupported version ${String(c.version)}`);
+  if (typeof c.task !== 'string') fail('missing task');
+  if (task !== undefined && c.task !== task) fail(`file is for task "${c.task}", not "${task}"`);
+  if (typeof c.schemaFingerprint !== 'string') fail('missing schemaFingerprint');
+  if (!isThreshold(c.threshold)) fail('threshold must be a number ≥ 0');
+  if (c.thresholds !== undefined) {
+    if (typeof c.thresholds !== 'object' || c.thresholds === null) fail('thresholds must be an object');
+    for (const [label, t] of Object.entries(c.thresholds as object)) {
+      if (!isThreshold(t)) fail(`thresholds["${label}"] must be a number ≥ 0`);
+    }
+  }
+  if (!c.expected || !isProb(c.expected.localShare) || !isProb(c.expected.accuracy)) {
+    fail('expected.accuracy and expected.localShare must be numbers in [0, 1]');
+  }
+  if (!Array.isArray(c.curve)) fail('curve must be an array');
+  if (!c.confidenceHistogram || !Array.isArray(c.confidenceHistogram.edges) || !Array.isArray(c.confidenceHistogram.counts)) {
+    fail('confidenceHistogram must have edges and counts');
+  } else if (c.confidenceHistogram.edges.length !== c.confidenceHistogram.counts.length + 1) {
+    fail('confidenceHistogram.edges must have one more entry than counts');
+  }
+  return c as CalibrationFile;
+}
+
+/** Threshold for a local top label: the per-label override if present, else the global one. */
+export function thresholdFor(calibration: Pick<CalibrationFile, 'threshold' | 'thresholds'>, label?: string): number {
+  if (label !== undefined && calibration.thresholds && Object.hasOwn(calibration.thresholds, label)) {
+    return calibration.thresholds[label]!;
+  }
+  return calibration.threshold;
+}
+
+export async function loadCalibration(
+  source: CalibrationFile | string | URL | (() => Promise<CalibrationFile>),
+  task?: string,
+): Promise<CalibrationFile> {
+  if (typeof source === 'function') return parseCalibration(await source(), task);
+  if (typeof source === 'string' || source instanceof URL) {
+    const res = await fetch(source);
+    if (!res.ok) {
+      throw new BelayError('invalid-calibration', `Failed to fetch calibration ${String(source)}: HTTP ${res.status}`);
+    }
+    return parseCalibration(await res.json(), task);
+  }
+  return parseCalibration(source, task);
+}

package/src/cloud.ts

@@ -0,0 +1,72 @@
+import type { CloudOutput, CloudRequest, CloudRunner, CloudUsage, TaskSchema } from './types.js';
+
+export type CloudFn<S extends TaskSchema = TaskSchema> = (
+  request: CloudRequest<S>,
+  options: {
+    signal?: AbortSignal;
+    /** Report the call's token usage, so calibration and `savingsMeter()` can measure its cost. */
+    reportUsage: (usage: CloudUsage | CloudUsage[]) => void;
+  },
+) => Promise<unknown>;
+
+/**
+ * Wraps a plain async function as a cloud runner. The function returns the raw output:
+ * the bare value, the `{ value }` wrapper, or a JSON string of either. The task validates it.
+ * Token usage goes through `reportUsage`, since the output itself may be a `{ value }` object.
+ *
+ * ```ts
+ * cloudAdapter(async ({ instruction, input, jsonSchema }, { signal }) => {
+ *   const res = await fetch('/api/classify', { method: 'POST', body: JSON.stringify({ instruction, input, jsonSchema }), signal });
+ *   return res.json();
+ * });
+ * ```
+ */
+export function cloudAdapter<S extends TaskSchema = TaskSchema>(
+  fn: CloudFn<S>,
+  options: { id?: string } = {},
+): CloudRunner<S> {
+  return {
+    id: options.id ?? 'cloud',
+    async run(request, opts): Promise<CloudOutput> {
+      let usage: CloudUsage | CloudUsage[] | undefined;
+      const value = await fn(request, { ...opts, reportUsage: (u) => void (usage = u) });
+      return usage ? { value, usage } : { value };
+    },
+  };
+}
+
+export interface FetchAdapterOptions<S extends TaskSchema = TaskSchema> {
+  /** Your backend endpoint. Never put provider API keys in the browser. */
+  url: string | URL;
+  headers?: Record<string, string>;
+  /** Builds the request body. Defaults to the JSON-serialized request (functions dropped). */
+  body?: (request: CloudRequest<S>) => unknown;
+  /** Extracts the output from the parsed JSON response. Defaults to the whole response. */
+  select?: (json: unknown) => unknown;
+  /** Extracts token usage from the parsed JSON response, if your backend passes it through. */
+  usage?: (json: unknown) => CloudUsage | CloudUsage[] | undefined;
+  id?: string;
+  fetch?: typeof fetch;
+}
+
+/** A cloud runner that POSTs the request as JSON to your endpoint. */
+export function fetchAdapter<S extends TaskSchema = TaskSchema>(options: FetchAdapterOptions<S>): CloudRunner<S> {
+  const doFetch = options.fetch ?? globalThis.fetch.bind(globalThis);
+  return cloudAdapter<S>(
+    async (request, { signal, reportUsage }) => {
+      const body = options.body ? options.body(request) : request;
+      const res = await doFetch(options.url, {
+        method: 'POST',
+        headers: { 'content-type': 'application/json', ...options.headers },
+        body: JSON.stringify(body),
+        ...(signal ? { signal } : {}),
+      });
+      if (!res.ok) throw new Error(`Cloud endpoint responded with HTTP ${res.status}`);
+      const json: unknown = await res.json();
+      const usage = options.usage?.(json);
+      if (usage) reportUsage(usage);
+      return options.select ? options.select(json) : json;
+    },
+    { id: options.id ?? 'fetch' },
+  );
+}

package/src/confidence.ts

@@ -0,0 +1,37 @@
+import type { ProbabilityEntry } from './types.js';
+
+/** Probability of the top option: the confidence signal for classifier tasks. */
+export function topProbability(probabilities: readonly ProbabilityEntry[]): number {
+  let top = 0;
+  for (const p of probabilities) if (p.probability > top) top = p.probability;
+  return clamp01(top);
+}
+
+export interface ConfidenceSignals {
+  /** Whether the output passed schema validation. */
+  valid: boolean;
+  /** Confidence reported by the local runner itself, if any. */
+  runner?: number | undefined;
+  /** P(correct) from the classifier-as-judge, if any. */
+  judge?: number | undefined;
+}
+
+/**
+ * Combines the signals into one confidence (ADR 0001, "Confidence signal"):
+ *
+ * - schema validation is a hard gate: an invalid output has confidence 0;
+ * - with only one of `runner` / `judge`, that value is the confidence;
+ * - with both, the minimum (the more pessimistic signal wins);
+ * - with neither, `undefined`: the task cannot decide and reports a configuration error.
+ */
+export function combineConfidence(signals: ConfidenceSignals): number | undefined {
+  if (!signals.valid) return 0;
+  const values = [signals.runner, signals.judge].filter((v): v is number => typeof v === 'number');
+  if (values.length === 0) return undefined;
+  return clamp01(Math.min(...values));
+}
+
+export function clamp01(n: number): number {
+  if (Number.isNaN(n)) return 0;
+  return n < 0 ? 0 : n > 1 ? 1 : n;
+}

package/src/cost.ts

@@ -0,0 +1,148 @@
+import type { BelayEvent, CloudUsage, PriceTable, TokenPrices } from './types.js';
+
+/** Normalizes a runner's usage report to a list of per-model parts. */
+export function usageParts(usage: CloudUsage | readonly CloudUsage[] | undefined): CloudUsage[] {
+  if (!usage) return [];
+  return Array.isArray(usage) ? [...usage] : [usage as CloudUsage];
+}
+
+function pricesFor(prices: PriceTable | TokenPrices, model: string | undefined): TokenPrices {
+  if (typeof (prices as TokenPrices).input === 'number') return prices as TokenPrices;
+  const table = prices as PriceTable;
+  if (model !== undefined && Object.hasOwn(table, model)) return table[model]!;
+  const known = Object.keys(table).join(', ') || 'none';
+  throw new Error(model === undefined ? `usage names no model and the price table has several (${known})` : `no price for model "${model}" (prices for: ${known})`);
+}
+
+/**
+ * Cost of one cloud call from its token usage. Prices are per million tokens. A price table is
+ * keyed by the model id the provider reports, so a call that a fallback model served is priced
+ * at that model's rates. Throws when a model has no price: guessing would make every saving wrong.
+ */
+export function costOf(usage: CloudUsage | readonly CloudUsage[], prices: PriceTable | TokenPrices): number {
+  let total = 0;
+  for (const part of usageParts(usage)) {
+    const p = pricesFor(prices, part.model);
+    total +=
+      part.inputTokens * p.input +
+      part.outputTokens * p.output +
+      (part.cacheReadInputTokens ?? 0) * (p.cacheRead ?? p.input) +
+      (part.cacheWriteInputTokens ?? 0) * (p.cacheWrite ?? p.input);
+  }
+  return total / 1e6;
+}
+
+export interface SavingsMeterOptions {
+  /** Per-token prices, to measure the cost of each cloud call from the usage its runner reports. */
+  prices?: PriceTable | TokenPrices;
+  /**
+   * Cost of one cloud call, used when calls report no usage, and to value the calls the cascade
+   * avoided before any cloud call has been measured. A calibration file's `cost.cloudPerRun` fits.
+   */
+  cloudPerRun?: number;
+  currency?: string;
+}
+
+export interface SavingsSummary {
+  currency: string | null;
+  runs: number;
+  /** Answered on device because the local confidence met the threshold. */
+  local: number;
+  /** Answered by the cloud runner. */
+  cloud: number;
+  /** Answered on device only because escalation was blocked (policy, consent, cloud error). */
+  blocked: number;
+  /** Spent on cloud calls: measured where usage was reported, otherwise `cloudPerRun` each. */
+  cloudSpend: number | null;
+  /** Mean cost of one cloud call (measured when possible). */
+  costPerCloudRun: number | null;
+  /** What the `local` runs would have cost in the cloud: `local × costPerCloudRun`. */
+  saved: number | null;
+  /** `saved / (saved + cloudSpend)`: the share of the cloud-only bill the cascade avoided. */
+  savedShare: number | null;
+  /** How many cloud calls were priced from reported usage. */
+  measuredCalls: number;
+}
+
+export interface SavingsMeter {
+  /** Pass as the task's `onEvent` (or call it from yours). */
+  readonly onEvent: (event: BelayEvent) => void;
+  /** Totals so far, for all tasks or one. */
+  summary(task?: string): SavingsSummary;
+  reset(): void;
+}
+
+interface Tally {
+  runs: number;
+  local: number;
+  cloud: number;
+  blocked: number;
+  measuredCalls: number;
+  measuredSpend: number;
+}
+
+/**
+ * Counts what the cascade saves in production. Only runs that stayed on device because the
+ * local answer was confident count as saved; runs kept local because escalation was blocked
+ * are reported separately. Unmeasured calls, and the value of avoided ones, use the mean
+ * measured cost per call, falling back to `cloudPerRun`.
+ *
+ * ```ts
+ * const meter = savingsMeter({ prices, cloudPerRun: calibration.cost?.cloudPerRun });
+ * const triage = task({ ..., onEvent: meter.onEvent });
+ * meter.summary(); // { runs, local, cloud, cloudSpend, saved, savedShare, ... }
+ * ```
+ */
+export function savingsMeter(options: SavingsMeterOptions = {}): SavingsMeter {
+  const tallies = new Map<string, Tally>();
+  const tallyFor = (task: string): Tally => {
+    let t = tallies.get(task);
+    if (!t) tallies.set(task, (t = { runs: 0, local: 0, cloud: 0, blocked: 0, measuredCalls: 0, measuredSpend: 0 }));
+    return t;
+  };
+
+  const onEvent = (event: BelayEvent): void => {
+    if (event.type !== 'run') return;
+    const t = tallyFor(event.task);
+    t.runs++;
+    if (event.source === 'cloud') {
+      t.cloud++;
+      if (options.prices && event.cloudUsage?.length) {
+        t.measuredSpend += costOf(event.cloudUsage, options.prices);
+        t.measuredCalls++;
+      }
+    } else if (event.escalationBlocked) {
+      t.blocked++;
+    } else if (event.escalationReason === null) {
+      t.local++;
+    }
+  };
+
+  const summary = (task?: string): SavingsSummary => {
+    const parts = task === undefined ? [...tallies.values()] : tallies.has(task) ? [tallies.get(task)!] : [];
+    const sum = (key: keyof Tally) => parts.reduce((a, t) => a + t[key], 0);
+    const runs = sum('runs');
+    const local = sum('local');
+    const cloud = sum('cloud');
+    const measuredCalls = sum('measuredCalls');
+    const measuredSpend = sum('measuredSpend');
+    const costPerCloudRun = measuredCalls ? measuredSpend / measuredCalls : (options.cloudPerRun ?? null);
+    const cloudSpend = costPerCloudRun === null ? null : measuredSpend + (cloud - measuredCalls) * costPerCloudRun;
+    const saved = costPerCloudRun === null ? null : local * costPerCloudRun;
+    const denominator = saved === null || cloudSpend === null ? 0 : saved + cloudSpend;
+    return {
+      currency: options.currency ?? null,
+      runs,
+      local,
+      cloud,
+      blocked: sum('blocked'),
+      cloudSpend,
+      costPerCloudRun,
+      saved,
+      savedShare: denominator > 0 ? saved! / denominator : null,
+      measuredCalls,
+    };
+  };
+
+  return { onEvent, summary, reset: () => tallies.clear() };
+}

package/src/errors.ts

@@ -0,0 +1,28 @@
+export type BelayErrorCode =
+  /** No local output and escalation was not possible or not permitted. */
+  | 'no-result'
+  /** The cloud runner threw or returned an output that does not match the schema. */
+  | 'cloud-error'
+  | 'cloud-invalid-output'
+  /** The task definition is invalid. */
+  | 'invalid-task'
+  /** The calibration file is malformed or does not belong to this task. */
+  | 'invalid-calibration';
+
+export class BelayError extends Error {
+  override readonly name = 'BelayError';
+  readonly code: BelayErrorCode;
+
+  constructor(code: BelayErrorCode, message: string, options?: { cause?: unknown }) {
+    super(message, options);
+    this.code = code;
+  }
+}
+
+/**
+ * Thrown by a local runner from `run()` when the model is not usable (e.g. it would need
+ * a download). The cascade treats it as escalation reason `local-unavailable`.
+ */
+export class LocalUnavailableError extends Error {
+  override readonly name = 'LocalUnavailableError';
+}