@stll/ai-catalog 0.3.2 → 0.5.0

package/dist/benchmark-frontier.d.ts

@@ -0,0 +1,78 @@
+import { D as ReasoningEffort, a as BYOKProvider } from "./index-D5gnRT1S.js";
+//#region src/benchmark-frontier.d.ts
+export type ModelBenchmarkMeasurement = {
+  sourceModelId: string;
+  reasoningEffort: ReasoningEffort | null;
+  rating: number;
+  ratingLower: number;
+  ratingUpper: number;
+};
+export declare const MODEL_BENCHMARK_AVAILABILITIES: readonly ["available", "provider_unconfigured"];
+export type ModelBenchmarkAvailability = (typeof MODEL_BENCHMARK_AVAILABILITIES)[number];
+/** One offered route with its measured variants and list price. */
+export type BenchmarkModelOption = {
+  availability: ModelBenchmarkAvailability;
+  costPerTypicalCallUsd: number | null;
+  measurements: readonly ModelBenchmarkMeasurement[];
+  /** Brand of the model's creator; differs from `provider` for aggregators. */
+  iconProvider: BYOKProvider;
+  provider: BYOKProvider;
+  value: string;
+};
+export type ModelBenchmarkPoint = {
+  availability: ModelBenchmarkAvailability;
+  costPerTypicalCallUsd: number;
+  id: string;
+  isDirectRoute: boolean;
+  modelValue: string;
+  reasoningEffort: ReasoningEffort | null;
+  rating: number;
+  ratingLower: number;
+  ratingUpper: number;
+  sourceModelId: string;
+};
+export declare const buildModelBenchmarkPoints: (options: readonly BenchmarkModelOption[]) => ModelBenchmarkPoint[];
+/**
+ * One point per Arena row: several routes (first-party, aggregator) can serve
+ * the same measured variant. Prefer a route the organization can use, then
+ * the model creator's own API, then the lexically first id for determinism.
+ */
+export declare const getUniqueModelBenchmarkPoints: <TPoint extends ModelBenchmarkPoint>(points: readonly TPoint[]) => TPoint[];
+/** Pareto dominance on point estimates: no costlier and no worse, one strictly. */
+export declare const dominatesBenchmarkPoint: (candidate: ModelBenchmarkPoint, target: ModelBenchmarkPoint) => boolean;
+/** No costlier, and better beyond both confidence intervals. */
+export declare const statisticallyDominatesBenchmarkPoint: (candidate: ModelBenchmarkPoint, target: ModelBenchmarkPoint) => boolean;
+/** Non-dominated points, ordered by cost then rating. */
+export declare const getParetoFrontier: <TPoint extends ModelBenchmarkPoint>(points: readonly TPoint[]) => TPoint[];
+export declare const MODEL_BENCHMARK_CLASSIFICATIONS: readonly ["frontier", "near_frontier", "dominated"];
+export type ModelBenchmarkClassification = (typeof MODEL_BENCHMARK_CLASSIFICATIONS)[number];
+/**
+ * Frontier: not dominated on point estimates. Dominated: some point is no
+ * costlier and its lower bound clears this point's upper bound. Everything
+ * else is within uncertainty of the frontier.
+ */
+export declare const classifyBenchmarkPoints: (points: readonly ModelBenchmarkPoint[]) => ReadonlyMap<string, ModelBenchmarkClassification>;
+export declare const PREMIUM_COST_MULTIPLIER = 3;
+/** Points costing at least three times the median cost per typical call. */
+export declare const getPremiumBenchmarkPointIds: (points: readonly ModelBenchmarkPoint[]) => ReadonlySet<string>;
+export type ClassifiedBenchmarkMeasurement = ModelBenchmarkMeasurement & {
+  classification: ModelBenchmarkClassification;
+  premium: boolean;
+};
+export type ModelBenchmarkTradeoff = {
+  dominatedReasoningEfforts: readonly (ReasoningEffort | null)[];
+  premiumReasoningEfforts: readonly (ReasoningEffort | null)[];
+  type: "dominated" | "near_frontier" | "pareto" | "unmeasured";
+};
+export type ClassifiedBenchmarkModelOption<TOption extends BenchmarkModelOption> = Omit<TOption, "measurements"> & {
+  measurements: ClassifiedBenchmarkMeasurement[];
+  tradeoff: ModelBenchmarkTradeoff;
+};
+export declare const getModelBenchmarkTradeoff: (measurements: readonly ClassifiedBenchmarkMeasurement[]) => ModelBenchmarkTradeoff;
+/**
+ * Classify every measured variant once over the whole deduplicated set, then
+ * project the result back onto each route. A route that shares an Arena row
+ * with the preferred route inherits that row's classification.
+ */
+export declare const classifyBenchmarkModelOptions: <TOption extends BenchmarkModelOption>(options: readonly TOption[]) => ClassifiedBenchmarkModelOption<TOption>[];
+//#endregion

package/dist/benchmark-frontier.js

@@ -0,0 +1,125 @@
+//#region src/benchmark-frontier.ts
+const MODEL_BENCHMARK_AVAILABILITIES = ["available", "provider_unconfigured"];
+const buildModelBenchmarkPoints = (options) => options.flatMap((option) => {
+	const cost = option.costPerTypicalCallUsd;
+	if (cost === null) return [];
+	return option.measurements.map((measurement) => ({
+		availability: option.availability,
+		costPerTypicalCallUsd: cost,
+		id: `${option.value}::${measurement.sourceModelId}`,
+		isDirectRoute: option.provider === option.iconProvider,
+		modelValue: option.value,
+		reasoningEffort: measurement.reasoningEffort,
+		rating: measurement.rating,
+		ratingLower: measurement.ratingLower,
+		ratingUpper: measurement.ratingUpper,
+		sourceModelId: measurement.sourceModelId
+	}));
+});
+const AVAILABILITY_PREFERENCE = {
+	available: 2,
+	provider_unconfigured: 0
+};
+const pointPreference = (point) => AVAILABILITY_PREFERENCE[point.availability] + (point.isDirectRoute ? 1 : 0);
+/**
+* One point per Arena row: several routes (first-party, aggregator) can serve
+* the same measured variant. Prefer a route the organization can use, then
+* the model creator's own API, then the lexically first id for determinism.
+*/
+const getUniqueModelBenchmarkPoints = (points) => {
+	const unique = /* @__PURE__ */ new Map();
+	for (const point of points) {
+		const existing = unique.get(point.sourceModelId);
+		if (existing === void 0) {
+			unique.set(point.sourceModelId, point);
+			continue;
+		}
+		const preference = pointPreference(point) - pointPreference(existing);
+		if (preference > 0 || preference === 0 && point.id < existing.id) unique.set(point.sourceModelId, point);
+	}
+	return [...unique.values()];
+};
+/** Pareto dominance on point estimates: no costlier and no worse, one strictly. */
+const dominatesBenchmarkPoint = (candidate, target) => candidate.costPerTypicalCallUsd <= target.costPerTypicalCallUsd && candidate.rating >= target.rating && (candidate.costPerTypicalCallUsd < target.costPerTypicalCallUsd || candidate.rating > target.rating);
+/** No costlier, and better beyond both confidence intervals. */
+const statisticallyDominatesBenchmarkPoint = (candidate, target) => candidate.costPerTypicalCallUsd <= target.costPerTypicalCallUsd && candidate.ratingLower > target.ratingUpper;
+const compareIds = (left, right) => {
+	if (left === right) return 0;
+	return left < right ? -1 : 1;
+};
+/** Non-dominated points, ordered by cost then rating. */
+const getParetoFrontier = (points) => points.filter((point) => !points.some((candidate) => dominatesBenchmarkPoint(candidate, point))).toSorted((left, right) => left.costPerTypicalCallUsd - right.costPerTypicalCallUsd || left.rating - right.rating || compareIds(left.id, right.id));
+const MODEL_BENCHMARK_CLASSIFICATIONS = [
+	"frontier",
+	"near_frontier",
+	"dominated"
+];
+/**
+* Frontier: not dominated on point estimates. Dominated: some point is no
+* costlier and its lower bound clears this point's upper bound. Everything
+* else is within uncertainty of the frontier.
+*/
+const classifyBenchmarkPoints = (points) => {
+	const frontierIds = new Set(getParetoFrontier(points).map(({ id }) => id));
+	const classifications = /* @__PURE__ */ new Map();
+	for (const point of points) {
+		if (frontierIds.has(point.id)) {
+			classifications.set(point.id, "frontier");
+			continue;
+		}
+		classifications.set(point.id, points.some((candidate) => statisticallyDominatesBenchmarkPoint(candidate, point)) ? "dominated" : "near_frontier");
+	}
+	return classifications;
+};
+const PREMIUM_COST_MULTIPLIER = 3;
+/** Points costing at least three times the median cost per typical call. */
+const getPremiumBenchmarkPointIds = (points) => {
+	const costs = points.map(({ costPerTypicalCallUsd }) => costPerTypicalCallUsd).toSorted((left, right) => left - right);
+	const middle = Math.floor(costs.length / 2);
+	const upperMiddle = costs.at(middle);
+	if (upperMiddle === void 0) return /* @__PURE__ */ new Set();
+	const threshold = (costs.length % 2 === 0 ? ((costs.at(middle - 1) ?? upperMiddle) + upperMiddle) / 2 : upperMiddle) * 3;
+	return new Set(points.filter(({ costPerTypicalCallUsd }) => costPerTypicalCallUsd >= threshold).map(({ id }) => id));
+};
+const tradeoffType = (measurements) => {
+	if (measurements.length === 0) return "unmeasured";
+	const classifications = new Set(measurements.map(({ classification }) => classification));
+	if (classifications.has("frontier")) return "pareto";
+	if (classifications.has("near_frontier")) return "near_frontier";
+	return "dominated";
+};
+const getModelBenchmarkTradeoff = (measurements) => ({
+	dominatedReasoningEfforts: measurements.filter(({ classification }) => classification === "dominated").map(({ reasoningEffort }) => reasoningEffort),
+	premiumReasoningEfforts: measurements.filter(({ premium }) => premium).map(({ reasoningEffort }) => reasoningEffort),
+	type: tradeoffType(measurements)
+});
+/**
+* Classify every measured variant once over the whole deduplicated set, then
+* project the result back onto each route. A route that shares an Arena row
+* with the preferred route inherits that row's classification.
+*/
+const classifyBenchmarkModelOptions = (options) => {
+	const points = getUniqueModelBenchmarkPoints(buildModelBenchmarkPoints(options));
+	const classifications = classifyBenchmarkPoints(points);
+	const premiumIds = getPremiumBenchmarkPointIds(points);
+	const bySourceModelId = new Map(points.map((point) => [point.sourceModelId, point]));
+	return options.map((option) => {
+		const measurements = option.measurements.flatMap((measurement) => {
+			const point = bySourceModelId.get(measurement.sourceModelId);
+			const classification = point === void 0 ? void 0 : classifications.get(point.id);
+			if (point === void 0 || classification === void 0) return [];
+			return [{
+				...measurement,
+				classification,
+				premium: premiumIds.has(point.id)
+			}];
+		});
+		return {
+			...option,
+			measurements,
+			tradeoff: getModelBenchmarkTradeoff(measurements)
+		};
+	});
+};
+//#endregion
+export { MODEL_BENCHMARK_AVAILABILITIES, MODEL_BENCHMARK_CLASSIFICATIONS, PREMIUM_COST_MULTIPLIER, buildModelBenchmarkPoints, classifyBenchmarkModelOptions, classifyBenchmarkPoints, dominatesBenchmarkPoint, getModelBenchmarkTradeoff, getParetoFrontier, getPremiumBenchmarkPointIds, getUniqueModelBenchmarkPoints, statisticallyDominatesBenchmarkPoint };