@danypops/jittor 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/package.json +4 -3
  2. package/src/adapters/artificial-analysis-direct-source.ts +42 -11
  3. package/src/adapters/lmarena-hf-source.ts +37 -15
  4. package/src/adapters/metric-benchmark-store.ts +29 -18
  5. package/src/adapters/openrouter-benchmark-source.ts +75 -19
  6. package/src/adapters/openrouter-design-arena-source.ts +36 -19
  7. package/src/adapters/sqlite-metric-store.ts +48 -25
  8. package/src/adapters/sqlite-session-identity-store.ts +13 -7
  9. package/src/cli-commands/benchmarks.ts +87 -22
  10. package/src/cli-commands/compaction.ts +7 -2
  11. package/src/cli-commands/context.ts +10 -2
  12. package/src/cli-commands/metrics.ts +128 -31
  13. package/src/cli-commands/op.ts +6 -1
  14. package/src/cli-commands/route-args.ts +5 -1
  15. package/src/cli-commands/router.ts +61 -18
  16. package/src/cli-commands/service-daemon.ts +28 -12
  17. package/src/cli-commands/session.ts +15 -4
  18. package/src/cli-commands/support.ts +1 -1
  19. package/src/cli.ts +30 -23
  20. package/src/client.ts +1 -1
  21. package/src/constants.ts +1 -1
  22. package/src/daemon.ts +44 -24
  23. package/src/domain/benchmark.ts +72 -40
  24. package/src/domain/codex-recovery.ts +34 -25
  25. package/src/domain/context-hub.ts +35 -26
  26. package/src/domain/context-telemetry.ts +106 -35
  27. package/src/domain/metric.ts +11 -11
  28. package/src/domain/model-observation.ts +139 -55
  29. package/src/domain/model-ranking-service.ts +13 -3
  30. package/src/domain/model-ranking.ts +126 -60
  31. package/src/domain/task-cost.ts +52 -11
  32. package/src/domain/task-focus.ts +11 -8
  33. package/src/domain/usage.ts +2 -2
  34. package/src/index.ts +69 -69
  35. package/src/log.ts +7 -2
  36. package/src/operations/benchmark-operations.ts +1 -1
  37. package/src/operations/context-operations.ts +15 -6
  38. package/src/operations/metrics-operations.ts +63 -28
  39. package/src/operations/model-ranking-operations.ts +9 -2
  40. package/src/operations/router-operations.ts +8 -4
  41. package/src/operations/session-identity-operations.ts +1 -1
  42. package/src/operations/session-scope.ts +5 -3
  43. package/src/policy.ts +22 -17
  44. package/src/ports/benchmark-controller.ts +1 -5
  45. package/src/ports/metric-store.ts +1 -1
  46. package/src/providers/anthropic-contracts.ts +13 -3
  47. package/src/providers/codex-contracts.ts +60 -52
  48. package/src/providers/codex.ts +16 -19
  49. package/src/providers/google-vertex-budget-contracts.ts +24 -14
  50. package/src/providers/google-vertex-budget.ts +15 -13
  51. package/src/providers/google-vertex-contracts.ts +36 -24
  52. package/src/providers/openrouter-contracts.ts +49 -51
  53. package/src/providers/openrouter.ts +21 -15
  54. package/src/providers/telemetry-sources.ts +13 -10
  55. package/src/router.ts +92 -43
  56. package/src/service.ts +93 -32
  57. package/src/session-identity-service.ts +10 -2
  58. package/src/state.ts +4 -10
  59. package/src/vehicle-registration.ts +154 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@danypops/jittor",
3
- "version": "0.14.0",
3
+ "version": "0.15.0",
4
4
  "description": "Just-in-Time Token Optimizing Router for Pi -- supervised daemon, router policy, and CLI",
5
5
  "type": "module",
6
6
  "main": "./src/index.ts",
@@ -16,8 +16,9 @@
16
16
  "service:install": "bun src/cli.ts service install"
17
17
  },
18
18
  "dependencies": {
19
- "@danypops/vehicle-server": "^0.3.1",
20
- "@danypops/vehicle-client": "^0.1.1",
19
+ "@danypops/vehicle-core": "^0.10.0",
20
+ "@danypops/vehicle-server": "^0.13.0",
21
+ "@danypops/vehicle-client": "^0.5.1",
21
22
  "google-auth-library": "^10.9.0"
22
23
  },
23
24
  "devDependencies": {
@@ -1,5 +1,10 @@
1
1
  import { BENCHMARK_MAX_MODELS_PER_SOURCE, BENCHMARK_REFRESH_INTERVAL_MS, BENCHMARK_SOURCE_MAX_RESPONSE_BYTES } from "../constants.ts";
2
- import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
2
+ import {
3
+ type BenchmarkObservation,
4
+ type BenchmarkSourceSnapshot,
5
+ normalizeModelIdentity,
6
+ validateBenchmarkObservation,
7
+ } from "../domain/benchmark.ts";
3
8
  import type { BenchmarkSource } from "../ports/benchmark-source.ts";
4
9
  import { contractRecord } from "../providers/openrouter-contracts.ts";
5
10
 
@@ -36,17 +41,25 @@ export class ArtificialAnalysisDirectSource implements BenchmarkSource {
36
41
  const response = await this.transport(new Request(ENDPOINT, { headers: { "x-api-key": this.apiKey } }));
37
42
  if (!response.ok) throw new Error(`Artificial Analysis benchmarks failed with HTTP ${response.status}`);
38
43
  const text = await response.text();
39
- if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("Artificial Analysis benchmark response exceeds the size limit");
44
+ if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
45
+ throw new Error("Artificial Analysis benchmark response exceeds the size limit");
40
46
  let payload: unknown;
41
- try { payload = JSON.parse(text); } catch { throw new Error("Artificial Analysis benchmark response is not valid JSON"); }
47
+ try {
48
+ payload = JSON.parse(text);
49
+ } catch {
50
+ throw new Error("Artificial Analysis benchmark response is not valid JSON");
51
+ }
42
52
  const root = contractRecord(payload, "benchmark response");
43
- if (!Array.isArray(root["data"]) || root["data"].length > BENCHMARK_MAX_MODELS_PER_SOURCE) throw new Error("Artificial Analysis benchmark result count is invalid");
53
+ if (!Array.isArray(root.data) || root.data.length > BENCHMARK_MAX_MODELS_PER_SOURCE)
54
+ throw new Error("Artificial Analysis benchmark result count is invalid");
44
55
  const revision = String(retrievedAt);
45
- const observations = root["data"].flatMap((value): BenchmarkObservation[] => {
56
+ const observations = root.data.flatMap((value): BenchmarkObservation[] => {
46
57
  const row = contractRecord(value, "benchmark row");
47
- const creator = contractRecord(row["model_creator"], "model creator");
48
- const evaluations = contractRecord(row["evaluations"], "evaluations");
49
- const identity = normalizeModelIdentity(requiredText(creator["slug"], "creator slug"), requiredText(row["slug"], "model slug"), [`artificial-analysis/${requiredText(row["id"], "model id")}`]);
58
+ const creator = contractRecord(row.model_creator, "model creator");
59
+ const evaluations = contractRecord(row.evaluations, "evaluations");
60
+ const identity = normalizeModelIdentity(requiredText(creator.slug, "creator slug"), requiredText(row.slug, "model slug"), [
61
+ `artificial-analysis/${requiredText(row.id, "model id")}`,
62
+ ]);
50
63
  const provenance = {
51
64
  sourceId: SOURCE_ID,
52
65
  sourceType: "creator" as const,
@@ -64,14 +77,32 @@ export class ArtificialAnalysisDirectSource implements BenchmarkSource {
64
77
  const index = (field: string, dimension: string): void => {
65
78
  const raw = evaluations[field];
66
79
  if (raw === undefined || raw === null) return;
67
- observations.push(validateBenchmarkObservation({ model: identity, dimension, value: requiredNumber(raw, field), unit: "ratio", provenance, methodology }));
80
+ observations.push(
81
+ validateBenchmarkObservation({
82
+ model: identity,
83
+ dimension,
84
+ value: requiredNumber(raw, field),
85
+ unit: "ratio",
86
+ provenance,
87
+ methodology,
88
+ }),
89
+ );
68
90
  };
69
91
  index("artificial_analysis_coding_index", "quality-coding");
70
92
  index("artificial_analysis_intelligence_index", "quality-general");
71
93
  index("artificial_analysis_math_index", "quality-math");
72
- const ttft = row["median_time_to_first_token_seconds"];
94
+ const ttft = row.median_time_to_first_token_seconds;
73
95
  if (typeof ttft === "number" && Number.isFinite(ttft)) {
74
- observations.push(validateBenchmarkObservation({ model: identity, dimension: "latency", value: ttft * 1_000, unit: "milliseconds", provenance, methodology }));
96
+ observations.push(
97
+ validateBenchmarkObservation({
98
+ model: identity,
99
+ dimension: "latency",
100
+ value: ttft * 1_000,
101
+ unit: "milliseconds",
102
+ provenance,
103
+ methodology,
104
+ }),
105
+ );
75
106
  }
76
107
  return observations;
77
108
  });
@@ -1,5 +1,10 @@
1
1
  import { BENCHMARK_MAX_MODELS_PER_SOURCE, BENCHMARK_REFRESH_INTERVAL_MS, BENCHMARK_SOURCE_MAX_RESPONSE_BYTES } from "../constants.ts";
2
- import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
2
+ import {
3
+ type BenchmarkObservation,
4
+ type BenchmarkSourceSnapshot,
5
+ normalizeModelIdentity,
6
+ validateBenchmarkObservation,
7
+ } from "../domain/benchmark.ts";
3
8
  import type { BenchmarkSource } from "../ports/benchmark-source.ts";
4
9
  import { contractRecord } from "../providers/openrouter-contracts.ts";
5
10
 
@@ -46,7 +51,10 @@ function requiredNumber(value: unknown, name: string): number {
46
51
  */
47
52
  function bestEffortIdentity(organization: string, displayName: string): { provider: string; model: string; aliases: string[] } {
48
53
  const base = displayName.replace(THINKING_SUFFIX, "").trim();
49
- const slug = base.toLowerCase().replace(/[^a-z0-9.]+/g, "-").replace(/^-+|-+$/g, "");
54
+ const slug = base
55
+ .toLowerCase()
56
+ .replace(/[^a-z0-9.]+/g, "-")
57
+ .replace(/^-+|-+$/g, "");
50
58
  return { provider: organization, model: slug, aliases: [displayName.toLowerCase()] };
51
59
  }
52
60
 
@@ -63,32 +71,44 @@ export class LmArenaHfSource implements BenchmarkSource {
63
71
  if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
64
72
  const perArena = await Promise.all(ARENAS.map((arena) => this.fetchArena(arena, retrievedAt)));
65
73
  const observations = perArena.flatMap((page) => page.observations).slice(0, BENCHMARK_MAX_MODELS_PER_SOURCE * ARENAS.length);
66
- const asOf = perArena.map((page) => page.asOf).sort().at(-1) ?? "unknown";
74
+ const asOf =
75
+ perArena
76
+ .map((page) => page.asOf)
77
+ .sort()
78
+ .at(-1) ?? "unknown";
67
79
  return { sourceId: this.id, snapshotId: `${this.id}:${asOf}`, retrievedAt, observations };
68
80
  }
69
81
 
70
- private async fetchArena(arena: { config: string; dimension: string; unit: "ratio" }, retrievedAt: number): Promise<{ asOf: string; observations: BenchmarkObservation[] }> {
82
+ private async fetchArena(
83
+ arena: { config: string; dimension: string; unit: "ratio" },
84
+ retrievedAt: number,
85
+ ): Promise<{ asOf: string; observations: BenchmarkObservation[] }> {
71
86
  const url = `${BASE_URL}?dataset=lmarena-ai%2Fleaderboard-dataset&config=${arena.config}&split=latest&length=${HF_ROWS_PER_PAGE}`;
72
87
  const response = await this.transport(new Request(url));
73
88
  if (!response.ok) throw new Error(`LMArena benchmark fetch failed with HTTP ${response.status}`);
74
89
  const text = await response.text();
75
- if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("LMArena benchmark response exceeds the size limit");
90
+ if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
91
+ throw new Error("LMArena benchmark response exceeds the size limit");
76
92
  let payload: unknown;
77
- try { payload = JSON.parse(text); } catch { throw new Error("LMArena benchmark response is not valid JSON"); }
93
+ try {
94
+ payload = JSON.parse(text);
95
+ } catch {
96
+ throw new Error("LMArena benchmark response is not valid JSON");
97
+ }
78
98
  const root = contractRecord(payload, "benchmark response");
79
- if (!Array.isArray(root["rows"]) || root["rows"].length > HF_ROWS_PER_PAGE) throw new Error("LMArena benchmark row count is invalid");
99
+ if (!Array.isArray(root.rows) || root.rows.length > HF_ROWS_PER_PAGE) throw new Error("LMArena benchmark row count is invalid");
80
100
  let asOf = "";
81
- const observations = root["rows"].flatMap((entry): BenchmarkObservation[] => {
101
+ const observations = root.rows.flatMap((entry): BenchmarkObservation[] => {
82
102
  const wrapper = contractRecord(entry, "benchmark row wrapper");
83
- const row = contractRecord(wrapper["row"], "benchmark row");
84
- const publishDate = requiredText(row["leaderboard_publish_date"], "publish date");
103
+ const row = contractRecord(wrapper.row, "benchmark row");
104
+ const publishDate = requiredText(row.leaderboard_publish_date, "publish date");
85
105
  if (publishDate > asOf) asOf = publishDate;
86
106
  // Skip finer per-category splits, if any exist for this config -- out of scope for now.
87
- if (requiredText(row["category"], "category") !== "overall") return [];
88
- const scoreValue = row["rating"] ?? row["score"];
107
+ if (requiredText(row.category, "category") !== "overall") return [];
108
+ const scoreValue = row.rating ?? row.score;
89
109
  const value = requiredNumber(scoreValue, "score");
90
- const organization = requiredText(row["organization"], "organization");
91
- const displayName = requiredText(row["model_name"], "model name");
110
+ const organization = requiredText(row.organization, "organization");
111
+ const displayName = requiredText(row.model_name, "model name");
92
112
  const guessed = bestEffortIdentity(organization, displayName);
93
113
  const identity = normalizeModelIdentity(guessed.provider, guessed.model, guessed.aliases);
94
114
  const provenance = {
@@ -104,7 +124,9 @@ export class LmArenaHfSource implements BenchmarkSource {
104
124
  confidence: 0.6,
105
125
  };
106
126
  const methodology = { basis: "LMArena human-preference battles", arena: arena.config, displayName, publishDate };
107
- return [validateBenchmarkObservation({ model: identity, dimension: arena.dimension, value, unit: arena.unit, provenance, methodology })];
127
+ return [
128
+ validateBenchmarkObservation({ model: identity, dimension: arena.dimension, value, unit: arena.unit, provenance, methodology }),
129
+ ];
108
130
  });
109
131
  return { asOf, observations };
110
132
  }
@@ -1,12 +1,5 @@
1
- import {
2
- BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT,
3
- BENCHMARK_STORE_QUERY_LIMIT,
4
- } from "../constants.ts";
5
- import {
6
- validateBenchmarkObservation,
7
- type BenchmarkObservation,
8
- type BenchmarkSnapshot,
9
- } from "../domain/benchmark.ts";
1
+ import { BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT, BENCHMARK_STORE_QUERY_LIMIT } from "../constants.ts";
2
+ import { type BenchmarkObservation, type BenchmarkSnapshot, validateBenchmarkObservation } from "../domain/benchmark.ts";
10
3
  import type { StoredMetricObservation } from "../domain/metric.ts";
11
4
  import type { BenchmarkStore } from "../ports/benchmark-store.ts";
12
5
  import type { MetricStore } from "../ports/metric-store.ts";
@@ -29,12 +22,12 @@ function observationAttributes(snapshotId: string, observation: BenchmarkObserva
29
22
 
30
23
  function decodeObservation(row: StoredMetricObservation): BenchmarkObservation {
31
24
  return validateBenchmarkObservation({
32
- model: row.attributes["model"],
25
+ model: row.attributes.model,
33
26
  dimension: row.metric,
34
27
  value: row.value,
35
28
  unit: row.unit,
36
- provenance: row.attributes["provenance"],
37
- methodology: row.attributes["methodology"],
29
+ provenance: row.attributes.provenance,
30
+ methodology: row.attributes.methodology,
38
31
  });
39
32
  }
40
33
 
@@ -47,7 +40,10 @@ export class MetricBenchmarkStore implements BenchmarkStore {
47
40
  const existing = this.latest(sourceId);
48
41
  if (existing?.snapshotId === snapshotId) return existing;
49
42
  const retrievedAt = observations[0]?.provenance.retrievedAt;
50
- if (retrievedAt === undefined || observations.some((observation) => observation.provenance.sourceId !== sourceId || observation.provenance.retrievedAt !== retrievedAt)) {
43
+ if (
44
+ retrievedAt === undefined ||
45
+ observations.some((observation) => observation.provenance.sourceId !== sourceId || observation.provenance.retrievedAt !== retrievedAt)
46
+ ) {
51
47
  throw new Error("benchmark snapshot provenance mismatch");
52
48
  }
53
49
  for (const observation of observations) {
@@ -74,15 +70,30 @@ export class MetricBenchmarkStore implements BenchmarkStore {
74
70
  }
75
71
 
76
72
  latest(sourceId: string): BenchmarkSnapshot | null {
77
- const marker = this.metrics.query({ source: metricSource(sourceId), scope: SNAPSHOT_SCOPE, metric: COMPLETE_METRIC, order: "desc", limit: 1 })[0];
73
+ const marker = this.metrics.query({
74
+ source: metricSource(sourceId),
75
+ scope: SNAPSHOT_SCOPE,
76
+ metric: COMPLETE_METRIC,
77
+ order: "desc",
78
+ limit: 1,
79
+ })[0];
78
80
  if (!marker) return null;
79
- const rows = this.metrics.query({ source: metricSource(sourceId), until: marker.observedAt, order: "desc", limit: BENCHMARK_STORE_QUERY_LIMIT });
81
+ const rows = this.metrics.query({
82
+ source: metricSource(sourceId),
83
+ until: marker.observedAt,
84
+ order: "desc",
85
+ limit: BENCHMARK_STORE_QUERY_LIMIT,
86
+ });
80
87
  const markerIndex = rows.findIndex((row) => row.id === marker.id);
81
88
  if (markerIndex < 0) return null;
82
- const snapshotId = marker.attributes["snapshotId"];
89
+ const snapshotId = marker.attributes.snapshotId;
83
90
  const expectedCount = marker.value;
84
- if (typeof snapshotId !== "string" || typeof expectedCount !== "number" || !Number.isSafeInteger(expectedCount) || expectedCount < 0) return null;
85
- const matching = rows.slice(markerIndex + 1).filter((row) => row.attributes["snapshotId"] === snapshotId).slice(0, expectedCount);
91
+ if (typeof snapshotId !== "string" || typeof expectedCount !== "number" || !Number.isSafeInteger(expectedCount) || expectedCount < 0)
92
+ return null;
93
+ const matching = rows
94
+ .slice(markerIndex + 1)
95
+ .filter((row) => row.attributes.snapshotId === snapshotId)
96
+ .slice(0, expectedCount);
86
97
  if (matching.length !== expectedCount) return null;
87
98
  try {
88
99
  return {
@@ -4,9 +4,14 @@ import {
4
4
  BENCHMARK_SOURCE_MAX_RESPONSE_BYTES,
5
5
  BENCHMARK_SOURCE_MAX_TOTAL_RESPONSE_BYTES,
6
6
  } from "../constants.ts";
7
- import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
7
+ import {
8
+ type BenchmarkObservation,
9
+ type BenchmarkSourceSnapshot,
10
+ normalizeModelIdentity,
11
+ validateBenchmarkObservation,
12
+ } from "../domain/benchmark.ts";
8
13
  import type { BenchmarkSource } from "../ports/benchmark-source.ts";
9
- import { contractRecord, parseOpenRouterModels, type OpenRouterModel } from "../providers/openrouter-contracts.ts";
14
+ import { contractRecord, type OpenRouterModel, parseOpenRouterModels } from "../providers/openrouter-contracts.ts";
10
15
 
11
16
  const OPENROUTER_MODELS_BASE_URL = "https://openrouter.ai/api/v1/models";
12
17
  const OPENROUTER_MODELS_URL = `${OPENROUTER_MODELS_BASE_URL}?limit=${BENCHMARK_MAX_MODELS_PER_SOURCE}`;
@@ -43,23 +48,68 @@ function provenance(retrievedAt: number, revision: string, url: string, confiden
43
48
  function modelObservations(model: OpenRouterModel, retrievedAt: number, revision: string): BenchmarkObservation[] {
44
49
  const common = { model: identity(model), provenance: provenance(retrievedAt, revision, OPENROUTER_MODELS_URL, 0.9) };
45
50
  return [
46
- validateBenchmarkObservation({ ...common, dimension: "context-window", value: model.contextLength, unit: "tokens", methodology: { basis: "OpenRouter model context_length" } }),
47
- ...(model.maxCompletionTokens === null ? [] : [validateBenchmarkObservation({ ...common, dimension: "max-output", value: model.maxCompletionTokens, unit: "tokens", methodology: { basis: "OpenRouter top_provider max_completion_tokens" } })]),
48
- validateBenchmarkObservation({ ...common, dimension: "price-input", value: model.pricing.prompt, unit: "usd", methodology: { basis: "USD per input token" } }),
49
- validateBenchmarkObservation({ ...common, dimension: "price-output", value: model.pricing.completion, unit: "usd", methodology: { basis: "USD per output token" } }),
50
- validateBenchmarkObservation({ ...common, dimension: "parameter-count", value: model.supportedParameters.length, unit: "count", methodology: { basis: "OpenRouter supported_parameters", parameters: [...model.supportedParameters].sort() } }),
51
+ validateBenchmarkObservation({
52
+ ...common,
53
+ dimension: "context-window",
54
+ value: model.contextLength,
55
+ unit: "tokens",
56
+ methodology: { basis: "OpenRouter model context_length" },
57
+ }),
58
+ ...(model.maxCompletionTokens === null
59
+ ? []
60
+ : [
61
+ validateBenchmarkObservation({
62
+ ...common,
63
+ dimension: "max-output",
64
+ value: model.maxCompletionTokens,
65
+ unit: "tokens",
66
+ methodology: { basis: "OpenRouter top_provider max_completion_tokens" },
67
+ }),
68
+ ]),
69
+ validateBenchmarkObservation({
70
+ ...common,
71
+ dimension: "price-input",
72
+ value: model.pricing.prompt,
73
+ unit: "usd",
74
+ methodology: { basis: "USD per input token" },
75
+ }),
76
+ validateBenchmarkObservation({
77
+ ...common,
78
+ dimension: "price-output",
79
+ value: model.pricing.completion,
80
+ unit: "usd",
81
+ methodology: { basis: "USD per output token" },
82
+ }),
83
+ validateBenchmarkObservation({
84
+ ...common,
85
+ dimension: "parameter-count",
86
+ value: model.supportedParameters.length,
87
+ unit: "count",
88
+ methodology: { basis: "OpenRouter supported_parameters", parameters: [...model.supportedParameters].sort() },
89
+ }),
51
90
  ];
52
91
  }
53
92
 
54
- function rankObservations(models: OpenRouterModel[], dimension: "latency-rank" | "throughput-rank", retrievedAt: number, revision: string, url: string): BenchmarkObservation[] {
55
- return models.map((model, index) => validateBenchmarkObservation({
56
- model: identity(model),
57
- dimension,
58
- value: index + 1,
59
- unit: "count",
60
- provenance: provenance(retrievedAt, revision, url, 0.7),
61
- methodology: { basis: dimension === "latency-rank" ? "OpenRouter p50 TTFT server-side ordering" : "OpenRouter p50 throughput server-side ordering", rank: index + 1 },
62
- }));
93
+ function rankObservations(
94
+ models: OpenRouterModel[],
95
+ dimension: "latency-rank" | "throughput-rank",
96
+ retrievedAt: number,
97
+ revision: string,
98
+ url: string,
99
+ ): BenchmarkObservation[] {
100
+ return models.map((model, index) =>
101
+ validateBenchmarkObservation({
102
+ model: identity(model),
103
+ dimension,
104
+ value: index + 1,
105
+ unit: "count",
106
+ provenance: provenance(retrievedAt, revision, url, 0.7),
107
+ methodology: {
108
+ basis: dimension === "latency-rank" ? "OpenRouter p50 TTFT server-side ordering" : "OpenRouter p50 throughput server-side ordering",
109
+ rank: index + 1,
110
+ },
111
+ }),
112
+ );
63
113
  }
64
114
 
65
115
  interface ModelResponse {
@@ -83,9 +133,14 @@ export class OpenRouterBenchmarkSource implements BenchmarkSource {
83
133
  const bytes = new TextEncoder().encode(text).byteLength;
84
134
  if (bytes > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("OpenRouter models response exceeds the size limit");
85
135
  let payload: unknown;
86
- try { payload = JSON.parse(text); } catch { throw new Error("OpenRouter models response is not valid JSON"); }
136
+ try {
137
+ payload = JSON.parse(text);
138
+ } catch {
139
+ throw new Error("OpenRouter models response is not valid JSON");
140
+ }
87
141
  const root = contractRecord(payload, "models response");
88
- if (!Array.isArray(root["data"]) || root["data"].length === 0 || root["data"].length > BENCHMARK_MAX_MODELS_PER_SOURCE) throw new Error("OpenRouter models response exceeds the model limit");
142
+ if (!Array.isArray(root.data) || root.data.length === 0 || root.data.length > BENCHMARK_MAX_MODELS_PER_SOURCE)
143
+ throw new Error("OpenRouter models response exceeds the model limit");
89
144
  return { models: parseOpenRouterModels(payload), etag: response.headers.get("etag")?.slice(0, 120) ?? null, bytes };
90
145
  }
91
146
 
@@ -95,7 +150,8 @@ export class OpenRouterBenchmarkSource implements BenchmarkSource {
95
150
  this.readModels(OPENROUTER_LATENCY_URL),
96
151
  this.readModels(OPENROUTER_THROUGHPUT_URL),
97
152
  ]);
98
- if (catalog.bytes + latency.bytes + throughput.bytes > BENCHMARK_SOURCE_MAX_TOTAL_RESPONSE_BYTES) throw new Error("OpenRouter model evidence exceeds the total size limit");
153
+ if (catalog.bytes + latency.bytes + throughput.bytes > BENCHMARK_SOURCE_MAX_TOTAL_RESPONSE_BYTES)
154
+ throw new Error("OpenRouter model evidence exceeds the total size limit");
99
155
  const retrievedAt = this.clock();
100
156
  if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
101
157
  const revision = catalog.etag || latency.etag || throughput.etag || `retrieved:${retrievedAt}`;
@@ -1,9 +1,10 @@
1
+ import { BENCHMARK_MAX_MODELS_PER_SOURCE, BENCHMARK_REFRESH_INTERVAL_MS, BENCHMARK_SOURCE_MAX_RESPONSE_BYTES } from "../constants.ts";
1
2
  import {
2
- BENCHMARK_MAX_MODELS_PER_SOURCE,
3
- BENCHMARK_REFRESH_INTERVAL_MS,
4
- BENCHMARK_SOURCE_MAX_RESPONSE_BYTES,
5
- } from "../constants.ts";
6
- import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
3
+ type BenchmarkObservation,
4
+ type BenchmarkSourceSnapshot,
5
+ normalizeModelIdentity,
6
+ validateBenchmarkObservation,
7
+ } from "../domain/benchmark.ts";
7
8
  import type { BenchmarkSource } from "../ports/benchmark-source.ts";
8
9
  import { contractRecord } from "../providers/openrouter-contracts.ts";
9
10
  import type { OpenRouterBenchmarkTransport } from "./openrouter-benchmark-source.ts";
@@ -22,7 +23,8 @@ const BASE_URL = `https://openrouter.ai/api/v1/benchmarks?source=design-arena&ma
22
23
  const RELEVANT_CATEGORIES = ["codecategories", "website", "uicomponent", "dataviz", "svg"] as const;
23
24
 
24
25
  function requiredText(value: unknown, name: string): string {
25
- if (typeof value !== "string" || value.length === 0 || value.length > 500) throw new Error(`OpenRouter Design Arena benchmark ${name} schema changed`);
26
+ if (typeof value !== "string" || value.length === 0 || value.length > 500)
27
+ throw new Error(`OpenRouter Design Arena benchmark ${name} schema changed`);
26
28
  return value;
27
29
  }
28
30
 
@@ -62,24 +64,32 @@ export class OpenRouterDesignArenaSource implements BenchmarkSource {
62
64
  const response = await this.transport(new Request(url, { headers: { authorization: `Bearer ${this.apiKey}` } }));
63
65
  if (!response.ok) throw new Error(`OpenRouter Design Arena benchmarks failed with HTTP ${response.status}`);
64
66
  const text = await response.text();
65
- if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("OpenRouter Design Arena benchmark response exceeds the size limit");
67
+ if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
68
+ throw new Error("OpenRouter Design Arena benchmark response exceeds the size limit");
66
69
  let payload: unknown;
67
- try { payload = JSON.parse(text); } catch { throw new Error("OpenRouter Design Arena benchmark response is not valid JSON"); }
70
+ try {
71
+ payload = JSON.parse(text);
72
+ } catch {
73
+ throw new Error("OpenRouter Design Arena benchmark response is not valid JSON");
74
+ }
68
75
  const root = contractRecord(payload, "benchmark response");
69
- const meta = contractRecord(root["meta"], "benchmark metadata");
70
- if (!Array.isArray(root["data"]) || root["data"].length > BENCHMARK_MAX_MODELS_PER_SOURCE) throw new Error("OpenRouter Design Arena benchmark result count is invalid");
71
- if (requiredText(meta["source"], "source") !== "design-arena") throw new Error("OpenRouter Design Arena benchmark source schema changed");
72
- const asOf = requiredText(meta["as_of"], "as-of date");
76
+ const meta = contractRecord(root.meta, "benchmark metadata");
77
+ if (!Array.isArray(root.data) || root.data.length > BENCHMARK_MAX_MODELS_PER_SOURCE)
78
+ throw new Error("OpenRouter Design Arena benchmark result count is invalid");
79
+ if (requiredText(meta.source, "source") !== "design-arena") throw new Error("OpenRouter Design Arena benchmark source schema changed");
80
+ const asOf = requiredText(meta.as_of, "as-of date");
73
81
  const publishedAt = Date.parse(asOf);
74
- if (!Number.isSafeInteger(publishedAt) || publishedAt <= 0) throw new Error("OpenRouter Design Arena benchmark publication date schema changed");
82
+ if (!Number.isSafeInteger(publishedAt) || publishedAt <= 0)
83
+ throw new Error("OpenRouter Design Arena benchmark publication date schema changed");
75
84
  const revision = `${category}:${asOf}`;
76
- const observations = root["data"].flatMap((value): BenchmarkObservation[] => {
85
+ const observations = root.data.flatMap((value): BenchmarkObservation[] => {
77
86
  const row = contractRecord(value, "benchmark row");
78
- if (requiredText(row["source"], "row source") !== "design-arena") throw new Error("OpenRouter Design Arena benchmark row source schema changed");
87
+ if (requiredText(row.source, "row source") !== "design-arena")
88
+ throw new Error("OpenRouter Design Arena benchmark row source schema changed");
79
89
  // A null open_router_id means the model isn't reachable through OpenRouter at all
80
90
  // (proprietary platform, image/video generator, deprecated model) -- not evidence
81
91
  // Jittor can ever route against, so the row is skipped rather than rejected.
82
- const openRouterId = row["open_router_id"];
92
+ const openRouterId = row.open_router_id;
83
93
  if (openRouterId === null) return [];
84
94
  const { provider, model } = identityFromOpenRouterId(requiredText(openRouterId, "open_router_id"));
85
95
  const identity = normalizeModelIdentity(provider, model, [`openrouter/${provider}/${model}`]);
@@ -96,9 +106,16 @@ export class OpenRouterDesignArenaSource implements BenchmarkSource {
96
106
  confidence: 0.7,
97
107
  };
98
108
  const methodology = { basis: "Design Arena Elo rating via OpenRouter", category, asOf };
99
- return [validateBenchmarkObservation({
100
- model: identity, dimension: "quality-design", value: requiredNumber(row["elo"], "elo rating"), unit: "elo", provenance, methodology,
101
- })];
109
+ return [
110
+ validateBenchmarkObservation({
111
+ model: identity,
112
+ dimension: "quality-design",
113
+ value: requiredNumber(row.elo, "elo rating"),
114
+ unit: "elo",
115
+ provenance,
116
+ methodology,
117
+ }),
118
+ ];
102
119
  });
103
120
  return { revision, observations };
104
121
  }
@@ -40,22 +40,26 @@ export class SQLiteMetricStore implements MetricStore {
40
40
 
41
41
  recordBatch(inputs: MetricObservation[]): StoredMetricObservation[] {
42
42
  const observations = inputs.map((input) => validateMetricObservation(input));
43
- return this.db.transaction((rows: typeof observations) => rows.map((observation) => this.get(Number(this.insert(observation).lastInsertRowid))))(observations);
43
+ return this.db.transaction((rows: typeof observations) =>
44
+ rows.map((observation) => this.get(Number(this.insert(observation).lastInsertRowid))),
45
+ )(observations);
44
46
  }
45
47
 
46
48
  private insert(observation: MetricObservation): { lastInsertRowid: number | bigint } {
47
- return this.db.query(`
49
+ return this.db
50
+ .query(`
48
51
  INSERT INTO metric_observations (source, scope, metric, value, unit, observed_at, attributes)
49
52
  VALUES (?, ?, ?, ?, ?, ?, ?)
50
- `).run(
51
- observation.source,
52
- observation.scope,
53
- observation.metric,
54
- observation.value,
55
- observation.unit,
56
- observation.observedAt,
57
- JSON.stringify(observation.attributes ?? {}),
58
- );
53
+ `)
54
+ .run(
55
+ observation.source,
56
+ observation.scope,
57
+ observation.metric,
58
+ observation.value,
59
+ observation.unit,
60
+ observation.observedAt,
61
+ JSON.stringify(observation.attributes ?? {}),
62
+ );
59
63
  }
60
64
 
61
65
  query(filter: MetricQuery = {}): StoredMetricObservation[] {
@@ -69,30 +73,40 @@ export class SQLiteMetricStore implements MetricStore {
69
73
  addEquals("source", filter.source);
70
74
  addEquals("scope", filter.scope);
71
75
  addEquals("metric", filter.metric);
72
- if (filter.since !== undefined) { conditions.push("observed_at >= ?"); parameters.push(filter.since); }
73
- if (filter.until !== undefined) { conditions.push("observed_at <= ?"); parameters.push(filter.until); }
76
+ if (filter.since !== undefined) {
77
+ conditions.push("observed_at >= ?");
78
+ parameters.push(filter.since);
79
+ }
80
+ if (filter.until !== undefined) {
81
+ conditions.push("observed_at <= ?");
82
+ parameters.push(filter.until);
83
+ }
74
84
  const requestedLimit = Number.isFinite(filter.limit) ? Math.floor(filter.limit!) : DEFAULT_QUERY_LIMIT;
75
85
  const limit = Math.max(1, Math.min(MAX_QUERY_LIMIT, requestedLimit));
76
86
  const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
77
87
  const order = filter.order === "desc" ? "DESC" : "ASC";
78
- const rows = this.db.query(`
88
+ const rows = this.db
89
+ .query(`
79
90
  SELECT id, source, scope, metric, value, unit, observed_at, attributes
80
91
  FROM metric_observations
81
92
  ${where}
82
93
  ORDER BY observed_at ${order}, id ${order}
83
94
  LIMIT ${limit}
84
- `).all(...parameters) as MetricRow[];
95
+ `)
96
+ .all(...parameters) as MetricRow[];
85
97
  return rows.map(fromRow);
86
98
  }
87
99
 
88
100
  distinctScopes(filter: DistinctScopesFilter): string[] {
89
101
  const limit = Math.max(1, Math.min(MAX_QUERY_LIMIT, Math.floor(filter.limit)));
90
- const rows = this.db.query(`
102
+ const rows = this.db
103
+ .query(`
91
104
  SELECT DISTINCT scope FROM metric_observations
92
105
  WHERE source = ? AND observed_at >= ? AND observed_at <= ?
93
106
  ORDER BY scope ASC
94
107
  LIMIT ?
95
- `).all(filter.source, filter.since, filter.until, limit) as Array<{ scope: string }>;
108
+ `)
109
+ .all(filter.source, filter.since, filter.until, limit) as Array<{ scope: string }>;
96
110
  return rows.map((row) => row.scope);
97
111
  }
98
112
 
@@ -104,7 +118,8 @@ export class SQLiteMetricStore implements MetricStore {
104
118
  // MIN(a, b) is SQLite's scalar two-argument form (smallest of the arguments), mirroring
105
119
  // usageBucketIndex's own `Math.min(bucketCount - 1, Math.floor(...))` clamp exactly, so a bucket
106
120
  // this returns always matches the same-indexed bucket the client-side window renders.
107
- const rows = this.db.query(`
121
+ const rows = this.db
122
+ .query(`
108
123
  SELECT scope, metric,
109
124
  MIN(CAST((observed_at - ?) AS REAL) / ?, ? - 1) AS bucket_index_raw,
110
125
  SUM(value) AS sum
@@ -112,11 +127,17 @@ export class SQLiteMetricStore implements MetricStore {
112
127
  WHERE source = ? AND observed_at >= ? AND observed_at <= ? AND value >= 0 AND scope IN (${scopePlaceholders})
113
128
  GROUP BY scope, metric, CAST(bucket_index_raw AS INTEGER)
114
129
  LIMIT ?
115
- `).all(
116
- filter.since, filter.bucketSizeMs, filter.bucketCount,
117
- filter.source, filter.since, filter.until, ...filter.scopes,
118
- USAGE_AGGREGATE_MAX_ROWS,
119
- ) as Array<{ scope: string; metric: string; bucket_index_raw: number; sum: number }>;
130
+ `)
131
+ .all(
132
+ filter.since,
133
+ filter.bucketSizeMs,
134
+ filter.bucketCount,
135
+ filter.source,
136
+ filter.since,
137
+ filter.until,
138
+ ...filter.scopes,
139
+ USAGE_AGGREGATE_MAX_ROWS,
140
+ ) as Array<{ scope: string; metric: string; bucket_index_raw: number; sum: number }>;
120
141
  return rows.map((row) => ({
121
142
  scope: row.scope,
122
143
  metric: row.metric,
@@ -139,10 +160,12 @@ export class SQLiteMetricStore implements MetricStore {
139
160
  }
140
161
 
141
162
  private get(id: number): StoredMetricObservation {
142
- const row = this.db.query(`
163
+ const row = this.db
164
+ .query(`
143
165
  SELECT id, source, scope, metric, value, unit, observed_at, attributes
144
166
  FROM metric_observations WHERE id = ?
145
- `).get(id) as MetricRow | null;
167
+ `)
168
+ .get(id) as MetricRow | null;
146
169
  if (!row) throw new Error(`metric observation ${id} was not persisted`);
147
170
  return fromRow(row);
148
171
  }