@danypops/jittor 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -3
- package/src/adapters/artificial-analysis-direct-source.ts +42 -11
- package/src/adapters/lmarena-hf-source.ts +37 -15
- package/src/adapters/metric-benchmark-store.ts +29 -18
- package/src/adapters/openrouter-benchmark-source.ts +75 -19
- package/src/adapters/openrouter-design-arena-source.ts +36 -19
- package/src/adapters/sqlite-metric-store.ts +48 -25
- package/src/adapters/sqlite-session-identity-store.ts +13 -7
- package/src/cli-commands/benchmarks.ts +87 -22
- package/src/cli-commands/compaction.ts +7 -2
- package/src/cli-commands/context.ts +10 -2
- package/src/cli-commands/metrics.ts +128 -31
- package/src/cli-commands/op.ts +6 -1
- package/src/cli-commands/route-args.ts +5 -1
- package/src/cli-commands/router.ts +61 -18
- package/src/cli-commands/service-daemon.ts +28 -12
- package/src/cli-commands/session.ts +15 -4
- package/src/cli-commands/support.ts +1 -1
- package/src/cli.ts +30 -23
- package/src/client.ts +1 -1
- package/src/constants.ts +6 -2
- package/src/daemon.ts +44 -24
- package/src/domain/benchmark.ts +72 -40
- package/src/domain/codex-recovery.ts +34 -25
- package/src/domain/context-hub.ts +44 -25
- package/src/domain/context-telemetry.ts +106 -35
- package/src/domain/metric.ts +11 -11
- package/src/domain/model-observation.ts +139 -55
- package/src/domain/model-ranking-service.ts +13 -3
- package/src/domain/model-ranking.ts +126 -60
- package/src/domain/task-cost.ts +52 -11
- package/src/domain/task-focus.ts +11 -8
- package/src/domain/usage.ts +2 -2
- package/src/index.ts +69 -69
- package/src/log.ts +7 -2
- package/src/operations/benchmark-operations.ts +1 -1
- package/src/operations/context-operations.ts +15 -6
- package/src/operations/metrics-operations.ts +63 -28
- package/src/operations/model-ranking-operations.ts +9 -2
- package/src/operations/router-operations.ts +8 -4
- package/src/operations/session-identity-operations.ts +1 -1
- package/src/operations/session-scope.ts +5 -3
- package/src/policy.ts +22 -17
- package/src/ports/benchmark-controller.ts +1 -5
- package/src/ports/metric-store.ts +1 -1
- package/src/providers/anthropic-contracts.ts +13 -3
- package/src/providers/codex-contracts.ts +60 -52
- package/src/providers/codex.ts +16 -19
- package/src/providers/google-vertex-budget-contracts.ts +24 -14
- package/src/providers/google-vertex-budget.ts +15 -13
- package/src/providers/google-vertex-contracts.ts +36 -24
- package/src/providers/openrouter-contracts.ts +49 -51
- package/src/providers/openrouter.ts +21 -15
- package/src/providers/telemetry-sources.ts +13 -10
- package/src/router.ts +92 -43
- package/src/service.ts +93 -32
- package/src/session-identity-service.ts +10 -2
- package/src/state.ts +4 -10
- package/src/vehicle-registration.ts +154 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@danypops/jittor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.15.0",
|
|
4
4
|
"description": "Just-in-Time Token Optimizing Router for Pi -- supervised daemon, router policy, and CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.ts",
|
|
@@ -16,8 +16,9 @@
|
|
|
16
16
|
"service:install": "bun src/cli.ts service install"
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"@danypops/vehicle-
|
|
20
|
-
"@danypops/vehicle-
|
|
19
|
+
"@danypops/vehicle-core": "^0.10.0",
|
|
20
|
+
"@danypops/vehicle-server": "^0.13.0",
|
|
21
|
+
"@danypops/vehicle-client": "^0.5.1",
|
|
21
22
|
"google-auth-library": "^10.9.0"
|
|
22
23
|
},
|
|
23
24
|
"devDependencies": {
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { BENCHMARK_MAX_MODELS_PER_SOURCE, BENCHMARK_REFRESH_INTERVAL_MS, BENCHMARK_SOURCE_MAX_RESPONSE_BYTES } from "../constants.ts";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
type BenchmarkObservation,
|
|
4
|
+
type BenchmarkSourceSnapshot,
|
|
5
|
+
normalizeModelIdentity,
|
|
6
|
+
validateBenchmarkObservation,
|
|
7
|
+
} from "../domain/benchmark.ts";
|
|
3
8
|
import type { BenchmarkSource } from "../ports/benchmark-source.ts";
|
|
4
9
|
import { contractRecord } from "../providers/openrouter-contracts.ts";
|
|
5
10
|
|
|
@@ -36,17 +41,25 @@ export class ArtificialAnalysisDirectSource implements BenchmarkSource {
|
|
|
36
41
|
const response = await this.transport(new Request(ENDPOINT, { headers: { "x-api-key": this.apiKey } }));
|
|
37
42
|
if (!response.ok) throw new Error(`Artificial Analysis benchmarks failed with HTTP ${response.status}`);
|
|
38
43
|
const text = await response.text();
|
|
39
|
-
if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
|
|
44
|
+
if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
|
|
45
|
+
throw new Error("Artificial Analysis benchmark response exceeds the size limit");
|
|
40
46
|
let payload: unknown;
|
|
41
|
-
try {
|
|
47
|
+
try {
|
|
48
|
+
payload = JSON.parse(text);
|
|
49
|
+
} catch {
|
|
50
|
+
throw new Error("Artificial Analysis benchmark response is not valid JSON");
|
|
51
|
+
}
|
|
42
52
|
const root = contractRecord(payload, "benchmark response");
|
|
43
|
-
if (!Array.isArray(root
|
|
53
|
+
if (!Array.isArray(root.data) || root.data.length > BENCHMARK_MAX_MODELS_PER_SOURCE)
|
|
54
|
+
throw new Error("Artificial Analysis benchmark result count is invalid");
|
|
44
55
|
const revision = String(retrievedAt);
|
|
45
|
-
const observations = root
|
|
56
|
+
const observations = root.data.flatMap((value): BenchmarkObservation[] => {
|
|
46
57
|
const row = contractRecord(value, "benchmark row");
|
|
47
|
-
const creator = contractRecord(row
|
|
48
|
-
const evaluations = contractRecord(row
|
|
49
|
-
const identity = normalizeModelIdentity(requiredText(creator
|
|
58
|
+
const creator = contractRecord(row.model_creator, "model creator");
|
|
59
|
+
const evaluations = contractRecord(row.evaluations, "evaluations");
|
|
60
|
+
const identity = normalizeModelIdentity(requiredText(creator.slug, "creator slug"), requiredText(row.slug, "model slug"), [
|
|
61
|
+
`artificial-analysis/${requiredText(row.id, "model id")}`,
|
|
62
|
+
]);
|
|
50
63
|
const provenance = {
|
|
51
64
|
sourceId: SOURCE_ID,
|
|
52
65
|
sourceType: "creator" as const,
|
|
@@ -64,14 +77,32 @@ export class ArtificialAnalysisDirectSource implements BenchmarkSource {
|
|
|
64
77
|
const index = (field: string, dimension: string): void => {
|
|
65
78
|
const raw = evaluations[field];
|
|
66
79
|
if (raw === undefined || raw === null) return;
|
|
67
|
-
observations.push(
|
|
80
|
+
observations.push(
|
|
81
|
+
validateBenchmarkObservation({
|
|
82
|
+
model: identity,
|
|
83
|
+
dimension,
|
|
84
|
+
value: requiredNumber(raw, field),
|
|
85
|
+
unit: "ratio",
|
|
86
|
+
provenance,
|
|
87
|
+
methodology,
|
|
88
|
+
}),
|
|
89
|
+
);
|
|
68
90
|
};
|
|
69
91
|
index("artificial_analysis_coding_index", "quality-coding");
|
|
70
92
|
index("artificial_analysis_intelligence_index", "quality-general");
|
|
71
93
|
index("artificial_analysis_math_index", "quality-math");
|
|
72
|
-
const ttft = row
|
|
94
|
+
const ttft = row.median_time_to_first_token_seconds;
|
|
73
95
|
if (typeof ttft === "number" && Number.isFinite(ttft)) {
|
|
74
|
-
observations.push(
|
|
96
|
+
observations.push(
|
|
97
|
+
validateBenchmarkObservation({
|
|
98
|
+
model: identity,
|
|
99
|
+
dimension: "latency",
|
|
100
|
+
value: ttft * 1_000,
|
|
101
|
+
unit: "milliseconds",
|
|
102
|
+
provenance,
|
|
103
|
+
methodology,
|
|
104
|
+
}),
|
|
105
|
+
);
|
|
75
106
|
}
|
|
76
107
|
return observations;
|
|
77
108
|
});
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { BENCHMARK_MAX_MODELS_PER_SOURCE, BENCHMARK_REFRESH_INTERVAL_MS, BENCHMARK_SOURCE_MAX_RESPONSE_BYTES } from "../constants.ts";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
type BenchmarkObservation,
|
|
4
|
+
type BenchmarkSourceSnapshot,
|
|
5
|
+
normalizeModelIdentity,
|
|
6
|
+
validateBenchmarkObservation,
|
|
7
|
+
} from "../domain/benchmark.ts";
|
|
3
8
|
import type { BenchmarkSource } from "../ports/benchmark-source.ts";
|
|
4
9
|
import { contractRecord } from "../providers/openrouter-contracts.ts";
|
|
5
10
|
|
|
@@ -46,7 +51,10 @@ function requiredNumber(value: unknown, name: string): number {
|
|
|
46
51
|
*/
|
|
47
52
|
function bestEffortIdentity(organization: string, displayName: string): { provider: string; model: string; aliases: string[] } {
|
|
48
53
|
const base = displayName.replace(THINKING_SUFFIX, "").trim();
|
|
49
|
-
const slug = base
|
|
54
|
+
const slug = base
|
|
55
|
+
.toLowerCase()
|
|
56
|
+
.replace(/[^a-z0-9.]+/g, "-")
|
|
57
|
+
.replace(/^-+|-+$/g, "");
|
|
50
58
|
return { provider: organization, model: slug, aliases: [displayName.toLowerCase()] };
|
|
51
59
|
}
|
|
52
60
|
|
|
@@ -63,32 +71,44 @@ export class LmArenaHfSource implements BenchmarkSource {
|
|
|
63
71
|
if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
|
|
64
72
|
const perArena = await Promise.all(ARENAS.map((arena) => this.fetchArena(arena, retrievedAt)));
|
|
65
73
|
const observations = perArena.flatMap((page) => page.observations).slice(0, BENCHMARK_MAX_MODELS_PER_SOURCE * ARENAS.length);
|
|
66
|
-
const asOf =
|
|
74
|
+
const asOf =
|
|
75
|
+
perArena
|
|
76
|
+
.map((page) => page.asOf)
|
|
77
|
+
.sort()
|
|
78
|
+
.at(-1) ?? "unknown";
|
|
67
79
|
return { sourceId: this.id, snapshotId: `${this.id}:${asOf}`, retrievedAt, observations };
|
|
68
80
|
}
|
|
69
81
|
|
|
70
|
-
private async fetchArena(
|
|
82
|
+
private async fetchArena(
|
|
83
|
+
arena: { config: string; dimension: string; unit: "ratio" },
|
|
84
|
+
retrievedAt: number,
|
|
85
|
+
): Promise<{ asOf: string; observations: BenchmarkObservation[] }> {
|
|
71
86
|
const url = `${BASE_URL}?dataset=lmarena-ai%2Fleaderboard-dataset&config=${arena.config}&split=latest&length=${HF_ROWS_PER_PAGE}`;
|
|
72
87
|
const response = await this.transport(new Request(url));
|
|
73
88
|
if (!response.ok) throw new Error(`LMArena benchmark fetch failed with HTTP ${response.status}`);
|
|
74
89
|
const text = await response.text();
|
|
75
|
-
if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
|
|
90
|
+
if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
|
|
91
|
+
throw new Error("LMArena benchmark response exceeds the size limit");
|
|
76
92
|
let payload: unknown;
|
|
77
|
-
try {
|
|
93
|
+
try {
|
|
94
|
+
payload = JSON.parse(text);
|
|
95
|
+
} catch {
|
|
96
|
+
throw new Error("LMArena benchmark response is not valid JSON");
|
|
97
|
+
}
|
|
78
98
|
const root = contractRecord(payload, "benchmark response");
|
|
79
|
-
if (!Array.isArray(root
|
|
99
|
+
if (!Array.isArray(root.rows) || root.rows.length > HF_ROWS_PER_PAGE) throw new Error("LMArena benchmark row count is invalid");
|
|
80
100
|
let asOf = "";
|
|
81
|
-
const observations = root
|
|
101
|
+
const observations = root.rows.flatMap((entry): BenchmarkObservation[] => {
|
|
82
102
|
const wrapper = contractRecord(entry, "benchmark row wrapper");
|
|
83
|
-
const row = contractRecord(wrapper
|
|
84
|
-
const publishDate = requiredText(row
|
|
103
|
+
const row = contractRecord(wrapper.row, "benchmark row");
|
|
104
|
+
const publishDate = requiredText(row.leaderboard_publish_date, "publish date");
|
|
85
105
|
if (publishDate > asOf) asOf = publishDate;
|
|
86
106
|
// Skip finer per-category splits, if any exist for this config -- out of scope for now.
|
|
87
|
-
if (requiredText(row
|
|
88
|
-
const scoreValue = row
|
|
107
|
+
if (requiredText(row.category, "category") !== "overall") return [];
|
|
108
|
+
const scoreValue = row.rating ?? row.score;
|
|
89
109
|
const value = requiredNumber(scoreValue, "score");
|
|
90
|
-
const organization = requiredText(row
|
|
91
|
-
const displayName = requiredText(row
|
|
110
|
+
const organization = requiredText(row.organization, "organization");
|
|
111
|
+
const displayName = requiredText(row.model_name, "model name");
|
|
92
112
|
const guessed = bestEffortIdentity(organization, displayName);
|
|
93
113
|
const identity = normalizeModelIdentity(guessed.provider, guessed.model, guessed.aliases);
|
|
94
114
|
const provenance = {
|
|
@@ -104,7 +124,9 @@ export class LmArenaHfSource implements BenchmarkSource {
|
|
|
104
124
|
confidence: 0.6,
|
|
105
125
|
};
|
|
106
126
|
const methodology = { basis: "LMArena human-preference battles", arena: arena.config, displayName, publishDate };
|
|
107
|
-
return [
|
|
127
|
+
return [
|
|
128
|
+
validateBenchmarkObservation({ model: identity, dimension: arena.dimension, value, unit: arena.unit, provenance, methodology }),
|
|
129
|
+
];
|
|
108
130
|
});
|
|
109
131
|
return { asOf, observations };
|
|
110
132
|
}
|
|
@@ -1,12 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
BENCHMARK_STORE_QUERY_LIMIT,
|
|
4
|
-
} from "../constants.ts";
|
|
5
|
-
import {
|
|
6
|
-
validateBenchmarkObservation,
|
|
7
|
-
type BenchmarkObservation,
|
|
8
|
-
type BenchmarkSnapshot,
|
|
9
|
-
} from "../domain/benchmark.ts";
|
|
1
|
+
import { BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT, BENCHMARK_STORE_QUERY_LIMIT } from "../constants.ts";
|
|
2
|
+
import { type BenchmarkObservation, type BenchmarkSnapshot, validateBenchmarkObservation } from "../domain/benchmark.ts";
|
|
10
3
|
import type { StoredMetricObservation } from "../domain/metric.ts";
|
|
11
4
|
import type { BenchmarkStore } from "../ports/benchmark-store.ts";
|
|
12
5
|
import type { MetricStore } from "../ports/metric-store.ts";
|
|
@@ -29,12 +22,12 @@ function observationAttributes(snapshotId: string, observation: BenchmarkObserva
|
|
|
29
22
|
|
|
30
23
|
function decodeObservation(row: StoredMetricObservation): BenchmarkObservation {
|
|
31
24
|
return validateBenchmarkObservation({
|
|
32
|
-
model: row.attributes
|
|
25
|
+
model: row.attributes.model,
|
|
33
26
|
dimension: row.metric,
|
|
34
27
|
value: row.value,
|
|
35
28
|
unit: row.unit,
|
|
36
|
-
provenance: row.attributes
|
|
37
|
-
methodology: row.attributes
|
|
29
|
+
provenance: row.attributes.provenance,
|
|
30
|
+
methodology: row.attributes.methodology,
|
|
38
31
|
});
|
|
39
32
|
}
|
|
40
33
|
|
|
@@ -47,7 +40,10 @@ export class MetricBenchmarkStore implements BenchmarkStore {
|
|
|
47
40
|
const existing = this.latest(sourceId);
|
|
48
41
|
if (existing?.snapshotId === snapshotId) return existing;
|
|
49
42
|
const retrievedAt = observations[0]?.provenance.retrievedAt;
|
|
50
|
-
if (
|
|
43
|
+
if (
|
|
44
|
+
retrievedAt === undefined ||
|
|
45
|
+
observations.some((observation) => observation.provenance.sourceId !== sourceId || observation.provenance.retrievedAt !== retrievedAt)
|
|
46
|
+
) {
|
|
51
47
|
throw new Error("benchmark snapshot provenance mismatch");
|
|
52
48
|
}
|
|
53
49
|
for (const observation of observations) {
|
|
@@ -74,15 +70,30 @@ export class MetricBenchmarkStore implements BenchmarkStore {
|
|
|
74
70
|
}
|
|
75
71
|
|
|
76
72
|
latest(sourceId: string): BenchmarkSnapshot | null {
|
|
77
|
-
const marker = this.metrics.query({
|
|
73
|
+
const marker = this.metrics.query({
|
|
74
|
+
source: metricSource(sourceId),
|
|
75
|
+
scope: SNAPSHOT_SCOPE,
|
|
76
|
+
metric: COMPLETE_METRIC,
|
|
77
|
+
order: "desc",
|
|
78
|
+
limit: 1,
|
|
79
|
+
})[0];
|
|
78
80
|
if (!marker) return null;
|
|
79
|
-
const rows = this.metrics.query({
|
|
81
|
+
const rows = this.metrics.query({
|
|
82
|
+
source: metricSource(sourceId),
|
|
83
|
+
until: marker.observedAt,
|
|
84
|
+
order: "desc",
|
|
85
|
+
limit: BENCHMARK_STORE_QUERY_LIMIT,
|
|
86
|
+
});
|
|
80
87
|
const markerIndex = rows.findIndex((row) => row.id === marker.id);
|
|
81
88
|
if (markerIndex < 0) return null;
|
|
82
|
-
const snapshotId = marker.attributes
|
|
89
|
+
const snapshotId = marker.attributes.snapshotId;
|
|
83
90
|
const expectedCount = marker.value;
|
|
84
|
-
if (typeof snapshotId !== "string" || typeof expectedCount !== "number" || !Number.isSafeInteger(expectedCount) || expectedCount < 0)
|
|
85
|
-
|
|
91
|
+
if (typeof snapshotId !== "string" || typeof expectedCount !== "number" || !Number.isSafeInteger(expectedCount) || expectedCount < 0)
|
|
92
|
+
return null;
|
|
93
|
+
const matching = rows
|
|
94
|
+
.slice(markerIndex + 1)
|
|
95
|
+
.filter((row) => row.attributes.snapshotId === snapshotId)
|
|
96
|
+
.slice(0, expectedCount);
|
|
86
97
|
if (matching.length !== expectedCount) return null;
|
|
87
98
|
try {
|
|
88
99
|
return {
|
|
@@ -4,9 +4,14 @@ import {
|
|
|
4
4
|
BENCHMARK_SOURCE_MAX_RESPONSE_BYTES,
|
|
5
5
|
BENCHMARK_SOURCE_MAX_TOTAL_RESPONSE_BYTES,
|
|
6
6
|
} from "../constants.ts";
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
type BenchmarkObservation,
|
|
9
|
+
type BenchmarkSourceSnapshot,
|
|
10
|
+
normalizeModelIdentity,
|
|
11
|
+
validateBenchmarkObservation,
|
|
12
|
+
} from "../domain/benchmark.ts";
|
|
8
13
|
import type { BenchmarkSource } from "../ports/benchmark-source.ts";
|
|
9
|
-
import { contractRecord,
|
|
14
|
+
import { contractRecord, type OpenRouterModel, parseOpenRouterModels } from "../providers/openrouter-contracts.ts";
|
|
10
15
|
|
|
11
16
|
const OPENROUTER_MODELS_BASE_URL = "https://openrouter.ai/api/v1/models";
|
|
12
17
|
const OPENROUTER_MODELS_URL = `${OPENROUTER_MODELS_BASE_URL}?limit=${BENCHMARK_MAX_MODELS_PER_SOURCE}`;
|
|
@@ -43,23 +48,68 @@ function provenance(retrievedAt: number, revision: string, url: string, confiden
|
|
|
43
48
|
function modelObservations(model: OpenRouterModel, retrievedAt: number, revision: string): BenchmarkObservation[] {
|
|
44
49
|
const common = { model: identity(model), provenance: provenance(retrievedAt, revision, OPENROUTER_MODELS_URL, 0.9) };
|
|
45
50
|
return [
|
|
46
|
-
validateBenchmarkObservation({
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
+
validateBenchmarkObservation({
|
|
52
|
+
...common,
|
|
53
|
+
dimension: "context-window",
|
|
54
|
+
value: model.contextLength,
|
|
55
|
+
unit: "tokens",
|
|
56
|
+
methodology: { basis: "OpenRouter model context_length" },
|
|
57
|
+
}),
|
|
58
|
+
...(model.maxCompletionTokens === null
|
|
59
|
+
? []
|
|
60
|
+
: [
|
|
61
|
+
validateBenchmarkObservation({
|
|
62
|
+
...common,
|
|
63
|
+
dimension: "max-output",
|
|
64
|
+
value: model.maxCompletionTokens,
|
|
65
|
+
unit: "tokens",
|
|
66
|
+
methodology: { basis: "OpenRouter top_provider max_completion_tokens" },
|
|
67
|
+
}),
|
|
68
|
+
]),
|
|
69
|
+
validateBenchmarkObservation({
|
|
70
|
+
...common,
|
|
71
|
+
dimension: "price-input",
|
|
72
|
+
value: model.pricing.prompt,
|
|
73
|
+
unit: "usd",
|
|
74
|
+
methodology: { basis: "USD per input token" },
|
|
75
|
+
}),
|
|
76
|
+
validateBenchmarkObservation({
|
|
77
|
+
...common,
|
|
78
|
+
dimension: "price-output",
|
|
79
|
+
value: model.pricing.completion,
|
|
80
|
+
unit: "usd",
|
|
81
|
+
methodology: { basis: "USD per output token" },
|
|
82
|
+
}),
|
|
83
|
+
validateBenchmarkObservation({
|
|
84
|
+
...common,
|
|
85
|
+
dimension: "parameter-count",
|
|
86
|
+
value: model.supportedParameters.length,
|
|
87
|
+
unit: "count",
|
|
88
|
+
methodology: { basis: "OpenRouter supported_parameters", parameters: [...model.supportedParameters].sort() },
|
|
89
|
+
}),
|
|
51
90
|
];
|
|
52
91
|
}
|
|
53
92
|
|
|
54
|
-
function rankObservations(
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
93
|
+
function rankObservations(
|
|
94
|
+
models: OpenRouterModel[],
|
|
95
|
+
dimension: "latency-rank" | "throughput-rank",
|
|
96
|
+
retrievedAt: number,
|
|
97
|
+
revision: string,
|
|
98
|
+
url: string,
|
|
99
|
+
): BenchmarkObservation[] {
|
|
100
|
+
return models.map((model, index) =>
|
|
101
|
+
validateBenchmarkObservation({
|
|
102
|
+
model: identity(model),
|
|
103
|
+
dimension,
|
|
104
|
+
value: index + 1,
|
|
105
|
+
unit: "count",
|
|
106
|
+
provenance: provenance(retrievedAt, revision, url, 0.7),
|
|
107
|
+
methodology: {
|
|
108
|
+
basis: dimension === "latency-rank" ? "OpenRouter p50 TTFT server-side ordering" : "OpenRouter p50 throughput server-side ordering",
|
|
109
|
+
rank: index + 1,
|
|
110
|
+
},
|
|
111
|
+
}),
|
|
112
|
+
);
|
|
63
113
|
}
|
|
64
114
|
|
|
65
115
|
interface ModelResponse {
|
|
@@ -83,9 +133,14 @@ export class OpenRouterBenchmarkSource implements BenchmarkSource {
|
|
|
83
133
|
const bytes = new TextEncoder().encode(text).byteLength;
|
|
84
134
|
if (bytes > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("OpenRouter models response exceeds the size limit");
|
|
85
135
|
let payload: unknown;
|
|
86
|
-
try {
|
|
136
|
+
try {
|
|
137
|
+
payload = JSON.parse(text);
|
|
138
|
+
} catch {
|
|
139
|
+
throw new Error("OpenRouter models response is not valid JSON");
|
|
140
|
+
}
|
|
87
141
|
const root = contractRecord(payload, "models response");
|
|
88
|
-
if (!Array.isArray(root
|
|
142
|
+
if (!Array.isArray(root.data) || root.data.length === 0 || root.data.length > BENCHMARK_MAX_MODELS_PER_SOURCE)
|
|
143
|
+
throw new Error("OpenRouter models response exceeds the model limit");
|
|
89
144
|
return { models: parseOpenRouterModels(payload), etag: response.headers.get("etag")?.slice(0, 120) ?? null, bytes };
|
|
90
145
|
}
|
|
91
146
|
|
|
@@ -95,7 +150,8 @@ export class OpenRouterBenchmarkSource implements BenchmarkSource {
|
|
|
95
150
|
this.readModels(OPENROUTER_LATENCY_URL),
|
|
96
151
|
this.readModels(OPENROUTER_THROUGHPUT_URL),
|
|
97
152
|
]);
|
|
98
|
-
if (catalog.bytes + latency.bytes + throughput.bytes > BENCHMARK_SOURCE_MAX_TOTAL_RESPONSE_BYTES)
|
|
153
|
+
if (catalog.bytes + latency.bytes + throughput.bytes > BENCHMARK_SOURCE_MAX_TOTAL_RESPONSE_BYTES)
|
|
154
|
+
throw new Error("OpenRouter model evidence exceeds the total size limit");
|
|
99
155
|
const retrievedAt = this.clock();
|
|
100
156
|
if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
|
|
101
157
|
const revision = catalog.etag || latency.etag || throughput.etag || `retrieved:${retrievedAt}`;
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import { BENCHMARK_MAX_MODELS_PER_SOURCE, BENCHMARK_REFRESH_INTERVAL_MS, BENCHMARK_SOURCE_MAX_RESPONSE_BYTES } from "../constants.ts";
|
|
1
2
|
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
3
|
+
type BenchmarkObservation,
|
|
4
|
+
type BenchmarkSourceSnapshot,
|
|
5
|
+
normalizeModelIdentity,
|
|
6
|
+
validateBenchmarkObservation,
|
|
7
|
+
} from "../domain/benchmark.ts";
|
|
7
8
|
import type { BenchmarkSource } from "../ports/benchmark-source.ts";
|
|
8
9
|
import { contractRecord } from "../providers/openrouter-contracts.ts";
|
|
9
10
|
import type { OpenRouterBenchmarkTransport } from "./openrouter-benchmark-source.ts";
|
|
@@ -22,7 +23,8 @@ const BASE_URL = `https://openrouter.ai/api/v1/benchmarks?source=design-arena&ma
|
|
|
22
23
|
const RELEVANT_CATEGORIES = ["codecategories", "website", "uicomponent", "dataviz", "svg"] as const;
|
|
23
24
|
|
|
24
25
|
function requiredText(value: unknown, name: string): string {
|
|
25
|
-
if (typeof value !== "string" || value.length === 0 || value.length > 500)
|
|
26
|
+
if (typeof value !== "string" || value.length === 0 || value.length > 500)
|
|
27
|
+
throw new Error(`OpenRouter Design Arena benchmark ${name} schema changed`);
|
|
26
28
|
return value;
|
|
27
29
|
}
|
|
28
30
|
|
|
@@ -62,24 +64,32 @@ export class OpenRouterDesignArenaSource implements BenchmarkSource {
|
|
|
62
64
|
const response = await this.transport(new Request(url, { headers: { authorization: `Bearer ${this.apiKey}` } }));
|
|
63
65
|
if (!response.ok) throw new Error(`OpenRouter Design Arena benchmarks failed with HTTP ${response.status}`);
|
|
64
66
|
const text = await response.text();
|
|
65
|
-
if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
|
|
67
|
+
if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES)
|
|
68
|
+
throw new Error("OpenRouter Design Arena benchmark response exceeds the size limit");
|
|
66
69
|
let payload: unknown;
|
|
67
|
-
try {
|
|
70
|
+
try {
|
|
71
|
+
payload = JSON.parse(text);
|
|
72
|
+
} catch {
|
|
73
|
+
throw new Error("OpenRouter Design Arena benchmark response is not valid JSON");
|
|
74
|
+
}
|
|
68
75
|
const root = contractRecord(payload, "benchmark response");
|
|
69
|
-
const meta = contractRecord(root
|
|
70
|
-
if (!Array.isArray(root
|
|
71
|
-
|
|
72
|
-
|
|
76
|
+
const meta = contractRecord(root.meta, "benchmark metadata");
|
|
77
|
+
if (!Array.isArray(root.data) || root.data.length > BENCHMARK_MAX_MODELS_PER_SOURCE)
|
|
78
|
+
throw new Error("OpenRouter Design Arena benchmark result count is invalid");
|
|
79
|
+
if (requiredText(meta.source, "source") !== "design-arena") throw new Error("OpenRouter Design Arena benchmark source schema changed");
|
|
80
|
+
const asOf = requiredText(meta.as_of, "as-of date");
|
|
73
81
|
const publishedAt = Date.parse(asOf);
|
|
74
|
-
if (!Number.isSafeInteger(publishedAt) || publishedAt <= 0)
|
|
82
|
+
if (!Number.isSafeInteger(publishedAt) || publishedAt <= 0)
|
|
83
|
+
throw new Error("OpenRouter Design Arena benchmark publication date schema changed");
|
|
75
84
|
const revision = `${category}:${asOf}`;
|
|
76
|
-
const observations = root
|
|
85
|
+
const observations = root.data.flatMap((value): BenchmarkObservation[] => {
|
|
77
86
|
const row = contractRecord(value, "benchmark row");
|
|
78
|
-
if (requiredText(row
|
|
87
|
+
if (requiredText(row.source, "row source") !== "design-arena")
|
|
88
|
+
throw new Error("OpenRouter Design Arena benchmark row source schema changed");
|
|
79
89
|
// A null open_router_id means the model isn't reachable through OpenRouter at all
|
|
80
90
|
// (proprietary platform, image/video generator, deprecated model) -- not evidence
|
|
81
91
|
// Jittor can ever route against, so the row is skipped rather than rejected.
|
|
82
|
-
const openRouterId = row
|
|
92
|
+
const openRouterId = row.open_router_id;
|
|
83
93
|
if (openRouterId === null) return [];
|
|
84
94
|
const { provider, model } = identityFromOpenRouterId(requiredText(openRouterId, "open_router_id"));
|
|
85
95
|
const identity = normalizeModelIdentity(provider, model, [`openrouter/${provider}/${model}`]);
|
|
@@ -96,9 +106,16 @@ export class OpenRouterDesignArenaSource implements BenchmarkSource {
|
|
|
96
106
|
confidence: 0.7,
|
|
97
107
|
};
|
|
98
108
|
const methodology = { basis: "Design Arena Elo rating via OpenRouter", category, asOf };
|
|
99
|
-
return [
|
|
100
|
-
|
|
101
|
-
|
|
109
|
+
return [
|
|
110
|
+
validateBenchmarkObservation({
|
|
111
|
+
model: identity,
|
|
112
|
+
dimension: "quality-design",
|
|
113
|
+
value: requiredNumber(row.elo, "elo rating"),
|
|
114
|
+
unit: "elo",
|
|
115
|
+
provenance,
|
|
116
|
+
methodology,
|
|
117
|
+
}),
|
|
118
|
+
];
|
|
102
119
|
});
|
|
103
120
|
return { revision, observations };
|
|
104
121
|
}
|
|
@@ -40,22 +40,26 @@ export class SQLiteMetricStore implements MetricStore {
|
|
|
40
40
|
|
|
41
41
|
recordBatch(inputs: MetricObservation[]): StoredMetricObservation[] {
|
|
42
42
|
const observations = inputs.map((input) => validateMetricObservation(input));
|
|
43
|
-
return this.db.transaction((rows: typeof observations) =>
|
|
43
|
+
return this.db.transaction((rows: typeof observations) =>
|
|
44
|
+
rows.map((observation) => this.get(Number(this.insert(observation).lastInsertRowid))),
|
|
45
|
+
)(observations);
|
|
44
46
|
}
|
|
45
47
|
|
|
46
48
|
private insert(observation: MetricObservation): { lastInsertRowid: number | bigint } {
|
|
47
|
-
return this.db
|
|
49
|
+
return this.db
|
|
50
|
+
.query(`
|
|
48
51
|
INSERT INTO metric_observations (source, scope, metric, value, unit, observed_at, attributes)
|
|
49
52
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
50
|
-
`)
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
53
|
+
`)
|
|
54
|
+
.run(
|
|
55
|
+
observation.source,
|
|
56
|
+
observation.scope,
|
|
57
|
+
observation.metric,
|
|
58
|
+
observation.value,
|
|
59
|
+
observation.unit,
|
|
60
|
+
observation.observedAt,
|
|
61
|
+
JSON.stringify(observation.attributes ?? {}),
|
|
62
|
+
);
|
|
59
63
|
}
|
|
60
64
|
|
|
61
65
|
query(filter: MetricQuery = {}): StoredMetricObservation[] {
|
|
@@ -69,30 +73,40 @@ export class SQLiteMetricStore implements MetricStore {
|
|
|
69
73
|
addEquals("source", filter.source);
|
|
70
74
|
addEquals("scope", filter.scope);
|
|
71
75
|
addEquals("metric", filter.metric);
|
|
72
|
-
if (filter.since !== undefined) {
|
|
73
|
-
|
|
76
|
+
if (filter.since !== undefined) {
|
|
77
|
+
conditions.push("observed_at >= ?");
|
|
78
|
+
parameters.push(filter.since);
|
|
79
|
+
}
|
|
80
|
+
if (filter.until !== undefined) {
|
|
81
|
+
conditions.push("observed_at <= ?");
|
|
82
|
+
parameters.push(filter.until);
|
|
83
|
+
}
|
|
74
84
|
const requestedLimit = Number.isFinite(filter.limit) ? Math.floor(filter.limit!) : DEFAULT_QUERY_LIMIT;
|
|
75
85
|
const limit = Math.max(1, Math.min(MAX_QUERY_LIMIT, requestedLimit));
|
|
76
86
|
const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
|
|
77
87
|
const order = filter.order === "desc" ? "DESC" : "ASC";
|
|
78
|
-
const rows = this.db
|
|
88
|
+
const rows = this.db
|
|
89
|
+
.query(`
|
|
79
90
|
SELECT id, source, scope, metric, value, unit, observed_at, attributes
|
|
80
91
|
FROM metric_observations
|
|
81
92
|
${where}
|
|
82
93
|
ORDER BY observed_at ${order}, id ${order}
|
|
83
94
|
LIMIT ${limit}
|
|
84
|
-
`)
|
|
95
|
+
`)
|
|
96
|
+
.all(...parameters) as MetricRow[];
|
|
85
97
|
return rows.map(fromRow);
|
|
86
98
|
}
|
|
87
99
|
|
|
88
100
|
distinctScopes(filter: DistinctScopesFilter): string[] {
|
|
89
101
|
const limit = Math.max(1, Math.min(MAX_QUERY_LIMIT, Math.floor(filter.limit)));
|
|
90
|
-
const rows = this.db
|
|
102
|
+
const rows = this.db
|
|
103
|
+
.query(`
|
|
91
104
|
SELECT DISTINCT scope FROM metric_observations
|
|
92
105
|
WHERE source = ? AND observed_at >= ? AND observed_at <= ?
|
|
93
106
|
ORDER BY scope ASC
|
|
94
107
|
LIMIT ?
|
|
95
|
-
`)
|
|
108
|
+
`)
|
|
109
|
+
.all(filter.source, filter.since, filter.until, limit) as Array<{ scope: string }>;
|
|
96
110
|
return rows.map((row) => row.scope);
|
|
97
111
|
}
|
|
98
112
|
|
|
@@ -104,7 +118,8 @@ export class SQLiteMetricStore implements MetricStore {
|
|
|
104
118
|
// MIN(a, b) is SQLite's scalar two-argument form (smallest of the arguments), mirroring
|
|
105
119
|
// usageBucketIndex's own `Math.min(bucketCount - 1, Math.floor(...))` clamp exactly, so a bucket
|
|
106
120
|
// this returns always matches the same-indexed bucket the client-side window renders.
|
|
107
|
-
const rows = this.db
|
|
121
|
+
const rows = this.db
|
|
122
|
+
.query(`
|
|
108
123
|
SELECT scope, metric,
|
|
109
124
|
MIN(CAST((observed_at - ?) AS REAL) / ?, ? - 1) AS bucket_index_raw,
|
|
110
125
|
SUM(value) AS sum
|
|
@@ -112,11 +127,17 @@ export class SQLiteMetricStore implements MetricStore {
|
|
|
112
127
|
WHERE source = ? AND observed_at >= ? AND observed_at <= ? AND value >= 0 AND scope IN (${scopePlaceholders})
|
|
113
128
|
GROUP BY scope, metric, CAST(bucket_index_raw AS INTEGER)
|
|
114
129
|
LIMIT ?
|
|
115
|
-
`)
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
130
|
+
`)
|
|
131
|
+
.all(
|
|
132
|
+
filter.since,
|
|
133
|
+
filter.bucketSizeMs,
|
|
134
|
+
filter.bucketCount,
|
|
135
|
+
filter.source,
|
|
136
|
+
filter.since,
|
|
137
|
+
filter.until,
|
|
138
|
+
...filter.scopes,
|
|
139
|
+
USAGE_AGGREGATE_MAX_ROWS,
|
|
140
|
+
) as Array<{ scope: string; metric: string; bucket_index_raw: number; sum: number }>;
|
|
120
141
|
return rows.map((row) => ({
|
|
121
142
|
scope: row.scope,
|
|
122
143
|
metric: row.metric,
|
|
@@ -139,10 +160,12 @@ export class SQLiteMetricStore implements MetricStore {
|
|
|
139
160
|
}
|
|
140
161
|
|
|
141
162
|
private get(id: number): StoredMetricObservation {
|
|
142
|
-
const row = this.db
|
|
163
|
+
const row = this.db
|
|
164
|
+
.query(`
|
|
143
165
|
SELECT id, source, scope, metric, value, unit, observed_at, attributes
|
|
144
166
|
FROM metric_observations WHERE id = ?
|
|
145
|
-
`)
|
|
167
|
+
`)
|
|
168
|
+
.get(id) as MetricRow | null;
|
|
146
169
|
if (!row) throw new Error(`metric observation ${id} was not persisted`);
|
|
147
170
|
return fromRow(row);
|
|
148
171
|
}
|