@danypops/jittor 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -5
- package/docs/PROVIDER_RESEARCH.md +21 -2
- package/extension/src/benchmark-tui.ts +7 -5
- package/extension/src/footer.ts +38 -22
- package/extension/src/index.ts +73 -9
- package/extension/src/tui.ts +29 -2
- package/extension/src/usage.ts +24 -15
- package/package.json +5 -1
- package/src/adapters/openrouter-benchmark-index-source.ts +2 -1
- package/src/adapters/openrouter-design-arena-source.ts +105 -0
- package/src/adapters/sqlite-metric-store.ts +43 -2
- package/src/cli.ts +139 -10
- package/src/client.ts +12 -44
- package/src/constants.ts +31 -4
- package/src/daemon.ts +52 -47
- package/src/db.ts +13 -30
- package/src/domain/metric.ts +1 -1
- package/src/domain/model-observation.ts +41 -19
- package/src/domain/model-ranking-service.ts +3 -2
- package/src/domain/model-ranking.ts +31 -12
- package/src/domain/task-cost.ts +70 -0
- package/src/domain/task-focus.ts +65 -0
- package/src/domain/usage.ts +80 -56
- package/src/log.ts +28 -0
- package/src/ports/metric-store.ts +30 -0
- package/src/providers/anthropic-contracts.ts +22 -11
- package/src/providers/google-adc-auth.ts +63 -0
- package/src/providers/google-vertex-budget-contracts.ts +181 -0
- package/src/providers/google-vertex-budget.ts +127 -0
- package/src/providers/google-vertex-contracts.ts +14 -2
- package/src/providers/telemetry-sources.ts +35 -0
- package/src/service.ts +80 -22
- package/src/state.ts +31 -57
- package/src/version.ts +2 -14
package/src/daemon.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { startDaemon as startDaemonKit, type RunningDaemon } from "@danypops/daemon-kit/daemon";
|
|
2
|
+
import { MAINTENANCE_INTERVAL_MS, TELEMETRY_POLL_INTERVAL_MS } from "./constants.ts";
|
|
2
3
|
import { DEFAULT_POLICY, UNCONFIGURED_ROUTE } from "./config.ts";
|
|
3
4
|
import { SQLiteMetricStore } from "./adapters/sqlite-metric-store.ts";
|
|
4
5
|
import { MetricBenchmarkStore } from "./adapters/metric-benchmark-store.ts";
|
|
5
6
|
import { OpenRouterBenchmarkIndexSource } from "./adapters/openrouter-benchmark-index-source.ts";
|
|
6
7
|
import { OpenRouterBenchmarkSource } from "./adapters/openrouter-benchmark-source.ts";
|
|
8
|
+
import { OpenRouterDesignArenaSource } from "./adapters/openrouter-design-arena-source.ts";
|
|
7
9
|
import { openJittorDb } from "./db.ts";
|
|
8
10
|
import { BenchmarkCatalog } from "./domain/benchmark.ts";
|
|
9
11
|
import { EvidenceModelRanker } from "./domain/model-ranking-service.ts";
|
|
@@ -11,25 +13,26 @@ import { createApp, JittorService } from "./service.ts";
|
|
|
11
13
|
import { JittorRouter } from "./router.ts";
|
|
12
14
|
import type { BenchmarkSource } from "./ports/benchmark-source.ts";
|
|
13
15
|
import type { TelemetrySource } from "./ports/telemetry-source.ts";
|
|
14
|
-
import { CodexTelemetrySource, OpenRouterTelemetrySource } from "./providers/telemetry-sources.ts";
|
|
15
|
-
import {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
type JittorPaths,
|
|
21
|
-
} from "./state.ts";
|
|
16
|
+
import { CodexTelemetrySource, GoogleVertexBudgetTelemetrySource, OpenRouterTelemetrySource } from "./providers/telemetry-sources.ts";
|
|
17
|
+
import { createGoogleAdcTokenProvider } from "./providers/google-adc-auth.ts";
|
|
18
|
+
import { GOOGLE_PUBSUB_READONLY_SCOPE } from "./providers/google-vertex-budget.ts";
|
|
19
|
+
import type { GoogleVertexMetricSource } from "./providers/google-vertex-contracts.ts";
|
|
20
|
+
import { ensureAuthToken, resolveJittorPaths, type JittorPaths } from "./state.ts";
|
|
21
|
+
import { logEvent, logger } from "./log.ts";
|
|
22
22
|
|
|
23
|
-
export
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
export type { RunningDaemon } from "@danypops/daemon-kit/daemon";
|
|
24
|
+
|
|
25
|
+
export function reportMaintenanceFailure(event: string, error: unknown): void {
|
|
26
|
+
logEvent("error", event, { message: error instanceof Error ? error.message : String(error) });
|
|
27
27
|
}
|
|
28
28
|
|
|
29
29
|
export function benchmarkSourcesFromEnvironment(env: Record<string, string | undefined> = process.env): BenchmarkSource[] {
|
|
30
30
|
if (env["JITTOR_OPENROUTER_BENCHMARKS"] !== "1") return [];
|
|
31
31
|
const sources: BenchmarkSource[] = [new OpenRouterBenchmarkSource()];
|
|
32
|
-
if (env["OPENROUTER_API_KEY"])
|
|
32
|
+
if (env["OPENROUTER_API_KEY"]) {
|
|
33
|
+
sources.push(new OpenRouterBenchmarkIndexSource(env["OPENROUTER_API_KEY"]));
|
|
34
|
+
sources.push(new OpenRouterDesignArenaSource(env["OPENROUTER_API_KEY"]));
|
|
35
|
+
}
|
|
33
36
|
return sources;
|
|
34
37
|
}
|
|
35
38
|
|
|
@@ -39,9 +42,27 @@ export function telemetrySourcesFromEnvironment(env: Record<string, string | und
|
|
|
39
42
|
if (codexAuthFile) sources.push(new CodexTelemetrySource(codexAuthFile));
|
|
40
43
|
const openRouterKey = env["OPENROUTER_API_KEY"];
|
|
41
44
|
if (openRouterKey) sources.push(new OpenRouterTelemetrySource(openRouterKey));
|
|
45
|
+
// Opt-in only: the Pub/Sub subscription is one-time GCP console/CLI setup outside Jittor (see
|
|
46
|
+
// docs/PROVIDER_RESEARCH.md), so its absence must never attempt ADC discovery or a network call.
|
|
47
|
+
const vertexBudgetSubscription = env["JITTOR_GOOGLE_VERTEX_BUDGET_SUBSCRIPTION"];
|
|
48
|
+
if (vertexBudgetSubscription) {
|
|
49
|
+
const source = (env["JITTOR_GOOGLE_VERTEX_BUDGET_SOURCE"] ?? "google-vertex") as GoogleVertexMetricSource;
|
|
50
|
+
const tokenProvider = createGoogleAdcTokenProvider([GOOGLE_PUBSUB_READONLY_SCOPE]);
|
|
51
|
+
sources.push(new GoogleVertexBudgetTelemetrySource(vertexBudgetSubscription, tokenProvider, Date.now, fetch, source));
|
|
52
|
+
}
|
|
42
53
|
return sources;
|
|
43
54
|
}
|
|
44
55
|
|
|
56
|
+
/**
|
|
57
|
+
* Composition root, now built on `@danypops/daemon-kit/daemon`'s `startDaemon` for binding,
|
|
58
|
+
* atomic handle write, maintenance-timer driving, and clean shutdown -- the skeleton that used to
|
|
59
|
+
* be hand-rolled here (and, byte-identically, in web-spider-daemon's and papyrus's daemon.ts; see
|
|
60
|
+
* daemon-kit's README). Each maintenance task still catches and classifies its own failure via
|
|
61
|
+
* `reportMaintenanceFailure` (preserving Jittor's specific `checkpoint_failed`/
|
|
62
|
+
* `benchmark_refresh_failed`/`telemetry_poll_failed` event taxonomy) rather than relying on
|
|
63
|
+
* daemon-kit's own generic "maintenance task failed: <name>" catch, which exists as a safety net
|
|
64
|
+
* for tasks that don't self-classify, not to replace a consumer's own richer classification.
|
|
65
|
+
*/
|
|
45
66
|
export function startDaemon(
|
|
46
67
|
paths: JittorPaths = resolveJittorPaths(),
|
|
47
68
|
env: Record<string, string | undefined> = process.env,
|
|
@@ -61,40 +82,24 @@ export function startDaemon(
|
|
|
61
82
|
currentRoute: UNCONFIGURED_ROUTE,
|
|
62
83
|
});
|
|
63
84
|
const service = new JittorService(metrics, router, benchmarks, modelRanker);
|
|
64
|
-
|
|
65
|
-
const
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
85
|
+
|
|
86
|
+
const daemon = startDaemonKit({
|
|
87
|
+
daemonLabel: "Jittor",
|
|
88
|
+
handlePath: paths.handle,
|
|
89
|
+
logger,
|
|
90
|
+
buildApp: () => createApp({ service, token }),
|
|
91
|
+
maintenanceTasks: [
|
|
92
|
+
{ name: "checkpoint", intervalMs: MAINTENANCE_INTERVAL_MS, run: async () => { await service.execute("service.checkpoint", {}).catch((error) => reportMaintenanceFailure("checkpoint_failed", error)); } },
|
|
93
|
+
{ name: "benchmark-refresh", intervalMs: MAINTENANCE_INTERVAL_MS, run: async () => { await benchmarks.refresh().catch((error) => reportMaintenanceFailure("benchmark_refresh_failed", error)); } },
|
|
94
|
+
{ name: "telemetry-poll", intervalMs: TELEMETRY_POLL_INTERVAL_MS, run: async () => { await router.poll().catch((error) => reportMaintenanceFailure("telemetry_poll_failed", error)); } },
|
|
95
|
+
],
|
|
96
|
+
onShutdown: () => { service.close(); },
|
|
69
97
|
});
|
|
70
|
-
|
|
71
|
-
if (
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
}
|
|
76
|
-
writeDaemonHandle(paths, { host: LOOPBACK_HOST, port, pid: process.pid });
|
|
77
|
-
const maintenance = setInterval(() => {
|
|
78
|
-
void service.execute("service.checkpoint", {});
|
|
79
|
-
void benchmarks.refresh();
|
|
80
|
-
}, MAINTENANCE_INTERVAL_MS);
|
|
81
|
-
const poll = setInterval(() => { void router.poll(); }, TELEMETRY_POLL_INTERVAL_MS);
|
|
82
|
-
if (sources.length > 0) void router.poll();
|
|
83
|
-
if (benchmarkSources.length > 0) void benchmarks.refresh();
|
|
84
|
-
let stopped = false;
|
|
85
|
-
return {
|
|
86
|
-
host: LOOPBACK_HOST,
|
|
87
|
-
port,
|
|
88
|
-
async stop(): Promise<void> {
|
|
89
|
-
if (stopped) return;
|
|
90
|
-
stopped = true;
|
|
91
|
-
clearInterval(maintenance);
|
|
92
|
-
clearInterval(poll);
|
|
93
|
-
await server.stop(true);
|
|
94
|
-
service.close();
|
|
95
|
-
removeDaemonHandle(paths);
|
|
96
|
-
},
|
|
97
|
-
};
|
|
98
|
+
|
|
99
|
+
if (sources.length > 0) router.poll().catch((error) => reportMaintenanceFailure("telemetry_poll_failed", error));
|
|
100
|
+
if (benchmarkSources.length > 0) benchmarks.refresh().catch((error) => reportMaintenanceFailure("benchmark_refresh_failed", error));
|
|
101
|
+
|
|
102
|
+
return daemon;
|
|
98
103
|
}
|
|
99
104
|
|
|
100
105
|
export function serveMain(): void {
|
package/src/db.ts
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Database } from "bun:sqlite";
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import { SQLITE_BUSY_TIMEOUT_MS, SQLITE_SCHEMA_VERSION } from "./constants.ts";
|
|
1
|
+
import type { Database } from "bun:sqlite";
|
|
2
|
+
import { openSqliteWithPragmas } from "@danypops/daemon-kit/storage";
|
|
3
|
+
import { SQLITE_BUSY_TIMEOUT_MS } from "./constants.ts";
|
|
5
4
|
|
|
6
5
|
const INITIAL_SCHEMA = `
|
|
7
6
|
CREATE TABLE metric_observations (
|
|
@@ -20,31 +19,15 @@ CREATE INDEX metric_observations_time_idx
|
|
|
20
19
|
ON metric_observations(observed_at);
|
|
21
20
|
`;
|
|
22
21
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
if (row.user_version < 1) {
|
|
29
|
-
const migration = db.transaction(() => {
|
|
30
|
-
db.exec(INITIAL_SCHEMA);
|
|
31
|
-
db.exec("PRAGMA user_version = 1");
|
|
32
|
-
});
|
|
33
|
-
migration.immediate();
|
|
34
|
-
}
|
|
35
|
-
const migrated = db.query("PRAGMA user_version").get() as { user_version: number };
|
|
36
|
-
if (migrated.user_version !== SQLITE_SCHEMA_VERSION) {
|
|
37
|
-
throw new Error(`missing migration from schema ${migrated.user_version} to ${SQLITE_SCHEMA_VERSION}`);
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Delegates bootstrap (pragmas, migration engine) to `@danypops/daemon-kit/storage`, which
|
|
24
|
+
* generalizes the byte-identical pragma/PRAGMA-user_version skeleton jittor's own db.ts used to
|
|
25
|
+
* hand-roll (see daemon-kit's README). Jittor's only remaining responsibility is its own schema.
|
|
26
|
+
*/
|
|
41
27
|
export function openJittorDb(path: string): Database {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
migrate(db);
|
|
48
|
-
db.exec("PRAGMA optimize=0x10002");
|
|
49
|
-
return db;
|
|
28
|
+
return openSqliteWithPragmas(path, {
|
|
29
|
+
databaseOptions: { create: true, strict: true },
|
|
30
|
+
busyTimeoutMs: SQLITE_BUSY_TIMEOUT_MS,
|
|
31
|
+
migrations: [{ version: 1, up: (db) => db.exec(INITIAL_SCHEMA) }],
|
|
32
|
+
});
|
|
50
33
|
}
|
package/src/domain/metric.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { METRIC_ATTRIBUTES_MAX_DEPTH, METRIC_ATTRIBUTES_MAX_SERIALIZED_CHARACTERS, METRIC_IDENTITY_MAX_CHARACTERS } from "../constants.ts";
|
|
2
2
|
|
|
3
|
-
export const METRIC_UNITS = ["ratio", "usd", "tokens", "tokens-per-second", "requests", "milliseconds", "count"] as const;
|
|
3
|
+
export const METRIC_UNITS = ["ratio", "usd", "tokens", "tokens-per-second", "requests", "milliseconds", "count", "elo"] as const;
|
|
4
4
|
export type MetricUnit = typeof METRIC_UNITS[number];
|
|
5
5
|
|
|
6
6
|
export interface MetricObservation {
|
|
@@ -7,8 +7,19 @@ import {
|
|
|
7
7
|
import { normalizeModelIdentity } from "./benchmark.ts";
|
|
8
8
|
import type { MetricObservation, MetricUnit, StoredMetricObservation } from "./metric.ts";
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Two independent axes, not one flat class: "coding" is a subject-matter domain (which
|
|
12
|
+
* benchmark quality evidence applies), while "research"/"planning" are activities that can
|
|
13
|
+
* happen inside any domain (which predict how much reasoning effort a task needs). The prior
|
|
14
|
+
* single ModelTaskClass conflated them -- an agentic/tool-use benchmark (a type signal) was
|
|
15
|
+
* being read as if it were a domain-quality signal. Both axes default to "general" when tool
|
|
16
|
+
* usage carries no distinguishing signal for that axis; a run can score coding on domain and
|
|
17
|
+
* research on type simultaneously (e.g. reading a file, then searching the web in one turn).
|
|
18
|
+
*/
|
|
19
|
+
export const TASK_DOMAINS = ["coding", "design", "general"] as const;
|
|
20
|
+
export type ModelTaskDomain = typeof TASK_DOMAINS[number];
|
|
21
|
+
export const TASK_TYPES = ["research", "planning", "general"] as const;
|
|
22
|
+
export type ModelTaskType = typeof TASK_TYPES[number];
|
|
12
23
|
export type ExplicitOutcome = "accepted" | "rejected" | "unknown";
|
|
13
24
|
|
|
14
25
|
export interface ModelRunObservation {
|
|
@@ -16,7 +27,8 @@ export interface ModelRunObservation {
|
|
|
16
27
|
provider: string;
|
|
17
28
|
model: string;
|
|
18
29
|
thinking: string;
|
|
19
|
-
|
|
30
|
+
domain: ModelTaskDomain;
|
|
31
|
+
type: ModelTaskType;
|
|
20
32
|
startedAt: number;
|
|
21
33
|
firstTokenAt: number | null;
|
|
22
34
|
completedAt: number;
|
|
@@ -36,7 +48,8 @@ export interface ModelMetricAggregate {
|
|
|
36
48
|
provider: string;
|
|
37
49
|
model: string;
|
|
38
50
|
thinking: string;
|
|
39
|
-
|
|
51
|
+
domain: ModelTaskDomain;
|
|
52
|
+
type: ModelTaskType;
|
|
40
53
|
dimension: string;
|
|
41
54
|
unit: MetricUnit;
|
|
42
55
|
sampleSize: number;
|
|
@@ -54,7 +67,7 @@ export interface ModelAggregateOptions {
|
|
|
54
67
|
}
|
|
55
68
|
|
|
56
69
|
const ALLOWED_FIELDS = new Set<keyof ModelRunObservation>([
|
|
57
|
-
"runId", "provider", "model", "thinking", "
|
|
70
|
+
"runId", "provider", "model", "thinking", "domain", "type", "startedAt", "firstTokenAt", "completedAt",
|
|
58
71
|
"inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "costUsd", "providerResponses",
|
|
59
72
|
"toolCalls", "toolFailures", "stopReason", "explicitOutcome",
|
|
60
73
|
]);
|
|
@@ -80,7 +93,8 @@ export function validateModelRunObservation(value: unknown): ModelRunObservation
|
|
|
80
93
|
const completedAt = nonNegative(input["completedAt"], "completion time", true);
|
|
81
94
|
const firstTokenAt = input["firstTokenAt"] === null ? null : nonNegative(input["firstTokenAt"], "first-token time", true);
|
|
82
95
|
if (completedAt < startedAt || (firstTokenAt !== null && (firstTokenAt < startedAt || firstTokenAt > completedAt))) throw new Error("model run timestamps are not ordered");
|
|
83
|
-
if (!
|
|
96
|
+
if (!TASK_DOMAINS.includes(input["domain"] as ModelTaskDomain)) throw new Error("task domain is invalid");
|
|
97
|
+
if (!TASK_TYPES.includes(input["type"] as ModelTaskType)) throw new Error("task type is invalid");
|
|
84
98
|
if (!STOP_REASONS.has(input["stopReason"] as ModelRunObservation["stopReason"])) throw new Error("stop reason is invalid");
|
|
85
99
|
if (!OUTCOMES.has(input["explicitOutcome"] as ExplicitOutcome)) throw new Error("explicit outcome is invalid");
|
|
86
100
|
const providerResponses = nonNegative(input["providerResponses"], "provider response count", true);
|
|
@@ -89,7 +103,7 @@ export function validateModelRunObservation(value: unknown): ModelRunObservation
|
|
|
89
103
|
if (providerResponses < 1 || toolFailures > toolCalls) throw new Error("model run counters are inconsistent");
|
|
90
104
|
return {
|
|
91
105
|
runId: text(input["runId"], "run id"), provider: identity.provider, model: identity.model,
|
|
92
|
-
thinking: text(input["thinking"], "thinking level"),
|
|
106
|
+
thinking: text(input["thinking"], "thinking level"), domain: input["domain"] as ModelTaskDomain, type: input["type"] as ModelTaskType,
|
|
93
107
|
startedAt, firstTokenAt, completedAt,
|
|
94
108
|
inputTokens: nonNegative(input["inputTokens"], "input tokens"),
|
|
95
109
|
outputTokens: nonNegative(input["outputTokens"], "output tokens"),
|
|
@@ -100,18 +114,25 @@ export function validateModelRunObservation(value: unknown): ModelRunObservation
|
|
|
100
114
|
};
|
|
101
115
|
}
|
|
102
116
|
|
|
103
|
-
export
|
|
117
|
+
export interface ModelTaskClassification {
|
|
118
|
+
domain: ModelTaskDomain;
|
|
119
|
+
type: ModelTaskType;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Domain and type are independent: a run can be domain=coding and type=research at once (e.g. reading a file, then searching the web in the same turn). */
|
|
123
|
+
export function classifyTaskFromTools(toolNames: string[]): ModelTaskClassification {
|
|
104
124
|
const names = new Set(toolNames.slice(0, 100).map((name) => name.toLowerCase()));
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
125
|
+
const domain: ModelTaskDomain = ["edit", "write", "read", "bash", "grep", "find", "ls"].some((name) => names.has(name)) ? "coding" : "general";
|
|
126
|
+
const type: ModelTaskType = ["web_fetch", "web_search"].some((name) => names.has(name))
|
|
127
|
+
? "research"
|
|
128
|
+
: ["tasks", "papyrus_create", "papyrus_graph"].some((name) => names.has(name)) ? "planning" : "general";
|
|
129
|
+
return { domain, type };
|
|
109
130
|
}
|
|
110
131
|
|
|
111
132
|
export function modelRunMetrics(value: ModelRunObservation): MetricObservation[] {
|
|
112
133
|
const run = validateModelRunObservation(value);
|
|
113
134
|
const scope = `${run.provider}/${run.model}`;
|
|
114
|
-
const attributes = { provider: run.provider, model: run.model, thinking: run.thinking,
|
|
135
|
+
const attributes = { provider: run.provider, model: run.model, thinking: run.thinking, domain: run.domain, type: run.type, runId: run.runId };
|
|
115
136
|
const metric = (name: string, amount: number, unit: MetricUnit): MetricObservation => ({ source: "local-model", scope, metric: name, value: amount, unit, observedAt: run.completedAt, attributes });
|
|
116
137
|
const wallMs = run.completedAt - run.startedAt;
|
|
117
138
|
const totalInput = run.inputTokens + run.cacheReadTokens;
|
|
@@ -155,16 +176,17 @@ export function aggregateModelMetrics(input: StoredMetricObservation[], options:
|
|
|
155
176
|
const provider = row.attributes["provider"];
|
|
156
177
|
const model = row.attributes["model"];
|
|
157
178
|
const thinking = row.attributes["thinking"];
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
|
|
179
|
+
const domain = row.attributes["domain"];
|
|
180
|
+
const type = row.attributes["type"];
|
|
181
|
+
if (typeof provider !== "string" || typeof model !== "string" || typeof thinking !== "string" || !TASK_DOMAINS.includes(domain as ModelTaskDomain) || !TASK_TYPES.includes(type as ModelTaskType)) continue;
|
|
182
|
+
const key = JSON.stringify([provider, model, thinking, domain, type, row.metric, row.unit]);
|
|
161
183
|
if (!groups.has(key) && groups.size >= MODEL_AGGREGATE_MAX_GROUPS) continue;
|
|
162
184
|
const rows = groups.get(key) ?? [];
|
|
163
185
|
rows.push(row);
|
|
164
186
|
groups.set(key, rows);
|
|
165
187
|
}
|
|
166
188
|
return [...groups.entries()].map(([key, rows]) => {
|
|
167
|
-
const [provider, model, thinking,
|
|
189
|
+
const [provider, model, thinking, domain, type, dimension, unit] = JSON.parse(key) as [string, string, string, ModelTaskDomain, ModelTaskType, string, MetricUnit];
|
|
168
190
|
const values = rows.map((row) => row.value as number).sort((left, right) => left - right);
|
|
169
191
|
const center = median(values);
|
|
170
192
|
const deviations = values.map((value) => Math.abs(value - center)).sort((left, right) => left - right);
|
|
@@ -172,10 +194,10 @@ export function aggregateModelMetrics(input: StoredMetricObservation[], options:
|
|
|
172
194
|
const age = Math.max(0, now - latestAt);
|
|
173
195
|
const recency = Math.max(0, 1 - (age / freshForMs));
|
|
174
196
|
return {
|
|
175
|
-
provider, model, thinking,
|
|
197
|
+
provider, model, thinking, domain, type, dimension, unit,
|
|
176
198
|
sampleSize: values.length, median: center, p90: percentile(values, 0.9), medianAbsoluteDeviation: median(deviations), latestAt,
|
|
177
199
|
freshness: age <= freshForMs ? "fresh" as const : "stale" as const,
|
|
178
200
|
confidence: Math.min(1, Math.sqrt(values.length / 20)) * recency,
|
|
179
201
|
};
|
|
180
|
-
}).sort((left, right) => left.provider.localeCompare(right.provider) || left.model.localeCompare(right.model) || left.dimension.localeCompare(right.dimension));
|
|
202
|
+
}).sort((left, right) => left.provider.localeCompare(right.provider) || left.model.localeCompare(right.model) || left.domain.localeCompare(right.domain) || left.type.localeCompare(right.type) || left.dimension.localeCompare(right.dimension));
|
|
181
203
|
}
|
|
@@ -3,12 +3,13 @@ import type { BenchmarkStore } from "../ports/benchmark-store.ts";
|
|
|
3
3
|
import type { MetricStore } from "../ports/metric-store.ts";
|
|
4
4
|
import { aggregateModelMetrics } from "./model-observation.ts";
|
|
5
5
|
import { rankModelCandidates, type ModelCandidate, type ModelRankingResult, type ScopeAuthority, type UtilityWeights } from "./model-ranking.ts";
|
|
6
|
-
import type {
|
|
6
|
+
import type { ModelTaskDomain, ModelTaskType } from "./model-observation.ts";
|
|
7
7
|
|
|
8
8
|
export interface ModelRecommendationInput {
|
|
9
9
|
candidates: ModelCandidate[];
|
|
10
10
|
scopeAuthority: ScopeAuthority;
|
|
11
|
-
|
|
11
|
+
domain: ModelTaskDomain;
|
|
12
|
+
type: ModelTaskType;
|
|
12
13
|
budgetPressure: number;
|
|
13
14
|
weights: UtilityWeights;
|
|
14
15
|
sourceIds: string[];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT, MAX_DYNAMIC_ROUTES, MODEL_AGGREGATE_MAX_ROWS } from "../constants.ts";
|
|
2
2
|
import { normalizeModelIdentity, type BenchmarkObservation } from "./benchmark.ts";
|
|
3
|
-
import {
|
|
3
|
+
import { TASK_DOMAINS, TASK_TYPES, type ModelMetricAggregate, type ModelTaskDomain, type ModelTaskType } from "./model-observation.ts";
|
|
4
4
|
|
|
5
5
|
export type ScopeAuthority = "exact-session" | "available-models";
|
|
6
6
|
export type UtilityComponentName = "quality" | "cost" | "latency" | "context" | "reliability";
|
|
@@ -22,7 +22,8 @@ export interface UtilityWeights {
|
|
|
22
22
|
export interface ModelRankingInput {
|
|
23
23
|
candidates: ModelCandidate[];
|
|
24
24
|
scopeAuthority: ScopeAuthority;
|
|
25
|
-
|
|
25
|
+
domain: ModelTaskDomain;
|
|
26
|
+
type: ModelTaskType;
|
|
26
27
|
budgetPressure: number;
|
|
27
28
|
weights: UtilityWeights;
|
|
28
29
|
externalEvidence: BenchmarkObservation[];
|
|
@@ -60,7 +61,8 @@ export interface RankedModel {
|
|
|
60
61
|
export interface ModelRankingResult {
|
|
61
62
|
scopeAuthority: ScopeAuthority;
|
|
62
63
|
scopeWarning: string | null;
|
|
63
|
-
|
|
64
|
+
domain: ModelTaskDomain;
|
|
65
|
+
type: ModelTaskType;
|
|
64
66
|
completeness: "complete" | "partial" | "insufficient-evidence";
|
|
65
67
|
ranked: RankedModel[];
|
|
66
68
|
automaticSelection: ModelCandidate | null;
|
|
@@ -101,22 +103,37 @@ function externalValues(candidate: ModelCandidate, evidence: BenchmarkObservatio
|
|
|
101
103
|
};
|
|
102
104
|
}
|
|
103
105
|
|
|
104
|
-
function localValues(candidate: ModelCandidate,
|
|
106
|
+
function localValues(candidate: ModelCandidate, domain: ModelTaskDomain, type: ModelTaskType, evidence: ModelMetricAggregate[], dimension: string): ModelMetricAggregate[] {
|
|
105
107
|
const identity = normalizeModelIdentity(candidate.provider, candidate.model);
|
|
106
|
-
return evidence.filter((item) => item.provider === identity.provider && item.model === identity.model && item.thinking === candidate.thinking && item.
|
|
108
|
+
return evidence.filter((item) => item.provider === identity.provider && item.model === identity.model && item.thinking === candidate.thinking && item.domain === domain && item.type === type && item.dimension === dimension);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* quality-{domain} is a subject-matter signal (e.g. quality-coding from a coding benchmark);
|
|
113
|
+
* quality-type-{type} is an activity signal (e.g. quality-type-planning from an agentic/tool-use
|
|
114
|
+
* benchmark). Both are optional and additive on top of the universal quality-general fallback --
|
|
115
|
+
* a candidate with no domain- or type-specific evidence still gets ranked on general quality
|
|
116
|
+
* rather than being treated as having zero evidence.
|
|
117
|
+
*/
|
|
118
|
+
function qualityDimensions(domain: ModelTaskDomain, type: ModelTaskType): string[] {
|
|
119
|
+
const dimensions: string[] = [];
|
|
120
|
+
if (domain !== "general") dimensions.push(`quality-${domain}`);
|
|
121
|
+
if (type !== "general") dimensions.push(`quality-type-${type}`);
|
|
122
|
+
dimensions.push("quality-general");
|
|
123
|
+
return dimensions;
|
|
107
124
|
}
|
|
108
125
|
|
|
109
126
|
function rawComponents(candidate: ModelCandidate, input: ModelRankingInput): { components: Record<UtilityComponentName, RawComponent>; provenance: RankingProvenance[] } {
|
|
110
|
-
const quality = externalValues(candidate, input.externalEvidence,
|
|
127
|
+
const quality = externalValues(candidate, input.externalEvidence, qualityDimensions(input.domain, input.type), input.now);
|
|
111
128
|
const priceInput = externalValues(candidate, input.externalEvidence, ["price-input"], input.now);
|
|
112
129
|
const priceOutput = externalValues(candidate, input.externalEvidence, ["price-output"], input.now);
|
|
113
130
|
const measuredLatency = externalValues(candidate, input.externalEvidence, ["latency"], input.now);
|
|
114
131
|
const rankedLatency = externalValues(candidate, input.externalEvidence, ["latency-rank", "throughput-rank"], input.now);
|
|
115
132
|
const latency = measuredLatency.values.length > 0 ? measuredLatency : rankedLatency;
|
|
116
133
|
const context = externalValues(candidate, input.externalEvidence, ["context-window"], input.now);
|
|
117
|
-
const localLatency = localValues(candidate, input.
|
|
118
|
-
const failures = localValues(candidate, input.
|
|
119
|
-
const outcomes = localValues(candidate, input.
|
|
134
|
+
const localLatency = localValues(candidate, input.domain, input.type, input.localEvidence, "wall-latency");
|
|
135
|
+
const failures = localValues(candidate, input.domain, input.type, input.localEvidence, "failure");
|
|
136
|
+
const outcomes = localValues(candidate, input.domain, input.type, input.localEvidence, "outcome-accepted");
|
|
120
137
|
const qualityValues = quality.values;
|
|
121
138
|
const prices = [...priceInput.values, ...priceOutput.values];
|
|
122
139
|
const latencyValues = localLatency.length > 0 ? localLatency.map((item) => item.median) : latency.values;
|
|
@@ -154,7 +171,8 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
154
171
|
if (!Array.isArray(value.externalEvidence) || value.externalEvidence.length > BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT * 4) throw new Error("external evidence exceeds the supported bound");
|
|
155
172
|
if (!Array.isArray(value.localEvidence) || value.localEvidence.length > MODEL_AGGREGATE_MAX_ROWS) throw new Error("local evidence exceeds the supported bound");
|
|
156
173
|
if (value.scopeAuthority !== "exact-session" && value.scopeAuthority !== "available-models") throw new Error("scope authority is invalid");
|
|
157
|
-
if (!
|
|
174
|
+
if (!TASK_DOMAINS.includes(value.domain)) throw new Error("task domain is invalid");
|
|
175
|
+
if (!TASK_TYPES.includes(value.type)) throw new Error("task type is invalid");
|
|
158
176
|
if (!Number.isSafeInteger(value.now) || value.now <= 0) throw new Error("ranking time is invalid");
|
|
159
177
|
const budgetPressure = finiteBound(value.budgetPressure, "budget pressure", 0, 2);
|
|
160
178
|
const weights = Object.fromEntries(COMPONENTS.map((name) => [name, finiteBound(value.weights[name], `${name} weight`, 0, 10)])) as unknown as UtilityWeights;
|
|
@@ -195,7 +213,7 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
195
213
|
confidence,
|
|
196
214
|
components,
|
|
197
215
|
provenance,
|
|
198
|
-
trace: [`
|
|
216
|
+
trace: [`domain ${value.domain}, type ${value.type}`, `budget pressure ${budgetPressure.toFixed(3)} makes cost weight ${effectiveWeights.cost.toFixed(3)}`, `${known.length}/${components.length} utility components have evidence`, `scope authority ${value.scopeAuthority}`],
|
|
199
217
|
};
|
|
200
218
|
}).sort((left, right) => (right.utility ?? -1) - (left.utility ?? -1) || right.confidence - left.confidence || left.identity.localeCompare(right.identity));
|
|
201
219
|
const knownComponents = ranked.reduce((sum, item) => sum + item.components.filter((component) => component.score !== null).length, 0);
|
|
@@ -205,7 +223,8 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
205
223
|
return {
|
|
206
224
|
scopeAuthority: value.scopeAuthority,
|
|
207
225
|
scopeWarning: exact ? null : "Pi available models are not the exact session scope; automatic selection is disabled",
|
|
208
|
-
|
|
226
|
+
domain: value.domain,
|
|
227
|
+
type: value.type,
|
|
209
228
|
completeness,
|
|
210
229
|
ranked,
|
|
211
230
|
automaticSelection: exact && ranked[0]?.utility !== null ? ranked[0]!.candidate : null,
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { StoredMetricObservation } from "./metric.ts";
|
|
2
|
+
|
|
3
|
+
export interface TaskCostEntry {
|
|
4
|
+
taskId: string;
|
|
5
|
+
costUsd: number;
|
|
6
|
+
inputTokens: number;
|
|
7
|
+
outputTokens: number;
|
|
8
|
+
cacheReadTokens: number;
|
|
9
|
+
cacheWriteTokens: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface TaskCostSummary {
|
|
13
|
+
since: number;
|
|
14
|
+
until: number;
|
|
15
|
+
entries: TaskCostEntry[];
|
|
16
|
+
unattributedCostUsd: number;
|
|
17
|
+
truncated: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface TaskCostSummaryOptions {
|
|
21
|
+
since: number;
|
|
22
|
+
until: number;
|
|
23
|
+
truncated?: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const TOKEN_METRICS = new Set(["input-tokens", "output-tokens", "cache-read-tokens", "cache-write-tokens"]);
|
|
27
|
+
|
|
28
|
+
function entryFor(byTask: Map<string, TaskCostEntry>, taskId: string): TaskCostEntry {
|
|
29
|
+
const existing = byTask.get(taskId);
|
|
30
|
+
if (existing) return existing;
|
|
31
|
+
const created: TaskCostEntry = { taskId, costUsd: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
|
|
32
|
+
byTask.set(taskId, created);
|
|
33
|
+
return created;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Groups already-recorded "pi" source cost/token metrics by the Papyrus task focused when each was
|
|
38
|
+
* recorded (see the papyrus.task-focus.v1 real-time tagging in the extension), rather than by
|
|
39
|
+
* provider/model identity. Rows recorded with nothing focused have no attributes.taskId and are
|
|
40
|
+
* reported separately as unattributedCostUsd -- they are real spend, just not attributable to any
|
|
41
|
+
* task, and must never be silently dropped or folded into an invented "unknown" task bucket.
|
|
42
|
+
*
|
|
43
|
+
* This queries a single bounded time window without per-task fairness partitioning (unlike
|
|
44
|
+
* buildUsageGraph/buildCostGraph's per-scope fetch): a task's own working period is typically a far
|
|
45
|
+
* narrower, more targeted window than "the whole usage dashboard, any period", so one task's volume
|
|
46
|
+
* crowding another out of the same query is a much smaller risk here. If this does become a problem
|
|
47
|
+
* in practice, the same per-distinct-value fetch strategy applies, keyed by attributes.taskId.
|
|
48
|
+
*/
|
|
49
|
+
export function buildTaskCostSummary(rows: StoredMetricObservation[], options: TaskCostSummaryOptions): TaskCostSummary {
|
|
50
|
+
const byTask = new Map<string, TaskCostEntry>();
|
|
51
|
+
let unattributedCostUsd = 0;
|
|
52
|
+
for (const row of rows) {
|
|
53
|
+
if (row.source !== "pi" || typeof row.value !== "number" || !Number.isFinite(row.value) || row.value < 0) continue;
|
|
54
|
+
if (row.observedAt < options.since || row.observedAt > options.until) continue;
|
|
55
|
+
const taskId = typeof row.attributes["taskId"] === "string" ? row.attributes["taskId"] : undefined;
|
|
56
|
+
if (row.metric === "cost" && row.unit === "usd") {
|
|
57
|
+
if (taskId === undefined) { unattributedCostUsd += row.value; continue; }
|
|
58
|
+
entryFor(byTask, taskId).costUsd += row.value;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (taskId === undefined || row.unit !== "tokens" || !TOKEN_METRICS.has(row.metric)) continue;
|
|
62
|
+
const entry = entryFor(byTask, taskId);
|
|
63
|
+
if (row.metric === "input-tokens") entry.inputTokens += row.value;
|
|
64
|
+
else if (row.metric === "output-tokens") entry.outputTokens += row.value;
|
|
65
|
+
else if (row.metric === "cache-read-tokens") entry.cacheReadTokens += row.value;
|
|
66
|
+
else entry.cacheWriteTokens += row.value;
|
|
67
|
+
}
|
|
68
|
+
const entries = [...byTask.values()].sort((left, right) => right.costUsd - left.costUsd || left.taskId.localeCompare(right.taskId));
|
|
69
|
+
return { since: options.since, until: options.until, entries, unattributedCostUsd, truncated: options.truncated === true };
|
|
70
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { PAPYRUS_TASK_FOCUS_SCHEMA, TASK_FOCUS_EVENT_MAX_AGE_MS, TASK_FOCUS_ID_MAX_LENGTH } from "../constants.ts";
|
|
2
|
+
|
|
3
|
+
export type TaskFocusStatus = "focused" | "paused" | "unpaused" | "cleared";
|
|
4
|
+
|
|
5
|
+
export interface TaskFocusEvent {
|
|
6
|
+
schema: typeof PAPYRUS_TASK_FOCUS_SCHEMA;
|
|
7
|
+
taskId: string | null;
|
|
8
|
+
sessionId?: string;
|
|
9
|
+
status: TaskFocusStatus;
|
|
10
|
+
observedAt: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const TOP_LEVEL_FIELDS = new Set(["schema", "taskId", "sessionId", "status", "observedAt"]);
|
|
14
|
+
const STATUSES = new Set<string>(["focused", "paused", "unpaused", "cleared"]);
|
|
15
|
+
|
|
16
|
+
function record(value: unknown): Record<string, unknown> {
|
|
17
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("task-focus event must be an object");
|
|
18
|
+
const result = value as Record<string, unknown>;
|
|
19
|
+
for (const key of Object.keys(result)) if (!TOP_LEVEL_FIELDS.has(key)) throw new Error(`task-focus event contains unexpected field: ${key}`);
|
|
20
|
+
return result;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function boundedId(value: unknown, name: string): string {
|
|
24
|
+
if (typeof value !== "string" || value.length === 0 || value.length > TASK_FOCUS_ID_MAX_LENGTH) throw new Error(`${name} must be a non-empty bounded string`);
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Validates the papyrus.task-focus.v1 shared-bus payload independently of whatever Papyrus itself
|
|
30
|
+
* guarantees -- this is a cross-extension trust boundary, so Jittor never trusts an unvalidated
|
|
31
|
+
* shape. Fails closed (throws) on schema drift, an unrecognized status, or a stale observation,
|
|
32
|
+
* mirroring validatePapyrusContextInjection's pattern for the same reason: a malformed or
|
|
33
|
+
* out-of-order cross-extension event must never silently corrupt Jittor's own state.
|
|
34
|
+
*/
|
|
35
|
+
export function validateTaskFocusEvent(value: unknown, now = Date.now()): TaskFocusEvent {
|
|
36
|
+
const input = record(value);
|
|
37
|
+
if (input["schema"] !== PAPYRUS_TASK_FOCUS_SCHEMA) throw new Error("task-focus event schema is not supported");
|
|
38
|
+
const status = input["status"];
|
|
39
|
+
if (typeof status !== "string" || !STATUSES.has(status)) throw new Error("task-focus event status is not supported");
|
|
40
|
+
const observedAt = input["observedAt"];
|
|
41
|
+
if (typeof observedAt !== "number" || !Number.isSafeInteger(observedAt) || observedAt < 0) throw new Error("task-focus event observedAt must be a non-negative integer");
|
|
42
|
+
if (Math.abs(now - observedAt) > TASK_FOCUS_EVENT_MAX_AGE_MS) throw new Error("task-focus event is stale");
|
|
43
|
+
const rawTaskId = input["taskId"];
|
|
44
|
+
const taskId = rawTaskId === null ? null : boundedId(rawTaskId, "taskId");
|
|
45
|
+
if (taskId === null && status !== "cleared") throw new Error(`task-focus event of status "${status}" requires a taskId`);
|
|
46
|
+
const rawSessionId = input["sessionId"];
|
|
47
|
+
const sessionId = rawSessionId === undefined ? undefined : boundedId(rawSessionId, "sessionId");
|
|
48
|
+
return {
|
|
49
|
+
schema: PAPYRUS_TASK_FOCUS_SCHEMA,
|
|
50
|
+
taskId,
|
|
51
|
+
status: status as TaskFocusStatus,
|
|
52
|
+
observedAt,
|
|
53
|
+
...(sessionId === undefined ? {} : { sessionId }),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Applies a validated event to the currently tracked focused task id. "paused" and "cleared" both
|
|
59
|
+
* mean "no task is actively being worked on right now" for cost-attribution purposes, even though
|
|
60
|
+
* Papyrus itself keeps a paused task's focus state around for later resumption -- Jittor only
|
|
61
|
+
* cares about whether to keep tagging new metrics, not about Papyrus's own pause bookkeeping.
|
|
62
|
+
*/
|
|
63
|
+
export function applyTaskFocusEvent(event: TaskFocusEvent): string | null {
|
|
64
|
+
return event.status === "focused" || event.status === "unpaused" ? event.taskId : null;
|
|
65
|
+
}
|