@danypops/jittor 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -4
- package/docs/PROVIDER_RESEARCH.md +21 -2
- package/extension/src/benchmark-tui.ts +6 -4
- package/extension/src/footer.ts +38 -22
- package/extension/src/index.ts +73 -9
- package/extension/src/tui.ts +29 -2
- package/extension/src/usage.ts +24 -15
- package/package.json +5 -1
- package/src/adapters/openrouter-benchmark-index-source.ts +2 -1
- package/src/adapters/sqlite-metric-store.ts +43 -2
- package/src/cli.ts +139 -10
- package/src/client.ts +12 -44
- package/src/constants.ts +31 -4
- package/src/daemon.ts +47 -46
- package/src/db.ts +13 -30
- package/src/domain/model-observation.ts +41 -19
- package/src/domain/model-ranking-service.ts +3 -2
- package/src/domain/model-ranking.ts +31 -12
- package/src/domain/task-cost.ts +70 -0
- package/src/domain/task-focus.ts +65 -0
- package/src/domain/usage.ts +80 -56
- package/src/log.ts +28 -0
- package/src/ports/metric-store.ts +30 -0
- package/src/providers/anthropic-contracts.ts +22 -11
- package/src/providers/google-adc-auth.ts +63 -0
- package/src/providers/google-vertex-budget-contracts.ts +181 -0
- package/src/providers/google-vertex-budget.ts +127 -0
- package/src/providers/google-vertex-contracts.ts +14 -2
- package/src/providers/telemetry-sources.ts +35 -0
- package/src/service.ts +80 -22
- package/src/state.ts +31 -57
- package/src/version.ts +2 -14
|
@@ -7,8 +7,19 @@ import {
|
|
|
7
7
|
import { normalizeModelIdentity } from "./benchmark.ts";
|
|
8
8
|
import type { MetricObservation, MetricUnit, StoredMetricObservation } from "./metric.ts";
|
|
9
9
|
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
/**
|
|
11
|
+
* Two independent axes, not one flat class: "coding" is a subject-matter domain (which
|
|
12
|
+
* benchmark quality evidence applies), while "research"/"planning" are activities that can
|
|
13
|
+
* happen inside any domain (which predict how much reasoning effort a task needs). The prior
|
|
14
|
+
* single ModelTaskClass conflated them -- an agentic/tool-use benchmark (a type signal) was
|
|
15
|
+
* being read as if it were a domain-quality signal. Both axes default to "general" when tool
|
|
16
|
+
* usage carries no distinguishing signal for that axis; a run can score coding on domain and
|
|
17
|
+
* research on type simultaneously (e.g. reading a file, then searching the web in one turn).
|
|
18
|
+
*/
|
|
19
|
+
export const TASK_DOMAINS = ["coding", "general"] as const;
|
|
20
|
+
export type ModelTaskDomain = typeof TASK_DOMAINS[number];
|
|
21
|
+
export const TASK_TYPES = ["research", "planning", "general"] as const;
|
|
22
|
+
export type ModelTaskType = typeof TASK_TYPES[number];
|
|
12
23
|
export type ExplicitOutcome = "accepted" | "rejected" | "unknown";
|
|
13
24
|
|
|
14
25
|
export interface ModelRunObservation {
|
|
@@ -16,7 +27,8 @@ export interface ModelRunObservation {
|
|
|
16
27
|
provider: string;
|
|
17
28
|
model: string;
|
|
18
29
|
thinking: string;
|
|
19
|
-
|
|
30
|
+
domain: ModelTaskDomain;
|
|
31
|
+
type: ModelTaskType;
|
|
20
32
|
startedAt: number;
|
|
21
33
|
firstTokenAt: number | null;
|
|
22
34
|
completedAt: number;
|
|
@@ -36,7 +48,8 @@ export interface ModelMetricAggregate {
|
|
|
36
48
|
provider: string;
|
|
37
49
|
model: string;
|
|
38
50
|
thinking: string;
|
|
39
|
-
|
|
51
|
+
domain: ModelTaskDomain;
|
|
52
|
+
type: ModelTaskType;
|
|
40
53
|
dimension: string;
|
|
41
54
|
unit: MetricUnit;
|
|
42
55
|
sampleSize: number;
|
|
@@ -54,7 +67,7 @@ export interface ModelAggregateOptions {
|
|
|
54
67
|
}
|
|
55
68
|
|
|
56
69
|
const ALLOWED_FIELDS = new Set<keyof ModelRunObservation>([
|
|
57
|
-
"runId", "provider", "model", "thinking", "
|
|
70
|
+
"runId", "provider", "model", "thinking", "domain", "type", "startedAt", "firstTokenAt", "completedAt",
|
|
58
71
|
"inputTokens", "outputTokens", "cacheReadTokens", "cacheWriteTokens", "costUsd", "providerResponses",
|
|
59
72
|
"toolCalls", "toolFailures", "stopReason", "explicitOutcome",
|
|
60
73
|
]);
|
|
@@ -80,7 +93,8 @@ export function validateModelRunObservation(value: unknown): ModelRunObservation
|
|
|
80
93
|
const completedAt = nonNegative(input["completedAt"], "completion time", true);
|
|
81
94
|
const firstTokenAt = input["firstTokenAt"] === null ? null : nonNegative(input["firstTokenAt"], "first-token time", true);
|
|
82
95
|
if (completedAt < startedAt || (firstTokenAt !== null && (firstTokenAt < startedAt || firstTokenAt > completedAt))) throw new Error("model run timestamps are not ordered");
|
|
83
|
-
if (!
|
|
96
|
+
if (!TASK_DOMAINS.includes(input["domain"] as ModelTaskDomain)) throw new Error("task domain is invalid");
|
|
97
|
+
if (!TASK_TYPES.includes(input["type"] as ModelTaskType)) throw new Error("task type is invalid");
|
|
84
98
|
if (!STOP_REASONS.has(input["stopReason"] as ModelRunObservation["stopReason"])) throw new Error("stop reason is invalid");
|
|
85
99
|
if (!OUTCOMES.has(input["explicitOutcome"] as ExplicitOutcome)) throw new Error("explicit outcome is invalid");
|
|
86
100
|
const providerResponses = nonNegative(input["providerResponses"], "provider response count", true);
|
|
@@ -89,7 +103,7 @@ export function validateModelRunObservation(value: unknown): ModelRunObservation
|
|
|
89
103
|
if (providerResponses < 1 || toolFailures > toolCalls) throw new Error("model run counters are inconsistent");
|
|
90
104
|
return {
|
|
91
105
|
runId: text(input["runId"], "run id"), provider: identity.provider, model: identity.model,
|
|
92
|
-
thinking: text(input["thinking"], "thinking level"),
|
|
106
|
+
thinking: text(input["thinking"], "thinking level"), domain: input["domain"] as ModelTaskDomain, type: input["type"] as ModelTaskType,
|
|
93
107
|
startedAt, firstTokenAt, completedAt,
|
|
94
108
|
inputTokens: nonNegative(input["inputTokens"], "input tokens"),
|
|
95
109
|
outputTokens: nonNegative(input["outputTokens"], "output tokens"),
|
|
@@ -100,18 +114,25 @@ export function validateModelRunObservation(value: unknown): ModelRunObservation
|
|
|
100
114
|
};
|
|
101
115
|
}
|
|
102
116
|
|
|
103
|
-
export
|
|
117
|
+
export interface ModelTaskClassification {
|
|
118
|
+
domain: ModelTaskDomain;
|
|
119
|
+
type: ModelTaskType;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Domain and type are independent: a run can be domain=coding and type=research at once (e.g. reading a file, then searching the web in the same turn). */
|
|
123
|
+
export function classifyTaskFromTools(toolNames: string[]): ModelTaskClassification {
|
|
104
124
|
const names = new Set(toolNames.slice(0, 100).map((name) => name.toLowerCase()));
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
125
|
+
const domain: ModelTaskDomain = ["edit", "write", "read", "bash", "grep", "find", "ls"].some((name) => names.has(name)) ? "coding" : "general";
|
|
126
|
+
const type: ModelTaskType = ["web_fetch", "web_search"].some((name) => names.has(name))
|
|
127
|
+
? "research"
|
|
128
|
+
: ["tasks", "papyrus_create", "papyrus_graph"].some((name) => names.has(name)) ? "planning" : "general";
|
|
129
|
+
return { domain, type };
|
|
109
130
|
}
|
|
110
131
|
|
|
111
132
|
export function modelRunMetrics(value: ModelRunObservation): MetricObservation[] {
|
|
112
133
|
const run = validateModelRunObservation(value);
|
|
113
134
|
const scope = `${run.provider}/${run.model}`;
|
|
114
|
-
const attributes = { provider: run.provider, model: run.model, thinking: run.thinking,
|
|
135
|
+
const attributes = { provider: run.provider, model: run.model, thinking: run.thinking, domain: run.domain, type: run.type, runId: run.runId };
|
|
115
136
|
const metric = (name: string, amount: number, unit: MetricUnit): MetricObservation => ({ source: "local-model", scope, metric: name, value: amount, unit, observedAt: run.completedAt, attributes });
|
|
116
137
|
const wallMs = run.completedAt - run.startedAt;
|
|
117
138
|
const totalInput = run.inputTokens + run.cacheReadTokens;
|
|
@@ -155,16 +176,17 @@ export function aggregateModelMetrics(input: StoredMetricObservation[], options:
|
|
|
155
176
|
const provider = row.attributes["provider"];
|
|
156
177
|
const model = row.attributes["model"];
|
|
157
178
|
const thinking = row.attributes["thinking"];
|
|
158
|
-
const
|
|
159
|
-
|
|
160
|
-
|
|
179
|
+
const domain = row.attributes["domain"];
|
|
180
|
+
const type = row.attributes["type"];
|
|
181
|
+
if (typeof provider !== "string" || typeof model !== "string" || typeof thinking !== "string" || !TASK_DOMAINS.includes(domain as ModelTaskDomain) || !TASK_TYPES.includes(type as ModelTaskType)) continue;
|
|
182
|
+
const key = JSON.stringify([provider, model, thinking, domain, type, row.metric, row.unit]);
|
|
161
183
|
if (!groups.has(key) && groups.size >= MODEL_AGGREGATE_MAX_GROUPS) continue;
|
|
162
184
|
const rows = groups.get(key) ?? [];
|
|
163
185
|
rows.push(row);
|
|
164
186
|
groups.set(key, rows);
|
|
165
187
|
}
|
|
166
188
|
return [...groups.entries()].map(([key, rows]) => {
|
|
167
|
-
const [provider, model, thinking,
|
|
189
|
+
const [provider, model, thinking, domain, type, dimension, unit] = JSON.parse(key) as [string, string, string, ModelTaskDomain, ModelTaskType, string, MetricUnit];
|
|
168
190
|
const values = rows.map((row) => row.value as number).sort((left, right) => left - right);
|
|
169
191
|
const center = median(values);
|
|
170
192
|
const deviations = values.map((value) => Math.abs(value - center)).sort((left, right) => left - right);
|
|
@@ -172,10 +194,10 @@ export function aggregateModelMetrics(input: StoredMetricObservation[], options:
|
|
|
172
194
|
const age = Math.max(0, now - latestAt);
|
|
173
195
|
const recency = Math.max(0, 1 - (age / freshForMs));
|
|
174
196
|
return {
|
|
175
|
-
provider, model, thinking,
|
|
197
|
+
provider, model, thinking, domain, type, dimension, unit,
|
|
176
198
|
sampleSize: values.length, median: center, p90: percentile(values, 0.9), medianAbsoluteDeviation: median(deviations), latestAt,
|
|
177
199
|
freshness: age <= freshForMs ? "fresh" as const : "stale" as const,
|
|
178
200
|
confidence: Math.min(1, Math.sqrt(values.length / 20)) * recency,
|
|
179
201
|
};
|
|
180
|
-
}).sort((left, right) => left.provider.localeCompare(right.provider) || left.model.localeCompare(right.model) || left.dimension.localeCompare(right.dimension));
|
|
202
|
+
}).sort((left, right) => left.provider.localeCompare(right.provider) || left.model.localeCompare(right.model) || left.domain.localeCompare(right.domain) || left.type.localeCompare(right.type) || left.dimension.localeCompare(right.dimension));
|
|
181
203
|
}
|
|
@@ -3,12 +3,13 @@ import type { BenchmarkStore } from "../ports/benchmark-store.ts";
|
|
|
3
3
|
import type { MetricStore } from "../ports/metric-store.ts";
|
|
4
4
|
import { aggregateModelMetrics } from "./model-observation.ts";
|
|
5
5
|
import { rankModelCandidates, type ModelCandidate, type ModelRankingResult, type ScopeAuthority, type UtilityWeights } from "./model-ranking.ts";
|
|
6
|
-
import type {
|
|
6
|
+
import type { ModelTaskDomain, ModelTaskType } from "./model-observation.ts";
|
|
7
7
|
|
|
8
8
|
export interface ModelRecommendationInput {
|
|
9
9
|
candidates: ModelCandidate[];
|
|
10
10
|
scopeAuthority: ScopeAuthority;
|
|
11
|
-
|
|
11
|
+
domain: ModelTaskDomain;
|
|
12
|
+
type: ModelTaskType;
|
|
12
13
|
budgetPressure: number;
|
|
13
14
|
weights: UtilityWeights;
|
|
14
15
|
sourceIds: string[];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT, MAX_DYNAMIC_ROUTES, MODEL_AGGREGATE_MAX_ROWS } from "../constants.ts";
|
|
2
2
|
import { normalizeModelIdentity, type BenchmarkObservation } from "./benchmark.ts";
|
|
3
|
-
import {
|
|
3
|
+
import { TASK_DOMAINS, TASK_TYPES, type ModelMetricAggregate, type ModelTaskDomain, type ModelTaskType } from "./model-observation.ts";
|
|
4
4
|
|
|
5
5
|
export type ScopeAuthority = "exact-session" | "available-models";
|
|
6
6
|
export type UtilityComponentName = "quality" | "cost" | "latency" | "context" | "reliability";
|
|
@@ -22,7 +22,8 @@ export interface UtilityWeights {
|
|
|
22
22
|
export interface ModelRankingInput {
|
|
23
23
|
candidates: ModelCandidate[];
|
|
24
24
|
scopeAuthority: ScopeAuthority;
|
|
25
|
-
|
|
25
|
+
domain: ModelTaskDomain;
|
|
26
|
+
type: ModelTaskType;
|
|
26
27
|
budgetPressure: number;
|
|
27
28
|
weights: UtilityWeights;
|
|
28
29
|
externalEvidence: BenchmarkObservation[];
|
|
@@ -60,7 +61,8 @@ export interface RankedModel {
|
|
|
60
61
|
export interface ModelRankingResult {
|
|
61
62
|
scopeAuthority: ScopeAuthority;
|
|
62
63
|
scopeWarning: string | null;
|
|
63
|
-
|
|
64
|
+
domain: ModelTaskDomain;
|
|
65
|
+
type: ModelTaskType;
|
|
64
66
|
completeness: "complete" | "partial" | "insufficient-evidence";
|
|
65
67
|
ranked: RankedModel[];
|
|
66
68
|
automaticSelection: ModelCandidate | null;
|
|
@@ -101,22 +103,37 @@ function externalValues(candidate: ModelCandidate, evidence: BenchmarkObservatio
|
|
|
101
103
|
};
|
|
102
104
|
}
|
|
103
105
|
|
|
104
|
-
function localValues(candidate: ModelCandidate,
|
|
106
|
+
function localValues(candidate: ModelCandidate, domain: ModelTaskDomain, type: ModelTaskType, evidence: ModelMetricAggregate[], dimension: string): ModelMetricAggregate[] {
|
|
105
107
|
const identity = normalizeModelIdentity(candidate.provider, candidate.model);
|
|
106
|
-
return evidence.filter((item) => item.provider === identity.provider && item.model === identity.model && item.thinking === candidate.thinking && item.
|
|
108
|
+
return evidence.filter((item) => item.provider === identity.provider && item.model === identity.model && item.thinking === candidate.thinking && item.domain === domain && item.type === type && item.dimension === dimension);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* quality-{domain} is a subject-matter signal (e.g. quality-coding from a coding benchmark);
|
|
113
|
+
* quality-type-{type} is an activity signal (e.g. quality-type-planning from an agentic/tool-use
|
|
114
|
+
* benchmark). Both are optional and additive on top of the universal quality-general fallback --
|
|
115
|
+
* a candidate with no domain- or type-specific evidence still gets ranked on general quality
|
|
116
|
+
* rather than being treated as having zero evidence.
|
|
117
|
+
*/
|
|
118
|
+
function qualityDimensions(domain: ModelTaskDomain, type: ModelTaskType): string[] {
|
|
119
|
+
const dimensions: string[] = [];
|
|
120
|
+
if (domain !== "general") dimensions.push(`quality-${domain}`);
|
|
121
|
+
if (type !== "general") dimensions.push(`quality-type-${type}`);
|
|
122
|
+
dimensions.push("quality-general");
|
|
123
|
+
return dimensions;
|
|
107
124
|
}
|
|
108
125
|
|
|
109
126
|
function rawComponents(candidate: ModelCandidate, input: ModelRankingInput): { components: Record<UtilityComponentName, RawComponent>; provenance: RankingProvenance[] } {
|
|
110
|
-
const quality = externalValues(candidate, input.externalEvidence,
|
|
127
|
+
const quality = externalValues(candidate, input.externalEvidence, qualityDimensions(input.domain, input.type), input.now);
|
|
111
128
|
const priceInput = externalValues(candidate, input.externalEvidence, ["price-input"], input.now);
|
|
112
129
|
const priceOutput = externalValues(candidate, input.externalEvidence, ["price-output"], input.now);
|
|
113
130
|
const measuredLatency = externalValues(candidate, input.externalEvidence, ["latency"], input.now);
|
|
114
131
|
const rankedLatency = externalValues(candidate, input.externalEvidence, ["latency-rank", "throughput-rank"], input.now);
|
|
115
132
|
const latency = measuredLatency.values.length > 0 ? measuredLatency : rankedLatency;
|
|
116
133
|
const context = externalValues(candidate, input.externalEvidence, ["context-window"], input.now);
|
|
117
|
-
const localLatency = localValues(candidate, input.
|
|
118
|
-
const failures = localValues(candidate, input.
|
|
119
|
-
const outcomes = localValues(candidate, input.
|
|
134
|
+
const localLatency = localValues(candidate, input.domain, input.type, input.localEvidence, "wall-latency");
|
|
135
|
+
const failures = localValues(candidate, input.domain, input.type, input.localEvidence, "failure");
|
|
136
|
+
const outcomes = localValues(candidate, input.domain, input.type, input.localEvidence, "outcome-accepted");
|
|
120
137
|
const qualityValues = quality.values;
|
|
121
138
|
const prices = [...priceInput.values, ...priceOutput.values];
|
|
122
139
|
const latencyValues = localLatency.length > 0 ? localLatency.map((item) => item.median) : latency.values;
|
|
@@ -154,7 +171,8 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
154
171
|
if (!Array.isArray(value.externalEvidence) || value.externalEvidence.length > BENCHMARK_MAX_OBSERVATIONS_PER_SNAPSHOT * 4) throw new Error("external evidence exceeds the supported bound");
|
|
155
172
|
if (!Array.isArray(value.localEvidence) || value.localEvidence.length > MODEL_AGGREGATE_MAX_ROWS) throw new Error("local evidence exceeds the supported bound");
|
|
156
173
|
if (value.scopeAuthority !== "exact-session" && value.scopeAuthority !== "available-models") throw new Error("scope authority is invalid");
|
|
157
|
-
if (!
|
|
174
|
+
if (!TASK_DOMAINS.includes(value.domain)) throw new Error("task domain is invalid");
|
|
175
|
+
if (!TASK_TYPES.includes(value.type)) throw new Error("task type is invalid");
|
|
158
176
|
if (!Number.isSafeInteger(value.now) || value.now <= 0) throw new Error("ranking time is invalid");
|
|
159
177
|
const budgetPressure = finiteBound(value.budgetPressure, "budget pressure", 0, 2);
|
|
160
178
|
const weights = Object.fromEntries(COMPONENTS.map((name) => [name, finiteBound(value.weights[name], `${name} weight`, 0, 10)])) as unknown as UtilityWeights;
|
|
@@ -195,7 +213,7 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
195
213
|
confidence,
|
|
196
214
|
components,
|
|
197
215
|
provenance,
|
|
198
|
-
trace: [`
|
|
216
|
+
trace: [`domain ${value.domain}, type ${value.type}`, `budget pressure ${budgetPressure.toFixed(3)} makes cost weight ${effectiveWeights.cost.toFixed(3)}`, `${known.length}/${components.length} utility components have evidence`, `scope authority ${value.scopeAuthority}`],
|
|
199
217
|
};
|
|
200
218
|
}).sort((left, right) => (right.utility ?? -1) - (left.utility ?? -1) || right.confidence - left.confidence || left.identity.localeCompare(right.identity));
|
|
201
219
|
const knownComponents = ranked.reduce((sum, item) => sum + item.components.filter((component) => component.score !== null).length, 0);
|
|
@@ -205,7 +223,8 @@ export function rankModelCandidates(value: ModelRankingInput): ModelRankingResul
|
|
|
205
223
|
return {
|
|
206
224
|
scopeAuthority: value.scopeAuthority,
|
|
207
225
|
scopeWarning: exact ? null : "Pi available models are not the exact session scope; automatic selection is disabled",
|
|
208
|
-
|
|
226
|
+
domain: value.domain,
|
|
227
|
+
type: value.type,
|
|
209
228
|
completeness,
|
|
210
229
|
ranked,
|
|
211
230
|
automaticSelection: exact && ranked[0]?.utility !== null ? ranked[0]!.candidate : null,
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { StoredMetricObservation } from "./metric.ts";
|
|
2
|
+
|
|
3
|
+
export interface TaskCostEntry {
|
|
4
|
+
taskId: string;
|
|
5
|
+
costUsd: number;
|
|
6
|
+
inputTokens: number;
|
|
7
|
+
outputTokens: number;
|
|
8
|
+
cacheReadTokens: number;
|
|
9
|
+
cacheWriteTokens: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface TaskCostSummary {
|
|
13
|
+
since: number;
|
|
14
|
+
until: number;
|
|
15
|
+
entries: TaskCostEntry[];
|
|
16
|
+
unattributedCostUsd: number;
|
|
17
|
+
truncated: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface TaskCostSummaryOptions {
|
|
21
|
+
since: number;
|
|
22
|
+
until: number;
|
|
23
|
+
truncated?: boolean;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const TOKEN_METRICS = new Set(["input-tokens", "output-tokens", "cache-read-tokens", "cache-write-tokens"]);
|
|
27
|
+
|
|
28
|
+
function entryFor(byTask: Map<string, TaskCostEntry>, taskId: string): TaskCostEntry {
|
|
29
|
+
const existing = byTask.get(taskId);
|
|
30
|
+
if (existing) return existing;
|
|
31
|
+
const created: TaskCostEntry = { taskId, costUsd: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 };
|
|
32
|
+
byTask.set(taskId, created);
|
|
33
|
+
return created;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Groups already-recorded "pi" source cost/token metrics by the Papyrus task focused when each was
|
|
38
|
+
* recorded (see the papyrus.task-focus.v1 real-time tagging in the extension), rather than by
|
|
39
|
+
* provider/model identity. Rows recorded with nothing focused have no attributes.taskId and are
|
|
40
|
+
* reported separately as unattributedCostUsd -- they are real spend, just not attributable to any
|
|
41
|
+
* task, and must never be silently dropped or folded into an invented "unknown" task bucket.
|
|
42
|
+
*
|
|
43
|
+
* This queries a single bounded time window without per-task fairness partitioning (unlike
|
|
44
|
+
* buildUsageGraph/buildCostGraph's per-scope fetch): a task's own working period is typically a far
|
|
45
|
+
* narrower, more targeted window than "the whole usage dashboard, any period", so one task's volume
|
|
46
|
+
* crowding another out of the same query is a much smaller risk here. If this does become a problem
|
|
47
|
+
* in practice, the same per-distinct-value fetch strategy applies, keyed by attributes.taskId.
|
|
48
|
+
*/
|
|
49
|
+
export function buildTaskCostSummary(rows: StoredMetricObservation[], options: TaskCostSummaryOptions): TaskCostSummary {
|
|
50
|
+
const byTask = new Map<string, TaskCostEntry>();
|
|
51
|
+
let unattributedCostUsd = 0;
|
|
52
|
+
for (const row of rows) {
|
|
53
|
+
if (row.source !== "pi" || typeof row.value !== "number" || !Number.isFinite(row.value) || row.value < 0) continue;
|
|
54
|
+
if (row.observedAt < options.since || row.observedAt > options.until) continue;
|
|
55
|
+
const taskId = typeof row.attributes["taskId"] === "string" ? row.attributes["taskId"] : undefined;
|
|
56
|
+
if (row.metric === "cost" && row.unit === "usd") {
|
|
57
|
+
if (taskId === undefined) { unattributedCostUsd += row.value; continue; }
|
|
58
|
+
entryFor(byTask, taskId).costUsd += row.value;
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
if (taskId === undefined || row.unit !== "tokens" || !TOKEN_METRICS.has(row.metric)) continue;
|
|
62
|
+
const entry = entryFor(byTask, taskId);
|
|
63
|
+
if (row.metric === "input-tokens") entry.inputTokens += row.value;
|
|
64
|
+
else if (row.metric === "output-tokens") entry.outputTokens += row.value;
|
|
65
|
+
else if (row.metric === "cache-read-tokens") entry.cacheReadTokens += row.value;
|
|
66
|
+
else entry.cacheWriteTokens += row.value;
|
|
67
|
+
}
|
|
68
|
+
const entries = [...byTask.values()].sort((left, right) => right.costUsd - left.costUsd || left.taskId.localeCompare(right.taskId));
|
|
69
|
+
return { since: options.since, until: options.until, entries, unattributedCostUsd, truncated: options.truncated === true };
|
|
70
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { PAPYRUS_TASK_FOCUS_SCHEMA, TASK_FOCUS_EVENT_MAX_AGE_MS, TASK_FOCUS_ID_MAX_LENGTH } from "../constants.ts";
|
|
2
|
+
|
|
3
|
+
export type TaskFocusStatus = "focused" | "paused" | "unpaused" | "cleared";
|
|
4
|
+
|
|
5
|
+
export interface TaskFocusEvent {
|
|
6
|
+
schema: typeof PAPYRUS_TASK_FOCUS_SCHEMA;
|
|
7
|
+
taskId: string | null;
|
|
8
|
+
sessionId?: string;
|
|
9
|
+
status: TaskFocusStatus;
|
|
10
|
+
observedAt: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const TOP_LEVEL_FIELDS = new Set(["schema", "taskId", "sessionId", "status", "observedAt"]);
|
|
14
|
+
const STATUSES = new Set<string>(["focused", "paused", "unpaused", "cleared"]);
|
|
15
|
+
|
|
16
|
+
function record(value: unknown): Record<string, unknown> {
|
|
17
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("task-focus event must be an object");
|
|
18
|
+
const result = value as Record<string, unknown>;
|
|
19
|
+
for (const key of Object.keys(result)) if (!TOP_LEVEL_FIELDS.has(key)) throw new Error(`task-focus event contains unexpected field: ${key}`);
|
|
20
|
+
return result;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function boundedId(value: unknown, name: string): string {
|
|
24
|
+
if (typeof value !== "string" || value.length === 0 || value.length > TASK_FOCUS_ID_MAX_LENGTH) throw new Error(`${name} must be a non-empty bounded string`);
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Validates the papyrus.task-focus.v1 shared-bus payload independently of whatever Papyrus itself
|
|
30
|
+
* guarantees -- this is a cross-extension trust boundary, so Jittor never trusts an unvalidated
|
|
31
|
+
* shape. Fails closed (throws) on schema drift, an unrecognized status, or a stale observation,
|
|
32
|
+
* mirroring validatePapyrusContextInjection's pattern for the same reason: a malformed or
|
|
33
|
+
* out-of-order cross-extension event must never silently corrupt Jittor's own state.
|
|
34
|
+
*/
|
|
35
|
+
export function validateTaskFocusEvent(value: unknown, now = Date.now()): TaskFocusEvent {
|
|
36
|
+
const input = record(value);
|
|
37
|
+
if (input["schema"] !== PAPYRUS_TASK_FOCUS_SCHEMA) throw new Error("task-focus event schema is not supported");
|
|
38
|
+
const status = input["status"];
|
|
39
|
+
if (typeof status !== "string" || !STATUSES.has(status)) throw new Error("task-focus event status is not supported");
|
|
40
|
+
const observedAt = input["observedAt"];
|
|
41
|
+
if (typeof observedAt !== "number" || !Number.isSafeInteger(observedAt) || observedAt < 0) throw new Error("task-focus event observedAt must be a non-negative integer");
|
|
42
|
+
if (Math.abs(now - observedAt) > TASK_FOCUS_EVENT_MAX_AGE_MS) throw new Error("task-focus event is stale");
|
|
43
|
+
const rawTaskId = input["taskId"];
|
|
44
|
+
const taskId = rawTaskId === null ? null : boundedId(rawTaskId, "taskId");
|
|
45
|
+
if (taskId === null && status !== "cleared") throw new Error(`task-focus event of status "${status}" requires a taskId`);
|
|
46
|
+
const rawSessionId = input["sessionId"];
|
|
47
|
+
const sessionId = rawSessionId === undefined ? undefined : boundedId(rawSessionId, "sessionId");
|
|
48
|
+
return {
|
|
49
|
+
schema: PAPYRUS_TASK_FOCUS_SCHEMA,
|
|
50
|
+
taskId,
|
|
51
|
+
status: status as TaskFocusStatus,
|
|
52
|
+
observedAt,
|
|
53
|
+
...(sessionId === undefined ? {} : { sessionId }),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Applies a validated event to the currently tracked focused task id. "paused" and "cleared" both
|
|
59
|
+
* mean "no task is actively being worked on right now" for cost-attribution purposes, even though
|
|
60
|
+
* Papyrus itself keeps a paused task's focus state around for later resumption -- Jittor only
|
|
61
|
+
* cares about whether to keep tagging new metrics, not about Papyrus's own pause bookkeeping.
|
|
62
|
+
*/
|
|
63
|
+
export function applyTaskFocusEvent(event: TaskFocusEvent): string | null {
|
|
64
|
+
return event.status === "focused" || event.status === "unpaused" ? event.taskId : null;
|
|
65
|
+
}
|
package/src/domain/usage.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { MAX_USAGE_BUCKETS, MILLISECONDS_PER_DAY, MILLISECONDS_PER_HOUR } from "../constants.ts";
|
|
2
|
-
import type { StoredMetricObservation } from "./metric.ts";
|
|
3
2
|
|
|
4
3
|
export const USAGE_PERIODS = [
|
|
5
4
|
{ id: "hourly", label: "Hourly", windowMs: MILLISECONDS_PER_HOUR, bucketCount: 12 },
|
|
@@ -14,6 +13,52 @@ export function usagePeriod(period: UsagePeriod): typeof USAGE_PERIODS[number] {
|
|
|
14
13
|
return USAGE_PERIODS.find((candidate) => candidate.id === period)!;
|
|
15
14
|
}
|
|
16
15
|
|
|
16
|
+
export function usagePeriodStart(period: UsagePeriod, now: number): number {
|
|
17
|
+
return Math.max(0, now - usagePeriod(period).windowMs);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* The bucket boundaries a chart period actually renders. Shared, byte-identical, by both sides of
|
|
22
|
+
* the wire: the daemon computes the SAME window from the same (period, now[, bucketCount]) before
|
|
23
|
+
* running its SQL-side GROUP BY, so a pre-aggregated bucket sum always lands in exactly the bucket
|
|
24
|
+
* the chart expects -- there is no second, independent bucketing pass left to disagree with it.
|
|
25
|
+
*/
|
|
26
|
+
export interface UsageBucketWindow {
|
|
27
|
+
start: number;
|
|
28
|
+
end: number;
|
|
29
|
+
bucketCount: number;
|
|
30
|
+
bucketSizeMs: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function resolveUsageWindow(period: UsagePeriod, now: number, bucketCount?: number): UsageBucketWindow {
|
|
34
|
+
const end = now;
|
|
35
|
+
const start = usagePeriodStart(period, end);
|
|
36
|
+
const requested = bucketCount ?? usagePeriod(period).bucketCount;
|
|
37
|
+
const count = Math.max(1, Math.min(MAX_USAGE_BUCKETS, Math.floor(requested)));
|
|
38
|
+
const bucketSizeMs = Math.max(1, (end - start) / count);
|
|
39
|
+
return { start, end, bucketCount: count, bucketSizeMs };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Mirrors the SQL-side `MIN(CAST((observed_at - start) / bucketSizeMs AS INTEGER), bucketCount - 1)` grouping exactly. */
|
|
43
|
+
export function usageBucketIndex(observedAt: number, window: UsageBucketWindow): number {
|
|
44
|
+
return Math.min(window.bucketCount - 1, Math.max(0, Math.floor((observedAt - window.start) / window.bucketSizeMs)));
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* One already-summed (scope, metric, bucket) cell -- what the daemon's SQL-side GROUP BY returns,
|
|
49
|
+
* replacing a bounded-but-still-truncatable fetch of raw per-observation rows. Result size scales
|
|
50
|
+
* with (distinct scopes x distinct metrics x bucket count), never with raw event count, so a
|
|
51
|
+
* heavy scope's full history is represented exactly regardless of how many observations it made
|
|
52
|
+
* (a real incident: a scope with 49,270 rows in a week had its "weekly" chart built from the 250
|
|
53
|
+
* most recent rows alone -- 3.3 minutes of real activity mislabeled as a full week).
|
|
54
|
+
*/
|
|
55
|
+
export interface UsageAggregateRow {
|
|
56
|
+
scope: string;
|
|
57
|
+
metric: string;
|
|
58
|
+
bucketIndex: number;
|
|
59
|
+
sum: number;
|
|
60
|
+
}
|
|
61
|
+
|
|
17
62
|
export interface UsageSeries {
|
|
18
63
|
key: string;
|
|
19
64
|
provider: string;
|
|
@@ -48,8 +93,6 @@ export interface UsageGraph {
|
|
|
48
93
|
|
|
49
94
|
export interface UsageGraphOptions {
|
|
50
95
|
period: UsagePeriod;
|
|
51
|
-
now: number;
|
|
52
|
-
bucketCount?: number;
|
|
53
96
|
truncated?: boolean;
|
|
54
97
|
}
|
|
55
98
|
|
|
@@ -60,69 +103,52 @@ const BREAKDOWN_KEYS = {
|
|
|
60
103
|
"cache-write-tokens": "cacheWrite",
|
|
61
104
|
} as const satisfies Record<string, keyof UsageBreakdown>;
|
|
62
105
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const
|
|
71
|
-
const provider =
|
|
72
|
-
const model =
|
|
106
|
+
/**
|
|
107
|
+
* Derives provider/model from `scope` alone (`"${provider}:${model}"`, see assistantUsageMetrics),
|
|
108
|
+
* rather than needing a row's `attributes.provider`/`attributes.model` -- source "pi" always
|
|
109
|
+
* constructs `scope` from those exact same two values, so the two are equivalent for this source,
|
|
110
|
+
* and an aggregated bucket sum has no per-row attributes left to read anyway.
|
|
111
|
+
*/
|
|
112
|
+
export function identity(scope: string): { key: string; provider: string; model: string } {
|
|
113
|
+
const separator = scope.indexOf(":");
|
|
114
|
+
const provider = separator >= 0 ? scope.slice(0, separator) : scope;
|
|
115
|
+
const model = separator >= 0 ? scope.slice(separator + 1) : "unknown";
|
|
73
116
|
return { key: `${provider}/${model}`, provider, model };
|
|
74
117
|
}
|
|
75
118
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
function bucketWindow(options: UsageGraphOptions): BucketWindow {
|
|
79
|
-
const end = options.now;
|
|
80
|
-
const start = usagePeriodStart(options.period, end);
|
|
81
|
-
const requestedBuckets = options.bucketCount ?? usagePeriod(options.period).bucketCount;
|
|
82
|
-
const bucketCount = Math.max(1, Math.min(MAX_USAGE_BUCKETS, Math.floor(requestedBuckets)));
|
|
83
|
-
const bucketSize = Math.max(1, (end - start) / bucketCount);
|
|
84
|
-
return { start, end, bucketCount, bucketSize };
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function emptyBuckets(window: BucketWindow): { start: number; end: number; total: number; series: Record<string, number> }[] {
|
|
119
|
+
function emptyBuckets(window: UsageBucketWindow): { start: number; end: number; total: number; series: Record<string, number> }[] {
|
|
88
120
|
return Array.from({ length: window.bucketCount }, (_, index) => ({
|
|
89
|
-
start: window.start + index * window.
|
|
90
|
-
end: index === window.bucketCount - 1 ? window.end : window.start + (index + 1) * window.
|
|
121
|
+
start: window.start + index * window.bucketSizeMs,
|
|
122
|
+
end: index === window.bucketCount - 1 ? window.end : window.start + (index + 1) * window.bucketSizeMs,
|
|
91
123
|
total: 0,
|
|
92
124
|
series: {},
|
|
93
125
|
}));
|
|
94
126
|
}
|
|
95
127
|
|
|
96
|
-
function
|
|
97
|
-
return Math.min(window.bucketCount - 1, Math.floor((observedAt - window.start) / window.bucketSize));
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
export function buildUsageGraph(rows: StoredMetricObservation[], options: UsageGraphOptions): UsageGraph {
|
|
101
|
-
const window = bucketWindow(options);
|
|
102
|
-
const { start, end } = window;
|
|
128
|
+
export function buildUsageGraph(rows: UsageAggregateRow[], window: UsageBucketWindow, options: UsageGraphOptions): UsageGraph {
|
|
103
129
|
const buckets: UsageBucket[] = emptyBuckets(window);
|
|
104
130
|
const breakdown: UsageBreakdown = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
|
|
105
131
|
const identities = new Map<string, UsageSeries>();
|
|
106
132
|
|
|
107
133
|
for (const row of rows) {
|
|
108
134
|
const breakdownKey = BREAKDOWN_KEYS[row.metric as keyof typeof BREAKDOWN_KEYS];
|
|
109
|
-
if (
|
|
110
|
-
if (row.
|
|
111
|
-
const bucket = buckets[
|
|
112
|
-
const series = identity(row);
|
|
113
|
-
bucket.total += row.
|
|
114
|
-
bucket.series[series.key] = (bucket.series[series.key] ?? 0) + row.
|
|
115
|
-
breakdown[breakdownKey] += row.
|
|
135
|
+
if (!breakdownKey || !Number.isFinite(row.sum) || row.sum < 0) continue;
|
|
136
|
+
if (!Number.isInteger(row.bucketIndex) || row.bucketIndex < 0 || row.bucketIndex >= buckets.length) continue;
|
|
137
|
+
const bucket = buckets[row.bucketIndex]!;
|
|
138
|
+
const series = identity(row.scope);
|
|
139
|
+
bucket.total += row.sum;
|
|
140
|
+
bucket.series[series.key] = (bucket.series[series.key] ?? 0) + row.sum;
|
|
141
|
+
breakdown[breakdownKey] += row.sum;
|
|
116
142
|
const current = identities.get(series.key) ?? { ...series, total: 0 };
|
|
117
|
-
current.total += row.
|
|
143
|
+
current.total += row.sum;
|
|
118
144
|
identities.set(series.key, current);
|
|
119
145
|
}
|
|
120
146
|
|
|
121
147
|
const series = [...identities.values()].sort((left, right) => right.total - left.total || left.key.localeCompare(right.key));
|
|
122
148
|
return {
|
|
123
149
|
period: options.period,
|
|
124
|
-
start,
|
|
125
|
-
end,
|
|
150
|
+
start: window.start,
|
|
151
|
+
end: window.end,
|
|
126
152
|
buckets,
|
|
127
153
|
series,
|
|
128
154
|
totalTokens: buckets.reduce((sum, bucket) => sum + bucket.total, 0),
|
|
@@ -162,29 +188,27 @@ export interface CostGraph {
|
|
|
162
188
|
* surface from day one (alongside raw token counts), since token counts alone do not reflect that
|
|
163
189
|
* output tokens and premium models cost disproportionately more per token.
|
|
164
190
|
*/
|
|
165
|
-
export function buildCostGraph(rows:
|
|
166
|
-
const window = bucketWindow(options);
|
|
167
|
-
const { start, end } = window;
|
|
191
|
+
export function buildCostGraph(rows: UsageAggregateRow[], window: UsageBucketWindow, options: UsageGraphOptions): CostGraph {
|
|
168
192
|
const buckets: CostBucket[] = emptyBuckets(window);
|
|
169
193
|
const identities = new Map<string, CostSeries>();
|
|
170
194
|
|
|
171
195
|
for (const row of rows) {
|
|
172
|
-
if (row.
|
|
173
|
-
if (row.
|
|
174
|
-
const bucket = buckets[
|
|
175
|
-
const series = identity(row);
|
|
176
|
-
bucket.total += row.
|
|
177
|
-
bucket.series[series.key] = (bucket.series[series.key] ?? 0) + row.
|
|
196
|
+
if (row.metric !== "cost" || !Number.isFinite(row.sum) || row.sum < 0) continue;
|
|
197
|
+
if (!Number.isInteger(row.bucketIndex) || row.bucketIndex < 0 || row.bucketIndex >= buckets.length) continue;
|
|
198
|
+
const bucket = buckets[row.bucketIndex]!;
|
|
199
|
+
const series = identity(row.scope);
|
|
200
|
+
bucket.total += row.sum;
|
|
201
|
+
bucket.series[series.key] = (bucket.series[series.key] ?? 0) + row.sum;
|
|
178
202
|
const current = identities.get(series.key) ?? { ...series, total: 0 };
|
|
179
|
-
current.total += row.
|
|
203
|
+
current.total += row.sum;
|
|
180
204
|
identities.set(series.key, current);
|
|
181
205
|
}
|
|
182
206
|
|
|
183
207
|
const series = [...identities.values()].sort((left, right) => right.total - left.total || left.key.localeCompare(right.key));
|
|
184
208
|
return {
|
|
185
209
|
period: options.period,
|
|
186
|
-
start,
|
|
187
|
-
end,
|
|
210
|
+
start: window.start,
|
|
211
|
+
end: window.end,
|
|
188
212
|
buckets,
|
|
189
213
|
series,
|
|
190
214
|
totalUsd: buckets.reduce((sum, bucket) => sum + bucket.total, 0),
|
package/src/log.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Structured daemon logging, now backed by `@danypops/daemon-kit/logging` (pino) instead of a
|
|
3
|
+
* hand-rolled `console.error(JSON.stringify(...))` -- daemon-kit's own module doc explains why:
|
|
4
|
+
* level ordering/filtering/child-scoping is exactly the kind of thing worth one shared,
|
|
5
|
+
* dependency-backed implementation instead of four independent hand-rolled ones. One deliberate,
|
|
6
|
+
* disclosed shape change from jittor's old bespoke format: the event name is now pino's `msg`
|
|
7
|
+
* field rather than a separate `event` field, matching daemon-kit's shared convention across all
|
|
8
|
+
* four daemons. `component`/`level`/`timestamp` and credential-safety (callers still must pass
|
|
9
|
+
* only bounded, non-sensitive fields) are unchanged.
|
|
10
|
+
*/
|
|
11
|
+
import { createLogger, type LogLevel as DaemonKitLogLevel, type Logger } from "@danypops/daemon-kit/logging";
|
|
12
|
+
|
|
13
|
+
export type LogLevel = Extract<DaemonKitLogLevel, "info" | "warn" | "error">;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Also passed directly as `StartDaemonOptions.logger` so daemon-kit's own maintenance-task
|
|
17
|
+
* failure logging shares this same sink/shape. `destination` is pinned to `console.error` rather
|
|
18
|
+
* than daemon-kit's own default (a raw fd 2 write via `pino.destination(2)`, which bypasses
|
|
19
|
+
* `console.error` entirely) so existing tooling/tests that intercept `console.error` keep working.
|
|
20
|
+
*/
|
|
21
|
+
export const logger: Logger = createLogger("jittor-daemon", {
|
|
22
|
+
destination: { write: (chunk: string) => { console.error(chunk.replace(/\n$/, "")); return true; } },
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
/** Credential-safe structured daemon event. Callers must pass bounded, non-sensitive fields. */
|
|
26
|
+
export function logEvent(level: LogLevel, event: string, fields: Record<string, unknown> = {}): void {
|
|
27
|
+
logger[level](event, fields);
|
|
28
|
+
}
|