@danypops/jittor 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -5
- package/docs/PROVIDER_RESEARCH.md +21 -2
- package/extension/src/benchmark-tui.ts +7 -5
- package/extension/src/footer.ts +38 -22
- package/extension/src/index.ts +73 -9
- package/extension/src/tui.ts +29 -2
- package/extension/src/usage.ts +24 -15
- package/package.json +5 -1
- package/src/adapters/openrouter-benchmark-index-source.ts +2 -1
- package/src/adapters/openrouter-design-arena-source.ts +105 -0
- package/src/adapters/sqlite-metric-store.ts +43 -2
- package/src/cli.ts +139 -10
- package/src/client.ts +12 -44
- package/src/constants.ts +31 -4
- package/src/daemon.ts +52 -47
- package/src/db.ts +13 -30
- package/src/domain/metric.ts +1 -1
- package/src/domain/model-observation.ts +41 -19
- package/src/domain/model-ranking-service.ts +3 -2
- package/src/domain/model-ranking.ts +31 -12
- package/src/domain/task-cost.ts +70 -0
- package/src/domain/task-focus.ts +65 -0
- package/src/domain/usage.ts +80 -56
- package/src/log.ts +28 -0
- package/src/ports/metric-store.ts +30 -0
- package/src/providers/anthropic-contracts.ts +22 -11
- package/src/providers/google-adc-auth.ts +63 -0
- package/src/providers/google-vertex-budget-contracts.ts +181 -0
- package/src/providers/google-vertex-budget.ts +127 -0
- package/src/providers/google-vertex-contracts.ts +14 -2
- package/src/providers/telemetry-sources.ts +35 -0
- package/src/service.ts +80 -22
- package/src/state.ts +31 -57
- package/src/version.ts +2 -14
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@danypops/jittor",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.8.0",
|
|
4
4
|
"description": "Just-in-Time Token Optimizing Router for Pi",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"keywords": ["pi-package", "llm-router", "token-budget"],
|
|
@@ -22,6 +22,10 @@
|
|
|
22
22
|
"@earendil-works/pi-tui": "*",
|
|
23
23
|
"typebox": "*"
|
|
24
24
|
},
|
|
25
|
+
"dependencies": {
|
|
26
|
+
"@danypops/daemon-kit": "^0.3.1",
|
|
27
|
+
"google-auth-library": "^10.9.0"
|
|
28
|
+
},
|
|
25
29
|
"devDependencies": {
|
|
26
30
|
"bun-types": "latest"
|
|
27
31
|
},
|
|
@@ -83,7 +83,8 @@ export class OpenRouterBenchmarkIndexSource implements BenchmarkSource {
|
|
|
83
83
|
return [
|
|
84
84
|
validateBenchmarkObservation({ ...common, dimension: "quality-coding", value: requiredNumber(row["coding_index"], "coding index"), unit: "ratio", methodology }),
|
|
85
85
|
validateBenchmarkObservation({ ...common, dimension: "quality-general", value: requiredNumber(row["intelligence_index"], "intelligence index"), unit: "ratio", methodology }),
|
|
86
|
-
|
|
86
|
+
// agentic_index measures tool-use/agentic execution style, an activity (type), not a subject-matter domain.
|
|
87
|
+
validateBenchmarkObservation({ ...common, dimension: "quality-type-planning", value: requiredNumber(row["agentic_index"], "agentic index"), unit: "ratio", methodology }),
|
|
87
88
|
validateBenchmarkObservation({ ...common, dimension: "price-input", value: price(pricing["prompt"], "prompt pricing"), unit: "usd", methodology: { ...methodology, basis: "OpenRouter USD per input token" } }),
|
|
88
89
|
validateBenchmarkObservation({ ...common, dimension: "price-output", value: price(pricing["completion"], "completion pricing"), unit: "usd", methodology: { ...methodology, basis: "OpenRouter USD per output token" } }),
|
|
89
90
|
];
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import {
|
|
2
|
+
BENCHMARK_MAX_MODELS_PER_SOURCE,
|
|
3
|
+
BENCHMARK_REFRESH_INTERVAL_MS,
|
|
4
|
+
BENCHMARK_SOURCE_MAX_RESPONSE_BYTES,
|
|
5
|
+
} from "../constants.ts";
|
|
6
|
+
import { normalizeModelIdentity, validateBenchmarkObservation, type BenchmarkObservation, type BenchmarkSourceSnapshot } from "../domain/benchmark.ts";
|
|
7
|
+
import type { BenchmarkSource } from "../ports/benchmark-source.ts";
|
|
8
|
+
import { contractRecord } from "../providers/openrouter-contracts.ts";
|
|
9
|
+
import type { OpenRouterBenchmarkTransport } from "./openrouter-benchmark-source.ts";
|
|
10
|
+
|
|
11
|
+
const SOURCE_ID = "openrouter-design-arena";
|
|
12
|
+
const BASE_URL = `https://openrouter.ai/api/v1/benchmarks?source=design-arena&max_results=${BENCHMARK_MAX_MODELS_PER_SOURCE}`;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Design Arena publishes rankings across dozens of arena/category pairs (music, video, TTS,
|
|
16
|
+
* ASCII art, ...) -- most of them have nothing to do with a text coding agent's model choice.
|
|
17
|
+
* This is a deliberate, bounded allowlist of the categories that measure frontend/UI-generation
|
|
18
|
+
* skill, the one facet of "design" quality relevant to routing a coding agent. Extending this
|
|
19
|
+
* list is a data curation decision (does the category's skill actually predict something a
|
|
20
|
+
* coding agent needs), not a mechanical one.
|
|
21
|
+
*/
|
|
22
|
+
const RELEVANT_CATEGORIES = ["codecategories", "website", "uicomponent", "dataviz", "svg"] as const;
|
|
23
|
+
|
|
24
|
+
function requiredText(value: unknown, name: string): string {
|
|
25
|
+
if (typeof value !== "string" || value.length === 0 || value.length > 500) throw new Error(`OpenRouter Design Arena benchmark ${name} schema changed`);
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function requiredNumber(value: unknown, name: string): number {
|
|
30
|
+
if (typeof value !== "number" || !Number.isFinite(value)) throw new Error(`OpenRouter Design Arena benchmark ${name} schema changed`);
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function identityFromOpenRouterId(openRouterId: string): { provider: string; model: string } {
|
|
35
|
+
const separator = openRouterId.indexOf("/");
|
|
36
|
+
if (separator <= 0 || separator === openRouterId.length - 1) throw new Error("OpenRouter Design Arena model identity schema changed");
|
|
37
|
+
return { provider: openRouterId.slice(0, separator), model: openRouterId.slice(separator + 1) };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export class OpenRouterDesignArenaSource implements BenchmarkSource {
|
|
41
|
+
readonly id = SOURCE_ID;
|
|
42
|
+
|
|
43
|
+
constructor(
|
|
44
|
+
private readonly apiKey: string,
|
|
45
|
+
private readonly transport: OpenRouterBenchmarkTransport = fetch,
|
|
46
|
+
private readonly clock: () => number = Date.now,
|
|
47
|
+
) {
|
|
48
|
+
if (apiKey.length === 0) throw new Error("OpenRouter API key is required for Design Arena benchmark evidence");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async fetch(): Promise<BenchmarkSourceSnapshot> {
|
|
52
|
+
const retrievedAt = this.clock();
|
|
53
|
+
if (!Number.isSafeInteger(retrievedAt) || retrievedAt <= 0) throw new Error("benchmark retrieval time is invalid");
|
|
54
|
+
const perCategory = await Promise.all(RELEVANT_CATEGORIES.map((category) => this.fetchCategory(category, retrievedAt)));
|
|
55
|
+
const revisions = new Set(perCategory.map((page) => page.revision));
|
|
56
|
+
const observations = perCategory.flatMap((page) => page.observations);
|
|
57
|
+
return { sourceId: this.id, snapshotId: `${this.id}:${[...revisions].sort().join(",")}`, retrievedAt, observations };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
private async fetchCategory(category: string, retrievedAt: number): Promise<{ revision: string; observations: BenchmarkObservation[] }> {
|
|
61
|
+
const url = `${BASE_URL}&task_type=${category}`;
|
|
62
|
+
const response = await this.transport(new Request(url, { headers: { authorization: `Bearer ${this.apiKey}` } }));
|
|
63
|
+
if (!response.ok) throw new Error(`OpenRouter Design Arena benchmarks failed with HTTP ${response.status}`);
|
|
64
|
+
const text = await response.text();
|
|
65
|
+
if (new TextEncoder().encode(text).byteLength > BENCHMARK_SOURCE_MAX_RESPONSE_BYTES) throw new Error("OpenRouter Design Arena benchmark response exceeds the size limit");
|
|
66
|
+
let payload: unknown;
|
|
67
|
+
try { payload = JSON.parse(text); } catch { throw new Error("OpenRouter Design Arena benchmark response is not valid JSON"); }
|
|
68
|
+
const root = contractRecord(payload, "benchmark response");
|
|
69
|
+
const meta = contractRecord(root["meta"], "benchmark metadata");
|
|
70
|
+
if (!Array.isArray(root["data"]) || root["data"].length > BENCHMARK_MAX_MODELS_PER_SOURCE) throw new Error("OpenRouter Design Arena benchmark result count is invalid");
|
|
71
|
+
if (requiredText(meta["source"], "source") !== "design-arena") throw new Error("OpenRouter Design Arena benchmark source schema changed");
|
|
72
|
+
const asOf = requiredText(meta["as_of"], "as-of date");
|
|
73
|
+
const publishedAt = Date.parse(asOf);
|
|
74
|
+
if (!Number.isSafeInteger(publishedAt) || publishedAt <= 0) throw new Error("OpenRouter Design Arena benchmark publication date schema changed");
|
|
75
|
+
const revision = `${category}:${asOf}`;
|
|
76
|
+
const observations = root["data"].flatMap((value): BenchmarkObservation[] => {
|
|
77
|
+
const row = contractRecord(value, "benchmark row");
|
|
78
|
+
if (requiredText(row["source"], "row source") !== "design-arena") throw new Error("OpenRouter Design Arena benchmark row source schema changed");
|
|
79
|
+
// A null open_router_id means the model isn't reachable through OpenRouter at all
|
|
80
|
+
// (proprietary platform, image/video generator, deprecated model) -- not evidence
|
|
81
|
+
// Jittor can ever route against, so the row is skipped rather than rejected.
|
|
82
|
+
const openRouterId = row["open_router_id"];
|
|
83
|
+
if (openRouterId === null) return [];
|
|
84
|
+
const { provider, model } = identityFromOpenRouterId(requiredText(openRouterId, "open_router_id"));
|
|
85
|
+
const identity = normalizeModelIdentity(provider, model, [`openrouter/${provider}/${model}`]);
|
|
86
|
+
const provenance = {
|
|
87
|
+
sourceId: SOURCE_ID,
|
|
88
|
+
sourceType: "independent" as const,
|
|
89
|
+
publisher: "Design Arena via OpenRouter",
|
|
90
|
+
url: BASE_URL,
|
|
91
|
+
revision,
|
|
92
|
+
publishedAt,
|
|
93
|
+
retrievedAt,
|
|
94
|
+
freshUntil: retrievedAt + BENCHMARK_REFRESH_INTERVAL_MS,
|
|
95
|
+
license: "OpenRouter API terms; Design Arena terms apply",
|
|
96
|
+
confidence: 0.7,
|
|
97
|
+
};
|
|
98
|
+
const methodology = { basis: "Design Arena Elo rating via OpenRouter", category, asOf };
|
|
99
|
+
return [validateBenchmarkObservation({
|
|
100
|
+
model: identity, dimension: "quality-design", value: requiredNumber(row["elo"], "elo rating"), unit: "elo", provenance, methodology,
|
|
101
|
+
})];
|
|
102
|
+
});
|
|
103
|
+
return { revision, observations };
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { Database } from "bun:sqlite";
|
|
2
|
-
import { DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT } from "../constants.ts";
|
|
2
|
+
import { DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT, USAGE_AGGREGATE_MAX_ROWS } from "../constants.ts";
|
|
3
3
|
import type { MetricObservation, MetricQuery, StoredMetricObservation } from "../domain/metric.ts";
|
|
4
4
|
import { validateMetricObservation } from "../domain/metric.ts";
|
|
5
|
-
import type {
|
|
5
|
+
import type { UsageAggregateRow } from "../domain/usage.ts";
|
|
6
|
+
import type { DistinctScopesFilter, MetricStore, UsageAggregateFilter } from "../ports/metric-store.ts";
|
|
6
7
|
|
|
7
8
|
interface MetricRow {
|
|
8
9
|
id: number;
|
|
@@ -75,6 +76,46 @@ export class SQLiteMetricStore implements MetricStore {
|
|
|
75
76
|
return rows.map(fromRow);
|
|
76
77
|
}
|
|
77
78
|
|
|
79
|
+
distinctScopes(filter: DistinctScopesFilter): string[] {
|
|
80
|
+
const limit = Math.max(1, Math.min(MAX_QUERY_LIMIT, Math.floor(filter.limit)));
|
|
81
|
+
const rows = this.db.query(`
|
|
82
|
+
SELECT DISTINCT scope FROM metric_observations
|
|
83
|
+
WHERE source = ? AND observed_at >= ? AND observed_at <= ?
|
|
84
|
+
ORDER BY scope ASC
|
|
85
|
+
LIMIT ?
|
|
86
|
+
`).all(filter.source, filter.since, filter.until, limit) as Array<{ scope: string }>;
|
|
87
|
+
return rows.map((row) => row.scope);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
aggregateUsage(filter: UsageAggregateFilter): UsageAggregateRow[] {
|
|
91
|
+
if (filter.scopes.length === 0) return [];
|
|
92
|
+
if (!Number.isFinite(filter.bucketSizeMs) || filter.bucketSizeMs <= 0) throw new Error("bucketSizeMs must be a positive number");
|
|
93
|
+
if (!Number.isInteger(filter.bucketCount) || filter.bucketCount <= 0) throw new Error("bucketCount must be a positive integer");
|
|
94
|
+
const scopePlaceholders = filter.scopes.map(() => "?").join(", ");
|
|
95
|
+
// MIN(a, b) is SQLite's scalar two-argument form (smallest of the arguments), mirroring
|
|
96
|
+
// usageBucketIndex's own `Math.min(bucketCount - 1, Math.floor(...))` clamp exactly, so a bucket
|
|
97
|
+
// this returns always matches the same-indexed bucket the client-side window renders.
|
|
98
|
+
const rows = this.db.query(`
|
|
99
|
+
SELECT scope, metric,
|
|
100
|
+
MIN(CAST((observed_at - ?) AS REAL) / ?, ? - 1) AS bucket_index_raw,
|
|
101
|
+
SUM(value) AS sum
|
|
102
|
+
FROM metric_observations
|
|
103
|
+
WHERE source = ? AND observed_at >= ? AND observed_at <= ? AND value >= 0 AND scope IN (${scopePlaceholders})
|
|
104
|
+
GROUP BY scope, metric, CAST(bucket_index_raw AS INTEGER)
|
|
105
|
+
LIMIT ?
|
|
106
|
+
`).all(
|
|
107
|
+
filter.since, filter.bucketSizeMs, filter.bucketCount,
|
|
108
|
+
filter.source, filter.since, filter.until, ...filter.scopes,
|
|
109
|
+
USAGE_AGGREGATE_MAX_ROWS,
|
|
110
|
+
) as Array<{ scope: string; metric: string; bucket_index_raw: number; sum: number }>;
|
|
111
|
+
return rows.map((row) => ({
|
|
112
|
+
scope: row.scope,
|
|
113
|
+
metric: row.metric,
|
|
114
|
+
bucketIndex: Math.max(0, Math.floor(row.bucket_index_raw)),
|
|
115
|
+
sum: row.sum,
|
|
116
|
+
}));
|
|
117
|
+
}
|
|
118
|
+
|
|
78
119
|
pruneBefore(cutoff: number): number {
|
|
79
120
|
if (!Number.isSafeInteger(cutoff) || cutoff < 0) throw new Error("cutoff must be a non-negative integer timestamp");
|
|
80
121
|
return this.db.query("DELETE FROM metric_observations WHERE observed_at < ?").run(cutoff).changes;
|
package/src/cli.ts
CHANGED
|
@@ -17,16 +17,19 @@ import {
|
|
|
17
17
|
MODEL_RANKING_DEFAULT_RELIABILITY_WEIGHT,
|
|
18
18
|
MODEL_RANKING_MAX_SOURCES,
|
|
19
19
|
SYSTEMD_UNIT_NAME,
|
|
20
|
+
MAX_USAGE_BUCKETS,
|
|
21
|
+
USAGE_MAX_DISTINCT_SCOPES,
|
|
20
22
|
} from "./constants.ts";
|
|
21
23
|
import { connectJittorClient, type JittorClient } from "./client.ts";
|
|
22
24
|
import { serveMain } from "./daemon.ts";
|
|
23
25
|
import type { BenchmarkQuery, BenchmarkQueryResult, BenchmarkRefreshResult } from "./domain/benchmark.ts";
|
|
24
26
|
import type { ModelRecommendationInput } from "./domain/model-ranking-service.ts";
|
|
25
27
|
import type { ModelCandidate, ModelRankingResult, ScopeAuthority, UtilityWeights } from "./domain/model-ranking.ts";
|
|
26
|
-
import {
|
|
28
|
+
import { TASK_DOMAINS, TASK_TYPES, type ModelTaskDomain, type ModelTaskType } from "./domain/model-observation.ts";
|
|
27
29
|
import type { ContextAssessment } from "./domain/context-telemetry.ts";
|
|
28
30
|
import { METRIC_UNITS, type MetricObservation, type MetricQuery, type MetricUnit, type StoredMetricObservation } from "./domain/metric.ts";
|
|
29
31
|
import type { CompactionDurationEstimate } from "./domain/context-telemetry.ts";
|
|
32
|
+
import type { TaskCostSummary } from "./domain/task-cost.ts";
|
|
30
33
|
import type { PolicyDecision, Route } from "./policy.ts";
|
|
31
34
|
import type { RouteOverride, RouterStatus, TelemetryPollResult } from "./ports/router-controller.ts";
|
|
32
35
|
import { EXPECTED_OPERATION_NAMES, type OperationInputs, type OperationName, type OperationOutputs } from "./service.ts";
|
|
@@ -104,7 +107,10 @@ function usage(stderr: (line: string) => void): number {
|
|
|
104
107
|
" benchmarks <status|refresh|list|rank> [options] [--json]",
|
|
105
108
|
" metrics record --source <s> --scope <s> --metric <s> --value <number|null> --unit <unit> [--observed-at <ms>] [--attributes <json>] [--json]",
|
|
106
109
|
" metrics query [--source <s>] [--scope <s>] [--metric <s>] [--since <ms>] [--until <ms>] [--limit <n>] [--order asc|desc] [--json]",
|
|
107
|
-
" metrics prune --before <ms> [--json]",
|
|
110
|
+
" metrics prune --before <ms> [--force] [--json] (force required if before is newer than 24h ago)",
|
|
111
|
+
` metrics distinct-scopes --source <s> --since <ms> --until <ms> [--limit 1..${USAGE_MAX_DISTINCT_SCOPES}] [--json]`,
|
|
112
|
+
` metrics usage-series --source <s> --since <ms> --until <ms> --bucket-size-ms <ms> --bucket-count 1..${MAX_USAGE_BUCKETS} [--scope-limit 1..${USAGE_MAX_DISTINCT_SCOPES}] [--json]`,
|
|
113
|
+
" metrics cost-by-task --since <ms> --until <ms> [--json]",
|
|
108
114
|
" telemetry poll [--json]",
|
|
109
115
|
" compaction estimate [--json]",
|
|
110
116
|
" router <status|decide|pause|resume|clear-override> [--json]",
|
|
@@ -225,14 +231,93 @@ function parseMetricsQueryArgs(args: string[]): MetricsQueryArgs | null {
|
|
|
225
231
|
return { input, json };
|
|
226
232
|
}
|
|
227
233
|
|
|
228
|
-
interface
|
|
234
|
+
interface MetricsDistinctScopesArgs { input: { source: string; since: number; until: number; limit?: number }; json: boolean }
|
|
235
|
+
|
|
236
|
+
function parseMetricsDistinctScopesArgs(args: string[]): MetricsDistinctScopesArgs | null {
|
|
237
|
+
let json = false;
|
|
238
|
+
let source: string | undefined;
|
|
239
|
+
let since: number | undefined;
|
|
240
|
+
let until: number | undefined;
|
|
241
|
+
let limit: number | undefined;
|
|
242
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
243
|
+
const argument = args[index];
|
|
244
|
+
if (argument === "--json") { json = true; continue; }
|
|
245
|
+
if (!["--source", "--since", "--until", "--limit"].includes(argument ?? "")) return null;
|
|
246
|
+
const raw = args[++index];
|
|
247
|
+
if (raw === undefined || raw.length === 0) return null;
|
|
248
|
+
if (argument === "--source") { source = raw; continue; }
|
|
249
|
+
const parsed = Number(raw);
|
|
250
|
+
if (!Number.isSafeInteger(parsed) || parsed < 0) return null;
|
|
251
|
+
if (argument === "--since") since = parsed;
|
|
252
|
+
else if (argument === "--until") until = parsed;
|
|
253
|
+
else {
|
|
254
|
+
if (parsed < 1 || parsed > USAGE_MAX_DISTINCT_SCOPES) return null;
|
|
255
|
+
limit = parsed;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
if (source === undefined || since === undefined || until === undefined || until < since) return null;
|
|
259
|
+
return { input: { source, since, until, ...(limit === undefined ? {} : { limit }) }, json };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
interface MetricsUsageSeriesArgs { input: { source: string; since: number; until: number; bucketSizeMs: number; bucketCount: number; scopeLimit?: number }; json: boolean }
|
|
263
|
+
|
|
264
|
+
function parseMetricsUsageSeriesArgs(args: string[]): MetricsUsageSeriesArgs | null {
|
|
265
|
+
let json = false;
|
|
266
|
+
let source: string | undefined;
|
|
267
|
+
let since: number | undefined;
|
|
268
|
+
let until: number | undefined;
|
|
269
|
+
let bucketSizeMs: number | undefined;
|
|
270
|
+
let bucketCount: number | undefined;
|
|
271
|
+
let scopeLimit: number | undefined;
|
|
272
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
273
|
+
const argument = args[index];
|
|
274
|
+
if (argument === "--json") { json = true; continue; }
|
|
275
|
+
if (!(["--source", "--since", "--until", "--bucket-size-ms", "--bucket-count", "--scope-limit"].includes(argument ?? ""))) return null;
|
|
276
|
+
const raw = args[++index];
|
|
277
|
+
if (raw === undefined || raw.length === 0) return null;
|
|
278
|
+
if (argument === "--source") { source = raw; continue; }
|
|
279
|
+
const parsed = Number(raw);
|
|
280
|
+
if (!Number.isSafeInteger(parsed) || parsed < 0) return null;
|
|
281
|
+
if (argument === "--since") since = parsed;
|
|
282
|
+
else if (argument === "--until") until = parsed;
|
|
283
|
+
else if (argument === "--bucket-size-ms") { if (parsed < 1) return null; bucketSizeMs = parsed; }
|
|
284
|
+
else if (argument === "--bucket-count") { if (parsed < 1 || parsed > MAX_USAGE_BUCKETS) return null; bucketCount = parsed; }
|
|
285
|
+
else { if (parsed < 1 || parsed > USAGE_MAX_DISTINCT_SCOPES) return null; scopeLimit = parsed; }
|
|
286
|
+
}
|
|
287
|
+
if (source === undefined || since === undefined || until === undefined || until < since || bucketSizeMs === undefined || bucketCount === undefined) return null;
|
|
288
|
+
return { input: { source, since, until, bucketSizeMs, bucketCount, ...(scopeLimit === undefined ? {} : { scopeLimit }) }, json };
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
interface CostByTaskArgs { input: { since: number; until: number }; json: boolean }
|
|
292
|
+
|
|
293
|
+
function parseCostByTaskArgs(args: string[]): CostByTaskArgs | null {
|
|
294
|
+
let json = false;
|
|
295
|
+
let since: number | undefined;
|
|
296
|
+
let until: number | undefined;
|
|
297
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
298
|
+
const argument = args[index];
|
|
299
|
+
if (argument === "--json") { json = true; continue; }
|
|
300
|
+
if (!["--since", "--until"].includes(argument ?? "")) return null;
|
|
301
|
+
const raw = args[++index];
|
|
302
|
+
const parsed = Number(raw);
|
|
303
|
+
if (!Number.isSafeInteger(parsed) || parsed < 0) return null;
|
|
304
|
+
if (argument === "--since") since = parsed;
|
|
305
|
+
else until = parsed;
|
|
306
|
+
}
|
|
307
|
+
if (since === undefined || until === undefined || until < since) return null;
|
|
308
|
+
return { input: { since, until }, json };
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
interface MetricsPruneArgs { input: { before: number; force?: boolean }; json: boolean }
|
|
229
312
|
|
|
230
313
|
function parseMetricsPruneArgs(args: string[]): MetricsPruneArgs | null {
|
|
231
314
|
let json = false;
|
|
315
|
+
let force = false;
|
|
232
316
|
let before: number | undefined;
|
|
233
317
|
for (let index = 0; index < args.length; index += 1) {
|
|
234
318
|
const argument = args[index];
|
|
235
319
|
if (argument === "--json") { json = true; continue; }
|
|
320
|
+
if (argument === "--force") { force = true; continue; }
|
|
236
321
|
if (argument !== "--before") return null;
|
|
237
322
|
const raw = args[++index];
|
|
238
323
|
const parsed = Number(raw);
|
|
@@ -240,7 +325,7 @@ function parseMetricsPruneArgs(args: string[]): MetricsPruneArgs | null {
|
|
|
240
325
|
before = parsed;
|
|
241
326
|
}
|
|
242
327
|
if (before === undefined) return null;
|
|
243
|
-
return { input: { before }, json };
|
|
328
|
+
return { input: { before, ...(force ? { force } : {}) }, json };
|
|
244
329
|
}
|
|
245
330
|
|
|
246
331
|
interface RouterOverrideArgs { input: RouteOverride; json: boolean }
|
|
@@ -356,7 +441,8 @@ function parseBenchmarkArgs(action: string | undefined, args: string[]): Benchma
|
|
|
356
441
|
const candidates: ModelCandidate[] = [];
|
|
357
442
|
const sourceIds: string[] = [];
|
|
358
443
|
let scopeAuthority: ScopeAuthority = "available-models";
|
|
359
|
-
let
|
|
444
|
+
let domain: ModelTaskDomain = "general";
|
|
445
|
+
let type: ModelTaskType = "general";
|
|
360
446
|
let budgetPressure = 0;
|
|
361
447
|
const weights: UtilityWeights = {
|
|
362
448
|
quality: MODEL_RANKING_DEFAULT_QUALITY_WEIGHT, cost: MODEL_RANKING_DEFAULT_COST_WEIGHT,
|
|
@@ -368,7 +454,7 @@ function parseBenchmarkArgs(action: string | undefined, args: string[]): Benchma
|
|
|
368
454
|
if (argument === "--json") { json = true; continue; }
|
|
369
455
|
if (argument === "--force" && action === "refresh") { force = true; continue; }
|
|
370
456
|
const allowed = action === "list" ? ["--source", "--model", "--dimension", "--limit"]
|
|
371
|
-
: action === "rank" ? ["--candidate", "--source", "--
|
|
457
|
+
: action === "rank" ? ["--candidate", "--source", "--domain", "--type", "--scope", "--budget", "--weight-quality", "--weight-cost", "--weight-latency", "--weight-context", "--weight-reliability"] : [];
|
|
372
458
|
if (!allowed.includes(argument ?? "")) return null;
|
|
373
459
|
const raw = args[++index];
|
|
374
460
|
if (raw === undefined || raw.length === 0) return null;
|
|
@@ -387,9 +473,12 @@ function parseBenchmarkArgs(action: string | undefined, args: string[]): Benchma
|
|
|
387
473
|
if (!candidate) return null;
|
|
388
474
|
candidates.push(candidate);
|
|
389
475
|
} else if (argument === "--source") sourceIds.push(raw);
|
|
390
|
-
else if (argument === "--
|
|
391
|
-
if (!
|
|
392
|
-
|
|
476
|
+
else if (argument === "--domain") {
|
|
477
|
+
if (!TASK_DOMAINS.includes(raw as ModelTaskDomain)) return null;
|
|
478
|
+
domain = raw as ModelTaskDomain;
|
|
479
|
+
} else if (argument === "--type") {
|
|
480
|
+
if (!TASK_TYPES.includes(raw as ModelTaskType)) return null;
|
|
481
|
+
type = raw as ModelTaskType;
|
|
393
482
|
} else if (argument === "--scope") {
|
|
394
483
|
if (raw !== "exact-session" && raw !== "available-models") return null;
|
|
395
484
|
scopeAuthority = raw;
|
|
@@ -405,7 +494,7 @@ function parseBenchmarkArgs(action: string | undefined, args: string[]): Benchma
|
|
|
405
494
|
return {
|
|
406
495
|
action, json, force,
|
|
407
496
|
...(action === "list" ? { query: query as BenchmarkQuery } : {}),
|
|
408
|
-
...(action === "rank" ? { recommendation: { candidates, sourceIds: [...new Set(sourceIds)], scopeAuthority,
|
|
497
|
+
...(action === "rank" ? { recommendation: { candidates, sourceIds: [...new Set(sourceIds)], scopeAuthority, domain, type, budgetPressure, weights } } : {}),
|
|
409
498
|
};
|
|
410
499
|
}
|
|
411
500
|
|
|
@@ -465,6 +554,31 @@ export function formatMetricsQuery(rows: StoredMetricObservation[]): string {
|
|
|
465
554
|
return lines.join("\n");
|
|
466
555
|
}
|
|
467
556
|
|
|
557
|
+
export function formatMetricsDistinctScopes(scopes: string[]): string {
|
|
558
|
+
if (scopes.length === 0) return "Scopes: none matched";
|
|
559
|
+
return [`Scopes: ${scopes.length.toLocaleString()}`, ...scopes.map((scope) => `- ${humanField(scope)}`)].join("\n");
|
|
560
|
+
}
|
|
561
|
+
|
|
562
|
+
export function formatMetricsUsageSeries(result: { rows: Array<{ scope: string; metric: string; bucketIndex: number; sum: number }>; truncated: boolean }): string {
|
|
563
|
+
if (result.rows.length === 0) return `Usage series: no data${result.truncated ? " (scope limit reached)" : ""}`;
|
|
564
|
+
const lines = [`Usage series: ${result.rows.length.toLocaleString()} bucket(s)${result.truncated ? " (scope limit reached)" : ""}`];
|
|
565
|
+
for (const row of result.rows) lines.push(`- ${humanField(row.scope)}/${humanField(row.metric)} bucket ${row.bucketIndex}: ${row.sum.toLocaleString()}`);
|
|
566
|
+
return lines.join("\n");
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
function formatUsdAmount(amount: number): string {
|
|
570
|
+
return `$${amount.toFixed(Math.abs(amount) < 0.01 && amount !== 0 ? 4 : 2)}`;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
export function formatCostByTask(summary: TaskCostSummary): string {
|
|
574
|
+
const lines = [
|
|
575
|
+
`Cost by task: ${summary.entries.length.toLocaleString()} task(s)${summary.truncated ? " (query limit reached; totals are a lower bound)" : ""}`,
|
|
576
|
+
...summary.entries.map((entry) => `- ${humanField(entry.taskId)}: ${formatUsdAmount(entry.costUsd)} · ↑${entry.inputTokens.toLocaleString()} ↓${entry.outputTokens.toLocaleString()} R${entry.cacheReadTokens.toLocaleString()} W${entry.cacheWriteTokens.toLocaleString()}`),
|
|
577
|
+
`Unattributed spend (no task was focused): ${formatUsdAmount(summary.unattributedCostUsd)}`,
|
|
578
|
+
];
|
|
579
|
+
return lines.join("\n");
|
|
580
|
+
}
|
|
581
|
+
|
|
468
582
|
export function formatTelemetryPoll(result: TelemetryPollResult): string {
|
|
469
583
|
if (result.sources.length === 0) return "Telemetry: no sources configured";
|
|
470
584
|
return ["Telemetry:", ...result.sources.map((source) => {
|
|
@@ -535,6 +649,21 @@ export async function runCli(args: string[], deps: CliDependencies = DEFAULT_DEP
|
|
|
535
649
|
if (!parsed) return usage(deps.stderr);
|
|
536
650
|
return callAndPrint(deps, "metrics.prune", parsed.input, parsed.json, (result) => `Pruned ${result.deleted.toLocaleString()} observation(s)`);
|
|
537
651
|
}
|
|
652
|
+
if (action === "distinct-scopes") {
|
|
653
|
+
const parsed = parseMetricsDistinctScopesArgs(rest);
|
|
654
|
+
if (!parsed) return usage(deps.stderr);
|
|
655
|
+
return callAndPrint(deps, "metrics.distinct_scopes", parsed.input, parsed.json, formatMetricsDistinctScopes);
|
|
656
|
+
}
|
|
657
|
+
if (action === "usage-series") {
|
|
658
|
+
const parsed = parseMetricsUsageSeriesArgs(rest);
|
|
659
|
+
if (!parsed) return usage(deps.stderr);
|
|
660
|
+
return callAndPrint(deps, "metrics.usage_series", parsed.input, parsed.json, formatMetricsUsageSeries);
|
|
661
|
+
}
|
|
662
|
+
if (action === "cost-by-task") {
|
|
663
|
+
const parsed = parseCostByTaskArgs(rest);
|
|
664
|
+
if (!parsed) return usage(deps.stderr);
|
|
665
|
+
return callAndPrint(deps, "metrics.cost_by_task", parsed.input, parsed.json, formatCostByTask);
|
|
666
|
+
}
|
|
538
667
|
return usage(deps.stderr);
|
|
539
668
|
}
|
|
540
669
|
if (command === "telemetry") {
|
package/src/client.ts
CHANGED
|
@@ -1,51 +1,19 @@
|
|
|
1
|
+
import { AuthenticatedRpcClient, type FetchTransport } from "@danypops/daemon-kit/rpc-client";
|
|
1
2
|
import type { OperationInputs, OperationName, OperationOutputs } from "./service.ts";
|
|
2
3
|
import { ensureAuthToken, readDaemonHandle, resolveJittorPaths, type JittorPaths } from "./state.ts";
|
|
3
4
|
|
|
4
|
-
export type FetchTransport
|
|
5
|
+
export type { FetchTransport };
|
|
5
6
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
headers: { authorization: `Bearer ${this.token}`, "content-type": "application/json" },
|
|
17
|
-
body: JSON.stringify({ op: operation, input }),
|
|
18
|
-
}));
|
|
19
|
-
const body = await response.json() as { result?: OperationOutputs[Name]; error?: string };
|
|
20
|
-
if (!response.ok) throw new Error(body.error ?? `Jittor operation failed with HTTP ${response.status}`);
|
|
21
|
-
return body.result as OperationOutputs[Name];
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
async operations(): Promise<OperationName[]> {
|
|
25
|
-
const response = await this.transport(new Request(`${this.baseUrl}/api/v1/ops`, {
|
|
26
|
-
headers: { authorization: `Bearer ${this.token}` },
|
|
27
|
-
}));
|
|
28
|
-
const body = await response.json() as { operations?: OperationName[]; error?: string };
|
|
29
|
-
if (!response.ok) throw new Error(body.error ?? `Jittor discovery failed with HTTP ${response.status}`);
|
|
30
|
-
return body.operations ?? [];
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
async ready(): Promise<boolean> {
|
|
34
|
-
const response = await this.transport(new Request(`${this.baseUrl}/ready`, {
|
|
35
|
-
headers: { authorization: `Bearer ${this.token}` },
|
|
36
|
-
}));
|
|
37
|
-
if (response.status === 503) return false;
|
|
38
|
-
if (!response.ok) throw new Error(`Jittor readiness check failed with HTTP ${response.status}`);
|
|
39
|
-
return true;
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
async health(): Promise<{ ok: true; version: string }> {
|
|
43
|
-
const response = await this.transport(new Request(`${this.baseUrl}/health`, {
|
|
44
|
-
headers: { authorization: `Bearer ${this.token}` },
|
|
45
|
-
}));
|
|
46
|
-
const body = await response.json() as { ok?: boolean; version?: string; error?: string };
|
|
47
|
-
if (!response.ok || body.ok !== true || typeof body.version !== "string") throw new Error(body.error ?? "Jittor health check failed");
|
|
48
|
-
return { ok: true, version: body.version };
|
|
7
|
+
/**
|
|
8
|
+
* Jittor's typed authenticated RPC client, now a thin named subclass of
|
|
9
|
+
* `@danypops/daemon-kit/rpc-client`'s `AuthenticatedRpcClient` -- the shared substrate factored
|
|
10
|
+
* out after jittor's own client.ts and web-spider-daemon's were found byte-identical (see
|
|
11
|
+
* daemon-kit's README). Keeps the old 3-positional-argument constructor so every existing call
|
|
12
|
+
* site is untouched by this migration.
|
|
13
|
+
*/
|
|
14
|
+
export class JittorClient extends AuthenticatedRpcClient<OperationName, OperationInputs, OperationOutputs> {
|
|
15
|
+
constructor(baseUrl: string, token: string, transport: FetchTransport = fetch) {
|
|
16
|
+
super(baseUrl, token, { label: "Jittor", transport });
|
|
49
17
|
}
|
|
50
18
|
}
|
|
51
19
|
|
package/src/constants.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
export const SQLITE_SCHEMA_VERSION = 1;
|
|
2
1
|
export const SQLITE_BUSY_TIMEOUT_MS = 5_000;
|
|
3
2
|
export const DEFAULT_QUERY_LIMIT = 1_000;
|
|
4
3
|
export const MAX_QUERY_LIMIT = 10_000;
|
|
@@ -39,7 +38,6 @@ export const FOOTER_BAR_MIN_WIDTH = 4;
|
|
|
39
38
|
export const FOOTER_BAR_MAX_WIDTH = 8;
|
|
40
39
|
export const FOOTER_WIDE_TERMINAL_WIDTH = 100;
|
|
41
40
|
export const FOOTER_COMPACTION_RENDER_INTERVAL_MS = 500;
|
|
42
|
-
export const FOOTER_COMPACTION_DRAIN_STEP_MS = 3_000;
|
|
43
41
|
/** Half-period of the compacting liveness indicator; equal to the render tick so it visibly alternates every repaint. */
|
|
44
42
|
export const FOOTER_COMPACTION_BLINK_HALF_PERIOD_MS = 500;
|
|
45
43
|
export const MILLISECONDS_PER_SECOND = 1_000;
|
|
@@ -47,13 +45,20 @@ export const MILLISECONDS_PER_MINUTE = 60 * MILLISECONDS_PER_SECOND;
|
|
|
47
45
|
export const MILLISECONDS_PER_HOUR = 60 * MILLISECONDS_PER_MINUTE;
|
|
48
46
|
export const PAPYRUS_CONTEXT_INJECTION_CHANNEL = "papyrus.context-injection.v1";
|
|
49
47
|
export const PAPYRUS_CONTEXT_INJECTION_SCHEMA = "papyrus.context-injection/v1";
|
|
48
|
+
/** Matches Papyrus's own constants by convention; Jittor does not depend on the Papyrus package. */
|
|
49
|
+
export const PAPYRUS_TASK_FOCUS_CHANNEL = "papyrus.task-focus.v1";
|
|
50
|
+
export const PAPYRUS_TASK_FOCUS_SCHEMA = "papyrus.task-focus/v1";
|
|
51
|
+
export const TASK_FOCUS_EVENT_MAX_AGE_MS = 5 * MILLISECONDS_PER_MINUTE;
|
|
52
|
+
export const TASK_FOCUS_ID_MAX_LENGTH = 200;
|
|
53
|
+
export const TASK_COST_QUERY_LIMIT = 10_000;
|
|
54
|
+
/** A `metrics prune --before` cutoff newer than this must pass `force: true`. Guards against accidentally wiping recent/live data with a too-recent cutoff (e.g. "now"), while still allowing routine cleanup of genuinely old rows without ceremony. */
|
|
55
|
+
export const PRUNE_MIN_AGE_MS = 24 * MILLISECONDS_PER_HOUR;
|
|
50
56
|
export const CONTEXT_OBSERVATION_MAX_CHARACTERS = 10_000_000;
|
|
51
57
|
export const CONTEXT_OBSERVATION_MAX_AGE_MS = 5 * MILLISECONDS_PER_MINUTE;
|
|
52
58
|
export const CONTEXT_ASSESSMENT_QUERY_LIMIT = 10_000;
|
|
53
59
|
export const CONTEXT_EVENT_DEDUP_LIMIT = 1_000;
|
|
54
60
|
export const CONTEXT_ASSESSMENT_DEFAULT_WINDOW_MS = 24 * MILLISECONDS_PER_HOUR;
|
|
55
61
|
export const MILLISECONDS_PER_DAY = 24 * MILLISECONDS_PER_HOUR;
|
|
56
|
-
export const LOOPBACK_HOST = "127.0.0.1";
|
|
57
62
|
export const JITTOR_STATE_DIRECTORY = "jittor";
|
|
58
63
|
export const JITTOR_EXTENSION_SETTINGS_FILENAME = "extension.json";
|
|
59
64
|
export const DATABASE_FILENAME = "jittor.db";
|
|
@@ -62,7 +67,17 @@ export const HANDLE_FILENAME = "daemon.json";
|
|
|
62
67
|
export const SYSTEMD_UNIT_NAME = "jittor.service";
|
|
63
68
|
export const USAGE_CHART_HEIGHT = 8;
|
|
64
69
|
export const USAGE_Y_AXIS_WIDTH = 7;
|
|
65
|
-
|
|
70
|
+
/**
|
|
71
|
+
* A single flat "most recent N rows" query lets one heavy provider/model monopolize the entire
|
|
72
|
+
* budget within the query window, silently starving every other series out of the chart no matter
|
|
73
|
+
* which time frame is selected (a real bug: a single long, heavy session can fill 10k rows within
|
|
74
|
+
* a few hours, hiding a whole other provider's usage from a day earlier even in the Monthly view).
|
|
75
|
+
* The usage/cost dashboard instead fetches per distinct scope (see distinctScopes), bounded by
|
|
76
|
+
* these two limits; the worst-case total row volume (40 * 250 = 10,000) matches the old flat cap,
|
|
77
|
+
* but is now fairly distributed across every active series instead of claimable by just one.
|
|
78
|
+
*/
|
|
79
|
+
export const USAGE_MAX_DISTINCT_SCOPES = 40;
|
|
80
|
+
export const USAGE_PER_SCOPE_QUERY_LIMIT = 250;
|
|
66
81
|
export const USAGE_RENDER_MAX_SERIES = 20;
|
|
67
82
|
export const HUMAN_STATUS_MAX_SOURCES = 20;
|
|
68
83
|
export const HUMAN_TEXT_FIELD_MAX_CHARACTERS = 160;
|
|
@@ -73,6 +88,8 @@ export const COMPACTION_DURATION_ESTIMATE_MAX_SAMPLES = 20;
|
|
|
73
88
|
/** Below this many samples the estimate stays explicit cold-start uncertainty rather than a guess. */
|
|
74
89
|
export const COMPACTION_DURATION_ESTIMATE_MIN_SAMPLES = 3;
|
|
75
90
|
export const MAX_USAGE_BUCKETS = 120;
|
|
91
|
+
/** Defense-in-depth cap on the SQL-side usage aggregation result: (scopes x metrics x buckets) is already small by construction, but this bounds it explicitly rather than trusting that alone. */
|
|
92
|
+
export const USAGE_AGGREGATE_MAX_ROWS = 25_000;
|
|
76
93
|
export const MAX_DYNAMIC_ROUTES = 100;
|
|
77
94
|
export const CODEX_ERROR_MESSAGE_LIMIT = 160;
|
|
78
95
|
export const CODEX_RETRY_AFTER_MAX_MS = 5 * MILLISECONDS_PER_MINUTE;
|
|
@@ -81,3 +98,13 @@ export const CODEX_RECOVERY_MAX_DELAY_MS = CODEX_RETRY_AFTER_MAX_MS;
|
|
|
81
98
|
export const CODEX_RECOVERY_MAX_ATTEMPTS = 3;
|
|
82
99
|
export const CODEX_RECOVERY_ATTEMPT_WINDOW_MS = 10 * MILLISECONDS_PER_MINUTE;
|
|
83
100
|
export const CODEX_RECOVERY_JITTER_RATIO = 0.2;
|
|
101
|
+
export const GOOGLE_VERTEX_BUDGET_DISPLAY_NAME_MAX_CHARACTERS = 160;
|
|
102
|
+
/** Bounded per-poll pull size; a budget subscription realistically holds at most a few pending notifications. */
|
|
103
|
+
export const GOOGLE_VERTEX_BUDGET_MAX_MESSAGES_PER_PULL = 20;
|
|
104
|
+
/**
|
|
105
|
+
* Lower than Codex's header-derived 0.8: this is Google's own documented "estimated ... subject to
|
|
106
|
+
* change until your invoice is finalized" data, delivered at-least-once and possibly out of order,
|
|
107
|
+
* multiple times per day rather than on every response.
|
|
108
|
+
*/
|
|
109
|
+
export const GOOGLE_VERTEX_BUDGET_CONFIDENCE = 0.6;
|
|
110
|
+
export const GOOGLE_ADC_TOKEN_REFRESH_SKEW_MS = 60_000;
|