dsh-auto-model-routing 0.4.0 → 0.4.2

package/lib/usage-report.js

@@ -0,0 +1,388 @@
+// Read-only, on-demand AUTO usage report (change auto-routing-usage-report).
+// Never writes, spawns, or touches the network; never echoes message text, paths or session ids.
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { parseArgs } from 'node:util';
+import { zstdDecompressSync } from 'node:zlib';
+import { FeedbackBook } from './feedback.js';
+const MAGIC = Buffer.from([0x28, 0xb5, 0x2f, 0xfd]);
+const FIELDS = ['inputTokens', 'outputTokens', 'cacheReadTokens', 'cacheWriteTokens', 'reasoningTokens'];
+const FEEDBACK = ['appropriate', 'too-high', 'too-low', 'unknown'];
+const CAVEATS = [
+    'observational-not-controlled',
+    'scope-main-session-assistant-calls-only',
+    'only-applied-decisions-recorded',
+    'token-semantics-differ-by-provider',
+    'reference-before-auto-effort-only-confounded',
+    'mode-history-not-persisted',
+    'implicit-ok-not-appropriateness',
+    'exposure-not-proof-of-reading',
+    'historical-exposure-unknown',
+    'exposure-after-continuation-excluded',
+];
+const sorted = (record) => Object.fromEntries(Object.entries(record).sort(([a], [b]) => a.localeCompare(b)));
+const round = (n) => Math.round(n * 100) / 100;
+const bump = (record, key, by = 1) => { record[key] = (record[key] ?? 0) + by; };
+/** Split concatenated zstd frames; only a failing last segment counts as an unfinished tail. */
+function decode(bytes) {
+    const starts = [];
+    for (let i = bytes.indexOf(MAGIC); i !== -1; i = bytes.indexOf(MAGIC, i + 1))
+        starts.push(i);
+    if (starts[0] !== 0)
+        return undefined;
+    let text = '';
+    for (let i = 0; i < starts.length; i++) {
+        try {
+            text += zstdDecompressSync(bytes.subarray(starts[i], starts[i + 1])).toString('utf8');
+        }
+        catch {
+            if (i === starts.length - 1)
+                return { text, tail: true };
+            return undefined;
+        }
+    }
+    return { text, tail: false };
+}
+function lines(file, failures) {
+    let text;
+    try {
+        text = readFileSync(file, 'utf8');
+    }
+    catch {
+        return [];
+    }
+    const out = [];
+    for (const line of text.split('\n')) {
+        if (!line.trim())
+            continue;
+        try {
+            out.push(JSON.parse(line));
+        }
+        catch {
+            failures['parse-failed']++;
+        }
+    }
+    return out;
+}
+function logPaths(home) {
+    const paths = new Map();
+    const root = join(home, 'sessions');
+    let dirs = [];
+    try {
+        dirs = readdirSync(root);
+    }
+    catch {
+        return paths;
+    }
+    for (const dir of dirs) {
+        let ids = [];
+        try {
+            ids = readdirSync(join(root, dir));
+        }
+        catch {
+            continue;
+        }
+        for (const id of ids) {
+            const file = join(root, dir, id, 'session.v4.jsonl.zstd');
+            if (existsSync(file))
+                paths.set(id, file);
+        }
+    }
+    return paths;
+}
+/** Calls in one session log, with the user turn each belongs to and the config in effect. */
+function readSession(file, failures) {
+    let decoded;
+    try {
+        decoded = decode(readFileSync(file));
+    }
+    catch {
+        decoded = undefined;
+    }
+    if (!decoded) {
+        failures['decode-failed']++;
+        return undefined;
+    }
+    if (decoded.tail)
+        failures['tail-incomplete']++;
+    const events = [];
+    for (const line of decoded.text.split('\n')) {
+        if (!line.trim())
+            continue;
+        try {
+            events.push(JSON.parse(line));
+        }
+        catch {
+            failures['parse-failed']++;
+        }
+    }
+    const header = events.shift();
+    if (header?.type !== 'session') {
+        failures['decode-failed']++;
+        return undefined;
+    }
+    const calls = [];
+    const seen = new Set();
+    const ends = new Map(), userStarts = new Map();
+    let startedAt;
+    let turn, userTurn;
+    let config = {};
+    for (const event of events) {
+        const data = event.data ?? {};
+        if (event.type === 'turn/start') {
+            turn = data.turn;
+            startedAt = event.time;
+        }
+        else if (event.type === 'user/message' && data.source?.kind === 'user' && turn !== undefined) {
+            userTurn = turn;
+            if (typeof startedAt === 'number')
+                userStarts.set(turn, startedAt);
+        }
+        else if (event.type === 'turn/end' && typeof data.turn === 'number') {
+            const reason = data.reason;
+            ends.set(data.turn, reason?.kind === 'aborted' ? reason.reason?.kind === 'user' ? 'aborted-user' : 'aborted-other' : ['completed', 'blocked'].includes(reason?.kind) ? reason.kind : 'other');
+        }
+        else if (event.type === 'request/header' && data.header?.config)
+            config = data.header.config;
+        else if (event.type === 'assistant/message') {
+            const op = event.surfaceOp;
+            const append = op === undefined || op === 'append' || (typeof op === 'object' && op?.op === 'append');
+            const id = data.message?.id;
+            if (!append || typeof id !== 'string' || seen.has(id))
+                continue;
+            seen.add(id);
+            const usage = {};
+            for (const field of FIELDS)
+                if (typeof data.usage?.[field] === 'number')
+                    usage[field] = data.usage[field];
+            calls.push({ id, ...(typeof data.turn === 'number' ? { turn: data.turn } : turn === undefined ? {} : { turn }), ...(userTurn === undefined ? {} : { userTurn }), time: event.time, usage, ...(config.model ? { model: config.model } : {}), ...(config.reasoningEffort ? { effort: config.reasoningEffort } : {}) });
+        }
+    }
+    return { header, calls, ends, userStarts };
+}
+function localMidnight(date) {
+    const [y, m, d] = date.split('-').map(Number);
+    const at = new Date(y, m - 1, d);
+    const offset = -at.getTimezoneOffset();
+    const sign = offset >= 0 ? '+' : '-';
+    const hh = String(Math.floor(Math.abs(offset) / 60)).padStart(2, '0'), mm = String(Math.abs(offset) % 60).padStart(2, '0');
+    return { ms: at.getTime(), label: `${date}T00:00:00${sign}${hh}:${mm}` };
+}
+function tokens(calls) {
+    const out = {};
+    for (const field of FIELDS) {
+        const values = calls.map(c => c.usage[field]).filter((v) => v !== undefined);
+        const sum = values.reduce((a, b) => a + b, 0);
+        out[field] = values.length ? { sum, mean: round(sum / values.length), present: values.length, calls: calls.length } : { sum: null, mean: null, present: 0, calls: calls.length };
+    }
+    return out;
+}
+function median(values) {
+    const s = [...values].sort((a, b) => a - b), mid = Math.floor(s.length / 2);
+    return s.length % 2 ? s[mid] : (s[mid - 1] + s[mid]) / 2;
+}
+export function buildReport(home, options = {}) {
+    const failures = { 'decode-failed': 0, 'tail-incomplete': 0, 'parse-failed': 0, 'missing-log': 0 };
+    const state = join(home, 'state/auto-model-routing');
+    const since = options.since ? localMidnight(options.since) : undefined;
+    const after = (time) => !since || time >= since.ms;
+    const feedbackFile = join(state, 'allocation-feedback.jsonl');
+    lines(feedbackFile, failures); // counts unparsable lines; FeedbackBook skips them silently
+    const book = new FeedbackBook(feedbackFile);
+    const bindings = book.entries();
+    const legacyFirst = new Map();
+    for (const line of lines(join(state, 'judgements.jsonl'), failures)) {
+        if (typeof line?.sessionId !== 'string' || !Number.isInteger(line.turn) || typeof line.tier !== 'string')
+            continue;
+        legacyFirst.set(line.sessionId, Math.min(legacyFirst.get(line.sessionId) ?? Infinity, line.turn));
+    }
+    const ledgerSessions = [...new Set([...bindings.map(b => b.sessionId), ...legacyFirst.keys()])].sort();
+    const paths = logPaths(home);
+    const excluded = { child: 0, seeded: 0 };
+    const decisions = new Map();
+    const reference = {};
+    const autoEffort = {};
+    let unlinkedBindings = 0, userTurns = 0, turnsWithoutRecord = 0;
+    for (const session of ledgerSessions) {
+        const path = paths.get(session);
+        if (!path) {
+            failures['missing-log']++;
+            continue;
+        }
+        const log = readSession(path, failures);
+        if (!log)
+            continue;
+        if (log.header.origin) {
+            excluded.child++;
+            continue;
+        }
+        if (log.header.isSeeded === true) {
+            excluded.seeded++;
+            continue;
+        }
+        const nextUserStart = new Map();
+        let next;
+        const turns = [...new Set([...log.calls.flatMap(call => call.turn === undefined ? [] : [call.turn]), ...log.userStarts.keys()])].sort((a, b) => b - a);
+        for (const turn of turns) {
+            if (next !== undefined)
+                nextUserStart.set(turn, next);
+            if (log.userStarts.has(turn))
+                next = log.userStarts.get(turn);
+        }
+        const linked = new Set();
+        let v2First;
+        for (const call of log.calls) {
+            const b = book.lookup(session, call.id);
+            if (!b)
+                continue;
+            linked.add(call.id);
+            if (call.userTurn !== undefined)
+                v2First = Math.min(v2First ?? Infinity, call.userTurn);
+            const key = JSON.stringify([session, b.decisionId]);
+            let d = decisions.get(key);
+            if (!d) {
+                d = { session, id: b.decisionId, version: b.snapshot ? `${b.snapshot.policyVersion}/${b.snapshot.mappingVersion}` : 'unknown', model: b.model, depth: b.depth, effort: b.effort, calls: [], feedback: new Set() };
+                decisions.set(key, d);
+            }
+            d.calls.push(call);
+            d.end = call.turn === undefined ? 'unknown' : log.ends.get(call.turn) ?? 'unknown';
+            d.continuation = call.turn === undefined ? undefined : nextUserStart.get(call.turn);
+            if (b.feedback)
+                d.feedback.add(b.feedback);
+        }
+        unlinkedBindings += bindings.filter(b => b.sessionId === session && !linked.has(b.messageId)).length;
+        // Coverage: user turns from the first v2 record on, with calls in range but none linked.
+        if (v2First !== undefined) {
+            const turns = new Map();
+            for (const call of log.calls) {
+                if (call.userTurn === undefined || call.userTurn < v2First || !after(call.time))
+                    continue;
+                turns.set(call.userTurn, (turns.get(call.userTurn) ?? false) || linked.has(call.id));
+            }
+            userTurns += turns.size;
+            turnsWithoutRecord += [...turns.values()].filter(v => !v).length;
+        }
+        // Reference: calls before the earliest record in either ledger (effort only, never tokens).
+        const first = Math.min(legacyFirst.get(session) ?? Infinity, v2First ?? Infinity);
+        for (const call of log.calls) {
+            if (call.userTurn === undefined || call.userTurn >= first || !after(call.time) || !call.model)
+                continue;
+            const r = (reference[call.model] ??= { calls: 0, sessions: new Set(), effort: {} });
+            r.calls++;
+            r.sessions.add(session);
+            bump(r.effort, call.effort ?? 'unknown');
+        }
+    }
+    const kept = [...decisions.values()].filter(d => after(Math.min(...d.calls.map(c => c.time))));
+    for (const d of kept)
+        bump((autoEffort[d.model] ??= {}), d.effort, d.calls.length);
+    const versions = new Map();
+    for (const d of kept) {
+        const byVersion = versions.get(d.version) ?? new Map();
+        versions.set(d.version, byVersion);
+        const key = JSON.stringify([d.model, d.depth]);
+        byVersion.set(key, [...(byVersion.get(key) ?? []), d]);
+    }
+    const groups = [...versions].sort(([a], [b]) => a.localeCompare(b)).map(([version, subs]) => ({
+        version,
+        subgroups: [...subs].sort(([a], [b]) => a.localeCompare(b)).map(([key, list]) => {
+            const [model, depth] = JSON.parse(key);
+            const calls = list.flatMap(d => d.calls);
+            const sessions = new Set(list.map(d => d.session)).size;
+            const feedback = { ...Object.fromEntries(FEEDBACK.map(f => [f, 0])), conflict: 0, unrated: 0 };
+            for (const d of list)
+                bump(feedback, d.feedback.size === 0 ? 'unrated' : d.feedback.size > 1 ? 'conflict' : [...d.feedback][0]);
+            const implicit = {
+                counts: { explicit: 0, negative: 0, 'implicit-ok': 0, inconclusive: 0, unobserved: 0 },
+                explicit: { ...Object.fromEntries(FEEDBACK.map(f => [f, 0])), conflict: 0 },
+                exposedExplicit: { ...Object.fromEntries(FEEDBACK.map(f => [f, 0])), conflict: 0 },
+                negative: { 'aborted-user': 0, 'manual-switch': 0 },
+            };
+            for (const d of list) {
+                const facts = book.facts(d.session, d.id);
+                const exposed = facts.exposed !== undefined && (d.continuation === undefined || facts.exposed < d.continuation);
+                const label = d.feedback.size > 1 ? 'conflict' : [...d.feedback][0];
+                if (label) {
+                    implicit.counts.explicit++;
+                    bump(implicit.explicit, label);
+                    if (exposed)
+                        bump(implicit.exposedExplicit, label);
+                }
+                else if (d.end === 'aborted-user' || facts['manual-switch'] !== undefined) {
+                    implicit.counts.negative++;
+                    if (d.end === 'aborted-user')
+                        implicit.negative['aborted-user']++;
+                    if (facts['manual-switch'] !== undefined)
+                        implicit.negative['manual-switch']++;
+                }
+                else if (exposed && d.end === 'completed' && d.continuation !== undefined)
+                    implicit.counts['implicit-ok']++;
+                else if (exposed)
+                    implicit.counts.inconclusive++;
+                else
+                    implicit.counts.unobserved++;
+            }
+            const byEffort = {};
+            for (const d of list) {
+                const e = (byEffort[d.effort] ??= { decisions: 0, calls: 0 });
+                e.decisions++;
+                e.calls += d.calls.length;
+            }
+            return {
+                model, depth, decisions: list.length, sessions, calls: calls.length,
+                callsPerDecision: { median: median(list.map(d => d.calls.length)), max: Math.max(...list.map(d => d.calls.length)) },
+                tokens: tokens(calls), byEffort: sorted(byEffort), feedback, implicit,
+                conclusion: list.length >= 20 && sessions >= 3 ? 'observed' : 'insufficient-sample',
+            };
+        }),
+    }));
+    const recorded = new Set(ledgerSessions.map(s => createHash('sha256').update(s).digest('hex')));
+    let autoSessionsWithoutRecord = 0;
+    let stateFiles = [];
+    try {
+        stateFiles = readdirSync(join(state, 'sessions')).filter(f => f.endsWith('.json')).sort();
+    }
+    catch { /* none */ }
+    for (const file of stateFiles) {
+        let mode;
+        try {
+            mode = JSON.parse(readFileSync(join(state, 'sessions', file), 'utf8'))?.mode;
+        }
+        catch {
+            failures['parse-failed']++;
+            continue;
+        }
+        if (mode === 'AUTO' && !recorded.has(file.slice(0, -5)))
+            autoSessionsWithoutRecord++;
+    }
+    return {
+        schema: 1,
+        since: since?.label ?? null,
+        failures, excluded,
+        coverage: { userTurns, turnsWithoutRecord, unlinkedBindings, autoSessionsWithoutRecord },
+        groups,
+        referenceBeforeAuto: Object.keys(reference).sort().map(model => ({
+            model, calls: reference[model].calls, sessions: reference[model].sessions.size,
+            effort: sorted(reference[model].effort), autoEffort: sorted(autoEffort[model] ?? {}),
+        })),
+        caveats: CAVEATS,
+    };
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+    try {
+        const { values } = parseArgs({ options: { since: { type: 'string' } } });
+        if (values.since !== undefined && !/^\d{4}-\d{2}-\d{2}$/.test(values.since)) {
+            process.stderr.write('usage-report: --since must be YYYY-MM-DD\n');
+            process.exit(2);
+        }
+        const home = process.env.DSH_HOME ?? join(homedir(), '.dsh');
+        process.stdout.write(`${JSON.stringify(buildReport(home, values.since ? { since: values.since } : {}), null, 2)}\n`);
+    }
+    catch {
+        process.stderr.write('usage-report: failed\n');
+        process.exit(1);
+    }
+}

package/package.json

@@ -1,13 +1,26 @@
 {
   "name": "dsh-auto-model-routing",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "DeepSeek Harness plugin: independent Jev capability and effort decisions, enforced fresh-child routes, decision visibility and allocation feedback",
   "license": "MIT",
   "author": "prgrmrwy",
-  "repository": { "type": "git", "url": "git+https://github.com/prgrmrwy/dsh-auto-model-routing.git" },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/prgrmrwy/dsh-auto-model-routing.git"
+  },
   "homepage": "https://github.com/prgrmrwy/dsh-auto-model-routing#readme",
-  "bugs": { "url": "https://github.com/prgrmrwy/dsh-auto-model-routing/issues" },
-  "keywords": ["dsh", "dsh-plugin", "deepseek-harness", "reasoning-effort", "model-routing", "jev", "subagent"],
+  "bugs": {
+    "url": "https://github.com/prgrmrwy/dsh-auto-model-routing/issues"
+  },
+  "keywords": [
+    "dsh",
+    "dsh-plugin",
+    "deepseek-harness",
+    "reasoning-effort",
+    "model-routing",
+    "jev",
+    "subagent"
+  ],
   "type": "module",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",

package/skills/auto-model-routing-report/SKILL.md

@@ -0,0 +1,55 @@
+---
+name: auto-model-routing-report
+description: 仅当用户明确要求评估 AUTO 效果、观察 AUTO 期间的 token 用量或判断分布，或直接点名 auto-model-routing-report 时使用：只读汇总本地 AUTO 决策记录与会话实际用量。不得主动执行或创建定时任务；会话处于 AUTO、普通 token 问题、路由判断或日常任务都不是触发条件。
+---
+
+# AUTO 用量观察报告
+
+**仅当用户明确要求时使用。** 不得主动执行，也不得创建定时任务或后台采集。会话处于 AUTO、用户询问 token 计价、讨论路由策略，都不是使用本 skill 的理由。
+
+## 它能回答什么
+
+- AUTO 期间做了哪些决策，按策略版本、模型和推理深度（depth）分组，以及这些决策实际消耗了多少 token。
+- 数据覆盖情况：哪些回合没有记录，以及有多少会话处于 AUTO 但完全没有记录。
+- 对照首个 AUTO 记录之前的回合，effort 起点发生了什么变化。
+
+**它不能回答“AUTO 省了多少”**：这是观察数据，不是受控对照。
+
+## 步骤
+
+1. 运行（只读：不写文件、不联网、不调用模型）：
+
+   ```bash
+   node "<插件目录>/lib/usage-report.js"                     # 全部数据
+   node "<插件目录>/lib/usage-report.js" --since 2026-11-01  # 从本地时区该日 00:00 起
+   ```
+
+   `<插件目录>` 是 `dsh-auto-model-routing` 包所在的目录。`$DSH_HOME` 未设置时，默认读取 `~/.dsh`。
+
+2. **先检查数据质量，再看数字：**
+   - `failures` 中任一项明显不为 0，或 `coverage.unlinkedBindings` 偏高时，说明日志格式可能已随 DSH 升级而变化。此时停止解读，先告诉用户需要核对。
+   - `failures.tail-incomplete` 为 1 通常是正常的，因为当前会话的日志还在写入。
+
+3. 转述结果时**保留以下含义，不得放大**：
+   - `conclusion: insufficient-sample` → “样本不足，尚不能判断”。门槛是 ≥20 个决策，且来自 ≥3 个会话。
+   - `caveats` 中的每一项都要用一句话说明：
+     - `observational-not-controlled`：观察数据，不是受控对照；判断因果需要专门对照实验。
+     - `scope-main-session-assistant-calls-only`：不含子会话、压缩摘要、标题生成、Jev 判档和失败重试的用量，因此不能推出“节省”。
+     - `only-applied-decisions-recorded`：只记录实际生效的决策，已统计部分可能偏向某类回合；`coverage.turnsWithoutRecord` 显示缺口大小。
+     - `token-semantics-differ-by-provider`：不同 provider 的 token 口径可能不同，不要跨模型比较。
+     - `reference-before-auto-effort-only-confounded`：`referenceBeforeAuto` 只用于看 effort 起点的变化；它受会话阶段、任务类型、模型切换等因素混杂，不能当作对照组。
+     - `mode-history-not-persisted`：会话模式的切换历史没有保存，会话计数只是粗略参考。
+     - `implicit-ok-not-appropriateness`：隐式无异议不等于分配合适，不能计作显式好评。
+     - `exposure-not-proof-of-reading`：前台可见 3 秒只是曝光，不证明用户读过。
+     - `historical-exposure-unknown`：上线前的曝光未知，不回填。
+     - `exposure-after-continuation-excluded`：下一用户回合之后才出现的曝光不算，避免翻看旧记录制造认可。
+   - `implicit.counts` 按优先级把决策分为显式评价、负向行为、隐式无异议、无定论、未曝光，五类合计为决策数；`implicit.negative` 的两个原因可重叠。`implicit.exposedExplicit` 是有效曝光且有显式评价的参照，不是准确率。
+   - 不要自行计算节省百分比，不要说“质量不变”，也不要把“首个记录之前”称为 MANUAL。
+   - token 均值的分母是**出现该字段的调用数**（`present`）；值为 `null` 表示该字段从未出现，不是 0。
+
+4. 如果要把报告原文贴到当前会话之外，例如飞书或 issue，**先请用户确认**。模型名、provider 名可能包含内部信息。
+
+## 前提
+
+- 报告每次都从原始记录和会话日志重新计算，不保存快照。用户删除旧会话后，对应历史无法再统计。
+- 旧版策略（legacy）的判档一致率请用同一插件目录下的 `scripts/label-report.mjs`。