dsh-tool-stats 0.1.0

package/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## 0.1.0 (2026-09-18)
+
+Initial release.

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 hj01857655
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,27 @@
+# dsh-tool-stats
+
+Every tool call is counted, and the ones that never fire are named so you can remove them.
+
+## Install
+
+```sh
+dsh plugin --profile web add dsh-tool-stats
+```
+
+## What it does
+
+- **Hook tool calls.** Records tool name, success/failure, latency, session.
+- **Dead-tool detection.** Tools configured but never invoked across N sessions are flagged.
+- **Failure analysis.** Tools with high failure rates are identified with their last error.
+- **Recommendations.** "Removing 12 unused tools would save ~1.8k tokens per context window."
+- **Panel.** Usage table, dead/failing sections, recommendations.
+
+## CLI
+
+```sh
+dsh-tool-stats summary    # show tool usage, dead tools, failures, recommendations
+```
+
+## License
+
+MIT

package/cordis.patch.yml

@@ -0,0 +1,4 @@
+# cordis patch for dsh-tool-stats
+insert:
+  - id: dsh-tool-stats
+    from: ./lib/index.js

package/lib/analyzer.d.ts

@@ -0,0 +1,5 @@
+import type { ToolCall, ToolCounter, DeadTool, FailingTool, Recommendation } from './types.js';
+export declare function aggregate(calls: ToolCall[]): Map<string, ToolCounter>;
+export declare function findDeadTools(configuredTools: string[], calls: ToolCall[], threshold?: number): DeadTool[];
+export declare function findFailingTools(calls: ToolCall[], threshold?: number): FailingTool[];
+export declare function recommend(configuredTools: string[], calls: ToolCall[], deadThreshold?: number, estimatedTokensPerTool?: number): Recommendation[];

package/lib/analyzer.js

@@ -0,0 +1,92 @@
+function percentile(sorted, p) {
+    if (sorted.length === 0)
+        return 0;
+    const idx = Math.ceil((p / 100) * sorted.length) - 1;
+    return sorted[Math.max(0, idx)];
+}
+export function aggregate(calls) {
+    const map = new Map();
+    for (const call of calls) {
+        const c = map.get(call.tool) ?? {
+            tool: call.tool,
+            invocations: 0,
+            successes: 0,
+            failures: 0,
+            failureRate: 0,
+            p50Latency: 0,
+            p95Latency: 0,
+            lastUsed: 0,
+        };
+        c.invocations++;
+        if (call.success)
+            c.successes++;
+        else {
+            c.failures++;
+            if (call.errorMessage)
+                c.lastError = call.errorMessage;
+        }
+        c.lastUsed = Math.max(c.lastUsed, call.timestamp);
+        map.set(call.tool, c);
+    }
+    // Compute derived fields
+    const latenciesByTool = new Map();
+    for (const call of calls) {
+        const arr = latenciesByTool.get(call.tool) ?? [];
+        arr.push(call.latencyMs);
+        latenciesByTool.set(call.tool, arr);
+    }
+    for (const c of map.values()) {
+        c.failureRate = c.invocations > 0 ? c.failures / c.invocations : 0;
+        const lats = (latenciesByTool.get(c.tool) ?? []).sort((a, b) => a - b);
+        c.p50Latency = percentile(lats, 50);
+        c.p95Latency = percentile(lats, 95);
+    }
+    return map;
+}
+export function findDeadTools(configuredTools, calls, threshold = 10) {
+    const counters = aggregate(calls);
+    const sessions = new Set(calls.map((c) => c.sessionId));
+    const sessionCount = sessions.size;
+    return configuredTools
+        .filter((tool) => {
+        const c = counters.get(tool);
+        return !c || c.invocations === 0 || (sessionCount - new Set(calls.filter((c) => c.tool === tool).map((c) => c.sessionId)).size) >= threshold;
+    })
+        .map((tool) => ({
+        tool,
+        sessionsSinceLastUse: sessionCount - (counters.get(tool)?.invocations ?? 0),
+    }));
+}
+export function findFailingTools(calls, threshold = 0.3) {
+    const counters = aggregate(calls);
+    return [...counters.values()]
+        .filter((c) => c.invocations >= 5 && c.failureRate >= threshold)
+        .map((c) => {
+        const ft = { tool: c.tool, failureRate: c.failureRate };
+        if (c.lastError !== undefined)
+            ft.lastError = c.lastError;
+        return ft;
+    })
+        .sort((a, b) => b.failureRate - a.failureRate);
+}
+export function recommend(configuredTools, calls, deadThreshold = 10, estimatedTokensPerTool = 150) {
+    const recs = [];
+    const dead = findDeadTools(configuredTools, calls, deadThreshold);
+    if (dead.length > 0) {
+        recs.push({
+            type: 'dead-tools',
+            message: `You have ${configuredTools.length} tools, but ${dead.length} haven't been used in ${deadThreshold}+ sessions. Removing them would save ~${dead.length * estimatedTokensPerTool} tokens per context window.`,
+            tools: dead.map((d) => d.tool),
+            estimatedTokenSavings: dead.length * estimatedTokensPerTool,
+        });
+    }
+    const failing = findFailingTools(calls);
+    if (failing.length > 0) {
+        recs.push({
+            type: 'failing-tools',
+            message: `${failing.length} tools have a failure rate above 30%. Check their configuration.`,
+            tools: failing.map((f) => f.tool),
+        });
+    }
+    return recs;
+}

package/lib/bin.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/lib/bin.js

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+import { run } from './cli.js';
+process.exitCode = run(process.argv.slice(2));

package/lib/cli.d.ts

@@ -0,0 +1 @@
+export declare function run(argv: string[]): number;

package/lib/cli.js

@@ -0,0 +1,58 @@
+import { parseArgs } from 'node:util';
+import { ToolStats } from './stats.js';
+export function run(argv) {
+    const { positionals } = parseArgs({
+        args: argv,
+        options: {
+            'tool': { type: 'string' },
+            'success': { type: 'boolean' },
+            'latency': { type: 'string' },
+            'session': { type: 'string' },
+        },
+        allowPositionals: true,
+    });
+    const projectDir = process.cwd();
+    const stats = new ToolStats(projectDir);
+    const cmd = positionals[0] ?? 'summary';
+    switch (cmd) {
+        case 'summary': {
+            const s = stats.summary();
+            if (s.tools.length === 0) {
+                console.log('No tool calls recorded.');
+                return 0;
+            }
+            console.log('Tool usage:');
+            for (const t of s.tools) {
+                console.log(`  ${t.tool}: ${t.invocations} calls, ${Math.round(t.failureRate * 100)}% fail, p50=${t.p50Latency}ms`);
+            }
+            if (s.deadTools.length > 0) {
+                console.log(`\nDead tools (${s.deadTools.length}):`);
+                for (const d of s.deadTools)
+                    console.log(`  ${d.tool}`);
+            }
+            if (s.failingTools.length > 0) {
+                console.log(`\nFailing tools (${s.failingTools.length}):`);
+                for (const f of s.failingTools)
+                    console.log(`  ${f.tool}: ${Math.round(f.failureRate * 100)}% fail`);
+            }
+            for (const r of s.recommendations) {
+                console.log(`\n💡 ${r.message}`);
+            }
+            return 0;
+        }
+        case 'record': {
+            const tool = positionals[1] ?? '';
+            const success = positionals.includes('--success');
+            const latency = parseInt(positionals[positionals.indexOf('--latency') + 1] ?? '0', 10);
+            const session = positionals[positionals.indexOf('--session') + 1] ?? 'cli';
+            stats.record({ tool, timestamp: Date.now(), success, latencyMs: latency, sessionId: session });
+            console.log(`Recorded: ${tool} ${success ? 'ok' : 'fail'}`);
+            return 0;
+        }
+        case 'help':
+        default:
+            console.log('Usage: dsh-tool-stats <command> [options]');
+            console.log('Commands: summary, record');
+            return 0;
+    }
+}

package/lib/client/index.d.ts

@@ -0,0 +1,4 @@
+export declare const inject: string[];
+export declare function apply(ctx: {
+    inject: (deps: string[], fn: (...services: unknown[]) => void) => void;
+}): void;

package/lib/client/index.js

@@ -0,0 +1,16 @@
+import { renderPanel } from './view.js';
+export const inject = ['@deepseek-ai/dsh-client-ui-settings', '@deepseek-ai/dsh-client-connection'];
+export function apply(ctx) {
+    ctx.inject(inject, (settings, connection) => {
+        const s = settings;
+        const c = connection;
+        s.section('tool-stats', {
+            title: 'Tool Stats',
+            render: async () => {
+                const res = await c.fetch('/api/stats.panel');
+                const payload = await res.json();
+                return renderPanel(payload);
+            },
+        });
+    });
+}

package/lib/client/view.d.ts

@@ -0,0 +1,2 @@
+import type { PanelPayload } from '../types.js';
+export declare function renderPanel(payload: PanelPayload): string;

package/lib/client/view.js

@@ -0,0 +1,19 @@
+export function renderPanel(payload) {
+    const toolRows = payload.tools
+        .map((t) => `<tr><td>${t.tool}</td><td>${t.invocations}</td><td>${Math.round(t.failureRate * 100)}%</td><td>${t.p50Latency}ms</td><td>${t.p95Latency}ms</td></tr>`)
+        .join('');
+    const deadSection = payload.deadTools.length > 0
+        ? `<h3>Dead tools</h3><ul>${payload.deadTools.map((d) => `<li>${d.tool}</li>`).join('')}</ul>`
+        : '';
+    const failSection = payload.failingTools.length > 0
+        ? `<h3>Failing tools</h3><ul>${payload.failingTools.map((f) => `<li>${f.tool}: ${Math.round(f.failureRate * 100)}%</li>`).join('')}</ul>`
+        : '';
+    const recSection = payload.recommendations.length > 0
+        ? `<h3>Recommendations</h3><ul>${payload.recommendations.map((r) => `<li>${r.message}</li>`).join('')}</ul>`
+        : '';
+    return `<div class="stats-panel">
+    <h2>Tool Stats</h2>
+    ${toolRows ? `<table><thead><tr><th>Tool</th><th>Calls</th><th>Fail</th><th>p50</th><th>p95</th></tr></thead><tbody>${toolRows}</tbody></table>` : '<p>No calls recorded.</p>'}
+    ${deadSection}${failSection}${recSection}
+  </div>`;
+}

package/lib/index.d.ts

@@ -0,0 +1,15 @@
+import type { Context } from '@deepseek-ai/cordis';
+import { ToolStats } from './stats.js';
+export declare const name = "dsh-tool-stats";
+export interface StatsService {
+    record(call: {
+        tool: string;
+        timestamp: number;
+        success: boolean;
+        latencyMs: number;
+        sessionId: string;
+        errorMessage?: string;
+    }): void;
+    summary(configuredTools?: string[]): ReturnType<ToolStats['summary']>;
+}
+export declare function apply(ctx: Context): void;

package/lib/index.js

@@ -0,0 +1,14 @@
+import { resolve } from 'node:path';
+import { ToolStats } from './stats.js';
+import { registerStatsRoutes } from './routes.js';
+export const name = 'dsh-tool-stats';
+export function apply(ctx) {
+    const root = resolve(process.cwd());
+    const stats = new ToolStats(root);
+    const service = {
+        record: (call) => stats.record(call),
+        summary: (configuredTools) => stats.summary(configuredTools ?? []),
+    };
+    ctx.provide('toolStats', service);
+    registerStatsRoutes(ctx, service);
+}

package/lib/routes.d.ts

@@ -0,0 +1,5 @@
+import type { Context } from '@deepseek-ai/cordis';
+import type { StatsService } from './index.js';
+import { STATS_PANEL_PATH } from './stats.js';
+export { STATS_PANEL_PATH };
+export declare function registerStatsRoutes(ctx: Context, stats: StatsService): void;

package/lib/routes.js

@@ -0,0 +1,15 @@
+import { STATS_PANEL_PATH } from './stats.js';
+export { STATS_PANEL_PATH };
+export function registerStatsRoutes(ctx, stats) {
+    ctx.inject(['connection'], (connectionCtx) => {
+        const connection = connectionCtx.connection;
+        connection.fetch.register({
+            path: STATS_PANEL_PATH,
+            methods: ['GET'],
+            requestBody: 'buffered',
+            fetch: () => Promise.resolve(Response.json(stats.summary(), {
+                headers: { 'cache-control': 'no-store' },
+            })),
+        });
+    });
+}

package/lib/stats.d.ts

@@ -0,0 +1,9 @@
+import type { ToolCall, PanelPayload } from './types.js';
+export declare const STATS_PANEL_PATH = "/api/stats.panel";
+export declare class ToolStats {
+    private projectDir;
+    private store;
+    constructor(projectDir: string);
+    record(call: ToolCall): void;
+    summary(configuredTools?: string[]): PanelPayload;
+}

package/lib/stats.js

@@ -0,0 +1,25 @@
+import { StatsStore } from './store.js';
+import { aggregate, findDeadTools, findFailingTools, recommend } from './analyzer.js';
+export const STATS_PANEL_PATH = '/api/stats.panel';
+export class ToolStats {
+    projectDir;
+    store;
+    constructor(projectDir) {
+        this.projectDir = projectDir;
+        this.store = new StatsStore(projectDir);
+    }
+    record(call) {
+        this.store.append(call);
+    }
+    summary(configuredTools = []) {
+        const calls = this.store.readAll();
+        const counters = aggregate(calls);
+        const tools = [...counters.values()].sort((a, b) => b.invocations - a.invocations);
+        return {
+            tools,
+            deadTools: findDeadTools(configuredTools, calls),
+            failingTools: findFailingTools(calls),
+            recommendations: recommend(configuredTools, calls),
+        };
+    }
+}

package/lib/stats.web.js

@@ -0,0 +1,33 @@
+// src/client/view.tsx
+function renderPanel(payload) {
+  const toolRows = payload.tools.map((t) => `<tr><td>${t.tool}</td><td>${t.invocations}</td><td>${Math.round(t.failureRate * 100)}%</td><td>${t.p50Latency}ms</td><td>${t.p95Latency}ms</td></tr>`).join("");
+  const deadSection = payload.deadTools.length > 0 ? `<h3>Dead tools</h3><ul>${payload.deadTools.map((d) => `<li>${d.tool}</li>`).join("")}</ul>` : "";
+  const failSection = payload.failingTools.length > 0 ? `<h3>Failing tools</h3><ul>${payload.failingTools.map((f) => `<li>${f.tool}: ${Math.round(f.failureRate * 100)}%</li>`).join("")}</ul>` : "";
+  const recSection = payload.recommendations.length > 0 ? `<h3>Recommendations</h3><ul>${payload.recommendations.map((r) => `<li>${r.message}</li>`).join("")}</ul>` : "";
+  return `<div class="stats-panel">
+    <h2>Tool Stats</h2>
+    ${toolRows ? `<table><thead><tr><th>Tool</th><th>Calls</th><th>Fail</th><th>p50</th><th>p95</th></tr></thead><tbody>${toolRows}</tbody></table>` : "<p>No calls recorded.</p>"}
+    ${deadSection}${failSection}${recSection}
+  </div>`;
+}
+
+// src/client/index.tsx
+var inject = ["@deepseek-ai/dsh-client-ui-settings", "@deepseek-ai/dsh-client-connection"];
+function apply(ctx) {
+  ctx.inject(inject, (settings, connection) => {
+    const s = settings;
+    const c = connection;
+    s.section("tool-stats", {
+      title: "Tool Stats",
+      render: async () => {
+        const res = await c.fetch("/api/stats.panel");
+        const payload = await res.json();
+        return renderPanel(payload);
+      }
+    });
+  });
+}
+export {
+  apply,
+  inject
+};

package/lib/store.d.ts

@@ -0,0 +1,10 @@
+import type { ToolCall } from './types.js';
+export declare class StatsStore {
+    private readonly projectDir;
+    private readonly statsDir;
+    private readonly callsPath;
+    constructor(projectDir: string);
+    private ensureDir;
+    append(call: ToolCall): void;
+    readAll(): ToolCall[];
+}

package/lib/store.js

@@ -0,0 +1,29 @@
+import { readFileSync, existsSync, mkdirSync, appendFileSync } from 'node:fs';
+import { join } from 'node:path';
+export class StatsStore {
+    projectDir;
+    statsDir;
+    callsPath;
+    constructor(projectDir) {
+        this.projectDir = projectDir;
+        this.statsDir = join(projectDir, '.toolstats');
+        this.callsPath = join(this.statsDir, 'calls.jsonl');
+    }
+    ensureDir() {
+        if (!existsSync(this.statsDir))
+            mkdirSync(this.statsDir, { recursive: true });
+    }
+    append(call) {
+        this.ensureDir();
+        appendFileSync(this.callsPath, JSON.stringify(call) + '\n', 'utf8');
+    }
+    readAll() {
+        if (!existsSync(this.callsPath))
+            return [];
+        return readFileSync(this.callsPath, 'utf8')
+            .trim()
+            .split('\n')
+            .filter(Boolean)
+            .map((line) => JSON.parse(line));
+    }
+}

package/lib/types.d.ts

@@ -0,0 +1,40 @@
+export interface ToolCall {
+    tool: string;
+    timestamp: number;
+    success: boolean;
+    latencyMs: number;
+    sessionId: string;
+    errorMessage?: string;
+}
+export interface ToolCounter {
+    tool: string;
+    invocations: number;
+    successes: number;
+    failures: number;
+    failureRate: number;
+    p50Latency: number;
+    p95Latency: number;
+    lastUsed: number;
+    lastError?: string;
+}
+export interface DeadTool {
+    tool: string;
+    sessionsSinceLastUse: number;
+}
+export interface FailingTool {
+    tool: string;
+    failureRate: number;
+    lastError?: string;
+}
+export interface Recommendation {
+    type: 'dead-tools' | 'failing-tools';
+    message: string;
+    tools: string[];
+    estimatedTokenSavings?: number;
+}
+export interface PanelPayload {
+    tools: ToolCounter[];
+    deadTools: DeadTool[];
+    failingTools: FailingTool[];
+    recommendations: Recommendation[];
+}

package/lib/types.js

@@ -0,0 +1,2 @@
+// Types for dsh-tool-stats
+export {};

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "dsh-tool-stats",
+  "version": "0.1.0",
+  "description": "Every tool call is counted, and the ones that never fire are named so you can remove them.",
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "bin": { "dsh-tool-stats": "lib/bin.js" },
+  "exports": {
+    ".": { "types": "./lib/index.d.ts", "default": "./lib/index.js" },
+    "./client": { "types": "./lib/client.d.ts", "default": "./lib/stats.web.js" },
+    "./package.json": "./package.json"
+  },
+  "files": ["lib", "cordis.patch.yml", "README.md", "CHANGELOG.md"],
+  "keywords": ["dsh", "dsh-plugin", "deepseek-harness", "tools", "analytics", "usage"],
+  "dsh": {
+    "bundle": { "patch": "./cordis.patch.yml" },
+    "client": {
+      "platform": "web",
+      "inject": ["@deepseek-ai/dsh-client-ui-settings", "@deepseek-ai/dsh-client-connection"]
+    }
+  },
+  "license": "MIT",
+  "author": "hj01857655",
+  "repository": { "type": "git", "url": "git+https://github.com/hj01857655/dsh-tool-stats.git" },
+  "bugs": { "url": "https://github.com/hj01857655/dsh-tool-stats/issues" },
+  "homepage": "https://github.com/hj01857655/dsh-tool-stats#readme",
+  "engines": { "node": ">=20" },
+  "peerDependencies": { "@deepseek-ai/cordis": "^4.0.1" },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.2",
+    "@types/node": "^22.20.3",
+    "@types/react": "^19.3.0",
+    "esbuild": "^0.28.2",
+    "react": "^19.3.0",
+    "react-dom": "^19.3.0",
+    "typescript": "^5.6.0"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json && node scripts/bundle-client.mjs",
+    "typecheck": "tsc -p tsconfig.json --noEmit && tsc -p tsconfig.client.json",
+    "test": "tsc -p tsconfig.json && node --test tests/*.test.mjs",
+    "prepublishOnly": "npm run build && npm test"
+  }
+}