@saifymatteo/pi-tool-usage-log 0.0.0-stage → 0.1.1

package/README.md

@@ -1,3 +1,32 @@
-# Temporary Holding Version
+# pi-tool-usage-log
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Passively record direct and nested tool executions and show aggregate usage, errors, duration, selected model and thinking level.
+
+## Running locally
+
+From the monorepo root:
+
+```sh
+pi -e ./packages/pi-tool-usage-log
+```
+
+**Warning:** this extension uses your existing Pi resources and settings unless you load it in an isolated home (see the root README). Do not load two copies of the same extension (for example an older standalone file alongside this package): commands and hooks can duplicate. The package never installs or relocates personal settings.
+
+## Commands and shortcuts
+
+- `/tool-usage` — opens the all-project report (no arguments).
+- Up/Down, PageUp/PageDown, Home/End scroll; Escape/Ctrl+C/Enter close.
+
+## Settings and storage
+
+Appends JSONL records to `<agent-dir>/tool-usage.log`; `<agent-dir>` is `PI_CODING_AGENT_DIR` or `~/.pi/agent`. No configuration file. The copied `skill-usage-log.ts` is a dialog sidecar only: only `src/index.ts` is a Pi manifest entry, so loading this package does not independently register skill logging.
+
+## Privacy and operations
+
+Log stores timestamps, session IDs, working directories, full tool/call/parent-call names, selected model/thinking level, status and durations, never tool arguments/results. No network requests. There is no rotation/deletion UI. Interrupted pending calls are flushed on orderly shutdown/session change; a hard crash can lose them. MCP tools aggregate by server in reports but keep full names on disk. Success means the tool returned, not that a background job eventually succeeded.
+
+## Compatibility
+
+Node.js >=24 for development checks; run extensions through Pi's TypeScript loader. Host imports are wildcard peers, not bundled runtime copies. Reports require interactive terminal mode; execution tracking needs Pi tool_execution_start/end hooks, including nested-event metadata for nested counts. Older hosts may omit these. Duration includes hooks/nested work and is not a pure tool benchmark.
+
+See [NOTICE.md](NOTICE.md) for provenance. Tests use the monorepo's TypeScript dev dependency and VM loader; they do not contact services or write to user-home resources.

package/package.json

@@ -1,6 +1,36 @@
 {
   "name": "@saifymatteo/pi-tool-usage-log",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "license": "AGPL-3.0-only",
+  "type": "module",
+  "keywords": [
+    "pi-package"
+  ],
+  "files": [
+    "src",
+    "README.md",
+    "NOTICE.md",
+    "LICENSE"
+  ],
+  "pi": {
+    "extensions": [
+      "./src/index.ts"
+    ]
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*"
+  },
+  "engines": {
+    "node": ">=24"
+  },
+  "description": "Passive log of tool executions (names and status, never contents).",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/saifymatteo/pi-extensions.git",
+    "directory": "packages/pi-tool-usage-log"
+  }
+}

package/src/index.ts

@@ -0,0 +1,233 @@
+/**
+ * Passive, forward-only tool usage log. Includes direct and nested executions,
+ * but never arguments/results. Success means the tool returned, not that a
+ * background command or delegated agent eventually succeeded.
+ */
+import type { ExtensionAPI, ExtensionContext } from '@earendil-works/pi-coding-agent'
+import { appendFileSync, mkdirSync } from 'node:fs'
+import { readFile } from 'node:fs/promises'
+import { homedir } from 'node:os'
+import { dirname, join } from 'node:path'
+import { SkillUsageDialog } from './skill-usage-log.ts'
+import { DIALOG_OVERLAY } from './lib/dialog.ts'
+import { agentMetadata, localTimestamp, logField, markdownCell, usageTimeFormat } from './lib/usage-log.ts'
+
+export interface ToolUsageRecord {
+  version: 1
+  timestamp: string
+  session: string
+  cwd: string
+  tool: string
+  callId: string
+  parentCallId?: string
+  model: string
+  thinking: string
+  status: 'success' | 'error' | 'interrupted'
+  durationMs: number
+}
+
+export interface ToolUsageRow {
+  tool: string
+  calls: number
+  sessions: number
+  errors: number
+  nested: number
+  durationMs: number
+  /** null = installed but never called. */
+  lastUsed: string | null
+}
+
+export interface ToolUsageBreakdown extends ToolUsageRow {
+  model: string
+  thinking: string
+}
+
+/** MCP tools (`mcp__<server>__<tool>`) report as one row per server, `mcp__<server>__*`; the log keeps full names. */
+const toolGroup = (tool: string) => tool.match(/^(mcp__.+?__)./)?.[1].concat('*') ?? tool
+
+/** Malformed/partial rows are ignored, not allowed to spoil the whole report.
+ *  When supplied, `installed` filters removed tools and adds unseen tools as zero rows. */
+export function summarizeToolUsage(log: string, installed?: string[]): {
+  tools: ToolUsageRow[]; breakdown: ToolUsageBreakdown[]; skipped: number
+} {
+  const tools = new Map<string, { row: ToolUsageRow; sessions: Set<string> }>()
+  const groups = new Map<string, { row: ToolUsageBreakdown; sessions: Set<string> }>()
+  let skipped = 0
+  for (const line of log.split(/\r?\n/)) {
+    if (!line.trim()) continue
+    let record: ToolUsageRecord
+    try {
+      const value = JSON.parse(line)
+      if (!value || value.version !== 1 ||
+          !['timestamp', 'session', 'cwd', 'tool', 'callId', 'model', 'thinking'].every(key => typeof value[key] === 'string') ||
+          !value.session || !value.tool || !value.callId ||
+          !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{2}:\d{2}$/.test(value.timestamp) ||
+          !Number.isFinite(Date.parse(value.timestamp)) ||
+          (value.parentCallId !== undefined && typeof value.parentCallId !== 'string') ||
+          !['success', 'error', 'interrupted'].includes(value.status) ||
+          typeof value.durationMs !== 'number' || !Number.isFinite(value.durationMs) || value.durationMs < 0) {
+        skipped++
+        continue
+      }
+      record = value
+    } catch {
+      skipped++
+      continue
+    }
+    const lastUsed = new Date(record.timestamp).toISOString()
+    const name = toolGroup(record.tool)
+    let entry = tools.get(name)
+    if (!entry) {
+      entry = { row: { tool: name, calls: 0, sessions: 0, errors: 0, nested: 0, durationMs: 0, lastUsed }, sessions: new Set() }
+      tools.set(name, entry)
+    }
+    const key = JSON.stringify([name, record.model, record.thinking])
+    let group = groups.get(key)
+    if (!group) {
+      group = { row: { ...entry.row, calls: 0, sessions: 0, errors: 0, nested: 0, durationMs: 0, lastUsed, model: record.model, thinking: record.thinking }, sessions: new Set() }
+      groups.set(key, group)
+    }
+    for (const aggregate of [entry, group]) {
+      aggregate.row.calls++
+      aggregate.sessions.add(record.session)
+      aggregate.row.sessions = aggregate.sessions.size
+      if (record.status !== 'success') aggregate.row.errors++
+      if (record.parentCallId) aggregate.row.nested++
+      aggregate.row.durationMs += record.durationMs
+      if (lastUsed > aggregate.row.lastUsed!) aggregate.row.lastUsed = lastUsed
+    }
+  }
+  if (installed !== undefined) {
+    const available = new Set(installed.map(toolGroup))
+    for (const tool of tools.keys()) if (!available.has(tool)) tools.delete(tool)
+    for (const [key, group] of groups) if (!available.has(group.row.tool)) groups.delete(key)
+    for (const tool of available) {
+      if (!tools.has(tool)) tools.set(tool, { row: { tool, calls: 0, sessions: 0, errors: 0, nested: 0, durationMs: 0, lastUsed: null }, sessions: new Set() })
+    }
+  }
+  const sort = (a: ToolUsageRow, b: ToolUsageRow) => b.calls - a.calls || a.tool.localeCompare(b.tool)
+  return {
+    tools: [...tools.values()].map(({ row }) => row).sort(sort),
+    breakdown: [...groups.values()].map(({ row }) => row).sort((a, b) => sort(a, b) || a.model.localeCompare(b.model) || a.thinking.localeCompare(b.thinking)),
+    skipped,
+  }
+}
+
+export default function register(pi: ExtensionAPI): void {
+  const agentDir = (process.env.PI_CODING_AGENT_DIR || join(homedir(), '.pi', 'agent')).replace(/^~(?=$|[\\/])/, homedir())
+  const logPath = join(agentDir, 'tool-usage.log')
+  const pending = new Map<string, { record: Omit<ToolUsageRecord, 'status' | 'durationMs'>; start: number }>()
+  let session: string | undefined
+  let logError: string | undefined
+  let warned = false
+
+  function forSession(ctx: ExtensionContext): void {
+    const sid = ctx.sessionManager.getSessionId()
+    if (session === sid) return
+    flushPending()
+    warn(ctx)
+    session = sid
+    logError = undefined
+    warned = false
+  }
+
+  function append(record: ToolUsageRecord): void {
+    try {
+      mkdirSync(dirname(logPath), { recursive: true })
+      appendFileSync(logPath, JSON.stringify(record) + '\n')
+    } catch (error) {
+      logError = error instanceof Error ? error.message : String(error)
+    }
+  }
+
+  function flushPending(): void {
+    for (const { record, start } of pending.values()) {
+      append({ ...record, status: 'interrupted', durationMs: Math.max(0, Math.round(performance.now() - start)) })
+    }
+    pending.clear()
+  }
+
+  function warn(ctx: ExtensionContext): void {
+    if (logError && !warned && ctx.hasUI) {
+      warned = true
+      try { ctx.ui.notify(`Tool usage could not be logged to ${logPath}: ${logError}`, 'warning') }
+      catch { /* A closed UI must not turn a logging failure into a tool failure. */ }
+    }
+  }
+
+  pi.on('session_start', (_event, ctx) => { forSession(ctx) })
+  pi.on('tool_execution_start', (event, ctx) => {
+    // Observation must never block a real tool, even if metadata is unavailable.
+    try {
+      forSession(ctx)
+      pending.set(event.toolCallId, {
+        start: performance.now(),
+        record: {
+          version: 1, timestamp: localTimestamp(), session: logField(session!),
+          cwd: logField(ctx.cwd), tool: logField(event.toolName), callId: logField(event.toolCallId),
+          ...(event.parentToolCallId ? { parentCallId: logField(event.parentToolCallId) } : {}),
+          ...agentMetadata(pi, ctx),
+        },
+      })
+    } catch (error) {
+      logError = error instanceof Error ? error.message : String(error)
+    }
+  })
+  pi.on('tool_execution_end', (event, ctx) => {
+    const call = pending.get(event.toolCallId)
+    if (!call) return // Started before reload, or already recorded.
+    pending.delete(event.toolCallId)
+    append({ ...call.record, status: event.isError ? 'error' : 'success', durationMs: Math.max(0, Math.round(performance.now() - call.start)) })
+    warn(ctx)
+  })
+  pi.on('turn_end', (_event, ctx) => { warn(ctx) })
+  pi.on('session_shutdown', (_event, ctx) => {
+    flushPending()
+    warn(ctx)
+  })
+
+  pi.registerCommand('tool-usage', {
+    description: 'List historical tool calls, sessions, errors, model and thinking level',
+    handler: async (args, ctx) => {
+      if (ctx.mode !== 'tui') {
+        ctx.ui.notify('Tool usage requires interactive terminal mode; no report was added to the transcript.', 'info')
+        return
+      }
+      if (args.trim()) {
+        ctx.ui.notify('Usage: /tool-usage (all logged sessions and projects)', 'info')
+        return
+      }
+      try {
+        let log = ''
+        try { log = await readFile(logPath, 'utf8') }
+        catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error }
+        const { tools, breakdown, skipped } = summarizeToolUsage(log, pi.getAllTools().map((tool) => tool.name))
+        const time = usageTimeFormat()
+        const lines = ['Tool usage — all logged sessions and projects', '',
+          'Dates follow your locale. Different historical daylight-saving offsets are shown beside the date.',
+          'Tracking starts when installed; no historical import. Every registered tool is listed; 0 / never = not called since then.',
+          'MCP tools are grouped per server as mcp__<server>__*.',
+          'Direct and nested calls are counted separately. Errors include interrupted calls; unfinished calls appear on orderly shutdown.',
+          'Success means the tool returned, not that a background job succeeded. Duration includes hooks and nested work.', '',
+          `| Tool | Calls | Sessions | Errors | Nested | Avg ms | Last used (${markdownCell(time.label)}) |`,
+          '| --- | ---: | ---: | ---: | ---: | ---: | --- |']
+        for (const row of tools) {
+          lines.push(`| ${markdownCell(row.tool)} | ${row.calls} | ${row.sessions} | ${row.errors} | ${row.nested} | ${row.calls ? Math.round(row.durationMs / row.calls) : '—'} | ${row.lastUsed ? markdownCell(time.format(new Date(row.lastUsed))) : 'never'} |`)
+        }
+        if (!tools.some((row) => row.calls)) lines.push('', 'No tool usage recorded yet.')
+        if (breakdown.length) {
+          lines.push('', 'Agent model / thinking at call start (not a tool’s internal service model).', '',
+            '| Tool | Model | Thinking | Calls | Sessions | Errors |',
+            '| --- | --- | --- | ---: | ---: | ---: |')
+          for (const row of breakdown) lines.push(`| ${markdownCell(row.tool)} | ${markdownCell(row.model)} | ${markdownCell(row.thinking)} | ${row.calls} | ${row.sessions} | ${row.errors} |`)
+        }
+        if (skipped) lines.push('', `Skipped ${skipped} malformed log row(s).`)
+        lines.push('', `Log: ${markdownCell(logPath.replaceAll('\\', '/'))}`)
+        await ctx.ui.custom<void>((tui, theme, _keybindings, done) =>
+          new SkillUsageDialog(tui, theme, lines.join('\n'), done, 'Tool usage'), DIALOG_OVERLAY)
+      } catch (error) {
+        ctx.ui.notify(`Could not show tool usage: ${error instanceof Error ? error.message : String(error)}`, 'error')
+      }
+    },
+  })
+}

package/src/lib/dialog.ts

@@ -0,0 +1,31 @@
+// One size for every overlay dialog in this package. Change DIALOG_HEIGHT here and every dialog follows:
+// the overlay is capped at that share of the terminal and each dialog pads its list to fill it.
+
+/** Share of the terminal height every dialog fills. */
+export const DIALOG_HEIGHT = 0.8
+
+/** `ctx.ui.custom(factory, DIALOG_OVERLAY)`: centered, 92% wide, DIALOG_HEIGHT tall. */
+export const DIALOG_OVERLAY = {
+  overlay: true,
+  overlayOptions: { anchor: 'center', width: '92%', maxHeight: `${DIALOG_HEIGHT * 100}%`, margin: 1 },
+} as const
+
+/**
+ * Scrolling rows that fit once `chrome` fixed lines (borders, search, header, footer) are drawn.
+ * Same floor as the overlay's own percentage, so dialog and overlay agree; read it in render() so a
+ * resize takes effect on the next frame.
+ */
+export const dialogRows = (tui: { terminal?: { rows: number } }, chrome: number, min = 1): number =>
+  Math.max(min, Math.floor((tui.terminal?.rows ?? 24) * DIALOG_HEIGHT) - chrome)
+
+/**
+ * Mouse wheel → ↑/↓ key presses, so a dialog reuses its own keyboard handling (clamping, render).
+ * Use as `handleMouse(e) { return wheelAsArrows(e, (k) => this.handleInput(k)) }`. Fullscreen mode only:
+ * regular mode leaves the mouse to the terminal. Handling the wheel also stops it scrolling the transcript behind.
+ */
+export function wheelAsArrows(event: { type: string; wheelDelta?: number }, handleInput: (data: string) => void): { handled: true } | undefined {
+  if (event.type !== 'wheel' || !event.wheelDelta) return undefined
+  const key = event.wheelDelta < 0 ? '\x1b[A' : '\x1b[B'
+  for (let i = 0; i < Math.min(Math.abs(event.wheelDelta), 20); i++) handleInput(key)
+  return { handled: true }
+}

package/src/lib/usage-log.ts

@@ -0,0 +1,62 @@
+/** Shared metadata and local-time formatting for the two passive usage logs. */
+export function userTimeZone(): string {
+  return Intl.DateTimeFormat().resolvedOptions().timeZone || 'UTC'
+}
+
+/** An ISO timestamp with an explicit offset; DST and fractional-hour zones work. */
+export function localTimestamp(date = new Date(), timeZone = userTimeZone()): string {
+  const parts = Object.fromEntries(new Intl.DateTimeFormat('en-GB', {
+    timeZone, year: 'numeric', month: '2-digit', day: '2-digit',
+    hour: '2-digit', minute: '2-digit', second: '2-digit', hourCycle: 'h23',
+    timeZoneName: 'longOffset',
+  }).formatToParts(date).map(({ type, value }) => [type, value]))
+  const offset = parts.timeZoneName.replace('GMT', '') || '+00:00'
+  return `${parts.year}-${parts.month}-${parts.day}T${parts.hour}:${parts.minute}:${parts.second}.${String(date.getUTCMilliseconds()).padStart(3, '0')}${offset}`
+}
+
+/** Human-readable report dates; keep machine-readable log timestamps separate. */
+export function usageTimeFormat(timeZone = userTimeZone(), locale?: string | string[], now = new Date()) {
+  const formatter = new Intl.DateTimeFormat(locale, { timeZone, dateStyle: 'medium', timeStyle: 'short' })
+  const offset = localTimestamp(now, timeZone).slice(-6)
+  const location = timeZone.split('/').pop()!.replaceAll('_', ' ')
+  return {
+    label: `${location} · UTC${offset}`,
+    format(date: Date): string {
+      const text = formatter.format(date)
+      const dateOffset = localTimestamp(date, timeZone).slice(-6)
+      // A historical DST offset can differ from the current offset in the header.
+      return dateOffset === offset ? text : `${text} (UTC${dateOffset})`
+    },
+  }
+}
+
+export interface AgentMetadata {
+  model: string
+  thinking: string
+}
+
+/** Read the selected agent settings, not a classifier/TTS model used by a tool. */
+export function agentMetadata(
+  pi: { getThinkingLevel(): string },
+  ctx: { model?: { provider: string; id: string } },
+): AgentMetadata {
+  let model = 'unknown'
+  let thinking = 'unknown'
+  // Logging must not block tool_call if settings are unavailable during a switch.
+  try {
+    const selected = ctx.model
+    if (selected) model = `${selected.provider}/${selected.id}`
+  } catch { /* Keep unknown metadata; the tool must still execute. */ }
+  try { thinking = pi.getThinkingLevel() || 'unknown' }
+  catch { /* Older runtimes may not expose the thinking getter. */ }
+  return { model, thinking }
+}
+
+/** Strip terminal controls as well as TSV separators from untrusted metadata. */
+export function logField(value: string): string {
+  return value.replace(/[\x00-\x1f\x7f-\x9f]/g, ' ')
+}
+
+export function markdownCell(value: string): string {
+  return logField(value).replace(/[\\`*_[\]<>|]/g, '\\$&')
+}