@lincoln504/pi-research 0.1.12 → 1.0.0

package/src/utils/metrics.ts

@@ -0,0 +1,302 @@
+/**
+ * Metrics Collection System — two-tier design
+ *
+ * Session registry: accumulates infrastructure and coordination events for the
+ * lifetime of one Pi session. Never cleared between research runs; only an
+ * explicit clearSession() / "Reset metrics" action resets it.
+ *
+ * Run registry: one per research invocation. Created fresh at tool entry,
+ * discarded after the run. A RunSummary snapshot is kept in the session-level
+ * run history (capped at MAX_RUN_HISTORY) for display.
+ *
+ * All emit callsites use the same `metrics` export — the active registry is
+ * resolved from AsyncLocalStorage. Inside runWithRunRegistry() the run
+ * registry receives the writes; outside (startup, infrastructure init) the
+ * session registry does.
+ *
+ * Metric types:
+ *  - Counter:   monotonically increasing integer (request counts, error counts)
+ *  - Gauge:     point-in-time scalar (active workers, lock held flag)
+ *  - Histogram: distribution of numeric observations (latency in ms)
+ */
+
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { logger } from '../logger.ts';
+
+export type Labels = Record<string, string>;
+
+export interface IMetricHistogram {
+  count: number;
+  min: number;
+  max: number;
+  avg: number;
+  p50: number;
+  p90: number;
+  p95: number;
+  p99: number;
+}
+
+export interface IMetricsSnapshot {
+  counters: Record<string, number>;
+  gauges: Record<string, number>;
+  histograms: Record<string, IMetricHistogram>;
+}
+
+/** Immutable snapshot captured at the end of a single research run. */
+export interface RunSummary {
+  readonly runId: string;
+  readonly startedAt: number;
+  readonly completedAt: number;
+  readonly durationMs: number;
+  readonly status: 'success' | 'error' | 'cancelled';
+  readonly snapshot: IMetricsSnapshot;
+}
+
+// ── Core registry ──────────────────────────────────────────────────────────
+
+/**
+ * Low-level metrics store. Holds counters, gauges and histograms in memory.
+ * Not directly exported as the singleton — use the `metrics` export instead.
+ */
+export class MetricsRegistry {
+  private counters = new Map<string, number>();
+  private gauges = new Map<string, number>();
+  private histograms = new Map<string, { 
+    values: number[]; 
+    pointer: number; 
+    limit: number; 
+    dirty: boolean;
+    cached: IMetricHistogram | null;
+  }>();
+
+  private serializeLabels(labels?: Labels): string {
+    if (!labels) return '';
+    const keys = Object.keys(labels).sort();
+    return keys.map(k => `${k}="${labels[k]}"`).join(',');
+  }
+
+  private getKey(name: string, labels?: Labels): string {
+    const labelStr = this.serializeLabels(labels);
+    return labelStr ? `${name}{${labelStr}}` : name;
+  }
+
+  public increment(name: string, value: number = 1, labels?: Labels): void {
+    const key = this.getKey(name, labels);
+    this.counters.set(key, (this.counters.get(key) ?? 0) + value);
+    if (name === 'llm_cost_total' || name === 'llm_tokens_total') {
+      logger.debug(`[Metrics] Incremented ${name} by ${value}${labels ? ` (labels: ${JSON.stringify(labels)})` : ''}`);
+    }
+  }
+
+  public setGauge(name: string, value: number, labels?: Labels): void {
+    this.gauges.set(this.getKey(name, labels), value);
+  }
+
+  public observe(name: string, value: number, labels?: Labels): void {
+    // FIX (#15): Guard against NaN/Infinity which would corrupt histogram calculations
+    if (!Number.isFinite(value)) return;
+    const key = this.getKey(name, labels);
+    let entry = this.histograms.get(key);
+    if (!entry) {
+      entry = { values: [], pointer: 0, limit: 10_000, dirty: true, cached: null };
+      this.histograms.set(key, entry);
+    }
+    entry.dirty = true;
+    if (entry.values.length < entry.limit) {
+      entry.values.push(value);
+    } else {
+      entry.values[entry.pointer] = value;
+      entry.pointer = (entry.pointer + 1) % entry.limit;
+    }
+  }
+
+  public async measure<T>(name: string, action: () => Promise<T>, labels?: Labels): Promise<T> {
+    const start = process.hrtime.bigint();
+    try {
+      const result = await action();
+      this.observe(name, Number(process.hrtime.bigint() - start) / 1_000_000, labels);
+      return result;
+    } catch (error) {
+      this.observe(name, Number(process.hrtime.bigint() - start) / 1_000_000, { ...labels, error: 'true' });
+      this.increment(`${name}_errors_total`, 1, labels);
+      throw error;
+    }
+  }
+
+  private percentile(sorted: number[], pct: number): number {
+    const len = sorted.length;
+    if (len === 0) return 0;
+    if (len === 1) return sorted[0] ?? 0;
+    if (pct <= 0) return sorted[0] ?? 0;
+    if (pct >= 100) return sorted[len - 1] ?? 0;
+    const rank = (pct / 100) * (len - 1);
+    const lo = Math.floor(rank);
+    const hi = Math.ceil(rank);
+    const w = rank - lo;
+    return (sorted[lo] ?? 0) + ((sorted[hi] ?? 0) - (sorted[lo] ?? 0)) * w;
+  }
+
+  public getSnapshot(): IMetricsSnapshot {
+    const snap: IMetricsSnapshot = { counters: {}, gauges: {}, histograms: {} };
+
+    for (const [k, v] of this.counters) snap.counters[k] = v;
+    for (const [k, v] of this.gauges)   snap.gauges[k]   = v;
+
+    for (const [k, entry] of this.histograms) {
+      if (entry.values.length === 0) continue;
+      
+      if (!entry.dirty && entry.cached) {
+        snap.histograms[k] = entry.cached;
+        continue;
+      }
+
+      const vals = entry.values;
+      const sorted = vals.slice().sort((a, b) => a - b);
+      const sum = sorted.reduce((a, b) => a + b, 0);
+      
+      const histogram = {
+        count: sorted.length,
+        min:   sorted[0] ?? 0,
+        max:   sorted[sorted.length - 1] ?? 0,
+        avg:   sum / sorted.length,
+        p50:   this.percentile(sorted, 50),
+        p90:   this.percentile(sorted, 90),
+        p95:   this.percentile(sorted, 95),
+        p99:   this.percentile(sorted, 99),
+      };
+
+      entry.cached = histogram;
+      entry.dirty = false;
+      snap.histograms[k] = histogram;
+    }
+    return snap;
+  }
+
+  public clear(): void {
+    this.counters.clear();
+    this.gauges.clear();
+    this.histograms.clear();
+  }
+}
+
+// ── Run-registry routing ───────────────────────────────────────────────────
+
+/** AsyncLocalStorage slot that holds the active run-scoped MetricsRegistry. */
+const runRegistryStorage = new AsyncLocalStorage<MetricsRegistry>();
+
+/**
+ * Execute `fn` with `registry` as the active run-scoped metrics target.
+ * Every `metrics.*` call inside `fn` — including all async continuations —
+ * will emit into `registry` rather than the session registry.
+ * The run context does NOT nest; a second runWithRunRegistry call inside an
+ * existing one replaces the active registry for that sub-scope.
+ */
+export function runWithRunRegistry<T>(
+  registry: MetricsRegistry,
+  fn: () => Promise<T>,
+): Promise<T> {
+  return runRegistryStorage.run(registry, fn);
+}
+
+/**
+ * Returns the run-scoped registry if called inside a runWithRunRegistry context,
+ * undefined otherwise.
+ */
+export function getCurrentRunRegistry(): MetricsRegistry | undefined {
+  return runRegistryStorage.getStore();
+}
+
+// ── Session-level manager ──────────────────────────────────────────────────
+
+const MAX_RUN_HISTORY = 10;
+
+/**
+ * Two-tier session-scoped metrics manager (the exported `metrics` singleton).
+ *
+ * Emit methods route to the run registry when inside a runWithRunRegistry
+ * context, falling back to the session registry otherwise:
+ *
+ *   • Infrastructure events at startup / health-check time  → session registry
+ *   • Everything emitted during a research invocation        → run registry
+ *
+ * Run registries are created by the research tool, passed to runWithRunRegistry,
+ * and discarded after the run. A RunSummary snapshot is appended to _runHistory
+ * by calling recordRunSummary() after the runWithRunRegistry call returns
+ * (i.e. outside the run context, so the write goes to session scope).
+ */
+class SessionMetrics {
+  private readonly _session = new MetricsRegistry();
+  private _sessionStartedAt = Date.now();
+  private _runHistory: RunSummary[] = [];
+
+  private getActive(): MetricsRegistry {
+    return runRegistryStorage.getStore() ?? this._session;
+  }
+
+  // ── Emit API (routes to active registry) ──────────────────────────────
+
+  public increment(name: string, value: number = 1, labels?: Labels): void {
+    this.getActive().increment(name, value, labels);
+  }
+
+  public setGauge(name: string, value: number, labels?: Labels): void {
+    this.getActive().setGauge(name, value, labels);
+  }
+
+  public observe(name: string, value: number, labels?: Labels): void {
+    this.getActive().observe(name, value, labels);
+  }
+
+  public async measure<T>(name: string, action: () => Promise<T>, labels?: Labels): Promise<T> {
+    return this.getActive().measure(name, action, labels);
+  }
+
+  // ── Session-level read API ─────────────────────────────────────────────
+
+  /** Snapshot of the session registry (infrastructure / cross-run metrics). */
+  public getSessionSnapshot(): IMetricsSnapshot {
+    return this._session.getSnapshot();
+  }
+
+  /** Millisecond timestamp when this session started or was last reset. */
+  public getSessionStartedAt(): number {
+    return this._sessionStartedAt;
+  }
+
+  /**
+   * Ordered run summaries, oldest first. Capped at MAX_RUN_HISTORY entries.
+   * Returns a shallow copy so callers cannot mutate the internal list.
+   */
+  public getRunHistory(): readonly RunSummary[] {
+    return [...this._runHistory];
+  }
+
+  // ── Run lifecycle API ──────────────────────────────────────────────────
+
+  /**
+   * Append a completed run's snapshot to the history.
+   * Call this AFTER runWithRunRegistry returns so the write lands in session
+   * scope. Automatically evicts the oldest entry when the cap is reached.
+   */
+  public recordRunSummary(summary: RunSummary): void {
+    this._runHistory.push(summary);
+    if (this._runHistory.length > MAX_RUN_HISTORY) {
+      this._runHistory.shift();
+    }
+  }
+
+  // ── Reset API ──────────────────────────────────────────────────────────
+
+  /**
+   * Clear the session registry, run history, and session start timestamp.
+   * In-flight run registries are NOT affected.
+   */
+  public clearSession(): void {
+    this._session.clear();
+    this._runHistory = [];
+    this._sessionStartedAt = Date.now();
+  }
+}
+
+/** Global session-scoped metrics instance. */
+export const metrics = new SessionMetrics();

package/src/utils/research-export.ts

@@ -16,11 +16,9 @@
 import { promises as fs } from 'node:fs';
 import { homedir, tmpdir, platform } from 'node:os';
 import { join, resolve, normalize, sep } from 'node:path';
+import { randomBytes } from 'node:crypto';
 import { logger } from '../logger.ts';
-import { MAX_QUERY_LENGTH, MAX_EXPORT_RETRIES } from '../constants.ts';
-
-const LETTERS = 'abcdefghijklmnopqrstuvwxyz';
-const DIGITS = '0123456789';
+import { MAX_FILENAME_QUERY_LENGTH, MAX_EXPORT_RETRIES } from '../constants.ts';
 
 // Subdirectory names to probe, in priority order. Matched case-insensitively.
 const PREFERRED_SUBDIRS = [
@@ -135,7 +133,7 @@ function sanitizeQuery(query: string): string {
   return query
     .toLowerCase()
     .trim()
-    .slice(0, MAX_QUERY_LENGTH)
+    .slice(0, MAX_FILENAME_QUERY_LENGTH)
     .replace(/[^a-z0-9\s-]/g, '')
     .replace(/\s+/g, '-')
     .replace(/-+/g, '-')
@@ -147,9 +145,7 @@ function sanitizeQuery(query: string): string {
  * Format: letter + number (e.g., a3, b7, x9)
  */
 function generateHash(): string {
-  const letter = LETTERS[Math.floor(Math.random() * LETTERS.length)]!;
-  const digit = DIGITS[Math.floor(Math.random() * DIGITS.length)]!;
-  return letter + digit;
+  return randomBytes(3).toString('hex'); // 6 hex chars ≈ 16M combinations
 }
 
 /**
@@ -166,10 +162,25 @@ export async function exportResearchReport(
   result: string,
   _mode: 'quick' | 'deep',
   cwd?: string,
+  explicitDir?: string,
 ): Promise<string | null> {
   const sanitizedQuery = sanitizeQuery(query);
   const baseFilename = `pi-research-${sanitizedQuery}`;
-  const targetDir = await resolveExportDir(cwd ?? tmpdir());
+  // An explicit directory (PI_RESEARCH_REPORT_EXPORT_DIR) is used verbatim —
+  // created if needed, no cwd-relative "smart" probing — so a
+  // host can pin a stable location instead of writing into the agent's arbitrary cwd.
+  let targetDir: string;
+  if (explicitDir && explicitDir.trim()) {
+    targetDir = resolve(explicitDir.trim());
+    try {
+      await fs.mkdir(targetDir, { recursive: true });
+    } catch (error) {
+      logger.error(`[export] Could not create export dir "${targetDir}":`, error);
+      return null;
+    }
+  } else {
+    targetDir = await resolveExportDir(cwd ?? tmpdir());
+  }
 
   for (let attempt = 0; attempt < MAX_EXPORT_RETRIES; attempt++) {
     const hash = generateHash();
@@ -192,8 +203,13 @@ export async function exportResearchReport(
 }
 
 /**
- * Append the saved-path footer to a research result string.
+ * Append the saved-path footer and total cost to a research result string.
  */
-export function appendExportMessage(result: string, filepath: string): string {
-  return result + `\n\n---\n\nResearch report saved to: ${filepath}`;
+export function appendExportMessage(result: string, filepath: string, totalCost?: number): string {
+  let footer = `\n\n---\n\nResearch report saved to: ${filepath}`;
+  if (totalCost !== undefined && totalCost > 0) {
+    const costStr = totalCost < 0.01 ? '<$0.01' : `$${totalCost.toFixed(4)}`.replace(/0+$/, '').replace(/\.$/, '');
+    footer += `\nTotal cost: ${costStr}`;
+  }
+  return result + footer;
 }

package/src/utils/safe-unref.ts

@@ -0,0 +1,31 @@
+/**
+ * Safe unref utility for Node.js timers.
+ *
+ * Node's setTimeout returns NodeJS.Timeout which has unref(), but some
+ * TypeScript configurations don't expose it. This helper avoids the
+ * repeated `(timer as any).unref()` pattern.
+ */
+export function safeUnref(timer: NodeJS.Timeout | null | undefined): void {
+  if (timer && typeof (timer as any).unref === 'function') {
+    (timer as any).unref();
+  }
+}
+
+/**
+ * Await `promise`, but give up after `deadlineMs` (resolving `undefined`).
+ *
+ * Unlike a bare `Promise.race([promise, setTimeout])`, the deadline timer is
+ * unref'd AND cleared as soon as `promise` settles, so a losing timeout never
+ * holds the event loop open after teardown work completes. Rejections of
+ * `promise` propagate unchanged (matching Promise.race semantics).
+ */
+export function raceWithDeadline<T>(promise: Promise<T>, deadlineMs: number): Promise<T | undefined> {
+  return new Promise<T | undefined>((resolve, reject) => {
+    const timer = setTimeout(() => resolve(undefined), deadlineMs);
+    safeUnref(timer);
+    promise.then(
+      (value) => { clearTimeout(timer); resolve(value); },
+      (err) => { clearTimeout(timer); reject(err); },
+    );
+  });
+}