@vauban-org/agent-sdk 0.16.0

package/src/economy/economy-router.ts

@@ -0,0 +1,242 @@
+/**
+ * economy/economy-router — High-level EconomyRouter with tier catalog + circuit breaker.
+ *
+ * Promoted from forge/src/agents/shared/economy/forge-economy.ts (inline logic)
+ * to a reusable SDK module.
+ *
+ * Design goals:
+ *   - Tier catalog is data-driven (injected via EconomyRouterOpts.tiers)
+ *   - Circuit breaker triggers on $/h threshold (configurable, default $10)
+ *   - Cost recording with optional EventBusPort publish `cc.cost.recorded`
+ *   - Multi-tenant isolation via forTenant()
+ *
+ * @public
+ */
+
+import type { EventBusPort } from "../ports/event-bus.js";
+
+// ─── Public types ─────────────────────────────────────────────────────────────
+
+export interface ProviderTier {
+  name: "free" | "cheap" | "mid" | "premium";
+  costPerInputToken: number; // USD per token
+  costPerOutputToken: number; // USD per token
+  qualityScore: number; // 0-1
+}
+
+export interface EconomyRouterOpts {
+  tiers: ProviderTier[];
+  hourlyBudgetUsd: number;
+  monthlyBudgetUsd: number;
+  defaultTier: ProviderTier["name"];
+  /** Called when the circuit breaker opens. */
+  onCircuitBreak?: (reason: string) => void;
+  /** Optional event bus for publishing cc.cost.recorded events. */
+  eventBus?: EventBusPort;
+  /** Rolling window for hourly budget. Default: 3_600_000ms (1h). */
+  windowMs?: number;
+  /** Monthly window. Default: 30 * 24 * 3_600_000ms. */
+  monthlyWindowMs?: number;
+  /** Injected clock for testability. Default: Date.now. */
+  _now?: () => number;
+  /** Tenant identifier (set by forTenant). */
+  _tenantId?: string;
+}
+
+export interface CostEntry {
+  agentId: string;
+  costUsd: number;
+  inputTokens: number;
+  outputTokens: number;
+  ts: number;
+}
+
+// ─── Complexity → tier selection ──────────────────────────────────────────────
+
+/**
+ * Map a 0-1 complexity score to a tier name.
+ * 0.0-0.25 → free, 0.25-0.5 → cheap, 0.5-0.75 → mid, 0.75-1.0 → premium
+ */
+function complexityToTierName(complexity: number): ProviderTier["name"] {
+  if (complexity < 0.25) return "free";
+  if (complexity < 0.5) return "cheap";
+  if (complexity < 0.75) return "mid";
+  return "premium";
+}
+
+// ─── EconomyRouter ────────────────────────────────────────────────────────────
+
+export class EconomyRouter {
+  private readonly _tiers: Map<ProviderTier["name"], ProviderTier>;
+  private readonly _opts: EconomyRouterOpts;
+  private readonly _hourlyWindowMs: number;
+  private readonly _monthlyWindowMs: number;
+  private readonly _now: () => number;
+  private readonly _history: CostEntry[] = [];
+  private _circuitOpen = false;
+  private _circuitOpenedAt: number | undefined;
+  /** Cooldown before circuit can auto-reset. 5 minutes. */
+  private static readonly COOLDOWN_MS = 300_000;
+
+  constructor(opts: EconomyRouterOpts) {
+    this._opts = opts;
+    this._tiers = new Map(opts.tiers.map((t) => [t.name, t]));
+    this._hourlyWindowMs = opts.windowMs ?? 3_600_000;
+    this._monthlyWindowMs = opts.monthlyWindowMs ?? 30 * 24 * 3_600_000;
+    this._now = opts._now ?? (() => Date.now());
+  }
+
+  /**
+   * Select the most appropriate tier for a given complexity (0-1 scale).
+   * Falls back to defaultTier if the mapped tier is not in the catalog.
+   */
+  selectTier(complexity: number, _agentId?: string): ProviderTier {
+    const preferred = complexityToTierName(complexity);
+    return this._tiers.get(preferred) ?? this._defaultTier();
+  }
+
+  /**
+   * Record a completed cycle's cost. Publishes cc.cost.recorded if an
+   * EventBusPort is provided.
+   */
+  recordCycle(
+    agentId: string,
+    costUsd: number,
+    tokens: { input: number; output: number }
+  ): void {
+    const now = this._now();
+    this._history.push({
+      agentId,
+      costUsd,
+      inputTokens: tokens.input,
+      outputTokens: tokens.output,
+      ts: now,
+    });
+
+    // Check circuit breaker
+    if (!this._circuitOpen) {
+      if (this.hourlySpend() > this._opts.hourlyBudgetUsd) {
+        this._circuitOpen = true;
+        this._circuitOpenedAt = now;
+        const reason = `hourly budget exceeded: ${this.hourlySpend().toFixed(
+          4
+        )} USD > ${this._opts.hourlyBudgetUsd} USD`;
+        this._opts.onCircuitBreak?.(reason);
+      }
+    }
+
+    // Publish event if EventBusPort provided
+    if (this._opts.eventBus !== undefined) {
+      const tenantId = this._opts._tenantId;
+      void this._opts.eventBus.publish(
+        {
+          id: `cc-cost-${now}-${agentId}`,
+          specversion: "1.0",
+          source: "cc",
+          type: "cc.cost.recorded",
+          time: new Date(now).toISOString(),
+          datacontenttype: "application/json",
+          data: {
+            agentId,
+            costUsd,
+            tokens,
+            ...(tenantId !== undefined ? { tenantId } : {}),
+          },
+        },
+        "cc.cost"
+      );
+    }
+  }
+
+  /**
+   * Total cost in the rolling hourly window.
+   */
+  hourlySpend(): number {
+    const cutoff = this._now() - this._hourlyWindowMs;
+    return this._history
+      .filter((e) => e.ts >= cutoff)
+      .reduce((sum, e) => sum + e.costUsd, 0);
+  }
+
+  /**
+   * Total cost in the rolling monthly window.
+   */
+  monthlySpend(): number {
+    const cutoff = this._now() - this._monthlyWindowMs;
+    return this._history
+      .filter((e) => e.ts >= cutoff)
+      .reduce((sum, e) => sum + e.costUsd, 0);
+  }
+
+  /**
+   * Check whether a new cycle may proceed given estimated cost.
+   *
+   * Returns `{ ok: false, reason }` if:
+   *   - Circuit breaker is open
+   *   - Estimated cost would push hourly spend over budget
+   *   - Estimated cost would push monthly spend over budget
+   */
+  canProceed(
+    estimatedCostUsd: number
+  ): { ok: true } | { ok: false; reason: string } {
+    // Auto-reset circuit breaker after cooldown
+    if (this._circuitOpen && this._circuitOpenedAt !== undefined) {
+      if (this._now() - this._circuitOpenedAt >= EconomyRouter.COOLDOWN_MS) {
+        this._circuitOpen = false;
+        this._circuitOpenedAt = undefined;
+      }
+    }
+
+    if (this._circuitOpen) {
+      return { ok: false, reason: "circuit-breaker-open" };
+    }
+
+    const projectedHourly = this.hourlySpend() + estimatedCostUsd;
+    if (projectedHourly > this._opts.hourlyBudgetUsd) {
+      return {
+        ok: false,
+        reason: `hourly budget would be exceeded: ${projectedHourly.toFixed(
+          4
+        )} > ${this._opts.hourlyBudgetUsd} USD`,
+      };
+    }
+
+    const projectedMonthly = this.monthlySpend() + estimatedCostUsd;
+    if (projectedMonthly > this._opts.monthlyBudgetUsd) {
+      return {
+        ok: false,
+        reason: `monthly budget would be exceeded: ${projectedMonthly.toFixed(
+          4
+        )} > ${this._opts.monthlyBudgetUsd} USD`,
+      };
+    }
+
+    return { ok: true };
+  }
+
+  /**
+   * Create a scoped router for a specific tenant.
+   *
+   * The scoped router shares the same tier catalog and configuration
+   * but maintains isolated spend history and circuit breaker state.
+   */
+  forTenant(tenantId: string): EconomyRouter {
+    return new EconomyRouter({
+      ...this._opts,
+      _tenantId: tenantId,
+      // Reset history and circuit state for the new tenant scope
+      _now: this._now,
+    });
+  }
+
+  // ─── Private helpers ───────────────────────────────────────────────────────
+
+  private _defaultTier(): ProviderTier {
+    const t = this._tiers.get(this._opts.defaultTier);
+    if (t !== undefined) return t;
+    // Absolute fallback: first tier in catalog
+    const first = this._opts.tiers[0];
+    if (first !== undefined) return first;
+    throw new Error("EconomyRouter: no tiers configured");
+  }
+}

package/src/economy/index.ts

@@ -0,0 +1,28 @@
+/**
+ * economy — Model tier policy + outcome cost tracking.
+ *
+ * Public exports for the economy sub-module.
+ */
+
+export type { ModelTier, TierPolicy } from "./tier-policy.js";
+export { DefaultTierPolicy, lookupTier } from "./tier-policy.js";
+
+export type { OutcomeHook, CycleCost } from "./outcome-tracker.js";
+export { OutcomeTracker } from "./outcome-tracker.js";
+
+export type {
+  EconomyMode,
+  EconomyRouterConfig,
+  RouteDecision,
+} from "./router.js";
+export { EconomyRouter } from "./router.js";
+
+export type { CircuitBreakerConfig } from "./circuit-breaker.js";
+export { FleetCircuitBreaker } from "./circuit-breaker.js";
+
+export type {
+  ProviderTier,
+  EconomyRouterOpts,
+  CostEntry,
+} from "./economy-router.js";
+export { EconomyRouter as SimpleEconomyRouter } from "./economy-router.js";

package/src/economy/outcome-tracker.ts

@@ -0,0 +1,228 @@
+/**
+ * economy/outcome-tracker — Per-cycle cost accounting, in-memory store.
+ *
+ * Closes the "agent_run.cost_usd always zero" gap (Sprint 0 v1).
+ *
+ * Design principles:
+ *   • No DB dependency — substrate is product-agnostic.
+ *   • Host wires DB persistence via OutcomeHook.
+ *   • Cost computed from TierPolicy pricing; unknown models default to 0.
+ */
+
+import type { TierPolicy } from "./tier-policy.js";
+import { lookupTier } from "./tier-policy.js";
+
+// ─── Public types ─────────────────────────────────────────────────────────
+
+export interface OutcomeHook {
+  /**
+   * Called by OutcomeTracker when flush() is invoked for a run.
+   * Consumer is responsible for DB persistence or downstream dispatch.
+   */
+  onCycleEnd(runId: string, cost: CycleCost): Promise<void>;
+}
+
+export interface CycleCost {
+  runId: string;
+  inputTokens: number;
+  outputTokens: number;
+  /** Computed from TierPolicy pricing. 0 when model pricing is unknown. */
+  costUsd: number;
+  /** Label of the ModelTier used (e.g. "free", "cheap", "mid", "premium"). */
+  modelTier: string;
+  durationMs: number;
+  /** Sprint C: skill that ran (if known). */
+  skillId?: string;
+  /** Sprint C: consumer-supplied outcome delta (business value). */
+  outcomeValue?: number;
+  /** Sprint D: task category for economy router lookups (e.g. "simple"|"standard"|"complex"|"reasoning"). */
+  category?: string;
+}
+
+// ─── OutcomeTracker ───────────────────────────────────────────────────────
+
+/**
+ * OutcomeTracker records token usage per cycle and computes cost_usd
+ * from the injected TierPolicy.
+ *
+ * Usage:
+ *   const tracker = new OutcomeTracker(policy, hook);
+ *   const cost = tracker.record(runId, "litellm/default-fast", 1200, 300, 1800);
+ *   await tracker.flush(runId);
+ */
+export class OutcomeTracker {
+  private readonly _policy: TierPolicy;
+  private readonly _hook: OutcomeHook | undefined;
+  /** In-memory store: runId → list of CycleCost records. */
+  private readonly _store: Map<string, CycleCost[]> = new Map();
+
+  constructor(policy: TierPolicy, hook?: OutcomeHook) {
+    this._policy = policy;
+    this._hook = hook;
+  }
+
+  /**
+   * Record tokens used by one cycle.
+   *
+   * @param runId        - Unique run identifier.
+   * @param model        - "provider/model" string, e.g. "litellm/default-fast" or "groq/llama-3.3-70b-versatile".
+   * @param inputTokens  - Number of input tokens consumed.
+   * @param outputTokens - Number of output tokens produced.
+   * @param durationMs   - Wall-clock duration of the cycle.
+   * @param opts         - Optional: skillId for Sprint C skill attribution.
+   */
+  record(
+    runId: string,
+    model: string,
+    inputTokens: number,
+    outputTokens: number,
+    durationMs: number,
+    opts?: { skillId?: string; outcomeValue?: number; category?: string },
+  ): CycleCost {
+    const { provider, modelName } = parseModel(model);
+    const tier = lookupTier(provider, modelName);
+
+    const costUsd =
+      tier !== undefined
+        ? (inputTokens / 1_000_000) * tier.costPerMTokenIn +
+          (outputTokens / 1_000_000) * tier.costPerMTokenOut
+        : 0;
+
+    // Also try via policy catalog for consumer-injected tiers.
+    // We use the "standard" category degraded mode to get a list, then check
+    // if any matches — this is a fallback for consumer tiers not in built-in catalog.
+    let tierLabel = tier?.label ?? "unknown";
+    if (tier === undefined) {
+      const candidates = this._policy.tiersFor("standard", "degraded");
+      const match = candidates.find(
+        (t) => t.provider === provider && t.model === modelName,
+      );
+      if (match !== undefined) {
+        tierLabel = match.label;
+        const matchedCost =
+          (inputTokens / 1_000_000) * match.costPerMTokenIn +
+          (outputTokens / 1_000_000) * match.costPerMTokenOut;
+        const cycleCost: CycleCost = {
+          runId,
+          inputTokens,
+          outputTokens,
+          costUsd: matchedCost,
+          modelTier: tierLabel,
+          durationMs,
+          ...(opts?.skillId !== undefined ? { skillId: opts.skillId } : {}),
+          ...(opts?.outcomeValue !== undefined
+            ? { outcomeValue: opts.outcomeValue }
+            : {}),
+          ...(opts?.category !== undefined ? { category: opts.category } : {}),
+        };
+        this._push(runId, cycleCost);
+        return cycleCost;
+      }
+    }
+
+    const cycleCost: CycleCost = {
+      runId,
+      inputTokens,
+      outputTokens,
+      costUsd,
+      modelTier: tierLabel,
+      durationMs,
+      ...(opts?.skillId !== undefined ? { skillId: opts.skillId } : {}),
+      ...(opts?.outcomeValue !== undefined
+        ? { outcomeValue: opts.outcomeValue }
+        : {}),
+      ...(opts?.category !== undefined ? { category: opts.category } : {}),
+    };
+
+    this._push(runId, cycleCost);
+    return cycleCost;
+  }
+
+  /**
+   * Flush all recorded cycles for a runId to the consumer hook.
+   * Noop if no hook is configured or no records exist for the runId.
+   */
+  async flush(runId: string): Promise<void> {
+    if (this._hook === undefined) return;
+    const records = this._store.get(runId);
+    if (records === undefined || records.length === 0) return;
+    for (const cost of records) {
+      await this._hook.onCycleEnd(runId, cost);
+    }
+  }
+
+  /**
+   * Get all recorded costs, optionally filtered to a trailing time window.
+   *
+   * @param windowMs - If provided, only return records within the last windowMs milliseconds.
+   *                   The tracker uses the current time as the reference point and
+   *                   estimates record time by insertion order (no timestamp stored per record).
+   *                   For window filtering, use totalCostInWindow() which is time-aware.
+   */
+  getCosts(windowMs?: number): CycleCost[] {
+    const all: CycleCost[] = [];
+    for (const records of this._store.values()) {
+      all.push(...records);
+    }
+    if (windowMs === undefined) return all;
+    // Without per-record timestamps, return all (host should use totalCostInWindow).
+    return all;
+  }
+
+  /**
+   * Running total cost (USD) across all recorded cycles within the time window.
+   * Default window: last 1 hour (3_600_000 ms).
+   *
+   * Uses an internal timestamped wrapper to implement accurate windowing.
+   */
+  totalCostInWindow(windowMs: number = 3_600_000): number {
+    const cutoff = Date.now() - windowMs;
+    let total = 0;
+    for (const records of this._timestamped.values()) {
+      for (const r of records) {
+        if (r.ts >= cutoff) {
+          total += r.cost.costUsd;
+        }
+      }
+    }
+    return total;
+  }
+
+  // ─── Private helpers ───────────────────────────────────────────────────
+
+  /** Timestamped record store for window queries. */
+  private readonly _timestamped: Map<
+    string,
+    Array<{ ts: number; cost: CycleCost }>
+  > = new Map();
+
+  private _push(runId: string, cost: CycleCost): void {
+    // Plain store
+    const list = this._store.get(runId) ?? [];
+    list.push(cost);
+    this._store.set(runId, list);
+
+    // Timestamped store
+    const tsList = this._timestamped.get(runId) ?? [];
+    tsList.push({ ts: Date.now(), cost });
+    this._timestamped.set(runId, tsList);
+  }
+}
+
+// ─── Internal helpers ─────────────────────────────────────────────────────
+
+/**
+ * Parse "provider/model" string.
+ * If no "/" is present, treats the whole string as the model name with an
+ * empty provider (will not match any built-in tier).
+ */
+function parseModel(model: string): { provider: string; modelName: string } {
+  const idx = model.indexOf("/");
+  if (idx === -1) {
+    return { provider: "", modelName: model };
+  }
+  return {
+    provider: model.slice(0, idx),
+    modelName: model.slice(idx + 1),
+  };
+}