@agentguard-run/spend 0.7.0 → 0.9.0

package/src/outcomes/runtime.ts

@@ -0,0 +1,396 @@
+import { randomUUID } from 'crypto';
+import { buildScopeKey } from '../policy';
+import { computeCallCents, inferProvider } from '../cost-table';
+import { recommend, type ModelRouteRecommendation } from '../router';
+import type { SpendDecision, SignedDecisionLogEntry } from '../types';
+import type { SpendGuard } from '../spend-guard';
+import { OutcomeSpendLedger, type OutcomeSpendHold } from './ledger';
+import { scoreOutcomeOutput, shouldRunReviewer, type OutcomeQualityGateResult } from './quality-gate';
+import {
+  InMemoryOutcomeReceiptStore,
+  type OutcomeDecisionLink,
+  type OutcomeExecutionContext,
+  type OutcomeModelRequest,
+  type OutcomeModelResponse,
+  type OutcomeModelRunner,
+  type OutcomeReceiptStore,
+  type OutcomeRuntimeReceipt,
+  type OutcomeTemplate,
+} from './types';
+
+export interface ExecuteOutcomeTaskOptions {
+  guard: SpendGuard;
+  template: OutcomeTemplate;
+  inputs: Record<string, unknown>;
+  runner: OutcomeModelRunner;
+  context: OutcomeExecutionContext;
+  ledger?: OutcomeSpendLedger;
+  receiptStore?: OutcomeReceiptStore;
+}
+
+export interface ExecuteOutcomeTaskResult {
+  receipt: OutcomeRuntimeReceipt;
+  signed: SignedDecisionLogEntry | null;
+  outputText: string;
+  reviewerOutputText?: string;
+}
+
+interface ModelRunResult {
+  response: OutcomeModelResponse;
+  decision: SpendDecision;
+  signed: SignedDecisionLogEntry | null;
+  actualInputTokens: number;
+  actualOutputTokens: number;
+  actualCents: number;
+  durationMs: number;
+}
+
+export async function executeOutcomeTask(options: ExecuteOutcomeTaskOptions): Promise<ExecuteOutcomeTaskResult> {
+  validateTemplate(options.template);
+  validateInputs(options.template, options.inputs);
+  const started = Date.now();
+  const receiptStore = options.receiptStore ?? new InMemoryOutcomeReceiptStore();
+  const ledger = options.ledger ?? new OutcomeSpendLedger(options.guard.getSpendStore());
+  const posture = options.context.posture ?? 'standard';
+  const budgetTier = options.context.budgetTier ?? 'solo';
+  const route = recommend({
+    vertical: options.template.vertical,
+    outcome: options.template.slug,
+    posture,
+    budgetTier,
+    imageCapable: options.context.imageCapable,
+  });
+  const blockedOrigins = [...new Set([...(options.template.blockedOrigins ?? []), ...route.blockedOrigins])];
+  const hold = await ledger.hold({
+    scope: options.context.scope,
+    window: options.template.spendCaps.window ?? 'per_call',
+    amountCents: options.template.spendCaps.maxCostCents,
+    limitCents: options.template.spendCaps.periodCapCents ?? options.template.spendCaps.maxCostCents,
+  });
+
+  const links: OutcomeDecisionLink[] = [];
+  let status: OutcomeRuntimeReceipt['status'] = 'failed';
+  let drafter: ModelRunResult | null = null;
+  let reviewer: ModelRunResult | null = null;
+  let fallback: OutcomeRuntimeReceipt['fallback'] = null;
+  let outputText = '';
+  let reviewerOutputText: string | undefined;
+  let gate: OutcomeQualityGateResult = { score: 0, riskScore: 0, passed: false, issues: ['not_run'], triggerFired: [] };
+
+  try {
+    drafter = await runGuardedModel({
+      guard: options.guard,
+      template: options.template,
+      context: options.context,
+      route,
+      role: 'drafter',
+      model: route.drafter,
+      inputs: options.inputs,
+      runner: options.runner,
+    });
+    links.push(linkFor('drafter', drafter));
+    outputText = drafter.response.text;
+    gate = scoreOutcomeOutput(outputText, options.template, drafter.response.qualityScore);
+
+    if (shouldRunReviewer(gate) && options.context.reviewerCascadeAuto === true) {
+      const reviewerModel = antiCorrelatedReviewer(route, drafter.decision.modelResolved);
+      reviewer = await runGuardedModel({
+        guard: options.guard,
+        template: options.template,
+        context: options.context,
+        route,
+        role: 'reviewer',
+        model: reviewerModel,
+        inputs: { outcome_metadata_only: true, trigger_fired: gate.triggerFired.join(',') },
+        runner: options.runner,
+        priorOutputText: outputText,
+      });
+      links.push(linkFor('reviewer', reviewer));
+      reviewerOutputText = reviewer.response.text;
+      status = 'completed';
+    } else {
+      status = gate.passed ? 'completed' : 'quality_failed';
+    }
+  } catch (err) {
+    if (!drafter) {
+      const fallbackModel = antiCorrelatedFallback(route, route.drafter);
+      if (fallbackModel) {
+        fallback = { model: fallbackModel, reason: errorReason(err) };
+        try {
+          drafter = await runGuardedModel({
+            guard: options.guard,
+            template: options.template,
+            context: options.context,
+            route,
+            role: 'fallback',
+            model: fallbackModel,
+            inputs: options.inputs,
+            runner: options.runner,
+          });
+          fallback.decisionId = drafter.decision.decisionId;
+          links.push(linkFor('fallback', drafter));
+          outputText = drafter.response.text;
+          gate = scoreOutcomeOutput(outputText, options.template, drafter.response.qualityScore);
+          status = gate.passed ? 'completed' : 'quality_failed';
+        } catch {
+          status = 'failed';
+        }
+      } else {
+        status = 'failed';
+      }
+    } else {
+      status = 'failed';
+    }
+  }
+
+  const totalCostCents = links.reduce((sum, item) => sum + item.actualCents, 0);
+  if (status === 'completed') await ledger.settle(hold, totalCostCents);
+  else await ledger.refund(hold);
+
+  const receipt = buildReceipt({
+    template: options.template,
+    context: options.context,
+    hold,
+    route,
+    blockedOrigins,
+    status,
+    drafter,
+    reviewer,
+    fallback,
+    links,
+    gate,
+    totalCostCents,
+    durationMs: Date.now() - started,
+    posture,
+    budgetTier,
+  });
+  await receiptStore.append(receipt);
+  const signedReceipt = await options.guard.recordOutcomeReceipt(receiptToMetadata(receipt));
+  return { receipt, signed: signedReceipt.signed, outputText, reviewerOutputText };
+}
+
+async function runGuardedModel(args: {
+  guard: SpendGuard;
+  template: OutcomeTemplate;
+  context: OutcomeExecutionContext;
+  route: ModelRouteRecommendation;
+  role: 'drafter' | 'reviewer' | 'fallback';
+  model: string;
+  inputs: Record<string, unknown>;
+  runner: OutcomeModelRunner;
+  priorOutputText?: string;
+}): Promise<ModelRunResult> {
+  const request = buildModelRequest(args);
+  const inputTokens = args.guard.estimateTokens(request.systemPrompt + '\n' + request.userPrompt);
+  const outputTokens = estimatedOutputTokens(args.template);
+  const { decision, signed } = await args.guard.decide({
+    provider: inferProvider(args.model),
+    model: args.model,
+    inputTokens,
+    outputTokens,
+    scope: args.context.scope,
+    capabilityClaim: args.template.capability,
+    label: args.template.slug + ':' + args.role,
+  });
+  if (decision.action === 'block') throw new Error('Outcome model call blocked by policy');
+
+  const started = Date.now();
+  let response: OutcomeModelResponse;
+  try {
+    response = await args.runner.run({ ...request, model: decision.modelResolved });
+  } catch (err) {
+    await args.guard.settleStreamUsage(decision.decisionId, 0, 0, { partial: true, reason: errorReason(err) });
+    throw err;
+  }
+  const actualInputTokens = validToken(response.inputTokens) ? response.inputTokens : inputTokens;
+  const actualOutputTokens = validToken(response.outputTokens) ? response.outputTokens : args.guard.estimateTokens(response.text);
+  const computed = computeCallCents(decision.modelResolved, actualInputTokens, actualOutputTokens);
+  const actualCents = validCents(response.costCents) ? response.costCents : computed ?? decision.projectedCents;
+  await args.guard.settleStreamUsage(decision.decisionId, actualInputTokens, actualOutputTokens, { partial: false });
+  return { response, decision, signed, actualInputTokens, actualOutputTokens, actualCents, durationMs: Date.now() - started };
+}
+
+function buildModelRequest(args: {
+  template: OutcomeTemplate;
+  context: OutcomeExecutionContext;
+  route: ModelRouteRecommendation;
+  role: 'drafter' | 'reviewer' | 'fallback';
+  model: string;
+  inputs: Record<string, unknown>;
+  priorOutputText?: string;
+}): OutcomeModelRequest {
+  const contextLines = (args.template.contextInjection ?? []).map((item) => {
+    const value = args.context.context?.[item.field] ?? item.example ?? '';
+    return item.label + ': ' + value;
+  });
+  const inputLines = Object.entries(args.inputs).map(([key, value]) => key + ': ' + JSON.stringify(value));
+  const userPrompt = [
+    'Outcome: ' + args.template.slug,
+    'Role: ' + args.role,
+    ...contextLines,
+    ...inputLines,
+    args.priorOutputText ? 'Prior drafter output:\n' + args.priorOutputText : '',
+    'Return format: ' + args.template.outputFormat,
+  ].filter(Boolean).join('\n');
+  return {
+    role: args.role,
+    model: args.model,
+    vertical: String(args.template.vertical),
+    outcome: args.template.slug,
+    promptVersion: args.template.promptVersion,
+    systemPrompt: args.template.systemPrompt + (args.template.disclaimer ? '\n' + args.template.disclaimer : ''),
+    userPrompt,
+    outputFormat: args.template.outputFormat,
+    metadata: {
+      vertical: String(args.template.vertical),
+      outcome: args.template.slug,
+      promptVersion: args.template.promptVersion,
+      need: args.route.need,
+      role: args.role,
+      effort: effortForRole(args.route, args.role),
+    },
+  };
+}
+
+function effortForRole(route: ModelRouteRecommendation, role: 'drafter' | 'reviewer' | 'fallback'): string {
+  return role === 'reviewer' ? route.reviewerEffort : route.drafterEffort;
+}
+
+function buildReceipt(args: {
+  template: OutcomeTemplate;
+  context: OutcomeExecutionContext;
+  hold: OutcomeSpendHold;
+  route: ModelRouteRecommendation;
+  blockedOrigins: string[];
+  status: OutcomeRuntimeReceipt['status'];
+  drafter: ModelRunResult | null;
+  reviewer: ModelRunResult | null;
+  fallback: OutcomeRuntimeReceipt['fallback'];
+  links: OutcomeDecisionLink[];
+  gate: OutcomeQualityGateResult;
+  totalCostCents: number;
+  durationMs: number;
+  posture: string;
+  budgetTier: string;
+}): OutcomeRuntimeReceipt {
+  return {
+    receiptId: 'ag_outcome_' + randomUUID(),
+    timestamp: new Date().toISOString(),
+    vertical: String(args.template.vertical),
+    outcome: args.template.slug,
+    promptVersion: args.template.promptVersion,
+    customerId: args.context.customerId,
+    scopeKey: buildScopeKey(args.context.scope),
+    status: args.status,
+    drafter: modelSummary(args.drafter),
+    reviewer: args.reviewer ? { ...modelSummary(args.reviewer), verdict: args.reviewer.response.verdict ?? 'reviewed' } : null,
+    fallback: args.fallback,
+    triggerFired: args.gate.triggerFired,
+    qualityScore: args.gate.score,
+    pass: args.status === 'completed',
+    totalCostCents: args.totalCostCents,
+    durationMs: args.durationMs,
+    decisionLinks: args.links,
+    metadata: {
+      need: args.route.need,
+      capability: args.template.capability,
+      posture: args.posture,
+      budgetTier: args.budgetTier,
+      blockedOrigins: args.blockedOrigins,
+      effort: args.route.effort,
+      drafterEffort: args.route.drafterEffort,
+      reviewerEffort: args.route.reviewerEffort,
+      provenance: provenanceSummary(args.drafter, args.reviewer),
+    },
+  };
+}
+
+function modelSummary(run: ModelRunResult | null): OutcomeRuntimeReceipt['drafter'] {
+  if (!run) return { model: 'none', costCents: 0, inputTokens: 0, outputTokens: 0 };
+  return {
+    model: run.decision.modelResolved,
+    decisionId: run.decision.decisionId,
+    costCents: run.actualCents,
+    inputTokens: run.actualInputTokens,
+    outputTokens: run.actualOutputTokens,
+  };
+}
+
+function provenanceSummary(drafter: ModelRunResult | null, reviewer: ModelRunResult | null): Record<string, string | number | boolean> {
+  const out: Record<string, string | number | boolean> = {};
+  if (drafter?.response.durationMs) out.drafterDurationMs = drafter.response.durationMs;
+  if (reviewer?.response.durationMs) out.reviewerDurationMs = reviewer.response.durationMs;
+  return out;
+}
+
+function receiptToMetadata(receipt: OutcomeRuntimeReceipt): Record<string, unknown> {
+  return JSON.parse(JSON.stringify(receipt)) as Record<string, unknown>;
+}
+
+function linkFor(role: 'drafter' | 'reviewer' | 'fallback', run: ModelRunResult): OutcomeDecisionLink {
+  return {
+    role,
+    decisionId: run.decision.decisionId,
+    entryHash: run.signed?.entryHash,
+    modelRequested: run.decision.modelRequested,
+    modelResolved: run.decision.modelResolved,
+    projectedCents: run.decision.projectedCents,
+    actualCents: run.actualCents,
+    inputTokens: run.actualInputTokens,
+    outputTokens: run.actualOutputTokens,
+  };
+}
+
+function antiCorrelatedReviewer(route: ModelRouteRecommendation, drafterModel: string): string {
+  if (familyOf(route.reviewer) !== familyOf(drafterModel)) return route.reviewer;
+  return antiCorrelatedFallback(route, drafterModel) ?? route.reviewer;
+}
+
+function antiCorrelatedFallback(route: ModelRouteRecommendation, failedModel: string): string | null {
+  const failedFamily = familyOf(failedModel);
+  return route.allowedModels.find((model) => model !== failedModel && familyOf(model) !== failedFamily && computeCallCents(model, 1, 1) !== null) ?? null;
+}
+
+function familyOf(model: string): string {
+  const id = model.toLowerCase();
+  if (id.includes('opus')) return 'opus';
+  if (id.includes('sonnet')) return 'sonnet';
+  if (id.includes('haiku')) return 'haiku';
+  if (id.includes('gpt-oss')) return 'gpt-oss';
+  if (id.includes('gpt-5')) return 'gpt-5';
+  if (id.includes('gpt-4')) return 'gpt-4';
+  if (id.includes('llama')) return 'llama';
+  if (id.includes('mistral')) return 'mistral';
+  if (id.includes('nova')) return 'nova';
+  return id.split('/')[0] || id.split('.')[0] || 'unknown';
+}
+
+function estimatedOutputTokens(template: OutcomeTemplate): number {
+  return Math.max(128, Math.min(4096, Math.ceil(template.spendCaps.maxCostCents * 64)));
+}
+
+function validateTemplate(template: OutcomeTemplate): void {
+  if (!template.slug || !template.vertical) throw new Error('Outcome template must include slug and vertical');
+  if (!Number.isSafeInteger(template.spendCaps.maxCostCents) || template.spendCaps.maxCostCents < 0) {
+    throw new Error('Outcome spend cap must be a non-negative safe integer');
+  }
+}
+
+function validateInputs(template: OutcomeTemplate, inputs: Record<string, unknown>): void {
+  for (const input of template.inputs) {
+    if (input.required && !(input.name in inputs)) throw new Error('Missing required outcome input: ' + input.name);
+  }
+}
+
+function validToken(value: unknown): value is number {
+  return Number.isSafeInteger(value) && (value as number) >= 0;
+}
+
+function validCents(value: unknown): value is number {
+  return Number.isSafeInteger(value) && (value as number) >= 0;
+}
+
+function errorReason(err: unknown): string {
+  if (err instanceof Error && err.message) return err.message.slice(0, 120);
+  return 'model_error';
+}

package/src/outcomes/types.ts

@@ -0,0 +1,176 @@
+import type { CapabilityTier, SpendScope, SpendWindow } from '../types';
+
+export type OutcomeVertical = 'law' | 'accounting' | 'insurance' | 'realestate' | 'ecommerce' | string;
+export type OutcomeRole = 'drafter' | 'reviewer' | 'fallback';
+export type OutcomeStatus = 'completed' | 'blocked' | 'failed' | 'quality_failed';
+export type OutcomeEffort = 'low' | 'medium' | 'high' | 'xhigh';
+export type OutcomeGuardrailTier = 'grounded_cite_or_refuse' | 'reviewer_required' | 'human_of_record';
+export type OutcomeTemplateStatus = 'active' | 'specced';
+
+export interface OutcomeInputDefinition {
+  name: string;
+  type: 'string' | 'number' | 'boolean' | 'json' | 'date';
+  required?: boolean;
+  description?: string;
+}
+
+export interface OutcomeSpendCaps {
+  maxCostCents: number;
+  window?: SpendWindow;
+  periodCapCents?: number;
+}
+
+export interface OutcomeQualityGateRubric {
+  name: string;
+  weight?: number;
+  passThreshold?: number;
+}
+
+export interface OutcomeQualityGateConfig {
+  passThreshold?: number;
+  reviewerTriggerScore?: number;
+  riskKeywords?: string[];
+  rubrics?: OutcomeQualityGateRubric[];
+}
+
+export interface OutcomeContextInjection {
+  field: string;
+  label: string;
+  required?: boolean;
+  example?: string;
+}
+
+export interface OutcomeTemplate {
+  slug: string;
+  vertical: OutcomeVertical;
+  systemPrompt: string;
+  inputs: OutcomeInputDefinition[];
+  outputFormat: string;
+  need: string;
+  capability: CapabilityTier;
+  spendCaps: OutcomeSpendCaps;
+  promptVersion: string;
+  effort?: OutcomeEffort;
+  guardrailTier?: OutcomeGuardrailTier;
+  status?: OutcomeTemplateStatus;
+  contextInjection?: OutcomeContextInjection[];
+  blockedOrigins?: string[];
+  premiumAgent?: string;
+  disclaimer?: string;
+  qualityGate?: OutcomeQualityGateConfig;
+}
+
+export interface OutcomeExecutionContext {
+  scope: SpendScope;
+  customerId?: string;
+  posture?: string;
+  budgetTier?: string;
+  imageCapable?: boolean;
+  context?: Record<string, string>;
+  reviewerCascadeAuto?: boolean;
+}
+
+export interface OutcomeModelRequest {
+  role: OutcomeRole;
+  model: string;
+  vertical: string;
+  outcome: string;
+  promptVersion: string;
+  systemPrompt: string;
+  userPrompt: string;
+  outputFormat: string;
+  metadata: Record<string, string | number | boolean>;
+}
+
+export interface OutcomeModelResponse {
+  text: string;
+  inputTokens?: number;
+  outputTokens?: number;
+  costCents?: number;
+  qualityScore?: number;
+  confidence?: number;
+  verdict?: string;
+  durationMs?: number;
+  provenance?: Record<string, string | number | boolean>;
+}
+
+export interface OutcomeModelRunner {
+  run(request: OutcomeModelRequest): Promise<OutcomeModelResponse>;
+}
+
+export interface OutcomeDecisionLink {
+  role: OutcomeRole;
+  decisionId: string;
+  entryHash?: string;
+  modelRequested: string;
+  modelResolved: string;
+  projectedCents: number;
+  actualCents: number;
+  inputTokens: number;
+  outputTokens: number;
+}
+
+export interface OutcomeRuntimeReceipt {
+  receiptId: string;
+  timestamp: string;
+  vertical: string;
+  outcome: string;
+  promptVersion: string;
+  customerId?: string;
+  scopeKey: string;
+  status: OutcomeStatus;
+  drafter: {
+    model: string;
+    decisionId?: string;
+    costCents: number;
+    inputTokens: number;
+    outputTokens: number;
+  };
+  reviewer: {
+    model: string;
+    decisionId?: string;
+    costCents: number;
+    inputTokens: number;
+    outputTokens: number;
+    verdict: string;
+  } | null;
+  fallback: {
+    model: string;
+    decisionId?: string;
+    reason: string;
+  } | null;
+  triggerFired: string[];
+  qualityScore: number;
+  pass: boolean;
+  totalCostCents: number;
+  durationMs: number;
+  decisionLinks: OutcomeDecisionLink[];
+  metadata: {
+    need: string;
+    capability: CapabilityTier;
+    posture: string;
+    budgetTier: string;
+    blockedOrigins: string[];
+    effort: OutcomeEffort;
+    drafterEffort: OutcomeEffort;
+    reviewerEffort: OutcomeEffort;
+    provenance: Record<string, string | number | boolean>;
+  };
+}
+
+export interface OutcomeReceiptStore {
+  append(receipt: OutcomeRuntimeReceipt): Promise<void>;
+  readAll(): Promise<OutcomeRuntimeReceipt[]>;
+}
+
+export class InMemoryOutcomeReceiptStore implements OutcomeReceiptStore {
+  private readonly receipts: OutcomeRuntimeReceipt[] = [];
+
+  async append(receipt: OutcomeRuntimeReceipt): Promise<void> {
+    this.receipts.push(receipt);
+  }
+
+  async readAll(): Promise<OutcomeRuntimeReceipt[]> {
+    return [...this.receipts];
+  }
+}