@graphorin/provider 0.6.1 → 0.7.0

package/src/middleware/compose.ts

@@ -0,0 +1,213 @@
+/**
+ * Canonical-order middleware composer. The composer enforces the
+ * documented ordering at startup and throws
+ * {@link MiddlewareOrderingError} on violation. Manual
+ * `withRetry(withRedaction(p))` style still works but emits a one-time
+ * WARN unless the consumer flips `strict: true`, in which case the
+ * composer also throws on manual nesting that violates the canonical
+ * order.
+ *
+ * @packageDocumentation
+ */
+
+import type { Provider, ProviderMiddleware } from '@graphorin/core';
+
+import { MiddlewareOrderingError } from '../errors/errors.js';
+
+/**
+ * Symbol attached to every middleware-produced provider so the
+ * composer can detect and validate the chain. The symbol is opaque
+ * and cross-realm safe via `Symbol.for`.
+ *
+ * @stable
+ */
+export const MIDDLEWARE_KIND: unique symbol = Symbol.for('graphorin.provider.middleware.kind');
+
+/**
+ * Canonical middleware ordering - outermost → innermost. The table
+ * is enforced by {@link composeProviderMiddleware} and is part of the
+ * provider layer's public contract (DEC-145 / ADR-039).
+ *
+ * ## Why this order
+ *
+ * Composition is outermost-first, so a request flows top→bottom and a
+ * response flows bottom→top:
+ *
+ * - `withTracing` is outermost so the span wraps everything below -
+ *   including retries - and records true end-to-end latency.
+ * - `withRetry` sits above the rate/cost limiters so each retry
+ *   attempt is independently counted and throttled.
+ * - `withRateLimit` → `withCostLimit` → `withCostTracking` form the
+ *   budget stack: throttle before admitting, reject over-budget calls
+ *   before spending, then meter what actually went through.
+ * - `withFallback` is just above redaction so a fallback to a
+ *   secondary provider still passes through the redactor.
+ * - `withRedaction` is **innermost** (closest to the provider) so it
+ *   is the last thing to touch the outbound payload and the first to
+ *   touch the inbound stream - guaranteeing every retry, fallback, and
+ *   cost-tracked request sees an already-redacted payload and no
+ *   secret can bypass it.
+ *
+ * @stable
+ */
+export const CANONICAL_MIDDLEWARE_ORDER: readonly string[] = [
+  'withTracing',
+  'withRetry',
+  'withRateLimit',
+  'withCostLimit',
+  'withCostTracking',
+  'withFallback',
+  'withRedaction',
+];
+
+const CANONICAL_INDEX: Readonly<Record<string, number>> = Object.freeze(
+  CANONICAL_MIDDLEWARE_ORDER.reduce<Record<string, number>>((acc, name, idx) => {
+    acc[name] = idx;
+    return acc;
+  }, {}),
+);
+
+interface KindedProvider extends Provider {
+  readonly [MIDDLEWARE_KIND]?: string;
+  readonly [INNER_PROVIDER]?: Provider;
+}
+
+/**
+ * Symbol used to walk the chain - every wrapper exposes the inner
+ * provider so the composer can introspect the full stack at startup.
+ *
+ * @stable
+ */
+export const INNER_PROVIDER: unique symbol = Symbol.for('graphorin.provider.middleware.inner');
+
+/**
+ * Read the discriminant kind attached to a middleware-produced
+ * provider. Returns `undefined` if the provider is the bare adapter
+ * or a custom wrapper that does not declare a kind.
+ *
+ * @stable
+ */
+export function getMiddlewareKind(provider: Provider): string | undefined {
+  return (provider as KindedProvider)[MIDDLEWARE_KIND];
+}
+
+/**
+ * Walk the middleware chain inside `provider` and return the array of
+ * declared kinds (outer → inner).
+ *
+ * @stable
+ */
+export function listMiddlewareKinds(provider: Provider): readonly string[] {
+  const kinds: string[] = [];
+  let cur: KindedProvider | undefined = provider as KindedProvider;
+  while (cur !== undefined) {
+    const kind = cur[MIDDLEWARE_KIND];
+    if (kind !== undefined) kinds.push(kind);
+    cur = cur[INNER_PROVIDER] as KindedProvider | undefined;
+  }
+  return kinds;
+}
+
+/**
+ * Return `true` iff the chain rooted at `provider` contains a
+ * middleware whose kind matches `name`.
+ *
+ * @stable
+ */
+export function providerHasMiddleware(provider: Provider, name: string): boolean {
+  return listMiddlewareKinds(provider).includes(name);
+}
+
+/**
+ * Wrap an adapter in a middleware chain whose order is validated
+ * against {@link CANONICAL_MIDDLEWARE_ORDER}. The argument array MUST
+ * be ordered outermost → innermost - the same way the layers appear
+ * in the documented composition example. The composer validates that
+ * every kind known to the canonical order is monotonically non-
+ * decreasing in index, throws otherwise.
+ *
+ * Custom middleware whose kind is NOT in the canonical order is
+ * silently allowed at any position - operators registering bespoke
+ * layers via {@link defineProviderMiddleware} carry the
+ * responsibility of placing them sensibly.
+ *
+ * @stable
+ */
+export function composeProviderMiddleware(
+  middlewares: ReadonlyArray<ProviderMiddleware>,
+): ProviderMiddleware {
+  return (next: Provider): Provider => {
+    let chain: Provider = next;
+    // Apply innermost → outermost so the composed call order matches
+    // the array's outer → inner declaration.
+    for (let i = middlewares.length - 1; i >= 0; i--) {
+      const factory = middlewares[i];
+      if (typeof factory !== 'function') {
+        throw new TypeError(
+          `composeProviderMiddleware: entry at index ${i} is not a ProviderMiddleware function.`,
+        );
+      }
+      chain = factory(chain);
+    }
+    validateMiddlewareOrder(chain);
+    return chain;
+  };
+}
+
+/**
+ * Validate the kinds present in `provider` against the canonical
+ * order. Exposed for tests; the composer calls it on every chain.
+ *
+ * @stable
+ */
+export function validateMiddlewareOrder(provider: Provider): void {
+  const kinds = listMiddlewareKinds(provider);
+  // Filter to the kinds we recognise.
+  const recognised: { kind: string; index: number }[] = [];
+  for (const kind of kinds) {
+    const idx = CANONICAL_INDEX[kind];
+    if (idx !== undefined) recognised.push({ kind, index: idx });
+  }
+  // The recognised list runs outer → inner; canonical indices must
+  // increase monotonically. The first decreasing pair is the
+  // violation.
+  for (let i = 1; i < recognised.length; i++) {
+    const prev = recognised[i - 1];
+    const cur = recognised[i];
+    if (prev !== undefined && cur !== undefined && cur.index < prev.index) {
+      throw new MiddlewareOrderingError({
+        offendingPair: [prev.kind, cur.kind],
+        canonicalOrder: CANONICAL_MIDDLEWARE_ORDER,
+      });
+    }
+  }
+}
+
+/**
+ * Decorator factory used internally by every built-in middleware. The
+ * returned function attaches the canonical kind discriminator and the
+ * inner-provider symbol so the composer can introspect chains.
+ *
+ * @stable
+ */
+export function defineProviderMiddleware<T>(args: {
+  readonly kind: string;
+  readonly factory: (opts: T) => ProviderMiddleware;
+}): (opts: T) => ProviderMiddleware {
+  return (opts: T): ProviderMiddleware => {
+    const built = args.factory(opts);
+    return (next: Provider): Provider => {
+      const wrapped = built(next);
+      const branded: Provider = Object.create(wrapped) as Provider;
+      Object.defineProperty(branded, MIDDLEWARE_KIND, {
+        value: args.kind,
+        enumerable: false,
+      });
+      Object.defineProperty(branded, INNER_PROVIDER, {
+        value: next,
+        enumerable: false,
+      });
+      return branded;
+    };
+  };
+}

package/src/middleware/index.ts

@@ -0,0 +1,37 @@
+/**
+ * Middleware barrel - the canonical-order composer plus seven built-in
+ * middlewares.
+ *
+ * @packageDocumentation
+ */
+
+export {
+  CANONICAL_MIDDLEWARE_ORDER,
+  composeProviderMiddleware,
+  defineProviderMiddleware,
+  getMiddlewareKind,
+  MIDDLEWARE_KIND,
+  providerHasMiddleware,
+} from './compose.js';
+export {
+  assertProductionMiddleware,
+  type ProductionStartupHookOptions,
+} from './production-hook.js';
+export { type WithCostLimitOptions, withCostLimit } from './with-cost-limit.js';
+export {
+  type CostAccumulator,
+  type CostTrackingTotals,
+  createCostAccumulator,
+  type WithCostTrackingOptions,
+  withCostTracking,
+} from './with-cost-tracking.js';
+export { type WithFallbackOptions, withFallback } from './with-fallback.js';
+export { type WithRateLimitOptions, withRateLimit } from './with-rate-limit.js';
+export {
+  type PromptRedactionPolicy,
+  type PromptRedactionScanScope,
+  type PromptRedactionViolation,
+  withRedaction,
+} from './with-redaction.js';
+export { type WithRetryOptions, withRetry } from './with-retry.js';
+export { type WithTracingOptions, withTracing } from './with-tracing.js';

package/src/middleware/production-hook.ts

@@ -0,0 +1,47 @@
+/**
+ * Production startup hook - fails fast when a security-critical
+ * middleware is missing from the composed chain. The default
+ * configuration enforces `withRedaction`; consumers can extend the
+ * `requiredKinds` list to lock additional middlewares.
+ *
+ * @packageDocumentation
+ */
+
+import type { Provider } from '@graphorin/core';
+
+import { MissingProductionMiddlewareError } from '../errors/errors.js';
+import { providerHasMiddleware } from './compose.js';
+
+/**
+ * Options for {@link assertProductionMiddleware}.
+ *
+ * @stable
+ */
+export interface ProductionStartupHookOptions {
+  /** Middleware kinds that must be present. Defaults to `['withRedaction']`. */
+  readonly requiredKinds?: ReadonlyArray<string>;
+  /** Force the check regardless of `NODE_ENV`. */
+  readonly force?: boolean;
+}
+
+/**
+ * Throw {@link MissingProductionMiddlewareError} if a required
+ * middleware is missing from the chain rooted at `provider`. The
+ * check runs only when `NODE_ENV === 'production'` unless `force` is
+ * `true`.
+ *
+ * @stable
+ */
+export function assertProductionMiddleware(
+  provider: Provider,
+  options: ProductionStartupHookOptions = {},
+): void {
+  const isProd = process.env.NODE_ENV === 'production';
+  if (!isProd && options.force !== true) return;
+  const required = options.requiredKinds ?? ['withRedaction'];
+  for (const kind of required) {
+    if (!providerHasMiddleware(provider, kind)) {
+      throw new MissingProductionMiddlewareError(kind);
+    }
+  }
+}

package/src/middleware/with-cost-limit.ts

@@ -0,0 +1,131 @@
+/**
+ * `withCostLimit` - enforce per-session / per-run / per-agent /
+ * per-hour cost ceilings. Couples with {@link withCostTracking} for
+ * the underlying accumulator. The middleware is positioned between
+ * `withRateLimit` and `withCostTracking` per the canonical order so a
+ * budget breach trips before the request hits the underlying
+ * provider.
+ *
+ * @packageDocumentation
+ */
+
+import type { Provider } from '@graphorin/core';
+
+import { CostBudgetExceededError } from '../errors/errors.js';
+import { defineProviderMiddleware } from './compose.js';
+
+/**
+ * Options for {@link withCostLimit}.
+ *
+ * @stable
+ */
+export interface WithCostLimitOptions {
+  /** Maximum cumulative USD cost per session. */
+  readonly maxPerSession?: number;
+  /** Maximum cumulative USD cost per run. */
+  readonly maxPerRun?: number;
+  /** Maximum cumulative USD cost per hour. */
+  readonly maxPerHour?: number;
+  /** What to do on breach. Default `'throw'`. */
+  readonly onExceed?: 'throw' | 'warn';
+  /**
+   * Resolver returning the current observed cost for the relevant
+   * scope. The resolver lets consumers wire any accumulator (the
+   * shipped `@graphorin/observability/cost.CostTracker` works out of
+   * the box). When unset, the middleware is a no-op (a placeholder
+   * for tooling that wires the accumulator later).
+   */
+  readonly resolveObservedCost?: (
+    scope: 'session' | 'run' | 'hour',
+    metadata: Readonly<Record<string, unknown>> | undefined,
+  ) => number;
+  /** Optional sink for `'warn'` mode. Defaults to `console.warn`. */
+  readonly logger?: (message: string, meta?: object) => void;
+}
+
+/**
+ * @stable
+ */
+export const withCostLimit = defineProviderMiddleware<WithCostLimitOptions>({
+  kind: 'withCostLimit',
+  factory: (opts: WithCostLimitOptions) => {
+    const onExceed = opts.onExceed ?? 'throw';
+    const resolver = opts.resolveObservedCost;
+    const logger = opts.logger ?? defaultLogger;
+    // PS-8: a ceiling without a resolver is a silent no-op - the limits look
+    // enforced but never trip. Warn loudly so the gap is visible. (No ceiling
+    // + no resolver is the documented inert placeholder, so it stays quiet.)
+    const hasCeiling =
+      opts.maxPerSession !== undefined ||
+      opts.maxPerRun !== undefined ||
+      opts.maxPerHour !== undefined;
+    if (hasCeiling && resolver === undefined) {
+      logger(
+        '[graphorin/provider] withCostLimit: a cost ceiling is configured but no ' +
+          '`resolveObservedCost` was supplied, so the limit is UNENFORCED (no-op). ' +
+          'Wire a resolver (e.g. from createCostAccumulator / @graphorin/observability).',
+      );
+    }
+    return (next: Provider): Provider => ({
+      name: next.name,
+      modelId: next.modelId,
+      capabilities: next.capabilities,
+      ...(next.acceptsSensitivity !== undefined
+        ? { acceptsSensitivity: next.acceptsSensitivity }
+        : {}),
+      stream(req) {
+        if (resolver !== undefined) {
+          check(asRecord(req.metadata), resolver, opts, onExceed, logger);
+        }
+        return next.stream(req);
+      },
+      async generate(req) {
+        if (resolver !== undefined) {
+          check(asRecord(req.metadata), resolver, opts, onExceed, logger);
+        }
+        return next.generate(req);
+      },
+      ...(next.countTokens ? { countTokens: next.countTokens.bind(next) } : {}),
+    });
+  },
+});
+
+function check(
+  metadata: Readonly<Record<string, unknown>> | undefined,
+  resolver: (
+    scope: 'session' | 'run' | 'hour',
+    metadata?: Readonly<Record<string, unknown>>,
+  ) => number,
+  opts: WithCostLimitOptions,
+  onExceed: 'throw' | 'warn',
+  logger: (message: string, meta?: object) => void,
+): void {
+  const checks: Array<['session' | 'run' | 'hour', number | undefined]> = [
+    ['session', opts.maxPerSession],
+    ['run', opts.maxPerRun],
+    ['hour', opts.maxPerHour],
+  ];
+  for (const [scope, limit] of checks) {
+    if (limit === undefined) continue;
+    const observed = resolver(scope, metadata);
+    if (observed > limit) {
+      if (onExceed === 'throw') {
+        throw new CostBudgetExceededError({ scope, limit, observed });
+      }
+      logger(
+        `[graphorin/provider] withCostLimit: ${scope} budget breach (${observed.toFixed(4)} > ${limit.toFixed(4)} USD).`,
+        { scope, limit, observed },
+      );
+    }
+  }
+}
+
+function defaultLogger(message: string, meta?: object): void {
+  if (meta !== undefined) console.warn(message, meta);
+  else console.warn(message);
+}
+
+function asRecord(value: unknown): Readonly<Record<string, unknown>> | undefined {
+  if (value === undefined || value === null) return undefined;
+  return value as Readonly<Record<string, unknown>>;
+}

package/src/middleware/with-cost-tracking.ts

@@ -0,0 +1,216 @@
+/**
+ * `withCostTracking` - reports `tokensUsed` + `cost` per `provider × model`
+ * from the `finish` event of every stream / one-shot generation via the
+ * `onUsage` hook. For a ready-made process-local accumulator, wire
+ * {@link createCostAccumulator}'s `onUsage` and read its `.totals()`; consumers
+ * that prefer the framework's `CostTracker` from `@graphorin/observability`
+ * pass their own delegate.
+ *
+ * @packageDocumentation
+ */
+
+import type { Provider, ProviderEvent, ProviderResponse } from '@graphorin/core';
+
+import { defineProviderMiddleware } from './compose.js';
+
+/**
+ * Aggregated totals for one `provider × model`, returned by
+ * {@link CostAccumulator.totalFor} / {@link CostAccumulator.totals}.
+ *
+ * @stable
+ */
+export interface CostTrackingTotals {
+  readonly callCount: number;
+  readonly promptTokens: number;
+  readonly completionTokens: number;
+  readonly totalTokens: number;
+  /** Prompt tokens served from the provider cache (subset of promptTokens). */
+  readonly cachedReadTokens: number;
+  /** Prompt tokens written to the provider cache (subset of promptTokens). */
+  readonly cacheWriteTokens: number;
+  readonly costUsd: number;
+}
+
+/**
+ * A process-local cost accumulator (PS-8). Wire {@link CostAccumulator.onUsage}
+ * into {@link withCostTracking} and read the running totals - keyed by
+ * `provider × model` - back via {@link CostAccumulator.totals} /
+ * {@link CostAccumulator.totalFor}.
+ *
+ * @stable
+ */
+export interface CostAccumulator {
+  /** Pass this to {@link withCostTracking}'s `onUsage`. */
+  readonly onUsage: NonNullable<WithCostTrackingOptions['onUsage']>;
+  /** Snapshot of every tracked `provider::model` → totals. */
+  totals(): ReadonlyMap<string, CostTrackingTotals>;
+  /** Running totals for one `provider × model` (zeros when unseen). */
+  totalFor(providerName: string, modelId: string): CostTrackingTotals;
+  /** Clear all accumulated totals. */
+  reset(): void;
+}
+
+const ZERO_TOTALS: CostTrackingTotals = Object.freeze({
+  callCount: 0,
+  promptTokens: 0,
+  completionTokens: 0,
+  totalTokens: 0,
+  cachedReadTokens: 0,
+  cacheWriteTokens: 0,
+  costUsd: 0,
+});
+
+/**
+ * Create a {@link CostAccumulator} - the process-local accumulator described on
+ * {@link withCostTracking}. Keys totals by `'<providerName>::<modelId>'`.
+ *
+ * @stable
+ */
+export function createCostAccumulator(): CostAccumulator {
+  const map = new Map<string, CostTrackingTotals>();
+  const keyOf = (providerName: string, modelId: string): string => `${providerName}::${modelId}`;
+  return {
+    onUsage: (info) => {
+      const key = keyOf(info.providerName, info.modelId);
+      const prev = map.get(key) ?? ZERO_TOTALS;
+      map.set(key, {
+        callCount: prev.callCount + 1,
+        promptTokens: prev.promptTokens + info.promptTokens,
+        completionTokens: prev.completionTokens + info.completionTokens,
+        totalTokens: prev.totalTokens + info.totalTokens,
+        cachedReadTokens: prev.cachedReadTokens + (info.cachedReadTokens ?? 0),
+        cacheWriteTokens: prev.cacheWriteTokens + (info.cacheWriteTokens ?? 0),
+        costUsd: prev.costUsd + info.costUsd,
+      });
+    },
+    totals: () => new Map(map),
+    totalFor: (providerName, modelId) => map.get(keyOf(providerName, modelId)) ?? ZERO_TOTALS,
+    reset: () => map.clear(),
+  };
+}
+
+/**
+ * Options for {@link withCostTracking}.
+ *
+ * @stable
+ */
+export interface WithCostTrackingOptions {
+  /**
+   * Hook fired on every `finish` event with the parsed usage. The
+   * hook receives the underlying provider's name + modelId so the
+   * caller can route into a per-model accumulator.
+   */
+  readonly onUsage?: (info: {
+    readonly providerName: string;
+    readonly modelId: string;
+    readonly promptTokens: number;
+    readonly completionTokens: number;
+    readonly totalTokens: number;
+    readonly cachedReadTokens?: number;
+    readonly cacheWriteTokens?: number;
+    readonly costUsd: number;
+    readonly metadata: Readonly<Record<string, unknown>> | undefined;
+  }) => void;
+  /**
+   * Optional pricing lookup. When set, the middleware computes
+   * `costUsd` from the returned price and surfaces it on the hook.
+   *
+   * `cachedReadPerMtok` / `cacheWritePerMtok` price the prompt-cache legs
+   * (core-provider-02); when omitted, cache tokens are billed at the full
+   * input rate (never cheaper than reality, so absent price data degrades
+   * to the pre-cache behaviour).
+   */
+  readonly priceLookup?: (info: { readonly providerName: string; readonly modelId: string }) => {
+    readonly inputPerMtok?: number;
+    readonly outputPerMtok?: number;
+    readonly cachedReadPerMtok?: number;
+    readonly cacheWritePerMtok?: number;
+  } | null;
+}
+
+/**
+ * @stable
+ */
+export const withCostTracking = defineProviderMiddleware<WithCostTrackingOptions>({
+  kind: 'withCostTracking',
+  factory: (opts: WithCostTrackingOptions) => {
+    return (next: Provider): Provider => ({
+      name: next.name,
+      modelId: next.modelId,
+      capabilities: next.capabilities,
+      ...(next.acceptsSensitivity !== undefined
+        ? { acceptsSensitivity: next.acceptsSensitivity }
+        : {}),
+      stream(req) {
+        return trackingStream(next, req, opts);
+      },
+      async generate(req) {
+        const result = await next.generate(req);
+        emitUsage(result, next, opts, asRecord(req.metadata));
+        return result;
+      },
+      ...(next.countTokens ? { countTokens: next.countTokens.bind(next) } : {}),
+    });
+  },
+});
+
+async function* trackingStream(
+  next: Provider,
+  req: import('@graphorin/core').ProviderRequest,
+  opts: WithCostTrackingOptions,
+): AsyncIterable<ProviderEvent> {
+  const metadata = asRecord(req.metadata);
+  for await (const event of next.stream(req)) {
+    yield event;
+    if (event.type === 'finish') {
+      emitUsage({ usage: event.usage, finishReason: event.finishReason }, next, opts, metadata);
+    }
+  }
+}
+
+function asRecord(value: unknown): Readonly<Record<string, unknown>> | undefined {
+  if (value === undefined || value === null) return undefined;
+  return value as Readonly<Record<string, unknown>>;
+}
+
+function emitUsage(
+  result:
+    | ProviderResponse
+    | { usage: ProviderResponse['usage']; finishReason: ProviderResponse['finishReason'] },
+  next: Provider,
+  opts: WithCostTrackingOptions,
+  metadata: Readonly<Record<string, unknown>> | undefined,
+): void {
+  if (opts.onUsage === undefined) return;
+  const usage = result.usage;
+  const price = opts.priceLookup?.({ providerName: next.name, modelId: next.modelId }) ?? null;
+  // Reasoning tokens are billed at the output rate (the PS-19 contract
+  // in @graphorin/pricing) - providers that report them separately from
+  // completionTokens must not get them for free.
+  // Cache legs (core-provider-02): promptTokens INCLUDES cache reads and
+  // writes, so subtract them from the base-rate leg and bill each at its
+  // own rate; a missing cache rate falls back to the full input rate.
+  const cachedRead = usage.cachedReadTokens ?? 0;
+  const cacheWrite = usage.cacheWriteTokens ?? 0;
+  const basePromptTokens = Math.max(0, usage.promptTokens - cachedRead - cacheWrite);
+  const inputRate = price?.inputPerMtok ?? 0;
+  const costUsd =
+    price !== null
+      ? (basePromptTokens * inputRate +
+          cachedRead * (price.cachedReadPerMtok ?? inputRate) +
+          cacheWrite * (price.cacheWritePerMtok ?? inputRate) +
+          (usage.completionTokens + (usage.reasoningTokens ?? 0)) * (price.outputPerMtok ?? 0)) /
+        1_000_000
+      : (usage.cost?.amount ?? 0);
+  opts.onUsage({
+    providerName: next.name,
+    modelId: next.modelId,
+    promptTokens: usage.promptTokens,
+    completionTokens: usage.completionTokens,
+    totalTokens: usage.totalTokens,
+    ...(usage.cachedReadTokens !== undefined ? { cachedReadTokens: usage.cachedReadTokens } : {}),
+    ...(usage.cacheWriteTokens !== undefined ? { cacheWriteTokens: usage.cacheWriteTokens } : {}),
+    costUsd,
+    metadata,
+  });
+}