floe-guard 0.15.7 → 0.16.0

package/README.md

@@ -34,6 +34,79 @@ The middleware sits in the call path: it `check()`s before `doGenerate` /
 `doStream` (throwing `BudgetExceeded` to halt the run) and `record()`s priced
 token usage after — for streaming it reads usage from the `finish` part.
 
+## Mid-stream budget enforcement
+
+`StreamGuard` and `guardStream` port Python's streaming USD guard. They price
+each text delta before it reaches the consumer. When a chunk crosses the
+shared ceiling, its partial spend is recorded in `guard.spendLog` **before**
+`BudgetExceeded` is thrown. Active streams on the same guard share the ceiling.
+
+```ts
+import { BudgetGuard, guardStream } from "floe-guard";
+
+const guard = new BudgetGuard(0.01);
+// textDeltas is your provider's Iterable<string> or AsyncIterable<string>.
+for await (const text of guardStream(guard, "gpt-4o", textDeltas)) {
+  consume(text);
+}
+```
+
+For structured chunks, supply `{ getText: chunk => chunk.delta.text ?? "" }`
+using your provider's actual shape. Without an extractor, non-string chunks
+throw instead of silently recording zero. Synchronous inputs return a
+synchronous iterator; asynchronous inputs return an asynchronous iterator.
+
+Use `StreamGuard` directly when the provider reports final usage:
+
+```ts
+import { StreamGuard } from "floe-guard";
+
+const reserved = guard.reserve(guard.estimateCall("gpt-4o", 100, 200));
+const stream = new StreamGuard(guard, "gpt-4o", { promptTokens: 100, reserved });
+try {
+  for await (const text of textDeltas) {
+    stream.feedText(text); // or feedTokens(n) with known per-chunk counts
+    consume(text);
+  }
+  stream.finish({ promptTokens: reportedPromptTokens, completionTokens: reportedCompletionTokens });
+} finally {
+  stream.close(); // idempotent; settles estimates if finish() was not reached
+}
+```
+
+- Options: `promptTokens`, `reserved`, `price`, `label`, and `countTokens(delta)`.
+  `guardStream` additionally accepts `getText(chunk)`.
+  Configuration is captured at construction; changing the caller's options or
+  manual price object later does not change an existing stream.
+- `approxTokens` defaults to roughly four Unicode characters per token, with
+  a minimum of one for a non-empty delta. A custom tokenizer can replace it.
+  `finish()` reconciles to reported usage; `completionTokens` exposes the running
+  estimate. Mid-stream checks cover the aggregate USD ceiling, matching Python;
+  token reservations are reconciled at settlement.
+- An unpriceable model fails at construction and releases its reservation.
+  With `failClosed: false`, unpriceable streams pass through and settlement
+  warns and skips accounting, matching the existing guard policy.
+- The wrapper settles on exhaustion, source/consumer errors, and early `break`,
+  and closes the source iterator when iteration ends early. Direct users must
+  call `close()` in `finally`. If a wrapper is **never iterated**, its reservation
+  remains the caller's responsibility: call `guard.release(reserved)`.
+- The crossing chunk has already been generated. With accurate counts, the
+  local cutoff can overshoot by that chunk; heuristic error, parallel streams,
+  provider buffering and delayed cancellation can cause additional actual
+  spend. Stopping iteration requests iterator cleanup, not guaranteed remote
+  cancellation. Connect cleanup to your provider's abort mechanism where needed.
+- Existing Vapi, LiveKit and middleware streaming behavior is unchanged; this
+  is the standalone primitive requested in issue #124.
+
+Run the no-key example from a repository checkout:
+
+```bash
+cd js
+npm ci
+npm run build
+node ../examples/streaming_guard.mjs
+```
+
 ## Pricing
 
 Tokens are priced **offline** from a bundled

package/dist/adapters/livekit.d.cts

@@ -1,4 +1,4 @@
-import { B as BudgetGuard, a as BudgetExceeded } from '../guard-BPIqdSVt.cjs';
+import { B as BudgetGuard, a as BudgetExceeded } from '../guard-Bbs5OxAF.cjs';
 
 /**
  * LiveKit Agents adapter (optional peer: `npm i @livekit/agents`).

package/dist/adapters/livekit.d.ts

@@ -1,4 +1,4 @@
-import { B as BudgetGuard, a as BudgetExceeded } from '../guard-BPIqdSVt.js';
+import { B as BudgetGuard, a as BudgetExceeded } from '../guard-Bbs5OxAF.js';
 
 /**
  * LiveKit Agents adapter (optional peer: `npm i @livekit/agents`).

package/dist/adapters/retell.d.cts

@@ -1,4 +1,4 @@
-import { B as BudgetGuard, a as BudgetExceeded } from '../guard-BPIqdSVt.cjs';
+import { B as BudgetGuard, a as BudgetExceeded } from '../guard-Bbs5OxAF.cjs';
 
 /**
  * Retell custom-LLM WebSocket adapter (optional peer: the `ws` server your custom

package/dist/adapters/retell.d.ts

@@ -1,4 +1,4 @@
-import { B as BudgetGuard, a as BudgetExceeded } from '../guard-BPIqdSVt.js';
+import { B as BudgetGuard, a as BudgetExceeded } from '../guard-Bbs5OxAF.js';
 
 /**
  * Retell custom-LLM WebSocket adapter (optional peer: the `ws` server your custom

package/dist/adapters/vapi.d.cts

@@ -1,5 +1,5 @@
-import { B as BudgetGuard, F as FloeGuardError } from '../guard-BPIqdSVt.cjs';
-export { a as BudgetExceeded } from '../guard-BPIqdSVt.cjs';
+import { B as BudgetGuard, F as FloeGuardError } from '../guard-Bbs5OxAF.cjs';
+export { a as BudgetExceeded } from '../guard-Bbs5OxAF.cjs';
 
 /**
  * Vapi custom-LLM adapter (no SDK dependency — typed structurally).

package/dist/adapters/vapi.d.ts

@@ -1,5 +1,5 @@
-import { B as BudgetGuard, F as FloeGuardError } from '../guard-BPIqdSVt.js';
-export { a as BudgetExceeded } from '../guard-BPIqdSVt.js';
+import { B as BudgetGuard, F as FloeGuardError } from '../guard-Bbs5OxAF.js';
+export { a as BudgetExceeded } from '../guard-Bbs5OxAF.js';
 
 /**
  * Vapi custom-LLM adapter (no SDK dependency — typed structurally).

package/dist/{guard-BPIqdSVt.d.cts → guard-Bbs5OxAF.d.cts}

@@ -347,6 +347,7 @@ declare class BudgetGuard {
     private lastToolCost;
     /** USD held for in-flight calls (reserved, not yet settled). Counts toward the ceiling. */
     private reserved;
+    private readonly streamCosts;
     /** Per-call ledger, oldest first; a ring buffer when maxLogEvents is set. */
     private readonly spendEvents;
     private readonly maxLogEvents?;
@@ -506,7 +507,7 @@ declare class BudgetGuard {
      * before producing usage). Safe to call with `0`.
      */
     release(reserved: ReservationHandle): void;
-    /** USD left before the ceiling, net of in-flight reservations (never negative). */
+    /** USD left, net of reservations and unsettled stream overages (never negative). */
     get remainingUsd(): number;
     /**
      * Per-tool running USD totals, keyed by the name given to `settleTool()` /
@@ -624,6 +625,21 @@ declare class BudgetGuard {
      * (called outside the lock) reads no shared state and stays race-free.
      */
     private blockingCross;
+    /** Accrued stream costs beyond existing holds, excluding a stream if requested. */
+    private streamOverage;
+    /** @internal Validate before transferring a reservation to a stream. */
+    _validateStreamReservation(reserved: ReservationHandle): void;
+    /** @internal Register accrued-but-unsettled streaming spend. */
+    _registerStream(reserved: ReservationHandle): symbol;
+    /** @internal Settlement moved this stream's accrual into spentUsd. */
+    _unregisterStream(key: symbol): void;
+    /** @internal Replace this stream's estimate with its cumulative actual estimate.
+     * Count other streams' overages once, in addition to their existing holds.
+     * Synchronous within one JS isolate, matching Python's locked registry.
+     */
+    _streamWouldCross(key: symbol, cumulative: number): boolean;
+    /** @internal Notify and throw after a stream has settled its partial spend. */
+    _blockStream(): never;
     /** Notify + throw the right error for a [dimension, scope, spent, limit] block. */
     private raiseBlock;
     /**

package/dist/{guard-BPIqdSVt.d.ts → guard-Bbs5OxAF.d.ts}

@@ -347,6 +347,7 @@ declare class BudgetGuard {
     private lastToolCost;
     /** USD held for in-flight calls (reserved, not yet settled). Counts toward the ceiling. */
     private reserved;
+    private readonly streamCosts;
     /** Per-call ledger, oldest first; a ring buffer when maxLogEvents is set. */
     private readonly spendEvents;
     private readonly maxLogEvents?;
@@ -506,7 +507,7 @@ declare class BudgetGuard {
      * before producing usage). Safe to call with `0`.
      */
     release(reserved: ReservationHandle): void;
-    /** USD left before the ceiling, net of in-flight reservations (never negative). */
+    /** USD left, net of reservations and unsettled stream overages (never negative). */
     get remainingUsd(): number;
     /**
      * Per-tool running USD totals, keyed by the name given to `settleTool()` /
@@ -624,6 +625,21 @@ declare class BudgetGuard {
      * (called outside the lock) reads no shared state and stays race-free.
      */
     private blockingCross;
+    /** Accrued stream costs beyond existing holds, excluding a stream if requested. */
+    private streamOverage;
+    /** @internal Validate before transferring a reservation to a stream. */
+    _validateStreamReservation(reserved: ReservationHandle): void;
+    /** @internal Register accrued-but-unsettled streaming spend. */
+    _registerStream(reserved: ReservationHandle): symbol;
+    /** @internal Settlement moved this stream's accrual into spentUsd. */
+    _unregisterStream(key: symbol): void;
+    /** @internal Replace this stream's estimate with its cumulative actual estimate.
+     * Count other streams' overages once, in addition to their existing holds.
+     * Synchronous within one JS isolate, matching Python's locked registry.
+     */
+    _streamWouldCross(key: symbol, cumulative: number): boolean;
+    /** @internal Notify and throw after a stream has settled its partial spend. */
+    _blockStream(): never;
     /** Notify + throw the right error for a [dimension, scope, spent, limit] block. */
     private raiseBlock;
     /**

package/dist/index.cjs

@@ -26,12 +26,15 @@ __export(index_exports, {
   FloeGuardError: () => FloeGuardError,
   LatencyBudget: () => LatencyBudget,
   LedgerSyncError: () => LedgerSyncError,
+  StreamGuard: () => StreamGuard,
   TokenBudgetExceeded: () => TokenBudgetExceeded,
   UnpriceableModelError: () => UnpriceableModelError,
   UnpriceableVoiceError: () => UnpriceableVoiceError,
+  approxTokens: () => approxTokens,
   budgetGuardMiddleware: () => budgetGuardMiddleware,
   costMapGeneratedAt: () => costMapGeneratedAt,
   gates: () => gates_exports,
+  guardStream: () => guardStream,
   lookupVoiceRate: () => lookupVoiceRate,
   priceTokens: () => priceTokens,
   priceVoiceLeg: () => priceVoiceLeg,
@@ -1736,6 +1739,7 @@ var BudgetGuard = class {
   lastToolCost = 0;
   /** USD held for in-flight calls (reserved, not yet settled). Counts toward the ceiling. */
   reserved = 0;
+  streamCosts = /* @__PURE__ */ new Map();
   /** Per-call ledger, oldest first; a ring buffer when maxLogEvents is set. */
   spendEvents = [];
   maxLogEvents;
@@ -2056,9 +2060,9 @@ var BudgetGuard = class {
     if (!reserved) return;
     this.consumeReservation(reserved);
   }
-  /** USD left before the ceiling, net of in-flight reservations (never negative). */
+  /** USD left, net of reservations and unsettled stream overages (never negative). */
   get remainingUsd() {
-    return Math.max(0, this.limitUsd - this.spentUsd - this.reserved);
+    return Math.max(0, this.limitUsd - this.spentUsd - this.reserved - this.streamOverage());
   }
   /**
    * Per-tool running USD totals, keyed by the name given to `settleTool()` /
@@ -2261,7 +2265,7 @@ var BudgetGuard = class {
    * (called outside the lock) reads no shared state and stays race-free.
    */
   blockingCross(estimateUsd, estimateTokens) {
-    const committed = this.spentUsd + this.reserved;
+    const committed = this.spentUsd + this.reserved + this.streamOverage();
     if (committed > this.limitUsd - EPS || committed + estimateUsd > this.limitUsd + EPS) {
       return ["usd", "aggregate", this.spentUsd, this.limitUsd];
     }
@@ -2288,6 +2292,43 @@ var BudgetGuard = class {
     }
     return null;
   }
+  /** Accrued stream costs beyond existing holds, excluding a stream if requested. */
+  streamOverage(exclude) {
+    let overage = 0;
+    for (const [key, stream] of this.streamCosts) {
+      if (key !== exclude) overage += Math.max(0, stream.accrued - stream.held);
+    }
+    return overage;
+  }
+  /** @internal Validate before transferring a reservation to a stream. */
+  _validateStreamReservation(reserved) {
+    this.reservedUsdOf(reserved);
+  }
+  /** @internal Register accrued-but-unsettled streaming spend. */
+  _registerStream(reserved) {
+    const held = this.reservedUsdOf(reserved);
+    const key = /* @__PURE__ */ Symbol();
+    this.streamCosts.set(key, { accrued: 0, held });
+    return key;
+  }
+  /** @internal Settlement moved this stream's accrual into spentUsd. */
+  _unregisterStream(key) {
+    this.streamCosts.delete(key);
+  }
+  /** @internal Replace this stream's estimate with its cumulative actual estimate.
+   * Count other streams' overages once, in addition to their existing holds.
+   * Synchronous within one JS isolate, matching Python's locked registry.
+   */
+  _streamWouldCross(key, cumulative) {
+    const own = this.streamCosts.get(key);
+    own.accrued = cumulative;
+    const others = this.spentUsd + Math.max(0, this.reserved - own.held) + this.streamOverage(key);
+    return others + cumulative > this.limitUsd + EPS;
+  }
+  /** @internal Notify and throw after a stream has settled its partial spend. */
+  _blockStream() {
+    return this.raiseBlock(["usd", "aggregate", this.spentUsd, this.limitUsd]);
+  }
   /** Notify + throw the right error for a [dimension, scope, spent, limit] block. */
   raiseBlock(blocked) {
     const [dimension, scope, spent, limit] = blocked;
@@ -2635,6 +2676,130 @@ async function nextPlan(guard, error, current, onDegrade) {
   return current;
 }
 
+// src/stream.ts
+function approxTokens(text) {
+  if (typeof text !== "string") throw new TypeError("stream text must be a string");
+  return text ? Math.max(1, Math.floor(Array.from(text).length / 4)) : 0;
+}
+function tokenCount(value) {
+  if (!Number.isFinite(value)) throw new RangeError("token count must be finite");
+  return Math.max(0, Math.trunc(value));
+}
+var StreamGuard = class {
+  /** Capture pricing and release the supplied reservation if validation fails. */
+  constructor(guard, model, options = {}) {
+    this.guard = guard;
+    this.model = model;
+    this.reserved = options.reserved ?? 0;
+    guard._validateStreamReservation(this.reserved);
+    try {
+      this.price = options.price === void 0 ? void 0 : { ...options.price };
+      this.label = options.label;
+      this.countTokens = options.countTokens ?? approxTokens;
+      this.promptTokens = tokenCount(options.promptTokens ?? 0);
+      this.priced = resolvePrice(
+        model,
+        this.price === void 0 ? guard.priceOverrides : { ...guard.priceOverrides, [model]: this.price }
+      );
+      if (this.priced === null && guard.failClosed) {
+        console.warn(`Cannot price model '${model}': pass a price override to enforce a streaming budget.`);
+        throw new UnpriceableModelError(model);
+      }
+    } catch (error) {
+      guard.release(this.reserved);
+      throw error;
+    }
+  }
+  guard;
+  model;
+  tokens = 0;
+  closed = false;
+  priced;
+  key;
+  promptTokens;
+  reserved;
+  price;
+  label;
+  countTokens;
+  /** Generated completion tokens counted so far (estimated until finish()). */
+  get completionTokens() {
+    return this.tokens;
+  }
+  /** Meter a text delta with the configured tokenizer. */
+  feedText(delta) {
+    if (typeof delta !== "string") throw new TypeError("stream text must be a string");
+    const count = this.countTokens;
+    this.feedTokens(count(delta));
+  }
+  /** Meter additional completion tokens, settling before a budget interruption. */
+  feedTokens(tokens) {
+    if (this.closed) throw new Error("stream already settled");
+    this.tokens = tokenCount(this.tokens + tokenCount(tokens));
+    if (this.priced === null) return;
+    const cost = priceTokens(this.priced, this.promptTokens, this.tokens);
+    this.key ??= this.guard._registerStream(this.reserved);
+    if (this.guard._streamWouldCross(this.key, cost)) {
+      this.finish();
+      this.guard._blockStream();
+    }
+  }
+  /** Reconcile estimates to provider usage, or settle accumulated estimates. */
+  finish(usage = {}) {
+    if (this.closed) throw new Error("stream already settled");
+    const prompt = tokenCount(usage.promptTokens ?? this.promptTokens);
+    const completion = tokenCount(usage.completionTokens ?? this.tokens);
+    this.promptTokens = prompt;
+    this.tokens = completion;
+    this.closed = true;
+    try {
+      if (this.priced === null) {
+        this.guard.release(this.reserved);
+        console.warn(`Cannot price model '${this.model}' at stream construction; skipping streaming spend.`);
+        return 0;
+      }
+      return this.guard.settle(this.model, prompt, completion, {
+        reserved: this.reserved,
+        price: this.priced,
+        label: this.label
+      });
+    } finally {
+      if (this.key !== void 0) this.guard._unregisterStream(this.key);
+    }
+  }
+  /** Idempotent cleanup for a finally block; partial generated usage is billed. */
+  close() {
+    if (!this.closed) this.finish();
+  }
+};
+function guardStream(guard, model, chunks, options = {}) {
+  const stream = new StreamGuard(guard, model, options);
+  const extract = options.getText ?? ((chunk) => {
+    if (typeof chunk !== "string") throw new TypeError("guardStream needs getText for non-string chunks");
+    return chunk;
+  });
+  function* run(source) {
+    try {
+      for (const chunk of source) {
+        stream.feedText(extract(chunk));
+        yield chunk;
+      }
+    } finally {
+      stream.close();
+    }
+  }
+  async function* runAsync(source) {
+    try {
+      for await (const chunk of source) {
+        stream.feedText(extract(chunk));
+        yield chunk;
+      }
+    } finally {
+      stream.close();
+    }
+  }
+  return Symbol.asyncIterator in Object(chunks) ? runAsync(chunks) : run(chunks);
+}
+
 // src/voice-pricing.ts
 var VOICE_MAP = cost_map_default["__voice__"] ?? {};
 var unitForMode = {
@@ -2743,12 +2908,15 @@ function vapi(guard, options = {}) {
   FloeGuardError,
   LatencyBudget,
   LedgerSyncError,
+  StreamGuard,
   TokenBudgetExceeded,
   UnpriceableModelError,
   UnpriceableVoiceError,
+  approxTokens,
   budgetGuardMiddleware,
   costMapGeneratedAt,
   gates,
+  guardStream,
   lookupVoiceRate,
   priceTokens,
   priceVoiceLeg,