@effect-uai/core 0.11.0 → 0.12.0

package/src/observability/Metrics.ts

@@ -12,7 +12,7 @@ import {
   Stream,
 } from "effect"
 import type { Usage } from "../domain/Items.js"
-import type { Turn, TurnEvent } from "../domain/Turn.js"
+import { type Turn, TurnEvent } from "../domain/Turn.js"
 
 // ---------------------------------------------------------------------------
 // Self-describing measurements (the exporter-facing payload)
@@ -126,6 +126,7 @@ const MetricName = {
   totalTokens: "effect_uai_total_tokens",
   reasoningTokens: "effect_uai_reasoning_tokens",
   cachedInputTokens: "effect_uai_cached_input_tokens",
+  cacheWriteTokens: "effect_uai_cache_write_tokens",
 } as const
 
 /** Read a structural `_tag` off any value without narrowing to a closed union. */
@@ -141,6 +142,34 @@ const contentDeltaKind = (ev: unknown): Option.Option<"text" | "reasoning"> => {
       : Option.none()
 }
 
+const isTextDelta = TurnEvent.$is("TextDelta")
+const isReasoningDelta = TurnEvent.$is("ReasoningDelta")
+const isRefusalDelta = TurnEvent.$is("RefusalDelta")
+const isToolCallArgsDelta = TurnEvent.$is("ToolCallArgsDelta")
+
+/** A delta carrying generated output, whatever the model was producing. */
+export type OutputDelta = Extract<
+  TurnEvent,
+  { readonly _tag: "TextDelta" | "ReasoningDelta" | "RefusalDelta" | "ToolCallArgsDelta" }
+>
+
+/**
+ * Narrow to a delta that contributes to the model's output stream. Tool-call
+ * arguments count: an agent turn can be almost entirely `ToolCallArgsDelta`,
+ * and measuring only prose reports a near-zero rate for it.
+ *
+ * The single place the output-delta tag list lives — the throughput
+ * accumulator and `unitCount` both read it. Distinct from `contentDeltaKind`,
+ * which answers a narrower question for first-token latency.
+ */
+const outputDelta = (ev: unknown): Option.Option<OutputDelta> =>
+  isTextDelta(ev) || isReasoningDelta(ev) || isRefusalDelta(ev) || isToolCallArgsDelta(ev)
+    ? Option.some(ev)
+    : Option.none()
+
+const outputTextOf = (delta: OutputDelta): string =>
+  isToolCallArgsDelta(delta) ? delta.delta : delta.text
+
 const isTurnCompleteEvent = (ev: unknown): boolean => tagOf(ev) === "TurnComplete"
 
 const turnOf = (ev: unknown): Turn =>
@@ -154,6 +183,10 @@ const addUsage = (a: Usage, b: Usage): Usage => {
     a.input_tokens_details?.cached_tokens,
     b.input_tokens_details?.cached_tokens,
   )
+  const cacheWrite = sumOptional(
+    a.input_tokens_details?.cache_write_tokens,
+    b.input_tokens_details?.cache_write_tokens,
+  )
   const reasoning = sumOptional(
     a.output_tokens_details?.reasoning_tokens,
     b.output_tokens_details?.reasoning_tokens,
@@ -162,7 +195,13 @@ const addUsage = (a: Usage, b: Usage): Usage => {
     input_tokens: sumOptional(a.input_tokens, b.input_tokens),
     output_tokens: sumOptional(a.output_tokens, b.output_tokens),
     total_tokens: sumOptional(a.total_tokens, b.total_tokens),
-    input_tokens_details: cached === undefined ? undefined : { cached_tokens: cached },
+    input_tokens_details:
+      cached === undefined && cacheWrite === undefined
+        ? undefined
+        : {
+            ...(cached !== undefined && { cached_tokens: cached }),
+            ...(cacheWrite !== undefined && { cache_write_tokens: cacheWrite }),
+          },
     output_tokens_details: reasoning === undefined ? undefined : { reasoning_tokens: reasoning },
   }
 }
@@ -175,6 +214,7 @@ const usageMeasurements = (usage: Usage): ReadonlyArray<Measurement> => {
     [MetricName.totalTokens, usage.total_tokens],
     [MetricName.reasoningTokens, usage.output_tokens_details?.reasoning_tokens],
     [MetricName.cachedInputTokens, usage.input_tokens_details?.cached_tokens],
+    [MetricName.cacheWriteTokens, usage.input_tokens_details?.cache_write_tokens],
   ]
   return Arr.filterMap(pairs, ([name, value]) =>
     value === undefined
@@ -349,8 +389,11 @@ export type ThroughputOptions = {
   readonly every?: Duration.Input
   /** What a "unit" is. Default `"char"` (exact and chunking-independent). */
   readonly unit?: "char" | "token" | "event"
-  /** Token counter for `unit: "token"`. Absent => approximate 1 per content delta. */
-  readonly tokenizer?: (event: TurnEvent) => Effect.Effect<number>
+  /**
+   * Token counter for `unit: "token"`. Called only for deltas that carry
+   * generated output. Absent => approximate 1 per such delta.
+   */
+  readonly tokenizer?: (event: OutputDelta) => Effect.Effect<number>
   /** Rate definition. Default `"windowed"` (current speed, no cold-start bias). */
   readonly mode?: "windowed" | "cumulative"
   /** EWMA smoothing. Omitted => off; `"default"` => a sensible factor. */
@@ -434,14 +477,13 @@ const unitCount = (
   unit: "char" | "token" | "event",
   tokenizer: ThroughputOptions["tokenizer"],
 ): Effect.Effect<number> => {
-  if (tagOf(ev) !== "TextDelta") return Effect.succeed(0)
-  const text = (ev as { readonly text: string }).text
+  const output = outputDelta(ev)
+  if (Option.isNone(output)) return Effect.succeed(0)
+  const delta = output.value
   return Match.value(unit).pipe(
-    Match.when("char", () => Effect.succeed([...text].length)),
+    Match.when("char", () => Effect.succeed([...outputTextOf(delta)].length)),
     Match.when("event", () => Effect.succeed(1)),
-    Match.when("token", () =>
-      tokenizer === undefined ? Effect.succeed(1) : tokenizer(ev as TurnEvent),
-    ),
+    Match.when("token", () => (tokenizer === undefined ? Effect.succeed(1) : tokenizer(delta))),
     Match.exhaustive,
   )
 }
@@ -465,7 +507,7 @@ export const throughput =
           Stream.tap((ev) =>
             isTurnCompleteEvent(ev)
               ? Ref.update(ref, resetTurn)
-              : tagOf(ev) === "TextDelta"
+              : Option.isSome(outputDelta(ev))
                 ? Effect.flatMap(Clock.currentTimeMillis, (now) =>
                     Effect.flatMap(unitCount(ev, unit, tokenizer), (n) =>
                       Ref.update(ref, (state) => addUnits(state, n, now)),

package/src/retry/Retry.ts

@@ -29,14 +29,55 @@ import type * as AiError from "../domain/AiError.js"
  * variants if needed (e.g. a custom schedule that observes `cause`).
  */
 export class Retryable extends Data.TaggedError("RetryableAi")<{
-  readonly cause: AiError.RateLimited | AiError.Unavailable | AiError.Timeout
+  readonly cause:
+    | AiError.RateLimited
+    | AiError.Unavailable
+    | AiError.Timeout
+    | AiError.InvalidRequest
 }> {}
 
+/**
+ * `InvalidRequest` payloads that are a transient fault on the provider's side
+ * rather than a defect in the request.
+ *
+ * The class exists because a provider can only answer within its own error
+ * vocabulary: a constrained-decoding grammar that times out while compiling is
+ * a cold-start timeout, but it arrives as `invalid_request_error` because that
+ * is where the vendor files schema problems. Replaying it succeeds — the same
+ * schema compiles on the next call.
+ *
+ * An allow-list rather than a general softening of `InvalidRequest`, and it
+ * must stay one. The whole point of that tag being fatal is that a genuinely
+ * malformed request retried on a schedule burns quota to arrive at the same
+ * rejection, slowly. An entry earns its place by having been observed to
+ * succeed on replay.
+ */
+const TRANSIENT_INVALID_REQUEST: ReadonlyArray<RegExp> = [/grammar compilation timed out/i]
+
+/** The payload as text, for matching. `raw` is whatever the provider handed
+ *  back — usually its response body as a string, sometimes a parsed object,
+ *  and occasionally something that will not serialise. */
+const rawText = (raw: unknown): string => {
+  if (typeof raw === "string") return raw
+  try {
+    return JSON.stringify(raw) ?? ""
+  } catch {
+    return String(raw)
+  }
+}
+
+const isTransientInvalidRequest = (e: AiError.AiError): boolean =>
+  e._tag === "InvalidRequest" &&
+  TRANSIENT_INVALID_REQUEST.some((pattern) => pattern.test(rawText(e.raw)))
+
 /** Type-narrowing predicate for the retryable subset. */
 export const isRetryable = (
   e: AiError.AiError,
-): e is AiError.RateLimited | AiError.Unavailable | AiError.Timeout =>
-  e._tag === "RateLimited" || e._tag === "Unavailable" || e._tag === "Timeout"
+): e is AiError.RateLimited | AiError.Unavailable | AiError.Timeout | AiError.InvalidRequest =>
+  e._tag === "RateLimited" ||
+  e._tag === "Unavailable" ||
+  e._tag === "Timeout" ||
+  isTransientInvalidRequest(e)
 
 // Lift events to Items, non-retryable failures to Terminal values
 // (escape retry), retryable failures to wrapped errors (only thing

package/dist/Items-BWNu6VGp.d.mts.map

@@ -1 +0,0 @@
-{"version":3,"file":"Items-BWNu6VGp.d.mts","names":[],"sources":["../src/domain/Items.ts"],"mappings":";;;;;;;cAOa,SAAA,EAAS,MAAA,CAAA,MAAA;EAAA;;;KAIV,SAAA,UAAmB,SAAA,CAAU,IAAA;;;;;;;;;cAU5B,UAAA,EAAU,MAAA,CAAA,MAAA;EAAA;;;KAIX,UAAA,UAAoB,UAAA,CAAW,IAAA;;;;;;;;;cAgB9B,WAAA,EAAW,MAAA,CAAA,MAAA;EAAA;;;;;;;;KASZ,WAAA,UAAqB,WAAA,CAAY,IAAA;AAAA,cAEhC,YAAA,EAAY,MAAA,CAAA,MAAA;EAAA;;;;KAKb,YAAA,UAAsB,YAAA,CAAa,IAAA;AAAA,cAElC,qBAAA,EAAqB,MAAA,CAAA,MAAA;EAAA;;;;;;KAOtB,qBAAA,UAA+B,qBAAA,CAAsB,IAAA;AAAA,cAEpD,QAAA,EAAQ,MAAA,CAAA,MAAA;EAAA;;;;KAKT,QAAA,UAAkB,QAAA,CAAS,IAAA;AAAA,cAE1B,UAAA,EAAU,MAAA,CAAA,KAAA,WAAA,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;KACX,UAAA,UAAoB,UAAA,CAAW,IAAA;AAAA,cAE9B,aAAA,MAAa,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA;EAAA;;;;;;;;cACb,cAAA,MAAc,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA,GAAA,MAAA,CAAA,MAAA,CAAA,YAAA;EAAA;;;;cACd,uBAAA,MAAuB,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA,GAAA,MAAA,CAAA,MAAA,CAAA,YAAA;EAAA;;;;;;cACvB,UAAA,MAAU,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA,GAAA,MAAA,CAAA,MAAA,CAAA,YAAA;EAAA;;;;cAEV,UAAA,EAAU,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;KAKX,UAAA,UAAoB,UAAA,CAAW,IAAA;;;;;;AAtC3C;cA8Ca,OAAA,EAAO,MAAA,CAAA,MAAA;EAAA;;;KAIR,OAAA,UAAiB,OAAA,CAAQ,IAAA;AAAA,cAExB,YAAA,EAAY,MAAA,CAAA,KAAA,WAAA,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KACb,YAAA,UAAsB,YAAA,CAAa,IAAA;AAAA,cAElC,IAAA,EAAI,MAAA,CAAA,QAAA;AAAA,KACL,IAAA,UAAc,IAAA,CAAK,IAAA;AAAA,cAelB,OAAA,EAAO,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAMR,OAAA,UAAiB,OAAA,CAAQ,IAAA;AAAA,cAExB,QAAA,EAAQ,MAAA,CAAA,MAAA;EAAA;;;;;;KAQT,QAAA,UAAkB,QAAA,CAAS,IAAA;AAAA,cAE1B,cAAA,EAAc,MAAA,CAAA,MAAA;EAAA;;;;;KAMf,cAAA,UAAwB,cAAA,CAAe,IAAA;;;;;;;cAQtC,SAAA,EAAS,MAAA,CAAA,MAAA;EAAA;;;;;;KAOV,SAAA,UAAmB,SAAA,CAAU,IAAA;AAAA,cAE5B,WAAA,EAAW,MAAA,CAAA,KAAA,WAAA,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KACZ,WAAA,UAAqB,WAAA,CAAY,IAAA;AAAA,cAMhC,WAAA,MAAW,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA,GAAA,MAAA,CAAA,MAAA,CAAA,YAAA;EAAA;;;cACX,YAAA,MAAY,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA,GAAA,MAAA,CAAA,MAAA,CAAA,YAAA;EAAA;;;cACZ,YAAA,MAAY,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;cACZ,SAAA,MAAS,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA,GAAA,MAAA,CAAA,MAAA,CAAA,YAAA;EAAA;;;cAET,SAAA,MAAS,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA;EAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cACT,UAAA,MAAU,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA;EAAA;;;;;;cACV,gBAAA,MAAgB,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA;EAAA;;;;;cAChB,WAAA,MAAW,KAAA,EAAA,CAAA,KAAA,KAAA,IAAA,CAAA;EAAA;;;;;;cAMX,kBAAA,EAAkB,MAAA,CAAA,MAAA;EAAA;;KAGnB,kBAAA,UAA4B,kBAAA,CAAmB,IAAA;AAAA,cAE9C,mBAAA,EAAmB,MAAA,CAAA,MAAA;EAAA;;KAGpB,mBAAA,UAA6B,mBAAA,CAAoB,IAAA;AAAA,cAEhD,KAAA,EAAK,MAAA,CAAA,MAAA;EAAA;;;;;;;;;;KAON,KAAA,UAAe,KAAA,CAAM,IAAA;AAAA,cAEpB,UAAA,EAAU,MAAA,CAAA,QAAA;AAAA,KAUX,UAAA,UAAoB,UAAA,CAAW,IAAA;AAAA,cAM9B,QAAA,GAAY,IAAA,aAAe,OAAA;AAAA,cAM3B,UAAA,GAAc,IAAA,aAAe,OAAA;AAAA,cAM7B,aAAA,GAAiB,IAAA,aAAe,OAAA;AAAA,cAMhC,cAAA,GAAkB,OAAA,UAAiB,MAAA,aAAiB,cAAA"}