@llm4ts/core 2.54.1 → 3.1.0

package/src/RateLimiter.ts

@@ -1,238 +0,0 @@
-import * as Clock from "effect/Clock"
-import * as Context from "effect/Context"
-import * as Duration from "effect/Duration"
-import * as Effect from "effect/Effect"
-import * as Layer from "effect/Layer"
-import * as Ref from "effect/Ref"
-import * as Schema from "effect/Schema"
-import type { LlmConfig } from "./Models.ts"
-
-export class RateLimiterAcquireTimeout extends Schema.TaggedError<RateLimiterAcquireTimeout>()(
-  "AcquireTimeout",
-  {
-    timeout: Schema.Duration
-  }
-) {}
-
-export class RateLimiterInvalidConfig extends Schema.TaggedError<RateLimiterInvalidConfig>()(
-  "InvalidConfig",
-  {
-    details: Schema.String
-  }
-) {}
-
-export const RateLimiterError = Schema.Union([RateLimiterAcquireTimeout, RateLimiterInvalidConfig])
-export type RateLimiterError = typeof RateLimiterError.Type
-
-export class RateLimiterConfig extends Schema.Class<RateLimiterConfig>("RateLimiterConfig")({
-  requestsPerMinute: Schema.Int.pipe(Schema.withConstructorDefault(Effect.succeed(60))),
-  burstSize: Schema.Int.pipe(Schema.withConstructorDefault(Effect.succeed(10))),
-  acquireTimeout: Schema.Duration.pipe(
-    Schema.withConstructorDefault(Effect.succeed(Duration.seconds(30)))
-  )
-}) {}
-
-export const rateLimiterConfigFromLlmConfig = (config: LlmConfig): RateLimiterConfig =>
-  RateLimiterConfig.make({
-    requestsPerMinute: config.requestsPerMinute,
-    burstSize: config.burstSize,
-    acquireTimeout: config.acquireTimeout
-  })
-
-export class RateLimiterMetrics extends Schema.Class<RateLimiterMetrics>("RateLimiterMetrics")({
-  totalRequests: Schema.Int,
-  throttledRequests: Schema.Int,
-  currentTokens: Schema.Int
-}) {}
-
-export interface RateLimiterShape {
-  readonly acquire: Effect.Effect<void, RateLimiterError>
-  readonly tryAcquire: Effect.Effect<boolean>
-  readonly metrics: Effect.Effect<RateLimiterMetrics>
-}
-
-export class RateLimiter extends Context.Service<RateLimiter, RateLimiterShape>()(
-  "@llm4ts/core/RateLimiter"
-) {}
-
-interface BucketState {
-  readonly tokens: number
-  readonly lastRefillNanos: bigint
-}
-
-interface TakeResult {
-  readonly acquired: boolean
-  readonly tokensAfter: number
-  readonly deficit: number
-}
-
-const nanosPerSecond = 1_000_000_000
-
-const configDetails = (config: RateLimiterConfig): string =>
-  `requestsPerMinute=${config.requestsPerMinute}, burstSize=${config.burstSize}, ` +
-  `acquireTimeout=${Duration.format(config.acquireTimeout)}`
-
-const isConfigValid = (config: RateLimiterConfig): boolean =>
-  config.requestsPerMinute > 0 &&
-  config.burstSize > 0 &&
-  Duration.toMillis(config.acquireTimeout) > 0
-
-export const makeRateLimiter = Effect.fn("@llm4ts/core/RateLimiter.make")(function* (
-  config: RateLimiterConfig
-) {
-  const initialNanos = yield* Clock.currentTimeNanos
-  const state = yield* Ref.make<BucketState>({
-    tokens: config.burstSize,
-    lastRefillNanos: initialNanos
-  })
-  const metrics = yield* Ref.make(
-    RateLimiterMetrics.make({
-      totalRequests: 0,
-      throttledRequests: 0,
-      currentTokens: config.burstSize
-    })
-  )
-  const capacity = config.burstSize
-  const ratePerSecond = config.requestsPerMinute / 60
-
-  const takeToken: Effect.Effect<TakeResult> = Effect.gen(function* () {
-    const now = yield* Clock.currentTimeNanos
-
-    return yield* Ref.modify(state, (current) => {
-      const elapsedNanos = now - current.lastRefillNanos
-      const tokensToAdd =
-        elapsedNanos <= 0n ? 0 : (Number(elapsedNanos) / nanosPerSecond) * ratePerSecond
-      const refilled =
-        tokensToAdd <= 0 ? current.tokens : Math.min(capacity, current.tokens + tokensToAdd)
-      const lastRefillNanos = tokensToAdd <= 0 ? current.lastRefillNanos : now
-
-      if (refilled >= 1) {
-        const remaining = refilled - 1
-        return [
-          {
-            acquired: true,
-            tokensAfter: remaining,
-            deficit: 0
-          },
-          {
-            tokens: remaining,
-            lastRefillNanos
-          }
-        ]
-      }
-
-      return [
-        {
-          acquired: false,
-          tokensAfter: refilled,
-          deficit: 1 - refilled
-        },
-        {
-          tokens: refilled,
-          lastRefillNanos
-        }
-      ]
-    })
-  })
-
-  const updateCurrentTokens = (tokens: number): Effect.Effect<void> =>
-    Ref.update(metrics, (current) =>
-      RateLimiterMetrics.make({
-        ...current,
-        currentTokens: Math.trunc(tokens)
-      })
-    )
-
-  const waitForToken = (deficit: number): Effect.Effect<void, RateLimiterInvalidConfig> => {
-    if (ratePerSecond <= 0) {
-      return Effect.fail(
-        RateLimiterInvalidConfig.make({
-          details: configDetails(config)
-        })
-      )
-    }
-
-    const nanoseconds = BigInt(Math.max(0, Math.ceil((deficit / ratePerSecond) * nanosPerSecond)))
-    return Effect.sleep(Duration.nanos(nanoseconds))
-  }
-
-  const acquireLoop = (alreadyThrottled: boolean): Effect.Effect<void, RateLimiterError> =>
-    Effect.gen(function* () {
-      const result = yield* takeToken
-      yield* updateCurrentTokens(result.tokensAfter)
-
-      if (result.acquired) {
-        return
-      }
-
-      if (!alreadyThrottled) {
-        yield* Ref.update(metrics, (current) =>
-          RateLimiterMetrics.make({
-            ...current,
-            throttledRequests: current.throttledRequests + 1
-          })
-        )
-      }
-
-      yield* waitForToken(result.deficit)
-      return yield* Effect.suspend(() => acquireLoop(true))
-    })
-
-  const acquire: Effect.Effect<void, RateLimiterError> = Effect.suspend(() => {
-    if (!isConfigValid(config)) {
-      return Effect.fail(
-        RateLimiterInvalidConfig.make({
-          details: configDetails(config)
-        })
-      )
-    }
-
-    return Ref.update(metrics, (current) =>
-      RateLimiterMetrics.make({
-        ...current,
-        totalRequests: current.totalRequests + 1
-      })
-    ).pipe(
-      Effect.andThen(acquireLoop(false)),
-      Effect.timeoutOrElse({
-        duration: config.acquireTimeout,
-        orElse: () =>
-          Effect.fail(
-            RateLimiterAcquireTimeout.make({
-              timeout: config.acquireTimeout
-            })
-          )
-      })
-    )
-  })
-
-  const tryAcquire: Effect.Effect<boolean> = Effect.suspend(() => {
-    if (!isConfigValid(config)) {
-      return Effect.logWarning(`Rate limiter config invalid: ${configDetails(config)}`).pipe(
-        Effect.as(false)
-      )
-    }
-
-    return Effect.gen(function* () {
-      const result = yield* takeToken
-      yield* updateCurrentTokens(result.tokensAfter)
-      yield* Ref.update(metrics, (current) =>
-        RateLimiterMetrics.make({
-          ...current,
-          totalRequests: current.totalRequests + 1,
-          throttledRequests: current.throttledRequests + (result.acquired ? 0 : 1)
-        })
-      )
-      return result.acquired
-    })
-  })
-
-  return RateLimiter.of({
-    acquire,
-    tryAcquire,
-    metrics: Ref.get(metrics)
-  })
-})
-
-export const rateLimiterLayer = (config: RateLimiterConfig): Layer.Layer<RateLimiter> =>
-  Layer.effect(RateLimiter, makeRateLimiter(config))

package/src/observability/MeteredLlmService.ts

@@ -1,168 +0,0 @@
-/**
- * @deprecated Since 2.30.0 (ADR 0026): llm4ts exports OpenTelemetry spans and
- * metrics over OTLP from `@llm4ts/runner/Otel`; this in-memory module is not
- * used by flow or runner and will be removed in the next major.
- */
-import * as Clock from "effect/Clock"
-import * as Effect from "effect/Effect"
-import * as Ref from "effect/Ref"
-import * as Schema from "effect/Schema"
-import * as Stream from "effect/Stream"
-import type { LlmError } from "../Errors.ts"
-import type { LlmServiceShape, StructuredResult } from "../LlmService.ts"
-import type { LlmChunk, TokenUsage } from "../Models.ts"
-import { RequestMetrics } from "./Metrics.ts"
-import type { MetricsCollectorShape, RequestLabels } from "./Metrics.ts"
-
-export class LlmPricing extends Schema.Class<LlmPricing>("LlmPricing")({
-  inputUsdPer1k: Schema.Number,
-  outputUsdPer1k: Schema.Number
-}) {}
-
-export interface MeteredLlmOptions {
-  readonly labels: RequestLabels
-  readonly pricing?: LlmPricing
-}
-
-const estimatedCost = (
-  usage: TokenUsage | undefined,
-  pricing: LlmPricing | undefined
-): number | undefined =>
-  usage === undefined || pricing === undefined
-    ? undefined
-    : (usage.prompt / 1_000) * pricing.inputUsdPer1k +
-      (usage.completion / 1_000) * pricing.outputUsdPer1k
-
-const complete = (
-  collector: MetricsCollectorShape,
-  options: MeteredLlmOptions,
-  startedAt: number,
-  endedAt: number,
-  success: boolean,
-  errorType: string | undefined,
-  usage: TokenUsage | undefined
-): Effect.Effect<void> => {
-  const cost = estimatedCost(usage, options.pricing)
-  return collector.recordCompleted(
-    RequestMetrics.make({
-      labels: options.labels,
-      latencyMs: Math.max(0, endedAt - startedAt),
-      success,
-      ...(errorType === undefined ? {} : { errorType }),
-      ...(usage === undefined ? {} : { tokenUsage: usage }),
-      ...(cost === undefined ? {} : { estimatedCostUsd: cost })
-    })
-  )
-}
-
-const meterEffect = <A, R>(
-  effect: Effect.Effect<A, LlmError, R>,
-  collector: MetricsCollectorShape,
-  options: MeteredLlmOptions,
-  usageOf: (value: A) => TokenUsage | undefined
-): Effect.Effect<A, LlmError, R> =>
-  Effect.gen(function* () {
-    const startedAt = yield* Clock.currentTimeMillis
-    yield* collector.markRequestStarted(options.labels)
-    const result = yield* Effect.result(effect)
-    const endedAt = yield* Clock.currentTimeMillis
-    yield* complete(
-      collector,
-      options,
-      startedAt,
-      endedAt,
-      result._tag === "Success",
-      result._tag === "Failure" ? result.failure._tag : undefined,
-      result._tag === "Success" ? usageOf(result.success) : undefined
-    )
-    return yield* Effect.fromResult(result)
-  })
-
-interface StreamMeasurement {
-  readonly success: boolean
-  readonly errorType: string | undefined
-  readonly usage: TokenUsage | undefined
-}
-
-const meterStream = (
-  stream: Stream.Stream<LlmChunk, LlmError>,
-  collector: MetricsCollectorShape,
-  options: MeteredLlmOptions
-): Stream.Stream<LlmChunk, LlmError> =>
-  Stream.unwrap(
-    Effect.gen(function* () {
-      const startedAt = yield* Clock.currentTimeMillis
-      yield* collector.markRequestStarted(options.labels)
-      const measurement = yield* Ref.make<StreamMeasurement>({
-        success: true,
-        errorType: undefined,
-        usage: undefined
-      })
-      return stream.pipe(
-        Stream.tap((chunk) =>
-          chunk.usage === undefined
-            ? Effect.void
-            : Ref.update(measurement, (current) => ({
-                ...current,
-                usage: chunk.usage
-              }))
-        ),
-        Stream.tapError((error) =>
-          Ref.update(measurement, (current) => ({
-            ...current,
-            success: false,
-            errorType: error._tag
-          }))
-        ),
-        Stream.ensuring(
-          Effect.gen(function* () {
-            const current = yield* Ref.get(measurement)
-            const endedAt = yield* Clock.currentTimeMillis
-            yield* complete(
-              collector,
-              options,
-              startedAt,
-              endedAt,
-              current.success,
-              current.errorType,
-              current.usage
-            )
-          })
-        )
-      )
-    })
-  )
-
-export const meterLlmService = (
-  service: LlmServiceShape,
-  collector: MetricsCollectorShape,
-  options: MeteredLlmOptions
-): LlmServiceShape => ({
-  executeStream: (prompt) => meterStream(service.executeStream(prompt), collector, options),
-  executeStreamWithHistory: (messages) =>
-    meterStream(service.executeStreamWithHistory(messages), collector, options),
-  executeWithTools: (prompt, tools) =>
-    meterEffect(service.executeWithTools(prompt, tools), collector, options, () => undefined),
-  executeStructured: (prompt, schema, jsonSchema) =>
-    meterEffect(
-      service.executeStructured(prompt, schema, jsonSchema),
-      collector,
-      options,
-      () => undefined
-    ),
-  executeStructuredWithUsage: (prompt, schema, jsonSchema) =>
-    meterEffect(
-      service.executeStructuredWithUsage(prompt, schema, jsonSchema),
-      collector,
-      options,
-      <A>(result: StructuredResult<A>) => result[1]
-    ),
-  scoreLabels: (prompt, labels) =>
-    meterEffect(
-      service.scoreLabels(prompt, labels),
-      collector,
-      options,
-      (distribution) => distribution.usage
-    ),
-  isAvailable: service.isAvailable
-})

package/src/observability/Metrics.ts

@@ -1,200 +0,0 @@
-/**
- * @deprecated Since 2.30.0 (ADR 0026): llm4ts exports OpenTelemetry spans and
- * metrics over OTLP from `@llm4ts/runner/Otel`; this in-memory module is not
- * used by flow or runner and will be removed in the next major.
- */
-import * as Context from "effect/Context"
-import * as Effect from "effect/Effect"
-import * as Layer from "effect/Layer"
-import * as Ref from "effect/Ref"
-import * as Schema from "effect/Schema"
-import { TokenUsage } from "../Models.ts"
-
-export class RequestLabels extends Schema.Class<RequestLabels>("RequestLabels")({
-  provider: Schema.String,
-  model: Schema.String,
-  agent: Schema.optionalKey(Schema.String),
-  runId: Schema.optionalKey(Schema.String),
-  workflowStep: Schema.optionalKey(Schema.String)
-}) {}
-
-export class RequestMetrics extends Schema.Class<RequestMetrics>("RequestMetrics")({
-  labels: RequestLabels,
-  tokenUsage: Schema.optionalKey(TokenUsage),
-  latencyMs: Schema.Number,
-  success: Schema.Boolean,
-  errorType: Schema.optionalKey(Schema.String),
-  estimatedCostUsd: Schema.optionalKey(Schema.Number)
-}) {}
-
-export class ProviderHealth extends Schema.Class<ProviderHealth>("ProviderHealth")({
-  provider: Schema.String,
-  requestCount: Schema.Int,
-  successRate: Schema.Number,
-  avgLatencyMs: Schema.Number,
-  p95LatencyMs: Schema.Number,
-  estimatedCostUsd: Schema.Number
-}) {}
-
-export class ObservabilitySnapshot extends Schema.Class<ObservabilitySnapshot>(
-  "ObservabilitySnapshot"
-)({
-  totalRequests: Schema.Int,
-  totalErrors: Schema.Int,
-  activeRequests: Schema.Int,
-  promptTokens: Schema.Int,
-  completionTokens: Schema.Int,
-  totalTokens: Schema.Int,
-  estimatedCostUsd: Schema.Number,
-  p50LatencyMs: Schema.Number,
-  p95LatencyMs: Schema.Number,
-  byProvider: Schema.Array(ProviderHealth),
-  byAgentRequests: Schema.Record(Schema.String, Schema.Int)
-}) {}
-
-export class DashboardMetricsSnapshot extends Schema.Class<DashboardMetricsSnapshot>(
-  "DashboardMetricsSnapshot"
-)({
-  providerHealth: Schema.Array(ProviderHealth),
-  totalTokens: Schema.Int,
-  totalCostUsd: Schema.Number,
-  totalRequests: Schema.Int,
-  errorRate: Schema.Number
-}) {}
-
-export interface MetricsCollectorShape {
-  readonly markRequestStarted: (labels: RequestLabels) => Effect.Effect<void>
-  readonly recordCompleted: (metrics: RequestMetrics) => Effect.Effect<void>
-  readonly snapshot: Effect.Effect<ObservabilitySnapshot>
-  readonly dashboardSnapshot: Effect.Effect<DashboardMetricsSnapshot>
-}
-
-export class MetricsCollector extends Context.Service<MetricsCollector, MetricsCollectorShape>()(
-  "@llm4ts/core/observability/MetricsCollector"
-) {}
-
-interface MetricsState {
-  readonly totalRequests: number
-  readonly activeRequests: number
-  readonly completed: ReadonlyArray<RequestMetrics>
-  readonly byAgentRequests: ReadonlyMap<string, number>
-}
-
-const percentile = (values: ReadonlyArray<number>, quantile: number): number => {
-  if (values.length === 0) {
-    return 0
-  }
-  const sorted = [...values].sort((left, right) => left - right)
-  const index = Math.max(0, Math.ceil(quantile * sorted.length) - 1)
-  return sorted[index] ?? 0
-}
-
-const providerHealth = (
-  completed: ReadonlyArray<RequestMetrics>
-): ReadonlyArray<ProviderHealth> => {
-  const providers = new Map<string, Array<RequestMetrics>>()
-  for (const metric of completed) {
-    const current = providers.get(metric.labels.provider) ?? []
-    current.push(metric)
-    providers.set(metric.labels.provider, current)
-  }
-  return [...providers.entries()]
-    .sort(([left], [right]) => left.localeCompare(right))
-    .map(([provider, metrics]) => {
-      const latencies = metrics.map((metric) => metric.latencyMs)
-      const successes = metrics.filter((metric) => metric.success).length
-      return ProviderHealth.make({
-        provider,
-        requestCount: metrics.length,
-        successRate: metrics.length === 0 ? 1 : successes / metrics.length,
-        avgLatencyMs:
-          metrics.length === 0
-            ? 0
-            : latencies.reduce((sum, latency) => sum + latency, 0) / metrics.length,
-        p95LatencyMs: percentile(latencies, 0.95),
-        estimatedCostUsd: metrics.reduce((sum, metric) => sum + (metric.estimatedCostUsd ?? 0), 0)
-      })
-    })
-}
-
-const toSnapshot = (state: MetricsState): ObservabilitySnapshot => {
-  const completed = state.completed
-  const promptTokens = completed.reduce((sum, metric) => sum + (metric.tokenUsage?.prompt ?? 0), 0)
-  const completionTokens = completed.reduce(
-    (sum, metric) => sum + (metric.tokenUsage?.completion ?? 0),
-    0
-  )
-  return ObservabilitySnapshot.make({
-    totalRequests: state.totalRequests,
-    totalErrors: completed.filter((metric) => !metric.success).length,
-    activeRequests: state.activeRequests,
-    promptTokens,
-    completionTokens,
-    totalTokens: promptTokens + completionTokens,
-    estimatedCostUsd: completed.reduce((sum, metric) => sum + (metric.estimatedCostUsd ?? 0), 0),
-    p50LatencyMs: percentile(
-      completed.map((metric) => metric.latencyMs),
-      0.5
-    ),
-    p95LatencyMs: percentile(
-      completed.map((metric) => metric.latencyMs),
-      0.95
-    ),
-    byProvider: providerHealth(completed),
-    byAgentRequests: Object.fromEntries(state.byAgentRequests)
-  })
-}
-
-export const makeMetricsCollector = Effect.fn("@llm4ts/core/observability/MetricsCollector.make")(
-  function* (): Effect.fn.Return<MetricsCollectorShape> {
-    const state = yield* Ref.make<MetricsState>({
-      totalRequests: 0,
-      activeRequests: 0,
-      completed: [],
-      byAgentRequests: new Map()
-    })
-
-    const markRequestStarted = (labels: RequestLabels): Effect.Effect<void> =>
-      Ref.update(state, (current) => {
-        const agents = new Map(current.byAgentRequests)
-        if (labels.agent !== undefined) {
-          agents.set(labels.agent, (agents.get(labels.agent) ?? 0) + 1)
-        }
-        return {
-          totalRequests: current.totalRequests + 1,
-          activeRequests: current.activeRequests + 1,
-          completed: current.completed,
-          byAgentRequests: agents
-        }
-      })
-
-    const recordCompleted = (metrics: RequestMetrics): Effect.Effect<void> =>
-      Ref.update(state, (current) => ({
-        ...current,
-        activeRequests: Math.max(0, current.activeRequests - 1),
-        completed: [...current.completed, metrics]
-      }))
-
-    const snapshot = Ref.get(state).pipe(Effect.map(toSnapshot))
-    const dashboardSnapshot = snapshot.pipe(
-      Effect.map((current) =>
-        DashboardMetricsSnapshot.make({
-          providerHealth: current.byProvider,
-          totalTokens: current.totalTokens,
-          totalCostUsd: current.estimatedCostUsd,
-          totalRequests: current.totalRequests,
-          errorRate: current.totalRequests === 0 ? 0 : current.totalErrors / current.totalRequests
-        })
-      )
-    )
-
-    return {
-      markRequestStarted,
-      recordCompleted,
-      snapshot,
-      dashboardSnapshot
-    }
-  }
-)
-
-export const MetricsCollectorLive = Layer.effect(MetricsCollector, makeMetricsCollector())

package/src/observability/StructuredLogger.ts

@@ -1,67 +0,0 @@
-/**
- * @deprecated Since 2.30.0 (ADR 0026): llm4ts exports OpenTelemetry spans and
- * metrics over OTLP from `@llm4ts/runner/Otel`; this in-memory module is not
- * used by flow or runner and will be removed in the next major.
- */
-import * as Clock from "effect/Clock"
-import * as Context from "effect/Context"
-import * as Effect from "effect/Effect"
-import * as Layer from "effect/Layer"
-import * as Schema from "effect/Schema"
-import type { JsonValue } from "../providers/CliSupport.ts"
-import { redactText, sanitizeFields, type RedactionOptions } from "./Redaction.ts"
-import type { TracingServiceShape } from "./Tracing.ts"
-
-export const LogLevel = Schema.Literals(["Debug", "Info", "Warning", "Error"])
-export type LogLevel = typeof LogLevel.Type
-
-export class StructuredLogEvent extends Schema.Class<StructuredLogEvent>("StructuredLogEvent")({
-  level: LogLevel,
-  message: Schema.String,
-  correlationId: Schema.String,
-  timestamp: Schema.Number,
-  fields: Schema.Record(Schema.String, Schema.Json)
-}) {}
-
-export interface StructuredLoggerShape {
-  readonly log: (
-    level: LogLevel,
-    message: string,
-    fields?: Readonly<Record<string, JsonValue>>
-  ) => Effect.Effect<void>
-}
-
-export class StructuredLogger extends Context.Service<StructuredLogger, StructuredLoggerShape>()(
-  "@llm4ts/core/observability/StructuredLogger"
-) {}
-
-export interface StructuredLoggerOptions extends RedactionOptions {
-  readonly sink: (event: StructuredLogEvent) => Effect.Effect<void>
-}
-
-export const makeStructuredLogger = (
-  tracing: TracingServiceShape,
-  options: StructuredLoggerOptions
-): StructuredLoggerShape => ({
-  log: Effect.fn("@llm4ts/core/observability/StructuredLogger.log")(function* (
-    level: LogLevel,
-    message: string,
-    fields: Readonly<Record<string, JsonValue>> = {}
-  ) {
-    const correlationId = yield* tracing.correlationId
-    const timestamp = yield* Clock.currentTimeMillis
-    yield* options.sink(
-      StructuredLogEvent.make({
-        level,
-        message: redactText(message, options),
-        correlationId,
-        timestamp,
-        fields: sanitizeFields(fields, options)
-      })
-    )
-  })
-})
-
-export const StructuredLoggerNoop = Layer.succeed(StructuredLogger)({
-  log: (_level, _message, _fields) => Effect.void
-})