@llm4ts/core 0.1.0

package/src/observability/MeteredLlmService.ts

@@ -0,0 +1,156 @@
+import * as Clock from "effect/Clock"
+import * as Effect from "effect/Effect"
+import * as Ref from "effect/Ref"
+import * as Schema from "effect/Schema"
+import * as Stream from "effect/Stream"
+import type { LlmError } from "../Errors.ts"
+import type { LlmServiceShape, StructuredResult } from "../LlmService.ts"
+import type { LlmChunk, TokenUsage } from "../Models.ts"
+import { RequestMetrics } from "./Metrics.ts"
+import type { MetricsCollectorShape, RequestLabels } from "./Metrics.ts"
+
+export class LlmPricing extends Schema.Class<LlmPricing>("LlmPricing")({
+  inputUsdPer1k: Schema.Number,
+  outputUsdPer1k: Schema.Number
+}) {}
+
+export interface MeteredLlmOptions {
+  readonly labels: RequestLabels
+  readonly pricing?: LlmPricing
+}
+
+const estimatedCost = (
+  usage: TokenUsage | undefined,
+  pricing: LlmPricing | undefined
+): number | undefined =>
+  usage === undefined || pricing === undefined
+    ? undefined
+    : (usage.prompt / 1_000) * pricing.inputUsdPer1k +
+      (usage.completion / 1_000) * pricing.outputUsdPer1k
+
+const complete = (
+  collector: MetricsCollectorShape,
+  options: MeteredLlmOptions,
+  startedAt: number,
+  endedAt: number,
+  success: boolean,
+  errorType: string | undefined,
+  usage: TokenUsage | undefined
+): Effect.Effect<void> => {
+  const cost = estimatedCost(usage, options.pricing)
+  return collector.recordCompleted(
+    RequestMetrics.make({
+      labels: options.labels,
+      latencyMs: Math.max(0, endedAt - startedAt),
+      success,
+      ...(errorType === undefined ? {} : { errorType }),
+      ...(usage === undefined ? {} : { tokenUsage: usage }),
+      ...(cost === undefined ? {} : { estimatedCostUsd: cost })
+    })
+  )
+}
+
+const meterEffect = <A, R>(
+  effect: Effect.Effect<A, LlmError, R>,
+  collector: MetricsCollectorShape,
+  options: MeteredLlmOptions,
+  usageOf: (value: A) => TokenUsage | undefined
+): Effect.Effect<A, LlmError, R> =>
+  Effect.gen(function* () {
+    const startedAt = yield* Clock.currentTimeMillis
+    yield* collector.markRequestStarted(options.labels)
+    const result = yield* Effect.result(effect)
+    const endedAt = yield* Clock.currentTimeMillis
+    yield* complete(
+      collector,
+      options,
+      startedAt,
+      endedAt,
+      result._tag === "Success",
+      result._tag === "Failure" ? result.failure._tag : undefined,
+      result._tag === "Success" ? usageOf(result.success) : undefined
+    )
+    return yield* Effect.fromResult(result)
+  })
+
+interface StreamMeasurement {
+  readonly success: boolean
+  readonly errorType: string | undefined
+  readonly usage: TokenUsage | undefined
+}
+
+const meterStream = (
+  stream: Stream.Stream<LlmChunk, LlmError>,
+  collector: MetricsCollectorShape,
+  options: MeteredLlmOptions
+): Stream.Stream<LlmChunk, LlmError> =>
+  Stream.unwrap(
+    Effect.gen(function* () {
+      const startedAt = yield* Clock.currentTimeMillis
+      yield* collector.markRequestStarted(options.labels)
+      const measurement = yield* Ref.make<StreamMeasurement>({
+        success: true,
+        errorType: undefined,
+        usage: undefined
+      })
+      return stream.pipe(
+        Stream.tap((chunk) =>
+          chunk.usage === undefined
+            ? Effect.void
+            : Ref.update(measurement, (current) => ({
+                ...current,
+                usage: chunk.usage
+              }))
+        ),
+        Stream.tapError((error) =>
+          Ref.update(measurement, (current) => ({
+            ...current,
+            success: false,
+            errorType: error._tag
+          }))
+        ),
+        Stream.ensuring(
+          Effect.gen(function* () {
+            const current = yield* Ref.get(measurement)
+            const endedAt = yield* Clock.currentTimeMillis
+            yield* complete(
+              collector,
+              options,
+              startedAt,
+              endedAt,
+              current.success,
+              current.errorType,
+              current.usage
+            )
+          })
+        )
+      )
+    })
+  )
+
+export const meterLlmService = (
+  service: LlmServiceShape,
+  collector: MetricsCollectorShape,
+  options: MeteredLlmOptions
+): LlmServiceShape => ({
+  executeStream: (prompt) => meterStream(service.executeStream(prompt), collector, options),
+  executeStreamWithHistory: (messages) =>
+    meterStream(service.executeStreamWithHistory(messages), collector, options),
+  executeWithTools: (prompt, tools) =>
+    meterEffect(service.executeWithTools(prompt, tools), collector, options, () => undefined),
+  executeStructured: (prompt, schema, jsonSchema) =>
+    meterEffect(
+      service.executeStructured(prompt, schema, jsonSchema),
+      collector,
+      options,
+      () => undefined
+    ),
+  executeStructuredWithUsage: (prompt, schema, jsonSchema) =>
+    meterEffect(
+      service.executeStructuredWithUsage(prompt, schema, jsonSchema),
+      collector,
+      options,
+      <A>(result: StructuredResult<A>) => result[1]
+    ),
+  isAvailable: service.isAvailable
+})

package/src/observability/Metrics.ts

@@ -0,0 +1,195 @@
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as Ref from "effect/Ref"
+import * as Schema from "effect/Schema"
+import { TokenUsage } from "../Models.ts"
+
+export class RequestLabels extends Schema.Class<RequestLabels>("RequestLabels")({
+  provider: Schema.String,
+  model: Schema.String,
+  agent: Schema.optionalKey(Schema.String),
+  runId: Schema.optionalKey(Schema.String),
+  workflowStep: Schema.optionalKey(Schema.String)
+}) {}
+
+export class RequestMetrics extends Schema.Class<RequestMetrics>("RequestMetrics")({
+  labels: RequestLabels,
+  tokenUsage: Schema.optionalKey(TokenUsage),
+  latencyMs: Schema.Number,
+  success: Schema.Boolean,
+  errorType: Schema.optionalKey(Schema.String),
+  estimatedCostUsd: Schema.optionalKey(Schema.Number)
+}) {}
+
+export class ProviderHealth extends Schema.Class<ProviderHealth>("ProviderHealth")({
+  provider: Schema.String,
+  requestCount: Schema.Int,
+  successRate: Schema.Number,
+  avgLatencyMs: Schema.Number,
+  p95LatencyMs: Schema.Number,
+  estimatedCostUsd: Schema.Number
+}) {}
+
+export class ObservabilitySnapshot extends Schema.Class<ObservabilitySnapshot>(
+  "ObservabilitySnapshot"
+)({
+  totalRequests: Schema.Int,
+  totalErrors: Schema.Int,
+  activeRequests: Schema.Int,
+  promptTokens: Schema.Int,
+  completionTokens: Schema.Int,
+  totalTokens: Schema.Int,
+  estimatedCostUsd: Schema.Number,
+  p50LatencyMs: Schema.Number,
+  p95LatencyMs: Schema.Number,
+  byProvider: Schema.Array(ProviderHealth),
+  byAgentRequests: Schema.Record(Schema.String, Schema.Int)
+}) {}
+
+export class DashboardMetricsSnapshot extends Schema.Class<DashboardMetricsSnapshot>(
+  "DashboardMetricsSnapshot"
+)({
+  providerHealth: Schema.Array(ProviderHealth),
+  totalTokens: Schema.Int,
+  totalCostUsd: Schema.Number,
+  totalRequests: Schema.Int,
+  errorRate: Schema.Number
+}) {}
+
+export interface MetricsCollectorShape {
+  readonly markRequestStarted: (labels: RequestLabels) => Effect.Effect<void>
+  readonly recordCompleted: (metrics: RequestMetrics) => Effect.Effect<void>
+  readonly snapshot: Effect.Effect<ObservabilitySnapshot>
+  readonly dashboardSnapshot: Effect.Effect<DashboardMetricsSnapshot>
+}
+
+export class MetricsCollector extends Context.Service<MetricsCollector, MetricsCollectorShape>()(
+  "@llm4ts/core/observability/MetricsCollector"
+) {}
+
+interface MetricsState {
+  readonly totalRequests: number
+  readonly activeRequests: number
+  readonly completed: ReadonlyArray<RequestMetrics>
+  readonly byAgentRequests: ReadonlyMap<string, number>
+}
+
+const percentile = (values: ReadonlyArray<number>, quantile: number): number => {
+  if (values.length === 0) {
+    return 0
+  }
+  const sorted = [...values].sort((left, right) => left - right)
+  const index = Math.max(0, Math.ceil(quantile * sorted.length) - 1)
+  return sorted[index] ?? 0
+}
+
+const providerHealth = (
+  completed: ReadonlyArray<RequestMetrics>
+): ReadonlyArray<ProviderHealth> => {
+  const providers = new Map<string, Array<RequestMetrics>>()
+  for (const metric of completed) {
+    const current = providers.get(metric.labels.provider) ?? []
+    current.push(metric)
+    providers.set(metric.labels.provider, current)
+  }
+  return [...providers.entries()]
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([provider, metrics]) => {
+      const latencies = metrics.map((metric) => metric.latencyMs)
+      const successes = metrics.filter((metric) => metric.success).length
+      return ProviderHealth.make({
+        provider,
+        requestCount: metrics.length,
+        successRate: metrics.length === 0 ? 1 : successes / metrics.length,
+        avgLatencyMs:
+          metrics.length === 0
+            ? 0
+            : latencies.reduce((sum, latency) => sum + latency, 0) / metrics.length,
+        p95LatencyMs: percentile(latencies, 0.95),
+        estimatedCostUsd: metrics.reduce((sum, metric) => sum + (metric.estimatedCostUsd ?? 0), 0)
+      })
+    })
+}
+
+const toSnapshot = (state: MetricsState): ObservabilitySnapshot => {
+  const completed = state.completed
+  const promptTokens = completed.reduce((sum, metric) => sum + (metric.tokenUsage?.prompt ?? 0), 0)
+  const completionTokens = completed.reduce(
+    (sum, metric) => sum + (metric.tokenUsage?.completion ?? 0),
+    0
+  )
+  return ObservabilitySnapshot.make({
+    totalRequests: state.totalRequests,
+    totalErrors: completed.filter((metric) => !metric.success).length,
+    activeRequests: state.activeRequests,
+    promptTokens,
+    completionTokens,
+    totalTokens: promptTokens + completionTokens,
+    estimatedCostUsd: completed.reduce((sum, metric) => sum + (metric.estimatedCostUsd ?? 0), 0),
+    p50LatencyMs: percentile(
+      completed.map((metric) => metric.latencyMs),
+      0.5
+    ),
+    p95LatencyMs: percentile(
+      completed.map((metric) => metric.latencyMs),
+      0.95
+    ),
+    byProvider: providerHealth(completed),
+    byAgentRequests: Object.fromEntries(state.byAgentRequests)
+  })
+}
+
+export const makeMetricsCollector = Effect.fn("@llm4ts/core/observability/MetricsCollector.make")(
+  function* (): Effect.fn.Return<MetricsCollectorShape> {
+    const state = yield* Ref.make<MetricsState>({
+      totalRequests: 0,
+      activeRequests: 0,
+      completed: [],
+      byAgentRequests: new Map()
+    })
+
+    const markRequestStarted = (labels: RequestLabels): Effect.Effect<void> =>
+      Ref.update(state, (current) => {
+        const agents = new Map(current.byAgentRequests)
+        if (labels.agent !== undefined) {
+          agents.set(labels.agent, (agents.get(labels.agent) ?? 0) + 1)
+        }
+        return {
+          totalRequests: current.totalRequests + 1,
+          activeRequests: current.activeRequests + 1,
+          completed: current.completed,
+          byAgentRequests: agents
+        }
+      })
+
+    const recordCompleted = (metrics: RequestMetrics): Effect.Effect<void> =>
+      Ref.update(state, (current) => ({
+        ...current,
+        activeRequests: Math.max(0, current.activeRequests - 1),
+        completed: [...current.completed, metrics]
+      }))
+
+    const snapshot = Ref.get(state).pipe(Effect.map(toSnapshot))
+    const dashboardSnapshot = snapshot.pipe(
+      Effect.map((current) =>
+        DashboardMetricsSnapshot.make({
+          providerHealth: current.byProvider,
+          totalTokens: current.totalTokens,
+          totalCostUsd: current.estimatedCostUsd,
+          totalRequests: current.totalRequests,
+          errorRate: current.totalRequests === 0 ? 0 : current.totalErrors / current.totalRequests
+        })
+      )
+    )
+
+    return {
+      markRequestStarted,
+      recordCompleted,
+      snapshot,
+      dashboardSnapshot
+    }
+  }
+)
+
+export const MetricsCollectorLive = Layer.effect(MetricsCollector, makeMetricsCollector())

package/src/observability/Redaction.ts

@@ -0,0 +1,55 @@
+import type { JsonValue } from "../providers/CliSupport.ts"
+
+export interface RedactionOptions {
+  readonly secrets?: ReadonlyArray<string>
+  readonly maxLength?: number
+}
+
+const sensitiveKey = /(?:api[-_]?key|authorization|cookie|password|secret|token)/i
+const credentialPatterns: ReadonlyArray<RegExp> = [
+  /\bBearer\s+[A-Za-z0-9._~+/=-]+\b/gi,
+  /\b(?:sk|xox[baprs]|gh[opusr])[-_][A-Za-z0-9_-]{8,}\b/g
+]
+
+export const redactText = (text: string, options: RedactionOptions = {}): string => {
+  let redacted = text
+  for (const secret of options.secrets ?? []) {
+    if (secret.length > 0) {
+      redacted = redacted.split(secret).join("[REDACTED]")
+    }
+  }
+  for (const pattern of credentialPatterns) {
+    redacted = redacted.replace(pattern, "[REDACTED]")
+  }
+  const maxLength = Math.max(0, options.maxLength ?? 2_048)
+  return redacted.length <= maxLength ? redacted : `${redacted.slice(0, maxLength)}…[TRUNCATED]`
+}
+
+const sanitizeValue = (value: JsonValue, options: RedactionOptions): JsonValue => {
+  if (typeof value === "string") {
+    return redactText(value, options)
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => sanitizeValue(item, options))
+  }
+  if (value !== null && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value).map(([key, child]) => [
+        key,
+        sensitiveKey.test(key) ? "[REDACTED]" : sanitizeValue(child, options)
+      ])
+    )
+  }
+  return value
+}
+
+export const sanitizeFields = (
+  fields: Readonly<Record<string, JsonValue>>,
+  options: RedactionOptions = {}
+): Readonly<Record<string, JsonValue>> =>
+  Object.fromEntries(
+    Object.entries(fields).map(([key, value]) => [
+      key,
+      sensitiveKey.test(key) ? "[REDACTED]" : sanitizeValue(value, options)
+    ])
+  )

package/src/observability/StreamRecorder.ts

@@ -0,0 +1,32 @@
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+
+export interface StreamRecorder {
+  readonly rawLine: (
+    provider: string,
+    model: string | undefined,
+    line: string
+  ) => Effect.Effect<void>
+  readonly streamError: (
+    provider: string,
+    model: string | undefined,
+    message: string
+  ) => Effect.Effect<void>
+}
+
+export const noopStreamRecorder: StreamRecorder = {
+  rawLine: (_provider, _model, _line) => Effect.void,
+  streamError: (_provider, _model, _message) => Effect.void
+}
+
+const CurrentStreamRecorder = Context.Reference<StreamRecorder>(
+  "@llm4ts/core/observability/CurrentStreamRecorder",
+  { defaultValue: () => noopStreamRecorder }
+)
+
+export const currentStreamRecorder: Effect.Effect<StreamRecorder> = CurrentStreamRecorder
+
+export const withStreamRecorder =
+  (recorder: StreamRecorder) =>
+  <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, E, R> =>
+    Effect.provideService(effect, CurrentStreamRecorder, recorder)

package/src/observability/StructuredLogger.ts

@@ -0,0 +1,62 @@
+import * as Clock from "effect/Clock"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as Schema from "effect/Schema"
+import type { JsonValue } from "../providers/CliSupport.ts"
+import { redactText, sanitizeFields, type RedactionOptions } from "./Redaction.ts"
+import type { TracingServiceShape } from "./Tracing.ts"
+
+export const LogLevel = Schema.Literals(["Debug", "Info", "Warning", "Error"])
+export type LogLevel = typeof LogLevel.Type
+
+export class StructuredLogEvent extends Schema.Class<StructuredLogEvent>("StructuredLogEvent")({
+  level: LogLevel,
+  message: Schema.String,
+  correlationId: Schema.String,
+  timestamp: Schema.Number,
+  fields: Schema.Record(Schema.String, Schema.Json)
+}) {}
+
+export interface StructuredLoggerShape {
+  readonly log: (
+    level: LogLevel,
+    message: string,
+    fields?: Readonly<Record<string, JsonValue>>
+  ) => Effect.Effect<void>
+}
+
+export class StructuredLogger extends Context.Service<StructuredLogger, StructuredLoggerShape>()(
+  "@llm4ts/core/observability/StructuredLogger"
+) {}
+
+export interface StructuredLoggerOptions extends RedactionOptions {
+  readonly sink: (event: StructuredLogEvent) => Effect.Effect<void>
+}
+
+export const makeStructuredLogger = (
+  tracing: TracingServiceShape,
+  options: StructuredLoggerOptions
+): StructuredLoggerShape => ({
+  log: Effect.fn("@llm4ts/core/observability/StructuredLogger.log")(function* (
+    level: LogLevel,
+    message: string,
+    fields: Readonly<Record<string, JsonValue>> = {}
+  ) {
+    const correlationId = yield* tracing.correlationId
+    const timestamp = yield* Clock.currentTimeMillis
+    yield* options.sink(
+      StructuredLogEvent.make({
+        level,
+        message: redactText(message, options),
+        correlationId,
+        timestamp,
+        fields: sanitizeFields(fields, options)
+      })
+    )
+  })
+})
+
+export const StructuredLoggerNoop = Layer.succeed(StructuredLogger)({
+  log: (_level, _message, _fields) => Effect.void
+})

package/src/observability/Tracing.ts

@@ -0,0 +1,141 @@
+import * as Clock from "effect/Clock"
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Exit from "effect/Exit"
+import * as Layer from "effect/Layer"
+import * as Ref from "effect/Ref"
+import * as Schema from "effect/Schema"
+import type { JsonValue } from "../providers/CliSupport.ts"
+import { redactText, sanitizeFields, type RedactionOptions } from "./Redaction.ts"
+
+export const TraceStatus = Schema.Literals(["Ok", "Error"])
+export type TraceStatus = typeof TraceStatus.Type
+
+export class TraceSpan extends Schema.Class<TraceSpan>("TraceSpan")({
+  traceId: Schema.String,
+  spanId: Schema.String,
+  parentSpanId: Schema.optionalKey(Schema.String),
+  correlationId: Schema.String,
+  name: Schema.String,
+  status: TraceStatus,
+  startedAt: Schema.Number,
+  endedAt: Schema.Number,
+  attributes: Schema.Record(Schema.String, Schema.Json),
+  errorMessage: Schema.optionalKey(Schema.String)
+}) {}
+
+interface TraceContext {
+  readonly traceId: string
+  readonly spanId: string
+  readonly correlationId: string
+}
+
+const CurrentTraceContext = Context.Reference<TraceContext | undefined>(
+  "@llm4ts/core/observability/CurrentTraceContext",
+  { defaultValue: () => undefined }
+)
+
+export interface TracingServiceShape {
+  readonly withCorrelationId: <A, E, R>(
+    id: string,
+    effect: Effect.Effect<A, E, R>
+  ) => Effect.Effect<A, E, R>
+  readonly correlationId: Effect.Effect<string>
+  readonly inSpan: <A, E, R>(
+    name: string,
+    attributes: Readonly<Record<string, JsonValue>>,
+    effect: Effect.Effect<A, E, R>
+  ) => Effect.Effect<A, E, R>
+  readonly recordedSpans: Effect.Effect<ReadonlyArray<TraceSpan>>
+}
+
+export class TracingService extends Context.Service<TracingService, TracingServiceShape>()(
+  "@llm4ts/core/observability/TracingService"
+) {}
+
+export interface TracingOptions extends RedactionOptions {
+  readonly makeId?: Effect.Effect<string>
+}
+
+const defaultId = Effect.sync(() => globalThis.crypto.randomUUID())
+
+export const makeTracingService = Effect.fn("@llm4ts/core/observability/TracingService.make")(
+  function* (options: TracingOptions = {}): Effect.fn.Return<TracingServiceShape> {
+    const spans = yield* Ref.make<ReadonlyArray<TraceSpan>>([])
+    const makeId = options.makeId ?? defaultId
+
+    const correlationId = Effect.flatMap(CurrentTraceContext, (current) =>
+      current === undefined ? makeId : Effect.succeed(current.correlationId)
+    )
+
+    const withCorrelationId = <A, E, R>(
+      id: string,
+      effect: Effect.Effect<A, E, R>
+    ): Effect.Effect<A, E, R> =>
+      Effect.flatMap(CurrentTraceContext, (current) =>
+        Effect.provideService(effect, CurrentTraceContext, {
+          traceId: current?.traceId ?? id,
+          spanId: current?.spanId ?? id,
+          correlationId: id
+        })
+      )
+
+    const inSpan = <A, E, R>(
+      name: string,
+      attributes: Readonly<Record<string, JsonValue>>,
+      effect: Effect.Effect<A, E, R>
+    ): Effect.Effect<A, E, R> =>
+      Effect.gen(function* () {
+        const parent = yield* CurrentTraceContext
+        const correlation = yield* correlationId
+        const traceId = parent?.traceId ?? (yield* makeId)
+        const spanId = yield* makeId
+        const startedAt = yield* Clock.currentTimeMillis
+        const next: TraceContext = {
+          traceId,
+          spanId,
+          correlationId: correlation
+        }
+        return yield* Effect.provideService(
+          effect.pipe(
+            Effect.onExit((exit) =>
+              Clock.currentTimeMillis.pipe(
+                Effect.flatMap((endedAt) =>
+                  Ref.update(spans, (current) => [
+                    ...current,
+                    TraceSpan.make({
+                      traceId,
+                      spanId,
+                      ...(parent === undefined ? {} : { parentSpanId: parent.spanId }),
+                      correlationId: correlation,
+                      name,
+                      status: Exit.isSuccess(exit) ? "Ok" : "Error",
+                      startedAt,
+                      endedAt,
+                      attributes: sanitizeFields(attributes, options),
+                      ...(Exit.isFailure(exit)
+                        ? {
+                            errorMessage: redactText(String(exit.cause), options)
+                          }
+                        : {})
+                    })
+                  ])
+                )
+              )
+            )
+          ),
+          CurrentTraceContext,
+          next
+        )
+      })
+
+    return {
+      withCorrelationId,
+      correlationId,
+      inSpan,
+      recordedSpans: Ref.get(spans)
+    }
+  }
+)
+
+export const TracingServiceLive = Layer.effect(TracingService, makeTracingService())