@llm4ts/core 0.1.0

package/src/TemporaryFiles.ts

@@ -0,0 +1,51 @@
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Ref from "effect/Ref"
+import type * as Scope from "effect/Scope"
+import type { LlmError } from "./Errors.ts"
+
+export interface TemporaryFilesShape {
+  readonly write: (
+    prefix: string,
+    suffix: string,
+    contents: string
+  ) => Effect.Effect<string, LlmError, Scope.Scope>
+}
+
+export class TemporaryFiles extends Context.Service<TemporaryFiles, TemporaryFilesShape>()(
+  "@llm4ts/core/TemporaryFiles"
+) {}
+
+export interface FakeTemporaryFiles {
+  readonly files: Effect.Effect<
+    ReadonlyArray<{
+      readonly path: string
+      readonly contents: string
+    }>
+  >
+  readonly temporaryFiles: TemporaryFilesShape
+}
+
+export const makeFakeTemporaryFiles = (
+  path = "/tmp/llm4ts-schema.json"
+): Effect.Effect<FakeTemporaryFiles> =>
+  Ref.make<
+    ReadonlyArray<{
+      readonly path: string
+      readonly contents: string
+    }>
+  >([]).pipe(
+    Effect.map((files) => ({
+      files: Ref.get(files),
+      temporaryFiles: {
+        write: (_prefix, _suffix, contents) =>
+          Ref.update(files, (current) => [
+            ...current,
+            {
+              path,
+              contents
+            }
+          ]).pipe(Effect.as(path))
+      }
+    }))
+  )

package/src/UsageLimits.ts

@@ -0,0 +1,242 @@
+import * as DateTime from "effect/DateTime"
+import * as Duration from "effect/Duration"
+import { RateLimitError, UsageLimitError, type LlmError } from "./Errors.ts"
+
+const codexAt = /try again at\s+(\d{1,2}:\d{2}\s*[AP]M)/i
+const claudeAt = /usage limit.*?(?:resets?|try again) at\s+(\d{1,2}(?::\d{2})?\s*[ap]m)/i
+const geminiReset = /reset after\s+((?:\d+(?:ms|[dhms]))+)/i
+const resetComponent = /(\d+)(ms|[dhms])/gi
+const shortResetLimit = Duration.seconds(120)
+
+const geminiQuotaSignals: ReadonlyArray<string> = [
+  "exhausted your capacity",
+  "quota",
+  "resource_exhausted",
+  "rate_limit",
+  "rate limit",
+  "429"
+]
+
+const genericQuotaSignals: ReadonlyArray<string> = [
+  ...geminiQuotaSignals,
+  "usage limit",
+  "out of credits",
+  "too many requests"
+]
+
+const includesSignal = (text: string, signals: ReadonlyArray<string>): boolean => {
+  const normalized = text.toLowerCase()
+  return signals.some((signal) => normalized.includes(signal))
+}
+
+export const isGeminiQuota = (text: string): boolean => includesSignal(text, geminiQuotaSignals)
+
+const isGenericQuota = (text: string): boolean => includesSignal(text, genericQuotaSignals)
+
+const resetDuration = (specification: string): Duration.Duration => {
+  let milliseconds = 0
+
+  for (const match of specification.matchAll(resetComponent)) {
+    const amount = Number(match[1] ?? "0")
+    const unit = (match[2] ?? "ms").toLowerCase()
+
+    switch (unit) {
+      case "d":
+        milliseconds += amount * 86_400_000
+        break
+      case "h":
+        milliseconds += amount * 3_600_000
+        break
+      case "m":
+        milliseconds += amount * 60_000
+        break
+      case "s":
+        milliseconds += amount * 1_000
+        break
+      default:
+        milliseconds += amount
+    }
+  }
+
+  return Duration.millis(milliseconds)
+}
+
+interface WallClockTime {
+  readonly hour: number
+  readonly minute: number
+}
+
+const parseWallClockTime = (raw: string): WallClockTime | undefined => {
+  const match = raw.trim().match(/^(\d{1,2})(?::(\d{2}))?\s*([ap])m$/i)
+  if (match === null) {
+    return undefined
+  }
+
+  const clockHour = Number(match[1])
+  const minute = Number(match[2] ?? "0")
+  const period = (match[3] ?? "").toLowerCase()
+  if (clockHour < 1 || clockHour > 12 || minute < 0 || minute > 59) {
+    return undefined
+  }
+
+  return {
+    hour: (clockHour % 12) + (period === "p" ? 12 : 0),
+    minute
+  }
+}
+
+interface CalendarDate {
+  readonly year: number
+  readonly month: number
+  readonly day: number
+}
+
+const dateTimeFormatter = (timeZone: string): Intl.DateTimeFormat =>
+  new Intl.DateTimeFormat("en-US-u-nu-latn", {
+    timeZone,
+    year: "numeric",
+    month: "2-digit",
+    day: "2-digit",
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+    hourCycle: "h23"
+  })
+
+const numericPart = (
+  parts: ReadonlyArray<Intl.DateTimeFormatPart>,
+  type: Intl.DateTimeFormatPartTypes
+): number => Number(parts.find((part) => part.type === type)?.value ?? "0")
+
+const localParts = (epochMillis: number, timeZone: string) => {
+  const parts = dateTimeFormatter(timeZone).formatToParts(new Date(epochMillis))
+  return {
+    year: numericPart(parts, "year"),
+    month: numericPart(parts, "month"),
+    day: numericPart(parts, "day"),
+    hour: numericPart(parts, "hour"),
+    minute: numericPart(parts, "minute"),
+    second: numericPart(parts, "second")
+  }
+}
+
+const timeZoneOffsetMillis = (epochMillis: number, timeZone: string): number => {
+  const parts = localParts(epochMillis, timeZone)
+  const representedAsUtc = Date.UTC(
+    parts.year,
+    parts.month - 1,
+    parts.day,
+    parts.hour,
+    parts.minute,
+    parts.second
+  )
+  return representedAsUtc - Math.trunc(epochMillis / 1_000) * 1_000
+}
+
+const epochForLocalTime = (date: CalendarDate, time: WallClockTime, timeZone: string): number => {
+  const utcGuess = Date.UTC(date.year, date.month - 1, date.day, time.hour, time.minute)
+  const firstCandidate = utcGuess - timeZoneOffsetMillis(utcGuess, timeZone)
+  return utcGuess - timeZoneOffsetMillis(firstCandidate, timeZone)
+}
+
+const nextCalendarDate = (date: CalendarDate): CalendarDate => {
+  const next = new Date(Date.UTC(date.year, date.month - 1, date.day + 1))
+  return {
+    year: next.getUTCFullYear(),
+    month: next.getUTCMonth() + 1,
+    day: next.getUTCDate()
+  }
+}
+
+const wallClockReset = (
+  text: string,
+  pattern: RegExp,
+  now: DateTime.Utc,
+  timeZone: string
+): DateTime.Utc | undefined => {
+  const rawTime = text.match(pattern)?.[1]
+  const time = rawTime === undefined ? undefined : parseWallClockTime(rawTime)
+  if (time === undefined) {
+    return undefined
+  }
+
+  try {
+    const nowMillis = DateTime.toEpochMillis(now)
+    const localNow = localParts(nowMillis, timeZone)
+    const today: CalendarDate = {
+      year: localNow.year,
+      month: localNow.month,
+      day: localNow.day
+    }
+    const todayCandidate = epochForLocalTime(today, time, timeZone)
+    const resetMillis =
+      todayCandidate > nowMillis
+        ? todayCandidate
+        : epochForLocalTime(nextCalendarDate(today), time, timeZone)
+
+    return DateTime.makeUnsafe(resetMillis)
+  } catch {
+    return undefined
+  }
+}
+
+export const classifyUsageLimit = (
+  provider: string,
+  text: string,
+  now: DateTime.Utc,
+  timeZone: string
+): LlmError | undefined => {
+  switch (provider) {
+    case "codex": {
+      const resetAt = wallClockReset(text, codexAt, now, timeZone)
+      return resetAt === undefined
+        ? undefined
+        : UsageLimitError.make({
+            resetAt,
+            provider,
+            message: text
+          })
+    }
+    case "claude": {
+      const resetAt = wallClockReset(text, claudeAt, now, timeZone)
+      return resetAt === undefined
+        ? undefined
+        : UsageLimitError.make({
+            resetAt,
+            provider,
+            message: text
+          })
+    }
+    case "gemini": {
+      const resetSpecification = text.match(geminiReset)?.[1]
+      if (resetSpecification !== undefined) {
+        const duration = resetDuration(resetSpecification)
+        return Duration.toMillis(duration) <= Duration.toMillis(shortResetLimit)
+          ? RateLimitError.make({ retryAfter: duration })
+          : UsageLimitError.make({
+              resetAt: DateTime.addDuration(now, duration),
+              provider,
+              message: text
+            })
+      }
+
+      return isGeminiQuota(text)
+        ? UsageLimitError.make({
+            provider,
+            message: text
+          })
+        : undefined
+    }
+    case "grok":
+    case "cursor":
+    case "opencode":
+      return isGenericQuota(text)
+        ? UsageLimitError.make({
+            provider,
+            message: text
+          })
+        : undefined
+    default:
+      return undefined
+  }
+}

package/src/eval/Checks.ts

@@ -0,0 +1,68 @@
+import * as Effect from "effect/Effect"
+import { DimensionScore, EvalResult } from "./Eval.ts"
+import { makeEvaluator, type Evaluator } from "./Evaluator.ts"
+
+const piiPatterns: ReadonlyArray<readonly [label: string, pattern: RegExp]> = [
+  ["email", /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/],
+  ["ssn", /\b\d{3}-\d{2}-\d{4}\b/],
+  ["credit-card", /\b(?:\d[ -]?){13,16}\b/],
+  ["phone", /\b\d{3}[-.\s]?\d{3}[-.\s]?\d{4}\b/],
+  ["ipv4", /\b(?:\d{1,3}\.){3}\d{1,3}\b/]
+]
+
+const one = (name: string, ok: boolean, maxScore: number, why: string): EvalResult =>
+  EvalResult.make({
+    scores: [
+      DimensionScore.make({
+        name,
+        score: ok ? maxScore : 0,
+        reasoning: why
+      })
+    ]
+  })
+
+export const noPii = (maxScore = 2): Evaluator<string> =>
+  makeEvaluator((input) => {
+    const hits = piiPatterns.flatMap(([label, pattern]) => (pattern.test(input) ? [label] : []))
+    return Effect.succeed(
+      one(
+        "no-pii",
+        hits.length === 0,
+        maxScore,
+        hits.length === 0 ? "no PII detected" : `matched: ${hits.join(", ")}`
+      )
+    )
+  })
+
+export const validJson = (maxScore = 2): Evaluator<string> =>
+  makeEvaluator((input) => {
+    let valid = true
+    try {
+      JSON.parse(input)
+    } catch {
+      valid = false
+    }
+    return Effect.succeed(
+      one("valid-json", valid, maxScore, valid ? "parses as JSON" : "not valid JSON")
+    )
+  })
+
+export const matches = (regex: string, name = "format", maxScore = 2): Evaluator<string> =>
+  makeEvaluator((input) => {
+    const ok = new RegExp(`^(?:${regex})$`).test(input)
+    return Effect.succeed(
+      one(name, ok, maxScore, ok ? `matches /${regex}/` : `does not match /${regex}/`)
+    )
+  })
+
+export const lengthBetween = (min: number, max: number, maxScore = 2): Evaluator<string> =>
+  makeEvaluator((input) =>
+    Effect.succeed(
+      one(
+        "length",
+        input.length >= min && input.length <= max,
+        maxScore,
+        `length ${input.length}; bounds [${min}, ${max}]`
+      )
+    )
+  )

package/src/eval/Eval.ts

@@ -0,0 +1,112 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import type { LlmError } from "../Errors.ts"
+import type { Evaluator } from "./Evaluator.ts"
+
+export class Dimension extends Schema.Class<Dimension>("Dimension")({
+  name: Schema.String,
+  rubric: Schema.String,
+  maxScore: Schema.Int.pipe(Schema.withConstructorDefault(Effect.succeed(2)))
+}) {}
+
+export class DimensionScore extends Schema.Class<DimensionScore>("DimensionScore")({
+  name: Schema.String,
+  score: Schema.Int,
+  reasoning: Schema.String.pipe(Schema.withConstructorDefault(Effect.succeed("")))
+}) {}
+
+export class EvalResult extends Schema.Class<EvalResult>("EvalResult")({
+  scores: Schema.Array(DimensionScore),
+  summary: Schema.String.pipe(Schema.withConstructorDefault(Effect.succeed("")))
+}) {
+  score(name: string): number | undefined {
+    return this.scores.find((item) => item.name === name)?.score
+  }
+
+  get total(): number {
+    return this.scores.reduce((sum, item) => sum + item.score, 0)
+  }
+
+  meets(minPerDimension: number): boolean {
+    return this.scores.every((item) => item.score >= minPerDimension)
+  }
+}
+
+export class Sample extends Schema.Class<Sample>("Sample")({
+  response: Schema.String,
+  query: Schema.optionalKey(Schema.String),
+  context: Schema.optionalKey(Schema.String),
+  expected: Schema.optionalKey(Schema.String)
+}) {}
+
+export class DimensionStats extends Schema.Class<DimensionStats>("DimensionStats")({
+  name: Schema.String,
+  median: Schema.Int,
+  min: Schema.Int,
+  max: Schema.Int
+}) {
+  get spread(): number {
+    return this.max - this.min
+  }
+
+  isFlaky(threshold: number): boolean {
+    return this.spread > threshold
+  }
+}
+
+export class RepeatedEval extends Schema.Class<RepeatedEval>("RepeatedEval")({
+  runs: Schema.Array(EvalResult),
+  stats: Schema.Array(DimensionStats),
+  spreadThreshold: Schema.Int
+}) {
+  get flakyDimensions(): ReadonlyArray<string> {
+    return this.stats.filter((stat) => stat.isFlaky(this.spreadThreshold)).map((stat) => stat.name)
+  }
+
+  get isFlaky(): boolean {
+    return this.flakyDimensions.length > 0
+  }
+
+  get aggregate(): EvalResult {
+    return EvalResult.make({
+      scores: this.stats.map((stat) =>
+        DimensionScore.make({
+          name: stat.name,
+          score: stat.median,
+          reasoning: `median of ${this.runs.length} runs`
+        })
+      ),
+      summary: `aggregate of ${this.runs.length} runs`
+    })
+  }
+}
+
+const median = (sorted: ReadonlyArray<number>): number =>
+  sorted.length === 0 ? 0 : (sorted[Math.floor((sorted.length - 1) / 2)] ?? 0)
+
+export const repeat = Effect.fn("@llm4ts/core/eval/Eval.repeat")(function* <In>(
+  evaluator: Evaluator<In>,
+  input: In,
+  n = 3,
+  spreadThreshold = 1
+): Effect.fn.Return<RepeatedEval, LlmError> {
+  const runs = yield* Effect.forEach(Array.from({ length: Math.max(0, n) }), () =>
+    evaluator.evaluate(input)
+  )
+  const names = [...new Set(runs.flatMap((run) => run.scores.map((score) => score.name)))]
+  const stats = names.map((name) => {
+    const values = runs
+      .flatMap((run) => {
+        const score = run.score(name)
+        return score === undefined ? [] : [score]
+      })
+      .sort((left, right) => left - right)
+    return DimensionStats.make({
+      name,
+      median: median(values),
+      min: values.at(0) ?? 0,
+      max: values.at(-1) ?? 0
+    })
+  })
+  return RepeatedEval.make({ runs, stats, spreadThreshold })
+})

package/src/eval/EvalSuite.ts

@@ -0,0 +1,63 @@
+import * as Effect from "effect/Effect"
+import type { LlmError } from "../Errors.ts"
+import { repeat } from "./Eval.ts"
+import type { EvalResult, RepeatedEval } from "./Eval.ts"
+import type { Evaluator } from "./Evaluator.ts"
+
+export interface EvalCase<In> {
+  readonly name: string
+  readonly input: In
+  readonly minPerDimension?: number
+}
+
+export interface CaseReport {
+  readonly name: string
+  readonly result: EvalResult
+  readonly repeated: RepeatedEval | undefined
+  readonly passed: boolean
+}
+
+export interface SuiteReport {
+  readonly cases: ReadonlyArray<CaseReport>
+  readonly passed: boolean
+  readonly failures: ReadonlyArray<CaseReport>
+}
+
+export const run = Effect.fn("@llm4ts/core/eval/EvalSuite.run")(function* <In>(
+  evaluator: Evaluator<In>,
+  cases: ReadonlyArray<EvalCase<In>>,
+  repeats = 1,
+  spreadThreshold = 1
+): Effect.fn.Return<SuiteReport, LlmError> {
+  const reports = yield* Effect.forEach(cases, (testCase) => {
+    const minimum = testCase.minPerDimension ?? 1
+    if (repeats <= 1) {
+      return evaluator.evaluate(testCase.input).pipe(
+        Effect.map(
+          (result): CaseReport => ({
+            name: testCase.name,
+            result,
+            repeated: undefined,
+            passed: result.meets(minimum)
+          })
+        )
+      )
+    }
+    return repeat(evaluator, testCase.input, repeats, spreadThreshold).pipe(
+      Effect.map((repeated): CaseReport => {
+        const result = repeated.aggregate
+        return {
+          name: testCase.name,
+          result,
+          repeated,
+          passed: result.meets(minimum) && !repeated.isFlaky
+        }
+      })
+    )
+  })
+  return {
+    cases: reports,
+    passed: reports.every((report) => report.passed),
+    failures: reports.filter((report) => !report.passed)
+  }
+})

package/src/eval/Evaluator.ts

@@ -0,0 +1,31 @@
+import * as Effect from "effect/Effect"
+import type { LlmError } from "../Errors.ts"
+import { EvalResult } from "./Eval.ts"
+
+export interface Evaluator<in In> {
+  readonly evaluate: (input: In) => Effect.Effect<EvalResult, LlmError>
+}
+
+export const makeEvaluator = <In>(evaluate: Evaluator<In>["evaluate"]): Evaluator<In> => ({
+  evaluate
+})
+
+export const contramap = <In, In2>(
+  evaluator: Evaluator<In>,
+  project: (input: In2) => In
+): Evaluator<In2> => makeEvaluator((input) => evaluator.evaluate(project(input)))
+
+export const all = <In>(...evaluators: ReadonlyArray<Evaluator<In>>): Evaluator<In> =>
+  makeEvaluator((input) =>
+    Effect.forEach(evaluators, (evaluator) => evaluator.evaluate(input)).pipe(
+      Effect.map((results) =>
+        EvalResult.make({
+          scores: results.flatMap((result) => result.scores),
+          summary: results
+            .map((result) => result.summary)
+            .filter((summary) => summary.length > 0)
+            .join("; ")
+        })
+      )
+    )
+  )

package/src/eval/Judge.ts

@@ -0,0 +1,92 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import type { LlmServiceShape } from "../LlmService.ts"
+import type { JsonSchema } from "../Models.ts"
+import { DimensionScore, EvalResult } from "./Eval.ts"
+import type { Dimension, Sample } from "./Eval.ts"
+import { makeEvaluator, type Evaluator } from "./Evaluator.ts"
+
+export class JudgeResponse extends Schema.Class<JudgeResponse>("JudgeResponse")({
+  scores: Schema.Array(DimensionScore)
+}) {}
+
+export const defaultSystem =
+  "You are an impartial evaluator. Score each dimension strictly on its rubric; when unsure, score lower. " +
+  "Return ONLY valid JSON, no prose."
+
+const judgeJsonSchema: JsonSchema = {
+  type: "object",
+  properties: {
+    scores: {
+      type: "array",
+      items: {
+        type: "object",
+        properties: {
+          name: { type: "string" },
+          score: { type: "integer" },
+          reasoning: { type: "string" }
+        },
+        required: ["name", "score"]
+      }
+    }
+  },
+  required: ["scores"],
+  additionalProperties: false
+}
+
+export const buildPrompt = (
+  system: string,
+  dimensions: ReadonlyArray<Dimension>,
+  sample: Sample
+): string => {
+  const rubric = dimensions
+    .map((dimension) => `- ${dimension.name} (0..${dimension.maxScore}): ${dimension.rubric}`)
+    .join("\n")
+  const user = [
+    sample.query === undefined ? undefined : `Query: ${sample.query}`,
+    sample.context === undefined ? undefined : `Context: ${sample.context}`,
+    `Response: ${sample.response}`,
+    sample.expected === undefined ? undefined : `Expected: ${sample.expected}`
+  ]
+    .filter((line): line is string => line !== undefined)
+    .join("\n")
+  return `${system}
+
+Score each dimension on its 0..max scale per the rubric:
+${rubric}
+
+Return ONLY this JSON: {"scores":[{"name":"<dimension>","score":<int>,"reasoning":"<one sentence>"}]}
+
+${user}`
+}
+
+const toResult = (dimensions: ReadonlyArray<Dimension>, response: JudgeResponse): EvalResult => {
+  const byName = new Map(response.scores.map((score) => [score.name, score]))
+  return EvalResult.make({
+    scores: dimensions.map((dimension) => {
+      const score = byName.get(dimension.name)
+      return score === undefined
+        ? DimensionScore.make({
+            name: dimension.name,
+            score: 0,
+            reasoning: "missing"
+          })
+        : DimensionScore.make({
+            name: dimension.name,
+            score: Math.max(0, Math.min(dimension.maxScore, score.score)),
+            reasoning: score.reasoning
+          })
+    })
+  })
+}
+
+export const judge = (
+  llm: LlmServiceShape,
+  dimensions: ReadonlyArray<Dimension>,
+  system = defaultSystem
+): Evaluator<Sample> =>
+  makeEvaluator((sample) =>
+    llm
+      .executeStructured(buildPrompt(system, dimensions, sample), JudgeResponse, judgeJsonSchema)
+      .pipe(Effect.map((response) => toResult(dimensions, response)))
+  )