@codilore/llm 1.15.13

package/src/protocols/bedrock-event-stream.ts

@@ -0,0 +1,87 @@
+import { EventStreamCodec } from "@smithy/eventstream-codec"
+import { fromUtf8, toUtf8 } from "@smithy/util-utf8"
+import { Effect, Stream } from "effect"
+import type { Framing } from "../route/framing"
+import { ProviderShared } from "./shared"
+
+// Bedrock streams responses using the AWS event stream binary protocol — each
+// frame is `[length:4][headers-length:4][prelude-crc:4][headers][payload][crc:4]`.
+// We use `@smithy/eventstream-codec` to validate framing and CRCs, then
+// reconstruct the JSON wrapping by `:event-type` so the chunk schema can match.
+const eventCodec = new EventStreamCodec(toUtf8, fromUtf8)
+const utf8 = new TextDecoder()
+
+// Cursor-tracking buffer state. Bytes accumulate in `buffer`; `offset` is the
+// read position. Reading by `subarray` is zero-copy. We only allocate a fresh
+// buffer when a new network chunk arrives and we need to append.
+interface FrameBufferState {
+  readonly buffer: Uint8Array
+  readonly offset: number
+}
+
+const initialFrameBuffer: FrameBufferState = { buffer: new Uint8Array(0), offset: 0 }
+
+const appendChunk = (state: FrameBufferState, chunk: Uint8Array): FrameBufferState => {
+  const remaining = state.buffer.length - state.offset
+  // Compact: drop the consumed prefix and append the new chunk in one alloc.
+  // This bounds buffer growth to at most one network chunk past the live
+  // window, regardless of stream length.
+  const next = new Uint8Array(remaining + chunk.length)
+  next.set(state.buffer.subarray(state.offset), 0)
+  next.set(chunk, remaining)
+  return { buffer: next, offset: 0 }
+}
+
+const consumeFrames = (route: string) => (state: FrameBufferState, chunk: Uint8Array) =>
+  Effect.gen(function* () {
+    let cursor = appendChunk(state, chunk)
+    const out: object[] = []
+    while (cursor.buffer.length - cursor.offset >= 4) {
+      const view = cursor.buffer.subarray(cursor.offset)
+      const totalLength = new DataView(view.buffer, view.byteOffset, view.byteLength).getUint32(0, false)
+      if (view.length < totalLength) break
+
+      const decoded = yield* Effect.try({
+        try: () => eventCodec.decode(view.subarray(0, totalLength)),
+        catch: (error) =>
+          ProviderShared.eventError(
+            route,
+            `Failed to decode Bedrock Converse event-stream frame: ${
+              error instanceof Error ? error.message : String(error)
+            }`,
+          ),
+      })
+      cursor = { buffer: cursor.buffer, offset: cursor.offset + totalLength }
+
+      if (decoded.headers[":message-type"]?.value !== "event") continue
+      const eventType = decoded.headers[":event-type"]?.value
+      if (typeof eventType !== "string") continue
+      const payload = utf8.decode(decoded.body)
+      if (!payload) continue
+      // The AWS event stream pads short payloads with a `p` field. Drop it
+      // before handing the object to the chunk schema. JSON decode goes
+      // through the shared Schema-driven codec to satisfy the package rule
+      // against ad-hoc `JSON.parse` calls.
+      const parsed = (yield* ProviderShared.parseJson(
+        route,
+        payload,
+        "Failed to parse Bedrock Converse event-stream payload",
+      )) as Record<string, unknown>
+      delete parsed.p
+      out.push({ [eventType]: parsed })
+    }
+    return [cursor, out] as const
+  })
+
+/**
+ * AWS event-stream framing for Bedrock Converse. Each frame is decoded by
+ * `@smithy/eventstream-codec` (length + header + payload + CRC) and rewrapped
+ * under its `:event-type` header so the chunk schema can match the JSON
+ * payload directly.
+ */
+export const framing = (route: string): Framing<object> => ({
+  id: "aws-event-stream",
+  frame: (bytes) => bytes.pipe(Stream.mapAccumEffect(() => initialFrameBuffer, consumeFrames(route))),
+})
+
+export * as BedrockEventStream from "./bedrock-event-stream"

package/src/protocols/gemini.ts

@@ -0,0 +1,465 @@
+import { Effect, Schema } from "effect"
+import { Route } from "../route/client"
+import { Auth } from "../route/auth"
+import { Endpoint } from "../route/endpoint"
+import { Framing } from "../route/framing"
+import { Protocol } from "../route/protocol"
+import {
+  LLMEvent,
+  Usage,
+  type FinishReason,
+  type LLMRequest,
+  type MediaPart,
+  type ProviderMetadata,
+  type TextPart,
+  type ToolCallPart,
+  type ToolDefinition,
+} from "../schema"
+import { JsonObject, optionalArray, ProviderShared } from "./shared"
+import { GeminiToolSchema } from "./utils/gemini-tool-schema"
+import { Lifecycle } from "./utils/lifecycle"
+
+const ADAPTER = "gemini"
+export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
+
+// =============================================================================
+// Request Body Schema
+// =============================================================================
+const GeminiTextPart = Schema.Struct({
+  text: Schema.String,
+  thought: Schema.optional(Schema.Boolean),
+  thoughtSignature: Schema.optional(Schema.String),
+})
+
+const GeminiInlineDataPart = Schema.Struct({
+  inlineData: Schema.Struct({
+    mimeType: Schema.String,
+    data: Schema.String,
+  }),
+})
+
+const GeminiFunctionCallPart = Schema.Struct({
+  functionCall: Schema.Struct({
+    name: Schema.String,
+    args: Schema.Unknown,
+  }),
+  thoughtSignature: Schema.optional(Schema.String),
+})
+
+const GeminiFunctionResponsePart = Schema.Struct({
+  functionResponse: Schema.Struct({
+    name: Schema.String,
+    response: Schema.Unknown,
+  }),
+})
+
+const GeminiContentPart = Schema.Union([
+  GeminiTextPart,
+  GeminiInlineDataPart,
+  GeminiFunctionCallPart,
+  GeminiFunctionResponsePart,
+])
+
+const GeminiContent = Schema.Struct({
+  role: Schema.Literals(["user", "model"]),
+  parts: Schema.Array(GeminiContentPart),
+})
+type GeminiContent = Schema.Schema.Type<typeof GeminiContent>
+
+const GeminiSystemInstruction = Schema.Struct({
+  parts: Schema.Array(Schema.Struct({ text: Schema.String })),
+})
+
+const GeminiFunctionDeclaration = Schema.Struct({
+  name: Schema.String,
+  description: Schema.String,
+  parameters: Schema.optional(JsonObject),
+})
+
+const GeminiTool = Schema.Struct({
+  functionDeclarations: Schema.Array(GeminiFunctionDeclaration),
+})
+
+const GeminiToolConfig = Schema.Struct({
+  functionCallingConfig: Schema.Struct({
+    mode: Schema.Literals(["AUTO", "NONE", "ANY"]),
+    allowedFunctionNames: optionalArray(Schema.String),
+  }),
+})
+
+const GeminiThinkingConfig = Schema.Struct({
+  thinkingBudget: Schema.optional(Schema.Number),
+  includeThoughts: Schema.optional(Schema.Boolean),
+})
+
+const GeminiGenerationConfig = Schema.Struct({
+  maxOutputTokens: Schema.optional(Schema.Number),
+  temperature: Schema.optional(Schema.Number),
+  topP: Schema.optional(Schema.Number),
+  topK: Schema.optional(Schema.Number),
+  stopSequences: optionalArray(Schema.String),
+  thinkingConfig: Schema.optional(GeminiThinkingConfig),
+})
+
+const GeminiBodyFields = {
+  contents: Schema.Array(GeminiContent),
+  systemInstruction: Schema.optional(GeminiSystemInstruction),
+  tools: optionalArray(GeminiTool),
+  toolConfig: Schema.optional(GeminiToolConfig),
+  generationConfig: Schema.optional(GeminiGenerationConfig),
+}
+const GeminiBody = Schema.Struct(GeminiBodyFields)
+export type GeminiBody = Schema.Schema.Type<typeof GeminiBody>
+
+const GeminiUsage = Schema.Struct({
+  cachedContentTokenCount: Schema.optional(Schema.Number),
+  thoughtsTokenCount: Schema.optional(Schema.Number),
+  promptTokenCount: Schema.optional(Schema.Number),
+  candidatesTokenCount: Schema.optional(Schema.Number),
+  totalTokenCount: Schema.optional(Schema.Number),
+})
+type GeminiUsage = Schema.Schema.Type<typeof GeminiUsage>
+
+const GeminiCandidate = Schema.Struct({
+  content: Schema.optional(GeminiContent),
+  finishReason: Schema.optional(Schema.String),
+})
+
+const GeminiEvent = Schema.Struct({
+  candidates: optionalArray(GeminiCandidate),
+  usageMetadata: Schema.optional(GeminiUsage),
+})
+type GeminiEvent = Schema.Schema.Type<typeof GeminiEvent>
+
+interface ParserState {
+  readonly finishReason?: string
+  readonly hasToolCalls: boolean
+  readonly nextToolCallId: number
+  readonly usage?: Usage
+  readonly lifecycle: Lifecycle.State
+  readonly reasoningSignature?: string
+}
+
+const mediaData = ProviderShared.mediaBytes
+
+// =============================================================================
+// Tool Schema Conversion
+// =============================================================================
+// Tool-schema conversion has two distinct concerns:
+//
+// 1. Sanitize — fix common authoring mistakes Gemini rejects: integer/number
+//    enums (must be strings), `required` entries that don't match a property,
+//    untyped arrays (`items` must be present), and `properties`/`required`
+//    keys on non-object scalars. Mirrors Codilore's historical Gemini rules.
+//
+// 2. Project — lossy mapping from JSON Schema to Gemini's schema dialect:
+//    drop empty objects, derive `nullable: true` from `type: [..., "null"]`,
+//    coerce `const` to `[const]` enum, recurse properties/items, propagate
+//    only an allowlisted set of keys (description, required, format, type,
+//    properties, items, allOf, anyOf, oneOf, minLength). Anything outside the
+//    allowlist (e.g. `additionalProperties`, `$ref`) is silently dropped.
+//
+// Sanitize runs first, then project. The implementation lives in
+// `utils/gemini-tool-schema` so this protocol keeps the same shape as the other
+// provider protocols.
+
+// =============================================================================
+// Request Lowering
+// =============================================================================
+const lowerTool = (tool: ToolDefinition) => ({
+  name: tool.name,
+  description: tool.description,
+  parameters: GeminiToolSchema.convert(tool.inputSchema),
+})
+
+const lowerToolConfig = (toolChoice: NonNullable<LLMRequest["toolChoice"]>) =>
+  ProviderShared.matchToolChoice("Gemini", toolChoice, {
+    auto: () => ({ functionCallingConfig: { mode: "AUTO" as const } }),
+    none: () => ({ functionCallingConfig: { mode: "NONE" as const } }),
+    required: () => ({ functionCallingConfig: { mode: "ANY" as const } }),
+    tool: (name) => ({ functionCallingConfig: { mode: "ANY" as const, allowedFunctionNames: [name] } }),
+  })
+
+const lowerUserPart = (part: TextPart | MediaPart) =>
+  part.type === "text" ? { text: part.text } : { inlineData: { mimeType: part.mediaType, data: mediaData(part) } }
+
+const googleMetadata = (metadata: Record<string, unknown>): ProviderMetadata => ({ google: metadata })
+
+const thoughtSignature = (providerMetadata: ProviderMetadata | undefined) => {
+  const google = providerMetadata?.google
+  return ProviderShared.isRecord(google) && typeof google.thoughtSignature === "string"
+    ? google.thoughtSignature
+    : undefined
+}
+
+const lowerToolCall = (part: ToolCallPart) => ({
+  functionCall: { name: part.name, args: part.input },
+  thoughtSignature: thoughtSignature(part.providerMetadata),
+})
+
+const lowerMessages = Effect.fn("Gemini.lowerMessages")(function* (request: LLMRequest) {
+  const contents: GeminiContent[] = []
+
+  for (const message of request.messages) {
+    if (message.role === "system") {
+      const part = yield* ProviderShared.wrappedSystemUpdate("Gemini", message)
+      const previous = contents.at(-1)
+      if (previous?.role === "user")
+        contents[contents.length - 1] = { role: "user", parts: [...previous.parts, { text: part.text }] }
+      else contents.push({ role: "user", parts: [{ text: part.text }] })
+      continue
+    }
+
+    if (message.role === "user") {
+      const parts: Array<Schema.Schema.Type<typeof GeminiContentPart>> = []
+      for (const part of message.content) {
+        if (!ProviderShared.supportsContent(part, ["text", "media"]))
+          return yield* ProviderShared.unsupportedContent("Gemini", "user", ["text", "media"])
+        parts.push(lowerUserPart(part))
+      }
+      contents.push({ role: "user", parts })
+      continue
+    }
+
+    if (message.role === "assistant") {
+      const parts: Array<Schema.Schema.Type<typeof GeminiContentPart>> = []
+      for (const part of message.content) {
+        if (!ProviderShared.supportsContent(part, ["text", "reasoning", "tool-call"]))
+          return yield* ProviderShared.unsupportedContent("Gemini", "assistant", ["text", "reasoning", "tool-call"])
+        if (part.type === "text") {
+          parts.push({ text: part.text })
+          continue
+        }
+        if (part.type === "reasoning") {
+          parts.push({ text: part.text, thought: true, thoughtSignature: thoughtSignature(part.providerMetadata) })
+          continue
+        }
+        if (part.type === "tool-call") {
+          parts.push(lowerToolCall(part))
+          continue
+        }
+      }
+      contents.push({ role: "model", parts })
+      continue
+    }
+
+    const parts: Array<Schema.Schema.Type<typeof GeminiContentPart>> = []
+    for (const part of message.content) {
+      if (!ProviderShared.supportsContent(part, ["tool-result"]))
+        return yield* ProviderShared.unsupportedContent("Gemini", "tool", ["tool-result"])
+      parts.push({
+        functionResponse: {
+          name: part.name,
+          response: {
+            name: part.name,
+            content: ProviderShared.toolResultText(part),
+          },
+        },
+      })
+    }
+    contents.push({ role: "user", parts })
+  }
+
+  return contents
+})
+
+const geminiOptions = (request: LLMRequest) => request.providerOptions?.gemini
+
+const thinkingConfig = (request: LLMRequest) => {
+  const value = geminiOptions(request)?.thinkingConfig
+  if (!ProviderShared.isRecord(value)) return undefined
+  const result = {
+    thinkingBudget: typeof value.thinkingBudget === "number" ? value.thinkingBudget : undefined,
+    includeThoughts: typeof value.includeThoughts === "boolean" ? value.includeThoughts : undefined,
+  }
+  return Object.values(result).some((item) => item !== undefined) ? result : undefined
+}
+
+const fromRequest = Effect.fn("Gemini.fromRequest")(function* (request: LLMRequest) {
+  const toolsEnabled = request.tools.length > 0 && request.toolChoice?.type !== "none"
+  const generation = request.generation
+  const generationConfig = {
+    maxOutputTokens: generation?.maxTokens,
+    temperature: generation?.temperature,
+    topP: generation?.topP,
+    topK: generation?.topK,
+    stopSequences: generation?.stop,
+    thinkingConfig: thinkingConfig(request),
+  }
+
+  return {
+    contents: yield* lowerMessages(request),
+    systemInstruction:
+      request.system.length === 0 ? undefined : { parts: [{ text: ProviderShared.joinText(request.system) }] },
+    tools: toolsEnabled ? [{ functionDeclarations: request.tools.map(lowerTool) }] : undefined,
+    toolConfig: toolsEnabled && request.toolChoice ? yield* lowerToolConfig(request.toolChoice) : undefined,
+    generationConfig: Object.values(generationConfig).some((value) => value !== undefined)
+      ? generationConfig
+      : undefined,
+  }
+})
+
+// =============================================================================
+// Stream Parsing
+// =============================================================================
+// Gemini reports `promptTokenCount` (inclusive total) with a
+// `cachedContentTokenCount` subset. `candidatesTokenCount` is *exclusive*
+// of `thoughtsTokenCount` — visible-only, not a total — so we sum the two
+// to produce the inclusive `outputTokens` the rest of the contract expects.
+const mapUsage = (usage: GeminiUsage | undefined) => {
+  if (!usage) return undefined
+  const cached = usage.cachedContentTokenCount
+  const nonCached = ProviderShared.subtractTokens(usage.promptTokenCount, cached)
+  // `candidatesTokenCount` is visible-only; sum with thoughts to produce the
+  // inclusive `outputTokens` the contract expects. Only compute the total
+  // when the visible component is reported — otherwise we'd fabricate an
+  // inclusive number from a partial breakdown.
+  const outputTokens =
+    usage.candidatesTokenCount !== undefined ? usage.candidatesTokenCount + (usage.thoughtsTokenCount ?? 0) : undefined
+  return new Usage({
+    inputTokens: usage.promptTokenCount,
+    outputTokens,
+    nonCachedInputTokens: nonCached,
+    cacheReadInputTokens: cached,
+    reasoningTokens: usage.thoughtsTokenCount,
+    totalTokens: ProviderShared.totalTokens(usage.promptTokenCount, outputTokens, usage.totalTokenCount),
+    providerMetadata: { google: usage },
+  })
+}
+
+const mapFinishReason = (finishReason: string | undefined, hasToolCalls: boolean): FinishReason => {
+  if (finishReason === "STOP") return hasToolCalls ? "tool-calls" : "stop"
+  if (finishReason === "MAX_TOKENS") return "length"
+  if (
+    finishReason === "IMAGE_SAFETY" ||
+    finishReason === "RECITATION" ||
+    finishReason === "SAFETY" ||
+    finishReason === "BLOCKLIST" ||
+    finishReason === "PROHIBITED_CONTENT" ||
+    finishReason === "SPII"
+  )
+    return "content-filter"
+  if (finishReason === "MALFORMED_FUNCTION_CALL") return "error"
+  return "unknown"
+}
+
+const finish = (state: ParserState): ReadonlyArray<LLMEvent> =>
+  state.finishReason || state.usage
+    ? (() => {
+        const events: LLMEvent[] = []
+        const lifecycle = state.reasoningSignature
+          ? Lifecycle.reasoningEnd(
+              state.lifecycle,
+              events,
+              "reasoning-0",
+              googleMetadata({ thoughtSignature: state.reasoningSignature }),
+            )
+          : state.lifecycle
+        Lifecycle.finish(lifecycle, events, {
+          reason: mapFinishReason(state.finishReason, state.hasToolCalls),
+          usage: state.usage,
+        })
+        return events
+      })()
+    : []
+
+const step = (state: ParserState, event: GeminiEvent) => {
+  const nextState = {
+    ...state,
+    usage: event.usageMetadata ? (mapUsage(event.usageMetadata) ?? state.usage) : state.usage,
+  }
+  const candidate = event.candidates?.[0]
+  if (!candidate?.content)
+    return Effect.succeed([
+      { ...nextState, finishReason: candidate?.finishReason ?? nextState.finishReason },
+      [],
+    ] as const)
+
+  const events: LLMEvent[] = []
+  let hasToolCalls = nextState.hasToolCalls
+  let lifecycle = nextState.lifecycle
+  let nextToolCallId = nextState.nextToolCallId
+  let reasoningSignature = nextState.reasoningSignature
+
+  for (const part of candidate.content.parts) {
+    if ("thoughtSignature" in part && part.thoughtSignature && "thought" in part && part.thought)
+      reasoningSignature = part.thoughtSignature
+    if ("text" in part && part.text.length > 0) {
+      lifecycle = part.thought
+        ? Lifecycle.reasoningDelta(
+            lifecycle,
+            events,
+            "reasoning-0",
+            part.text,
+            part.thoughtSignature ? googleMetadata({ thoughtSignature: part.thoughtSignature }) : undefined,
+          )
+        : Lifecycle.textDelta(lifecycle, events, "text-0", part.text)
+      continue
+    }
+
+    if ("functionCall" in part) {
+      const input = part.functionCall.args
+      const id = `tool_${nextToolCallId++}`
+      lifecycle = Lifecycle.stepStart(lifecycle, events)
+      events.push(
+        LLMEvent.toolCall({
+          id,
+          name: part.functionCall.name,
+          input,
+          providerMetadata: part.thoughtSignature
+            ? googleMetadata({ thoughtSignature: part.thoughtSignature })
+            : undefined,
+        }),
+      )
+      hasToolCalls = true
+    }
+  }
+
+  return Effect.succeed([
+    {
+      ...nextState,
+      hasToolCalls,
+      lifecycle,
+      nextToolCallId,
+      reasoningSignature,
+      finishReason: candidate.finishReason ?? nextState.finishReason,
+    },
+    events,
+  ] as const)
+}
+
+// =============================================================================
+// Protocol And Gemini Route
+// =============================================================================
+/**
+ * The Gemini protocol — request body construction, body schema, and the
+ * streaming-event state machine. Used by Google AI Studio Gemini and (once
+ * registered) Vertex Gemini.
+ */
+export const protocol = Protocol.make({
+  id: ADAPTER,
+  body: {
+    schema: GeminiBody,
+    from: fromRequest,
+  },
+  stream: {
+    event: Protocol.jsonEvent(GeminiEvent),
+    initial: () => ({ hasToolCalls: false, nextToolCallId: 0, lifecycle: Lifecycle.initial() }),
+    step,
+    onHalt: finish,
+  },
+})
+
+export const route = Route.make({
+  id: ADAPTER,
+  provider: "google",
+  protocol,
+  // Gemini's path embeds the model id and pins SSE framing at the URL level.
+  endpoint: Endpoint.path(({ request }) => `/models/${request.model.id}:streamGenerateContent?alt=sse`, {
+    baseURL: DEFAULT_BASE_URL,
+  }),
+  auth: Auth.none,
+  framing: Framing.sse,
+})
+
+export * as Gemini from "./gemini"

package/src/protocols/index.ts

@@ -0,0 +1,6 @@
+export * as AnthropicMessages from "./anthropic-messages"
+export * as BedrockConverse from "./bedrock-converse"
+export * as Gemini from "./gemini"
+export * as OpenAIChat from "./openai-chat"
+export * as OpenAICompatibleChat from "./openai-compatible-chat"
+export * as OpenAIResponses from "./openai-responses"