opencode-plugboard 0.6.0 → 0.8.0

package/src/openai/fanout.ts

@@ -0,0 +1,278 @@
+/**
+ * Several choices for one request (`n` > 1 on Chat Completions, `n` or several prompts on legacy
+ * completions): one independent `ModelCaller` call per choice, run concurrently, each in its own
+ * session, merged into one response.
+ *
+ * Every choice is a full model call and a kept history session, which is why `maxN` caps them.
+ * The choices share nothing: no history, no seed, no ordering.
+ *
+ * Failure policy: if any choice fails, the request fails and the other choices are cancelled
+ * (their sessions are interrupted). A partial answer set would look like a complete one, and
+ * callers that sample n times for evaluation would silently average over fewer samples.
+ *   - non-stream: the first failure becomes the HTTP error.
+ *   - stream, nothing sent yet: the same HTTP error.
+ *   - stream, already committed: an `error` frame and `[DONE]`, as for a single stream.
+ * Client disconnect and the request deadline cancel every choice (the group signal includes the
+ * request's).
+ *
+ * The wire encoders stay per-choice and come in through `ChoiceCodec`: Chat Completions here,
+ * legacy completions in `src/completions/response.ts`. This module owns what is shared: the
+ * group cancellation, the merge of JSON bodies and the interleaving of SSE chunks.
+ */
+
+import type { ServerResponse } from "node:http"
+import type { PlugError, PlugEvent, PlugRequest, PlugUsage } from "../contract.js"
+import { RequestAborted, boundEvents, sendJson, sseWriter, type RequestScope } from "../http.js"
+import { chatUsageChunk, collectCompletion, encodeError, streamCompletion, type OpenAiExchange, type StreamChoice, type StreamResult } from "./response.js"
+import type { RouteHost } from "./route.js"
+
+const SESSION_HEADER = "x-plugboard-session"
+
+/** The per-choice wire encoder a front-end supplies. */
+export interface ChoiceCodec {
+  readonly stream: boolean
+  readonly includeUsage: boolean
+  /** Prefix of the shared stream id (`chatcmpl-`, `cmpl-`). */
+  readonly idPrefix: string
+  /** One choice as a complete JSON body with `choices[0]` and `usage`. */
+  collect(events: AsyncIterable<PlugEvent>, index: number): Promise<{ status: number; body: object }>
+  /** One choice of the shared stream: this choice's chunks only (see `StreamChoice`). */
+  streamChoice(events: AsyncIterable<PlugEvent>, write: (chunk: string) => void, choice: StreamChoice): Promise<StreamResult>
+  /** The final chunk with the summed usage (`include_usage`). */
+  usageChunk(choice: Omit<StreamChoice, "index">, usage: PlugUsage): object
+}
+
+/** The Chat Completions encoder as a `ChoiceCodec`. */
+export const chatCodec = (exchange: OpenAiExchange): ChoiceCodec => ({
+  stream: exchange.stream,
+  includeUsage: exchange.includeUsage,
+  idPrefix: "chatcmpl-",
+  collect: (events) => collectCompletion(events, exchange),
+  streamChoice: (events, write, choice) => streamCompletion(events, exchange, write, choice),
+  usageChunk: (choice, usage) => chatUsageChunk(exchange.model, choice.id, choice.created, usage),
+})
+
+export interface ChoicesCall {
+  readonly app: string
+  /** One resolved model call per choice, in choice order. Every entry is a full call with its own session. */
+  readonly calls: ReadonlyArray<PlugRequest>
+  readonly response: ServerResponse
+  readonly scope: RequestScope
+  /** The route's log fields: the outcome of a stream that failed after it committed. */
+  readonly info: { failed?: string }
+  readonly codec: ChoiceCodec
+}
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value)
+
+/** A queue that is an async iterable: the encoder of one choice reads what the interleaver feeds it. */
+function channel<T>() {
+  const items: T[] = []
+  let closed = false
+  let wake: (() => void) | undefined
+  return {
+    push(item: T) {
+      items.push(item)
+      wake?.()
+    },
+    close() {
+      closed = true
+      wake?.()
+    },
+    async *[Symbol.asyncIterator](): AsyncGenerator<T> {
+      while (true) {
+        if (items.length > 0) {
+          yield items.shift()!
+          continue
+        }
+        if (closed) return
+        await new Promise<void>((resolve) => {
+          wake = resolve
+        })
+        wake = undefined
+      }
+    },
+  }
+}
+
+/** A scope whose signal also aborts when one choice fails; deadline and error stay the request's. */
+function groupScope(scope: RequestScope, signal: AbortSignal): RequestScope {
+  return {
+    signal,
+    get timedOut() {
+      return scope.timedOut
+    },
+    timeoutError: scope.timeoutError,
+    race<T>(promise: Promise<T>): Promise<T> {
+      promise.catch(() => {})
+      if (signal.aborted) return Promise.reject(signal.reason)
+      return new Promise<T>((resolve, reject) => {
+        const onAbort = () => reject(signal.reason)
+        signal.addEventListener("abort", onAbort, { once: true })
+        promise.then(
+          (value) => {
+            signal.removeEventListener("abort", onAbort)
+            resolve(value)
+          },
+          (error) => {
+            signal.removeEventListener("abort", onAbort)
+            reject(error)
+          },
+        )
+      })
+    },
+    dispose() {},
+  }
+}
+
+/** Numbers add up, objects add up per key. */
+const sumDeep = (a: unknown, b: unknown): unknown => {
+  if (typeof a === "number" && typeof b === "number") return a + b
+  if (isRecord(a) && isRecord(b)) return Object.fromEntries(Object.keys(a).map((key) => [key, sumDeep(a[key], b[key])]))
+  return a ?? b
+}
+
+/** The bodies of successful choices as one: `choices` in order with their `index`, `usage` summed, the rest from the first. */
+function mergeBodies(bodies: ReadonlyArray<object>): object {
+  const [head] = bodies as ReadonlyArray<Record<string, unknown>>
+  const choices = (bodies as ReadonlyArray<{ choices: object[] }>).map((body, index) => ({ ...body.choices[0], index }))
+  const usage = (bodies as ReadonlyArray<{ usage?: unknown }>).map((body) => body.usage).reduce((total, next) => (total === undefined ? next : sumDeep(total, next)), undefined as unknown)
+  return { ...head, choices, ...(usage === undefined ? {} : { usage }) }
+}
+
+const addUsage = (a: PlugUsage, b: PlugUsage): PlugUsage => {
+  const out: { -readonly [K in keyof PlugUsage]: PlugUsage[K] } = {}
+  for (const key of ["inputTokens", "outputTokens", "reasoningTokens", "cacheReadTokens", "cacheWriteTokens"] as const) {
+    if (a[key] !== undefined || b[key] !== undefined) out[key] = (a[key] ?? 0) + (b[key] ?? 0)
+  }
+  return out
+}
+
+export async function serveChoices(host: Pick<RouteHost, "caller">, { app, calls, response, scope, info, codec }: ChoicesCall): Promise<void> {
+  const fail = (error: PlugError) => sendJson(response, error.status, encodeError(error))
+  // Abandoned requests (client gone, deadline passed) must never reach the model caller.
+  if (scope.signal.aborted) throw scope.signal.reason
+
+  const group = new AbortController()
+  const cancel = () => group.abort(new RequestAborted("disconnected"))
+  const gscope = groupScope(scope, AbortSignal.any([scope.signal, group.signal]))
+  /** History record ids as the callers report them, in choice order, for the response header (comma separated). */
+  const records: Array<string | undefined> = []
+  const open = (index: number) =>
+    boundEvents(
+      host.caller.stream(calls[index]!, {
+        app,
+        signal: gscope.signal,
+        onRecord: (recordID) => {
+          if (response.headersSent || response.destroyed || !/^[\x21-\x7e]+$/.test(recordID)) return
+          records[index] = recordID
+          response.setHeader(SESSION_HEADER, records.filter((id) => id !== undefined).join(","))
+        },
+      }),
+      gscope,
+    )
+
+  if (!codec.stream) {
+    const results = await new Promise<Array<{ status: number; body: object }>>((resolve, reject) => {
+      const done: Array<{ status: number; body: object }> = []
+      let left = calls.length
+      calls.forEach((_, index) => {
+        codec.collect(open(index), index).then(
+          (result) => {
+            if (result.status !== 200) {
+              cancel()
+              return resolve([result])
+            }
+            done[index] = result
+            if (--left === 0) resolve(done)
+          },
+          (error) => {
+            cancel()
+            reject(error)
+          },
+        )
+      })
+    })
+    const first = results[0]!
+    return sendJson(response, first.status, results.length === 1 ? first.body : mergeBodies(results.map((result) => result.body)))
+  }
+
+  const sse = sseWriter(response, scope.signal)
+  const shared = { id: `${codec.idPrefix}${crypto.randomUUID().replaceAll("-", "").slice(0, 29)}`, created: Math.floor(Date.now() / 1000) }
+  const channels = calls.map(() => channel<PlugEvent>())
+  let failure: PlugError | undefined
+  let usage: PlugUsage | undefined
+  const runs = channels.map((events, index) =>
+    codec.streamChoice(events, sse.write, { ...shared, index }).then((result) => {
+      if (result.failure && !failure) {
+        failure = result.failure
+        cancel()
+      }
+      if (result.usage) usage = usage ? addUsage(usage, result.usage) : result.usage
+    }),
+  )
+
+  // The choice an event belongs to. Set just before it is yielded; the consumer reads it before it
+  // pulls again, so it cannot change in between.
+  let current = 0
+  /**
+   * The choices' events as one stream, so one `sse.paced` serves all: backpressure, and the
+   * keepalive commit at the first non-terminal event of any choice. At most one event per choice
+   * is pulled ahead. A choice's stream ending closes its channel.
+   */
+  async function* merged(): AsyncGenerator<PlugEvent> {
+    const iterators = calls.map((_, index) => open(index)[Symbol.asyncIterator]())
+    type Step = { index: number; result?: IteratorResult<PlugEvent>; error?: unknown; threw?: true }
+    const pending = new Map<number, Promise<Step>>()
+    const arm = (index: number) =>
+      pending.set(
+        index,
+        iterators[index]!.next().then(
+          (result): Step => ({ index, result }),
+          (error): Step => ({ index, error, threw: true }),
+        ),
+      )
+    try {
+      for (let index = 0; index < iterators.length; index++) arm(index)
+      while (pending.size > 0 && !failure) {
+        const step = await Promise.race(pending.values())
+        pending.delete(step.index)
+        if (step.threw) throw step.error
+        if (step.result!.done) {
+          channels[step.index]!.close()
+          continue
+        }
+        current = step.index
+        yield step.result!.value
+        arm(step.index)
+      }
+    } finally {
+      for (const iterator of iterators) void iterator.return?.(undefined)?.catch?.(() => {})
+      for (const events of channels) events.close()
+    }
+  }
+
+  try {
+    for await (const event of sse.paced(merged())) channels[current]!.push(event)
+  } catch (error) {
+    cancel()
+    await Promise.all(runs)
+    throw error
+  }
+  await Promise.all(runs)
+
+  if (!failure && !sse.started && !response.headersSent) failure = { status: 502, kind: "server", message: "No response from the model" }
+  if (failure) {
+    if (!sse.started && !response.headersSent) {
+      if (!scope.signal.aborted || scope.timedOut) fail(failure)
+    } else {
+      info.failed = failure.code ?? failure.kind
+      sse.write(`data: ${JSON.stringify(encodeError(failure))}\n\ndata: [DONE]\n\n`)
+      if (!response.writableEnded) response.end()
+    }
+    return
+  }
+  if (codec.includeUsage && usage) sse.write(`data: ${JSON.stringify(codec.usageChunk(shared, usage))}\n\n`)
+  sse.write("data: [DONE]\n\n")
+  if (!response.writableEnded) response.end()
+}

package/src/openai/request.ts

@@ -25,6 +25,8 @@ export interface ParsedChat {
   readonly request: Omit<PlugRequest, "ref">
   /** Response presentation state for the encoders; never reaches the model call. */
   readonly exchange: OpenAiExchange
+  /** `n`: independent model calls whose answers become `choices` (1..`maxN`). The route fans out when it is above 1. */
+  readonly n: number
   readonly reasoningEffort?: string
 }
 
@@ -226,7 +228,16 @@ const parseResponseFormat = (raw: unknown): { structured?: PlugTool; instruction
   return bad(`Unsupported response_format type "${String(raw.type)}"`, "response_format.type")
 }
 
-const parseGeneration = (body: JsonObject): PlugGeneration | PlugError => {
+/** `n` is how many completions to return. Each one is a full model call (see `src/openai/fanout.ts`), so it is capped by the `maxN` option. */
+export const parseChoiceCount = (body: JsonObject, options: PlugboardOptions): number | PlugError => {
+  if (body.n == null) return 1
+  if (typeof body.n !== "number" || !Number.isInteger(body.n) || body.n < 1) return bad("n must be a positive integer", "n")
+  if (body.n > options.maxN)
+    return bad(`n=${body.n} is above the limit of ${options.maxN} (option \`maxN\`; each choice is a full model call)`, "n", "too_many_choices")
+  return body.n
+}
+
+export const parseGeneration = (body: JsonObject): PlugGeneration | PlugError => {
   const out: { -readonly [K in keyof PlugGeneration]: PlugGeneration[K] } = {}
   const number = (key: string, min: number, max: number, integer: boolean): number | undefined | PlugError => {
     const value = body[key]
@@ -265,10 +276,8 @@ export function parseChatRequest(body: unknown, options: PlugboardOptions): Pars
   if (!Array.isArray(body.messages) || body.messages.length === 0) return bad("messages must be a non-empty array", "messages")
 
   // Parameters the session backend cannot honour are rejected, not ignored.
-  if (body.n != null) {
-    if (typeof body.n !== "number" || !Number.isInteger(body.n) || body.n < 1) return bad("n must be a positive integer", "n")
-    if (body.n > 1) return bad("Only n=1 is supported", "n", "unsupported_parameter")
-  }
+  const n = parseChoiceCount(body, options)
+  if (typeof n === "object") return n
   if (body.logprobs === true) return bad("logprobs is not supported", "logprobs", "unsupported_parameter")
   if (body.top_logprobs != null) return bad("top_logprobs is not supported", "top_logprobs", "unsupported_parameter")
 
@@ -323,6 +332,7 @@ export function parseChatRequest(body: unknown, options: PlugboardOptions): Pars
   const streamOptions = isRecord(body.stream_options) ? body.stream_options : {}
   return {
     request,
+    n,
     exchange: {
       model: body.model,
       stream: body.stream === true,

package/src/openai/response.ts

@@ -40,6 +40,20 @@ export interface StreamResult {
   readonly failure?: PlugError
   /** Whether any frame was written (the response is committed as a 200 SSE stream). */
   readonly started: boolean
+  /** The usage the model reported, when the exchange succeeded. */
+  readonly usage?: PlugUsage
+}
+
+/**
+ * One choice of a fanned-out stream (`n` > 1, see `src/openai/fanout.ts`): the chunks share the
+ * stream's `id` and `created` and carry this choice's `index`. The encoder then writes this
+ * choice's chunks only. The error frame, the usage chunk and `[DONE]` belong to the whole
+ * stream and are written by the fan-out.
+ */
+export interface StreamChoice {
+  readonly id: string
+  readonly created: number
+  readonly index: number
 }
 
 const ERROR_TYPES: Record<PlugErrorKind, string> = {
@@ -246,6 +260,16 @@ export async function collectCompletion(
   }
 }
 
+/** The final chunk of a stream with `include_usage`: no choices, the totals. */
+export const chatUsageChunk = (model: string, id: string, created: number, usage: PlugUsage): object => ({
+  id,
+  object: "chat.completion.chunk",
+  created,
+  model,
+  choices: [],
+  usage: encodeUsage(usage),
+})
+
 /**
  * Writes SSE frames via `write` (each frame is one complete `data: ...\n\n` string) and reports
  * how the exchange ended. A failure before the first frame is not written (the first wire-visible
@@ -256,9 +280,11 @@ export async function streamCompletion(
   events: AsyncIterable<PlugEvent>,
   exchange: OpenAiExchange,
   write: (chunk: string) => void,
+  choice?: StreamChoice,
 ): Promise<StreamResult> {
-  const id = newId()
-  const created = now()
+  const id = choice?.id ?? newId()
+  const created = choice?.created ?? now()
+  const index = choice?.index ?? 0
   const frame = (value: object) => write(`data: ${JSON.stringify(value)}\n\n`)
   const chunk = (delta: Record<string, unknown>, finish: string | null = null) =>
     frame({
@@ -266,7 +292,7 @@ export async function streamCompletion(
       object: "chat.completion.chunk",
       created,
       model: exchange.model,
-      choices: [{ index: 0, delta, finish_reason: finish, logprobs: null }],
+      choices: [{ index, delta, finish_reason: finish, logprobs: null }],
       ...(exchange.includeUsage ? { usage: null } : {}),
     })
 
@@ -348,16 +374,18 @@ export async function streamCompletion(
   failure ??= finish ? (passthrough ? undefined : structured.failure(exchange, finish, answered)) : noFinish()
   if (failure) {
     if (!started) return { failure, started }
-    frame(encodeError(failure))
-    write("data: [DONE]\n\n")
+    if (!choice) {
+      frame(encodeError(failure))
+      write("data: [DONE]\n\n")
+    }
     return { failure, started }
   }
   flush()
   start()
   if (held && isStructured) chunk({ content: structured.content(held) })
   chunk({}, finishReason(finish!, calls, isStructured))
-  if (exchange.includeUsage && usage)
-    frame({ id, object: "chat.completion.chunk", created, model: exchange.model, choices: [], usage: encodeUsage(usage) })
+  if (choice) return { started, ...(usage ? { usage } : {}) }
+  if (exchange.includeUsage && usage) frame(chatUsageChunk(exchange.model, id, created, usage))
   write("data: [DONE]\n\n")
   return { started }
 }

package/src/openai/route.ts

@@ -1,6 +1,7 @@
 /**
  * OpenAI front-end route table: `GET /healthz`, `GET /v1/models`, `GET /v1/models/{id}`,
- * `POST /v1/chat/completions`, and the dispatch of `POST /v1/responses` to `src/responses/route.ts`.
+ * `POST /v1/chat/completions` (`n` > 1 through `./fanout.ts`), and the dispatch of `POST /v1/responses`
+ * and `POST /v1/completions` to `src/responses/route.ts` and `src/completions/route.ts`.
  * Observability and first-run routes (`src/discovery.ts`, `src/metrics.ts`): `GET /` and `GET /v1`
  * (keyless discovery), the authenticated `GET /v1/status`, and `GET /metrics` (option `metrics`).
  *
@@ -38,8 +39,11 @@ import {
 import { errorMessage, log, logError, withRequestId } from "../log.js"
 import { NO_APP, outcomeLabel, type Metrics } from "../metrics.js"
 import { listAvailableModels, resolveModel } from "../models.js"
+import { isMessagesPath, serveMessages } from "../anthropic/route.js"
+import { serveCompletions } from "../completions/route.js"
 import { serveResponses } from "../responses/route.js"
 import { processMetrics, processStatus, type RuntimeStatus } from "../runtime.js"
+import { chatCodec, serveChoices } from "./fanout.js"
 import { parseChatRequest } from "./request.js"
 import { collectCompletion, encodeError, encodeModelList, streamCompletion } from "./response.js"
 import { dropRejected } from "./sampling.js"
@@ -60,7 +64,7 @@ const SESSION_HEADER = "x-plugboard-session"
 const REQUEST_ID_HEADER = "x-plugboard-request-id"
 const DROPPED_HEADER = "x-plugboard-dropped-params"
 const MODEL_PREFIX = "/v1/models/"
-const KNOWN_ROUTES = ["GET /v1/models", "POST /v1/chat/completions", "POST /v1/responses", "GET /v1/status"]
+const KNOWN_ROUTES = ["GET /v1/models", "POST /v1/chat/completions", "POST /v1/completions", "POST /v1/responses", "GET /v1/status"]
 /** Pages anyone may fetch; they decide for themselves how much to show (see `src/discovery.ts`). */
 const DISCOVERY_ROUTES = ["GET /", "GET /v1"]
 /** Allows a page's inline style and nothing else: no scripts, images, fonts or frames. */
@@ -135,6 +139,8 @@ export function createOpenAiRoute(routeHost: RouteHost) {
     const route = `${request.method} ${modelId !== undefined ? "/v1/models/{id}" : path}`
     // Arbitrary client paths stay out of the log.
     info.route = modelId !== undefined || known(route) || route === "GET /healthz" ? route : "unknown"
+    // Anthropic Messages: its own error shape and x-api-key auth, so it takes the request from here.
+    if (isMessagesPath(path)) return serveMessages(host, { path, request, response, scope, info })
 
     const rejected = checkOrigin(request, host.options)
     if (rejected) return fail(response, rejected)
@@ -202,6 +208,7 @@ export function createOpenAiRoute(routeHost: RouteHost) {
 
     const body = await readJsonBody(request, scope.signal)
     if (route === "POST /v1/responses") return serveResponses(host, { app: auth.app, body, response, scope, info })
+    if (route === "POST /v1/completions") return serveCompletions(host, { app: auth.app, body, response, scope, info })
     const parsed = parseChatRequest(body, host.options)
     if (isPlugError(parsed)) return fail(response, parsed)
     const catalog = await scope.race(host.loadCatalog())
@@ -217,6 +224,9 @@ export function createOpenAiRoute(routeHost: RouteHost) {
     }
     const call: PlugRequest = { ...parsed.request, ...(sampled ? { generation: sampled.generation } : {}), ref }
     const { exchange } = parsed
+    // n > 1: one independent call per choice, merged (src/openai/fanout.ts).
+    if (parsed.n > 1)
+      return serveChoices(host, { app: auth.app, calls: Array.from({ length: parsed.n }, () => call), response, scope, info, codec: chatCodec(exchange) })
 
     // Abandoned requests (client gone, deadline passed) must never reach the model caller.
     if (scope.signal.aborted) throw scope.signal.reason