@acentric/llm 0.0.0-stage → 0.1.0

package/src/convert/responses.ts

@@ -0,0 +1,203 @@
+// The Responses format's side of conversion (OpenAI, Fireworks).
+
+import type { Json, JsonObject } from '@acentric/types';
+
+import { isArray, isObject } from '../json.js';
+import type { Common, Context, FormatConverter } from './types.js';
+
+function texts(value: Json | undefined): string[] {
+  return isArray(value)
+    ? value.flatMap((part) =>
+        isObject(part) && typeof part['text'] === 'string' ? [part['text']] : [],
+      )
+    : [];
+}
+
+/** A reasoning item's readable text: its summary, or else its reasoning text. */
+function reasoningText(item: JsonObject): string {
+  const summary = texts(item['summary']);
+  return (summary.length > 0 ? summary : texts(item['content'])).join('\n\n');
+}
+
+function message(content: JsonObject[]): JsonObject {
+  return { type: 'message', role: 'assistant', content };
+}
+
+function outputText(text: string): JsonObject {
+  return { type: 'output_text', text, annotations: [] };
+}
+
+/** Reasoning the target cannot use as it is: dropped, or kept as text. */
+function carryReasoning(text: string, context: Context): JsonObject | undefined {
+  if (context.reasoning === 'drop' || text === '') {
+    context.change('dropped', 'reasoning', text === '' ? 'no readable text' : 'bound to its model');
+    return undefined;
+  }
+  if (context.target.plainReasoning) {
+    context.change('rewritten', 'reasoning', 'as a plain reasoning item');
+    return { type: 'reasoning', summary: [{ type: 'summary_text', text }] };
+  }
+  context.change('rewritten', 'reasoning', 'as assistant text');
+  return message([outputText(text)]);
+}
+
+/** An item without the IDs and status its provider gave it. */
+function withoutIds(item: JsonObject): JsonObject {
+  const { id: _id, status: _status, ...rest } = item;
+  return rest;
+}
+
+function string(value: Json | undefined, otherwise: string): string {
+  return typeof value === 'string' ? value : otherwise;
+}
+
+export const responsesConverter: FormatConverter = {
+  sameFormat(items, sameProvider, context) {
+    const out: JsonObject[] = [];
+    for (const item of items) {
+      switch (item['type']) {
+        case 'message':
+          out.push(sameProvider ? item : withoutIds(item));
+          break;
+        case 'reasoning': {
+          // A provider's own plain reasoning reads the same on any of its models.
+          const encrypted = item['encrypted_content'];
+          if (sameProvider && context.target.plainReasoning && !encrypted) {
+            out.push(item);
+            break;
+          }
+          const carried = carryReasoning(reasoningText(item), context);
+          if (carried !== undefined) out.push(carried);
+          break;
+        }
+        case 'function_call':
+          out.push(
+            sameProvider
+              ? item
+              : {
+                  type: 'function_call',
+                  call_id: string(item['call_id'], ''),
+                  name: string(item['name'], ''),
+                  arguments: string(item['arguments'], '{}'),
+                },
+          );
+          break;
+        case 'custom_tool_call':
+          if (context.target.customTools) {
+            out.push(sameProvider ? item : withoutIds(item));
+          } else {
+            const callId = item['call_id'];
+            if (typeof callId === 'string') context.droppedCalls.add(callId);
+            context.change('dropped', 'tool_call', `custom tool ${string(item['name'], '')}`);
+          }
+          break;
+        default:
+          if (sameProvider) out.push(item);
+          else context.change('dropped', 'provider_item', string(item['type'], 'unknown'));
+      }
+    }
+    return out;
+  },
+
+  read(items, context) {
+    const common: Common[] = [];
+    for (const item of items) {
+      switch (item['type']) {
+        case 'message': {
+          const content = item['content'];
+          for (const part of isArray(content) ? content : []) {
+            if (!isObject(part)) continue;
+            const text = part['text'];
+            const refusal = part['refusal'];
+            if (part['type'] === 'output_text' && typeof text === 'string') {
+              const annotations = part['annotations'];
+              if (isArray(annotations) && annotations.length > 0) {
+                context.change('dropped', 'citations', `${annotations.length} annotations`);
+              }
+              common.push({ kind: 'text', text });
+            } else if (part['type'] === 'refusal' && typeof refusal === 'string') {
+              common.push({ kind: 'refusal', text: refusal });
+            } else {
+              common.push({ kind: 'other', type: `message ${string(part['type'], 'part')}` });
+            }
+          }
+          break;
+        }
+        case 'reasoning':
+          common.push({ kind: 'reasoning', text: reasoningText(item) });
+          break;
+        case 'function_call':
+          common.push({
+            kind: 'call',
+            id: string(item['call_id'], ''),
+            name: string(item['name'], ''),
+            arguments: string(item['arguments'], '{}'),
+          });
+          break;
+        case 'custom_tool_call':
+          common.push({
+            kind: 'custom_call',
+            id: string(item['call_id'], ''),
+            name: string(item['name'], ''),
+            input: string(item['input'], ''),
+          });
+          break;
+        default:
+          common.push({ kind: 'other', type: string(item['type'], 'unknown') });
+      }
+    }
+    return common;
+  },
+
+  write(items, context) {
+    const out: JsonObject[] = [];
+    let open: JsonObject[] = [];
+    const close = () => {
+      if (open.length > 0) out.push(message(open));
+      open = [];
+    };
+    for (const item of items) {
+      switch (item.kind) {
+        case 'text':
+          open.push(outputText(item.text));
+          break;
+        case 'refusal':
+          open.push({ type: 'refusal', refusal: item.text });
+          break;
+        case 'reasoning': {
+          close();
+          const carried = carryReasoning(item.text, context);
+          if (carried !== undefined) out.push(carried);
+          break;
+        }
+        case 'call':
+          close();
+          out.push({
+            type: 'function_call',
+            call_id: context.callId(item.id),
+            name: item.name,
+            arguments: item.arguments,
+          });
+          break;
+        case 'custom_call':
+          close();
+          if (context.target.customTools) {
+            out.push({
+              type: 'custom_tool_call',
+              call_id: context.callId(item.id),
+              name: item.name,
+              input: item.input,
+            });
+          } else {
+            context.droppedCalls.add(item.id);
+            context.change('dropped', 'tool_call', `custom tool ${item.name}`);
+          }
+          break;
+        case 'other':
+          context.change('dropped', 'provider_item', item.type);
+      }
+    }
+    close();
+    return out;
+  },
+};

package/src/convert/types.ts

@@ -0,0 +1,104 @@
+// The shapes conversion works with.
+
+import type { JsonObject } from '@acentric/types';
+
+import type { ApiFormat, Provider } from '../providers.js';
+
+/** The provider and model a conversation is converted for. */
+export type ConvertTarget<P extends Provider = Provider> = {
+  readonly provider: P;
+  readonly model: string;
+};
+
+export type ConvertOptions = {
+  /**
+   * Reasoning the target cannot use as it is: another model's signed or
+   * encrypted reasoning, and any reasoning sent to another provider. `drop`,
+   * the default, leaves it out. `text` keeps its readable text: as a plain
+   * reasoning item where the target reads one (Fireworks), otherwise as
+   * assistant text. Reasoning with no readable text is dropped either way.
+   */
+  readonly reasoning?: 'drop' | 'text';
+};
+
+/** One thing `convert` dropped or rewrote. */
+export type ConversionChange = {
+  /** The message's index in the input. */
+  readonly index: number;
+  readonly action: 'dropped' | 'rewritten';
+  readonly what:
+    | 'message'
+    | 'reasoning'
+    | 'tool_call'
+    | 'tool_result'
+    | 'tool_call_id'
+    | 'tool_arguments'
+    | 'provider_item'
+    | 'citations'
+    | 'refusal'
+    | 'part'
+    | 'part_option';
+  /** What exactly, for logs, such as `web_search_call` or `cache_control`. */
+  readonly detail: string;
+};
+
+/** The converted messages, and every change made to get them. */
+export type Conversion<M> = {
+  readonly messages: M[];
+  readonly changes: readonly ConversionChange[];
+};
+
+/** What a target provider takes, as far as conversion cares. */
+export type Capabilities = {
+  readonly api: ApiFormat;
+  /** Free-form custom tools and their calls. */
+  readonly customTools: boolean;
+  /** File parts. */
+  readonly files: boolean;
+  /** Images and files by a provider's file ID. */
+  readonly fileIds: boolean;
+  /** Reads reasoning items that hold only text. */
+  readonly plainReasoning: boolean;
+};
+
+export type Target = Capabilities & ConvertTarget;
+
+/** One assistant item read without its format, on its way to another format. Never stored. */
+export type Common =
+  | { readonly kind: 'text'; readonly text: string }
+  | { readonly kind: 'refusal'; readonly text: string }
+  | { readonly kind: 'reasoning'; readonly text: string }
+  | {
+      readonly kind: 'call';
+      readonly id: string;
+      readonly name: string;
+      readonly arguments: string;
+    }
+  | {
+      readonly kind: 'custom_call';
+      readonly id: string;
+      readonly name: string;
+      readonly input: string;
+    }
+  | { readonly kind: 'other'; readonly type: string };
+
+/** What a conversation's conversion shares from message to message. */
+export type Context = {
+  readonly target: Target;
+  readonly reasoning: 'drop' | 'text';
+  /** Call IDs whose calls were dropped; their results are dropped too. */
+  readonly droppedCalls: Set<string>;
+  change(action: ConversionChange['action'], what: ConversionChange['what'], detail: string): void;
+  /** A call ID as the target takes it. Results are mapped the same way. */
+  callId(id: string): string;
+};
+
+/** One API format's side of conversion. */
+export type FormatConverter = {
+  /** The items of a message from another provider or model in this same format. */
+  sameFormat(items: readonly JsonObject[], sameProvider: boolean, context: Context): JsonObject[];
+  /** The items of a message in this format, read for another format. */
+  read(items: readonly JsonObject[], context: Context): Common[];
+  /** Common items written as this format's. */
+  write(items: readonly Common[], context: Context): JsonObject[];
+};

package/src/cost.ts

@@ -0,0 +1,71 @@
+// What a completion cost, estimated from its tokens and the model's prices,
+// for providers that do not say what they charged.
+
+import type { ModelCost, ModelPricing } from './model.js';
+import type { Usage, UsageCost } from './usage.js';
+
+export type CostOptions = {
+  /**
+   * Scales every price, for where and how the request ran, such as 0.5 for a
+   * batch or 1.1 for inference kept in the US. Defaults to 1.
+   */
+  readonly multiplier?: number;
+  /** The part of `usage.cacheWrite` written for an hour, priced at `cacheWrite1h`. */
+  readonly cacheWrite1h?: number;
+};
+
+/**
+ * The estimated cost, or undefined when it cannot be known: no prices, a count
+ * that is missing, or tokens in a part the prices do not cover. The prices for
+ * the whole request are those of the highest tier its prompt is over.
+ */
+export function estimateCost(
+  usage: Usage,
+  pricing: ModelPricing | undefined,
+  { multiplier = 1, cacheWrite1h = 0 }: CostOptions = {},
+): UsageCost | undefined {
+  const { totalInput, input, output, cacheRead, cacheWrite } = usage;
+  if (pricing === undefined || totalInput === undefined || input === undefined) return undefined;
+  if (output === undefined) return undefined;
+  if (cacheWrite1h > (cacheWrite ?? 0)) return undefined;
+
+  let prices: ModelCost = pricing.base;
+  for (const tier of pricing.tiers ?? []) {
+    if (totalInput > tier.aboveInputTokens) prices = tier.cost;
+  }
+
+  const read = part(cacheRead, prices.cacheRead);
+  const shortWrite = part(
+    cacheWrite === undefined ? undefined : cacheWrite - cacheWrite1h,
+    prices.cacheWrite,
+  );
+  const hourWrite = part(cacheWrite1h, prices.cacheWrite1h);
+  if (read === null || shortWrite === null || hourWrite === null) return undefined;
+  const written =
+    shortWrite === undefined && hourWrite === undefined
+      ? undefined
+      : (shortWrite ?? 0) + (hourWrite ?? 0);
+
+  const parts = {
+    input: price(input, prices.input) * multiplier,
+    output: price(output, prices.output) * multiplier,
+    ...(read === undefined ? {} : { cacheRead: read * multiplier }),
+    ...(written === undefined ? {} : { cacheWrite: written * multiplier }),
+  };
+  const total = parts.input + parts.output + (parts.cacheRead ?? 0) + (parts.cacheWrite ?? 0);
+  return { source: 'estimated', ...parts, total };
+}
+
+/** The price of some tokens: undefined when there are none to price, null when they have no price. */
+function part(
+  tokens: number | undefined,
+  perMillion: number | undefined,
+): number | undefined | null {
+  if (tokens === undefined) return undefined;
+  if (tokens === 0) return perMillion === undefined ? undefined : 0;
+  return perMillion === undefined ? null : price(tokens, perMillion);
+}
+
+function price(tokens: number, perMillion: number): number {
+  return (tokens * perMillion) / 1_000_000;
+}

package/src/failure.ts

@@ -0,0 +1,164 @@
+// Every way a completion can fail, as an error the client throws and as the
+// Json a step reads. An LlmError's toJSON() is what a runner reports as the
+// failure's error (RUNNERS.md §8), so step learns its kind, whether a retry
+// may help and how long to wait. Retrying is the agent's decision.
+
+import type { Json, Outcome } from '@acentric/types';
+
+import { isObject } from './json.js';
+import type { Usage } from './usage.js';
+
+/** Whether each kind is worth retrying when the error does not say otherwise. */
+const RETRYABLE = {
+  /** The client, the gateway or the provider refused the request. */
+  invalid_request: false,
+  /** Credentials missing, not valid or not allowed. */
+  auth: false,
+  /** Throttled; see `retryAfterMs`. */
+  rate_limited: true,
+  /** Out of credits or over a billing limit. */
+  quota_exceeded: false,
+  /** The prompt is too long for the model. */
+  context_overflow: false,
+  /** A provider's safety policy refused it; sending it again as it is will not help. */
+  policy_violation: false,
+  /** An unknown model, or a previous response that expired or is not valid; see `providerParam`. */
+  not_found: false,
+  /** The provider is out of capacity. */
+  overloaded: true,
+  /** Any other failure at the provider. */
+  provider_error: true,
+  /** The provider answered with something that cannot be read. */
+  invalid_response: false,
+  /** The provider or the gateway could not be reached. */
+  network_error: true,
+  /** The call took longer than allowed. */
+  timeout: true,
+  /** The handler's signal fired: the effect was cancelled, aborted or timed out. */
+  cancelled: false,
+  /** A bug in the gateway. */
+  internal_error: false,
+} as const satisfies { readonly [kind: string]: boolean };
+
+export type LlmFailureKind = keyof typeof RETRYABLE;
+
+export const LLM_FAILURE_KINDS: readonly LlmFailureKind[] = Object.keys(RETRYABLE).filter(
+  (kind): kind is LlmFailureKind => kind in RETRYABLE,
+);
+
+/** A failed completion as Json: what an LlmError's toJSON() returns, and what step reads. */
+export type LlmFailure = {
+  /** Always `LlmError`, which tells an LLM failure from other thrown values. */
+  readonly name: 'LlmError';
+  readonly message: string;
+  readonly kind: LlmFailureKind;
+  /** Whether sending the same request again may succeed. */
+  readonly retryable: boolean;
+  /** Left out for a failure before a provider was chosen. */
+  readonly provider?: string;
+  readonly providerCode?: string;
+  readonly providerType?: string;
+  /** The request field the provider blamed, such as `previous_response_id`. */
+  readonly providerParam?: string;
+  /** The provider's error as it answered, cut short when long. */
+  readonly providerError?: Json;
+  /** The provider's HTTP status, not the gateway's. */
+  readonly httpStatus?: number;
+  /** How long the provider asked to wait before trying again, in milliseconds. */
+  readonly retryAfterMs?: number;
+  /** The provider's request ID, for its support. */
+  readonly requestId?: string;
+  /** The provider's response ID, when a response was created before it failed. */
+  readonly responseId?: string;
+  /** Tokens the provider reported, and may bill, for a failed generation. */
+  readonly usage?: Usage;
+};
+
+type Detail = Omit<LlmFailure, 'name' | 'message' | 'kind' | 'retryable'>;
+
+/** What an LlmError is made from. `retryable` defaults by kind. */
+export type LlmErrorOptions = Detail & {
+  readonly kind: LlmFailureKind;
+  readonly retryable?: boolean;
+};
+
+const DETAIL: readonly (keyof Detail)[] = [
+  'provider',
+  'providerCode',
+  'providerType',
+  'providerParam',
+  'providerError',
+  'httpStatus',
+  'retryAfterMs',
+  'requestId',
+  'responseId',
+  'usage',
+];
+
+/** A failed completion. The client throws it; a runner reports its toJSON() to step. */
+export class LlmError extends Error {
+  override readonly name = 'LlmError';
+  readonly failure: LlmFailure;
+
+  constructor(message: string, options: LlmErrorOptions) {
+    super(message);
+    const { kind, retryable, ...detail } = options;
+    const failure: { [key: string]: Json | undefined } = {
+      name: 'LlmError',
+      message,
+      kind,
+      retryable: retryable ?? RETRYABLE[kind],
+    };
+    for (const key of DETAIL) if (detail[key] !== undefined) failure[key] = detail[key];
+    // oxlint-disable-next-line typescript/no-unsafe-type-assertion
+    this.failure = failure as LlmFailure;
+  }
+
+  get kind(): LlmFailureKind {
+    return this.failure.kind;
+  }
+
+  get retryable(): boolean {
+    return this.failure.retryable;
+  }
+
+  /** The failure as Json, which a runner reports as the effect's error. */
+  toJSON(): LlmFailure {
+    return this.failure;
+  }
+
+  /** An LlmError from its Json, such as a gateway's answer. */
+  static fromJSON({ message, kind, retryable, ...detail }: LlmFailure): LlmError {
+    const options: LlmErrorOptions = { ...omitName(detail), kind, retryable };
+    return new LlmError(message, options);
+  }
+}
+
+function omitName(detail: Detail & { readonly name?: string }): Detail {
+  const { name: _name, ...rest } = detail;
+  return rest;
+}
+
+/** Whether a value is an LlmFailure, such as a failed effect's error. */
+export function isLlmFailure(value: Json | undefined): value is LlmFailure {
+  if (!isObject(value)) return false;
+  const { name, message, kind, retryable } = value;
+  return (
+    name === 'LlmError' &&
+    typeof message === 'string' &&
+    typeof kind === 'string' &&
+    kind in RETRYABLE &&
+    typeof retryable === 'boolean'
+  );
+}
+
+/**
+ * The LLM failure in an effect's outcome, or null: for a result, for a
+ * failure the runner reported itself (a timeout, a crash) and for one thrown
+ * by something other than this package.
+ */
+export function llmFailure(outcome: Outcome): LlmFailure | null {
+  if (!('failed' in outcome)) return null;
+  const { error } = outcome.failed;
+  return isLlmFailure(error) ? error : null;
+}

package/src/fields.ts

@@ -0,0 +1,13 @@
+// Open records of Json: fields a shape passes through without interpreting.
+
+import type { Json } from '@acentric/types';
+
+/**
+ * Fields merged as they are into the provider's own block for a shared shape,
+ * such as Anthropic's `cache_control` on a content part or OpenAI's `detail`
+ * on an image. The client never interprets them.
+ */
+export type ProviderFields = { readonly [key: string]: Json };
+
+/** The application's own attributes on a part or message. Never sent to a provider. */
+export type Metadata = { readonly [key: string]: Json };