@volcengine/ark-runtime 1.0.0

package/src/types/tokenization.ts

@@ -0,0 +1,24 @@
+import type { HttpHeaders } from "./common";
+
+export interface TokenizationRequest {
+  text: string | string[];
+  model: string;
+  user?: string;
+}
+
+export interface Tokenization {
+  index: number;
+  object: string;
+  total_tokens: number;
+  token_ids: number[];
+  offset_mapping: number[][];
+}
+
+export interface TokenizationResponse {
+  id: string;
+  created: number;
+  model: string;
+  object: string;
+  data: Tokenization[];
+  headers?: HttpHeaders;
+}

package/src/utils/breaker-provider.ts

@@ -0,0 +1,17 @@
+import { Breaker } from "./breaker";
+
+/**
+ * Maintains a per-model Breaker instance.
+ */
+export class ModelBreakerProvider {
+  private breakers = new Map<string, Breaker>();
+
+  getOrCreate(model: string): Breaker {
+    let breaker = this.breakers.get(model);
+    if (!breaker) {
+      breaker = new Breaker();
+      this.breakers.set(model, breaker);
+    }
+    return breaker;
+  }
+}

package/src/utils/breaker.ts

@@ -0,0 +1,56 @@
+/**
+ * Circuit breaker with slow-start strategy.
+ *
+ * After reset(durationMs):
+ * - No requests allowed during the cooldown period
+ * - After cooldown: allow 1, then 2, 4, 8... requests (exponential)
+ * - After 10s past cooldown: allow all
+ */
+export class Breaker {
+  private allowTime: number; // timestamp (ms)
+  private waiters = new Map<string, number>(); // id → index
+  private nextIndex = 0;
+
+  constructor() {
+    this.allowTime = Date.now();
+  }
+
+  /**
+   * Check if a request with given waitIndex is allowed through.
+   */
+  private allow(waitIndex: number): boolean {
+    const elapsed = (Date.now() - this.allowTime) / 1000;
+    if (elapsed <= 0) return false;
+    if (elapsed > 10) return true;
+    return waitIndex < Math.pow(2, elapsed);
+  }
+
+  private getAllowedDelayMs(): number {
+    const delay = this.allowTime - Date.now();
+    return delay < 1000 ? 1000 : delay;
+  }
+
+  /**
+   * Reset the breaker — block all requests for `durationMs`.
+   */
+  reset(durationMs: number): void {
+    this.allowTime = Date.now() + durationMs;
+  }
+
+  /**
+   * Wait until the breaker allows this request through.
+   */
+  async wait(): Promise<void> {
+    const id = String(this.nextIndex++);
+    const idx = this.waiters.size;
+    this.waiters.set(id, idx);
+
+    try {
+      while (!this.allow(idx)) {
+        await new Promise((r) => setTimeout(r, this.getAllowedDelayMs()));
+      }
+    } finally {
+      this.waiters.delete(id);
+    }
+  }
+}

package/src/utils/normalize.ts

@@ -0,0 +1,154 @@
+/**
+ * Response normalization utilities.
+ *
+ * The Go SDK serializes responses through Go structs, which means:
+ *   - Fields WITHOUT `omitempty` are always present (even when zero/null)
+ *   - Fields WITH `omitempty` are omitted when zero/null/empty
+ *
+ * The Node.js SDK receives raw JSON from the server (via Axios / JSON.parse),
+ * so fields may or may not be present depending on what the server returns.
+ *
+ * These functions align Node.js responses to match Go SDK serialization behavior.
+ */
+
+import type {
+  ChatCompletionResponse,
+  ChatCompletionStreamResponse,
+  ChatCompletionChoice,
+  ChatCompletionStreamChoice,
+} from "../types/chat-completion";
+import type { EmbeddingResponse } from "../types/embeddings";
+import type { Usage } from "../types/common";
+
+// ─── Chat Completion (non-stream) ────────────────────────────
+
+/**
+ * Normalize a ChatCompletionResponse to match Go SDK output.
+ *
+ * Go struct tags:
+ *   ChatCompletionMessage.Name      *string  `json:"name"`            → always present
+ *   ChatCompletionChoice.LogProbs   *LogProbs `json:"logprobs,omitempty"` → omitted when nil
+ *   ChatCompletionChoice.FinishReason FinishReason `json:"finish_reason"` → always present (custom MarshalJSON: empty → null)
+ */
+export function normalizeChatCompletionResponse(
+  resp: ChatCompletionResponse,
+): ChatCompletionResponse {
+  if (resp.choices) {
+    for (const choice of resp.choices) {
+      normalizeChatCompletionChoice(choice);
+    }
+  }
+  return resp;
+}
+
+function normalizeChatCompletionChoice(choice: ChatCompletionChoice): void {
+  // Go: Name *string `json:"name"` — no omitempty, always serialized
+  if (choice.message && !("name" in choice.message)) {
+    (choice.message as any).name = null;
+  }
+
+  // Go: LogProbs *LogProbs `json:"logprobs,omitempty"` — omitted when nil
+  if ("logprobs" in choice && (choice.logprobs === null || choice.logprobs === undefined)) {
+    delete (choice as any).logprobs;
+  }
+
+  // Go: FinishReason `json:"finish_reason"` — always present, custom MarshalJSON
+  // (already present in server response, but ensure it exists)
+  if (!("finish_reason" in choice)) {
+    (choice as any).finish_reason = null;
+  }
+}
+
+// ─── Chat Completion Stream ──────────────────────────────────
+
+/**
+ * Normalize a ChatCompletionStreamResponse chunk to match Go SDK output.
+ *
+ * Go struct tags:
+ *   ChatCompletionStreamChoiceDelta.Content  string   `json:"content,omitempty"`  → omitted when ""
+ *   ChatCompletionStreamChoiceDelta.Role     string   `json:"role,omitempty"`     → omitted when ""
+ *   ChatCompletionStreamChoice.FinishReason  FinishReason `json:"finish_reason"`  → always present (null when empty)
+ *   ChatCompletionStreamChoice.LogProbs      *LogProbs `json:"logprobs,omitempty"` → omitted when nil
+ *   ChatCompletionStreamResponse.Usage       *Usage   `json:"usage,omitempty"`    → omitted when nil
+ */
+export function normalizeChatCompletionStreamChunk(
+  chunk: ChatCompletionStreamResponse,
+): ChatCompletionStreamResponse {
+  if (chunk.choices) {
+    for (const choice of chunk.choices) {
+      normalizeStreamChoice(choice);
+    }
+  }
+
+  // Go: Usage *Usage `json:"usage,omitempty"` — omitted when nil
+  if ("usage" in chunk && (chunk.usage === null || chunk.usage === undefined)) {
+    delete (chunk as any).usage;
+  }
+
+  return chunk;
+}
+
+function normalizeStreamChoice(choice: ChatCompletionStreamChoice): void {
+  // Go: FinishReason `json:"finish_reason"` — no omitempty, custom MarshalJSON outputs null for empty
+  if (!("finish_reason" in choice)) {
+    (choice as any).finish_reason = null;
+  }
+
+  // Go: LogProbs *LogProbs `json:"logprobs,omitempty"` — omitted when nil
+  if ("logprobs" in choice && (choice.logprobs === null || choice.logprobs === undefined)) {
+    delete (choice as any).logprobs;
+  }
+
+  if (choice.delta) {
+    // Go: Content string `json:"content,omitempty"` — omitted when empty string
+    if ("content" in choice.delta && choice.delta.content === "") {
+      delete (choice.delta as any).content;
+    }
+
+    // Go: Role string `json:"role,omitempty"` — omitted when empty string
+    if ("role" in choice.delta && choice.delta.role === "") {
+      delete (choice.delta as any).role;
+    }
+  }
+}
+
+// ─── Embeddings ──────────────────────────────────────────────
+
+/**
+ * Normalize an EmbeddingResponse to match Go SDK output.
+ *
+ * Go struct tags (Usage):
+ *   PromptTokens            int                    `json:"prompt_tokens"`              → always present
+ *   CompletionTokens        int                    `json:"completion_tokens"`          → always present (even 0)
+ *   TotalTokens             int                    `json:"total_tokens"`               → always present
+ *   PromptTokensDetails     PromptTokensDetail     `json:"prompt_tokens_details"`      → always present (even zero struct)
+ *   CompletionTokensDetails CompletionTokensDetails `json:"completion_tokens_details"` → always present (even zero struct)
+ */
+export function normalizeEmbeddingResponse(
+  resp: EmbeddingResponse,
+): EmbeddingResponse {
+  if (resp.usage) {
+    normalizeUsage(resp.usage);
+  }
+  return resp;
+}
+
+/**
+ * Ensure Usage has all fields that Go always serializes (no omitempty).
+ */
+export function normalizeUsage(usage: Usage): void {
+  // Go: CompletionTokens int `json:"completion_tokens"` — no omitempty
+  if (usage.completion_tokens === undefined || usage.completion_tokens === null) {
+    usage.completion_tokens = 0;
+  }
+
+  // Go: PromptTokensDetails PromptTokensDetail `json:"prompt_tokens_details"` — no omitempty
+  if (!usage.prompt_tokens_details) {
+    usage.prompt_tokens_details = { cached_tokens: 0 };
+  }
+
+  // Go: CompletionTokensDetails CompletionTokensDetails `json:"completion_tokens_details"` — no omitempty
+  if (!usage.completion_tokens_details) {
+    usage.completion_tokens_details = { reasoning_tokens: 0 };
+  }
+}

package/src/utils/request-builder.ts

@@ -0,0 +1,51 @@
+import type { AxiosRequestConfig } from "axios";
+import { genRequestId } from "./request-id";
+import { ClientRequestHeader } from "../types/common";
+
+export interface RequestOptions {
+  body?: unknown;
+  contentType?: string;
+  projectName?: string;
+  customHeaders?: Record<string, string>;
+  query?: Record<string, string>;
+  signal?: AbortSignal;
+  stream?: boolean;
+}
+
+/**
+ * Build an AxiosRequestConfig from method, url, and request options.
+ */
+export function buildRequest(
+  method: string,
+  url: string,
+  authHeader: Record<string, string>,
+  opts: RequestOptions = {},
+): AxiosRequestConfig {
+  const headers: Record<string, string> = {
+    [ClientRequestHeader]: genRequestId(),
+    "Content-Type": opts.contentType ?? "application/json",
+    ...authHeader,
+    ...opts.customHeaders,
+  };
+
+  const config: AxiosRequestConfig = {
+    method: method as any,
+    url,
+    headers,
+    signal: opts.signal,
+  };
+
+  if (opts.body !== undefined) {
+    config.data = opts.body;
+  }
+
+  if (opts.query) {
+    config.params = opts.query;
+  }
+
+  if (opts.stream) {
+    config.responseType = "stream";
+  }
+
+  return config;
+}

package/src/utils/request-id.ts

@@ -0,0 +1,17 @@
+import { randomBytes } from "crypto";
+
+/**
+ * Generate a request ID in the format: YYYYMMDDHHmmss + 20 hex digits.
+ */
+export function genRequestId(): string {
+  const now = new Date();
+  const ts =
+    now.getFullYear().toString() +
+    String(now.getMonth() + 1).padStart(2, "0") +
+    String(now.getDate()).padStart(2, "0") +
+    String(now.getHours()).padStart(2, "0") +
+    String(now.getMinutes()).padStart(2, "0") +
+    String(now.getSeconds()).padStart(2, "0");
+  const hex = randomBytes(10).toString("hex");
+  return ts + hex;
+}

package/src/utils/retry.ts

@@ -0,0 +1,76 @@
+export interface RetryPolicy {
+  maxAttempts: number;
+  initialBackoffMs: number;
+  maxBackoffMs: number;
+}
+
+/**
+ * Calculate retry delay using ExponentialWithRandomJitter strategy.
+ * Algorithm aligned with sdk-core calculateRetryDelay
+ * (packages/sdk-core/src/utils/retry.ts).
+ */
+function calculateRetryDelay(
+  attemptNumber: number,
+  minRetryDelay: number,
+  maxRetryDelay: number,
+): number {
+  const base = Math.min(
+    minRetryDelay * Math.pow(2, attemptNumber - 1),
+    maxRetryDelay,
+  );
+  return Math.floor(Math.min(maxRetryDelay, base + Math.random() * base));
+}
+
+/**
+ * Retry a function with exponential backoff and jitter.
+ * Respects AbortSignal for cancellation.
+ */
+export async function retry<T>(
+  policy: RetryPolicy,
+  fn: () => Promise<T>,
+  isRetryable: (err: unknown) => boolean,
+  signal?: AbortSignal,
+): Promise<T> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt <= policy.maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastError = err;
+
+      if (!isRetryable(err)) {
+        throw err;
+      }
+      if (attempt === policy.maxAttempts) {
+        break;
+      }
+
+      const delayMs = calculateRetryDelay(
+        attempt + 1,
+        policy.initialBackoffMs,
+        policy.maxBackoffMs,
+      );
+
+      await sleep(delayMs, signal);
+    }
+  }
+  throw lastError;
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
+      return;
+    }
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(timer);
+        reject(signal.reason ?? new DOMException("Aborted", "AbortError"));
+      },
+      { once: true },
+    );
+  });
+}

package/src/utils/sse-decoder.ts

@@ -0,0 +1,140 @@
+/**
+ * SSE event parsed from a text/event-stream.
+ */
+export interface SSEEvent {
+  event: string;
+  data: string;
+}
+
+/**
+ * EventStreamDecoder parses a text/event-stream from a Node readable stream.
+ * Yields SSEEvent objects via the async iterator protocol.
+ */
+export class EventStreamDecoder {
+  private reader: ReadableStreamDefaultReader<Uint8Array> | null = null;
+  private decoder = new TextDecoder();
+  private buffer = "";
+
+  constructor(
+    private stream:
+      | ReadableStream<Uint8Array>
+      | NodeJS.ReadableStream,
+  ) {}
+
+  async *[Symbol.asyncIterator](): AsyncIterator<SSEEvent> {
+    const lines = this.readLines();
+    let event = "";
+    let dataLines: string[] = [];
+
+    for await (const line of lines) {
+      // Empty line → dispatch event
+      if (line === "") {
+        if (dataLines.length > 0) {
+          yield {
+            event,
+            data: dataLines.join("\n"),
+          };
+        }
+        event = "";
+        dataLines = [];
+        continue;
+      }
+
+      // Comment line
+      if (line.startsWith(":")) {
+        continue;
+      }
+
+      const colonIdx = line.indexOf(":");
+      let field: string;
+      let value: string;
+
+      if (colonIdx === -1) {
+        field = line;
+        value = "";
+      } else {
+        field = line.slice(0, colonIdx);
+        value = line.slice(colonIdx + 1);
+        // Remove optional leading space after colon
+        if (value.startsWith(" ")) {
+          value = value.slice(1);
+        }
+      }
+
+      switch (field) {
+        case "event":
+          event = value;
+          break;
+        case "data":
+          dataLines.push(value);
+          break;
+        // id, retry etc. — ignored for our use case
+      }
+    }
+
+    // Flush remaining
+    if (dataLines.length > 0) {
+      yield { event, data: dataLines.join("\n") };
+    }
+  }
+
+  private async *readLines(): AsyncGenerator<string> {
+    // Handle both Web ReadableStream and Node ReadableStream
+    const chunks = this.iterateStream();
+
+    for await (const chunk of chunks) {
+      this.buffer += typeof chunk === "string"
+        ? chunk
+        : this.decoder.decode(chunk, { stream: true });
+
+      const lines = this.buffer.split("\n");
+      // Keep the last partial line in the buffer
+      this.buffer = lines.pop() ?? "";
+
+      for (const line of lines) {
+        // Remove trailing \r if present
+        yield line.endsWith("\r") ? line.slice(0, -1) : line;
+      }
+    }
+
+    // Flush remaining buffer
+    if (this.buffer.length > 0) {
+      yield this.buffer;
+      this.buffer = "";
+    }
+  }
+
+  private async *iterateStream(): AsyncGenerator<Uint8Array | string> {
+    const stream = this.stream as any;
+
+    // Web ReadableStream
+    if (typeof stream.getReader === "function") {
+      this.reader = stream.getReader();
+      try {
+        while (true) {
+          const { done, value } = await this.reader!.read();
+          if (done) break;
+          if (value) yield value;
+        }
+      } finally {
+        this.reader!.releaseLock();
+        this.reader = null;
+      }
+      return;
+    }
+
+    // Node ReadableStream (async iterable)
+    if (Symbol.asyncIterator in stream) {
+      for await (const chunk of stream) {
+        yield chunk;
+      }
+      return;
+    }
+
+    throw new Error("Unsupported stream type");
+  }
+
+  cancel(): void {
+    this.reader?.cancel();
+  }
+}