@agorapete/wllama 3.5.1-q2.0

package/src/storage/opfs.ts

@@ -0,0 +1,119 @@
+import { createWorker, isSafariMobile } from '../utils';
+import { OPFS_UTILS_WORKER_CODE } from '../workers-code/generated';
+import type { StorageBackend } from './index';
+
+export class OPFSBackend implements StorageBackend {
+  isSupported(): boolean {
+    return (
+      typeof navigator !== 'undefined' &&
+      'storage' in navigator &&
+      !!navigator.storage?.getDirectory
+    );
+  }
+
+  async read(key: string): Promise<Blob | null> {
+    try {
+      const cacheDir = await getCacheDir();
+      const fileHandle = await cacheDir.getFileHandle(key);
+      return await fileHandle.getFile();
+    } catch (e) {
+      // NotFoundError or similar
+      return null;
+    }
+  }
+
+  async write(key: string, stream: ReadableStream): Promise<void> {
+    const writable = await openWritable(key);
+    await writable.truncate(0);
+    const reader = stream.getReader();
+    try {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        await writable.write(value);
+      }
+    } finally {
+      await writable.close();
+    }
+  }
+
+  async getSize(key: string): Promise<number> {
+    try {
+      const cacheDir = await getCacheDir();
+      const fileHandle = await cacheDir.getFileHandle(key);
+      const file = await fileHandle.getFile();
+      return file.size;
+    } catch (e) {
+      return -1;
+    }
+  }
+
+  async list(): Promise<Array<{ key: string; size: number }>> {
+    const cacheDir = await getCacheDir();
+    const result: Array<{ key: string; size: number }> = [];
+    // @ts-ignore
+    for await (const [name, handle] of cacheDir.entries()) {
+      if (handle.kind === 'file') {
+        const file = await (handle as FileSystemFileHandle).getFile();
+        result.push({ key: name, size: file.size });
+      }
+    }
+    return result;
+  }
+
+  async delete(key: string): Promise<void> {
+    try {
+      const cacheDir = await getCacheDir();
+      await cacheDir.removeEntry(key);
+    } catch (e: any) {
+      if (e?.name !== 'NotFoundError') throw e;
+    }
+  }
+}
+
+async function getCacheDir(): Promise<FileSystemDirectoryHandle> {
+  const opfsRoot = await navigator.storage.getDirectory();
+  return opfsRoot.getDirectoryHandle('cache', { create: true });
+}
+
+async function openWritable(fileName: string): Promise<{
+  truncate(offset: number): Promise<void>;
+  write(value: Uint8Array): Promise<void>;
+  close(): Promise<void>;
+}> {
+  const worker = createWorker(OPFS_UTILS_WORKER_CODE);
+  let pResolve: (v: any) => void;
+  let pReject: (v: any) => void;
+  worker.onmessage = (e: MessageEvent<any>) => {
+    if (e.data.ok) pResolve(null);
+    else if (e.data.err) pReject(e.data.err);
+  };
+  worker.onerror = (e) => pReject?.(e.message ?? e);
+  const workerExec = (
+    data:
+      | { action: 'open'; filename: string }
+      | { action: 'write'; buf: Uint8Array }
+      | { action: 'close' }
+  ) =>
+    new Promise<void>((resolve, reject) => {
+      pResolve = resolve;
+      pReject = reject;
+      worker.postMessage(
+        data,
+        isSafariMobile()
+          ? undefined
+          : { transfer: 'buf' in data && data.buf ? [data.buf.buffer] : [] }
+      );
+    });
+  await workerExec({ action: 'open', filename: fileName });
+  return {
+    truncate: async () => {
+      /* worker's openFile already calls truncate(0) on open */
+    },
+    write: (value) => workerExec({ action: 'write', buf: value }),
+    close: async () => {
+      await workerExec({ action: 'close' });
+      worker.terminate();
+    },
+  };
+}

package/src/types/oai-compat.ts

@@ -0,0 +1,342 @@
+import type { SamplingParams } from './types';
+
+// Message content types
+
+export interface ChatCompletionMessageText {
+  type: 'text';
+  text: string;
+}
+
+export interface ChatCompletionMessageImage {
+  type: 'image';
+  data: ArrayBuffer;
+}
+
+export interface ChatCompletionMessageAudio {
+  type: 'audio';
+  data: ArrayBuffer;
+}
+
+export type ChatCompletionMessageContent =
+  | ChatCompletionMessageText
+  | ChatCompletionMessageImage
+  | ChatCompletionMessageAudio;
+
+// Tool definitions
+
+export interface ChatCompletionToolFunctionParameters {
+  type: 'object';
+  properties: Record<
+    string,
+    {
+      type: string;
+      description?: string;
+      enum?: string[];
+      [key: string]: unknown;
+    }
+  >;
+  required?: string[];
+  additionalProperties?: boolean;
+}
+
+export interface ChatCompletionToolFunction {
+  name: string;
+  description?: string;
+  parameters?: ChatCompletionToolFunctionParameters;
+  strict?: boolean;
+}
+
+export interface ChatCompletionTool {
+  type: 'function';
+  function: ChatCompletionToolFunction;
+}
+
+export type ChatCompletionToolChoice =
+  | 'none'
+  | 'auto'
+  | 'required'
+  | { type: 'function'; function: { name: string } };
+
+// Message types
+
+export interface ChatCompletionSystemMessage {
+  role: 'system';
+  content: string;
+  name?: string;
+}
+
+export interface ChatCompletionUserMessage {
+  role: 'user';
+  content: string | ChatCompletionMessageContent[];
+  name?: string;
+}
+
+export interface ChatCompletionToolCall {
+  id: string;
+  type: 'function';
+  function: {
+    name: string;
+    arguments: string; // JSON-encoded string
+  };
+}
+
+export interface ChatCompletionAssistantMessage {
+  role: 'assistant';
+  content?: string | null;
+  name?: string;
+  tool_calls?: ChatCompletionToolCall[];
+}
+
+export interface ChatCompletionToolMessage {
+  role: 'tool';
+  content: string;
+  tool_call_id: string;
+}
+
+export type ChatCompletionMessage =
+  | ChatCompletionSystemMessage
+  | ChatCompletionUserMessage
+  | ChatCompletionAssistantMessage
+  | ChatCompletionToolMessage;
+
+// Request params
+
+export type ChatCompletionParams = {
+  messages: ChatCompletionMessage[];
+  stream?: boolean;
+  model?: string;
+  abortSignal?: AbortSignal;
+  // sampling
+  temperature?: number;
+  max_tokens?: number;
+  // stop?: string | string[];
+  // n?: number;
+  logprobs?: boolean;
+  top_logprobs?: number;
+  logit_bias?: Record<string, number>;
+  // tools
+  tools?: ChatCompletionTool[];
+  tool_choice?: ChatCompletionToolChoice;
+  // parallel_tool_calls?: boolean;
+  // response format
+  response_format?: {
+    type: 'text' | 'json_object' | 'json_schema';
+    json_schema?: { name: string; schema: unknown; strict?: boolean };
+  };
+  // user-facing
+  user?: string;
+  // llama-server-specific
+  chat_template_kwargs?: Record<string, any>;
+  cache_prompt?: boolean;
+  return_tokens?: boolean;
+  timings_per_token?: boolean;
+} & SamplingParams;
+
+// Response types----------
+
+export interface ChatCompletionLogprob {
+  token: string;
+  logprob: number;
+  bytes: number[] | null;
+}
+
+export interface ChatCompletionLogprobsContent extends ChatCompletionLogprob {
+  top_logprobs: ChatCompletionLogprob[];
+}
+
+export interface ChatCompletionChoiceLogprobs {
+  content: ChatCompletionLogprobsContent[] | null;
+  refusal: ChatCompletionLogprobsContent[] | null;
+}
+
+export interface ChatCompletionChoice {
+  index: number;
+  message: ChatCompletionAssistantMessage;
+  finish_reason: 'stop' | 'length' | 'tool_calls' | 'content_filter' | null;
+  logprobs: ChatCompletionChoiceLogprobs | null;
+}
+
+export interface ChatCompletionUsage {
+  prompt_tokens: number;
+  completion_tokens: number;
+  total_tokens: number;
+  prompt_tokens_details?: { cached_tokens: number; audio_tokens: number };
+  completion_tokens_details?: {
+    reasoning_tokens: number;
+    audio_tokens: number;
+    accepted_prediction_tokens: number;
+    rejected_prediction_tokens: number;
+  };
+}
+
+/** Response when stream=false (or omitted) */
+export interface ChatCompletionResponse {
+  id: string;
+  object: 'chat.completion';
+  created: number;
+  model: string;
+  choices: ChatCompletionChoice[];
+  usage: ChatCompletionUsage;
+  system_fingerprint?: string;
+  service_tier?: string;
+}
+
+// Streaming response types
+
+export interface ChatCompletionChunkDelta {
+  role?: 'assistant';
+  content?: string | null;
+  tool_calls?: Array<{
+    index: number;
+    id?: string;
+    type?: 'function';
+    function?: { name?: string; arguments?: string };
+  }>;
+  refusal?: string | null;
+}
+
+export interface ChatCompletionChunkChoice {
+  index: number;
+  delta: ChatCompletionChunkDelta;
+  finish_reason: 'stop' | 'length' | 'tool_calls' | 'content_filter' | null;
+  logprobs: ChatCompletionChoiceLogprobs | null;
+}
+
+export interface ResultTimings {
+  cache_n: number;
+  prompt_n: number;
+  prompt_ms: number;
+  prompt_per_token_ms: number;
+  prompt_per_second: number;
+  predicted_n: number;
+  predicted_ms: number;
+  predicted_per_token_ms: number;
+  predicted_per_second: number;
+}
+
+/** Response when stream=true — one chunk per SSE event */
+export interface ChatCompletionChunk {
+  id: string;
+  object: 'chat.completion.chunk';
+  created: number;
+  model: string;
+  choices: ChatCompletionChunkChoice[];
+  usage?: ChatCompletionUsage | null;
+  timings?: ResultTimings;
+}
+
+// Raw (text) completion
+
+export type RawCompletionParams = {
+  prompt: string | string[];
+  stream?: boolean;
+  model?: string;
+  abortSignal?: AbortSignal;
+  suffix?: string;
+  max_tokens?: number;
+  temperature?: number;
+  top_p?: number;
+  n?: number;
+  logprobs?: number | null;
+  echo?: boolean;
+  stop?: string | string[];
+  presence_penalty?: number;
+  frequency_penalty?: number;
+  best_of?: number;
+  logit_bias?: Record<string, number>;
+  seed?: number;
+  user?: string;
+} & SamplingParams;
+
+export interface RawCompletionChoice {
+  text: string;
+  index: number;
+  finish_reason: 'stop' | 'length' | 'content_filter' | null;
+  logprobs: {
+    tokens: string[];
+    token_logprobs: number[];
+    top_logprobs: Array<Record<string, number>>;
+    text_offset: number[];
+  } | null;
+}
+
+/** Response when stream=false */
+export interface RawCompletionResponse {
+  id: string;
+  object: 'text_completion';
+  created: number;
+  model: string;
+  choices: RawCompletionChoice[];
+  usage: ChatCompletionUsage;
+  system_fingerprint?: string;
+  timings?: ResultTimings;
+}
+
+/** One chunk when stream=true */
+export interface RawCompletionChunk {
+  id: string;
+  object: 'text_completion';
+  created: number;
+  model: string;
+  choices: Array<{
+    text: string;
+    index: number;
+    finish_reason: 'stop' | 'length' | 'content_filter' | null;
+    logprobs: null;
+  }>;
+  usage?: ChatCompletionUsage | null;
+  timings?: ResultTimings;
+}
+
+// Embeddings
+
+export interface EmbeddingCreateParams {
+  input: string | string[] | number[] | number[][];
+  model?: string;
+  encoding_format?: 'float' | 'base64';
+  // dimensions?: number; // unsupported by llama.cpp
+  // user?: string;
+}
+
+export interface Embedding {
+  object: 'embedding';
+  index: number;
+  embedding: number[] | string; // float array or base64 string depending on encoding_format
+}
+
+export interface EmbeddingUsage {
+  prompt_tokens: number;
+  total_tokens: number;
+}
+
+export interface CreateEmbeddingResponse {
+  object: 'list';
+  data: Embedding[];
+  model: string;
+  usage: EmbeddingUsage;
+}
+
+// Reranking (NOT official OAI-compat, but is a commonly-used API schema)
+
+export interface RerankParams {
+  query: string;
+  documents: string[];
+  top_n?: number;
+}
+
+export interface RerankResult {
+  index: number;
+  relevance_score: number;
+}
+
+export interface RerankUsage {
+  prompt_tokens: number;
+  total_tokens: number;
+}
+
+export interface RerankResponse {
+  model: string;
+  object: 'list';
+  usage: RerankUsage;
+  results: RerankResult[];
+}

package/src/types/types.ts

@@ -0,0 +1,133 @@
+// Note: snake_case is used to match llama.cpp's naming convention
+export interface LoadModelParams {
+  log_level?: LogLevel;
+  seed?: number;
+  n_ctx?: number;
+  n_batch?: number;
+  // by default, all layers are offloaded if WebGPU is available
+  n_gpu_layers?: number;
+  // by default, on multi-thread build, we take half number of available threads (hardwareConcurrency / 2)
+  n_threads?: number;
+  embeddings?: boolean;
+  offload_kqv?: boolean;
+  pooling_type?: // legacy values
+  | 'LLAMA_POOLING_TYPE_UNSPECIFIED'
+    | 'LLAMA_POOLING_TYPE_NONE'
+    | 'LLAMA_POOLING_TYPE_MEAN'
+    | 'LLAMA_POOLING_TYPE_CLS'
+    // new values
+    | 'unspecified'
+    | 'none'
+    | 'mean'
+    | 'cls'
+    | 'last'
+    | 'rank';
+  // context extending
+  rope_scaling_type?:
+    | 'LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED'
+    | 'LLAMA_ROPE_SCALING_TYPE_NONE'
+    | 'LLAMA_ROPE_SCALING_TYPE_LINEAR'
+    | 'LLAMA_ROPE_SCALING_TYPE_YARN';
+  rope_freq_base?: number;
+  rope_freq_scale?: number;
+  yarn_ext_factor?: number;
+  yarn_attn_factor?: number;
+  yarn_beta_fast?: number;
+  yarn_beta_slow?: number;
+  yarn_orig_ctx?: number;
+  // optimizations
+  cache_type_k?: 'f32' | 'f16' | 'q8_0' | 'q5_1' | 'q5_0' | 'q4_1' | 'q4_0';
+  cache_type_v?: 'f32' | 'f16' | 'q8_0' | 'q5_1' | 'q5_0' | 'q4_1' | 'q4_0';
+  flash_attn?: boolean; // true is auto, false is disabled
+  swa_full?: boolean;
+  chat_template?: string;
+  jinja?: boolean;
+  reasoning?: boolean;
+  image_min_tokens?: number;
+  image_max_tokens?: number;
+  warmup?: boolean;
+  no_kv_offload?: boolean;
+  mmproj_offload?: boolean;
+  cont_batching?: boolean;
+  n_keep?: number;
+  ctx_shift?: boolean;
+  cache_idle_slots?: boolean;
+  n_cache_reuse?: number;
+  lora_adapters?: { path: string; scale?: number }[];
+  lora_init_without_apply?: boolean;
+  spec_draft_model?: string;
+  spec_draft_ngl?: number;
+  spec_draft_n_max?: number;
+  spec_draft_n_min?: number;
+  spec_draft_p_min?: number;
+  spec_draft_threads?: number;
+  spec_draft_threads_batch?: number;
+  kv_overrides?: Record<string, string>;
+  reasoning_budget_tokens?: number;
+  reasoning_budget_message?: string;
+  reasoning_format?: 'none' | 'deepseek-legacy' | 'deepseek';
+  skip_chat_parsing?: boolean;
+  prefill_assistant?: boolean;
+  default_template_kwargs?: Record<string, any>;
+}
+
+// Note: snake_case is used to match llama.cpp's naming convention
+export interface LoadedContextInfo {
+  n_vocab: number;
+  n_ctx: number;
+  n_batch: number;
+  n_ubatch: number;
+  n_ctx_train: number;
+  n_embd: number;
+  n_layer: number;
+  metadata: Record<string, string>;
+  token_bos: number;
+  token_eos: number;
+  token_eot: number;
+  list_tokens_eog: number[];
+  has_encoder: boolean;
+  token_decoder_start: number;
+  add_bos_token: boolean;
+  add_eos_token: boolean;
+  has_image_input: boolean;
+  has_audio_input: boolean;
+}
+
+// Note: snake_case is used to match llama.cpp's naming convention
+export interface SamplingParams {
+  // See sampling.h for more details
+  seed?: number;
+  mirostat?: number | undefined; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0
+  mirostat_eta?: number | undefined;
+  mirostat_tau?: number | undefined;
+  samplers_sequence?: string[] | undefined; // unused for now
+  temp?: number | undefined; // temperature
+  top_p?: number | undefined;
+  top_k?: number | undefined;
+  penalty_last_n?: number | undefined;
+  penalty_repeat?: number | undefined;
+  penalty_freq?: number | undefined;
+  penalty_present?: number | undefined;
+  dynatemp_range?: number | undefined;
+  dynatemp_exponent?: number | undefined;
+  grammar?: string;
+  n_prev?: number | undefined;
+  n_probs?: number | undefined;
+  min_p?: number | undefined;
+  typ_p?: number | undefined;
+  typical_p?: number | undefined;
+  logit_bias?: { token: number; bias: number }[] | undefined;
+  ignore_eos?: boolean | undefined;
+}
+
+export interface StreamParams<T> {
+  stream: true;
+  onData: (data: T) => void;
+}
+
+export enum LogLevel {
+  DEBUG = 1,
+  INFO = 2,
+  WARN = 3,
+  ERROR = 4,
+}