cui-llama.rn 1.6.0 → 1.7.0

package/src/NativeRNLlama.ts

@@ -68,6 +68,11 @@ export type NativeContextParams = {
 
   pooling_type?: number
 
+  /**
+   * Enable context shifting to handle prompts larger than context size
+   */
+  ctx_shift?: boolean
+
   // Embedding params
   embedding?: boolean
   embd_normalize?: number
@@ -99,6 +104,12 @@ export type NativeCompletionParams = {
   }>
   preserved_tokens?: Array<string>
   chat_format?: number
+  /**
+   * Path to an image file to process before generating text.
+   * When provided, the image will be processed and added to the context.
+   * Requires multimodal support to be enabled via initMultimodal.
+   */
+  media_paths?: Array<string>
   /**
    * Specify a JSON array of stopping strings.
    * These words will not be included in the completion, so make sure to add them to the prompt for the next iteration. Default: `[]`
@@ -278,6 +289,22 @@ export type NativeCompletionResult = {
 
 export type NativeTokenizeResult = {
   tokens: Array<number>
+  /**
+   * Whether the tokenization contains images
+   */
+  has_images: boolean
+  /**
+   * Bitmap hashes of the images
+   */
+  bitmap_hashes: Array<number>
+  /**
+   * Chunk positions of the text and images
+   */
+  chunk_pos: Array<number>
+  /**
+   * Chunk positions of the images
+   */
+  chunk_pos_images: Array<number>
 }
 
 export type NativeEmbeddingResult = {
@@ -331,9 +358,14 @@ export type NativeSessionLoadResult = {
   prompt: string
 }
 
+export type NativeLlamaMessagePart = {
+  type: 'text'
+  text: string
+}
+
 export type NativeLlamaChatMessage = {
   role: string
-  content: string
+  content: string | Array<NativeLlamaMessagePart>
 }
 
 export type NativeCPUFeatures = {
@@ -342,8 +374,14 @@ export type NativeCPUFeatures = {
   dotprod: boolean
 }
 
-export type JinjaFormattedChatResult = {
+export type FormattedChatResult = {
+  type: 'jinja' | 'llama-chat'
   prompt: string
+  has_media: boolean
+  media_paths?: Array<string>
+}
+
+export type JinjaFormattedChatResult = FormattedChatResult & {
   chat_format?: number
   grammar?: string
   grammar_lazy?: boolean
@@ -356,6 +394,12 @@ export type JinjaFormattedChatResult = {
   additional_stops?: Array<string>
 }
 
+export type NativeImageProcessingResult = {
+  success: boolean
+  prompt: string
+  error?: string
+}
+
 export interface Spec extends TurboModule {
   toggleNativeLog(enabled: boolean): Promise<void>
   setContextLimit(limit: number): Promise<void>
@@ -392,8 +436,8 @@ export interface Spec extends TurboModule {
     params: NativeCompletionParams,
   ): Promise<NativeCompletionResult>
   stopCompletion(contextId: number): Promise<void>
-  tokenizeAsync(contextId: number, text: string): Promise<NativeTokenizeResult>
-  tokenizeSync(contextId: number, text: string): NativeTokenizeResult
+  tokenizeAsync(contextId: number, text: string, imagePaths?: Array<string>): Promise<NativeTokenizeResult>
+  tokenizeSync(contextId: number, text: string, imagePaths?: Array<string>): NativeTokenizeResult
   getCpuFeatures() : Promise<NativeCPUFeatures>
   detokenize(contextId: number, tokens: number[]): Promise<string>
   embedding(
@@ -418,6 +462,30 @@ export interface Spec extends TurboModule {
     contextId: number,
   ): Promise<Array<{ path: string; scaled?: number }>>
 
+  // Multimodal methods
+  initMultimodal(
+    contextId: number,
+    params: {
+      path: string
+      use_gpu: boolean
+    },
+  ): Promise<boolean>
+
+  isMultimodalEnabled(
+    contextId: number,
+  ): Promise<boolean>
+
+  getMultimodalSupport(
+    contextId: number,
+  ): Promise<{
+    vision: boolean
+    audio: boolean
+  }>
+
+  releaseMultimodal(
+    contextId: number,
+  ): Promise<void>
+
   releaseContext(contextId: number): Promise<void>
 
   releaseAllContexts(): Promise<void>

package/src/index.ts

@@ -15,14 +15,33 @@ import type {
   NativeCompletionTokenProbItem,
   NativeCompletionResultTimings,
   JinjaFormattedChatResult,
+  FormattedChatResult,
+  NativeImageProcessingResult,
+  NativeLlamaChatMessage,
 } from './NativeRNLlama'
 import type {
   SchemaGrammarConverterPropOrder,
   SchemaGrammarConverterBuiltinRule,
 } from './grammar'
 import { SchemaGrammarConverter, convertJsonSchemaToGrammar } from './grammar'
-import type { RNLlamaMessagePart, RNLlamaOAICompatibleMessage } from './chat'
-import { formatChat } from './chat'
+
+export type RNLlamaMessagePart = {
+  type: string
+  text?: string
+  image_url?: {
+    url?: string
+  }
+  input_audio?: {
+    format: string
+    data?: string
+    url?: string
+  }
+}
+
+export type RNLlamaOAICompatibleMessage = {
+  role: string
+  content?: string | RNLlamaMessagePart[]
+}
 
 export type {
   NativeContextParams,
@@ -36,15 +55,17 @@ export type {
   NativeEmbeddingParams,
   NativeCompletionTokenProbItem,
   NativeCompletionResultTimings,
-  RNLlamaMessagePart,
-  RNLlamaOAICompatibleMessage,
+  FormattedChatResult,
   JinjaFormattedChatResult,
+  NativeImageProcessingResult,
 
   // Deprecated
   SchemaGrammarConverterPropOrder,
   SchemaGrammarConverterBuiltinRule,
 }
 
+export const RNLLAMA_MTMD_DEFAULT_MEDIA_MARKER = '<__media__>'
+
 export { SchemaGrammarConverter, convertJsonSchemaToGrammar }
 
 const EVENT_ON_INIT_CONTEXT_PROGRESS = '@RNLlama_onInitContextProgress'
@@ -126,6 +147,7 @@ export type CompletionBaseParams = {
   parallel_tool_calls?: object
   tool_choice?: string
   response_format?: CompletionResponseFormat
+  media_paths?: string | string[]
 }
 export type CompletionParams = Omit<
   NativeCompletionParams,
@@ -207,23 +229,94 @@ export class LlamaContext {
       parallel_tool_calls?: object
       tool_choice?: string
     },
-  ): Promise<JinjaFormattedChatResult | string> {
-    const chat = formatChat(messages)
+  ): Promise<FormattedChatResult | JinjaFormattedChatResult> {
+    const mediaPaths: string[] = []
+    const chat = messages.map((msg) => {
+      if (Array.isArray(msg.content)) {
+        const content = msg.content.map((part) => {
+          // Handle multimodal content
+          if (part.type === 'image_url') {
+            let path = part.image_url?.url || ''
+            if (path?.startsWith('file://')) path = path.slice(7)
+            mediaPaths.push(path)
+            return {
+              type: 'text',
+              text: RNLLAMA_MTMD_DEFAULT_MEDIA_MARKER,
+            }
+          } else if (part.type === 'input_audio') {
+            const { input_audio: audio } = part
+            if (!audio) throw new Error('input_audio is required')
+
+            const { format } = audio
+            if (format != 'wav' && format != 'mp3') {
+              throw new Error(`Unsupported audio format: ${format}`)
+            }
+            if (audio.url) {
+              const path = audio.url.replace(/file:\/\//, '')
+              mediaPaths.push(path)
+            } else if (audio.data) {
+              mediaPaths.push(audio.data)
+            }
+            return {
+              type: 'text',
+              text: RNLLAMA_MTMD_DEFAULT_MEDIA_MARKER,
+            }
+          }
+          return part
+        })
+
+        return {
+          ...msg,
+          content,
+        }
+      }
+      return msg
+    }) as NativeLlamaChatMessage[]
+
     const useJinja = this.isJinjaSupported() && params?.jinja
-    let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml'
+    let tmpl
     if (template) tmpl = template // Force replace if provided
     const jsonSchema = getJsonSchema(params?.response_format)
-    return RNLlama.getFormattedChat(this.id, JSON.stringify(chat), tmpl, {
-      jinja: useJinja,
-      json_schema: jsonSchema ? JSON.stringify(jsonSchema) : undefined,
-      tools: params?.tools ? JSON.stringify(params.tools) : undefined,
-      parallel_tool_calls: params?.parallel_tool_calls
-        ? JSON.stringify(params.parallel_tool_calls)
-        : undefined,
-      tool_choice: params?.tool_choice,
-    })
+
+    const result = await RNLlama.getFormattedChat(
+      this.id,
+      JSON.stringify(chat),
+      tmpl,
+      {
+        jinja: useJinja,
+        json_schema: jsonSchema ? JSON.stringify(jsonSchema) : undefined,
+        tools: params?.tools ? JSON.stringify(params.tools) : undefined,
+        parallel_tool_calls: params?.parallel_tool_calls
+          ? JSON.stringify(params.parallel_tool_calls)
+          : undefined,
+        tool_choice: params?.tool_choice,
+      },
+    )
+    if (!useJinja) {
+      return {
+        type: 'llama-chat',
+        prompt: result as string,
+        has_media: mediaPaths.length > 0,
+        media_paths: mediaPaths,
+      }
+    }
+    const jinjaResult = result as JinjaFormattedChatResult
+    jinjaResult.type = 'jinja'
+    jinjaResult.has_media = mediaPaths.length > 0
+    jinjaResult.media_paths = mediaPaths
+    return jinjaResult
   }
 
+  /**
+   * Generate a completion based on the provided parameters
+   * @param params Completion parameters including prompt or messages
+   * @param callback Optional callback for token-by-token streaming
+   * @returns Promise resolving to the completion result
+   *
+   * Note: For multimodal support, you can include an media_paths parameter.
+   * This will process the images and add them to the context before generating text.
+   * Multimodal support must be enabled via initMultimodal() first.
+   */
   async completion(
     params: CompletionParams,
     callback?: (data: TokenData) => void,
@@ -233,8 +326,8 @@ export class LlamaContext {
       prompt: params.prompt || '',
       emit_partial_completion: !!callback,
     }
+
     if (params.messages) {
-      // messages always win
       const formattedResult = await this.getFormattedChat(
         params.messages,
         params.chat_template || params.chatTemplate,
@@ -245,29 +338,42 @@ export class LlamaContext {
           tool_choice: params.tool_choice,
         },
       )
-      if (typeof formattedResult === 'string') {
-        nativeParams.prompt = formattedResult || ''
-      } else {
-        nativeParams.prompt = formattedResult.prompt || ''
-        if (typeof formattedResult.chat_format === 'number')
-          nativeParams.chat_format = formattedResult.chat_format
-        if (formattedResult.grammar)
-          nativeParams.grammar = formattedResult.grammar
-        if (typeof formattedResult.grammar_lazy === 'boolean')
-          nativeParams.grammar_lazy = formattedResult.grammar_lazy
-        if (formattedResult.grammar_triggers)
-          nativeParams.grammar_triggers = formattedResult.grammar_triggers
-        if (formattedResult.preserved_tokens)
-          nativeParams.preserved_tokens = formattedResult.preserved_tokens
-        if (formattedResult.additional_stops) {
+      if (formattedResult.type === 'jinja') {
+        const jinjaResult = formattedResult as JinjaFormattedChatResult
+
+        nativeParams.prompt = jinjaResult.prompt || ''
+        if (typeof jinjaResult.chat_format === 'number')
+          nativeParams.chat_format = jinjaResult.chat_format
+        if (jinjaResult.grammar) nativeParams.grammar = jinjaResult.grammar
+        if (typeof jinjaResult.grammar_lazy === 'boolean')
+          nativeParams.grammar_lazy = jinjaResult.grammar_lazy
+        if (jinjaResult.grammar_triggers)
+          nativeParams.grammar_triggers = jinjaResult.grammar_triggers
+        if (jinjaResult.preserved_tokens)
+          nativeParams.preserved_tokens = jinjaResult.preserved_tokens
+        if (jinjaResult.additional_stops) {
           if (!nativeParams.stop) nativeParams.stop = []
-          nativeParams.stop.push(...formattedResult.additional_stops)
+          nativeParams.stop.push(...jinjaResult.additional_stops)
+        }
+        if (jinjaResult.has_media) {
+          nativeParams.media_paths = jinjaResult.media_paths
+        }
+      } else if (formattedResult.type === 'llama-chat') {
+        const llamaChatResult = formattedResult as FormattedChatResult
+        nativeParams.prompt = llamaChatResult.prompt || ''
+        if (llamaChatResult.has_media) {
+          nativeParams.media_paths = llamaChatResult.media_paths
         }
       }
     } else {
       nativeParams.prompt = params.prompt || ''
     }
 
+    // If media_paths were explicitly provided or extracted from messages, use them
+    if (!nativeParams.media_paths && params.media_paths) {
+      nativeParams.media_paths = params.media_paths
+    }
+
     if (nativeParams.response_format && !nativeParams.grammar) {
       const jsonSchema = getJsonSchema(params.response_format)
       if (jsonSchema) nativeParams.json_schema = JSON.stringify(jsonSchema)
@@ -301,12 +407,32 @@ export class LlamaContext {
     return RNLlama.stopCompletion(this.id)
   }
 
-  tokenizeAsync(text: string): Promise<NativeTokenizeResult> {
-    return RNLlama.tokenizeAsync(this.id, text)
+  /**
+   * Tokenize text or text with images
+   * @param text Text to tokenize
+   * @param params.media_paths Array of image paths to tokenize (if multimodal is enabled)
+   * @returns Promise resolving to the tokenize result
+   */
+  tokenizeAsync(
+    text: string,
+    {
+      media_paths: mediaPaths,
+    }: {
+      media_paths?: string[]
+    } = {},
+  ): Promise<NativeTokenizeResult> {
+    return RNLlama.tokenizeAsync(this.id, text, mediaPaths)
   }
 
-  tokenizeSync(text: string): NativeTokenizeResult {
-    return RNLlama.tokenizeSync(this.id, text)
+  tokenizeSync(
+    text: string,
+    {
+      media_paths: mediaPaths,
+    }: {
+      media_paths?: string[]
+    } = {},
+  ): NativeTokenizeResult {
+    return RNLlama.tokenizeSync(this.id, text, mediaPaths)
   }
 
   detokenize(tokens: number[]): Promise<string> {
@@ -362,6 +488,54 @@ export class LlamaContext {
     return RNLlama.getLoadedLoraAdapters(this.id)
   }
 
+  /**
+   * Initialize multimodal support with a mmproj file
+   * @param params Parameters for multimodal support
+   * @param params.path Path to the multimodal projector file
+   * @param params.use_gpu Whether to use GPU
+   * @returns Promise resolving to true if initialization was successful
+   */
+  async initMultimodal({
+    path,
+    use_gpu: useGpu,
+  }: {
+    path: string
+    use_gpu?: boolean
+  }): Promise<boolean> {
+    if (path.startsWith('file://')) path = path.slice(7)
+    return RNLlama.initMultimodal(this.id, {
+      path,
+      use_gpu: useGpu ?? true,
+    })
+  }
+
+  /**
+   * Check if multimodal support is enabled
+   * @returns Promise resolving to true if multimodal is enabled
+   */
+  async isMultimodalEnabled(): Promise<boolean> {
+    return await RNLlama.isMultimodalEnabled(this.id)
+  }
+
+  /**
+   * Check multimodal support
+   * @returns Promise resolving to an object with vision and audio support
+   */
+  async getMultimodalSupport(): Promise<{
+    vision: boolean
+    audio: boolean
+  }> {
+    return await RNLlama.getMultimodalSupport(this.id)
+  }
+
+  /**
+   * Release multimodal support
+   * @returns Promise resolving to void
+   */
+  async releaseMultimodal(): Promise<void> {
+    return await RNLlama.releaseMultimodal(this.id)
+  }
+
   async release(): Promise<void> {
     return RNLlama.releaseContext(this.id)
   }
@@ -407,7 +581,7 @@ const modelInfoSkip = [
   'tokenizer.ggml.tokens',
   'tokenizer.ggml.token_type',
   'tokenizer.ggml.merges',
-  'tokenizer.ggml.scores'
+  'tokenizer.ggml.scores',
 ]
 export async function loadLlamaModelInfo(model: string): Promise<Object> {
   let path = model

package/cpp/binary-ops.h

@@ -1,16 +0,0 @@
-#pragma once
-
-#include "cpu-common.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void lm_ggml_compute_forward_add_non_quantized(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_sub(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_mul(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_div(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-
-#ifdef __cplusplus
-}
-#endif

package/cpp/ops.h

@@ -1,128 +0,0 @@
-#pragma once
-
-#include "ggml.h"
-
-//
-// cache line
-//
-
-#if defined(__cpp_lib_hardware_interference_size)
-#define CACHE_LINE_SIZE std::hardware_destructive_interference_size
-#else
-#if defined(__POWER9_VECTOR__)
-#define CACHE_LINE_SIZE 128
-#elif defined(__VXE__) || defined(__VXE2__)
-#define CACHE_LINE_SIZE 256
-#else
-#define CACHE_LINE_SIZE 64
-#endif
-#endif
-
-static const size_t CACHE_LINE_SIZE_F32 = CACHE_LINE_SIZE/sizeof(float);
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void lm_ggml_compute_forward_dup(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_add(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_add1(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_acc(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_sum(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_sum_rows(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_mean(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_argmax(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_count_equal(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_repeat(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_repeat_back(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_concat(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_silu_back(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_norm(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_rms_norm(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_rms_norm_back(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_group_norm(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_l2_norm(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_out_prod(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_scale(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_set(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_cpy(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_cont(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_reshape(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_view(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_permute(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_transpose(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_get_rows(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_get_rows_back(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_diag(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_diag_mask_inf(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_diag_mask_zero(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_soft_max(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_soft_max_ext_back(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_rope(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_rope_back(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_clamp(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_conv_transpose_1d(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_im2col(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_im2col_back_f32(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_conv_transpose_2d(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_pool_1d(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_pool_2d(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_pool_2d_back(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_upscale(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_pad(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_pad_reflect_1d(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_arange(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_timestep_embedding(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_argsort(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_leaky_relu(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_flash_attn_ext(
-    const struct lm_ggml_compute_params * params,
-    const struct lm_ggml_tensor * q,
-    const struct lm_ggml_tensor * k,
-    const struct lm_ggml_tensor * v,
-    const struct lm_ggml_tensor * mask,
-    struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_flash_attn_back(
-        const struct lm_ggml_compute_params * params,
-        const bool masked,
-        struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_ssm_conv(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_ssm_scan(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_win_part(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_win_unpart(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_unary(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_get_rel_pos(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_add_rel_pos(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_rwkv_wkv6(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_rwkv_wkv7(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_gla(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_map_unary(
-    const struct lm_ggml_compute_params * params,
-    struct lm_ggml_tensor * dst,
-    const lm_ggml_unary_op_f32_t fun);
-void lm_ggml_compute_forward_map_binary(
-    const struct lm_ggml_compute_params * params,
-    struct lm_ggml_tensor * dst,
-    const lm_ggml_binary_op_f32_t fun);
-void lm_ggml_compute_forward_map_custom1_f32(
-    const struct lm_ggml_compute_params * params,
-    struct lm_ggml_tensor * dst,
-    const lm_ggml_custom1_op_f32_t fun);
-void lm_ggml_compute_forward_map_custom2_f32(
-    const struct lm_ggml_compute_params * params,
-    struct lm_ggml_tensor * dst,
-    const lm_ggml_custom2_op_f32_t fun);
-void lm_ggml_compute_forward_map_custom3_f32(
-    const struct lm_ggml_compute_params * params,
-    struct lm_ggml_tensor * dst,
-    const lm_ggml_custom3_op_f32_t fun);
-void lm_ggml_compute_forward_map_custom1(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_map_custom2(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_map_custom3(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_cross_entropy_loss(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_cross_entropy_loss_back(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-void lm_ggml_compute_forward_opt_step_adamw(const struct lm_ggml_compute_params * params, struct lm_ggml_tensor * dst);
-
-#ifdef __cplusplus
-}
-#endif