cui-llama.rn 1.4.3 → 1.4.6

package/src/NativeRNLlama.ts

@@ -7,6 +7,13 @@ export type NativeEmbeddingParams = {
 
 export type NativeContextParams = {
   model: string
+  /**
+   * Chat template to override the default one from the model.
+   */
+  chat_template?: string
+
+  reasoning_format?: string
+
   is_model_asset?: boolean
   use_progress_callback?: boolean
 
@@ -15,7 +22,15 @@ export type NativeContextParams = {
   n_ubatch?: number
 
   n_threads?: number
+
+  /**
+   * Number of layers to store in VRAM (Currently only for iOS)
+   */
   n_gpu_layers?: number
+  /**
+   * Skip GPU devices (iOS only)
+   */
+  no_gpu_devices?: boolean
 
   /**
    * Enable flash attention, only recommended in GPU device (Experimental in llama.cpp)
@@ -61,10 +76,28 @@ export type NativeContextParams = {
 export type NativeCompletionParams = {
   prompt: string
   n_threads?: number
+  /**
+   * JSON schema for convert to grammar for structured JSON output.
+   * It will be override by grammar if both are set.
+   */
+  json_schema?: string
   /**
    * Set grammar for grammar-based sampling.  Default: no grammar
    */
   grammar?: string
+  /**
+   * Lazy grammar sampling, trigger by grammar_triggers. Default: false
+   */
+  grammar_lazy?: boolean
+  /**
+   * Lazy grammar triggers. Default: []
+   */
+  grammar_triggers?: Array<{
+    at_start: boolean
+    word: string
+  }>
+  preserved_tokens?: Array<string>
+  chat_format?: number
   /**
    * Specify a JSON array of stopping strings.
    * These words will not be included in the completion, so make sure to add them to the prompt for the next iteration. Default: `[]`
@@ -158,6 +191,11 @@ export type NativeCompletionParams = {
    * Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\n', ':', '"', '*']`
    */
   dry_sequence_breakers?: Array<string>
+  /**
+   * Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641. Default: `-1.0` (Disabled)
+   */
+  top_n_sigma?: number
+
   /**
    * Ignore end of stream token and continue generating. Default: `false`
    */
@@ -200,7 +238,29 @@ export type NativeCompletionResultTimings = {
 }
 
 export type NativeCompletionResult = {
+  /**
+   * Original text (Ignored reasoning_content / tool_calls)
+   */
   text: string
+  /**
+   * Reasoning content (parsed for reasoning model)
+   */
+  reasoning_content: string
+  /**
+   * Tool calls
+   */
+  tool_calls: Array<{
+    type: 'function'
+    function: {
+      name: string
+      arguments: string
+    }
+    id?: string
+  }>
+  /**
+   * Content text (Filtered text by reasoning_content / tool_calls)
+   */
+  content: string
 
   tokens_predicted: number
   tokens_evaluated: number
@@ -225,7 +285,38 @@ export type NativeEmbeddingResult = {
 
 export type NativeLlamaContext = {
   contextId: number
-  model: Object
+  model: {
+    desc: string
+    size: number
+    nEmbd: number
+    nParams: number
+    chatTemplates: {
+      llamaChat: boolean // Chat template in llama-chat.cpp
+      minja: {
+        // Chat template supported by minja.hpp
+        default: boolean
+        defaultCaps: {
+          tools: boolean
+          toolCalls: boolean
+          toolResponses: boolean
+          systemRole: boolean
+          parallelToolCalls: boolean
+          toolCallId: boolean
+        }
+        toolUse: boolean
+        toolUseCaps: {
+          tools: boolean
+          toolCalls: boolean
+          toolResponses: boolean
+          systemRole: boolean
+          parallelToolCalls: boolean
+          toolCallId: boolean
+        }
+      }
+    }
+    metadata: Object
+    isChatTemplateSupported: boolean // Deprecated
+  }
   /**
    * Loaded library name for Android
    */
@@ -250,7 +341,21 @@ export type NativeCPUFeatures = {
   dotprod: boolean
 }
 
+export type JinjaFormattedChatResult = {
+  prompt: string
+  chat_format?: number
+  grammar?: string
+  grammar_lazy?: boolean
+  grammar_triggers?: Array<{
+    at_start: boolean
+    word: string
+  }>
+  preserved_tokens?: Array<string>
+  additional_stops?: Array<string>
+}
+
 export interface Spec extends TurboModule {
+  toggleNativeLog(enabled: boolean): Promise<void>
   setContextLimit(limit: number): Promise<void>
 
   modelInfo(path: string, skip?: string[]): Promise<Object>
@@ -261,9 +366,16 @@ export interface Spec extends TurboModule {
 
   getFormattedChat(
     contextId: number,
-    messages: NativeLlamaChatMessage[],
+    messages: string,
     chatTemplate?: string,
-  ): Promise<string>
+    params?: {
+      jinja?: boolean
+      json_schema?: string
+      tools?: string
+      parallel_tool_calls?: string
+      tool_choice?: string
+    },
+  ): Promise<JinjaFormattedChatResult | string>
   loadSession(
     contextId: number,
     filepath: string,

package/src/grammar.ts

@@ -1,5 +1,8 @@
 /* eslint-disable no-restricted-syntax */
 /* eslint-disable no-underscore-dangle */
+
+// NOTE: Deprecated, please use tools or response_format with json_schema instead
+
 const SPACE_RULE = '" "?'
 
 function buildRepetition(

package/src/index.ts

@@ -14,6 +14,7 @@ import type {
   NativeEmbeddingParams,
   NativeCompletionTokenProbItem,
   NativeCompletionResultTimings,
+  JinjaFormattedChatResult,
 } from './NativeRNLlama'
 import type {
   SchemaGrammarConverterPropOrder,
@@ -37,6 +38,9 @@ export type {
   NativeCompletionResultTimings,
   RNLlamaMessagePart,
   RNLlamaOAICompatibleMessage,
+  JinjaFormattedChatResult,
+
+  // Deprecated
   SchemaGrammarConverterPropOrder,
   SchemaGrammarConverterBuiltinRule,
 }
@@ -45,6 +49,7 @@ export { SchemaGrammarConverter, convertJsonSchemaToGrammar }
 
 const EVENT_ON_INIT_CONTEXT_PROGRESS = '@RNLlama_onInitContextProgress'
 const EVENT_ON_TOKEN = '@RNLlama_onToken'
+const EVENT_ON_NATIVE_LOG = '@RNLlama_onNativeLog'
 
 let EventEmitter: NativeEventEmitter | DeviceEventEmitterStatic
 if (Platform.OS === 'ios') {
@@ -55,6 +60,19 @@ if (Platform.OS === 'android') {
   EventEmitter = DeviceEventEmitter
 }
 
+const logListeners: Array<(level: string, text: string) => void> = []
+
+// @ts-ignore
+if (EventEmitter) {
+  EventEmitter.addListener(
+    EVENT_ON_NATIVE_LOG,
+    (evt: { level: string; text: string }) => {
+      logListeners.forEach((listener) => listener(evt.level, evt.text))
+    },
+  )
+  RNLlama?.toggleNativeLog?.(false) // Trigger unset to use default log callback
+}
+
 export type TokenData = {
   token: string
   completion_probabilities?: Array<NativeCompletionTokenProb>
@@ -88,14 +106,31 @@ export type ContextParams = Omit<
 
 export type EmbeddingParams = NativeEmbeddingParams
 
-export type CompletionParams = Omit<
-  NativeCompletionParams,
-  'emit_partial_completion' | 'prompt'
-> & {
+export type CompletionResponseFormat = {
+  type: 'text' | 'json_object' | 'json_schema'
+  json_schema?: {
+    strict?: boolean
+    schema: object
+  }
+  schema?: object // for json_object type
+}
+
+export type CompletionBaseParams = {
   prompt?: string
   messages?: RNLlamaOAICompatibleMessage[]
-  chatTemplate?: string
+  chatTemplate?: string // deprecated
+  chat_template?: string
+  jinja?: boolean
+  tools?: object
+  parallel_tool_calls?: object
+  tool_choice?: string
+  response_format?: CompletionResponseFormat
 }
+export type CompletionParams = Omit<
+  NativeCompletionParams,
+  'emit_partial_completion' | 'prompt'
+> &
+  CompletionBaseParams
 
 export type BenchResult = {
   modelDesc: string
@@ -107,6 +142,16 @@ export type BenchResult = {
   tgStd: number
 }
 
+const getJsonSchema = (responseFormat?: CompletionResponseFormat) => {
+  if (responseFormat?.type === 'json_schema') {
+    return responseFormat.json_schema?.schema
+  }
+  if (responseFormat?.type === 'json_object') {
+    return responseFormat.schema || {}
+  }
+  return null
+}
+
 export class LlamaContext {
   id: number
 
@@ -114,9 +159,7 @@ export class LlamaContext {
 
   reasonNoGPU: string = ''
 
-  model: {
-    isChatTemplateSupported?: boolean
-  } = {}
+  model: NativeLlamaContext['model']
 
   constructor({ contextId, gpu, reasonNoGPU, model }: NativeLlamaContext) {
     this.id = contextId
@@ -144,27 +187,89 @@ export class LlamaContext {
     return RNLlama.saveSession(this.id, filepath, options?.tokenSize || -1)
   }
 
+  isLlamaChatSupported(): boolean {
+    return !!this.model.chatTemplates.llamaChat
+  }
+
+  isJinjaSupported(): boolean {
+    const { minja } = this.model.chatTemplates
+    return !!minja?.toolUse || !!minja?.default
+  }
+
   async getFormattedChat(
     messages: RNLlamaOAICompatibleMessage[],
-    template?: string,
-  ): Promise<string> {
+    template?: string | null,
+    params?: {
+      jinja?: boolean
+      response_format?: CompletionResponseFormat
+      tools?: object
+      parallel_tool_calls?: object
+      tool_choice?: string
+    },
+  ): Promise<JinjaFormattedChatResult | string> {
     const chat = formatChat(messages)
-    let tmpl = this.model?.isChatTemplateSupported ? undefined : 'chatml'
+    const useJinja = this.isJinjaSupported() && params?.jinja
+    let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml'
     if (template) tmpl = template // Force replace if provided
-    return RNLlama.getFormattedChat(this.id, chat, tmpl)
+    const jsonSchema = getJsonSchema(params?.response_format)
+    return RNLlama.getFormattedChat(this.id, JSON.stringify(chat), tmpl, {
+      jinja: useJinja,
+      json_schema: jsonSchema ? JSON.stringify(jsonSchema) : undefined,
+      tools: params?.tools ? JSON.stringify(params.tools) : undefined,
+      parallel_tool_calls: params?.parallel_tool_calls
+        ? JSON.stringify(params.parallel_tool_calls)
+        : undefined,
+      tool_choice: params?.tool_choice,
+    })
   }
 
   async completion(
     params: CompletionParams,
     callback?: (data: TokenData) => void,
   ): Promise<NativeCompletionResult> {
-    let finalPrompt = params.prompt
+    const nativeParams = {
+      ...params,
+      prompt: params.prompt || '',
+      emit_partial_completion: !!callback,
+    }
     if (params.messages) {
       // messages always win
-      finalPrompt = await this.getFormattedChat(
+      const formattedResult = await this.getFormattedChat(
         params.messages,
-        params.chatTemplate,
+        params.chat_template || params.chatTemplate,
+        {
+          jinja: params.jinja,
+          tools: params.tools,
+          parallel_tool_calls: params.parallel_tool_calls,
+          tool_choice: params.tool_choice,
+        },
       )
+      if (typeof formattedResult === 'string') {
+        nativeParams.prompt = formattedResult || ''
+      } else {
+        nativeParams.prompt = formattedResult.prompt || ''
+        if (typeof formattedResult.chat_format === 'number')
+          nativeParams.chat_format = formattedResult.chat_format
+        if (formattedResult.grammar)
+          nativeParams.grammar = formattedResult.grammar
+        if (typeof formattedResult.grammar_lazy === 'boolean')
+          nativeParams.grammar_lazy = formattedResult.grammar_lazy
+        if (formattedResult.grammar_triggers)
+          nativeParams.grammar_triggers = formattedResult.grammar_triggers
+        if (formattedResult.preserved_tokens)
+          nativeParams.preserved_tokens = formattedResult.preserved_tokens
+        if (formattedResult.additional_stops) {
+          if (!nativeParams.stop) nativeParams.stop = []
+          nativeParams.stop.push(...formattedResult.additional_stops)
+        }
+      }
+    } else {
+      nativeParams.prompt = params.prompt || ''
+    }
+
+    if (nativeParams.response_format && !nativeParams.grammar) {
+      const jsonSchema = getJsonSchema(params.response_format)
+      if (jsonSchema) nativeParams.json_schema = JSON.stringify(jsonSchema)
     }
 
     let tokenListener: any =
@@ -175,12 +280,9 @@ export class LlamaContext {
         callback(tokenResult)
       })
 
-    if (!finalPrompt) throw new Error('Prompt is required')
-    const promise = RNLlama.completion(this.id, {
-      ...params,
-      prompt: finalPrompt,
-      emit_partial_completion: !!callback,
-    })
+    if (!nativeParams.prompt) throw new Error('Prompt is required')
+
+    const promise = RNLlama.completion(this.id, nativeParams)
     return promise
       .then((completionResult) => {
         tokenListener?.remove()
@@ -268,6 +370,21 @@ export async function getCpuFeatures() : Promise<NativeCPUFeatures> {
   return RNLlama.getCpuFeatures()
 }
 
+export async function toggleNativeLog(enabled: boolean): Promise<void> {
+  return RNLlama.toggleNativeLog(enabled)
+}
+
+export function addNativeLogListener(
+  listener: (level: string, text: string) => void,
+): { remove: () => void } {
+  logListeners.push(listener)
+  return {
+    remove: () => {
+      logListeners.splice(logListeners.indexOf(listener), 1)
+    },
+  }
+}
+
 export async function setContextLimit(limit: number): Promise<void> {
   return RNLlama.setContextLimit(limit)
 }

package/android/src/main/build-arm64/CMakeFiles/3.31.4/CMakeCCompiler.cmake

@@ -1,81 +0,0 @@
-set(CMAKE_C_COMPILER "D:/Android/Sdk/ndk/26.3.11579264/toolchains/llvm/prebuilt/windows-x86_64/bin/clang.exe")
-set(CMAKE_C_COMPILER_ARG1 "")
-set(CMAKE_C_COMPILER_ID "Clang")
-set(CMAKE_C_COMPILER_VERSION "17.0.2")
-set(CMAKE_C_COMPILER_VERSION_INTERNAL "")
-set(CMAKE_C_COMPILER_WRAPPER "")
-set(CMAKE_C_STANDARD_COMPUTED_DEFAULT "17")
-set(CMAKE_C_EXTENSIONS_COMPUTED_DEFAULT "ON")
-set(CMAKE_C_STANDARD_LATEST "")
-set(CMAKE_C_COMPILE_FEATURES "")
-set(CMAKE_C90_COMPILE_FEATURES "")
-set(CMAKE_C99_COMPILE_FEATURES "")
-set(CMAKE_C11_COMPILE_FEATURES "")
-set(CMAKE_C17_COMPILE_FEATURES "")
-set(CMAKE_C23_COMPILE_FEATURES "")
-
-set(CMAKE_C_PLATFORM_ID "Linux")
-set(CMAKE_C_SIMULATE_ID "")
-set(CMAKE_C_COMPILER_FRONTEND_VARIANT "GNU")
-set(CMAKE_C_SIMULATE_VERSION "")
-
-
-
-
-set(CMAKE_AR "D:/Android/Sdk/ndk/26.3.11579264/toolchains/llvm/prebuilt/windows-x86_64/bin/llvm-ar.exe")
-set(CMAKE_C_COMPILER_AR "D:/Android/Sdk/ndk/26.3.11579264/toolchains/llvm/prebuilt/windows-x86_64/bin/llvm-ar.exe")
-set(CMAKE_RANLIB "D:/Android/Sdk/ndk/26.3.11579264/toolchains/llvm/prebuilt/windows-x86_64/bin/llvm-ranlib.exe")
-set(CMAKE_C_COMPILER_RANLIB "D:/Android/Sdk/ndk/26.3.11579264/toolchains/llvm/prebuilt/windows-x86_64/bin/llvm-ranlib.exe")
-set(CMAKE_LINKER "D:/Android/Sdk/ndk/26.3.11579264/toolchains/llvm/prebuilt/windows-x86_64/bin/ld.lld.exe")
-set(CMAKE_LINKER_LINK "")
-set(CMAKE_LINKER_LLD "")
-set(CMAKE_C_COMPILER_LINKER "")
-set(CMAKE_C_COMPILER_LINKER_ID "")
-set(CMAKE_C_COMPILER_LINKER_VERSION )
-set(CMAKE_C_COMPILER_LINKER_FRONTEND_VARIANT )
-set(CMAKE_MT "")
-set(CMAKE_TAPI "CMAKE_TAPI-NOTFOUND")
-set(CMAKE_COMPILER_IS_GNUCC )
-set(CMAKE_C_COMPILER_LOADED 1)
-set(CMAKE_C_COMPILER_WORKS )
-set(CMAKE_C_ABI_COMPILED )
-
-set(CMAKE_C_COMPILER_ENV_VAR "CC")
-
-set(CMAKE_C_COMPILER_ID_RUN 1)
-set(CMAKE_C_SOURCE_FILE_EXTENSIONS c;m)
-set(CMAKE_C_IGNORE_EXTENSIONS h;H;o;O;obj;OBJ;def;DEF;rc;RC)
-set(CMAKE_C_LINKER_PREFERENCE 10)
-set(CMAKE_C_LINKER_DEPFILE_SUPPORTED )
-
-# Save compiler ABI information.
-set(CMAKE_C_SIZEOF_DATA_PTR "")
-set(CMAKE_C_COMPILER_ABI "")
-set(CMAKE_C_BYTE_ORDER "")
-set(CMAKE_C_LIBRARY_ARCHITECTURE "")
-
-if(CMAKE_C_SIZEOF_DATA_PTR)
-  set(CMAKE_SIZEOF_VOID_P "${CMAKE_C_SIZEOF_DATA_PTR}")
-endif()
-
-if(CMAKE_C_COMPILER_ABI)
-  set(CMAKE_INTERNAL_PLATFORM_ABI "${CMAKE_C_COMPILER_ABI}")
-endif()
-
-if(CMAKE_C_LIBRARY_ARCHITECTURE)
-  set(CMAKE_LIBRARY_ARCHITECTURE "")
-endif()
-
-set(CMAKE_C_CL_SHOWINCLUDES_PREFIX "")
-if(CMAKE_C_CL_SHOWINCLUDES_PREFIX)
-  set(CMAKE_CL_SHOWINCLUDES_PREFIX "${CMAKE_C_CL_SHOWINCLUDES_PREFIX}")
-endif()
-
-
-
-
-
-set(CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES "")
-set(CMAKE_C_IMPLICIT_LINK_LIBRARIES "")
-set(CMAKE_C_IMPLICIT_LINK_DIRECTORIES "")
-set(CMAKE_C_IMPLICIT_LINK_FRAMEWORK_DIRECTORIES "")

package/android/src/main/build-arm64/CMakeFiles/3.31.4/CMakeSystem.cmake

@@ -1,15 +0,0 @@
-set(CMAKE_HOST_SYSTEM "Windows-10.0.26120")
-set(CMAKE_HOST_SYSTEM_NAME "Windows")
-set(CMAKE_HOST_SYSTEM_VERSION "10.0.26120")
-set(CMAKE_HOST_SYSTEM_PROCESSOR "AMD64")
-
-include("D:/Android/Sdk/ndk/26.3.11579264/build/cmake/android.toolchain.cmake")
-
-set(CMAKE_SYSTEM "Android-1")
-set(CMAKE_SYSTEM_NAME "Android")
-set(CMAKE_SYSTEM_VERSION "1")
-set(CMAKE_SYSTEM_PROCESSOR "aarch64")
-
-set(CMAKE_CROSSCOMPILING "TRUE")
-
-set(CMAKE_SYSTEM_LOADED 1)