cactus-react-native 0.1.1 → 0.1.3

package/lib/module/index.js

@@ -1,363 +0,0 @@
-"use strict";
-
-import { NativeEventEmitter, DeviceEventEmitter, Platform } from 'react-native';
-import Cactus from './NativeCactus';
-import { SchemaGrammarConverter, convertJsonSchemaToGrammar } from './grammar';
-import { formatChat } from './chat';
-import { Tools, injectToolsIntoMessages, parseAndExecuteTool, updateMessagesWithToolCall } from './tools';
-export { SchemaGrammarConverter, convertJsonSchemaToGrammar, Tools };
-const EVENT_ON_INIT_CONTEXT_PROGRESS = '@Cactus_onInitContextProgress';
-const EVENT_ON_TOKEN = '@Cactus_onToken';
-const EVENT_ON_NATIVE_LOG = '@Cactus_onNativeLog';
-let EventEmitter;
-if (Platform.OS === 'ios') {
-  // @ts-ignore
-  EventEmitter = new NativeEventEmitter(Cactus);
-}
-if (Platform.OS === 'android') {
-  EventEmitter = DeviceEventEmitter;
-}
-const logListeners = [];
-
-// @ts-ignore
-if (EventEmitter) {
-  EventEmitter.addListener(EVENT_ON_NATIVE_LOG, evt => {
-    logListeners.forEach(listener => listener(evt.level, evt.text));
-  });
-  // Trigger unset to use default log callback
-  Cactus?.toggleNativeLog?.(false)?.catch?.(() => {});
-}
-const getJsonSchema = responseFormat => {
-  if (responseFormat?.type === 'json_schema') {
-    return responseFormat.json_schema?.schema;
-  }
-  if (responseFormat?.type === 'json_object') {
-    return responseFormat.schema || {};
-  }
-  return null;
-};
-export class LlamaContext {
-  gpu = false;
-  reasonNoGPU = '';
-  constructor({
-    contextId,
-    gpu,
-    reasonNoGPU,
-    model
-  }) {
-    this.id = contextId;
-    this.gpu = gpu;
-    this.reasonNoGPU = reasonNoGPU;
-    this.model = model;
-  }
-
-  /**
-   * Load cached prompt & completion state from a file.
-   */
-  async loadSession(filepath) {
-    let path = filepath;
-    if (path.startsWith('file://')) path = path.slice(7);
-    return Cactus.loadSession(this.id, path);
-  }
-
-  /**
-   * Save current cached prompt & completion state to a file.
-   */
-  async saveSession(filepath, options) {
-    return Cactus.saveSession(this.id, filepath, options?.tokenSize || -1);
-  }
-  isLlamaChatSupported() {
-    return !!this.model.chatTemplates.llamaChat;
-  }
-  isJinjaSupported() {
-    const {
-      minja
-    } = this.model.chatTemplates;
-    return !!minja?.toolUse || !!minja?.default;
-  }
-  async getFormattedChat(messages, template, params) {
-    const chat = formatChat(messages);
-    const useJinja = this.isJinjaSupported() && params?.jinja;
-    let tmpl = this.isLlamaChatSupported() || useJinja ? undefined : 'chatml';
-    if (template) tmpl = template; // Force replace if provided
-    const jsonSchema = getJsonSchema(params?.response_format);
-    return Cactus.getFormattedChat(this.id, JSON.stringify(chat), tmpl, {
-      jinja: useJinja,
-      json_schema: jsonSchema ? JSON.stringify(jsonSchema) : undefined,
-      tools: params?.tools ? JSON.stringify(params.tools) : undefined,
-      parallel_tool_calls: params?.parallel_tool_calls ? JSON.stringify(params.parallel_tool_calls) : undefined,
-      tool_choice: params?.tool_choice
-    });
-  }
-  async completionWithTools(params, callback, recursionCount = 0, recursionLimit = 3) {
-    if (!params.messages) {
-      // tool calling only works with messages
-      return this.completion(params, callback);
-    }
-    if (!params.tools) {
-      // no tools => default completion
-      return this.completion(params, callback);
-    }
-    if (recursionCount >= recursionLimit) {
-      return this.completion(params, callback);
-    }
-    let messages = [...(params?.messages || [])]; // avoid mutating the original
-
-    if (recursionCount === 0) {
-      messages = injectToolsIntoMessages(messages, params.tools);
-    }
-    const result = await this.completion({
-      ...params,
-      messages
-    }, callback);
-    const {
-      toolCalled,
-      toolName,
-      toolInput,
-      toolOutput
-    } = await parseAndExecuteTool(result, params.tools);
-    if (toolCalled && toolName && toolInput) {
-      const newMessages = updateMessagesWithToolCall(messages, toolName, toolInput, toolOutput);
-      return await this.completionWithTools({
-        ...params,
-        messages: newMessages
-      }, callback, recursionCount + 1, recursionLimit);
-    }
-    return result;
-  }
-  async completion(params, callback) {
-    const nativeParams = {
-      ...params,
-      prompt: params.prompt || '',
-      emit_partial_completion: !!callback
-    };
-    if (params.messages) {
-      // messages always win
-      const formattedResult = await this.getFormattedChat(params.messages, params.chat_template || params.chatTemplate, {
-        jinja: params.jinja,
-        tools: params.tools,
-        parallel_tool_calls: params.parallel_tool_calls,
-        tool_choice: params.tool_choice
-      });
-      if (typeof formattedResult === 'string') {
-        nativeParams.prompt = formattedResult || '';
-      } else {
-        nativeParams.prompt = formattedResult.prompt || '';
-        if (typeof formattedResult.chat_format === 'number') nativeParams.chat_format = formattedResult.chat_format;
-        if (formattedResult.grammar) nativeParams.grammar = formattedResult.grammar;
-        if (typeof formattedResult.grammar_lazy === 'boolean') nativeParams.grammar_lazy = formattedResult.grammar_lazy;
-        if (formattedResult.grammar_triggers) nativeParams.grammar_triggers = formattedResult.grammar_triggers;
-        if (formattedResult.preserved_tokens) nativeParams.preserved_tokens = formattedResult.preserved_tokens;
-        if (formattedResult.additional_stops) {
-          if (!nativeParams.stop) nativeParams.stop = [];
-          nativeParams.stop.push(...formattedResult.additional_stops);
-        }
-      }
-    } else {
-      nativeParams.prompt = params.prompt || '';
-    }
-    if (nativeParams.response_format && !nativeParams.grammar) {
-      const jsonSchema = getJsonSchema(params.response_format);
-      if (jsonSchema) nativeParams.json_schema = JSON.stringify(jsonSchema);
-    }
-    let tokenListener = callback && EventEmitter.addListener(EVENT_ON_TOKEN, evt => {
-      const {
-        contextId,
-        tokenResult
-      } = evt;
-      if (contextId !== this.id) return;
-      callback(tokenResult);
-    });
-    if (!nativeParams.prompt) throw new Error('Prompt is required');
-    const promise = Cactus.completion(this.id, nativeParams);
-    return promise.then(completionResult => {
-      tokenListener?.remove();
-      tokenListener = null;
-      return completionResult;
-    }).catch(err => {
-      tokenListener?.remove();
-      tokenListener = null;
-      throw err;
-    });
-  }
-  stopCompletion() {
-    return Cactus.stopCompletion(this.id);
-  }
-  tokenize(text) {
-    return Cactus.tokenize(this.id, text);
-  }
-  detokenize(tokens) {
-    return Cactus.detokenize(this.id, tokens);
-  }
-  embedding(text, params) {
-    return Cactus.embedding(this.id, text, params || {});
-  }
-  async bench(pp, tg, pl, nr) {
-    const result = await Cactus.bench(this.id, pp, tg, pl, nr);
-    const [modelDesc, modelSize, modelNParams, ppAvg, ppStd, tgAvg, tgStd] = JSON.parse(result);
-    return {
-      modelDesc,
-      modelSize,
-      modelNParams,
-      ppAvg,
-      ppStd,
-      tgAvg,
-      tgStd
-    };
-  }
-  async applyLoraAdapters(loraList) {
-    let loraAdapters = [];
-    if (loraList) loraAdapters = loraList.map(l => ({
-      path: l.path.replace(/file:\/\//, ''),
-      scaled: l.scaled
-    }));
-    return Cactus.applyLoraAdapters(this.id, loraAdapters);
-  }
-  async removeLoraAdapters() {
-    return Cactus.removeLoraAdapters(this.id);
-  }
-  async getLoadedLoraAdapters() {
-    return Cactus.getLoadedLoraAdapters(this.id);
-  }
-  async release() {
-    return Cactus.releaseContext(this.id);
-  }
-}
-export async function toggleNativeLog(enabled) {
-  return Cactus.toggleNativeLog(enabled);
-}
-export function addNativeLogListener(listener) {
-  logListeners.push(listener);
-  return {
-    remove: () => {
-      logListeners.splice(logListeners.indexOf(listener), 1);
-    }
-  };
-}
-export async function setContextLimit(limit) {
-  return Cactus.setContextLimit(limit);
-}
-let contextIdCounter = 0;
-const contextIdRandom = () => process.env.NODE_ENV === 'test' ? 0 : Math.floor(Math.random() * 100000);
-const modelInfoSkip = [
-// Large fields
-'tokenizer.ggml.tokens', 'tokenizer.ggml.token_type', 'tokenizer.ggml.merges', 'tokenizer.ggml.scores'];
-export async function loadLlamaModelInfo(model) {
-  let path = model;
-  if (path.startsWith('file://')) path = path.slice(7);
-  return Cactus.modelInfo(path, modelInfoSkip);
-}
-const poolTypeMap = {
-  // -1 is unspecified as undefined
-  none: 0,
-  mean: 1,
-  cls: 2,
-  last: 3,
-  rank: 4
-};
-export async function initLlama({
-  model,
-  is_model_asset: isModelAsset,
-  pooling_type: poolingType,
-  lora,
-  lora_list: loraList,
-  ...rest
-}, onProgress) {
-  let path = model;
-  if (path.startsWith('file://')) path = path.slice(7);
-  let loraPath = lora;
-  if (loraPath?.startsWith('file://')) loraPath = loraPath.slice(7);
-  let loraAdapters = [];
-  if (loraList) loraAdapters = loraList.map(l => ({
-    path: l.path.replace(/file:\/\//, ''),
-    scaled: l.scaled
-  }));
-  const contextId = contextIdCounter + contextIdRandom();
-  contextIdCounter += 1;
-  let removeProgressListener = null;
-  if (onProgress) {
-    removeProgressListener = EventEmitter.addListener(EVENT_ON_INIT_CONTEXT_PROGRESS, evt => {
-      if (evt.contextId !== contextId) return;
-      onProgress(evt.progress);
-    });
-  }
-  const poolType = poolTypeMap[poolingType];
-  const {
-    gpu,
-    reasonNoGPU,
-    model: modelDetails,
-    androidLib
-  } = await Cactus.initContext(contextId, {
-    model: path,
-    is_model_asset: !!isModelAsset,
-    use_progress_callback: !!onProgress,
-    pooling_type: poolType,
-    lora: loraPath,
-    lora_list: loraAdapters,
-    ...rest
-  }).catch(err => {
-    removeProgressListener?.remove();
-    throw err;
-  });
-  removeProgressListener?.remove();
-  return new LlamaContext({
-    contextId,
-    gpu,
-    reasonNoGPU,
-    model: modelDetails,
-    androidLib
-  });
-}
-export async function releaseAllLlama() {
-  return Cactus.releaseAllContexts();
-}
-export const initContext = async params => {
-  return await Cactus.initContext(contextIdCounter++, params);
-};
-export const initMultimodal = async (contextId, mmprojPath, useGpu = false) => {
-  return await Cactus.initMultimodal(contextId, mmprojPath, useGpu);
-};
-export const isMultimodalEnabled = async contextId => {
-  return await Cactus.isMultimodalEnabled(contextId);
-};
-export const isMultimodalSupportVision = async contextId => {
-  return await Cactus.isMultimodalSupportVision(contextId);
-};
-export const isMultimodalSupportAudio = async contextId => {
-  return await Cactus.isMultimodalSupportAudio(contextId);
-};
-export const releaseMultimodal = async contextId => {
-  return await Cactus.releaseMultimodal(contextId);
-};
-export const multimodalCompletion = async (contextId, prompt, mediaPaths, params) => {
-  return await Cactus.multimodalCompletion(contextId, prompt, mediaPaths, params);
-};
-export const initVocoder = async (contextId, vocoderModelPath) => {
-  return await Cactus.initVocoder(contextId, vocoderModelPath);
-};
-export const isVocoderEnabled = async contextId => {
-  return await Cactus.isVocoderEnabled(contextId);
-};
-export const getTTSType = async contextId => {
-  return await Cactus.getTTSType(contextId);
-};
-export const getFormattedAudioCompletion = async (contextId, speakerJsonStr, textToSpeak) => {
-  return await Cactus.getFormattedAudioCompletion(contextId, speakerJsonStr, textToSpeak);
-};
-export const getAudioCompletionGuideTokens = async (contextId, textToSpeak) => {
-  return await Cactus.getAudioCompletionGuideTokens(contextId, textToSpeak);
-};
-export const decodeAudioTokens = async (contextId, tokens) => {
-  return await Cactus.decodeAudioTokens(contextId, tokens);
-};
-export const releaseVocoder = async contextId => {
-  return await Cactus.releaseVocoder(contextId);
-};
-export const tokenize = async (contextId, text, mediaPaths) => {
-  if (mediaPaths && mediaPaths.length > 0) {
-    return await Cactus.tokenize(contextId, text, mediaPaths);
-  } else {
-    return await Cactus.tokenize(contextId, text);
-  }
-};
-//# sourceMappingURL=index.js.map

package/lib/module/tools.js

@@ -1,110 +0,0 @@
-"use strict";
-
-export class Tools {
-  tools = new Map();
-  add(func, description, parameters) {
-    this.tools.set(func.name, {
-      func,
-      description,
-      parameters,
-      required: Object.entries(parameters).filter(([_, param]) => param.required).map(([key, _]) => key)
-    });
-    return func;
-  }
-  getSchemas() {
-    return Array.from(this.tools.entries()).map(([name, {
-      description,
-      parameters,
-      required
-    }]) => ({
-      type: "function",
-      function: {
-        name,
-        description,
-        parameters: {
-          type: "object",
-          properties: parameters,
-          required
-        }
-      }
-    }));
-  }
-  async execute(name, args) {
-    const tool = this.tools.get(name);
-    if (!tool) throw new Error(`Tool ${name} not found`);
-    return await tool.func(...Object.values(args));
-  }
-}
-export function injectToolsIntoMessages(messages, tools) {
-  const newMessages = [...messages];
-  const toolsSchemas = tools.getSchemas();
-  const promptToolInjection = `You have access to the following functions. Use them if required - 
-${JSON.stringify(toolsSchemas, null, 2)}
-Only use an available tool if needed. If a tool is chosen, respond ONLY with a JSON object matching the following schema:
-\`\`\`json
-{
-"tool_name": "<name of the tool>",
-"tool_input": {
-"<parameter_name>": "<parameter_value>",
-...
-}
-}
-\`\`\`
-Remember, if you are calling a tool, you must respond with the JSON object and the JSON object ONLY!
-If no tool is needed, respond normally.
-  `;
-  const systemMessage = newMessages.find(m => m.role === 'system');
-  if (!systemMessage) {
-    newMessages.unshift({
-      role: 'system',
-      content: promptToolInjection
-    });
-  } else {
-    systemMessage.content = `${systemMessage.content}\n\n${promptToolInjection}`;
-  }
-  return newMessages;
-}
-export async function parseAndExecuteTool(result, tools) {
-  const match = result.content.match(/```json\s*([\s\S]*?)\s*```/);
-  if (!match || !match[1]) return {
-    toolCalled: false
-  };
-  try {
-    const jsonContent = JSON.parse(match[1]);
-    const {
-      tool_name,
-      tool_input
-    } = jsonContent;
-    // console.log('Calling tool:', tool_name, tool_input);
-    const toolOutput = (await tools.execute(tool_name, tool_input)) || true;
-    // console.log('Tool called result:', toolOutput);
-
-    return {
-      toolCalled: true,
-      toolName: tool_name,
-      toolInput: tool_input,
-      toolOutput
-    };
-  } catch (error) {
-    // console.error('Error parsing JSON:', match, error);
-    return {
-      toolCalled: false
-    };
-  }
-}
-export function updateMessagesWithToolCall(messages, toolName, toolInput, toolOutput) {
-  const newMessages = [...messages];
-  newMessages.push({
-    role: 'function-call',
-    content: JSON.stringify({
-      name: toolName,
-      arguments: toolInput
-    }, null, 2)
-  });
-  newMessages.push({
-    role: 'function-response',
-    content: JSON.stringify(toolOutput, null, 2)
-  });
-  return newMessages;
-}
-//# sourceMappingURL=tools.js.map

package/lib/module/tools.js.map

@@ -1 +0,0 @@
-{"version":3,"names":["Tools","tools","Map","add","func","description","parameters","set","name","required","Object","entries","filter","_","param","map","key","getSchemas","Array","from","type","function","properties","execute","args","tool","get","Error","values","injectToolsIntoMessages","messages","newMessages","toolsSchemas","promptToolInjection","JSON","stringify","systemMessage","find","m","role","unshift","content","parseAndExecuteTool","result","match","toolCalled","jsonContent","parse","tool_name","tool_input","toolOutput","toolName","toolInput","error","updateMessagesWithToolCall","push","arguments"],"sourceRoot":"../../src","sources":["tools.ts"],"mappings":";;AAgBA,OAAO,MAAMA,KAAK,CAAC;EACTC,KAAK,GAAG,IAAIC,GAAG,CAAe,CAAC;EAEvCC,GAAGA,CACCC,IAAc,EACdC,WAAmB,EACnBC,UAAsC,EACtC;IACA,IAAI,CAACL,KAAK,CAACM,GAAG,CAACH,IAAI,CAACI,IAAI,EAAE;MACxBJ,IAAI;MACJC,WAAW;MACXC,UAAU;MACVG,QAAQ,EAAEC,MAAM,CAACC,OAAO,CAACL,UAAU,CAAC,CACjCM,MAAM,CAAC,CAAC,CAACC,CAAC,EAAEC,KAAK,CAAC,KAAKA,KAAK,CAACL,QAAQ,CAAC,CACtCM,GAAG,CAAC,CAAC,CAACC,GAAG,EAAEH,CAAC,CAAC,KAAKG,GAAG;IAC1B,CAAC,CAAC;IACF,OAAOZ,IAAI;EACb;EAEFa,UAAUA,CAAA,EAAG;IACT,OAAOC,KAAK,CAACC,IAAI,CAAC,IAAI,CAAClB,KAAK,CAACU,OAAO,CAAC,CAAC,CAAC,CAACI,GAAG,CAAC,CAAC,CAACP,IAAI,EAAE;MAAEH,WAAW;MAAEC,UAAU;MAAEG;IAAS,CAAC,CAAC,MAAM;MAC9FW,IAAI,EAAE,UAAU;MAChBC,QAAQ,EAAE;QACRb,IAAI;QACJH,WAAW;QACXC,UAAU,EAAE;UACVc,IAAI,EAAE,QAAQ;UACdE,UAAU,EAAEhB,UAAU;UACtBG;QACF;MACF;IACF,CAAC,CAAC,CAAC;EACL;EAEF,MAAMc,OAAOA,CAACf,IAAY,EAAEgB,IAAS,EAAE;IACnC,MAAMC,IAAI,GAAG,IAAI,CAACxB,KAAK,CAACyB,GAAG,CAAClB,IAAI,CAAC;IACjC,IAAI,CAACiB,IAAI,EAAE,MAAM,IAAIE,KAAK,CAAC,QAAQnB,IAAI,YAAY,CAAC;IACpD,OAAO,MAAMiB,IAAI,CAACrB,IAAI,CAAC,GAAGM,MAAM,CAACkB,MAAM,CAACJ,IAAI,CAAC,CAAC;EAClD;AACF;AAEA,OAAO,SAASK,uBAAuBA,CAACC,QAAsC,EAAE7B,KAAY,EAAgC;EAC1H,MAAM8B,WAAW,GAAG,CAAC,GAAGD,QAAQ,CAAC;EACjC,MAAME,YAAY,GAAG/B,KAAK,CAACgB,UAAU,CAAC,CAAC;EACvC,MAAMgB,mBAAmB,GAAG;AAC9B,EAAEC,IAAI,CAACC,SAAS,CAACH,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC;AACvC;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA,GAAG;EAED,MAAMI,aAAa,GAAGL,WAAW,CAACM,IAAI,CAACC,CAAC,IAAIA,CAAC,CAACC,IAAI,KAAK,QAAQ,CAAC;EAChE,IAAI,CAACH,aAAa,EAAE;IAChBL,WAAW,CAACS,OAAO,CAAC;MAChBD,IAAI,EAAE,QAAQ;MACdE,OAAO,EAAER;IACb,CAAC,CAAC;EACN,CAAC,MAAM;IACHG,aAAa,CAACK,OAAO,GAAG,GAAGL,aAAa,CAACK,OAAO,OAAOR,mBAAmB,EAAE;EAChF;EAEA,OAAOF,WAAW;AACpB;AAEA,OAAO,eAAeW,mBAAmBA,CAACC,MAA8B,EAAE1C,KAAY,EAAwF;EAC5K,MAAM2C,KAAK,GAAGD,MAAM,CAACF,OAAO,CAACG,KAAK,CAAC,4BAA4B,CAAC;EAEhE,IAAI,CAACA,KAAK,IAAI,CAACA,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO;IAACC,UAAU,EAAE;EAAK,CAAC;EAEnD,IAAI;IACA,MAAMC,WAAW,GAAGZ,IAAI,CAACa,KAAK,CAACH,KAAK,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM;MAAEI,SAAS;MAAEC;IAAW,CAAC,GAAGH,WAAW;IAC7C;IACA,MAAMI,UAAU,GAAG,OAAMjD,KAAK,CAACsB,OAAO,CAACyB,SAAS,EAAEC,UAAU,CAAC,KAAI,IAAI;IACrE;;IAEA,OAAO;MACHJ,UAAU,EAAE,IAAI;MAChBM,QAAQ,EAAEH,SAAS;MACnBI,SAAS,EAAEH,UAAU;MACrBC;IACJ,CAAC;EACL,CAAC,CAAC,OAAOG,KAAK,EAAE;IACZ;IACA,OAAO;MAACR,UAAU,EAAE;IAAK,CAAC;EAC9B;AACF;AAEA,OAAO,SAASS,0BAA0BA,CAACxB,QAAsC,EAAEqB,QAAgB,EAAEC,SAAc,EAAEF,UAAe,EAAgC;EAClK,MAAMnB,WAAW,GAAG,CAAC,GAAGD,QAAQ,CAAC;EAEjCC,WAAW,CAACwB,IAAI,CAAC;IACbhB,IAAI,EAAE,eAAe;IACrBE,OAAO,EAAEP,IAAI,CAACC,SAAS,CAAC;MAAC3B,IAAI,EAAE2C,QAAQ;MAAEK,SAAS,EAAEJ;IAAS,CAAC,EAAE,IAAI,EAAE,CAAC;EAC3E,CAAC,CAAC;EACFrB,WAAW,CAACwB,IAAI,CAAC;IACbhB,IAAI,EAAE,mBAAmB;IACzBE,OAAO,EAAEP,IAAI,CAACC,SAAS,CAACe,UAAU,EAAE,IAAI,EAAE,CAAC;EAC/C,CAAC,CAAC;EAEF,OAAOnB,WAAW;AACpB","ignoreList":[]}