@fugood/bricks-ctor 2.24.0-beta.40

package/types/generators/LlmGgml.ts

@@ -0,0 +1,861 @@
+/* Auto generated by build script
+ *
+ * Local Large Language Model (LLM) inference based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)
+ *
+ *  ## Notice
+ *  - The device RAM must be larger than 8GB
+ *  - iOS: Recommended use M1+ / A17+ chip device. Supported GPU acceleration by Metal.
+ *  - macOS: Recommended use M1+ chip device. Supported GPU acceleration by Metal.
+ *  - Android: Recommended use Android 13+ system.
+ *    - Supported GPU acceleration by OpenCL, currently only for Qualcomm Adreno 700+ GPUs, other GPUs are not supported.
+ *    - Supported Hexagon NPU for Qualcomm Snapdragon 8 Gen 1+ GPUs.
+ *  - Linux / Windows  [@nextline    - Supported GPU acceleration, you can choose `vulkan` or `cuda` backend in Accel Variant property
+ *    - Supported Hexagon NPU for Qualcomm Dragonwing IQ9 series+ (Linux)
+ */
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Brick,
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+import type { TemplateEventPropsMap } from '../../utils/event-props'
+
+/* Load the model */
+export type GeneratorLLMActionLoadModel = Action & {
+  __actionName: 'GENERATOR_LLM_LOAD_MODEL'
+}
+
+/* Load multimodal (vision) model (PREVIEW FEATURE) */
+export type GeneratorLLMActionLoadMultimodalModel = Action & {
+  __actionName: 'GENERATOR_LLM_LOAD_MULTIMODAL_MODEL'
+}
+
+/* Tokenize the prompt */
+export type GeneratorLLMActionTokenize = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_TOKENIZE'
+  params?: Array<
+    | {
+        input: 'mode'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'promptMediaPaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Detokenize the tokens to text */
+export type GeneratorLLMActionDetokenize = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_DETOKENIZE'
+  params?: Array<{
+    input: 'tokens'
+    value?: Array<any> | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Pre-process the prompt, this can speed up the completion action */
+export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_PROCESS_PROMPT'
+  params?: Array<
+    | {
+        input: 'sessionKey'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mode'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'parallelToolCalls'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'toolChoice'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'enableThinking'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'thinkingBudgetTokens'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'thinkingBudgetMessage'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'promptMediaPaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'promptTemplateData'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'promptTemplateType'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'responseFormat'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'chatTemplateKwargs'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'addGenerationPrompt'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'now'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'forcePureContent'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Run text completion */
+export type GeneratorLLMActionCompletion = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_COMPLETION'
+  params?: Array<
+    | {
+        input: 'sessionKey'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mode'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'parallelToolCalls'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'toolChoice'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'enableThinking'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'thinkingBudgetTokens'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'thinkingBudgetMessage'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'useReasoningFormat'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'promptMediaPaths'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'promptTemplateData'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'promptTemplateType'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'responseFormat'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'chatTemplateKwargs'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'addGenerationPrompt'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'now'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'forcePureContent'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'grammar'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'stopWords'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'predict'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'temperature'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'probs'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'topK'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'topP'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'xtcThreshold'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'xtcProbability'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'dryMultiplier'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'dryBase'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'dryAllowedLength'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'dryPenaltyLastN'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'drySequenceBreakers'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mirostat'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mirostatTau'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mirostatEta'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'penaltyLastN'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'penaltyRepeat'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'penaltyFrequency'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'penaltyPresent'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'penalizeNewline'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'seed'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'typicalP'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'ignoreEos'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'functionCallEnabled'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'functionCallSchema'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Clear session with session key or session ID */
+export type GeneratorLLMActionClearSession = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_CLEAR_SESSION'
+  params?: Array<
+    | {
+        input: 'sessionId'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'sessionCustomKey'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Stop text completion */
+export type GeneratorLLMActionStopCompletion = Action & {
+  __actionName: 'GENERATOR_LLM_STOP_COMPLETION'
+}
+
+/* Clear KV cache */
+export type GeneratorLLMActionClearCache = Action & {
+  __actionName: 'GENERATOR_LLM_CLEAR_CACHE'
+}
+
+/* Clear downloaded models & current jobs */
+export type GeneratorLLMActionClearDownload = Action & {
+  __actionName: 'GENERATOR_LLM_CLEAR_DOWNLOAD'
+}
+
+/* Release multimodal (vision) context (PREVIEW FEATURE) */
+export type GeneratorLLMActionReleaseMultimodalContext = Action & {
+  __actionName: 'GENERATOR_LLM_RELEASE_MULTIMODAL_CONTEXT'
+}
+
+/* Release context */
+export type GeneratorLLMActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_LLM_RELEASE_CONTEXT'
+}
+
+interface GeneratorLLMDef {
+  /*
+Default property:
+{
+  "init": false,
+  "contextSize": 512,
+  "batchSize": 512,
+  "uBatchSize": 512,
+  "accelVariant": "default",
+  "mainGpu": 0,
+  "gpuLayers": 0,
+  "useMlock": true,
+  "useMmap": true,
+  "cacheKType": "f16",
+  "cacheVType": "f16",
+  "ctxShift": true,
+  "cpuMoeLayers": 0,
+  "transformScriptEnabled": false,
+  "transformScriptCode": "\/\* Global variable: inputs = { prompt, messages, variables }, members = { llmUtils } \*\/\nreturn inputs.prompt",
+  "transformScriptVariables": {},
+  "sessionMinSaveSize": 50,
+  "sessionRemain": 10,
+  "completionMode": "auto",
+  "completionPrompt": "",
+  "completionPromptTemplateType": "${}",
+  "completionEnableThinking": true,
+  "completionAddGenerationPrompt": true,
+  "completionChatTemplateKwargs": {},
+  "completionForcePureContent": false,
+  "completionUseReasoningFormat": "auto",
+  "completionStopWords": [],
+  "completionPredict": 400,
+  "completionTopK": 40,
+  "completionTopP": 0.95,
+  "completionMinP": 0.05,
+  "completionDryMultiplier": 0,
+  "completionDryBase": 1.75,
+  "completionDryAllowedLength": 2,
+  "completionDrySequenceBreakers": [
+    "\n",
+    ":",
+    "\"",
+    "*"
+  ],
+  "completionMirostat": 0,
+  "completionMirostatTau": 5,
+  "completionMirostatEta": 0.1,
+  "completionPenaltyLastN": 64,
+  "completionPenaltyRepeat": 1,
+  "completionPenaltyFrequency": 0,
+  "completionPenaltyPresent": 0,
+  "completionPenalizeNewline": false,
+  "completionTypicalP": 1
+}
+  */
+  property?: {
+    /* Initialize the Llama context on generator initialization
+ Please note that it will take some RAM depending on the model size */
+    init?: boolean | DataLink
+    /* The URL or path of model
+ We used GGUF format model, please refer to https://github.com/ggerganov/llama.cpp/tree/master#description */
+    modelUrl?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
+    /* Load multimodal (vision) context after model loaded (PREVIEW FEATURE) */
+    initMultimodal?: boolean | DataLink
+    /* The URL or path of mmproj file for multimodal vision support (PREVIEW FEATURE) */
+    mmprojUrl?: string | DataLink
+    /* Hash type of mmproj file (PREVIEW FEATURE) */
+    mmprojHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of mmproj file (PREVIEW FEATURE) */
+    mmprojHash?: string | DataLink
+    /* Minimum tokens for image encoding in multimodal (PREVIEW FEATURE)
+ Useful for dynamic resolution models (e.g. Qwen-VL). Default: -1 (auto) */
+    imageMinTokens?: number | DataLink
+    /* Maximum tokens for image encoding in multimodal (PREVIEW FEATURE)
+ Limit tokens for dynamic resolution models to balance speed vs. detail. Default: -1 (auto) */
+    imageMaxTokens?: number | DataLink
+    /* Chat Template (Jinja format) to override the default template from model */
+    chatTemplate?: string | DataLink
+    /* Context size (0 ~ 4096) (Default to 512) */
+    contextSize?: number | DataLink
+    /* Logical batch size for prompt processing */
+    batchSize?: number | DataLink
+    /* Physical batch size for prompt processing */
+    uBatchSize?: number | DataLink
+    /* Number of threads */
+    maxThreads?: number | DataLink
+    /* Accelerator variant (Only for desktop)
+ `default` - CPU / Metal (macOS)  
+ `vulkan` - Use Vulkan  
+ `cuda` - Use CUDA  
+ `snapdragon` - Use OpenCL/Hexagon of Qualcomm Snapdragon */
+    accelVariant?: 'default' | 'vulkan' | 'cuda' | 'snapdragon' | DataLink
+    /* Devices. For example:
+  
+ Metal or CPU for iOS/tvOS/MacOS  
+ OpenCL or CPU for Android  
+   - Add `HTP0`, `HTP1`, `...` for OpenCL/Hexagon devices (Use HTP* for all HTP devices)  
+ For Desktop, you may want to include/exclude GPUs */
+    devices?: Array<string | DataLink> | DataLink
+    /* Main GPU index */
+    mainGpu?: number | DataLink
+    /* Number of GPU layers */
+    gpuLayers?: number | DataLink
+    /* Use memory lock */
+    useMlock?: boolean | DataLink
+    /* Use mmap */
+    useMmap?: boolean | DataLink
+    /* Disable extra buffer types for weight repacking. Reduces memory usage at the cost of slower prompt processing. */
+    noExtraBuffs?: boolean | DataLink
+    /* Use Flash Attention for inference (Recommended with GPU enabled) */
+    useFlashAttn?: 'auto' | 'on' | 'off' | DataLink
+    /* KV cache data type for the K (Default: f16) */
+    cacheKType?: 'f16' | 'f32' | 'q8_0' | 'q4_0' | 'q4_1' | 'iq4_nl' | 'q5_0' | 'q5_1' | DataLink
+    /* KV cache data type for the V (Default: f16) */
+    cacheVType?: 'f16' | 'f32' | 'q8_0' | 'q4_0' | 'q4_1' | 'iq4_nl' | 'q5_0' | 'q5_1' | DataLink
+    /* Use a unified buffer across the input sequences when computing the attention */
+    useKVUnified?: boolean | DataLink
+    /* Use full-size SWA cache. May improve performance for multiple sequences but uses more memory. */
+    useSwaFull?: boolean | DataLink
+    /* Enable context shift */
+    ctxShift?: boolean | DataLink
+    /* Number of layers to keep MoE weights on CPU */
+    cpuMoeLayers?: number | DataLink
+    /* Enable Transform Script for processing the prompt */
+    transformScriptEnabled?: boolean | DataLink
+    /* Code of Transform Script */
+    transformScriptCode?: string | DataLink
+    /* Variables used in Transform Script (object) */
+    transformScriptVariables?: {} | DataLink
+    /* Session save mode
+ `none` - No session saving  
+ `prompt` - Save session when prompt processed  
+ `completion` - Save session when completion finished  
+ `all` - Save session when prompt processed and completion finished */
+    sessionSaveMode?: 'none' | 'prompt' | 'completion' | 'all' | DataLink
+    /* Minimum processed/generated size to determine whether to save session (Unit: token) */
+    sessionMinSaveSize?: number | DataLink
+    /* Session file remain count (Default to 10) */
+    sessionRemain?: number | DataLink
+    /* TODO:loran_gqarms_norm_epsrope_freq_baserope_freq_scale */
+    completionMode?: 'auto' | 'chat' | 'text' | DataLink
+    /* Tools for chat mode using OpenAI-compatible function calling format
+ Format: Array of objects with {type, function: {name, description, parameters}} structure  
+ See: https://platform.openai.com/docs/guides/function-calling */
+    completionTools?: Array<{} | DataLink> | DataLink
+    /* Enable parallel tool calls */
+    completionParallelToolCalls?: boolean | DataLink
+    /* Tool choice for chat mode */
+    completionToolChoice?: 'none' | 'auto' | 'required' | DataLink
+    /* Messages (chat mode) */
+    completionMessages?:
+      | Array<
+          | DataLink
+          | {
+              role?: string | DataLink
+              content?: string | DataLink
+            }
+        >
+      | DataLink
+    /* Prompt (text mode) */
+    completionPrompt?: string | DataLink
+    /* Media paths to be used in the prompt template (PREVIEW FEATURE)
+ In prompt, use `<__media__>` for position of media content */
+    completionPromptMediaPaths?: Array<string | DataLink> | DataLink
+    /* Data to be used in the prompt template (e.g. `Hello ${name}`). Supports nested data, such as `Hello ${user.name}`. */
+    completionPromptTemplateData?: {} | DataLink
+    /* The prompt template type */
+    completionPromptTemplateType?: '${}' | '{{}}' | DataLink
+    /* Response format */
+    completionResponseFormat?:
+      | DataLink
+      | {
+          type?: 'text' | 'json_schema' | 'json_object' | DataLink
+          json_schema?:
+            | DataLink
+            | {
+                strict?: boolean | DataLink
+                schema?: {} | DataLink
+              }
+          schema?: {} | DataLink
+        }
+    /* Enable thinking */
+    completionEnableThinking?: boolean | DataLink
+    /* Maximum tokens allowed inside the model's thinking block before forcing it closed. Only applies when chat formatting exposes thinking tags. */
+    completionThinkingBudgetTokens?: number | DataLink
+    /* Message injected before the thinking end tag when the thinking budget is exhausted. */
+    completionThinkingBudgetMessage?: string | DataLink
+    /* Add generation prompt */
+    completionAddGenerationPrompt?: boolean | DataLink
+    /* Now (for fill current date in chat template if supported) */
+    completionNow?: string | DataLink
+    /* Additional keyword arguments for chat template (object) */
+    completionChatTemplateKwargs?: {} | DataLink
+    /* Force pure content (accept any model that has a chat_template without requiring template validation) */
+    completionForcePureContent?: boolean | DataLink
+    /* Use reasoning format for enhanced response structure
+ `auto` - Auto-determine the reasoning format of the model  
+ `none` - Disable reasoning format */
+    completionUseReasoningFormat?: 'auto' | 'none' | DataLink
+    /* Stop words */
+    completionStopWords?: Array<string | DataLink> | DataLink
+    /* Number of tokens to predict */
+    completionPredict?: number | DataLink
+    /* Throttle time for completion result (in milliseconds) */
+    completionResultThrottle?: number | DataLink
+    /* Grammar (GBNF: Please refer to https://github.com/ggerganov/llama.cpp/tree/master/grammars) */
+    completionGrammar?: string | DataLink
+    /* Temperature */
+    completionTemperature?: number | DataLink
+    /* Number of probablites to show for each token in the completion details */
+    completionProbs?: number | DataLink
+    /* Top K sampling */
+    completionTopK?: number | DataLink
+    /* Top P sampling */
+    completionTopP?: number | DataLink
+    /* Min P sampling */
+    completionMinP?: number | DataLink
+    /* Sets a minimum probability threshold for tokens to be removed */
+    completionXtcThreshold?: number | DataLink
+    /* Sets the chance for token removal (checked once on sampler start) */
+    completionXtcProbability?: number | DataLink
+    /* Set the DRY (Don't Repeat Yourself) repetition penalty multiplier. Default: `0.0`, which is disabled. */
+    completionDryMultiplier?: number | DataLink
+    /* Set the DRY repetition penalty base value. Default: `1.75` */
+    completionDryBase?: number | DataLink
+    /* Tokens that extend repetition beyond this receive exponentially increasing penalty: multiplier * base ^ (length of repeating sequence before token - allowed length). Default: `2` */
+    completionDryAllowedLength?: number | DataLink
+    /* How many tokens to scan for repetitions. Default: `-1`, where `0` is disabled and `-1` is context size. */
+    completionDryPenaltyLastN?: number | DataLink
+    /* Specify an array of sequence breakers for DRY sampling. Only a JSON array of strings is accepted. Default: `['\n', ':', '"', '*']` */
+    completionDrySequenceBreakers?: Array<string | DataLink> | DataLink
+    /* Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641. Default: `-1.0` (Disabled) */
+    completionTopNSigma?: number | DataLink
+    /* Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally Typical samplers are ignored if used. */
+    completionMirostat?: number | DataLink
+    /* Mirostat target entropy, parameter tau */
+    completionMirostatTau?: number | DataLink
+    /* Mirostat learning rate, parameter eta */
+    completionMirostatEta?: number | DataLink
+    /* Last n tokens to consider for penalize */
+    completionPenaltyLastN?: number | DataLink
+    /* Penalize repeat sequence of tokens (default: 0.1, 1.0 = disabled) */
+    completionPenaltyRepeat?: number | DataLink
+    /* Repeat alpha frequency penalty (default: 0.1, 0.0 = disabled) */
+    completionPenaltyFrequency?: number | DataLink
+    /* Repeat alpha presence penalty (default: 0.1, 0.0 = disabled) */
+    completionPenaltyPresent?: number | DataLink
+    /* Penalize newline tokens when applying the repeat penalty (default: true) */
+    completionPenalizeNewline?: boolean | DataLink
+    /* Set the random number generator (RNG) seed (default: -1, -1 = random seed) */
+    completionSeed?: number | DataLink
+    /* locally typical sampling, parameter p (default: 0.1, 1.0 = disabled) */
+    completionTypicalP?: number | DataLink
+    /* Repeat alpha frequency penalty (default: 0.1, 0.0 = disabled) */
+    completionIgnoreEOS?: boolean | DataLink
+    /* Buttress connection settings for remote inference */
+    buttressConnectionSettings?:
+      | DataLink
+      | {
+          enabled?: boolean | DataLink
+          url?: string | DataLink
+          fallbackType?: 'use-local' | 'no-op' | DataLink
+          strategy?: 'prefer-local' | 'prefer-buttress' | 'prefer-best' | DataLink
+        }
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['Llm']['onContextStateChange']>
+    >
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction<string & keyof TemplateEventPropsMap['Llm']['onError']>>
+    /* Event triggered when completion */
+    onCompletion?: Array<EventAction<string & keyof TemplateEventPropsMap['Llm']['onCompletion']>>
+    /* Event triggered when completion finished */
+    onCompletionFinished?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['Llm']['onCompletionFinished']>
+    >
+    /* Event triggered on get function call request */
+    onCompletionFunctionCall?: Array<
+      EventAction<string & keyof TemplateEventPropsMap['Llm']['onCompletionFunctionCall']>
+    >
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data<string>
+    /* Context load progress (0-100) */
+    contextLoadProgress?: () => Data<number>
+    /* Context details */
+    contextDetails?: () => Data<{
+      state?: string
+      contextId?: string
+      gpu?: string
+      reasonNoGPU?: string
+      model?: { [key: string]: any }
+      isMultimodalEnabled?: boolean
+      [key: string]: any
+    }>
+    /* Session details */
+    sessions?: () => Data<{
+      last_session_id?: string
+      sessions?: Array<{
+        id?: string
+        type?: string
+        prompt?: string
+        sessionKey?: string
+        model_instance_id?: string
+        tokens_evaluated?: number
+        t?: number
+        [key: string]: any
+      }>
+      last_custom_session_id?: string
+      custom_sessions?: Array<{
+        id?: string
+        type?: string
+        prompt?: string
+        sessionKey?: string
+        model_instance_id?: string
+        tokens_evaluated?: number
+        t?: number
+        [key: string]: any
+      }>
+      [key: string]: any
+    }>
+    /* Is evaluating */
+    isEvaluating?: () => Data<boolean>
+    /* Tokenize result */
+    tokenizeResult?: () => Data<Array<number>>
+    /* Detokenize result */
+    detokenizeResult?: () => Data<string>
+    /* Last formatted prompt (messages or prompt) */
+    completionLastFormattedPrompt?: () => Data<string>
+    /* Last completion token */
+    completionLastToken?: () => Data<string>
+    /* Completion result */
+    completionResult?: () => Data<string>
+    /* Reasoning content from model responses */
+    completionReasoningContent?: () => Data<string>
+    /* Full context (Prompt + Completion) */
+    completionFullContext?: () => Data<string>
+    /* Inference result details */
+    completionResultDetails?: () => Data<{
+      prompt?: string
+      full_context?: string
+      text?: string
+      content?: string
+      reasoning_content?: string
+      token?: string
+      tool_calls?: Array<{
+        id?: string
+        type?: string
+        function?: {
+          name?: string
+          arguments?: string
+          [key: string]: any
+        }
+        [key: string]: any
+      }>
+      [key: string]: any
+    }>
+  }
+}
+
+/* Local Large Language Model (LLM) inference based on GGML and [llama.cpp](https://github.com/ggerganov/llama.cpp)
+
+ ## Notice  
+ - The device RAM must be larger than 8GB  
+ - iOS: Recommended use M1+ / A17+ chip device. Supported GPU acceleration by Metal.  
+ - macOS: Recommended use M1+ chip device. Supported GPU acceleration by Metal.  
+ - Android: Recommended use Android 13+ system.  
+   - Supported GPU acceleration by OpenCL, currently only for Qualcomm Adreno 700+ GPUs, other GPUs are not supported.  
+   - Supported Hexagon NPU for Qualcomm Snapdragon 8 Gen 1+ GPUs.  
+ - Linux / Windows  [@nextline    - Supported GPU acceleration, you can choose `vulkan` or `cuda` backend in Accel Variant property  
+   - Supported Hexagon NPU for Qualcomm Dragonwing IQ9 series+ (Linux) */
+export type GeneratorLLM = Generator &
+  GeneratorLLMDef & {
+    templateKey: 'GENERATOR_LLM'
+    switches?: Array<
+      SwitchDef &
+        GeneratorLLMDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'contextLoadProgress'
+                    | 'contextDetails'
+                    | 'sessions'
+                    | 'isEvaluating'
+                    | 'tokenizeResult'
+                    | 'detokenizeResult'
+                    | 'completionLastFormattedPrompt'
+                    | 'completionLastToken'
+                    | 'completionResult'
+                    | 'completionReasoningContent'
+                    | 'completionFullContext'
+                    | 'completionResultDetails'
+                  value: any
+                }
+          }>
+        }
+    >
+  }