@fugood/bricks-project 2.22.0-beta.9 → 2.22.0

package/types/generators/LlmOnnx.ts

@@ -0,0 +1,184 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load the model */
+export type GeneratorOnnxLLMActionLoadModel = Action & {
+  __actionName: 'GENERATOR_ONNX_LLM_LOAD_MODEL'
+}
+
+/* Inference */
+export type GeneratorOnnxLLMActionInfer = ActionWithParams & {
+  __actionName: 'GENERATOR_ONNX_LLM_INFER'
+  params?: Array<
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'chat'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'images'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'toolChoice'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Clean cache */
+export type GeneratorOnnxLLMActionCleanCache = Action & {
+  __actionName: 'GENERATOR_ONNX_LLM_CLEAN_CACHE'
+}
+
+/* Release context */
+export type GeneratorOnnxLLMActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_ONNX_LLM_RELEASE_CONTEXT'
+}
+
+interface GeneratorOnnxLLMDef {
+  /*
+Default property:
+{
+  "modelType": "auto",
+  "toolCallParser": "llama3_json",
+  "toolChoice": "auto",
+  "maxNewTokens": 256,
+  "temperature": 0.7,
+  "topK": 50,
+  "topP": 0.9,
+  "repetitionPenalty": 1,
+  "noRepeatNgramSize": 0,
+  "numBeams": 1,
+  "doSample": true,
+  "executionMode": "sequential"
+}
+  */
+  property?: {
+    /* Initialize the TTS context on generator initialization */
+    init?: boolean | DataLink
+    /* LLM model */
+    model?: string | DataLink
+    /* Model type */
+    modelType?: string | DataLink
+    /* Quantize type */
+    quantizeType?:
+      | 'auto'
+      | 'none'
+      | 'fp16'
+      | 'q8'
+      | 'int8'
+      | 'uint8'
+      | 'q4'
+      | 'bnb4'
+      | 'q4f16'
+      | DataLink
+    /* Prompt to inference */
+    prompt?: string | DataLink
+    /* Messages to inference */
+    messages?: Array<DataLink | {}> | DataLink
+    /* Images with message to inference */
+    images?: Array<string | DataLink> | DataLink
+    /* Tool call parser */
+    toolCallParser?: 'llama3_json' | 'mistral' | 'hermes' | 'internlm' | 'phi4' | DataLink
+    /* Tools for chat mode using OpenAI-compatible function calling format  
+ Format: Array of objects with {type, function: {name, description, parameters}} structure  
+ See: https://platform.openai.com/docs/guides/function-calling */
+    tools?: Array<{} | DataLink> | DataLink
+    /* Tool choice for chat mode */
+    toolChoice?: 'none' | 'auto' | DataLink
+    /* Max new tokens to generate */
+    maxNewTokens?: number | DataLink
+    /* Temperature */
+    temperature?: number | DataLink
+    /* Top k */
+    topK?: number | DataLink
+    /* Top p */
+    topP?: number | DataLink
+    /* Repetition penalty */
+    repetitionPenalty?: number | DataLink
+    /* No repeat ngram size */
+    noRepeatNgramSize?: number | DataLink
+    /* Number of beams */
+    numBeams?: number | DataLink
+    /* Do sampling */
+    doSample?: boolean | DataLink
+    /* Executor candidates, descending order of priority  
+  Default will be xnnpack, wasm, cpu */
+    executors?:
+      | Array<'qnn' | 'dml' | 'nnapi' | 'xnnpack' | 'coreml' | 'cpu' | 'wasm' | 'webgpu' | DataLink>
+      | DataLink
+    /* Execution mode  
+  Usually when the model has many branches, setting this option to `parallel` will give you better performance. */
+    executionMode?: 'sequential' | 'parallel' | DataLink
+    /* QNN backend */
+    qnnBackend?: 'HTP' | 'HTA' | 'DSP' | 'GPU' | 'CPU' | DataLink
+    /* Enable FP16 for QNN HTP */
+    qnnHtpEnableFp16?: boolean | DataLink
+    /* Enable QNN debug */
+    qnnEnableDebug?: boolean | DataLink
+  }
+  events?: {
+    /* Event triggered when state change */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered on get function call request */
+    onFunctionCall?: Array<EventAction>
+    /* Event triggered on completion finished */
+    onCompletionFinished?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Generated output */
+    generated?: () => Data
+    /* Full result of generation */
+    fullResult?: () => Data
+    /* Last function call */
+    lastFunctionCall?: () => Data
+  }
+}
+
+/* Local LLM inference based on [transformers.js](https://huggingface.co/docs/transformers.js)  
+ You can use any converted model on HuggingFace. */
+export type GeneratorOnnxLLM = Generator &
+  GeneratorOnnxLLMDef & {
+    templateKey: 'GENERATOR_ONNX_LLM'
+    switches: Array<
+      SwitchDef &
+        GeneratorOnnxLLMDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'contextState' | 'generated' | 'fullResult' | 'lastFunctionCall'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/LlmOpenAiCompat.ts

@@ -0,0 +1,206 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Run text completion */
+export type GeneratorOpenAILLMActionCompletion = ActionWithParams & {
+  __actionName: 'GENERATOR_OPENAI_LLM_COMPLETION'
+  params?: Array<
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'maxTokens'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'temperature'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'topP'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'frequencyPenalty'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'presencePenalty'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'stop'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'tools'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'toolChoice'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'parallelToolCalls'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'responseFormat'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Stop text completion */
+export type GeneratorOpenAILLMActionStopCompletion = Action & {
+  __actionName: 'GENERATOR_OPENAI_LLM_STOP_COMPLETION'
+}
+
+interface GeneratorOpenAILLMDef {
+  /*
+Default property:
+{
+  "apiEndpoint": "https://api.openai.com/v1",
+  "model": "gpt-4o",
+  "completionMessages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    }
+  ],
+  "completionMaxTokens": 1024,
+  "completionTemperature": 1,
+  "completionTopP": 1,
+  "completionStop": []
+}
+  */
+  property?: {
+    /* API endpoint URL */
+    apiEndpoint?: string | DataLink
+    /* API key */
+    apiKey?: string | DataLink
+    /* Model name (Default: gpt-4o-mini) */
+    model?: string | DataLink
+    /* Chat messages */
+    completionMessages?:
+      | Array<
+          | DataLink
+          | {
+              role?: string | DataLink
+              content?:
+                | string
+                | DataLink
+                | DataLink
+                | {
+                    type?: string | DataLink
+                    text?: string | DataLink
+                    image_url?: string | DataLink
+                  }
+                | DataLink
+            }
+        >
+      | DataLink
+    /* Tools for chat mode following OpenAI function calling format  
+ Format: Array of objects with {type, function: {name, description, parameters}} structure  
+ See: https://platform.openai.com/docs/guides/function-calling */
+    completionTools?: Array<{} | DataLink> | DataLink
+    /* Enable parallel tool calls */
+    completionParallelToolCalls?: boolean | DataLink
+    /* Tool choice for chat mode */
+    completionToolChoice?: 'none' | 'auto' | 'required' | DataLink
+    /* Response format */
+    completionResponseFormat?:
+      | DataLink
+      | {
+          type?: 'text' | 'json_schema' | 'json_object' | DataLink
+          json_schema?:
+            | DataLink
+            | {
+                strict?: boolean | DataLink
+                schema?: {} | DataLink
+              }
+        }
+    /* Maximum tokens to generate */
+    completionMaxTokens?: number | DataLink
+    /* Temperature */
+    completionTemperature?: number | DataLink
+    /* Top P sampling */
+    completionTopP?: number | DataLink
+    /* Frequency penalty */
+    completionFrequencyPenalty?: number | DataLink
+    /* Presence penalty */
+    completionPresencePenalty?: number | DataLink
+    /* Stop sequences */
+    completionStop?: Array<string | DataLink> | DataLink
+  }
+  events?: {
+    /* Error event */
+    onError?: Array<EventAction>
+    /* Completion event */
+    onCompletion?: Array<EventAction>
+    /* Completion finished event */
+    onCompletionFinished?: Array<EventAction>
+    /* Completion function call event */
+    onCompletionFunctionCall?: Array<EventAction>
+  }
+  outlets?: {
+    /* Evaluating outlet */
+    isEvaluating?: () => Data
+    /* Completion result outlet */
+    completionResult?: () => Data
+    /* Completion details outlet */
+    completionDetails?: () => Data
+  }
+}
+
+/* LLM inference using OpenAI-compatible API endpoints  
+
+ ## Features  
+ - Compatible with OpenAI API format  
+ - Supports function calling  
+ - Streaming responses  
+ - Custom API endpoints, like  
+   - OpenAI API: https://platform.openai.com/docs/guides/text?api-mode=chat  
+   - Anthropic API: https://docs.anthropic.com/en/api/openai-sdk  
+   - Gemini API: https://ai.google.dev/gemini-api/docs/openai  
+   - llama.cpp server: https://github.com/ggml-org/llama.cpp/tree/master/tools/server */
+export type GeneratorOpenAILLM = Generator &
+  GeneratorOpenAILLMDef & {
+    templateKey: 'GENERATOR_OPENAI_LLM'
+    switches: Array<
+      SwitchDef &
+        GeneratorOpenAILLMDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isEvaluating' | 'completionResult' | 'completionDetails'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/generators/LlmQualcommAiEngine.ts

@@ -0,0 +1,213 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Load the model */
+export type GeneratorQnnLlmActionLoadModel = Action & {
+  __actionName: 'GENERATOR_QNN_LLM_LOAD_MODEL'
+}
+
+/* Abort model download */
+export type GeneratorQnnLlmActionAbortModelDownload = Action & {
+  __actionName: 'GENERATOR_QNN_LLM_ABORT_MODEL_DOWNLOAD'
+}
+
+/* Pre-process the prompt, to prepare KV cache */
+export type GeneratorQnnLlmActionProcess = ActionWithParams & {
+  __actionName: 'GENERATOR_QNN_LLM_PROCESS'
+  params?: Array<
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Generate text */
+export type GeneratorQnnLlmActionGenerate = ActionWithParams & {
+  __actionName: 'GENERATOR_QNN_LLM_GENERATE'
+  params?: Array<
+    | {
+        input: 'prompt'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'messages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'tools'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Abort generation */
+export type GeneratorQnnLlmActionAbortGeneration = Action & {
+  __actionName: 'GENERATOR_QNN_LLM_ABORT_GENERATION'
+}
+
+/* Release context */
+export type GeneratorQnnLlmActionReleaseContext = Action & {
+  __actionName: 'GENERATOR_QNN_LLM_RELEASE_CONTEXT'
+}
+
+interface GeneratorQnnLlmDef {
+  /*
+Default property:
+{
+  "modelType": "Llama 3.2 3B Chat",
+  "chatFormat": "Llama 3.x",
+  "toolsInUserMessage": true,
+  "toolCallParser": "llama3_json",
+  "toolChoice": "auto",
+  "parallelToolCalls": false,
+  "temperature": 0.8,
+  "seed": 42,
+  "topK": 40,
+  "topP": 0.95,
+  "greedy": false
+}
+  */
+  property?: {
+    /* Load model context when generator is initialized */
+    init?: boolean | DataLink
+    /* Model type */
+    modelType?:
+      | 'Llama 3 8B Chat'
+      | 'Llama 3.1 8B Chat'
+      | 'Llama 3.2 3B Chat'
+      | 'Mistral 7B Instruct v0.3'
+      | 'Qwen 2 7B Chat'
+      | 'Phi 3.5 Mini'
+      | 'Granite v3.1 8B Instruct'
+      | 'Custom'
+      | DataLink
+    /* SOC model */
+    socModel?: 'X Elite' | 'X Plus' | '8 Elite' | '8 Gen 3' | 'QCS8550' | DataLink
+    /* Custom model base URL  
+ The model should be bundled, for details see https://github.com/mybigday/node-qnn-llm?tab=readme-ov-file#bundled-file */
+    customModelUrl?: string | DataLink
+    /* Custom model MD5 */
+    customModelMd5?: string | DataLink
+    /* Chat format */
+    chatFormat?:
+      | 'Llama 2'
+      | 'Llama 3'
+      | 'Llama 3.x'
+      | 'Mistral v0.3'
+      | 'Qwen 2'
+      | 'Custom'
+      | DataLink
+    /* Custom chat format template */
+    customChatFormat?: string | DataLink
+    /* Put tools in user message */
+    toolsInUserMessage?: boolean | DataLink
+    /* Prompt to generate */
+    prompt?: string | DataLink
+    /* Chat messages */
+    messages?:
+      | Array<
+          | DataLink
+          | {
+              role?: string | DataLink
+              content?: string | DataLink
+            }
+        >
+      | DataLink
+    /* Stop words */
+    stopWords?: Array<string | DataLink> | DataLink
+    /* Tool call parser */
+    toolCallParser?: 'llama3_json' | 'mistral' | 'hermes' | 'internlm' | 'phi4' | DataLink
+    /* Tools for chat mode using OpenAI-compatible function calling format  
+ Format: Array of objects with {type, function: {name, description, parameters}} structure  
+ See: https://platform.openai.com/docs/guides/function-calling */
+    tools?: Array<{} | DataLink> | DataLink
+    /* Tool choice for chat mode */
+    toolChoice?: 'none' | 'auto' | 'required' | DataLink
+    /* Enable parallel tool calls */
+    parallelToolCalls?: boolean | DataLink
+    /* Number of threads, -1 to use n-threads from model config */
+    nThreads?: number | DataLink
+    /* Temperature, -1 to use temperature from model config */
+    temperature?: number | DataLink
+    /* Seed, -1 to use seed from model config */
+    seed?: number | DataLink
+    /* Top K, -1 to use top-k from model config */
+    topK?: number | DataLink
+    /* Top P, -1 to use top-p from model config */
+    topP?: number | DataLink
+    /* Greedy, use greedy sampling */
+    greedy?: boolean | DataLink
+  }
+  events?: {
+    /* Event triggered when context state changes */
+    onContextStateChange?: Array<EventAction>
+    /* Event triggered when generate is done */
+    onGenerate?: Array<EventAction>
+    /* Event triggered on get function call request */
+    onFunctionCall?: Array<EventAction>
+    /* Event triggered when error occurs */
+    onError?: Array<EventAction>
+  }
+  outlets?: {
+    /* Context state */
+    contextState?: () => Data
+    /* Generation result */
+    result?: () => Data
+    /* Full context (Prompt + Generation Result) */
+    fullContext?: () => Data
+    /* Last function call details */
+    lastFunctionCall?: () => Data
+    /* Completion details */
+    completionDetails?: () => Data
+  }
+}
+
+/* Local LLM inference using Qualcomm AI Engine */
+export type GeneratorQnnLlm = Generator &
+  GeneratorQnnLlmDef & {
+    templateKey: 'GENERATOR_QNN_LLM'
+    switches: Array<
+      SwitchDef &
+        GeneratorQnnLlmDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'contextState'
+                    | 'result'
+                    | 'fullContext'
+                    | 'lastFunctionCall'
+                    | 'completionDetails'
+                  value: any
+                }
+          }>
+        }
+    >
+  }