@fugood/bricks-project 2.21.0-beta.21 → 2.21.0-beta.23

package/compile/action-name-map.ts

@@ -441,11 +441,18 @@ export const templateActionNameMap = {
   },
   GENERATOR_SPEECH_INFERENCE: {
     GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_FILE: {
+      fileUrl: 'GENERATOR_SPEECH_INFERENCE_FILE_URL',
+      prompt: 'GENERATOR_SPEECH_INFERENCE_PROMPT',
+      beamSize: 'GENERATOR_SPEECH_INFERENCE_BEAM_SIZE',
+      language: 'GENERATOR_SPEECH_INFERENCE_LANGUAGE',
+      translate: 'GENERATOR_SPEECH_INFERENCE_TRANSLATE',
+    },
+    GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_DATA: {
+      data: 'GENERATOR_SPEECH_INFERENCE_DATA',
       prompt: 'GENERATOR_SPEECH_INFERENCE_PROMPT',
       beamSize: 'GENERATOR_SPEECH_INFERENCE_BEAM_SIZE',
       language: 'GENERATOR_SPEECH_INFERENCE_LANGUAGE',
       translate: 'GENERATOR_SPEECH_INFERENCE_TRANSLATE',
-      fileUrl: 'GENERATOR_SPEECH_INFERENCE_FILE_URL',
     },
     GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_REALTIME: {
       prompt: 'GENERATOR_SPEECH_INFERENCE_PROMPT',
@@ -464,6 +471,7 @@ export const templateActionNameMap = {
   },
   GENERATOR_LLM: {
     GENERATOR_LLM_PROCESS_PROMPT: {
+      sessionKey: 'GENERATOR_LLM_SESSION_KEY',
       mode: 'GENERATOR_LLM_MODE',
       messages: 'GENERATOR_LLM_MESSAGES',
       prompt: 'GENERATOR_LLM_PROMPT',
@@ -471,6 +479,7 @@ export const templateActionNameMap = {
       promptTemplateType: 'GENERATOR_LLM_PROMPT_TEMPLATE_TYPE',
     },
     GENERATOR_LLM_COMPLETION: {
+      sessionKey: 'GENERATOR_LLM_SESSION_KEY',
       mode: 'GENERATOR_LLM_MODE',
       messages: 'GENERATOR_LLM_MESSAGES',
       prompt: 'GENERATOR_LLM_PROMPT',
@@ -499,5 +508,9 @@ export const templateActionNameMap = {
       functionCallEnabled: 'GENERATOR_LLM_FUNCTION_CALL_ENABLED',
       functionCallSchema: 'GENERATOR_LLM_FUNCTION_CALL_SCHEMA',
     },
+    GENERATOR_LLM_CLEAR_SESSION: {
+      sessionId: 'GENERATOR_LLM_SESSION_ID',
+      sessionCustomKey: 'GENERATOR_LLM_SESSION_CUSTOM_KEY',
+    },
   },
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.21.0-beta.21",
+  "version": "2.21.0-beta.23",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"
@@ -13,5 +13,5 @@
     "lodash": "^4.17.4",
     "uuid": "^8.3.1"
   },
-  "gitHead": "129ece680c77c677a3b643a1102d056533e5792c"
+  "gitHead": "d7d83d4425e118960bcbde76a7e66a2c2f4e7bdb"
 }

package/types/data.ts

@@ -72,7 +72,7 @@ export type DataAssetKind = {
     | 'binary-asset'
   preload?: {
     type: 'url'
-    hashType: 'md5' | 'sha256'
+    hashType: 'md5' | 'sha256' | 'sha1'
     hash: string
   }
   metadata?: { [key: string]: any }

package/types/generators.ts

@@ -438,7 +438,7 @@ Default property:
     /* Hash value */
     hash?: string | DataLink
     /* Hash type */
-    hashType?: 'md5' | 'sha256' | DataLink
+    hashType?: 'md5' | 'sha256' | 'sha1' | DataLink
     /* Request method */
     method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | DataLink
     /* Header */
@@ -455,7 +455,7 @@ Default property:
               encoding?: 'utf8' | 'base64' | DataLink
               url?: string | DataLink
               hash?: string | DataLink
-              hashType?: 'sha256' | 'md5' | DataLink
+              hashType?: 'sha256' | 'sha1' | 'md5' | DataLink
               method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | DataLink
               header?: {} | DataLink
               fields?: {} | DataLink
@@ -4260,6 +4260,7 @@ interface GeneratorQuestionDef {
   /*
 Default property:
 {
+  "modalMode": "root",
   "showClose": true,
   "showBack": true,
   "cameraType": "back",
@@ -4271,6 +4272,8 @@ Default property:
 }
   */
   property?: {
+    /* Modal mode */
+    modalMode?: 'root' | 'in-subspace' | DataLink
     /* Inquirer schema  
  `key`: Field key (unique, required)  
  `kind`: Field type (required)  
@@ -5711,6 +5714,11 @@ export type GeneratorSpeechInferenceActionLoadModel = Action & {
 export type GeneratorSpeechInferenceActionTranscribeFile = ActionWithParams & {
   __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_FILE'
   params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink
+        mapping?: string
+      }
     | {
         input: 'prompt'
         value?: string | DataLink
@@ -5731,11 +5739,38 @@ export type GeneratorSpeechInferenceActionTranscribeFile = ActionWithParams & {
         value?: boolean | DataLink
         mapping?: string
       }
+  >
+}
+
+/* Transcribe audio data. Currently only support base64 encoded audio data (16-bit PCM, mono, 16kHz) */
+export type GeneratorSpeechInferenceActionTranscribeData = ActionWithParams & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_DATA'
+  params?: Array<
     | {
-        input: 'fileUrl'
+        input: 'data'
+        value?: any
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
         value?: string | DataLink
         mapping?: string
       }
+    | {
+        input: 'beamSize'
+        value?: number | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'language'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'translate'
+        value?: boolean | DataLink
+        mapping?: string
+      }
   >
 }
 
@@ -5826,10 +5861,10 @@ interface GeneratorSpeechInferenceDef {
 Default property:
 {
   "init": false,
-  "modelType": "Base",
-  "modelQuantizedType": "none",
+  "modelName": "base-q8_0",
   "modelUseCoreML": false,
   "modelUseGPU": true,
+  "modelUseFlashAttn": false,
   "inferLanguage": "Auto",
   "inferRealtimeAudioSeconds": 30,
   "inferRealtimeAudioSliceSeconds": 30,
@@ -5844,35 +5879,71 @@ Default property:
     /* Initialize the Whisper context on generator initialization  
  Please note that it will take some RAM depending on the model size */
     init?: boolean | DataLink
-    /* Use model type,  the model download progress will be done in preload stage or  the generator initialization stage.  
- We used `ggml` format model, please refer to https://github.com/ggerganov/whisper.cpp/tree/master/models  
- You can also choose `Custom` option and set `Model URL` and `Model MD5` to use your own model */
-    modelType?:
-      | 'Custom'
-      | 'Tiny'
-      | 'Base'
-      | 'Small'
-      | 'Medium'
-      | 'Large'
-      | 'Distil Large v2'
-      | 'Tiny (English only)'
-      | 'Base (English only)'
-      | 'Small (English only)'
-      | 'Medium (English only)'
-      | 'Distil Small (English only)'
-      | 'Distil Medium (English only)'
+    /* Use model name,  the model download progress will be done in preload stage or  the generator initialization stage.  
+ We used `ggml` format model, please refer to https://huggingface.co/BricksDisplay/whisper-ggml  
+ You can also choose `custom` option and set `Model URL` and `Model MD5` to use your own model */
+    modelName?:
+      | 'custom'
+      | 'tiny'
+      | 'tiny-q5_1'
+      | 'tiny-q8_0'
+      | 'tiny.en'
+      | 'tiny.en-q5_1'
+      | 'tiny.en-q8_0'
+      | 'base'
+      | 'base-q5_1'
+      | 'base-q8_0'
+      | 'base.en'
+      | 'base.en-q5_1'
+      | 'base.en-q8_0'
+      | 'small'
+      | 'small-q5_1'
+      | 'small-q8_0'
+      | 'small.en'
+      | 'small.en-q5_1'
+      | 'small.en-q8_0'
+      | 'medium'
+      | 'medium-q5_0'
+      | 'medium-q8_0'
+      | 'medium.en'
+      | 'medium.en-q5_1'
+      | 'medium.en-q8_0'
+      | 'large-v1'
+      | 'large-v2'
+      | 'large-v2-q5_0'
+      | 'large-v2-q8_0'
+      | 'large-v3'
+      | 'large-v3-q5_0'
+      | 'large-v3-q8_0'
+      | 'large-v3-turbo'
+      | 'large-v3-turbo-q5_0'
+      | 'large-v3-turbo-q8_0'
+      | 'small.en-tdrz'
+      | 'small.en-tdrz-q5_0'
+      | 'small.en-tdrz-q8_0'
+      | 'distil-small.en'
+      | 'distil-small.en-q5_0'
+      | 'distil-small.en-q8_0'
+      | 'distil-medium.en'
+      | 'distil-medium.en-q5_0'
+      | 'distil-medium.en-q8_0'
+      | 'distil-large-v3'
+      | 'distil-large-v3-q5_0'
+      | 'distil-large-v3-q8_0'
       | DataLink
-    /* Use quantized model */
-    modelQuantizedType?: 'none' | 'q4_0' | 'q4_1' | 'q5_0' | 'q5_1' | 'q8_0' | DataLink
     /* The URL or path of model  
  We used `ggml` format model, please refer to https://github.com/ggerganov/whisper.cpp/tree/master/models */
     modelUrl?: string | DataLink
-    /* MD5 of model JSON */
-    modelMd5?: string | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
     /* [Unstable] iOS: Use CoreML model for inference */
     modelUseCoreML?: boolean | DataLink
     /* Use GPU Acceleration for inference. Currently iOS only, if it's enabled, Core ML option will be ignored. */
     modelUseGPU?: boolean | DataLink
+    /* Use Flash Attention for inference (Recommended with GPU enabled) */
+    modelUseFlashAttn?: boolean | DataLink
     /* The language of the file to be inferred */
     inferLanguage?:
       | 'Auto'
@@ -5986,6 +6057,8 @@ Default property:
     inferMaxThreads?: number | DataLink
     /* Output token-level timestamps in details outlet */
     inferTokenTimestamps?: boolean | DataLink
+    /* Speaker diarization (Please use small.en-tdrz model) */
+    inferTdrz?: boolean | DataLink
     /* Maximum segment length in characters */
     inferMaxLength?: number | DataLink
     /* Audio time offset in milliseconds */
@@ -6076,7 +6149,7 @@ Default property:
     /* Is transcribing */
     isTranscribing?: () => Data
     /* Progress of transcribe audio (0-100) */
-    transcribeFileProgress?: () => Data
+    transcribeProgress?: () => Data
     /* Inference result */
     transcribeResult?: () => Data
     /* Inference result details */
@@ -6104,7 +6177,7 @@ export type GeneratorSpeechInference = Generator &
                     | 'contextState'
                     | 'contextDetails'
                     | 'isTranscribing'
-                    | 'transcribeFileProgress'
+                    | 'transcribeProgress'
                     | 'transcribeResult'
                     | 'transcribeDetails'
                     | 'recordedPath'
@@ -6124,6 +6197,11 @@ export type GeneratorLLMActionLoadModel = Action & {
 export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
   __actionName: 'GENERATOR_LLM_PROCESS_PROMPT'
   params?: Array<
+    | {
+        input: 'sessionKey'
+        value?: string | DataLink
+        mapping?: string
+      }
     | {
         input: 'mode'
         value?: string | DataLink
@@ -6156,6 +6234,11 @@ export type GeneratorLLMActionProcessPrompt = ActionWithParams & {
 export type GeneratorLLMActionCompletion = ActionWithParams & {
   __actionName: 'GENERATOR_LLM_COMPLETION'
   params?: Array<
+    | {
+        input: 'sessionKey'
+        value?: string | DataLink
+        mapping?: string
+      }
     | {
         input: 'mode'
         value?: string | DataLink
@@ -6294,6 +6377,23 @@ export type GeneratorLLMActionCompletion = ActionWithParams & {
   >
 }
 
+/* Clear session with session key or session ID */
+export type GeneratorLLMActionClearSession = ActionWithParams & {
+  __actionName: 'GENERATOR_LLM_CLEAR_SESSION'
+  params?: Array<
+    | {
+        input: 'sessionId'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'sessionCustomKey'
+        value?: string | DataLink
+        mapping?: string
+      }
+  >
+}
+
 /* Stop text completion */
 export type GeneratorLLMActionStopCompletion = Action & {
   __actionName: 'GENERATOR_LLM_STOP_COMPLETION'
@@ -6314,8 +6414,6 @@ interface GeneratorLLMDef {
 Default property:
 {
   "init": false,
-  "modelType": "Custom",
-  "modelQuantizedType": "q2_k",
   "contextSize": 512,
   "batchSize": 512,
   "accelVariant": "default",
@@ -6363,38 +6461,13 @@ Default property:
     /* Initialize the Llama context on generator initialization  
  Please note that it will take some RAM depending on the model size */
     init?: boolean | DataLink
-    /* Use model type,  the model download progress will be done in preload stage or  the generator initialization stage.  
- We used GGUF format model, please refer to https://github.com/ggerganov/llama.cpp/tree/master#description  
- You can also choose `Custom` option and set `Model URL` and `Model MD5` to use your own model.  
- Please also read model license for fair use. */
-    modelType?:
-      | 'Custom'
-      | '[Local] Llama 3 8B'
-      | '[Local] Llama 3 8B Instruct'
-      | '[Local] Llama 2 7B Chat'
-      | '[Local] Llama 2 7B'
-      | '[Local] Mistral 7B Instruct v0.2'
-      | '[Local] Mistral 7B v0.1'
-      | '[Local] Breeze 7B Instruct 64k v0.1'
-      | '[Local] Falcon 7B'
-      | '[Local] Gemma 2B Instruct'
-      | '[Local] Gemma 2B'
-      | '[Local] Phi-2 3B'
-      | '[Local] OpenLLaMA 2 3B'
-      | '[Local] TinyLlama 1.1B intermediate-step-1195k-token-2.5T'
-      | DataLink
-    /* Select quantized model, currently only provided `q2_k` / `q3_k` / `q4_0` (Default to `q2_k`, will use `q4_0` instead if K-Quants not supported) */
-    modelQuantizedType?: 'q2_k' | 'q3_k' | 'q4_0' | DataLink
     /* The URL or path of model  
  We used GGUF format model, please refer to https://github.com/ggerganov/llama.cpp/tree/master#description */
     modelUrl?: string | DataLink
-    /* Hash of model JSON */
-    modelMd5?: string | DataLink
-    /* Hash of model JSON */
-    modelHashType?: 'md5' | 'sha256' | DataLink
-    /* Pre-process the prompt after model loaded, this can speed up the completion action.  
- This doing the same thing as PROCESS_PROMPT action. */
-    modelPreprocessPrompt?: boolean | DataLink
+    /* Hash type of model */
+    modelHashType?: 'md5' | 'sha256' | 'sha1' | DataLink
+    /* Hash of model */
+    modelHash?: string | DataLink
     /* Context size (0 ~ 4096) (Default to 512) */
     contextSize?: number | DataLink
     /* Batch size */
@@ -6451,12 +6524,14 @@ Default property:
     completionPromptTemplateData?: {} | DataLink
     /* The prompt template type */
     completionPromptTemplateType?: '${}' | '{{}}' | DataLink
-    /* Grammar */
-    completionGrammar?: string | DataLink
     /* Stop words */
     completionStopWords?: Array<string | DataLink> | DataLink
     /* Number of tokens to predict */
     completionPredict?: number | DataLink
+    /* Throttle time for completion result (in milliseconds) */
+    completionResultThrottle?: number | DataLink
+    /* Grammar (GBNF: Please refer to https://github.com/ggerganov/llama.cpp/tree/master/grammars) */
+    completionGrammar?: string | DataLink
     /* Temperature */
     completionTemperature?: number | DataLink
     /* Number of probablites to show for each token in the completion details */
@@ -6493,8 +6568,6 @@ Default property:
     completionTypicalP?: number | DataLink
     /* Repeat alpha frequency penalty (default: 0.1, 0.0 = disabled) */
     completionIgnoreEOS?: boolean | DataLink
-    /* Throttle time for completion result (in milliseconds) */
-    completionResultThrottle?: number | DataLink
   }
   events?: {
     /* Event triggered when load is done */