@fugood/bricks-project 2.21.0-beta.21 → 2.21.0-beta.22

package/compile/action-name-map.ts

@@ -441,11 +441,18 @@ export const templateActionNameMap = {
   },
   GENERATOR_SPEECH_INFERENCE: {
     GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_FILE: {
+      fileUrl: 'GENERATOR_SPEECH_INFERENCE_FILE_URL',
+      prompt: 'GENERATOR_SPEECH_INFERENCE_PROMPT',
+      beamSize: 'GENERATOR_SPEECH_INFERENCE_BEAM_SIZE',
+      language: 'GENERATOR_SPEECH_INFERENCE_LANGUAGE',
+      translate: 'GENERATOR_SPEECH_INFERENCE_TRANSLATE',
+    },
+    GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_DATA: {
+      data: 'GENERATOR_SPEECH_INFERENCE_DATA',
       prompt: 'GENERATOR_SPEECH_INFERENCE_PROMPT',
       beamSize: 'GENERATOR_SPEECH_INFERENCE_BEAM_SIZE',
       language: 'GENERATOR_SPEECH_INFERENCE_LANGUAGE',
       translate: 'GENERATOR_SPEECH_INFERENCE_TRANSLATE',
-      fileUrl: 'GENERATOR_SPEECH_INFERENCE_FILE_URL',
     },
     GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_REALTIME: {
       prompt: 'GENERATOR_SPEECH_INFERENCE_PROMPT',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.21.0-beta.21",
+  "version": "2.21.0-beta.22",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"
@@ -13,5 +13,5 @@
     "lodash": "^4.17.4",
     "uuid": "^8.3.1"
   },
-  "gitHead": "129ece680c77c677a3b643a1102d056533e5792c"
+  "gitHead": "3e8cf8167c1692f5e981c122f144ef4680cac28d"
 }

package/types/data.ts

@@ -72,7 +72,7 @@ export type DataAssetKind = {
     | 'binary-asset'
   preload?: {
     type: 'url'
-    hashType: 'md5' | 'sha256'
+    hashType: 'md5' | 'sha256' | 'sha1'
     hash: string
   }
   metadata?: { [key: string]: any }

package/types/generators.ts

@@ -438,7 +438,7 @@ Default property:
     /* Hash value */
     hash?: string | DataLink
     /* Hash type */
-    hashType?: 'md5' | 'sha256' | DataLink
+    hashType?: 'md5' | 'sha256' | 'sha1' | DataLink
     /* Request method */
     method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | DataLink
     /* Header */
@@ -455,7 +455,7 @@ Default property:
               encoding?: 'utf8' | 'base64' | DataLink
               url?: string | DataLink
               hash?: string | DataLink
-              hashType?: 'sha256' | 'md5' | DataLink
+              hashType?: 'sha256' | 'sha1' | 'md5' | DataLink
               method?: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | DataLink
               header?: {} | DataLink
               fields?: {} | DataLink
@@ -4260,6 +4260,7 @@ interface GeneratorQuestionDef {
   /*
 Default property:
 {
+  "modalMode": "root",
   "showClose": true,
   "showBack": true,
   "cameraType": "back",
@@ -4271,6 +4272,8 @@ Default property:
 }
   */
   property?: {
+    /* Modal mode */
+    modalMode?: 'root' | 'in-subspace' | DataLink
     /* Inquirer schema  
  `key`: Field key (unique, required)  
  `kind`: Field type (required)  
@@ -5711,6 +5714,11 @@ export type GeneratorSpeechInferenceActionLoadModel = Action & {
 export type GeneratorSpeechInferenceActionTranscribeFile = ActionWithParams & {
   __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_FILE'
   params?: Array<
+    | {
+        input: 'fileUrl'
+        value?: string | DataLink
+        mapping?: string
+      }
     | {
         input: 'prompt'
         value?: string | DataLink
@@ -5731,11 +5739,38 @@ export type GeneratorSpeechInferenceActionTranscribeFile = ActionWithParams & {
         value?: boolean | DataLink
         mapping?: string
       }
+  >
+}
+
+/* Transcribe audio data. Currently only support base64 encoded audio data (16-bit PCM, mono, 16kHz) */
+export type GeneratorSpeechInferenceActionTranscribeData = ActionWithParams & {
+  __actionName: 'GENERATOR_SPEECH_INFERENCE_TRANSCRIBE_DATA'
+  params?: Array<
     | {
-        input: 'fileUrl'
+        input: 'data'
+        value?: any
+        mapping?: string
+      }
+    | {
+        input: 'prompt'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'beamSize'
+        value?: number | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'language'
         value?: string | DataLink
         mapping?: string
       }
+    | {
+        input: 'translate'
+        value?: boolean | DataLink
+        mapping?: string
+      }
   >
 }
 
@@ -5826,10 +5861,10 @@ interface GeneratorSpeechInferenceDef {
 Default property:
 {
   "init": false,
-  "modelType": "Base",
-  "modelQuantizedType": "none",
+  "modelName": "base-q8_0",
   "modelUseCoreML": false,
   "modelUseGPU": true,
+  "modelUseFlashAttn": false,
   "inferLanguage": "Auto",
   "inferRealtimeAudioSeconds": 30,
   "inferRealtimeAudioSliceSeconds": 30,
@@ -5844,35 +5879,71 @@ Default property:
     /* Initialize the Whisper context on generator initialization  
  Please note that it will take some RAM depending on the model size */
     init?: boolean | DataLink
-    /* Use model type,  the model download progress will be done in preload stage or  the generator initialization stage.  
- We used `ggml` format model, please refer to https://github.com/ggerganov/whisper.cpp/tree/master/models  
- You can also choose `Custom` option and set `Model URL` and `Model MD5` to use your own model */
-    modelType?:
-      | 'Custom'
-      | 'Tiny'
-      | 'Base'
-      | 'Small'
-      | 'Medium'
-      | 'Large'
-      | 'Distil Large v2'
-      | 'Tiny (English only)'
-      | 'Base (English only)'
-      | 'Small (English only)'
-      | 'Medium (English only)'
-      | 'Distil Small (English only)'
-      | 'Distil Medium (English only)'
+    /* Use model name,  the model download progress will be done in preload stage or  the generator initialization stage.  
+ We used `ggml` format model, please refer to https://huggingface.co/BricksDisplay/whisper-ggml  
+ You can also choose `custom` option and set `Model URL` and `Model MD5` to use your own model */
+    modelName?:
+      | 'custom'
+      | 'tiny'
+      | 'tiny-q5_1'
+      | 'tiny-q8_0'
+      | 'tiny.en'
+      | 'tiny.en-q5_1'
+      | 'tiny.en-q8_0'
+      | 'base'
+      | 'base-q5_1'
+      | 'base-q8_0'
+      | 'base.en'
+      | 'base.en-q5_1'
+      | 'base.en-q8_0'
+      | 'small'
+      | 'small-q5_0'
+      | 'small-q5_1'
+      | 'small-q8_0'
+      | 'small.en'
+      | 'small.en-q5_1'
+      | 'small.en-q8_0'
+      | 'medium'
+      | 'medium-q5_0'
+      | 'medium-q5_1'
+      | 'medium-q8_0'
+      | 'medium.en'
+      | 'medium.en-q5_1'
+      | 'medium.en-q8_0'
+      | 'large-v1'
+      | 'large-v2'
+      | 'large-v2-q5_0'
+      | 'large-v2-q8_0'
+      | 'large-v3'
+      | 'large-v3-q5_0'
+      | 'large-v3-q8_0'
+      | 'large-v3-turbo'
+      | 'large-v3-turbo-q5_0'
+      | 'large-v3-turbo-q8_0'
+      | 'small.en-tdrz'
+      | 'small.en-tdrz-q5_0'
+      | 'small.en-tdrz-q8_0'
+      | 'distil-small.en'
+      | 'distil-small.en-q5_0'
+      | 'distil-small.en-q8_0'
+      | 'distil-medium.en'
+      | 'distil-medium.en-q5_0'
+      | 'distil-medium.en-q8_0'
+      | 'distil-large-v3'
+      | 'distil-large-v3-q5_0'
+      | 'distil-large-v3-q8_0'
       | DataLink
-    /* Use quantized model */
-    modelQuantizedType?: 'none' | 'q4_0' | 'q4_1' | 'q5_0' | 'q5_1' | 'q8_0' | DataLink
     /* The URL or path of model  
  We used `ggml` format model, please refer to https://github.com/ggerganov/whisper.cpp/tree/master/models */
     modelUrl?: string | DataLink
-    /* MD5 of model JSON */
-    modelMd5?: string | DataLink
+    /* SHA1 of model */
+    modelSha1?: string | DataLink
     /* [Unstable] iOS: Use CoreML model for inference */
     modelUseCoreML?: boolean | DataLink
     /* Use GPU Acceleration for inference. Currently iOS only, if it's enabled, Core ML option will be ignored. */
     modelUseGPU?: boolean | DataLink
+    /* Use Flash Attention for inference (Recommended with GPU enabled) */
+    modelUseFlashAttn?: boolean | DataLink
     /* The language of the file to be inferred */
     inferLanguage?:
       | 'Auto'
@@ -5986,6 +6057,8 @@ Default property:
     inferMaxThreads?: number | DataLink
     /* Output token-level timestamps in details outlet */
     inferTokenTimestamps?: boolean | DataLink
+    /* Speaker diarization (Please use small.en-tdrz model) */
+    inferTdrz?: boolean | DataLink
     /* Maximum segment length in characters */
     inferMaxLength?: number | DataLink
     /* Audio time offset in milliseconds */
@@ -6076,7 +6149,7 @@ Default property:
     /* Is transcribing */
     isTranscribing?: () => Data
     /* Progress of transcribe audio (0-100) */
-    transcribeFileProgress?: () => Data
+    transcribeProgress?: () => Data
     /* Inference result */
     transcribeResult?: () => Data
     /* Inference result details */
@@ -6104,7 +6177,7 @@ export type GeneratorSpeechInference = Generator &
                     | 'contextState'
                     | 'contextDetails'
                     | 'isTranscribing'
-                    | 'transcribeFileProgress'
+                    | 'transcribeProgress'
                     | 'transcribeResult'
                     | 'transcribeDetails'
                     | 'recordedPath'
@@ -6451,12 +6524,14 @@ Default property:
     completionPromptTemplateData?: {} | DataLink
     /* The prompt template type */
     completionPromptTemplateType?: '${}' | '{{}}' | DataLink
-    /* Grammar */
-    completionGrammar?: string | DataLink
     /* Stop words */
     completionStopWords?: Array<string | DataLink> | DataLink
     /* Number of tokens to predict */
     completionPredict?: number | DataLink
+    /* Throttle time for completion result (in milliseconds) */
+    completionResultThrottle?: number | DataLink
+    /* Grammar (GBNF: Please refer to https://github.com/ggerganov/llama.cpp/tree/master/grammars) */
+    completionGrammar?: string | DataLink
     /* Temperature */
     completionTemperature?: number | DataLink
     /* Number of probablites to show for each token in the completion details */
@@ -6493,8 +6568,6 @@ Default property:
     completionTypicalP?: number | DataLink
     /* Repeat alpha frequency penalty (default: 0.1, 0.0 = disabled) */
     completionIgnoreEOS?: boolean | DataLink
-    /* Throttle time for completion result (in milliseconds) */
-    completionResultThrottle?: number | DataLink
   }
   events?: {
     /* Event triggered when load is done */