@fugood/bricks-project 2.21.0-beta.27 → 2.21.0-beta.31

package/compile/action-name-map.ts

@@ -47,6 +47,7 @@ export const templateActionNameMap = {
       selections: 'ALERT_SELECTIONS',
       selectionResult: 'ALERT_SELECTION_RESULT',
       timeout: 'ALERT_TIMEOUT',
+      hideSelections: 'ALERT_HIDE_SELECTIONS',
     },
     POPUP_RESET: {
       popupClearType: 'POPUP_CLEAR_TYPE',
@@ -54,6 +55,7 @@ export const templateActionNameMap = {
     },
     TAKE_SCREENSHOT: {
       currentSubspace: 'TAKE_SCREENSHOT_CURRENT_SUBSPACE',
+      brickId: 'TAKE_SCREENSHOT_BRICK_ID',
       format: 'TAKE_SCREENSHOT_FORMAT',
       quality: 'TAKE_SCREENSHOT_QUALITY',
       width: 'TAKE_SCREENSHOT_WIDTH',
@@ -65,7 +67,9 @@ export const templateActionNameMap = {
       type: 'CHANNEL_SUBSCRIBE_TYPE',
       livetime: 'CHANNEL_SUBSCRIBE_LIVETIME',
       dataResult: 'CHANNEL_SUBSCRIBE_DATA_RESULT',
+      dataAssignList: 'CHANNEL_SUBSCRIBE_DATA_ASSIGN_LIST',
       progressResult: 'CHANNEL_SUBSCRIBE_PROGRESS_RESULT',
+      progressAssignList: 'CHANNEL_SUBSCRIBE_PROGRESS_ASSIGN_LIST',
       errorResult: 'CHANNEL_SUBSCRIBE_ERROR_RESULT',
     },
     CHANNEL_UNSUBSCRIBE: {
@@ -557,4 +561,31 @@ export const templateActionNameMap = {
       sessionCustomKey: 'GENERATOR_LLM_SESSION_CUSTOM_KEY',
     },
   },
+  GENERATOR_ASSISTANT: {
+    GENERATOR_ASSISTANT_ADD_MESSAGE: {
+      role: 'GENERATOR_ASSISTANT_ROLE',
+      content: 'GENERATOR_ASSISTANT_CONTENT',
+      payload: 'GENERATOR_ASSISTANT_PAYLOAD',
+    },
+    GENERATOR_ASSISTANT_UPDATE_MESSAGE_AT_INDEX: {
+      index: 'GENERATOR_ASSISTANT_INDEX',
+      content: 'GENERATOR_ASSISTANT_CONTENT',
+      payload: 'GENERATOR_ASSISTANT_PAYLOAD',
+    },
+    GENERATOR_ASSISTANT_ADD_AUDIO_MESSAGE: {
+      role: 'GENERATOR_ASSISTANT_ROLE',
+      contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
+      contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      payload: 'GENERATOR_ASSISTANT_PAYLOAD',
+    },
+    GENERATOR_ASSISTANT_UPDATE_AUDIO_MESSAGE_AT_INDEX: {
+      index: 'GENERATOR_ASSISTANT_INDEX',
+      contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
+      contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      payload: 'GENERATOR_ASSISTANT_PAYLOAD',
+    },
+    GENERATOR_ASSISTANT_REMOVE_MESSAGE_AT_INDEX: {
+      index: 'GENERATOR_ASSISTANT_INDEX',
+    },
+  },
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.21.0-beta.27",
+  "version": "2.21.0-beta.31",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"
@@ -13,5 +13,5 @@
     "lodash": "^4.17.4",
     "uuid": "^8.3.1"
   },
-  "gitHead": "85b6bb3416b693de316511738097112f89e764e7"
+  "gitHead": "366ad791d1217e2a5b7dc9ab7ba65196023f2357"
 }

package/types/bricks.ts

@@ -3153,6 +3153,17 @@ Default property:
         }
       }
     >
+    /* Event of the Camera mount error */
+    mountError?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          mountError: {
+            type: 'string'
+            path: string
+          }
+        }
+      }
+    >
   }
   outlets?: {
     /* Picture taken result */
@@ -3171,6 +3182,7 @@ Default property:
     barcodeRead?: Animation
     pictureTaken?: Animation
     recordFinish?: Animation
+    mountError?: Animation
   }
 }
 

package/types/generators.ts

@@ -3547,6 +3547,10 @@ Default property:
   outlets?: {
     /* Signal to create peer connection */
     signal?: () => Data
+    /* Local SDP offer */
+    sdpOffer?: () => Data
+    /* Local SDP answer */
+    sdpAnswer?: () => Data
     /* Peer connection state */
     connectionState?: () => Data
     /* ICE connection state */
@@ -3576,6 +3580,8 @@ export type GeneratorWebRTC = Generator &
                   __typename: 'SwitchCondInnerStateOutlet'
                   outlet:
                     | 'signal'
+                    | 'sdpOffer'
+                    | 'sdpAnswer'
                     | 'connectionState'
                     | 'iceConnectionState'
                     | 'signalingState'
@@ -3588,121 +3594,6 @@ export type GeneratorWebRTC = Generator &
     >
   }
 
-interface GeneratorTensorFlowInferenceDef {
-  /*
-Default property:
-{
-  "inputType": "camera",
-  "minScore": 0.5,
-  "objectLimit": 20,
-  "imageChannelFormat": "rgb",
-  "imageNormalize": "off",
-  "cameraInit": false,
-  "cameraFps": 10,
-  "cameraRotate": 0
-}
-  */
-  property?: {
-    /* URL to download TF.js graph model  
-  You can find pre-trained model on [TensorFlow Hub](https://tfhub.dev) */
-    modelUrl?: string | DataLink
-    /* MD5 of model JSON */
-    modelMd5?: string | DataLink
-    /* Model input source */
-    inputType?: 'raw' | 'media-image' | 'camera' | DataLink
-    /* Model output type  
-  label: Classification problem (model output number or score array)  
-  object-boxes: Object detection (model output [labels, boxes])  
-  raw: No transform */
-    outputType?: 'label' | 'object-boxes' | 'raw' | DataLink
-    /* Model output labels */
-    labels?: Array<string | DataLink> | DataLink
-    /* Download labels from URL */
-    labelsUrl?: string | DataLink
-    /* MD5 checksum of label file */
-    labelsMd5?: string | DataLink
-    /* Output score limit */
-    minScore?: number | DataLink
-    /* Object limit for object detection */
-    objectLimit?: number | DataLink
-    /* Input channel format */
-    imageChannelFormat?: 'rgb' | 'grayscale' | DataLink
-    /* Input data format of the model is channel first (Commonly is channel last) */
-    imageChannelFirst?: boolean | DataLink
-    /* Resize to input layer height */
-    imageResizeHeight?: number | DataLink
-    /* Resize to input layer width */
-    imageResizeWidth?: number | DataLink
-    /* Input data normalize */
-    imageNormalize?: 'off' | '0~1' | '-1~1' | DataLink
-    /* Start infer on camera when initialized */
-    cameraInit?: boolean | DataLink
-    /* Target Camera brick ID */
-    cameraTarget?: string | DataLink
-    /* Infer fps, 0 is unlimited */
-    cameraFps?: number | DataLink
-    /* Camera rotate correction */
-    cameraRotate?: number | DataLink
-    /* Show detect object bounding box on Camera brick */
-    cameraShowObjectBox?: boolean | DataLink
-    /* Bonding box border size */
-    cameraBoxBorderSize?: number | DataLink
-    /* Bonding box border color */
-    cameraBoxBorderColor?: Array<string | DataLink> | DataLink
-    /* The URL of image to infer */
-    mediaImageUrl?: string | DataLink
-    /* MD5 checksum of image */
-    mediaImageMd5?: string | DataLink
-    /* The data input to infer */
-    rawInput?: any
-  }
-  events?: {
-    /* Event of TensorFlow inference error */
-    onError?: Array<
-      EventAction & {
-        eventPropertyMapping?: {
-          errorMessage: {
-            type: 'string'
-            path: string
-          }
-        }
-      }
-    >
-  }
-  outlets?: {
-    /* Model inference spend time in ms */
-    modelPerformance?: () => Data
-    /* Model state */
-    modelState?: () => Data
-    /* Transformed infer result */
-    result?: () => Data
-    /* Raw infer result */
-    rawResult?: () => Data
-  }
-}
-
-/* TensorFlow Inference */
-export type GeneratorTensorFlowInference = Generator &
-  GeneratorTensorFlowInferenceDef & {
-    templateKey: 'GENERATOR_TENSORFLOW_INFERENCE'
-    switches: Array<
-      SwitchDef &
-        GeneratorTensorFlowInferenceDef & {
-          conds?: Array<{
-            method: '==' | '!=' | '>' | '<' | '>=' | '<='
-            cond:
-              | SwitchCondInnerStateCurrentCanvas
-              | SwitchCondData
-              | {
-                  __typename: 'SwitchCondInnerStateOutlet'
-                  outlet: 'modelPerformance' | 'modelState' | 'result' | 'rawResult'
-                  value: any
-                }
-          }>
-        }
-    >
-  }
-
 /* Run Crawler request with defined properties */
 export type GeneratorWebCrawlerActionRunRequest = Action & {
   __actionName: 'GENERATOR_WEB_CRAWLER_RUN_REQUEST'
@@ -3853,6 +3744,17 @@ Default property:
         }
       }
     >
+    /* Event for recorded file */
+    recorded?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          recordedPath: {
+            type: 'string'
+            path: string
+          }
+        }
+      }
+    >
   }
   outlets?: {
     /* Is recording */
@@ -4105,6 +4007,7 @@ Default property:
   "cameraType": "back",
   "cameraPosition": "top",
   "nextButtonText": "NEXT",
+  "skipButtonText": "SKIP",
   "doneButtonText": "DONE",
   "selectColumns": 3,
   "sizeFactor": 2
@@ -4184,12 +4087,13 @@ Default property:
               selectOptions?:
                 | DataLink
                 | {
+                    withConfirmButton?: boolean | DataLink
                     options?:
                       | Array<
                           | DataLink
                           | {
                               value?: string | DataLink
-                              image?: string | DataLink
+                              image_uri?: string | DataLink
                               label?: string | DataLink
                             }
                         >
@@ -4203,7 +4107,7 @@ Default property:
                           | DataLink
                           | {
                               value?: string | DataLink
-                              image?: string | DataLink
+                              image_uri?: string | DataLink
                               label?: string | DataLink
                             }
                         >
@@ -4214,6 +4118,7 @@ Default property:
               yesnoOptions?:
                 | DataLink
                 | {
+                    withConfirmButton?: boolean | DataLink
                     yesBtnText?: string | DataLink
                     noBtnText?: string | DataLink
                   }
@@ -4250,6 +4155,8 @@ Default property:
     cameraPosition?: 'top' | 'left' | 'bottom' | 'right' | DataLink
     /* "Next" button text */
     nextButtonText?: string | DataLink
+    /* "Skip" button text */
+    skipButtonText?: string | DataLink
     /* "Done" button text */
     doneButtonText?: string | DataLink
     /* Select columns */
@@ -6491,6 +6398,10 @@ Default property:
     onCompletion?: Array<
       EventAction & {
         eventPropertyMapping?: {
+          completionSessionKey: {
+            type: 'string'
+            path: string
+          }
           completionResult: {
             type: 'string'
             path: string
@@ -6510,6 +6421,10 @@ Default property:
     onCompletionFinished?: Array<
       EventAction & {
         eventPropertyMapping?: {
+          completionSessionKey: {
+            type: 'string'
+            path: string
+          }
           completionResult: {
             type: 'string'
             path: string
@@ -6529,6 +6444,10 @@ Default property:
     onCompletionFunctionCall?: Array<
       EventAction & {
         eventPropertyMapping?: {
+          completionSessionKey: {
+            type: 'string'
+            path: string
+          }
           completionFunctionCallName: {
             type: 'string'
             path: string
@@ -6610,3 +6529,243 @@ export type GeneratorLLM = Generator &
         }
     >
   }
+
+/* Add a message to the assistant */
+export type GeneratorAssistantActionAddMessage = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_ADD_MESSAGE'
+  params?: Array<
+    | {
+        input: 'role'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'content'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink
+        mapping?: string
+      }
+  >
+}
+
+/* Update a message at a specific index */
+export type GeneratorAssistantActionUpdateMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_UPDATE_MESSAGE_AT_INDEX'
+  params?: Array<
+    | {
+        input: 'index'
+        value?: number | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'content'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink
+        mapping?: string
+      }
+  >
+}
+
+/* Add an audio message and convert it to text to the assistant (if STT enabled) */
+export type GeneratorAssistantActionAddAudioMessage = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_ADD_AUDIO_MESSAGE'
+  params?: Array<
+    | {
+        input: 'role'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'contentFile'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'contentBase64'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink
+        mapping?: string
+      }
+  >
+}
+
+/* Update a audio message at a specific index */
+export type GeneratorAssistantActionUpdateAudioMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_UPDATE_AUDIO_MESSAGE_AT_INDEX'
+  params?: Array<
+    | {
+        input: 'index'
+        value?: number | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'contentFile'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'contentBase64'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink
+        mapping?: string
+      }
+  >
+}
+
+/* Remove a message at a specific index */
+export type GeneratorAssistantActionRemoveMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_REMOVE_MESSAGE_AT_INDEX'
+  params?: Array<{
+    input: 'index'
+    value?: number | DataLink
+    mapping?: string
+  }>
+}
+
+/* Reset all messages of the assistant */
+export type GeneratorAssistantActionResetMessages = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_RESET_MESSAGES'
+}
+
+/* Submit the assistant */
+export type GeneratorAssistantActionSubmit = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_SUBMIT'
+}
+
+/* Cancel the assistant responding */
+export type GeneratorAssistantActionCancel = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_CANCEL'
+}
+
+interface GeneratorAssistantDef {
+  /*
+Default property:
+{
+  "initialMessages": [
+    null
+  ],
+  "cacheMessages": false,
+  "llmLivePolicy": "only-in-use",
+  "llmSessionKey": "default-assistant",
+  "sttEnabled": true,
+  "sttLivePolicy": "only-in-use",
+  "ttsEnabled": false,
+  "ttsLivePolicy": "only-in-use"
+}
+  */
+  property?: {
+    /* Whether to heat up generators on initialization for better performance */
+    heatup?: boolean | DataLink
+    /* Messages */
+    initialMessages?:
+      | Array<
+          | DataLink
+          | {
+              role?: string | DataLink
+              content?: string | DataLink
+            }
+        >
+      | DataLink
+    /* Whether to cache messages */
+    cacheMessages?: boolean | DataLink
+    /* LLM Generator (Currently only support `LLM (GGML)` generator) */
+    llmGeneratorId?: string | DataLink
+    /* LLM Live Policy. If the policy is `only-in-use`, the LLM context will be released when the assistant is not in use. */
+    llmLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* LLM main session key */
+    llmSessionKey?: string | DataLink
+    /* STT Generator use for transcribing audio message (Currently only support `STT (GGML)` generator) */
+    sttGeneratorId?: string | DataLink
+    /* STT Enabled */
+    sttEnabled?: boolean | DataLink
+    /* STT Live Policy. If the policy is `only-in-use`, the STT context will be released when the assistant is not in use. */
+    sttLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* TTS Generator use for generating LLM response audio message (Currently only support `TTS (ONNX)` generator) */
+    ttsGeneratorId?: string | DataLink
+    /* TTS Enabled */
+    ttsEnabled?: boolean | DataLink
+    /* TTS Live Policy. If the policy is `only-in-use`, the TTS context will be released when the assistant is not in use. */
+    ttsLivePolicy?: 'only-in-use' | 'manual' | DataLink
+  }
+  events?: {
+    /* Error event */
+    onError?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          error: {
+            type: 'string'
+            path: string
+          }
+        }
+      }
+    >
+    /* Log event */
+    onLogs?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          logs: {
+            type: 'array'
+            path: string
+          }
+        }
+      }
+    >
+    /* Messages update event */
+    onMessagesUpdate?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          messages: {
+            type: 'array'
+            path: string
+          }
+        }
+      }
+    >
+  }
+  outlets?: {
+    /* Whether the assistant is heating up */
+    isHeatingUp?: () => Data
+    /* Whether the assistant is responding */
+    isResponding?: () => Data
+    /* Messages of the assistant */
+    messages?: () => Data
+  }
+}
+
+/* AI Assistant */
+export type GeneratorAssistant = Generator &
+  GeneratorAssistantDef & {
+    templateKey: 'GENERATOR_ASSISTANT'
+    switches: Array<
+      SwitchDef &
+        GeneratorAssistantDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isHeatingUp' | 'isResponding' | 'messages'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/system.ts

@@ -222,6 +222,11 @@ export type SystemActionAlert = ActionWithParams & {
         value?: number | DataLink
         mapping?: string
       }
+    | {
+        input: 'hideSelections'
+        value?: boolean | DataLink
+        mapping?: string
+      }
   >
 }
 
@@ -251,6 +256,11 @@ export type SystemActionTakeScreenshot = ActionWithParams & {
         value?: boolean | DataLink
         mapping?: string
       }
+    | {
+        input: 'brickId'
+        value?: string | DataLink | (() => Brick)
+        mapping?: string
+      }
     | {
         input: 'format'
         value?: 'jpg' | 'png' | DataLink
@@ -303,11 +313,21 @@ export type SystemActionChannelSubscribe = ActionWithParams & {
         value?: string | DataLink | (() => Data)
         mapping?: string
       }
+    | {
+        input: 'dataAssignList'
+        value?: {} | DataLink
+        mapping?: string
+      }
     | {
         input: 'progressResult'
         value?: string | DataLink | (() => Data)
         mapping?: string
       }
+    | {
+        input: 'progressAssignList'
+        value?: {} | DataLink
+        mapping?: string
+      }
     | {
         input: 'errorResult'
         value?: string | DataLink | (() => Data)
@@ -475,3 +495,8 @@ export type SystemActionThrowException = ActionWithParams & {
       }
   >
 }
+
+/* Dismiss keyboard */
+export type SystemActionDismissKeyboard = Action & {
+  __actionName: 'DISMISS_KEYBOARD'
+}