@fugood/bricks-project 2.21.0-beta.27 → 2.21.0-beta.28

package/compile/action-name-map.ts

@@ -557,4 +557,31 @@ export const templateActionNameMap = {
       sessionCustomKey: 'GENERATOR_LLM_SESSION_CUSTOM_KEY',
     },
   },
+  GENERATOR_ASSISTANT: {
+    GENERATOR_ASSISTANT_ADD_MESSAGE: {
+      role: 'GENERATOR_ASSISTANT_ROLE',
+      content: 'GENERATOR_ASSISTANT_CONTENT',
+      payload: 'GENERATOR_ASSISTANT_PAYLOAD',
+    },
+    GENERATOR_ASSISTANT_UPDATE_MESSAGE_AT_INDEX: {
+      index: 'GENERATOR_ASSISTANT_INDEX',
+      content: 'GENERATOR_ASSISTANT_CONTENT',
+      payload: 'GENERATOR_ASSISTANT_PAYLOAD',
+    },
+    GENERATOR_ASSISTANT_ADD_AUDIO_MESSAGE: {
+      role: 'GENERATOR_ASSISTANT_ROLE',
+      contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
+      contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      payload: 'GENERATOR_ASSISTANT_PAYLOAD',
+    },
+    GENERATOR_ASSISTANT_UPDATE_AUDIO_MESSAGE_AT_INDEX: {
+      index: 'GENERATOR_ASSISTANT_INDEX',
+      contentFile: 'GENERATOR_ASSISTANT_CONTENT_FILE',
+      contentBase64: 'GENERATOR_ASSISTANT_CONTENT_BASE64',
+      payload: 'GENERATOR_ASSISTANT_PAYLOAD',
+    },
+    GENERATOR_ASSISTANT_REMOVE_MESSAGE_AT_INDEX: {
+      index: 'GENERATOR_ASSISTANT_INDEX',
+    },
+  },
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fugood/bricks-project",
-  "version": "2.21.0-beta.27",
+  "version": "2.21.0-beta.28",
   "main": "index.ts",
   "scripts": {
     "build": "node scripts/build.js"
@@ -13,5 +13,5 @@
     "lodash": "^4.17.4",
     "uuid": "^8.3.1"
   },
-  "gitHead": "85b6bb3416b693de316511738097112f89e764e7"
+  "gitHead": "fcdff86f963446d3489e4b1cccd697e6d04e4f39"
 }

package/types/generators.ts

@@ -3588,121 +3588,6 @@ export type GeneratorWebRTC = Generator &
     >
   }
 
-interface GeneratorTensorFlowInferenceDef {
-  /*
-Default property:
-{
-  "inputType": "camera",
-  "minScore": 0.5,
-  "objectLimit": 20,
-  "imageChannelFormat": "rgb",
-  "imageNormalize": "off",
-  "cameraInit": false,
-  "cameraFps": 10,
-  "cameraRotate": 0
-}
-  */
-  property?: {
-    /* URL to download TF.js graph model  
-  You can find pre-trained model on [TensorFlow Hub](https://tfhub.dev) */
-    modelUrl?: string | DataLink
-    /* MD5 of model JSON */
-    modelMd5?: string | DataLink
-    /* Model input source */
-    inputType?: 'raw' | 'media-image' | 'camera' | DataLink
-    /* Model output type  
-  label: Classification problem (model output number or score array)  
-  object-boxes: Object detection (model output [labels, boxes])  
-  raw: No transform */
-    outputType?: 'label' | 'object-boxes' | 'raw' | DataLink
-    /* Model output labels */
-    labels?: Array<string | DataLink> | DataLink
-    /* Download labels from URL */
-    labelsUrl?: string | DataLink
-    /* MD5 checksum of label file */
-    labelsMd5?: string | DataLink
-    /* Output score limit */
-    minScore?: number | DataLink
-    /* Object limit for object detection */
-    objectLimit?: number | DataLink
-    /* Input channel format */
-    imageChannelFormat?: 'rgb' | 'grayscale' | DataLink
-    /* Input data format of the model is channel first (Commonly is channel last) */
-    imageChannelFirst?: boolean | DataLink
-    /* Resize to input layer height */
-    imageResizeHeight?: number | DataLink
-    /* Resize to input layer width */
-    imageResizeWidth?: number | DataLink
-    /* Input data normalize */
-    imageNormalize?: 'off' | '0~1' | '-1~1' | DataLink
-    /* Start infer on camera when initialized */
-    cameraInit?: boolean | DataLink
-    /* Target Camera brick ID */
-    cameraTarget?: string | DataLink
-    /* Infer fps, 0 is unlimited */
-    cameraFps?: number | DataLink
-    /* Camera rotate correction */
-    cameraRotate?: number | DataLink
-    /* Show detect object bounding box on Camera brick */
-    cameraShowObjectBox?: boolean | DataLink
-    /* Bonding box border size */
-    cameraBoxBorderSize?: number | DataLink
-    /* Bonding box border color */
-    cameraBoxBorderColor?: Array<string | DataLink> | DataLink
-    /* The URL of image to infer */
-    mediaImageUrl?: string | DataLink
-    /* MD5 checksum of image */
-    mediaImageMd5?: string | DataLink
-    /* The data input to infer */
-    rawInput?: any
-  }
-  events?: {
-    /* Event of TensorFlow inference error */
-    onError?: Array<
-      EventAction & {
-        eventPropertyMapping?: {
-          errorMessage: {
-            type: 'string'
-            path: string
-          }
-        }
-      }
-    >
-  }
-  outlets?: {
-    /* Model inference spend time in ms */
-    modelPerformance?: () => Data
-    /* Model state */
-    modelState?: () => Data
-    /* Transformed infer result */
-    result?: () => Data
-    /* Raw infer result */
-    rawResult?: () => Data
-  }
-}
-
-/* TensorFlow Inference */
-export type GeneratorTensorFlowInference = Generator &
-  GeneratorTensorFlowInferenceDef & {
-    templateKey: 'GENERATOR_TENSORFLOW_INFERENCE'
-    switches: Array<
-      SwitchDef &
-        GeneratorTensorFlowInferenceDef & {
-          conds?: Array<{
-            method: '==' | '!=' | '>' | '<' | '>=' | '<='
-            cond:
-              | SwitchCondInnerStateCurrentCanvas
-              | SwitchCondData
-              | {
-                  __typename: 'SwitchCondInnerStateOutlet'
-                  outlet: 'modelPerformance' | 'modelState' | 'result' | 'rawResult'
-                  value: any
-                }
-          }>
-        }
-    >
-  }
-
 /* Run Crawler request with defined properties */
 export type GeneratorWebCrawlerActionRunRequest = Action & {
   __actionName: 'GENERATOR_WEB_CRAWLER_RUN_REQUEST'
@@ -3853,6 +3738,17 @@ Default property:
         }
       }
     >
+    /* Event for recorded file */
+    recorded?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          recordedPath: {
+            type: 'string'
+            path: string
+          }
+        }
+      }
+    >
   }
   outlets?: {
     /* Is recording */
@@ -6491,6 +6387,10 @@ Default property:
     onCompletion?: Array<
       EventAction & {
         eventPropertyMapping?: {
+          completionSessionKey: {
+            type: 'string'
+            path: string
+          }
           completionResult: {
             type: 'string'
             path: string
@@ -6510,6 +6410,10 @@ Default property:
     onCompletionFinished?: Array<
       EventAction & {
         eventPropertyMapping?: {
+          completionSessionKey: {
+            type: 'string'
+            path: string
+          }
           completionResult: {
             type: 'string'
             path: string
@@ -6529,6 +6433,10 @@ Default property:
     onCompletionFunctionCall?: Array<
       EventAction & {
         eventPropertyMapping?: {
+          completionSessionKey: {
+            type: 'string'
+            path: string
+          }
           completionFunctionCallName: {
             type: 'string'
             path: string
@@ -6610,3 +6518,239 @@ export type GeneratorLLM = Generator &
         }
     >
   }
+
+/* Add a message to the assistant */
+export type GeneratorAssistantActionAddMessage = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_ADD_MESSAGE'
+  params?: Array<
+    | {
+        input: 'role'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'content'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink
+        mapping?: string
+      }
+  >
+}
+
+/* Update a message at a specific index */
+export type GeneratorAssistantActionUpdateMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_UPDATE_MESSAGE_AT_INDEX'
+  params?: Array<
+    | {
+        input: 'index'
+        value?: number | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'content'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink
+        mapping?: string
+      }
+  >
+}
+
+/* Add an audio message and convert it to text to the assistant (if STT enabled) */
+export type GeneratorAssistantActionAddAudioMessage = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_ADD_AUDIO_MESSAGE'
+  params?: Array<
+    | {
+        input: 'role'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'contentFile'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'contentBase64'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink
+        mapping?: string
+      }
+  >
+}
+
+/* Update a audio message at a specific index */
+export type GeneratorAssistantActionUpdateAudioMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_UPDATE_AUDIO_MESSAGE_AT_INDEX'
+  params?: Array<
+    | {
+        input: 'index'
+        value?: number | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'contentFile'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'contentBase64'
+        value?: string | DataLink
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink
+        mapping?: string
+      }
+  >
+}
+
+/* Remove a message at a specific index */
+export type GeneratorAssistantActionRemoveMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_REMOVE_MESSAGE_AT_INDEX'
+  params?: Array<{
+    input: 'index'
+    value?: number | DataLink
+    mapping?: string
+  }>
+}
+
+/* Reset all messages of the assistant */
+export type GeneratorAssistantActionResetMessages = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_RESET_MESSAGES'
+}
+
+/* Submit the assistant */
+export type GeneratorAssistantActionSubmit = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_SUBMIT'
+}
+
+/* Cancel the assistant responding */
+export type GeneratorAssistantActionCancel = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_CANCEL'
+}
+
+interface GeneratorAssistantDef {
+  /*
+Default property:
+{
+  "initialMessages": [
+    null
+  ],
+  "cacheMessages": false,
+  "llmLivePolicy": "only-in-use",
+  "llmSessionKey": "default-assistant",
+  "sttEnabled": true,
+  "sttLivePolicy": "only-in-use",
+  "ttsEnabled": false,
+  "ttsLivePolicy": "only-in-use"
+}
+  */
+  property?: {
+    /* Messages */
+    initialMessages?:
+      | Array<
+          | DataLink
+          | {
+              role?: string | DataLink
+              content?: string | DataLink
+            }
+        >
+      | DataLink
+    /* Whether to cache messages */
+    cacheMessages?: boolean | DataLink
+    /* LLM Generator (Currently only support `LLM (GGML)` generator) */
+    llmGeneratorId?: string | DataLink
+    /* LLM Live Policy. If the policy is `only-in-use`, the LLM context will be released when the assistant is not in use. */
+    llmLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* LLM main session key */
+    llmSessionKey?: string | DataLink
+    /* STT Generator use for transcribing audio message (Currently only support `STT (GGML)` generator) */
+    sttGeneratorId?: string | DataLink
+    /* STT Enabled */
+    sttEnabled?: boolean | DataLink
+    /* STT Live Policy. If the policy is `only-in-use`, the STT context will be released when the assistant is not in use. */
+    sttLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* TTS Generator use for generating LLM response audio message (Currently only support `TTS (ONNX)` generator) */
+    ttsGeneratorId?: string | DataLink
+    /* TTS Enabled */
+    ttsEnabled?: boolean | DataLink
+    /* TTS Live Policy. If the policy is `only-in-use`, the TTS context will be released when the assistant is not in use. */
+    ttsLivePolicy?: 'only-in-use' | 'manual' | DataLink
+  }
+  events?: {
+    /* Error event */
+    onError?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          error: {
+            type: 'string'
+            path: string
+          }
+        }
+      }
+    >
+    /* Log event */
+    onLogs?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          logs: {
+            type: 'array'
+            path: string
+          }
+        }
+      }
+    >
+    /* Messages update event */
+    onMessagesUpdate?: Array<
+      EventAction & {
+        eventPropertyMapping?: {
+          messages: {
+            type: 'array'
+            path: string
+          }
+        }
+      }
+    >
+  }
+  outlets?: {
+    /* Whether the assistant is responding */
+    isResponding?: () => Data
+    /* Messages of the assistant */
+    messages?: () => Data
+  }
+}
+
+/* AI Assistant */
+export type GeneratorAssistant = Generator &
+  GeneratorAssistantDef & {
+    templateKey: 'GENERATOR_ASSISTANT'
+    switches: Array<
+      SwitchDef &
+        GeneratorAssistantDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet: 'isResponding' | 'messages'
+                  value: any
+                }
+          }>
+        }
+    >
+  }

package/types/system.ts

@@ -475,3 +475,8 @@ export type SystemActionThrowException = ActionWithParams & {
       }
   >
 }
+
+/* Dismiss keyboard */
+export type SystemActionDismissKeyboard = Action & {
+  __actionName: 'DISMISS_KEYBOARD'
+}