@fugood/bricks-project 2.22.0-beta.9 → 2.22.0

package/types/generators/Assistant.ts

@@ -0,0 +1,546 @@
+import type { SwitchCondInnerStateCurrentCanvas, SwitchCondData, SwitchDef } from '../switch'
+import type { Data, DataLink } from '../data'
+import type {
+  Generator,
+  EventAction,
+  ActionWithDataParams,
+  ActionWithParams,
+  Action,
+  EventProperty,
+} from '../common'
+
+/* Add a message to the assistant */
+export type GeneratorAssistantActionAddMessage = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_ADD_MESSAGE'
+  params?: Array<
+    | {
+        input: 'role'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'content'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'useFileSearch'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'filePath'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileExtension'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'filePayload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileSearchCitationCount'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileSearchInsertMethod'
+        value?: 'in-user-message' | 'new-assistant-message' | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Initialize messages from MCP prompt */
+export type GeneratorAssistantActionInitMcpPrompt = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_INIT_MCP_PROMPT'
+  params?: Array<
+    | {
+        input: 'mcpClientName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mcpPromptName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mcpArguments'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'firstMessageAsSystem'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Add messages from MCP prompt */
+export type GeneratorAssistantActionAddMcpPromptMessage = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_ADD_MCP_PROMPT_MESSAGE'
+  params?: Array<
+    | {
+        input: 'mcpClientName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mcpPromptName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mcpArguments'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Update a message at a specific index */
+export type GeneratorAssistantActionUpdateMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_UPDATE_MESSAGE_AT_INDEX'
+  params?: Array<
+    | {
+        input: 'index'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'content'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Add an audio message and convert it to text to the assistant (if STT enabled) */
+export type GeneratorAssistantActionAddAudioMessage = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_ADD_AUDIO_MESSAGE'
+  params?: Array<
+    | {
+        input: 'role'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'contentFile'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'contentBase64'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'useFileSearch'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'filePath'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileExtension'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'filePayload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileSearchCitationCount'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileSearchInsertMethod'
+        value?: 'in-user-message' | 'new-assistant-message' | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Add a file to File Search (Vector Store) */
+export type GeneratorAssistantActionAddFile = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_ADD_FILE'
+  params?: Array<
+    | {
+        input: 'filePath'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'fileExtension'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'filePayload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Remove a file from File Search (Vector Store) */
+export type GeneratorAssistantActionRemoveFile = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_REMOVE_FILE'
+  params?: Array<{
+    input: 'fileName'
+    value?: string | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Update a audio message at a specific index */
+export type GeneratorAssistantActionUpdateAudioMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_UPDATE_AUDIO_MESSAGE_AT_INDEX'
+  params?: Array<
+    | {
+        input: 'index'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'contentFile'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'contentBase64'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'image'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'payload'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Remove a message at a specific index */
+export type GeneratorAssistantActionRemoveMessageAtIndex = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_REMOVE_MESSAGE_AT_INDEX'
+  params?: Array<{
+    input: 'index'
+    value?: number | DataLink | EventProperty
+    mapping?: string
+  }>
+}
+
+/* Reset all messages of the assistant */
+export type GeneratorAssistantActionResetMessages = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_RESET_MESSAGES'
+}
+
+/* Reset assistant (Reset all messages and files) */
+export type GeneratorAssistantActionReset = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_RESET'
+}
+
+/* Submit the assistant */
+export type GeneratorAssistantActionSubmit = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_SUBMIT'
+  params?: Array<
+    | {
+        input: 'continueOnToolCallConfirm'
+        value?: boolean | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'continueOnToolCallStrategy'
+        value?: 'never' | 'success' | 'always' | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'continueOnToolCallLimit'
+        value?: number | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Cancel the assistant responding */
+export type GeneratorAssistantActionCancel = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_CANCEL'
+}
+
+/* Check the enabled MCP clients connection status and available tools */
+export type GeneratorAssistantActionCheckMcpServers = Action & {
+  __actionName: 'GENERATOR_ASSISTANT_CHECK_MCP_SERVERS'
+}
+
+/* Insert an MCP resource as a new assistant message */
+export type GeneratorAssistantActionInsertMcpResource = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_INSERT_MCP_RESOURCE'
+  params?: Array<
+    | {
+        input: 'mcpClientName'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mcpResourceUri'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'mcpVariables'
+        value?: {} | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'role'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+/* Summarize messages based on the conversation  
+
+ Note: Summary uses the same LLM context size, so it is recommended only to use it when the system prompt (in Initial Messages) is long,  otherwise it may still fail when the context is full (Ctx Shift is NO). */
+export type GeneratorAssistantActionSummaryMessages = ActionWithParams & {
+  __actionName: 'GENERATOR_ASSISTANT_SUMMARY_MESSAGES'
+  params?: Array<
+    | {
+        input: 'summaryMessages'
+        value?: Array<any> | DataLink | EventProperty
+        mapping?: string
+      }
+    | {
+        input: 'summarySessionKey'
+        value?: string | DataLink | EventProperty
+        mapping?: string
+      }
+  >
+}
+
+interface GeneratorAssistantDef {
+  /*
+Default property:
+{
+  "initialMessages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant."
+    }
+  ],
+  "cacheMessages": false,
+  "llmLivePolicy": "only-in-use",
+  "llmSessionKey": "default-assistant",
+  "llmAutoSummaryMessages": false,
+  "llmSummaryMessages": [
+    {
+      "role": "system",
+      "content": "You are a helpful assistant specialized in summarizing conversations. Create a concise summary of the conversation that captures the key points while maintaining important context. The summary should be clear, accurate, and briefer than the original conversation."
+    },
+    {
+      "role": "user",
+      "content": "Please summarize the following conversation into a concise system message that can replace the previous conversation context while maintaining all important information. Here is the conversation to summarize:\n\n"
+    }
+  ],
+  "llmSummarySessionKey": "assistant-default-summary",
+  "fileSearchEnabled": false,
+  "fileSearchLivePolicy": "only-in-use",
+  "sttEnabled": true,
+  "sttLivePolicy": "only-in-use",
+  "ttsEnabled": false,
+  "ttsLivePolicy": "only-in-use"
+}
+  */
+  property?: {
+    /* Whether to heat up generators on initialization for better performance */
+    heatup?: boolean | DataLink
+    /* Messages */
+    initialMessages?:
+      | Array<
+          | DataLink
+          | {
+              role?: string | DataLink
+              content?: string | DataLink
+            }
+        >
+      | DataLink
+    /* Whether to cache messages */
+    cacheMessages?: boolean | DataLink
+    /* LLM Generator (Supports `LLM (GGML)` and `OpenAI LLM` generators) */
+    llmGeneratorId?: string | DataLink
+    /* LLM Live Policy. If the policy is `only-in-use`, the LLM context will be released when the assistant is not in use.  
+
+ Note: LLM (Qualcomm AI Engine) recommend use `manual` and loaded constantly. */
+    llmLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* LLM main session key */
+    llmSessionKey?: string | DataLink
+    /* Auto Summary Messages (Automatically summarize messages when the LLM context is full or content gets truncated, currently only supported with LLM (GGML) generators)  
+
+ Note: Summary uses the same LLM context size, so it is recommended only to use it when the system prompt (in Initial Messages) is long,  otherwise it may still fail when the context is full (Ctx Shift is NO). */
+    llmAutoSummaryMessages?: boolean | DataLink
+    /* Summary Messages (Messages used for summarization prompt, conversation will be appended to the last message) */
+    llmSummaryMessages?:
+      | Array<
+          | DataLink
+          | {
+              role?: string | DataLink
+              content?: string | DataLink
+            }
+        >
+      | DataLink
+    /* Summary Session Key (Custom session key for summarization) */
+    llmSummarySessionKey?: string | DataLink
+    /* File Search (Vector Store) Enabled */
+    fileSearchEnabled?: boolean | DataLink
+    /* File Search (Vector Store) Generator */
+    fileSearchGeneratorId?: string | DataLink
+    /* File Search (Vector Store) Live Policy. If the policy is `only-in-use`, the File Search (Vector Store) context will be released when the assistant is not in use. */
+    fileSearchLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* File Search Citation Count. (Default: 3) */
+    fileSearchCitationCount?: number | DataLink
+    /* File Search Threshold. (Default: 1.0) */
+    fileSearchThreshold?: number | DataLink
+    /* File Search Ignore Threshold. (Default: false) */
+    fileSearchIgnoreThreshold?: boolean | DataLink
+    /* STT Generator use for transcribing audio message (Supports `STT (GGML)` generators) */
+    sttGeneratorId?: string | DataLink
+    /* STT Enabled */
+    sttEnabled?: boolean | DataLink
+    /* STT Live Policy. If the policy is `only-in-use`, the STT context will be released when the assistant is not in use. */
+    sttLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* TTS Generator use for generating LLM response audio message (Supports `TTS (ONNX)` and `OpenAI TTS` generators) */
+    ttsGeneratorId?: string | DataLink
+    /* TTS Enabled */
+    ttsEnabled?: boolean | DataLink
+    /* TTS Live Policy. If the policy is `only-in-use`, the TTS context will be released when the assistant is not in use. */
+    ttsLivePolicy?: 'only-in-use' | 'manual' | DataLink
+    /* MCP Generators (Add a unique name if generator name property are duplicate) */
+    mcpGenerators?:
+      | Array<
+          | DataLink
+          | {
+              generatorId?: string | DataLink
+              generatorKey?: string | DataLink
+              name?: string | DataLink
+              enabled?: boolean | DataLink
+            }
+        >
+      | DataLink
+  }
+  events?: {
+    /* Error event */
+    onError?: Array<EventAction>
+    /* Log event */
+    onLogs?: Array<EventAction>
+    /* Messages update event */
+    onMessagesUpdate?: Array<EventAction>
+  }
+  outlets?: {
+    /* Whether the assistant is heating up */
+    isHeatingUp?: () => Data
+    /* Whether the assistant is file processing */
+    isFileProcessing?: () => Data
+    /* Whether the assistant is responding */
+    isResponding?: () => Data
+    /* Whether the assistant is busy */
+    isBusy?: () => Data
+    /* Embedding files of the assistant */
+    files?: () => Data
+    /* Messages of the assistant */
+    messages?: () => Data
+    /* MCP servers status and available tools */
+    mcpServers?: () => Data
+  }
+}
+
+/* AI Assistant */
+export type GeneratorAssistant = Generator &
+  GeneratorAssistantDef & {
+    templateKey: 'GENERATOR_ASSISTANT'
+    switches: Array<
+      SwitchDef &
+        GeneratorAssistantDef & {
+          conds?: Array<{
+            method: '==' | '!=' | '>' | '<' | '>=' | '<='
+            cond:
+              | SwitchCondInnerStateCurrentCanvas
+              | SwitchCondData
+              | {
+                  __typename: 'SwitchCondInnerStateOutlet'
+                  outlet:
+                    | 'isHeatingUp'
+                    | 'isFileProcessing'
+                    | 'isResponding'
+                    | 'isBusy'
+                    | 'files'
+                    | 'messages'
+                    | 'mcpServers'
+                  value: any
+                }
+          }>
+        }
+    >
+  }