@aiscene/core 1.7.0

package/dist/es/ai-model/conversation-history.mjs

@@ -0,0 +1,195 @@
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+var _computedKey;
+_computedKey = Symbol.iterator;
+let _computedKey1 = _computedKey;
+class ConversationHistory {
+    resetPendingFeedbackMessageIfExists() {
+        if (this.pendingFeedbackMessage) this.pendingFeedbackMessage = '';
+    }
+    append(message) {
+        this.messages.push(message);
+    }
+    seed(messages) {
+        this.reset();
+        messages.forEach((message)=>{
+            this.append(message);
+        });
+    }
+    reset() {
+        this.messages.length = 0;
+        this.memories.length = 0;
+        this.subGoals.length = 0;
+        this.historicalLogs.length = 0;
+        this.pendingFeedbackMessage = '';
+    }
+    snapshot(maxImages) {
+        if (void 0 === maxImages) return [
+            ...this.messages
+        ];
+        const clonedMessages = structuredClone(this.messages);
+        let imageCount = 0;
+        for(let i = clonedMessages.length - 1; i >= 0; i--){
+            const message = clonedMessages[i];
+            const content = message.content;
+            if (Array.isArray(content)) for(let j = 0; j < content.length; j++){
+                const item = content[j];
+                if ('image_url' === item.type) {
+                    imageCount++;
+                    if (imageCount > maxImages) content[j] = {
+                        type: 'text',
+                        text: '(image ignored due to size optimization)'
+                    };
+                }
+            }
+        }
+        return clonedMessages;
+    }
+    get length() {
+        return this.messages.length;
+    }
+    [_computedKey1]() {
+        return this.messages[Symbol.iterator]();
+    }
+    toJSON() {
+        return this.snapshot();
+    }
+    setSubGoals(subGoals) {
+        this.subGoals = subGoals.map((goal)=>({
+                ...goal
+            }));
+        this.markFirstPendingAsRunning();
+    }
+    mergeSubGoals(subGoals) {
+        if (0 === this.subGoals.length) return void this.setSubGoals(subGoals);
+        const existingByIndex = new Map(this.subGoals.map((goal)=>[
+                goal.index,
+                goal
+            ]));
+        const mergedSubGoals = subGoals.map((goal)=>{
+            const existingGoal = existingByIndex.get(goal.index);
+            const hasNonEmptyDescription = goal.description.trim().length > 0;
+            if (!existingGoal && !hasNonEmptyDescription) return null;
+            return {
+                ...goal,
+                description: hasNonEmptyDescription || !existingGoal ? goal.description : existingGoal.description
+            };
+        });
+        const validSubGoals = mergedSubGoals.filter((goal)=>null !== goal);
+        if (0 === validSubGoals.length) return;
+        this.setSubGoals(validSubGoals);
+    }
+    updateSubGoal(index, updates) {
+        const goal = this.subGoals.find((g)=>g.index === index);
+        if (!goal) return false;
+        let changed = false;
+        if (void 0 !== updates.status && updates.status !== goal.status) {
+            goal.status = updates.status;
+            changed = true;
+        }
+        if (void 0 !== updates.description && updates.description !== goal.description) {
+            goal.description = updates.description;
+            changed = true;
+        }
+        if (changed) goal.logs = [];
+        return true;
+    }
+    markFirstPendingAsRunning() {
+        const firstPending = this.subGoals.find((g)=>'pending' === g.status);
+        if (firstPending) {
+            firstPending.status = 'running';
+            firstPending.logs = [];
+        }
+    }
+    markSubGoalFinished(index) {
+        const result = this.updateSubGoal(index, {
+            status: 'finished'
+        });
+        if (result) this.markFirstPendingAsRunning();
+        return result;
+    }
+    markAllSubGoalsFinished() {
+        for (const goal of this.subGoals){
+            if ('finished' !== goal.status) goal.logs = [];
+            goal.status = 'finished';
+        }
+    }
+    appendSubGoalLog(log) {
+        if (!log) return;
+        const runningGoal = this.subGoals.find((g)=>'running' === g.status);
+        if (runningGoal) {
+            if (!runningGoal.logs) runningGoal.logs = [];
+            runningGoal.logs.push(log);
+        }
+    }
+    subGoalsToText() {
+        if (0 === this.subGoals.length) return '';
+        const lines = this.subGoals.map((goal)=>`${goal.index}. ${goal.description} (${goal.status})`);
+        const currentGoal = this.subGoals.find((goal)=>'running' === goal.status) || this.subGoals.find((goal)=>'pending' === goal.status);
+        let currentGoalText = '';
+        if (currentGoal) {
+            currentGoalText = `\nCurrent sub-goal is: ${currentGoal.description}`;
+            if (currentGoal.logs && currentGoal.logs.length > 0) {
+                const logLines = currentGoal.logs.map((log)=>`- ${log}`).join('\n');
+                currentGoalText += `\nActions performed for current sub-goal:\n${logLines}`;
+            }
+        }
+        return `Sub-goals:\n${lines.join('\n')}${currentGoalText}`;
+    }
+    appendHistoricalLog(log) {
+        if (log) this.historicalLogs.push(log);
+    }
+    historicalLogsToText() {
+        if (0 === this.historicalLogs.length) return '';
+        const logLines = this.historicalLogs.map((log)=>`- ${log}`).join('\n');
+        return `Here are the steps that have been executed:\n${logLines}`;
+    }
+    appendMemory(memory) {
+        if (memory) this.memories.push(memory);
+    }
+    getMemories() {
+        return [
+            ...this.memories
+        ];
+    }
+    memoriesToText() {
+        if (0 === this.memories.length) return '';
+        return `Memories from previous steps:\n---\n${this.memories.join('\n---\n')}\n`;
+    }
+    clearMemories() {
+        this.memories.length = 0;
+    }
+    compressHistory(threshold, keepCount) {
+        if (this.messages.length <= threshold) return false;
+        const omittedCount = this.messages.length - keepCount;
+        const omittedPlaceholder = {
+            role: 'user',
+            content: `(${omittedCount} previous conversation messages have been omitted)`
+        };
+        const recentMessages = this.messages.slice(-keepCount);
+        this.messages.length = 0;
+        this.messages.push(omittedPlaceholder);
+        for (const msg of recentMessages)this.messages.push(msg);
+        return true;
+    }
+    constructor(options){
+        _define_property(this, "messages", []);
+        _define_property(this, "subGoals", []);
+        _define_property(this, "memories", []);
+        _define_property(this, "historicalLogs", []);
+        _define_property(this, "pendingFeedbackMessage", void 0);
+        if (options?.initialMessages?.length) this.seed(options.initialMessages);
+        this.pendingFeedbackMessage = '';
+    }
+}
+export { ConversationHistory };
+
+//# sourceMappingURL=conversation-history.mjs.map

package/dist/es/ai-model/conversation-history.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model/conversation-history.mjs","sources":["../../../src/ai-model/conversation-history.ts"],"sourcesContent":["import type { SubGoal } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ChatCompletionMessageParam[] = [];\n  private subGoals: SubGoal[] = [];\n  private memories: string[] = [];\n  private historicalLogs: string[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n    this.memories.length = 0;\n    this.subGoals.length = 0;\n    this.historicalLogs.length = 0;\n    this.pendingFeedbackMessage = '';\n  }\n\n  /**\n   * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n   * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n   * @returns The snapshot of the conversation history.\n   */\n  snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n    if (maxImages === undefined) {\n      return [...this.messages];\n    }\n\n    const clonedMessages = structuredClone(this.messages);\n    let imageCount = 0;\n\n    // Traverse from the end to the beginning\n    for (let i = clonedMessages.length - 1; i >= 0; i--) {\n      const message = clonedMessages[i];\n      const content = message.content;\n\n      // Only process if content is an array\n      if (Array.isArray(content)) {\n        for (let j = 0; j < content.length; j++) {\n          const item = content[j];\n\n          // Check if this is an image\n          if (item.type === 'image_url') {\n            imageCount++;\n\n            // If we've exceeded the limit, replace with text\n            if (imageCount > maxImages) {\n              content[j] = {\n                type: 'text',\n                text: '(image ignored due to size optimization)',\n              };\n            }\n          }\n        }\n      }\n    }\n\n    return clonedMessages;\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n\n  // Sub-goal management methods\n\n  /**\n   * Set all sub-goals, replacing any existing ones.\n   * Automatically marks the first pending goal as running.\n   */\n  setSubGoals(subGoals: SubGoal[]): void {\n    this.subGoals = subGoals.map((goal) => ({ ...goal }));\n    this.markFirstPendingAsRunning();\n  }\n\n  /**\n   * Merge sub-goals from update-plan-content.\n   * Preserves existing descriptions when incoming description is empty.\n   *\n   * This handles compact XML updates like:\n   * <sub-goal index=\"1\" status=\"finished\" />\n   */\n  mergeSubGoals(subGoals: SubGoal[]): void {\n    if (this.subGoals.length === 0) {\n      this.setSubGoals(subGoals);\n      return;\n    }\n\n    const existingByIndex = new Map(\n      this.subGoals.map((goal) => [goal.index, goal] as const),\n    );\n\n    const mergedSubGoals = subGoals.map((goal) => {\n      const existingGoal = existingByIndex.get(goal.index);\n      const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n      if (!existingGoal && !hasNonEmptyDescription) {\n        return null;\n      }\n\n      return {\n        ...goal,\n        description:\n          hasNonEmptyDescription || !existingGoal\n            ? goal.description\n            : existingGoal.description,\n      };\n    });\n\n    const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n    if (validSubGoals.length === 0) {\n      return;\n    }\n\n    this.setSubGoals(validSubGoals);\n  }\n\n  /**\n   * Update a single sub-goal by index.\n   * Clears logs if status or description actually changes.\n   * @returns true if the sub-goal was found and updated, false otherwise\n   */\n  updateSubGoal(\n    index: number,\n    updates: Partial<Omit<SubGoal, 'index'>>,\n  ): boolean {\n    const goal = this.subGoals.find((g) => g.index === index);\n    if (!goal) {\n      return false;\n    }\n\n    let changed = false;\n\n    if (updates.status !== undefined && updates.status !== goal.status) {\n      goal.status = updates.status;\n      changed = true;\n    }\n    if (\n      updates.description !== undefined &&\n      updates.description !== goal.description\n    ) {\n      goal.description = updates.description;\n      changed = true;\n    }\n\n    if (changed) {\n      goal.logs = [];\n    }\n\n    return true;\n  }\n\n  /**\n   * Mark the first pending sub-goal as running.\n   * Clears logs since status changes.\n   */\n  markFirstPendingAsRunning(): void {\n    const firstPending = this.subGoals.find((g) => g.status === 'pending');\n    if (firstPending) {\n      firstPending.status = 'running';\n      firstPending.logs = [];\n    }\n  }\n\n  /**\n   * Mark a sub-goal as finished.\n   * Automatically marks the next pending goal as running.\n   * @returns true if the sub-goal was found and updated, false otherwise\n   */\n  markSubGoalFinished(index: number): boolean {\n    const result = this.updateSubGoal(index, { status: 'finished' });\n    if (result) {\n      this.markFirstPendingAsRunning();\n    }\n    return result;\n  }\n\n  /**\n   * Mark all sub-goals as finished.\n   * Clears logs for any goal whose status actually changes.\n   */\n  markAllSubGoalsFinished(): void {\n    for (const goal of this.subGoals) {\n      if (goal.status !== 'finished') {\n        goal.logs = [];\n      }\n      goal.status = 'finished';\n    }\n  }\n\n  /**\n   * Append a log entry to the currently running sub-goal.\n   * The log describes an action performed while working on the sub-goal.\n   */\n  appendSubGoalLog(log: string): void {\n    if (!log) {\n      return;\n    }\n    const runningGoal = this.subGoals.find((g) => g.status === 'running');\n    if (runningGoal) {\n      if (!runningGoal.logs) {\n        runningGoal.logs = [];\n      }\n      runningGoal.logs.push(log);\n    }\n  }\n\n  /**\n   * Convert sub-goals to text representation.\n   * Includes actions performed (logs) for the current sub-goal.\n   */\n  subGoalsToText(): string {\n    if (this.subGoals.length === 0) {\n      return '';\n    }\n\n    const lines = this.subGoals.map((goal) => {\n      return `${goal.index}. ${goal.description} (${goal.status})`;\n    });\n\n    // Running goal takes priority, otherwise show first pending\n    const currentGoal =\n      this.subGoals.find((goal) => goal.status === 'running') ||\n      this.subGoals.find((goal) => goal.status === 'pending');\n\n    let currentGoalText = '';\n    if (currentGoal) {\n      currentGoalText = `\\nCurrent sub-goal is: ${currentGoal.description}`;\n      if (currentGoal.logs && currentGoal.logs.length > 0) {\n        const logLines = currentGoal.logs.map((log) => `- ${log}`).join('\\n');\n        currentGoalText += `\\nActions performed for current sub-goal:\\n${logLines}`;\n      }\n    }\n\n    return `Sub-goals:\\n${lines.join('\\n')}${currentGoalText}`;\n  }\n\n  // Historical log management methods (used in non-deepThink mode)\n\n  /**\n   * Append a log entry to the historical logs list.\n   * Used in non-deepThink mode to track executed steps across planning rounds.\n   */\n  appendHistoricalLog(log: string): void {\n    if (log) {\n      this.historicalLogs.push(log);\n    }\n  }\n\n  /**\n   * Convert historical logs to text representation.\n   * Provides context about previously executed steps to the model.\n   */\n  historicalLogsToText(): string {\n    if (this.historicalLogs.length === 0) {\n      return '';\n    }\n\n    const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n    return `Here are the steps that have been executed:\\n${logLines}`;\n  }\n\n  // Memory management methods\n\n  /**\n   * Append a memory to the memories list\n   */\n  appendMemory(memory: string): void {\n    if (memory) {\n      this.memories.push(memory);\n    }\n  }\n\n  /**\n   * Get all memories\n   */\n  getMemories(): string[] {\n    return [...this.memories];\n  }\n\n  /**\n   * Convert memories to text representation\n   */\n  memoriesToText(): string {\n    if (this.memories.length === 0) {\n      return '';\n    }\n\n    return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n  }\n\n  /**\n   * Clear all memories\n   */\n  clearMemories(): void {\n    this.memories.length = 0;\n  }\n\n  /**\n   * Compress the conversation history if it exceeds the threshold.\n   * Removes the oldest messages and replaces them with a single placeholder message.\n   * @param threshold - The number of messages that triggers compression.\n   * @param keepCount - The number of recent messages to keep after compression.\n   * @returns true if compression was performed, false otherwise.\n   */\n  compressHistory(threshold: number, keepCount: number): boolean {\n    if (this.messages.length <= threshold) {\n      return false;\n    }\n\n    const omittedCount = this.messages.length - keepCount;\n    const omittedPlaceholder: ChatCompletionMessageParam = {\n      role: 'user',\n      content: `(${omittedCount} previous conversation messages have been omitted)`,\n    };\n\n    // Keep only the last `keepCount` messages\n    const recentMessages = this.messages.slice(-keepCount);\n\n    // Reset and rebuild with placeholder + recent messages\n    this.messages.length = 0;\n    this.messages.push(omittedPlaceholder);\n    for (const msg of recentMessages) {\n      this.messages.push(msg);\n    }\n\n    return true;\n  }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","lines","currentGoal","currentGoalText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options"],"mappings":";;;;;;;;;;;eA6FGA,OAAO,QAAQ;;AAtFX,MAAMC;IAeX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYc,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBlB,WAAnBkB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBlB,WAAxBkB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,MAAME,QAAQ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACf,OACxB,GAAGA,KAAK,KAAK,CAAC,EAAE,EAAEA,KAAK,WAAW,CAAC,EAAE,EAAEA,KAAK,MAAM,CAAC,CAAC,CAAC;QAI9D,MAAMgB,cACJ,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAChB,OAASA,AAAgB,cAAhBA,KAAK,MAAM,KACxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACA,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAE1C,IAAIiB,kBAAkB;QACtB,IAAID,aAAa;YACfC,kBAAkB,CAAC,uBAAuB,EAAED,YAAY,WAAW,EAAE;YACrE,IAAIA,YAAY,IAAI,IAAIA,YAAY,IAAI,CAAC,MAAM,GAAG,GAAG;gBACnD,MAAME,WAAWF,YAAY,IAAI,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;gBAChEI,mBAAmB,CAAC,2CAA2C,EAAEC,UAAU;YAC7E;QACF;QAEA,OAAO,CAAC,YAAY,EAAEH,MAAM,IAAI,CAAC,QAAQE,iBAAiB;IAC5D;IAQA,oBAAoBJ,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMK,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACL,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEK,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAAiD;YACrD,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IA7VA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAyC,EAAE;QAC5D,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAyVF"}

package/dist/es/ai-model/index.mjs

@@ -0,0 +1,11 @@
+import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
+import { systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
+import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/playwright-generator.mjs";
+import { generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
+import { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection } from "./inspect.mjs";
+import { plan } from "./llm-planning.mjs";
+import { autoGLMPlanning } from "./auto-glm/planning.mjs";
+import { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, parseActionParam } from "../common.mjs";
+import { uiTarsPlanning } from "./ui-tars-planning.mjs";
+import { ConversationHistory } from "./conversation-history.mjs";
+export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBboxToRect, autoGLMPlanning, callAI, callAIWithObjectResponse, callAIWithStringResponse, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, parseActionParam, plan, systemPromptToLocateElement, uiTarsPlanning };

package/dist/es/ai-model/inspect.mjs

@@ -0,0 +1,394 @@
+import { generateElementByPoint, generateElementByRect } from "@midscene/shared/extractor/dom-util";
+import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl, scaleImage } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { adaptBboxToRect, expandSearchArea, mergeRects } from "../common.mjs";
+import { parseAutoGLMLocateResponse } from "./auto-glm/parser.mjs";
+import { getAutoGLMLocatePrompt } from "./auto-glm/prompt.mjs";
+import { isAutoGLM } from "./auto-glm/util.mjs";
+import { extractDataQueryPrompt, parseXMLExtractionResponse, systemPromptToExtract } from "./prompt/extraction.mjs";
+import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
+import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
+import { orderSensitiveJudgePrompt, systemPromptToJudgeOrderSensitive } from "./prompt/order-sensitive-judge.mjs";
+import { AIResponseParseError, callAI, callAIWithObjectResponse, callAIWithStringResponse } from "./service-caller/index.mjs";
+const debugInspect = getDebug('ai:inspect');
+const debugSection = getDebug('ai:section');
+async function buildSearchAreaConfig(options) {
+    const { context, baseRect, modelFamily } = options;
+    const scaleRatio = 2;
+    const sectionRect = expandSearchArea(baseRect, context.shotSize);
+    const croppedResult = await cropByRect(context.screenshot.base64, sectionRect, 'qwen2.5-vl' === modelFamily);
+    const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);
+    sectionRect.width = scaledResult.width;
+    sectionRect.height = scaledResult.height;
+    return {
+        rect: sectionRect,
+        imageBase64: scaledResult.imageBase64,
+        scale: scaleRatio
+    };
+}
+const extraTextFromUserPrompt = (prompt)=>{
+    if ('string' == typeof prompt) return prompt;
+    return prompt.prompt;
+};
+const promptsToChatParam = async (multimodalPrompt)=>{
+    const msgs = [];
+    if (multimodalPrompt?.images?.length) {
+        msgs.push({
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: 'Next, I will provide all the reference images.'
+                }
+            ]
+        });
+        for (const item of multimodalPrompt.images){
+            const base64 = await preProcessImageUrl(item.url, !!multimodalPrompt.convertHttpImage2Base64);
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `this is the reference image named '${item.name}':`
+                    }
+                ]
+            });
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: base64,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            });
+        }
+    }
+    return msgs;
+};
+async function AiLocateElement(options) {
+    const { context, targetElementDescription, modelConfig } = options;
+    const { modelFamily } = modelConfig;
+    const screenshotBase64 = context.screenshot.base64;
+    assert(targetElementDescription, "cannot find the target element description");
+    const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
+    const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);
+    const systemPrompt = isAutoGLM(modelFamily) ? getAutoGLMLocatePrompt(modelFamily) : systemPromptToLocateElement(modelFamily);
+    let imagePayload = screenshotBase64;
+    let imageWidth = context.shotSize.width;
+    let imageHeight = context.shotSize.height;
+    let originalImageWidth = imageWidth;
+    let originalImageHeight = imageHeight;
+    if (options.searchConfig) {
+        assert(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
+        assert(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
+        imagePayload = options.searchConfig.imageBase64;
+        imageWidth = options.searchConfig.rect?.width;
+        imageHeight = options.searchConfig.rect?.height;
+        originalImageWidth = imageWidth;
+        originalImageHeight = imageHeight;
+    } else if ('qwen2.5-vl' === modelFamily) {
+        const paddedResult = await paddingToMatchBlockByBase64(imagePayload);
+        imageWidth = paddedResult.width;
+        imageHeight = paddedResult.height;
+        imagePayload = paddedResult.imageBase64;
+    }
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: isAutoGLM(modelFamily) ? `Tap: ${userInstructionPrompt}` : userInstructionPrompt
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof targetElementDescription) {
+        const addOns = await promptsToChatParam({
+            images: targetElementDescription.images,
+            convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    if (isAutoGLM(modelFamily)) {
+        const { content: rawResponseContent, usage } = await callAIWithStringResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+        debugInspect('auto-glm rawResponse:', rawResponseContent);
+        const parsed = parseAutoGLMLocateResponse(rawResponseContent);
+        debugInspect('auto-glm thinking:', parsed.think);
+        debugInspect('auto-glm coordinates:', parsed.coordinates);
+        let resRect;
+        let matchedElements = [];
+        let errors = [];
+        if (parsed.error || !parsed.coordinates) {
+            errors = [
+                parsed.error || 'Failed to parse auto-glm response'
+            ];
+            debugInspect('auto-glm parse error:', errors[0]);
+        } else {
+            const { x, y } = parsed.coordinates;
+            debugInspect('auto-glm coordinates [0-999]:', {
+                x,
+                y
+            });
+            const pixelX = Math.round(x * imageWidth / 1000);
+            const pixelY = Math.round(y * imageHeight / 1000);
+            debugInspect('auto-glm pixel coordinates:', {
+                pixelX,
+                pixelY
+            });
+            let finalX = pixelX;
+            let finalY = pixelY;
+            if (options.searchConfig?.rect) {
+                finalX += options.searchConfig.rect.left;
+                finalY += options.searchConfig.rect.top;
+            }
+            const element = generateElementByPoint([
+                finalX,
+                finalY
+            ], targetElementDescriptionText);
+            resRect = element.rect;
+            debugInspect('auto-glm resRect:', resRect);
+            if (element) matchedElements = [
+                element
+            ];
+        }
+        return {
+            rect: resRect,
+            parseResult: {
+                elements: matchedElements,
+                errors
+            },
+            rawResponse: rawResponseContent,
+            usage,
+            reasoning_content: parsed.think
+        };
+    }
+    let res;
+    try {
+        res = await callAIWithObjectResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+    } catch (callError) {
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
+        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
+        return {
+            rect: void 0,
+            parseResult: {
+                elements: [],
+                errors: [
+                    `AI call error: ${errorMessage}`
+                ]
+            },
+            rawResponse,
+            usage,
+            reasoning_content: void 0
+        };
+    }
+    const rawResponse = JSON.stringify(res.content);
+    let resRect;
+    let matchedElements = [];
+    let errors = 'errors' in res.content ? res.content.errors : [];
+    try {
+        if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
+            resRect = adaptBboxToRect(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, modelFamily, options.searchConfig?.scale);
+            debugInspect('resRect', resRect);
+            const element = generateElementByRect(resRect, targetElementDescriptionText);
+            errors = [];
+            if (element) matchedElements = [
+                element
+            ];
+        }
+    } catch (e) {
+        const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
+        if (errors && errors?.length !== 0) errors.push(`(${msg})`);
+        else errors = [
+            msg
+        ];
+    }
+    return {
+        rect: resRect,
+        parseResult: {
+            elements: matchedElements,
+            errors: errors
+        },
+        rawResponse,
+        usage: res.usage,
+        reasoning_content: res.reasoning_content
+    };
+}
+async function AiLocateSection(options) {
+    const { context, sectionDescription, modelConfig } = options;
+    const { modelFamily } = modelConfig;
+    const screenshotBase64 = context.screenshot.base64;
+    const systemPrompt = systemPromptToLocateSection(modelFamily);
+    const sectionLocatorInstructionText = sectionLocatorInstruction(extraTextFromUserPrompt(sectionDescription));
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshotBase64,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: sectionLocatorInstructionText
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof sectionDescription) {
+        const addOns = await promptsToChatParam({
+            images: sectionDescription.images,
+            convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    let result;
+    try {
+        result = await callAIWithObjectResponse(msgs, modelConfig, {
+            abortSignal: options.abortSignal
+        });
+    } catch (callError) {
+        const errorMessage = callError instanceof Error ? callError.message : String(callError);
+        const rawResponse = callError instanceof AIResponseParseError ? callError.rawResponse : errorMessage;
+        const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
+        return {
+            rect: void 0,
+            imageBase64: void 0,
+            error: `AI call error: ${errorMessage}`,
+            rawResponse,
+            usage
+        };
+    }
+    let searchAreaConfig;
+    const sectionBbox = result.content.bbox;
+    if (sectionBbox) {
+        const targetRect = adaptBboxToRect(sectionBbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily);
+        debugSection('original targetRect %j', targetRect);
+        const referenceBboxList = result.content.references_bbox || [];
+        debugSection('referenceBboxList %j', referenceBboxList);
+        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>adaptBboxToRect(bbox, context.shotSize.width, context.shotSize.height, 0, 0, context.shotSize.width, context.shotSize.height, modelFamily));
+        debugSection('referenceRects %j', referenceRects);
+        const mergedRect = mergeRects([
+            targetRect,
+            ...referenceRects
+        ]);
+        debugSection('mergedRect %j', mergedRect);
+        const expandedRect = expandSearchArea(mergedRect, context.shotSize);
+        const originalWidth = expandedRect.width;
+        const originalHeight = expandedRect.height;
+        debugSection('expanded sectionRect %j', expandedRect);
+        searchAreaConfig = await buildSearchAreaConfig({
+            context,
+            baseRect: mergedRect,
+            modelFamily
+        });
+        debugSection('scaled sectionRect from %dx%d to %dx%d (scale=%d)', originalWidth, originalHeight, searchAreaConfig.rect.width, searchAreaConfig.rect.height, searchAreaConfig.scale);
+    }
+    return {
+        rect: searchAreaConfig?.rect,
+        imageBase64: searchAreaConfig?.imageBase64,
+        scale: searchAreaConfig?.scale,
+        error: result.content.error,
+        rawResponse: JSON.stringify(result.content),
+        usage: result.usage
+    };
+}
+async function AiExtractElementInfo(options) {
+    const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
+    const systemPrompt = systemPromptToExtract();
+    const screenshotBase64 = context.screenshot.base64;
+    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery);
+    const userContent = [];
+    if (extractOption?.screenshotIncluded !== false) userContent.push({
+        type: 'image_url',
+        image_url: {
+            url: screenshotBase64,
+            detail: 'high'
+        }
+    });
+    userContent.push({
+        type: 'text',
+        text: extractDataPromptText
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userContent
+        }
+    ];
+    if (multimodalPrompt) {
+        const addOns = await promptsToChatParam({
+            images: multimodalPrompt.images,
+            convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const { content: rawResponse, usage, reasoning_content } = await callAI(msgs, modelConfig);
+    let parseResult;
+    try {
+        parseResult = parseXMLExtractionResponse(rawResponse);
+    } catch (parseError) {
+        const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
+        throw new AIResponseParseError(`XML parse error: ${errorMessage}`, rawResponse, usage);
+    }
+    return {
+        parseResult,
+        rawResponse,
+        usage,
+        reasoning_content
+    };
+}
+async function AiJudgeOrderSensitive(description, callAIFn, modelConfig) {
+    const systemPrompt = systemPromptToJudgeOrderSensitive();
+    const userPrompt = orderSensitiveJudgePrompt(description);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userPrompt
+        }
+    ];
+    const result = await callAIFn(msgs, modelConfig);
+    return {
+        isOrderSensitive: result.content.isOrderSensitive ?? false,
+        usage: result.usage
+    };
+}
+export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, buildSearchAreaConfig };
+
+//# sourceMappingURL=inspect.mjs.map