@midscene/core 1.13.1 → 1.13.2-beta-20260923031905.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/es/agent/task-builder.mjs +1 -1
- package/dist/es/agent/task-builder.mjs.map +1 -1
- package/dist/es/agent/tasks.mjs +4 -4
- package/dist/es/agent/tasks.mjs.map +1 -1
- package/dist/es/agent/utils.mjs +1 -1
- package/dist/es/ai-model/connectivity/run-connectivity-test.mjs +2 -5
- package/dist/es/ai-model/connectivity/run-connectivity-test.mjs.map +1 -1
- package/dist/es/ai-model/model-adapter/chat-completion.mjs +8 -11
- package/dist/es/ai-model/model-adapter/chat-completion.mjs.map +1 -1
- package/dist/es/ai-model/model-adapter/resolve.mjs +10 -0
- package/dist/es/ai-model/model-adapter/resolve.mjs.map +1 -1
- package/dist/es/ai-model/model-adapter/responses.mjs +38 -0
- package/dist/es/ai-model/model-adapter/responses.mjs.map +1 -0
- package/dist/es/ai-model/model-adapter/temperature-with-semantic-retry.mjs +7 -0
- package/dist/es/ai-model/model-adapter/temperature-with-semantic-retry.mjs.map +1 -0
- package/dist/es/ai-model/models/gpt.mjs +73 -9
- package/dist/es/ai-model/models/gpt.mjs.map +1 -1
- package/dist/es/ai-model/models/registry.mjs +2 -2
- package/dist/es/ai-model/models/registry.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/call-ai.mjs +22 -13
- package/dist/es/ai-model/service-caller/call-ai.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/codex/call-codex.mjs +10 -8
- package/dist/es/ai-model/service-caller/codex/call-codex.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/codex/codex-app-server.mjs +29 -28
- package/dist/es/ai-model/service-caller/codex/codex-app-server.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/index.mjs +1 -2
- package/dist/es/ai-model/service-caller/model-call.mjs +4 -4
- package/dist/es/ai-model/service-caller/model-call.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/openai/call-openai.mjs +13 -8
- package/dist/es/ai-model/service-caller/openai/call-openai.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/openai/chat-completion/chat-completion.mjs +11 -7
- package/dist/es/ai-model/service-caller/openai/chat-completion/chat-completion.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/openai/chat-completion/non-stream.mjs +9 -9
- package/dist/es/ai-model/service-caller/openai/chat-completion/non-stream.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/openai/chat-completion/stream.mjs +4 -8
- package/dist/es/ai-model/service-caller/openai/chat-completion/stream.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/openai/chat-completion/utils.mjs +28 -1
- package/dist/es/ai-model/service-caller/openai/chat-completion/utils.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/openai/openai-client.mjs +3 -3
- package/dist/es/ai-model/service-caller/openai/openai-client.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/openai/responses/non-stream.mjs +21 -0
- package/dist/es/ai-model/service-caller/openai/responses/non-stream.mjs.map +1 -0
- package/dist/es/ai-model/service-caller/openai/responses/responses.mjs +57 -0
- package/dist/es/ai-model/service-caller/openai/responses/responses.mjs.map +1 -0
- package/dist/es/ai-model/service-caller/openai/responses/stream.mjs +56 -0
- package/dist/es/ai-model/service-caller/openai/responses/stream.mjs.map +1 -0
- package/dist/es/ai-model/service-caller/openai/responses/types.mjs +0 -0
- package/dist/es/ai-model/service-caller/openai/responses/utils.mjs +72 -0
- package/dist/es/ai-model/service-caller/openai/responses/utils.mjs.map +1 -0
- package/dist/es/ai-model/service-caller/semantic-retry.mjs.map +1 -1
- package/dist/es/ai-model/service-caller/utils.mjs +6 -25
- package/dist/es/ai-model/service-caller/utils.mjs.map +1 -1
- package/dist/es/ai-model/shared/multimodal-prompt.mjs +5 -8
- package/dist/es/ai-model/shared/multimodal-prompt.mjs.map +1 -1
- package/dist/es/ai-model/workflows/grounding/locate.mjs +6 -6
- package/dist/es/ai-model/workflows/grounding/locate.mjs.map +1 -1
- package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs +5 -5
- package/dist/es/ai-model/workflows/grounding/planning-action-locate.mjs.map +1 -1
- package/dist/es/ai-model/workflows/grounding/section-locate.mjs +5 -5
- package/dist/es/ai-model/workflows/grounding/section-locate.mjs.map +1 -1
- package/dist/es/ai-model/workflows/grounding/utils.mjs +4 -7
- package/dist/es/ai-model/workflows/grounding/utils.mjs.map +1 -1
- package/dist/es/ai-model/workflows/insight/extraction.mjs +9 -15
- package/dist/es/ai-model/workflows/insight/extraction.mjs.map +1 -1
- package/dist/es/ai-model/workflows/insight/order-sensitive.mjs.map +1 -1
- package/dist/es/ai-model/workflows/planning/conversation-history.mjs +17 -9
- package/dist/es/ai-model/workflows/planning/conversation-history.mjs.map +1 -1
- package/dist/es/ai-model/workflows/planning/custom-planning.mjs +11 -12
- package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map +1 -1
- package/dist/es/ai-model/workflows/planning/standard-planning.mjs +24 -26
- package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map +1 -1
- package/dist/es/ai-model/workflows/recorder-generation/common.mjs +2 -4
- package/dist/es/ai-model/workflows/recorder-generation/common.mjs.map +1 -1
- package/dist/es/ai-model/workflows/recorder-generation/markdown.mjs +3 -5
- package/dist/es/ai-model/workflows/recorder-generation/markdown.mjs.map +1 -1
- package/dist/es/ai-model/workflows/recorder-generation/metadata.mjs +2 -4
- package/dist/es/ai-model/workflows/recorder-generation/metadata.mjs.map +1 -1
- package/dist/es/ai-model/workflows/recorder-generation/playwright.mjs.map +1 -1
- package/dist/es/ai-model/workflows/recorder-generation/yaml.mjs +2 -4
- package/dist/es/ai-model/workflows/recorder-generation/yaml.mjs.map +1 -1
- package/dist/es/report-html-template.mjs +1 -1
- package/dist/es/service/index.mjs +17 -26
- package/dist/es/service/index.mjs.map +1 -1
- package/dist/es/types.mjs.map +1 -1
- package/dist/es/utils.mjs +1 -1
- package/dist/lib/agent/task-builder.js +1 -1
- package/dist/lib/agent/task-builder.js.map +1 -1
- package/dist/lib/agent/tasks.js +4 -4
- package/dist/lib/agent/tasks.js.map +1 -1
- package/dist/lib/agent/utils.js +1 -1
- package/dist/lib/ai-model/connectivity/run-connectivity-test.js +2 -5
- package/dist/lib/ai-model/connectivity/run-connectivity-test.js.map +1 -1
- package/dist/lib/ai-model/model-adapter/chat-completion.js +8 -11
- package/dist/lib/ai-model/model-adapter/chat-completion.js.map +1 -1
- package/dist/lib/ai-model/model-adapter/resolve.js +10 -0
- package/dist/lib/ai-model/model-adapter/resolve.js.map +1 -1
- package/dist/lib/ai-model/model-adapter/responses.js +75 -0
- package/dist/lib/ai-model/model-adapter/responses.js.map +1 -0
- package/dist/lib/ai-model/model-adapter/temperature-with-semantic-retry.js +41 -0
- package/dist/lib/ai-model/model-adapter/temperature-with-semantic-retry.js.map +1 -0
- package/dist/lib/ai-model/models/gpt.js +73 -9
- package/dist/lib/ai-model/models/gpt.js.map +1 -1
- package/dist/lib/ai-model/models/registry.js +2 -2
- package/dist/lib/ai-model/models/registry.js.map +1 -1
- package/dist/lib/ai-model/service-caller/call-ai.js +22 -13
- package/dist/lib/ai-model/service-caller/call-ai.js.map +1 -1
- package/dist/lib/ai-model/service-caller/codex/call-codex.js +10 -8
- package/dist/lib/ai-model/service-caller/codex/call-codex.js.map +1 -1
- package/dist/lib/ai-model/service-caller/codex/codex-app-server.js +29 -28
- package/dist/lib/ai-model/service-caller/codex/codex-app-server.js.map +1 -1
- package/dist/lib/ai-model/service-caller/index.js +0 -4
- package/dist/lib/ai-model/service-caller/model-call.js +4 -4
- package/dist/lib/ai-model/service-caller/model-call.js.map +1 -1
- package/dist/lib/ai-model/service-caller/openai/call-openai.js +12 -7
- package/dist/lib/ai-model/service-caller/openai/call-openai.js.map +1 -1
- package/dist/lib/ai-model/service-caller/openai/chat-completion/chat-completion.js +10 -6
- package/dist/lib/ai-model/service-caller/openai/chat-completion/chat-completion.js.map +1 -1
- package/dist/lib/ai-model/service-caller/openai/chat-completion/non-stream.js +9 -9
- package/dist/lib/ai-model/service-caller/openai/chat-completion/non-stream.js.map +1 -1
- package/dist/lib/ai-model/service-caller/openai/chat-completion/stream.js +4 -8
- package/dist/lib/ai-model/service-caller/openai/chat-completion/stream.js.map +1 -1
- package/dist/lib/ai-model/service-caller/openai/chat-completion/utils.js +32 -2
- package/dist/lib/ai-model/service-caller/openai/chat-completion/utils.js.map +1 -1
- package/dist/lib/ai-model/service-caller/openai/openai-client.js +5 -5
- package/dist/lib/ai-model/service-caller/openai/openai-client.js.map +1 -1
- package/dist/lib/ai-model/service-caller/openai/responses/non-stream.js +55 -0
- package/dist/lib/ai-model/service-caller/openai/responses/non-stream.js.map +1 -0
- package/dist/lib/ai-model/service-caller/openai/responses/responses.js +94 -0
- package/dist/lib/ai-model/service-caller/openai/responses/responses.js.map +1 -0
- package/dist/lib/ai-model/service-caller/openai/responses/stream.js +90 -0
- package/dist/lib/ai-model/service-caller/openai/responses/stream.js.map +1 -0
- package/dist/lib/ai-model/service-caller/openai/responses/types.js +20 -0
- package/dist/lib/ai-model/service-caller/openai/responses/types.js.map +1 -0
- package/dist/lib/ai-model/service-caller/openai/responses/utils.js +112 -0
- package/dist/lib/ai-model/service-caller/openai/responses/utils.js.map +1 -0
- package/dist/lib/ai-model/service-caller/semantic-retry.js.map +1 -1
- package/dist/lib/ai-model/service-caller/utils.js +5 -27
- package/dist/lib/ai-model/service-caller/utils.js.map +1 -1
- package/dist/lib/ai-model/shared/multimodal-prompt.js +11 -14
- package/dist/lib/ai-model/shared/multimodal-prompt.js.map +1 -1
- package/dist/lib/ai-model/workflows/grounding/locate.js +6 -6
- package/dist/lib/ai-model/workflows/grounding/locate.js.map +1 -1
- package/dist/lib/ai-model/workflows/grounding/planning-action-locate.js +5 -5
- package/dist/lib/ai-model/workflows/grounding/planning-action-locate.js.map +1 -1
- package/dist/lib/ai-model/workflows/grounding/section-locate.js +5 -5
- package/dist/lib/ai-model/workflows/grounding/section-locate.js.map +1 -1
- package/dist/lib/ai-model/workflows/grounding/utils.js +3 -6
- package/dist/lib/ai-model/workflows/grounding/utils.js.map +1 -1
- package/dist/lib/ai-model/workflows/insight/extraction.js +8 -14
- package/dist/lib/ai-model/workflows/insight/extraction.js.map +1 -1
- package/dist/lib/ai-model/workflows/insight/order-sensitive.js.map +1 -1
- package/dist/lib/ai-model/workflows/planning/conversation-history.js +17 -9
- package/dist/lib/ai-model/workflows/planning/conversation-history.js.map +1 -1
- package/dist/lib/ai-model/workflows/planning/custom-planning.js +10 -11
- package/dist/lib/ai-model/workflows/planning/custom-planning.js.map +1 -1
- package/dist/lib/ai-model/workflows/planning/standard-planning.js +23 -25
- package/dist/lib/ai-model/workflows/planning/standard-planning.js.map +1 -1
- package/dist/lib/ai-model/workflows/recorder-generation/common.js +2 -4
- package/dist/lib/ai-model/workflows/recorder-generation/common.js.map +1 -1
- package/dist/lib/ai-model/workflows/recorder-generation/markdown.js +3 -5
- package/dist/lib/ai-model/workflows/recorder-generation/markdown.js.map +1 -1
- package/dist/lib/ai-model/workflows/recorder-generation/metadata.js +2 -4
- package/dist/lib/ai-model/workflows/recorder-generation/metadata.js.map +1 -1
- package/dist/lib/ai-model/workflows/recorder-generation/playwright.js.map +1 -1
- package/dist/lib/ai-model/workflows/recorder-generation/yaml.js +2 -4
- package/dist/lib/ai-model/workflows/recorder-generation/yaml.js.map +1 -1
- package/dist/lib/report-html-template.js +1 -1
- package/dist/lib/service/index.js +17 -26
- package/dist/lib/service/index.js.map +1 -1
- package/dist/lib/types.js.map +1 -1
- package/dist/lib/utils.js +1 -1
- package/dist/types/ai-model/connectivity/run-connectivity-test.d.ts.map +1 -1
- package/dist/types/ai-model/index.d.ts +1 -1
- package/dist/types/ai-model/index.d.ts.map +1 -1
- package/dist/types/ai-model/model-adapter/chat-completion.d.ts.map +1 -1
- package/dist/types/ai-model/model-adapter/resolve.d.ts +4 -1
- package/dist/types/ai-model/model-adapter/resolve.d.ts.map +1 -1
- package/dist/types/ai-model/model-adapter/responses.d.ts +4 -0
- package/dist/types/ai-model/model-adapter/responses.d.ts.map +1 -0
- package/dist/types/ai-model/model-adapter/temperature-with-semantic-retry.d.ts +6 -0
- package/dist/types/ai-model/model-adapter/temperature-with-semantic-retry.d.ts.map +1 -0
- package/dist/types/ai-model/model-adapter/types.d.ts +42 -13
- package/dist/types/ai-model/model-adapter/types.d.ts.map +1 -1
- package/dist/types/ai-model/models/gpt.d.ts +15 -3
- package/dist/types/ai-model/models/gpt.d.ts.map +1 -1
- package/dist/types/ai-model/models/registry.d.ts +14 -2
- package/dist/types/ai-model/models/registry.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/call-ai.d.ts +2 -2
- package/dist/types/ai-model/service-caller/call-ai.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/codex/call-codex.d.ts +4 -2
- package/dist/types/ai-model/service-caller/codex/call-codex.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/codex/codex-app-server.d.ts +5 -4
- package/dist/types/ai-model/service-caller/codex/codex-app-server.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/index.d.ts +0 -1
- package/dist/types/ai-model/service-caller/index.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/model-call.d.ts +6 -6
- package/dist/types/ai-model/service-caller/model-call.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/openai/call-openai.d.ts +9 -1
- package/dist/types/ai-model/service-caller/openai/call-openai.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/openai/chat-completion/chat-completion.d.ts +3 -4
- package/dist/types/ai-model/service-caller/openai/chat-completion/chat-completion.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/openai/chat-completion/non-stream.d.ts +1 -1
- package/dist/types/ai-model/service-caller/openai/chat-completion/non-stream.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/openai/chat-completion/stream.d.ts +1 -1
- package/dist/types/ai-model/service-caller/openai/chat-completion/stream.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/openai/chat-completion/types.d.ts +5 -5
- package/dist/types/ai-model/service-caller/openai/chat-completion/types.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/openai/chat-completion/utils.d.ts +5 -0
- package/dist/types/ai-model/service-caller/openai/chat-completion/utils.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/openai/openai-client.d.ts +2 -2
- package/dist/types/ai-model/service-caller/openai/openai-client.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/openai/responses/non-stream.d.ts +4 -0
- package/dist/types/ai-model/service-caller/openai/responses/non-stream.d.ts.map +1 -0
- package/dist/types/ai-model/service-caller/openai/responses/responses.d.ts +8 -0
- package/dist/types/ai-model/service-caller/openai/responses/responses.d.ts.map +1 -0
- package/dist/types/ai-model/service-caller/openai/responses/stream.d.ts +4 -0
- package/dist/types/ai-model/service-caller/openai/responses/stream.d.ts.map +1 -0
- package/dist/types/ai-model/service-caller/openai/responses/types.d.ts +8 -0
- package/dist/types/ai-model/service-caller/openai/responses/types.d.ts.map +1 -0
- package/dist/types/ai-model/service-caller/openai/responses/utils.d.ts +8 -0
- package/dist/types/ai-model/service-caller/openai/responses/utils.d.ts.map +1 -0
- package/dist/types/ai-model/service-caller/semantic-retry.d.ts +2 -2
- package/dist/types/ai-model/service-caller/semantic-retry.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/types.d.ts +35 -5
- package/dist/types/ai-model/service-caller/types.d.ts.map +1 -1
- package/dist/types/ai-model/service-caller/utils.d.ts +5 -9
- package/dist/types/ai-model/service-caller/utils.d.ts.map +1 -1
- package/dist/types/ai-model/shared/multimodal-prompt.d.ts +3 -3
- package/dist/types/ai-model/shared/multimodal-prompt.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/grounding/planning-action-locate.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/grounding/section-locate.d.ts +2 -1
- package/dist/types/ai-model/workflows/grounding/section-locate.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/grounding/types.d.ts +3 -2
- package/dist/types/ai-model/workflows/grounding/types.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/grounding/utils.d.ts +3 -3
- package/dist/types/ai-model/workflows/grounding/utils.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/insight/extraction.d.ts +1 -1
- package/dist/types/ai-model/workflows/insight/extraction.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/insight/order-sensitive.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/planning/conversation-history.d.ts +10 -8
- package/dist/types/ai-model/workflows/planning/conversation-history.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/planning/custom-planning.d.ts +2 -2
- package/dist/types/ai-model/workflows/planning/custom-planning.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/planning/standard-planning.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/recorder-generation/common.d.ts +2 -1
- package/dist/types/ai-model/workflows/recorder-generation/common.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/recorder-generation/markdown.d.ts +2 -2
- package/dist/types/ai-model/workflows/recorder-generation/markdown.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/recorder-generation/metadata.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/recorder-generation/playwright.d.ts.map +1 -1
- package/dist/types/ai-model/workflows/recorder-generation/yaml.d.ts.map +1 -1
- package/dist/types/service/index.d.ts.map +1 -1
- package/dist/types/types.d.ts +11 -2
- package/dist/types/types.d.ts.map +1 -1
- package/package.json +2 -2
|
@@ -16,14 +16,21 @@ class ConversationHistory {
|
|
|
16
16
|
resetPendingFeedbackMessageIfExists() {
|
|
17
17
|
if (this.pendingFeedbackMessage) this.pendingFeedbackMessage = '';
|
|
18
18
|
}
|
|
19
|
-
|
|
20
|
-
this.messages.push(
|
|
19
|
+
appendMessage(message) {
|
|
20
|
+
this.messages.push({
|
|
21
|
+
type: 'input-message',
|
|
22
|
+
message
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
appendModelOutput(output) {
|
|
26
|
+
this.messages.push({
|
|
27
|
+
type: 'model-output',
|
|
28
|
+
output
|
|
29
|
+
});
|
|
21
30
|
}
|
|
22
31
|
seed(messages) {
|
|
23
32
|
this.reset();
|
|
24
|
-
messages.
|
|
25
|
-
this.append(message);
|
|
26
|
-
});
|
|
33
|
+
this.messages.push(...messages);
|
|
27
34
|
}
|
|
28
35
|
reset() {
|
|
29
36
|
this.messages.length = 0;
|
|
@@ -39,11 +46,12 @@ class ConversationHistory {
|
|
|
39
46
|
const clonedMessages = structuredClone(this.messages);
|
|
40
47
|
let imageCount = 0;
|
|
41
48
|
for(let i = clonedMessages.length - 1; i >= 0; i--){
|
|
42
|
-
const
|
|
43
|
-
|
|
49
|
+
const entry = clonedMessages[i];
|
|
50
|
+
if ('input-message' !== entry.type) continue;
|
|
51
|
+
const content = entry.message.content;
|
|
44
52
|
if (Array.isArray(content)) for(let j = 0; j < content.length; j++){
|
|
45
53
|
const item = content[j];
|
|
46
|
-
if ('
|
|
54
|
+
if ('image' === item.type) {
|
|
47
55
|
imageCount++;
|
|
48
56
|
if (imageCount > maxImages) content[j] = {
|
|
49
57
|
type: 'text',
|
|
@@ -166,7 +174,7 @@ class ConversationHistory {
|
|
|
166
174
|
};
|
|
167
175
|
const recentMessages = this.messages.slice(-keepCount);
|
|
168
176
|
this.messages.length = 0;
|
|
169
|
-
this.
|
|
177
|
+
this.appendMessage(omittedPlaceholder);
|
|
170
178
|
for (const msg of recentMessages)this.messages.push(msg);
|
|
171
179
|
return true;
|
|
172
180
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai-model/workflows/planning/conversation-history.mjs","sources":["../../../../../src/ai-model/workflows/planning/conversation-history.ts"],"sourcesContent":["import type { SubGoal } from '@/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildSubGoalsText } from '../../prompt/planning/sub-goals-text';\n\nexport interface ConversationHistoryOptions {\n initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n private readonly messages: ChatCompletionMessageParam[] = [];\n private subGoals: SubGoal[] = [];\n private memories: string[] = [];\n private historicalLogs: string[] = [];\n\n public pendingFeedbackMessage: string;\n\n constructor(options?: ConversationHistoryOptions) {\n if (options?.initialMessages?.length) {\n this.seed(options.initialMessages);\n }\n this.pendingFeedbackMessage = '';\n }\n\n resetPendingFeedbackMessageIfExists() {\n if (this.pendingFeedbackMessage) {\n this.pendingFeedbackMessage = '';\n }\n }\n\n append(message: ChatCompletionMessageParam) {\n this.messages.push(message);\n }\n\n seed(messages: ChatCompletionMessageParam[]) {\n this.reset();\n messages.forEach((message) => {\n this.append(message);\n });\n }\n\n reset() {\n this.messages.length = 0;\n this.memories.length = 0;\n this.subGoals.length = 0;\n this.historicalLogs.length = 0;\n this.pendingFeedbackMessage = '';\n }\n\n /**\n * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n * @returns The snapshot of the conversation history.\n */\n snapshot(maxImages?: number): ChatCompletionMessageParam[] {\n if (maxImages === undefined) {\n return [...this.messages];\n }\n\n const clonedMessages = structuredClone(this.messages);\n let imageCount = 0;\n\n // Traverse from the end to the beginning\n for (let i = clonedMessages.length - 1; i >= 0; i--) {\n const message = clonedMessages[i];\n const content = message.content;\n\n // Only process if content is an array\n if (Array.isArray(content)) {\n for (let j = 0; j < content.length; j++) {\n const item = content[j];\n\n // Check if this is an image\n if (item.type === 'image_url') {\n imageCount++;\n\n // If we've exceeded the limit, replace with text\n if (imageCount > maxImages) {\n content[j] = {\n type: 'text',\n text: '(image ignored due to size optimization)',\n };\n }\n }\n }\n }\n }\n\n return clonedMessages;\n }\n\n get length(): number {\n return this.messages.length;\n }\n\n [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n return this.messages[Symbol.iterator]();\n }\n\n toJSON(): ChatCompletionMessageParam[] {\n return this.snapshot();\n }\n\n // Planning sub-goal management methods\n\n /**\n * Set all sub-goals, replacing any existing ones.\n * Automatically marks the first pending goal as running.\n */\n setSubGoals(subGoals: SubGoal[]): void {\n this.subGoals = subGoals.map((goal) => ({ ...goal }));\n this.markFirstPendingAsRunning();\n }\n\n /**\n * Merge sub-goals from update-plan-content.\n * Preserves existing descriptions when incoming description is empty.\n *\n * This handles compact XML updates like:\n * <sub-goal index=\"1\" status=\"finished\" />\n */\n mergeSubGoals(subGoals: SubGoal[]): void {\n if (this.subGoals.length === 0) {\n this.setSubGoals(subGoals);\n return;\n }\n\n const existingByIndex = new Map(\n this.subGoals.map((goal) => [goal.index, goal] as const),\n );\n\n const mergedSubGoals = subGoals.map((goal) => {\n const existingGoal = existingByIndex.get(goal.index);\n const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n if (!existingGoal && !hasNonEmptyDescription) {\n return null;\n }\n\n return {\n ...goal,\n description:\n hasNonEmptyDescription || !existingGoal\n ? goal.description\n : existingGoal.description,\n };\n });\n\n const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n if (validSubGoals.length === 0) {\n return;\n }\n\n this.setSubGoals(validSubGoals);\n }\n\n /**\n * Update a single sub-goal by index.\n * Clears logs if status or description actually changes.\n * @returns true if the sub-goal was found and updated, false otherwise\n */\n updateSubGoal(\n index: number,\n updates: Partial<Omit<SubGoal, 'index'>>,\n ): boolean {\n const goal = this.subGoals.find((g) => g.index === index);\n if (!goal) {\n return false;\n }\n\n let changed = false;\n\n if (updates.status !== undefined && updates.status !== goal.status) {\n goal.status = updates.status;\n changed = true;\n }\n if (\n updates.description !== undefined &&\n updates.description !== goal.description\n ) {\n goal.description = updates.description;\n changed = true;\n }\n\n if (changed) {\n goal.logs = [];\n }\n\n return true;\n }\n\n /**\n * Mark the first pending sub-goal as running.\n * Clears logs since status changes.\n */\n markFirstPendingAsRunning(): void {\n const firstPending = this.subGoals.find((g) => g.status === 'pending');\n if (firstPending) {\n firstPending.status = 'running';\n firstPending.logs = [];\n }\n }\n\n /**\n * Mark a sub-goal as finished.\n * Automatically marks the next pending goal as running.\n * @returns true if the sub-goal was found and updated, false otherwise\n */\n markSubGoalFinished(index: number): boolean {\n const result = this.updateSubGoal(index, { status: 'finished' });\n if (result) {\n this.markFirstPendingAsRunning();\n }\n return result;\n }\n\n /**\n * Mark all sub-goals as finished.\n * Clears logs for any goal whose status actually changes.\n */\n markAllSubGoalsFinished(): void {\n for (const goal of this.subGoals) {\n if (goal.status !== 'finished') {\n goal.logs = [];\n }\n goal.status = 'finished';\n }\n }\n\n /**\n * Append a log entry to the currently running sub-goal.\n * The log describes an action performed while working on the sub-goal.\n */\n appendSubGoalLog(log: string): void {\n if (!log) {\n return;\n }\n const runningGoal = this.subGoals.find((g) => g.status === 'running');\n if (runningGoal) {\n if (!runningGoal.logs) {\n runningGoal.logs = [];\n }\n runningGoal.logs.push(log);\n }\n }\n\n /**\n * Convert sub-goals to text representation.\n * Includes actions performed (logs) for the current sub-goal.\n */\n subGoalsToText(): string {\n return buildSubGoalsText(this.subGoals);\n }\n\n // Historical log management methods (used in non-deepThink mode)\n\n /**\n * Append a log entry to the historical logs list.\n * Used in non-deepThink mode to track executed steps across planning rounds.\n */\n appendHistoricalLog(log: string): void {\n if (log) {\n this.historicalLogs.push(log);\n }\n }\n\n /**\n * Convert historical logs to text representation.\n * Provides context about previously executed steps to the model.\n */\n historicalLogsToText(): string {\n if (this.historicalLogs.length === 0) {\n return '';\n }\n\n const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n return `Here are the steps that have been executed:\\n${logLines}`;\n }\n\n // Memory management methods\n\n /**\n * Append a memory to the memories list\n */\n appendMemory(memory: string): void {\n if (memory) {\n this.memories.push(memory);\n }\n }\n\n /**\n * Get all memories\n */\n getMemories(): string[] {\n return [...this.memories];\n }\n\n /**\n * Convert memories to text representation\n */\n memoriesToText(): string {\n if (this.memories.length === 0) {\n return '';\n }\n\n return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n }\n\n /**\n * Clear all memories\n */\n clearMemories(): void {\n this.memories.length = 0;\n }\n\n /**\n * Compress the conversation history if it exceeds the threshold.\n * Removes the oldest messages and replaces them with a single placeholder message.\n * @param threshold - The number of messages that triggers compression.\n * @param keepCount - The number of recent messages to keep after compression.\n * @returns true if compression was performed, false otherwise.\n */\n compressHistory(threshold: number, keepCount: number): boolean {\n if (this.messages.length <= threshold) {\n return false;\n }\n\n const omittedCount = this.messages.length - keepCount;\n const omittedPlaceholder: ChatCompletionMessageParam = {\n role: 'user',\n content: `(${omittedCount} previous conversation messages have been omitted)`,\n };\n\n // Keep only the last `keepCount` messages\n const recentMessages = this.messages.slice(-keepCount);\n\n // Reset and rebuild with placeholder + recent messages\n this.messages.length = 0;\n this.messages.push(omittedPlaceholder);\n for (const msg of recentMessages) {\n this.messages.push(msg);\n }\n\n return true;\n }\n}\n"],"names":["Symbol","ConversationHistory","message","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","buildSubGoalsText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options"],"mappings":";;;;;;;;;;;;eA8FGA,OAAO,QAAQ;;AAtFX,MAAMC;IAeX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASE,SAAkB,EAAgC;QACzD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMP,UAAUI,cAAc,CAACG,EAAE;YACjC,MAAMC,UAAUR,QAAQ,OAAO;YAG/B,IAAIS,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,gBAAdA,KAAK,IAAI,EAAkB;oBAC7BL;oBAGA,IAAIA,aAAaJ,WACfM,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAON;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACN,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYc,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBlB,WAAnBkB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBlB,WAAxBkB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,OAAOE,kBAAkB,IAAI,CAAC,QAAQ;IACxC;IAQA,oBAAoBF,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMG,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEG,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAAiD;YACrD,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IAvUA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAyC,EAAE;QAC5D,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAmUF"}
|
|
1
|
+
{"version":3,"file":"ai-model/workflows/planning/conversation-history.mjs","sources":["../../../../../src/ai-model/workflows/planning/conversation-history.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type { RawAssistantOutput, SubGoal } from '@/types';\nimport { buildSubGoalsText } from '../../prompt/planning/sub-goals-text';\nimport type { ConversationEntry } from '../../service-caller/types';\n\nexport interface ConversationHistoryOptions {\n initialMessages?: ConversationEntry[];\n}\n\nexport class ConversationHistory {\n private readonly messages: ConversationEntry[] = [];\n private subGoals: SubGoal[] = [];\n private memories: string[] = [];\n private historicalLogs: string[] = [];\n\n public pendingFeedbackMessage: string;\n\n constructor(options?: ConversationHistoryOptions) {\n if (options?.initialMessages?.length) {\n this.seed(options.initialMessages);\n }\n this.pendingFeedbackMessage = '';\n }\n\n resetPendingFeedbackMessageIfExists() {\n if (this.pendingFeedbackMessage) {\n this.pendingFeedbackMessage = '';\n }\n }\n\n appendMessage(message: ConversationMessage) {\n this.messages.push({ type: 'input-message', message });\n }\n\n appendModelOutput(output: RawAssistantOutput) {\n this.messages.push({ type: 'model-output', output });\n }\n\n seed(messages: ConversationEntry[]) {\n this.reset();\n this.messages.push(...messages);\n }\n\n reset() {\n this.messages.length = 0;\n this.memories.length = 0;\n this.subGoals.length = 0;\n this.historicalLogs.length = 0;\n this.pendingFeedbackMessage = '';\n }\n\n /**\n * Snapshot the conversation history, and replace the images with text if the number of images exceeds the limit.\n * @param maxImages - The maximum number of images to include in the snapshot. Undefined means no limit.\n * @returns The snapshot of the conversation history.\n */\n snapshot(maxImages?: number): ConversationEntry[] {\n if (maxImages === undefined) {\n return [...this.messages];\n }\n\n const clonedMessages = structuredClone(this.messages);\n let imageCount = 0;\n\n // Traverse from the end to the beginning\n for (let i = clonedMessages.length - 1; i >= 0; i--) {\n const entry = clonedMessages[i];\n if (entry.type !== 'input-message') {\n continue;\n }\n const content = entry.message.content;\n\n // Only process if content is an array\n if (Array.isArray(content)) {\n for (let j = 0; j < content.length; j++) {\n const item = content[j];\n\n // Check if this is an image\n if (item.type === 'image') {\n imageCount++;\n\n // If we've exceeded the limit, replace with text\n if (imageCount > maxImages) {\n content[j] = {\n type: 'text',\n text: '(image ignored due to size optimization)',\n };\n }\n }\n }\n }\n }\n\n return clonedMessages;\n }\n\n get length(): number {\n return this.messages.length;\n }\n\n [Symbol.iterator](): IterableIterator<ConversationEntry> {\n return this.messages[Symbol.iterator]();\n }\n\n toJSON(): ConversationEntry[] {\n return this.snapshot();\n }\n\n // Planning sub-goal management methods\n\n /**\n * Set all sub-goals, replacing any existing ones.\n * Automatically marks the first pending goal as running.\n */\n setSubGoals(subGoals: SubGoal[]): void {\n this.subGoals = subGoals.map((goal) => ({ ...goal }));\n this.markFirstPendingAsRunning();\n }\n\n /**\n * Merge sub-goals from update-plan-content.\n * Preserves existing descriptions when incoming description is empty.\n *\n * This handles compact XML updates like:\n * <sub-goal index=\"1\" status=\"finished\" />\n */\n mergeSubGoals(subGoals: SubGoal[]): void {\n if (this.subGoals.length === 0) {\n this.setSubGoals(subGoals);\n return;\n }\n\n const existingByIndex = new Map(\n this.subGoals.map((goal) => [goal.index, goal] as const),\n );\n\n const mergedSubGoals = subGoals.map((goal) => {\n const existingGoal = existingByIndex.get(goal.index);\n const hasNonEmptyDescription = goal.description.trim().length > 0;\n\n if (!existingGoal && !hasNonEmptyDescription) {\n return null;\n }\n\n return {\n ...goal,\n description:\n hasNonEmptyDescription || !existingGoal\n ? goal.description\n : existingGoal.description,\n };\n });\n\n const validSubGoals = mergedSubGoals.filter((goal) => goal !== null);\n if (validSubGoals.length === 0) {\n return;\n }\n\n this.setSubGoals(validSubGoals);\n }\n\n /**\n * Update a single sub-goal by index.\n * Clears logs if status or description actually changes.\n * @returns true if the sub-goal was found and updated, false otherwise\n */\n updateSubGoal(\n index: number,\n updates: Partial<Omit<SubGoal, 'index'>>,\n ): boolean {\n const goal = this.subGoals.find((g) => g.index === index);\n if (!goal) {\n return false;\n }\n\n let changed = false;\n\n if (updates.status !== undefined && updates.status !== goal.status) {\n goal.status = updates.status;\n changed = true;\n }\n if (\n updates.description !== undefined &&\n updates.description !== goal.description\n ) {\n goal.description = updates.description;\n changed = true;\n }\n\n if (changed) {\n goal.logs = [];\n }\n\n return true;\n }\n\n /**\n * Mark the first pending sub-goal as running.\n * Clears logs since status changes.\n */\n markFirstPendingAsRunning(): void {\n const firstPending = this.subGoals.find((g) => g.status === 'pending');\n if (firstPending) {\n firstPending.status = 'running';\n firstPending.logs = [];\n }\n }\n\n /**\n * Mark a sub-goal as finished.\n * Automatically marks the next pending goal as running.\n * @returns true if the sub-goal was found and updated, false otherwise\n */\n markSubGoalFinished(index: number): boolean {\n const result = this.updateSubGoal(index, { status: 'finished' });\n if (result) {\n this.markFirstPendingAsRunning();\n }\n return result;\n }\n\n /**\n * Mark all sub-goals as finished.\n * Clears logs for any goal whose status actually changes.\n */\n markAllSubGoalsFinished(): void {\n for (const goal of this.subGoals) {\n if (goal.status !== 'finished') {\n goal.logs = [];\n }\n goal.status = 'finished';\n }\n }\n\n /**\n * Append a log entry to the currently running sub-goal.\n * The log describes an action performed while working on the sub-goal.\n */\n appendSubGoalLog(log: string): void {\n if (!log) {\n return;\n }\n const runningGoal = this.subGoals.find((g) => g.status === 'running');\n if (runningGoal) {\n if (!runningGoal.logs) {\n runningGoal.logs = [];\n }\n runningGoal.logs.push(log);\n }\n }\n\n /**\n * Convert sub-goals to text representation.\n * Includes actions performed (logs) for the current sub-goal.\n */\n subGoalsToText(): string {\n return buildSubGoalsText(this.subGoals);\n }\n\n // Historical log management methods (used in non-deepThink mode)\n\n /**\n * Append a log entry to the historical logs list.\n * Used in non-deepThink mode to track executed steps across planning rounds.\n */\n appendHistoricalLog(log: string): void {\n if (log) {\n this.historicalLogs.push(log);\n }\n }\n\n /**\n * Convert historical logs to text representation.\n * Provides context about previously executed steps to the model.\n */\n historicalLogsToText(): string {\n if (this.historicalLogs.length === 0) {\n return '';\n }\n\n const logLines = this.historicalLogs.map((log) => `- ${log}`).join('\\n');\n return `Here are the steps that have been executed:\\n${logLines}`;\n }\n\n // Memory management methods\n\n /**\n * Append a memory to the memories list\n */\n appendMemory(memory: string): void {\n if (memory) {\n this.memories.push(memory);\n }\n }\n\n /**\n * Get all memories\n */\n getMemories(): string[] {\n return [...this.memories];\n }\n\n /**\n * Convert memories to text representation\n */\n memoriesToText(): string {\n if (this.memories.length === 0) {\n return '';\n }\n\n return `Memories from previous steps:\\n---\\n${this.memories.join('\\n---\\n')}\\n`;\n }\n\n /**\n * Clear all memories\n */\n clearMemories(): void {\n this.memories.length = 0;\n }\n\n /**\n * Compress the conversation history if it exceeds the threshold.\n * Removes the oldest messages and replaces them with a single placeholder message.\n * @param threshold - The number of messages that triggers compression.\n * @param keepCount - The number of recent messages to keep after compression.\n * @returns true if compression was performed, false otherwise.\n */\n compressHistory(threshold: number, keepCount: number): boolean {\n if (this.messages.length <= threshold) {\n return false;\n }\n\n const omittedCount = this.messages.length - keepCount;\n const omittedPlaceholder: ConversationMessage = {\n role: 'user',\n content: `(${omittedCount} previous conversation messages have been omitted)`,\n };\n\n // Keep only the last `keepCount` messages\n const recentMessages = this.messages.slice(-keepCount);\n\n // Reset and rebuild with placeholder + recent messages\n this.messages.length = 0;\n this.appendMessage(omittedPlaceholder);\n for (const msg of recentMessages) {\n this.messages.push(msg);\n }\n\n return true;\n }\n}\n"],"names":["Symbol","ConversationHistory","message","output","messages","maxImages","undefined","clonedMessages","structuredClone","imageCount","i","entry","content","Array","j","item","subGoals","goal","existingByIndex","Map","mergedSubGoals","existingGoal","hasNonEmptyDescription","validSubGoals","index","updates","g","changed","firstPending","result","log","runningGoal","buildSubGoalsText","logLines","memory","threshold","keepCount","omittedCount","omittedPlaceholder","recentMessages","msg","options"],"mappings":";;;;;;;;;;;;eAoGGA,OAAO,QAAQ;;AA3FX,MAAMC;IAeX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,cAAcC,OAA4B,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM;YAAiBA;QAAQ;IACtD;IAEA,kBAAkBC,MAA0B,EAAE;QAC5C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM;YAAgBA;QAAO;IACpD;IAEA,KAAKC,QAA6B,EAAE;QAClC,IAAI,CAAC,KAAK;QACV,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAIA;IACxB;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,cAAc,CAAC,MAAM,GAAG;QAC7B,IAAI,CAAC,sBAAsB,GAAG;IAChC;IAOA,SAASC,SAAkB,EAAuB;QAChD,IAAIA,AAAcC,WAAdD,WACF,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;QAG3B,MAAME,iBAAiBC,gBAAgB,IAAI,CAAC,QAAQ;QACpD,IAAIC,aAAa;QAGjB,IAAK,IAAIC,IAAIH,eAAe,MAAM,GAAG,GAAGG,KAAK,GAAGA,IAAK;YACnD,MAAMC,QAAQJ,cAAc,CAACG,EAAE;YAC/B,IAAIC,AAAe,oBAAfA,MAAM,IAAI,EACZ;YAEF,MAAMC,UAAUD,MAAM,OAAO,CAAC,OAAO;YAGrC,IAAIE,MAAM,OAAO,CAACD,UAChB,IAAK,IAAIE,IAAI,GAAGA,IAAIF,QAAQ,MAAM,EAAEE,IAAK;gBACvC,MAAMC,OAAOH,OAAO,CAACE,EAAE;gBAGvB,IAAIC,AAAc,YAAdA,KAAK,IAAI,EAAc;oBACzBN;oBAGA,IAAIA,aAAaJ,WACfO,OAAO,CAACE,EAAE,GAAG;wBACX,MAAM;wBACN,MAAM;oBACR;gBAEJ;YACF;QAEJ;QAEA,OAAOP;IACT;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAyD;QACvD,OAAO,IAAI,CAAC,QAAQ,CAACP,OAAO,QAAQ,CAAC;IACvC;IAEA,SAA8B;QAC5B,OAAO,IAAI,CAAC,QAAQ;IACtB;IAQA,YAAYgB,QAAmB,EAAQ;QACrC,IAAI,CAAC,QAAQ,GAAGA,SAAS,GAAG,CAAC,CAACC,OAAU;gBAAE,GAAGA,IAAI;YAAC;QAClD,IAAI,CAAC,yBAAyB;IAChC;IASA,cAAcD,QAAmB,EAAQ;QACvC,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAQ,YAC9B,IAAI,CAAC,WAAW,CAACA;QAInB,MAAME,kBAAkB,IAAIC,IAC1B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAACF,OAAS;gBAACA,KAAK,KAAK;gBAAEA;aAAK;QAGhD,MAAMG,iBAAiBJ,SAAS,GAAG,CAAC,CAACC;YACnC,MAAMI,eAAeH,gBAAgB,GAAG,CAACD,KAAK,KAAK;YACnD,MAAMK,yBAAyBL,KAAK,WAAW,CAAC,IAAI,GAAG,MAAM,GAAG;YAEhE,IAAI,CAACI,gBAAgB,CAACC,wBACpB,OAAO;YAGT,OAAO;gBACL,GAAGL,IAAI;gBACP,aACEK,0BAA0B,CAACD,eACvBJ,KAAK,WAAW,GAChBI,aAAa,WAAW;YAChC;QACF;QAEA,MAAME,gBAAgBH,eAAe,MAAM,CAAC,CAACH,OAASA,AAAS,SAATA;QACtD,IAAIM,AAAyB,MAAzBA,cAAc,MAAM,EACtB;QAGF,IAAI,CAAC,WAAW,CAACA;IACnB;IAOA,cACEC,KAAa,EACbC,OAAwC,EAC/B;QACT,MAAMR,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACS,IAAMA,EAAE,KAAK,KAAKF;QACnD,IAAI,CAACP,MACH,OAAO;QAGT,IAAIU,UAAU;QAEd,IAAIF,AAAmBnB,WAAnBmB,QAAQ,MAAM,IAAkBA,QAAQ,MAAM,KAAKR,KAAK,MAAM,EAAE;YAClEA,KAAK,MAAM,GAAGQ,QAAQ,MAAM;YAC5BE,UAAU;QACZ;QACA,IACEF,AAAwBnB,WAAxBmB,QAAQ,WAAW,IACnBA,QAAQ,WAAW,KAAKR,KAAK,WAAW,EACxC;YACAA,KAAK,WAAW,GAAGQ,QAAQ,WAAW;YACtCE,UAAU;QACZ;QAEA,IAAIA,SACFV,KAAK,IAAI,GAAG,EAAE;QAGhB,OAAO;IACT;IAMA,4BAAkC;QAChC,MAAMW,eAAe,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACF,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACvD,IAAIE,cAAc;YAChBA,aAAa,MAAM,GAAG;YACtBA,aAAa,IAAI,GAAG,EAAE;QACxB;IACF;IAOA,oBAAoBJ,KAAa,EAAW;QAC1C,MAAMK,SAAS,IAAI,CAAC,aAAa,CAACL,OAAO;YAAE,QAAQ;QAAW;QAC9D,IAAIK,QACF,IAAI,CAAC,yBAAyB;QAEhC,OAAOA;IACT;IAMA,0BAAgC;QAC9B,KAAK,MAAMZ,QAAQ,IAAI,CAAC,QAAQ,CAAE;YAChC,IAAIA,AAAgB,eAAhBA,KAAK,MAAM,EACbA,KAAK,IAAI,GAAG,EAAE;YAEhBA,KAAK,MAAM,GAAG;QAChB;IACF;IAMA,iBAAiBa,GAAW,EAAQ;QAClC,IAAI,CAACA,KACH;QAEF,MAAMC,cAAc,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAACL,IAAMA,AAAa,cAAbA,EAAE,MAAM;QACtD,IAAIK,aAAa;YACf,IAAI,CAACA,YAAY,IAAI,EACnBA,YAAY,IAAI,GAAG,EAAE;YAEvBA,YAAY,IAAI,CAAC,IAAI,CAACD;QACxB;IACF;IAMA,iBAAyB;QACvB,OAAOE,kBAAkB,IAAI,CAAC,QAAQ;IACxC;IAQA,oBAAoBF,GAAW,EAAQ;QACrC,IAAIA,KACF,IAAI,CAAC,cAAc,CAAC,IAAI,CAACA;IAE7B;IAMA,uBAA+B;QAC7B,IAAI,AAA+B,MAA/B,IAAI,CAAC,cAAc,CAAC,MAAM,EAC5B,OAAO;QAGT,MAAMG,WAAW,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAACH,MAAQ,CAAC,EAAE,EAAEA,KAAK,EAAE,IAAI,CAAC;QACnE,OAAO,CAAC,6CAA6C,EAAEG,UAAU;IACnE;IAOA,aAAaC,MAAc,EAAQ;QACjC,IAAIA,QACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IAEvB;IAKA,cAAwB;QACtB,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAKA,iBAAyB;QACvB,IAAI,AAAyB,MAAzB,IAAI,CAAC,QAAQ,CAAC,MAAM,EACtB,OAAO;QAGT,OAAO,CAAC,oCAAoC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;IACjF;IAKA,gBAAsB;QACpB,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IASA,gBAAgBC,SAAiB,EAAEC,SAAiB,EAAW;QAC7D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,IAAID,WAC1B,OAAO;QAGT,MAAME,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAGD;QAC5C,MAAME,qBAA0C;YAC9C,MAAM;YACN,SAAS,CAAC,CAAC,EAAED,aAAa,kDAAkD,CAAC;QAC/E;QAGA,MAAME,iBAAiB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAACH;QAG5C,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;QACvB,IAAI,CAAC,aAAa,CAACE;QACnB,KAAK,MAAME,OAAOD,eAChB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACC;QAGrB,OAAO;IACT;IA5UA,YAAYC,OAAoC,CAAE;QAPlD,uBAAiB,YAAgC,EAAE;QACnD,uBAAQ,YAAsB,EAAE;QAChC,uBAAQ,YAAqB,EAAE;QAC/B,uBAAQ,kBAA2B,EAAE;QAErC,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAwUF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ScreenshotItem } from "../../../screenshot-item.mjs";
|
|
2
2
|
import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
|
|
3
3
|
import { AIResponseParseError, callAIWithStringResponse } from "../../service-caller/index.mjs";
|
|
4
|
-
import {
|
|
4
|
+
import { preparedReferenceImagesToMessages } from "../../shared/multimodal-prompt.mjs";
|
|
5
5
|
import { normalizePlanningActionLocateFields } from "./locate-normalization.mjs";
|
|
6
6
|
function appendActionContext(systemPrompt, actionContext) {
|
|
7
7
|
return systemPrompt + (actionContext ? `${actionContext}\n` : '');
|
|
@@ -12,9 +12,9 @@ function buildCustomPlanningMessages(input, config) {
|
|
|
12
12
|
const systemPrompt = appendActionContext(config.buildSystemPrompt(), actionContext);
|
|
13
13
|
const userInstructionText = input.userInstruction.text;
|
|
14
14
|
const userInstruction = config.buildUserInstruction ? config.buildUserInstruction(userInstructionText) : userInstructionText;
|
|
15
|
-
const referenceImageMessages =
|
|
15
|
+
const referenceImageMessages = preparedReferenceImagesToMessages(input.userInstruction.referenceImages);
|
|
16
16
|
if (conversationHistory.pendingFeedbackMessage) {
|
|
17
|
-
conversationHistory.
|
|
17
|
+
conversationHistory.appendMessage({
|
|
18
18
|
role: 'user',
|
|
19
19
|
content: [
|
|
20
20
|
{
|
|
@@ -25,14 +25,12 @@ function buildCustomPlanningMessages(input, config) {
|
|
|
25
25
|
});
|
|
26
26
|
conversationHistory.resetPendingFeedbackMessageIfExists();
|
|
27
27
|
}
|
|
28
|
-
conversationHistory.
|
|
28
|
+
conversationHistory.appendMessage({
|
|
29
29
|
role: 'user',
|
|
30
30
|
content: [
|
|
31
31
|
{
|
|
32
|
-
type: '
|
|
33
|
-
|
|
34
|
-
url: context.screenshot.base64
|
|
35
|
-
}
|
|
32
|
+
type: 'image',
|
|
33
|
+
url: context.screenshot.base64
|
|
36
34
|
}
|
|
37
35
|
]
|
|
38
36
|
});
|
|
@@ -84,7 +82,7 @@ async function runCustomPlanning(userInstruction, options, config) {
|
|
|
84
82
|
coordinateSystem: config.coordinateSystem
|
|
85
83
|
};
|
|
86
84
|
const messages = buildCustomPlanningMessages(input, config.messages);
|
|
87
|
-
const { content, usage,
|
|
85
|
+
const { content, usage, rawAssistantOutput } = await callAIWithStringResponse(messages, preparedOptions.modelRuntime, {
|
|
88
86
|
abortSignal: preparedOptions.abortSignal,
|
|
89
87
|
requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning
|
|
90
88
|
});
|
|
@@ -107,10 +105,11 @@ async function runCustomPlanning(userInstruction, options, config) {
|
|
|
107
105
|
shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);
|
|
108
106
|
} catch (parseError) {
|
|
109
107
|
const errorMessage = `Parse error: ${parseError instanceof Error ? parseError.message : String(parseError)}`;
|
|
110
|
-
throw new AIResponseParseError(errorMessage, JSON.stringify(content, void 0, 2), usage,
|
|
108
|
+
throw new AIResponseParseError(errorMessage, JSON.stringify(content, void 0, 2), usage, rawAssistantOutput);
|
|
111
109
|
}
|
|
112
110
|
const assistantContent = config.messages.buildAssistantContent?.(parsed, content, input);
|
|
113
|
-
if (
|
|
111
|
+
if (rawAssistantOutput?.type === 'responses' && options.modelRuntime.adapter.responses.replayRawAssistantOutput) options.conversationHistory.appendModelOutput(rawAssistantOutput);
|
|
112
|
+
else if (assistantContent) options.conversationHistory.appendMessage({
|
|
114
113
|
role: 'assistant',
|
|
115
114
|
content: assistantContent
|
|
116
115
|
});
|
|
@@ -120,7 +119,7 @@ async function runCustomPlanning(userInstruction, options, config) {
|
|
|
120
119
|
usage,
|
|
121
120
|
shouldContinuePlanning,
|
|
122
121
|
rawResponse: JSON.stringify(content, void 0, 2),
|
|
123
|
-
|
|
122
|
+
rawAssistantOutput
|
|
124
123
|
};
|
|
125
124
|
}
|
|
126
125
|
export { buildCustomPlanningMessages, runCustomPlanning };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport
|
|
1
|
+
{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n CustomPlanningInput,\n CustomPlanningMessageConfig,\n ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport {\n AIResponseParseError,\n callAIWithStringResponse,\n} from '../../service-caller/index';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport {\n type PreparedUserPrompt,\n preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n systemPrompt: string,\n actionContext?: string,\n): string {\n return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n input: CustomPlanningInput,\n config: CustomPlanningMessageConfig<TParsed>,\n): ModelCallMessages {\n const { options } = input;\n const { conversationHistory, context, actionContext } = options;\n const systemPrompt = appendActionContext(\n config.buildSystemPrompt(),\n actionContext,\n );\n const userInstructionText = input.userInstruction.text;\n const userInstruction = config.buildUserInstruction\n ? config.buildUserInstruction(userInstructionText)\n : userInstructionText;\n const referenceImageMessages = preparedReferenceImagesToMessages(\n input.userInstruction.referenceImages,\n );\n\n if (conversationHistory.pendingFeedbackMessage) {\n conversationHistory.appendMessage({\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n },\n ],\n });\n conversationHistory.resetPendingFeedbackMessageIfExists();\n }\n\n conversationHistory.appendMessage({\n role: 'user',\n content: [\n {\n type: 'image',\n url: context.screenshot.base64,\n },\n ],\n });\n\n if (config.systemPromptPlacement === 'system-message') {\n return [\n { role: 'system', content: systemPrompt },\n {\n role: 'user',\n content: [{ type: 'text', text: userInstruction }],\n },\n ...referenceImageMessages,\n ...conversationHistory.snapshot(config.historyImageLimit),\n ];\n }\n\n return [\n {\n role: 'user',\n content: `${systemPrompt}${userInstruction}`,\n },\n ...referenceImageMessages,\n ...conversationHistory.snapshot(config.historyImageLimit),\n ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n userInstruction: PreparedUserPrompt,\n options: PlanOptions,\n config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n const { context } = options;\n const preparedImage = await prepareModelImage({\n imageBase64: context.screenshot.base64,\n width: context.shotSize.width,\n height: context.shotSize.height,\n policy: options.modelRuntime.adapter.imagePreprocess,\n });\n const preparedOptions: PlanOptions = {\n ...options,\n context: {\n ...context,\n screenshot: ScreenshotItem.create(\n preparedImage.imageBase64,\n context.screenshot.capturedAt,\n ),\n shotSize: preparedImage.preparedSize,\n },\n };\n const input: CustomPlanningInput = {\n userInstruction,\n options: preparedOptions,\n coordinateSystem: config.coordinateSystem,\n };\n\n const messages = buildCustomPlanningMessages(input, config.messages);\n const { content, usage, rawAssistantOutput } = await callAIWithStringResponse(\n messages,\n preparedOptions.modelRuntime,\n {\n abortSignal: preparedOptions.abortSignal,\n requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n },\n );\n\n let parsed: TParsed;\n let actions: PlanningAction[];\n let shouldContinuePlanning: boolean;\n\n try {\n parsed = config.parseResponse(content, input);\n actions = config.transformActions(parsed, input);\n normalizePlanningActionLocateFields(actions, {\n actionSpace: preparedOptions.actionSpace,\n includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n locateResultCodec: config.coordinateNormalizer,\n locateResultContext: {\n preparedSize: preparedImage.preparedSize,\n contentSize: preparedImage.contentSize,\n },\n acceptBbox2dAlias: preparedOptions.modelRuntime.adapter.acceptBbox2dAlias,\n });\n shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n } catch (parseError) {\n const errorMessage = `Parse error: ${\n parseError instanceof Error ? parseError.message : String(parseError)\n }`;\n throw new AIResponseParseError(\n errorMessage,\n JSON.stringify(content, undefined, 2),\n usage,\n rawAssistantOutput,\n );\n }\n\n const assistantContent = config.messages.buildAssistantContent?.(\n parsed,\n content,\n input,\n );\n // UI-TARS and Auto-GLM define their own assistant history format.\n if (\n rawAssistantOutput?.type === 'responses' &&\n options.modelRuntime.adapter.responses.replayRawAssistantOutput\n ) {\n options.conversationHistory.appendModelOutput(rawAssistantOutput);\n } else if (assistantContent) {\n options.conversationHistory.appendMessage({\n role: 'assistant',\n content: assistantContent,\n });\n }\n\n return {\n actions,\n log: config.buildResponseLog(parsed, content),\n usage,\n shouldContinuePlanning,\n rawResponse: JSON.stringify(content, undefined, 2),\n rawAssistantOutput,\n };\n}\n"],"names":["appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawAssistantOutput","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAoBA,SAASA,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,kCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,aAAa,CAAC;QAChC,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,KAAKC,QAAQ,UAAU,CAAC,MAAM;YAChC;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,kBAAkB,EAAE,GAAG,MAAMC,yBACnDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;YACA,mBAAmBE,gBAAgB,YAAY,CAAC,OAAO,CAAC,iBAAiB;QAC3E;QACAS,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOG,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACb,SAASc,QAAW,IACnCb,OACAC;IAEJ;IAEA,MAAMa,mBAAmB7B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IACEiB,oBAAoB,SAAS,eAC7Bf,QAAQ,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,wBAAwB,EAE/DA,QAAQ,mBAAmB,CAAC,iBAAiB,CAACe;SACzC,IAAIa,kBACT5B,QAAQ,mBAAmB,CAAC,aAAa,CAAC;QACxC,MAAM;QACN,SAAS4B;IACX;IAGF,OAAO;QACLV;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaO,KAAK,SAAS,CAACb,SAASc,QAAW;QAChDZ;IACF;AACF"}
|
|
@@ -6,7 +6,7 @@ import { buildStandardPlanningSystemPrompt } from "../../prompt/planning/index.m
|
|
|
6
6
|
import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
|
|
7
7
|
import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
|
|
8
8
|
import { planningModelFamilyRequiredForLocateMessage } from "../../shared/model-locate-result/errors.mjs";
|
|
9
|
-
import {
|
|
9
|
+
import { preparedReferenceImagesToMessages } from "../../shared/multimodal-prompt.mjs";
|
|
10
10
|
import { parsePlanningActions } from "./parse-planning-actions.mjs";
|
|
11
11
|
import { parseStandardPlanningResponse } from "./standard-planning-parser.mjs";
|
|
12
12
|
const debug = getDebug('planning');
|
|
@@ -56,7 +56,7 @@ async function callAndParsePlanningResponse(options) {
|
|
|
56
56
|
},
|
|
57
57
|
toParseError: (parseError, response)=>{
|
|
58
58
|
const errorMessage = parseError instanceof Error ? parseError.message : String(parseError);
|
|
59
|
-
return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.
|
|
59
|
+
return new AIResponseParseError(`XML parse error: ${errorMessage}`, response.content, response.usage, response.rawAssistantOutput, response.reasoning_content);
|
|
60
60
|
},
|
|
61
61
|
parseRetryTimes: modelRuntime.config.retryCount,
|
|
62
62
|
parseRetryInterval: modelRuntime.config.retryInterval,
|
|
@@ -101,7 +101,7 @@ async function standardPlan(userInstruction, opts) {
|
|
|
101
101
|
});
|
|
102
102
|
const imagePayload = preparedImage.imageBase64;
|
|
103
103
|
const actionContext = opts.actionContext ? `${opts.actionContext}\n` : '';
|
|
104
|
-
const referenceImageMessages =
|
|
104
|
+
const referenceImageMessages = preparedReferenceImagesToMessages(userInstruction.referenceImages);
|
|
105
105
|
const instruction = [
|
|
106
106
|
{
|
|
107
107
|
role: 'user',
|
|
@@ -128,11 +128,8 @@ async function standardPlan(userInstruction, opts) {
|
|
|
128
128
|
text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`
|
|
129
129
|
},
|
|
130
130
|
{
|
|
131
|
-
type: '
|
|
132
|
-
|
|
133
|
-
url: imagePayload,
|
|
134
|
-
detail: 'high'
|
|
135
|
-
}
|
|
131
|
+
type: 'image',
|
|
132
|
+
url: imagePayload
|
|
136
133
|
}
|
|
137
134
|
]
|
|
138
135
|
};
|
|
@@ -145,15 +142,12 @@ async function standardPlan(userInstruction, opts) {
|
|
|
145
142
|
text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`
|
|
146
143
|
},
|
|
147
144
|
{
|
|
148
|
-
type: '
|
|
149
|
-
|
|
150
|
-
url: imagePayload,
|
|
151
|
-
detail: 'high'
|
|
152
|
-
}
|
|
145
|
+
type: 'image',
|
|
146
|
+
url: imagePayload
|
|
153
147
|
}
|
|
154
148
|
]
|
|
155
149
|
};
|
|
156
|
-
conversationHistory.
|
|
150
|
+
conversationHistory.appendMessage(latestFeedbackMessage);
|
|
157
151
|
conversationHistory.compressHistory(50, 20);
|
|
158
152
|
const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);
|
|
159
153
|
const msgs = [
|
|
@@ -164,7 +158,7 @@ async function standardPlan(userInstruction, opts) {
|
|
|
164
158
|
...instruction,
|
|
165
159
|
...historyLog
|
|
166
160
|
];
|
|
167
|
-
const { response: { content: rawResponse, usage, reasoning_content,
|
|
161
|
+
const { response: { content: rawResponse, usage, reasoning_content, rawAssistantOutput }, planFromAI, actions } = await callAndParsePlanningResponse({
|
|
168
162
|
messages: msgs,
|
|
169
163
|
modelRuntime,
|
|
170
164
|
abortSignal: opts.abortSignal,
|
|
@@ -189,7 +183,7 @@ async function standardPlan(userInstruction, opts) {
|
|
|
189
183
|
...planFromAI,
|
|
190
184
|
actions,
|
|
191
185
|
rawResponse,
|
|
192
|
-
|
|
186
|
+
rawAssistantOutput,
|
|
193
187
|
usage,
|
|
194
188
|
reasoning_content,
|
|
195
189
|
yamlFlow,
|
|
@@ -202,16 +196,20 @@ async function standardPlan(userInstruction, opts) {
|
|
|
202
196
|
if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
|
|
203
197
|
} else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
|
|
204
198
|
if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
199
|
+
const shouldReplayRawOutput = rawAssistantOutput?.type === 'responses' && modelRuntime.adapter.responses.replayRawAssistantOutput || rawAssistantOutput?.type === 'chat-completion' && modelRuntime.adapter.chatCompletion.replayRawAssistantMessage;
|
|
200
|
+
if (rawAssistantOutput && shouldReplayRawOutput) conversationHistory.appendModelOutput(rawAssistantOutput);
|
|
201
|
+
else {
|
|
202
|
+
if (!rawAssistantOutput) debug('Raw assistant output is unavailable; saving only response text in conversation history.');
|
|
203
|
+
conversationHistory.appendMessage({
|
|
204
|
+
role: 'assistant',
|
|
205
|
+
content: [
|
|
206
|
+
{
|
|
207
|
+
type: 'text',
|
|
208
|
+
text: rawResponse
|
|
209
|
+
}
|
|
210
|
+
]
|
|
211
|
+
});
|
|
212
|
+
}
|
|
215
213
|
return returnValue;
|
|
216
214
|
}
|
|
217
215
|
export { standardPlan };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n PlanningAIResponse,\n PlanningAction,\n RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n callAiAndParseWithRetry,\n withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n LocateResultCodec,\n LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n type PreparedUserPrompt,\n preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n 'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n messages: ChatCompletionMessageParam[];\n modelRuntime: PlanOptions['modelRuntime'];\n abortSignal?: AbortSignal;\n includeLocateInPlanning: boolean;\n actionSpace: PlanOptions['actionSpace'];\n locateResultCodec?: LocateResultCodec;\n locateResultContext: LocateResultContext;\n includeThought: boolean;\n includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n options: CallAndParsePlanningResponseOptions,\n): Promise<{\n response: PlanningCallResponse;\n planFromAI: RawResponsePlanningAIResponse;\n actions: PlanningAction[];\n}> {\n const {\n messages,\n modelRuntime,\n abortSignal,\n includeLocateInPlanning,\n actionSpace,\n locateResultCodec,\n locateResultContext,\n includeThought,\n includeLog,\n } = options;\n assert(\n modelRuntime.adapter.planning.kind === 'standard',\n 'callAndParsePlanningResponse requires a standard planning adapter',\n );\n const actionOutputProtocol =\n modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n return callAiAndParseWithRetry({\n callAi: (retryAttempt, previousParseError) =>\n callAI(\n withSemanticRetryFeedback(messages, previousParseError),\n modelRuntime,\n {\n abortSignal,\n requiresOriginalImageDetail: includeLocateInPlanning,\n semanticRetryAttempt: retryAttempt,\n },\n ),\n parseResponse: (response) => {\n const planFromAI = parseStandardPlanningResponse(response.content, {\n includeThought,\n actionOutputProtocol,\n actionSpace,\n logSource: includeLog ? 'model' : 'action',\n });\n if (\n !planFromAI.action &&\n planFromAI.finalizeSuccess === undefined &&\n !planFromAI.error\n ) {\n throw new Error(\n 'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n );\n }\n if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n warnLog(\n 'Planning response included both an action and <complete>; ignoring <complete> output.',\n );\n planFromAI.finalizeMessage = undefined;\n planFromAI.finalizeSuccess = undefined;\n }\n\n const actions = planFromAI.action ? [planFromAI.action] : [];\n parsePlanningActions(actions, {\n parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n actionSpace,\n includeLocateInPlanning,\n locateResultCodec,\n locateResultContext,\n acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n });\n return { response, planFromAI, actions };\n },\n toParseError: (parseError, response) => {\n const errorMessage =\n parseError instanceof Error ? parseError.message : String(parseError);\n return new AIResponseParseError(\n `XML parse error: ${errorMessage}`,\n response.content,\n response.usage,\n response.rawChoiceMessage,\n response.reasoning_content,\n );\n },\n parseRetryTimes: modelRuntime.config.retryCount,\n parseRetryInterval: modelRuntime.config.retryInterval,\n abortSignal,\n onParseRetry: (parseError) => {\n debug(\n 'retrying plan after response parsing failed: %s',\n parseError instanceof Error ? parseError.message : String(parseError),\n );\n },\n });\n}\n\nexport async function standardPlan(\n userInstruction: PreparedUserPrompt,\n opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n const { context, conversationHistory } = opts;\n const modelRuntime = opts.modelRuntime;\n const { adapter } = modelRuntime;\n const { shotSize } = context;\n const screenshotBase64 = context.screenshot.base64;\n assert(\n adapter.planning.kind === 'standard',\n 'standardPlan requires a standard planning adapter',\n );\n const planningProtocol = adapter.planning.protocol;\n\n if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n throw new Error(\n planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n );\n }\n\n const locateResultCodec = modelRuntime.config.modelFamily\n ? adapter.planning.locateResultCodec\n : undefined;\n\n // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n const includeSubGoals = opts.effort === 'deepThink';\n const includeThought = opts.effort !== 'fast';\n const includeLog = opts.effort !== 'fast';\n\n if (opts.includeLocateInPlanning && !locateResultCodec) {\n throw new Error(\n planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n );\n }\n\n const systemPrompt = await buildStandardPlanningSystemPrompt({\n actionSpace: opts.actionSpace,\n includeThought,\n includeLog,\n includeSubGoals,\n planningProtocol,\n ...(opts.includeLocateInPlanning && locateResultCodec\n ? {\n includeLocateInPlanning: true,\n locatePromptSpec: locateResultCodec.promptSpec,\n }\n : { includeLocateInPlanning: false }),\n });\n\n const preparedImage = await prepareModelImage({\n imageBase64: screenshotBase64,\n width: shotSize.width,\n height: shotSize.height,\n policy: adapter.imagePreprocess,\n });\n const imagePayload = preparedImage.imageBase64;\n\n const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n const referenceImageMessages = preparedReferenceImagesToChatMessages(\n userInstruction.referenceImages,\n );\n const instruction: ChatCompletionMessageParam[] = [\n {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n },\n ],\n },\n ...referenceImageMessages,\n ];\n\n let latestFeedbackMessage: ChatCompletionMessageParam;\n\n // Build sub-goal status text to include in the message\n // In planning deep-think mode: show full sub-goals with logs\n // Otherwise: show historical execution logs\n const executionProgressText = includeSubGoals\n ? conversationHistory.subGoalsToText()\n : conversationHistory.historicalLogsToText();\n const executionProgressSection = executionProgressText\n ? `\\n\\n${executionProgressText}`\n : conversationHistory.pendingFeedbackMessage\n ? ''\n : `\\n\\n${noPreviousActionsText}`;\n\n // Build memories text to include in the message\n const memoriesText = conversationHistory.memoriesToText();\n const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n if (conversationHistory.pendingFeedbackMessage) {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n },\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n };\n\n conversationHistory.resetPendingFeedbackMessageIfExists();\n } else {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n },\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n };\n }\n conversationHistory.append(latestFeedbackMessage);\n\n // Compress history if it exceeds the threshold to avoid context overflow\n conversationHistory.compressHistory(50, 20);\n\n const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n const msgs: ChatCompletionMessageParam[] = [\n { role: 'system', content: systemPrompt },\n ...instruction,\n ...historyLog,\n ];\n\n const {\n response: {\n content: rawResponse,\n usage,\n reasoning_content,\n rawChoiceMessage,\n },\n planFromAI,\n actions,\n } = await callAndParsePlanningResponse({\n messages: msgs,\n modelRuntime,\n abortSignal: opts.abortSignal,\n includeLocateInPlanning: opts.includeLocateInPlanning,\n actionSpace: opts.actionSpace,\n locateResultCodec,\n locateResultContext: {\n preparedSize: preparedImage.preparedSize,\n contentSize: preparedImage.contentSize,\n },\n includeThought,\n includeLog,\n });\n\n // YAML is derived from validated actions outside the model-response retry scope.\n // dumpActionParam omits runtime-only locatedPixelResult fields.\n const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n let shouldContinuePlanning = true;\n\n // Check if task is completed via <complete> tag\n if (planFromAI.finalizeSuccess !== undefined) {\n debug('task completed via <complete> tag, stop planning');\n shouldContinuePlanning = false;\n // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n if (includeSubGoals) {\n conversationHistory.markAllSubGoalsFinished();\n }\n }\n\n const returnValue: PlanningAIResponse = {\n ...planFromAI,\n actions,\n rawResponse,\n rawChoiceMessage,\n usage,\n reasoning_content,\n yamlFlow,\n shouldContinuePlanning,\n };\n\n assert(planFromAI, \"can't get plans from AI\");\n\n // TODO: The plan log is recorded before its action has executed, so a failed\n // action may still appear in the next round as an action already performed.\n // Move this write to the successful action execution path in TaskExecutor.action.\n // Update sub-goals in conversation history only in planning deep-think mode.\n if (includeSubGoals) {\n if (planFromAI.updateSubGoals?.length) {\n conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n }\n if (planFromAI.markFinishedIndexes?.length) {\n for (const index of planFromAI.markFinishedIndexes) {\n conversationHistory.markSubGoalFinished(index);\n }\n }\n // Append the planning log to the currently running sub-goal\n if (planFromAI.log) {\n conversationHistory.appendSubGoalLog(planFromAI.log);\n }\n } else {\n // Without planning deep-think mode, accumulate logs as historical execution steps.\n if (planFromAI.log) {\n conversationHistory.appendHistoricalLog(planFromAI.log);\n }\n }\n\n // Append memory to conversation history if present\n if (planFromAI.memory) {\n conversationHistory.appendMemory(planFromAI.memory);\n }\n\n // Some model providers require opaque assistant fields to be replayed\n // verbatim in later turns. Keep this opt-in per model adapter so that an\n // unverified provider does not receive non-standard response fields.\n if (\n modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n rawChoiceMessage\n ) {\n conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n } else {\n conversationHistory.append({\n role: 'assistant',\n content: [\n {\n type: 'text',\n text: rawResponse,\n },\n ],\n });\n }\n\n return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;YAC3D;YACA,OAAO;gBAAEe;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,uBAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7DwD,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOO;AACT"}
|
|
1
|
+
{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type { ConversationMessage } from '@/ai-model/service-caller/types';\nimport type {\n PlanningAIResponse,\n PlanningAction,\n RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n callAiAndParseWithRetry,\n withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport type {\n LocateResultCodec,\n LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n type PreparedUserPrompt,\n preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { parsePlanningActions } from './parse-planning-actions';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n 'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n messages: ModelCallMessages;\n modelRuntime: PlanOptions['modelRuntime'];\n abortSignal?: AbortSignal;\n includeLocateInPlanning: boolean;\n actionSpace: PlanOptions['actionSpace'];\n locateResultCodec?: LocateResultCodec;\n locateResultContext: LocateResultContext;\n includeThought: boolean;\n includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n options: CallAndParsePlanningResponseOptions,\n): Promise<{\n response: PlanningCallResponse;\n planFromAI: RawResponsePlanningAIResponse;\n actions: PlanningAction[];\n}> {\n const {\n messages,\n modelRuntime,\n abortSignal,\n includeLocateInPlanning,\n actionSpace,\n locateResultCodec,\n locateResultContext,\n includeThought,\n includeLog,\n } = options;\n assert(\n modelRuntime.adapter.planning.kind === 'standard',\n 'callAndParsePlanningResponse requires a standard planning adapter',\n );\n const actionOutputProtocol =\n modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n return callAiAndParseWithRetry({\n callAi: (retryAttempt, previousParseError) =>\n callAI(\n withSemanticRetryFeedback(messages, previousParseError),\n modelRuntime,\n {\n abortSignal,\n requiresOriginalImageDetail: includeLocateInPlanning,\n semanticRetryAttempt: retryAttempt,\n },\n ),\n parseResponse: (response) => {\n const planFromAI = parseStandardPlanningResponse(response.content, {\n includeThought,\n actionOutputProtocol,\n actionSpace,\n logSource: includeLog ? 'model' : 'action',\n });\n if (\n !planFromAI.action &&\n planFromAI.finalizeSuccess === undefined &&\n !planFromAI.error\n ) {\n throw new Error(\n 'Incomplete planning response: provide an action, <complete>, or <error>. A <planning> explanation alone is not sufficient.',\n );\n }\n if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n warnLog(\n 'Planning response included both an action and <complete>; ignoring <complete> output.',\n );\n planFromAI.finalizeMessage = undefined;\n planFromAI.finalizeSuccess = undefined;\n }\n\n const actions = planFromAI.action ? [planFromAI.action] : [];\n parsePlanningActions(actions, {\n parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n actionSpace,\n includeLocateInPlanning,\n locateResultCodec,\n locateResultContext,\n acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n });\n return { response, planFromAI, actions };\n },\n toParseError: (parseError, response) => {\n const errorMessage =\n parseError instanceof Error ? parseError.message : String(parseError);\n return new AIResponseParseError(\n `XML parse error: ${errorMessage}`,\n response.content,\n response.usage,\n response.rawAssistantOutput,\n response.reasoning_content,\n );\n },\n parseRetryTimes: modelRuntime.config.retryCount,\n parseRetryInterval: modelRuntime.config.retryInterval,\n abortSignal,\n onParseRetry: (parseError) => {\n debug(\n 'retrying plan after response parsing failed: %s',\n parseError instanceof Error ? parseError.message : String(parseError),\n );\n },\n });\n}\n\nexport async function standardPlan(\n userInstruction: PreparedUserPrompt,\n opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n const { context, conversationHistory } = opts;\n const modelRuntime = opts.modelRuntime;\n const { adapter } = modelRuntime;\n const { shotSize } = context;\n const screenshotBase64 = context.screenshot.base64;\n assert(\n adapter.planning.kind === 'standard',\n 'standardPlan requires a standard planning adapter',\n );\n const planningProtocol = adapter.planning.protocol;\n\n if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n throw new Error(\n planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n );\n }\n\n const locateResultCodec = modelRuntime.config.modelFamily\n ? adapter.planning.locateResultCodec\n : undefined;\n\n // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n const includeSubGoals = opts.effort === 'deepThink';\n const includeThought = opts.effort !== 'fast';\n const includeLog = opts.effort !== 'fast';\n\n if (opts.includeLocateInPlanning && !locateResultCodec) {\n throw new Error(\n planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n );\n }\n\n const systemPrompt = await buildStandardPlanningSystemPrompt({\n actionSpace: opts.actionSpace,\n includeThought,\n includeLog,\n includeSubGoals,\n planningProtocol,\n ...(opts.includeLocateInPlanning && locateResultCodec\n ? {\n includeLocateInPlanning: true,\n locatePromptSpec: locateResultCodec.promptSpec,\n }\n : { includeLocateInPlanning: false }),\n });\n\n const preparedImage = await prepareModelImage({\n imageBase64: screenshotBase64,\n width: shotSize.width,\n height: shotSize.height,\n policy: adapter.imagePreprocess,\n });\n const imagePayload = preparedImage.imageBase64;\n\n const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n const referenceImageMessages = preparedReferenceImagesToMessages(\n userInstruction.referenceImages,\n );\n const instruction: ConversationMessage[] = [\n {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n },\n ],\n },\n ...referenceImageMessages,\n ];\n\n let latestFeedbackMessage: ConversationMessage;\n\n // Build sub-goal status text to include in the message\n // In planning deep-think mode: show full sub-goals with logs\n // Otherwise: show historical execution logs\n const executionProgressText = includeSubGoals\n ? conversationHistory.subGoalsToText()\n : conversationHistory.historicalLogsToText();\n const executionProgressSection = executionProgressText\n ? `\\n\\n${executionProgressText}`\n : conversationHistory.pendingFeedbackMessage\n ? ''\n : `\\n\\n${noPreviousActionsText}`;\n\n // Build memories text to include in the message\n const memoriesText = conversationHistory.memoriesToText();\n const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n if (conversationHistory.pendingFeedbackMessage) {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n },\n {\n type: 'image',\n url: imagePayload,\n },\n ],\n };\n\n conversationHistory.resetPendingFeedbackMessageIfExists();\n } else {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n },\n {\n type: 'image',\n url: imagePayload,\n },\n ],\n };\n }\n conversationHistory.appendMessage(latestFeedbackMessage);\n\n // Compress history if it exceeds the threshold to avoid context overflow\n conversationHistory.compressHistory(50, 20);\n\n const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n const msgs: ModelCallMessages = [\n { role: 'system', content: systemPrompt },\n ...instruction,\n ...historyLog,\n ];\n\n const {\n response: {\n content: rawResponse,\n usage,\n reasoning_content,\n rawAssistantOutput,\n },\n planFromAI,\n actions,\n } = await callAndParsePlanningResponse({\n messages: msgs,\n modelRuntime,\n abortSignal: opts.abortSignal,\n includeLocateInPlanning: opts.includeLocateInPlanning,\n actionSpace: opts.actionSpace,\n locateResultCodec,\n locateResultContext: {\n preparedSize: preparedImage.preparedSize,\n contentSize: preparedImage.contentSize,\n },\n includeThought,\n includeLog,\n });\n\n // YAML is derived from validated actions outside the model-response retry scope.\n // dumpActionParam omits runtime-only locatedPixelResult fields.\n const yamlFlow = buildYamlFlowFromPlans(actions, opts.actionSpace);\n\n let shouldContinuePlanning = true;\n\n // Check if task is completed via <complete> tag\n if (planFromAI.finalizeSuccess !== undefined) {\n debug('task completed via <complete> tag, stop planning');\n shouldContinuePlanning = false;\n // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n if (includeSubGoals) {\n conversationHistory.markAllSubGoalsFinished();\n }\n }\n\n const returnValue: PlanningAIResponse = {\n ...planFromAI,\n actions,\n rawResponse,\n rawAssistantOutput,\n usage,\n reasoning_content,\n yamlFlow,\n shouldContinuePlanning,\n };\n\n assert(planFromAI, \"can't get plans from AI\");\n\n // TODO: The plan log is recorded before its action has executed, so a failed\n // action may still appear in the next round as an action already performed.\n // Move this write to the successful action execution path in TaskExecutor.action.\n // Update sub-goals in conversation history only in planning deep-think mode.\n if (includeSubGoals) {\n if (planFromAI.updateSubGoals?.length) {\n conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n }\n if (planFromAI.markFinishedIndexes?.length) {\n for (const index of planFromAI.markFinishedIndexes) {\n conversationHistory.markSubGoalFinished(index);\n }\n }\n // Append the planning log to the currently running sub-goal\n if (planFromAI.log) {\n conversationHistory.appendSubGoalLog(planFromAI.log);\n }\n } else {\n // Without planning deep-think mode, accumulate logs as historical execution steps.\n if (planFromAI.log) {\n conversationHistory.appendHistoricalLog(planFromAI.log);\n }\n }\n\n // Append memory to conversation history if present\n if (planFromAI.memory) {\n conversationHistory.appendMemory(planFromAI.memory);\n }\n\n const shouldReplayRawOutput =\n (rawAssistantOutput?.type === 'responses' &&\n modelRuntime.adapter.responses.replayRawAssistantOutput) ||\n (rawAssistantOutput?.type === 'chat-completion' &&\n modelRuntime.adapter.chatCompletion.replayRawAssistantMessage);\n if (rawAssistantOutput && shouldReplayRawOutput) {\n conversationHistory.appendModelOutput(rawAssistantOutput);\n } else {\n if (!rawAssistantOutput) {\n debug(\n 'Raw assistant output is unavailable; saving only response text in conversation history.',\n );\n }\n conversationHistory.appendMessage({\n role: 'assistant',\n content: [\n {\n type: 'text',\n text: rawResponse,\n },\n ],\n });\n }\n\n return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","Error","actions","parsePlanningActions","parseError","errorMessage","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawAssistantOutput","yamlFlow","buildYamlFlowFromPlans","shouldContinuePlanning","returnValue","index","shouldReplayRawOutput"],"mappings":";;;;;;;;;;;AA8BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAM5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IACE,CAACS,WAAW,MAAM,IAClBA,AAA+BE,WAA/BF,WAAW,eAAe,IAC1B,CAACA,WAAW,KAAK,EAEjB,MAAM,IAAIG,MACR;YAGJ,IAAIH,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAME,UAAUJ,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DK,qBAAqBD,SAAS;gBAC5B,yBAAyBX,qBAAqB,uBAAuB;gBACrEN;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;YAC3D;YACA,OAAO;gBAAEe;gBAAUC;gBAAYI;YAAQ;QACzC;QACA,cAAc,CAACE,YAAYP;YACzB,MAAMQ,eACJD,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;YAC5D,OAAO,IAAIG,qBACT,CAAC,iBAAiB,EAAEF,cAAc,EAClCR,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,kBAAkB,EAC3BA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACqB;YACb7B,MACE,mDACA6B,sBAAsBH,QAAQG,WAAW,OAAO,GAAGE,OAAOF;QAE9D;IACF;AACF;AAEO,eAAeI,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM5B,eAAe4B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAG/B;IACpB,MAAM,EAAEgC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDrB,OACEuB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC5B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAImB,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrD+B,QAAQ,QAAQ,CAAC,iBAAiB,GAClCb;IAGJ,MAAMkB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMtB,iBAAiBsB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMrB,aAAaqB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAACxB,mBACnC,MAAM,IAAIe,MACRgB,4CAA4CnC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMqC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BtB;QACAC;QACA6B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAIxB,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMmC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,kCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAAqC;QACzC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAElC,uBAAuB;IAGpC,MAAMqD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,KAAKP;gBACP;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,KAAKP;YACP;SACD;IACH;IAEFX,oBAAoB,aAAa,CAACgB;IAGlChB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAA0B;QAC9B;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,kBAAkB,EACnB,EACDxC,UAAU,EACVI,OAAO,EACR,GAAG,MAAMvB,6BAA6B;QACrC,UAAUuD;QACVpD;QACA,aAAa4B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7BxB;QACA,qBAAqB;YACnB,cAAcmC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAjC;QACAC;IACF;IAIA,MAAMkD,WAAWC,uBAAuBtC,SAASQ,KAAK,WAAW;IAEjE,IAAI+B,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIvB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM8B,cAAkC;QACtC,GAAG5C,UAAU;QACbI;QACAiC;QACAG;QACAF;QACAC;QACAE;QACAE;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIoB,iBAAiB;QACnB,IAAIpB,WAAW,cAAc,EAAE,QAC7Bc,oBAAoB,aAAa,CAACd,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDc,oBAAoB,mBAAmB,CAAC+B;QAI5C,IAAI7C,WAAW,GAAG,EAChBc,oBAAoB,gBAAgB,CAACd,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBc,oBAAoB,mBAAmB,CAACd,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBc,oBAAoB,YAAY,CAACd,WAAW,MAAM;IAGpD,MAAM8C,wBACHN,oBAAoB,SAAS,eAC5BxD,aAAa,OAAO,CAAC,SAAS,CAAC,wBAAwB,IACxDwD,oBAAoB,SAAS,qBAC5BxD,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB;IACjE,IAAIwD,sBAAsBM,uBACxBhC,oBAAoB,iBAAiB,CAAC0B;SACjC;QACL,IAAI,CAACA,oBACH/D,MACE;QAGJqC,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMuB;gBACR;aACD;QACH;IACF;IAEA,OAAOO;AACT"}
|
|
@@ -157,10 +157,8 @@ const createMessageContent = (promptText, screenshots = [], includeScreenshots =
|
|
|
157
157
|
});
|
|
158
158
|
screenshots.forEach((screenshot)=>{
|
|
159
159
|
messageContent.push({
|
|
160
|
-
type: '
|
|
161
|
-
|
|
162
|
-
url: screenshot
|
|
163
|
-
}
|
|
160
|
+
type: 'image',
|
|
161
|
+
url: screenshot
|
|
164
162
|
});
|
|
165
163
|
});
|
|
166
164
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai-model/workflows/recorder-generation/common.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/common.ts"],"sourcesContent":["import {\n DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS,\n type MidsceneRecorderEvent,\n type MidsceneRecorderMarkdownScreenshotAsset,\n type MidsceneRecorderSemantic,\n type MidsceneRecorderTarget,\n createMidsceneRecorderMarkdownScreenshotAssets,\n getMidsceneRecorderEventDescription,\n getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\n\nexport interface EventCounts {\n navigation: number;\n click: number;\n input: number;\n scroll: number;\n total: number;\n}\n\nexport interface InputDescription {\n description: string;\n value: string;\n}\n\nexport interface ProcessedEvent {\n hashId: string;\n type: string;\n timestamp: number;\n source?: string;\n actionType?: string;\n url?: string;\n title?: string;\n semantic?: MidsceneRecorderSemantic;\n description?: string;\n value?: string;\n typedText?: string;\n inputIndex?: number;\n isSequentialInput?: boolean;\n hasNeighborInput?: boolean;\n previousInputDescription?: string;\n previousActionDescription?: string;\n nextActionDescription?: string;\n neighborInputValues?: string[];\n pageInfo?: any;\n elementRect?: any;\n screenshotPath?: string;\n}\n\nexport interface EventSummary {\n testName: string;\n startUrl: string;\n eventCounts: EventCounts;\n urls: string[];\n clickDescriptions: string[];\n inputDescriptions: InputDescription[];\n events: ProcessedEvent[];\n}\n\nexport interface RecorderGenerationContext {\n summary: EventSummary;\n screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n}\n\nexport type ChromeRecordedEvent = MidsceneRecorderEvent;\n\nconst MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH = 1200;\nconst MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH = 400;\n\nexport interface RecorderGenerationOptions {\n testName?: string;\n includeTimestamps?: boolean;\n maxScreenshots?: number;\n description?: string;\n /** Language for human-readable generated content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n language?: string;\n navigationInfo?: {\n urls?: string[];\n titles?: string[];\n initialViewport?: {\n width?: number;\n height?: number;\n };\n };\n}\n\nexport interface RecorderGenerationInput extends RecorderGenerationOptions {\n target: MidsceneRecorderTarget;\n events: MidsceneRecorderEvent[];\n}\n\nexport interface FilteredEvents {\n navigationEvents: ChromeRecordedEvent[];\n clickEvents: ChromeRecordedEvent[];\n inputEvents: ChromeRecordedEvent[];\n scrollEvents: ChromeRecordedEvent[];\n}\n\nfunction cleanRecorderSemanticField(value?: string) {\n return value?.trim() === 'AI is analyzing element...' ? undefined : value;\n}\n\nfunction truncateRecorderGenerationText(value: string, maxLength: number) {\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, maxLength)}... [truncated ${value.length - maxLength} chars]`;\n}\n\nfunction compactRecorderSemanticText(value?: string) {\n const cleaned = cleanRecorderSemanticField(value);\n return cleaned\n ? truncateRecorderGenerationText(\n cleaned,\n MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH,\n )\n : undefined;\n}\n\nfunction compactRecorderSemanticError(value?: string) {\n return value\n ? truncateRecorderGenerationText(\n value,\n MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH,\n )\n : undefined;\n}\n\nexport function compactRecorderSemanticForGeneration(\n semantic?: MidsceneRecorderSemantic,\n): MidsceneRecorderSemantic | undefined {\n if (!semantic) {\n return undefined;\n }\n\n return {\n source: semantic.source,\n status: semantic.status,\n confidence: semantic.confidence,\n elementDescription: compactRecorderSemanticText(\n semantic.elementDescription,\n ),\n replayInstruction: compactRecorderSemanticText(semantic.replayInstruction),\n actionSummary: compactRecorderSemanticText(semantic.actionSummary),\n error: compactRecorderSemanticError(semantic.error),\n ...(semantic.aiDescribe\n ? {\n aiDescribe: {\n verifyPrompt: semantic.aiDescribe.verifyPrompt,\n verifyPassed: semantic.aiDescribe.verifyPassed,\n deepLocate: semantic.aiDescribe.deepLocate,\n centerDistance: semantic.aiDescribe.centerDistance,\n expectedCenter: semantic.aiDescribe.expectedCenter,\n actualCenter: semantic.aiDescribe.actualCenter,\n },\n }\n : {}),\n };\n}\n\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n if (!events.length) {\n throw new Error('No events provided for test generation');\n }\n};\n\nexport const getScreenshotsForLLM = (\n events: ChromeRecordedEvent[],\n maxScreenshots = 1,\n): string[] => {\n return createMidsceneRecorderMarkdownScreenshotAssets(events, {\n baseDir: './screenshots',\n maxScreenshots,\n }).map((asset) => asset.dataUrl);\n};\n\nexport const filterEventsByType = (\n events: ChromeRecordedEvent[],\n): FilteredEvents => {\n return {\n navigationEvents: events.filter((event) => event.type === 'navigation'),\n clickEvents: events.filter((event) => event.type === 'click'),\n inputEvents: events.filter((event) => event.type === 'input'),\n scrollEvents: events.filter((event) => event.type === 'scroll'),\n };\n};\n\nexport const createEventCounts = (\n filteredEvents: FilteredEvents,\n totalEvents: number,\n): EventCounts => {\n return {\n navigation: filteredEvents.navigationEvents.length,\n click: filteredEvents.clickEvents.length,\n input: filteredEvents.inputEvents.length,\n scroll: filteredEvents.scrollEvents.length,\n total: totalEvents,\n };\n};\n\nexport const extractInputDescriptions = (\n inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n return inputEvents\n .map((event) => {\n const semantic = getMidsceneRecorderSemantic(event);\n return {\n description:\n cleanRecorderSemanticField(semantic?.elementDescription) || '',\n value: event.value || '',\n };\n })\n .filter((item) => item.description && item.value);\n};\n\nexport const processEventsForLLM = (\n events: ChromeRecordedEvent[],\n screenshotPathByEventHash: Map<string, string> = new Map(),\n): ProcessedEvent[] => {\n let inputIndex = 0;\n return events.map((event, index) => {\n const previousEvent = events[index - 1];\n const nextEvent = events[index + 1];\n const previousInput = events\n .slice(0, index)\n .reverse()\n .find((candidate) => candidate.type === 'input');\n const nextInput = events\n .slice(index + 1)\n .find((candidate) => candidate.type === 'input');\n const isInput = event.type === 'input';\n const inputSequenceIndex = isInput ? ++inputIndex : undefined;\n const hasNeighborInput = Boolean(previousInput || nextInput);\n const neighborInputValues = isInput\n ? [previousInput?.value, nextInput?.value].filter(\n (value): value is string => Boolean(value),\n )\n : undefined;\n const semantic = compactRecorderSemanticForGeneration(\n getMidsceneRecorderSemantic(event),\n );\n\n return {\n hashId: event.hashId,\n type: event.type,\n timestamp: event.timestamp,\n source: event.source,\n actionType: event.actionType,\n url: event.url,\n title: event.title,\n semantic,\n description: getMidsceneRecorderEventDescription(event),\n value: event.value,\n previousActionDescription: previousEvent\n ? getMidsceneRecorderEventDescription(previousEvent)\n : undefined,\n nextActionDescription: nextEvent\n ? getMidsceneRecorderEventDescription(nextEvent)\n : undefined,\n ...(isInput\n ? {\n typedText: event.value || '',\n inputIndex: inputSequenceIndex,\n isSequentialInput:\n previousEvent?.type === 'input' || nextEvent?.type === 'input',\n hasNeighborInput,\n previousInputDescription: previousInput\n ? getMidsceneRecorderEventDescription(previousInput)\n : undefined,\n neighborInputValues:\n neighborInputValues && neighborInputValues.length > 0\n ? neighborInputValues\n : undefined,\n }\n : {}),\n pageInfo: event.pageInfo,\n elementRect: event.elementRect,\n screenshotPath: screenshotPathByEventHash.get(event.hashId),\n };\n });\n};\n\nexport const prepareEventSummary = (\n events: ChromeRecordedEvent[],\n options: {\n testName?: string;\n maxScreenshots?: number;\n screenshotPathByEventHash?: Map<string, string>;\n } = {},\n): EventSummary => {\n const filteredEvents = filterEventsByType(events);\n const eventCounts = createEventCounts(filteredEvents, events.length);\n\n const startUrl =\n filteredEvents.navigationEvents.length > 0\n ? filteredEvents.navigationEvents[0].url || ''\n : '';\n\n const clickDescriptions = filteredEvents.clickEvents\n .map((event) => getMidsceneRecorderSemantic(event)?.elementDescription)\n .filter((desc): desc is string => Boolean(desc))\n .slice(0, 10);\n\n const inputDescriptions = extractInputDescriptions(\n filteredEvents.inputEvents,\n ).slice(0, 10);\n\n const urls = filteredEvents.navigationEvents\n .map((e) => e.url)\n .filter((url): url is string => Boolean(url))\n .slice(0, 5);\n\n const processedEvents = processEventsForLLM(\n events,\n options.screenshotPathByEventHash,\n );\n\n return {\n testName: options.testName || 'Automated test from recorded events',\n startUrl,\n eventCounts,\n urls,\n clickDescriptions,\n inputDescriptions,\n events: processedEvents,\n };\n};\n\nexport function prepareRecorderGenerationContext(\n input: RecorderGenerationInput,\n): RecorderGenerationContext {\n validateEvents(input.events);\n\n const maxScreenshots =\n input.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;\n const screenshotAssets = createMidsceneRecorderMarkdownScreenshotAssets(\n input.events,\n {\n baseDir: './screenshots',\n maxScreenshots,\n },\n );\n const screenshotPathByEventHash = new Map(\n screenshotAssets.map((asset) => [asset.eventHashId, asset.relativePath]),\n );\n\n return {\n summary: prepareEventSummary(input.events, {\n testName: input.testName,\n screenshotPathByEventHash,\n }),\n screenshotAssets,\n };\n}\n\nexport const createMessageContent = (\n promptText: string,\n screenshots: string[] = [],\n includeScreenshots = true,\n) => {\n const messageContent: any[] = [\n {\n type: 'text',\n text: promptText,\n },\n ];\n\n if (includeScreenshots && screenshots.length > 0) {\n messageContent.unshift({\n type: 'text',\n text: 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n screenshots.forEach((screenshot) => {\n messageContent.push({\n type: 'image_url',\n image_url: {\n url: screenshot,\n },\n });\n });\n }\n\n return messageContent;\n};\n"],"names":["MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH","MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH","cleanRecorderSemanticField","value","undefined","truncateRecorderGenerationText","maxLength","compactRecorderSemanticText","cleaned","compactRecorderSemanticError","compactRecorderSemanticForGeneration","semantic","validateEvents","events","Error","getScreenshotsForLLM","maxScreenshots","createMidsceneRecorderMarkdownScreenshotAssets","asset","filterEventsByType","event","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","getMidsceneRecorderSemantic","item","processEventsForLLM","screenshotPathByEventHash","Map","inputIndex","index","previousEvent","nextEvent","previousInput","candidate","nextInput","isInput","inputSequenceIndex","hasNeighborInput","Boolean","neighborInputValues","getMidsceneRecorderEventDescription","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","inputDescriptions","urls","e","url","processedEvents","prepareRecorderGenerationContext","input","DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS","screenshotAssets","createMessageContent","promptText","screenshots","includeScreenshots","messageContent","screenshot"],"mappings":";AAiEA,MAAMA,+CAA+C;AACrD,MAAMC,gDAAgD;AA+BtD,SAASC,2BAA2BC,KAAc;IAChD,OAAOA,OAAO,WAAW,+BAA+BC,SAAYD;AACtE;AAEA,SAASE,+BAA+BF,KAAa,EAAEG,SAAiB;IACtE,IAAIH,MAAM,MAAM,IAAIG,WAClB,OAAOH;IAET,OAAO,GAAGA,MAAM,KAAK,CAAC,GAAGG,WAAW,eAAe,EAAEH,MAAM,MAAM,GAAGG,UAAU,OAAO,CAAC;AACxF;AAEA,SAASC,4BAA4BJ,KAAc;IACjD,MAAMK,UAAUN,2BAA2BC;IAC3C,OAAOK,UACHH,+BACEG,SACAR,gDAEFI;AACN;AAEA,SAASK,6BAA6BN,KAAc;IAClD,OAAOA,QACHE,+BACEF,OACAF,iDAEFG;AACN;AAEO,SAASM,qCACdC,QAAmC;IAEnC,IAAI,CAACA,UACH;IAGF,OAAO;QACL,QAAQA,SAAS,MAAM;QACvB,QAAQA,SAAS,MAAM;QACvB,YAAYA,SAAS,UAAU;QAC/B,oBAAoBJ,4BAClBI,SAAS,kBAAkB;QAE7B,mBAAmBJ,4BAA4BI,SAAS,iBAAiB;QACzE,eAAeJ,4BAA4BI,SAAS,aAAa;QACjE,OAAOF,6BAA6BE,SAAS,KAAK;QAClD,GAAIA,SAAS,UAAU,GACnB;YACE,YAAY;gBACV,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,YAAYA,SAAS,UAAU,CAAC,UAAU;gBAC1C,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,cAAcA,SAAS,UAAU,CAAC,YAAY;YAChD;QACF,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAMC,iBAAiB,CAACC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIC,MAAM;AAEpB;AAEO,MAAMC,uBAAuB,CAClCF,QACAG,iBAAiB,CAAC,GAEXC,+CAA+CJ,QAAQ;QAC5D,SAAS;QACTG;IACF,GAAG,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO;AAG1B,MAAMC,qBAAqB,CAChCN,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAGK,MAAMC,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAGK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACL;QACJ,MAAMT,WAAWe,4BAA4BN;QAC7C,OAAO;YACL,aACElB,2BAA2BS,UAAU,uBAAuB;YAC9D,OAAOS,MAAM,KAAK,IAAI;QACxB;IACF,GACC,MAAM,CAAC,CAACO,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAG7C,MAAMC,sBAAsB,CACjCf,QACAgB,4BAAiD,IAAIC,KAAK;IAE1D,IAAIC,aAAa;IACjB,OAAOlB,OAAO,GAAG,CAAC,CAACO,OAAOY;QACxB,MAAMC,gBAAgBpB,MAAM,CAACmB,QAAQ,EAAE;QACvC,MAAME,YAAYrB,MAAM,CAACmB,QAAQ,EAAE;QACnC,MAAMG,gBAAgBtB,OACnB,KAAK,CAAC,GAAGmB,OACT,OAAO,GACP,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAMC,YAAYxB,OACf,KAAK,CAACmB,QAAQ,GACd,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAME,UAAUlB,AAAe,YAAfA,MAAM,IAAI;QAC1B,MAAMmB,qBAAqBD,UAAU,EAAEP,aAAa3B;QACpD,MAAMoC,mBAAmBC,QAAQN,iBAAiBE;QAClD,MAAMK,sBAAsBJ,UACxB;YAACH,eAAe;YAAOE,WAAW;SAAM,CAAC,MAAM,CAC7C,CAAClC,QAA2BsC,QAAQtC,UAEtCC;QACJ,MAAMO,WAAWD,qCACfgB,4BAA4BN;QAG9B,OAAO;YACL,QAAQA,MAAM,MAAM;YACpB,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,QAAQA,MAAM,MAAM;YACpB,YAAYA,MAAM,UAAU;YAC5B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClBT;YACA,aAAagC,oCAAoCvB;YACjD,OAAOA,MAAM,KAAK;YAClB,2BAA2Ba,gBACvBU,oCAAoCV,iBACpC7B;YACJ,uBAAuB8B,YACnBS,oCAAoCT,aACpC9B;YACJ,GAAIkC,UACA;gBACE,WAAWlB,MAAM,KAAK,IAAI;gBAC1B,YAAYmB;gBACZ,mBACEN,eAAe,SAAS,WAAWC,WAAW,SAAS;gBACzDM;gBACA,0BAA0BL,gBACtBQ,oCAAoCR,iBACpC/B;gBACJ,qBACEsC,uBAAuBA,oBAAoB,MAAM,GAAG,IAChDA,sBACAtC;YACR,IACA,CAAC,CAAC;YACN,UAAUgB,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;YAC9B,gBAAgBS,0BAA0B,GAAG,CAACT,MAAM,MAAM;QAC5D;IACF;AACF;AAEO,MAAMwB,sBAAsB,CACjC/B,QACAgC,UAII,CAAC,CAAC;IAEN,MAAMvB,iBAAiBH,mBAAmBN;IAC1C,MAAMiC,cAAczB,kBAAkBC,gBAAgBT,OAAO,MAAM;IAEnE,MAAMkC,WACJzB,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAM0B,oBAAoB1B,eAAe,WAAW,CACjD,GAAG,CAAC,CAACF,QAAUM,4BAA4BN,QAAQ,oBACnD,MAAM,CAAC,CAAC6B,OAAyBR,QAAQQ,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAMC,oBAAoB1B,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAM6B,OAAO7B,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAAC8B,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBZ,QAAQY,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkB1B,oBACtBf,QACAgC,QAAQ,yBAAyB;IAGnC,OAAO;QACL,UAAUA,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAK;QACAH;QACAE;QACA,QAAQI;IACV;AACF;AAEO,SAASC,iCACdC,KAA8B;IAE9B5C,eAAe4C,MAAM,MAAM;IAE3B,MAAMxC,iBACJwC,MAAM,cAAc,IAAIC;IAC1B,MAAMC,mBAAmBzC,+CACvBuC,MAAM,MAAM,EACZ;QACE,SAAS;QACTxC;IACF;IAEF,MAAMa,4BAA4B,IAAIC,IACpC4B,iBAAiB,GAAG,CAAC,CAACxC,QAAU;YAACA,MAAM,WAAW;YAAEA,MAAM,YAAY;SAAC;IAGzE,OAAO;QACL,SAAS0B,oBAAoBY,MAAM,MAAM,EAAE;YACzC,UAAUA,MAAM,QAAQ;YACxB3B;QACF;QACA6B;IACF;AACF;AAEO,MAAMC,uBAAuB,CAClCC,YACAC,cAAwB,EAAE,EAC1BC,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMH;QACR;KACD;IAED,IAAIE,sBAAsBD,YAAY,MAAM,GAAG,GAAG;QAChDE,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAF,YAAY,OAAO,CAAC,CAACG;YACnBD,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKC;gBACP;YACF;QACF;IACF;IAEA,OAAOD;AACT"}
|
|
1
|
+
{"version":3,"file":"ai-model/workflows/recorder-generation/common.mjs","sources":["../../../../../src/ai-model/workflows/recorder-generation/common.ts"],"sourcesContent":["import {\n DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS,\n type MidsceneRecorderEvent,\n type MidsceneRecorderMarkdownScreenshotAsset,\n type MidsceneRecorderSemantic,\n type MidsceneRecorderTarget,\n createMidsceneRecorderMarkdownScreenshotAssets,\n getMidsceneRecorderEventDescription,\n getMidsceneRecorderSemantic,\n} from '@midscene/shared/recorder';\nimport type { MessageContent } from '../../service-caller/types';\n\nexport interface EventCounts {\n navigation: number;\n click: number;\n input: number;\n scroll: number;\n total: number;\n}\n\nexport interface InputDescription {\n description: string;\n value: string;\n}\n\nexport interface ProcessedEvent {\n hashId: string;\n type: string;\n timestamp: number;\n source?: string;\n actionType?: string;\n url?: string;\n title?: string;\n semantic?: MidsceneRecorderSemantic;\n description?: string;\n value?: string;\n typedText?: string;\n inputIndex?: number;\n isSequentialInput?: boolean;\n hasNeighborInput?: boolean;\n previousInputDescription?: string;\n previousActionDescription?: string;\n nextActionDescription?: string;\n neighborInputValues?: string[];\n pageInfo?: any;\n elementRect?: any;\n screenshotPath?: string;\n}\n\nexport interface EventSummary {\n testName: string;\n startUrl: string;\n eventCounts: EventCounts;\n urls: string[];\n clickDescriptions: string[];\n inputDescriptions: InputDescription[];\n events: ProcessedEvent[];\n}\n\nexport interface RecorderGenerationContext {\n summary: EventSummary;\n screenshotAssets: MidsceneRecorderMarkdownScreenshotAsset[];\n}\n\nexport type ChromeRecordedEvent = MidsceneRecorderEvent;\n\nconst MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH = 1200;\nconst MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH = 400;\n\nexport interface RecorderGenerationOptions {\n testName?: string;\n includeTimestamps?: boolean;\n maxScreenshots?: number;\n description?: string;\n /** Language for human-readable generated content (e.g. 'English', 'Chinese'). Keys and API names are kept as-is. */\n language?: string;\n navigationInfo?: {\n urls?: string[];\n titles?: string[];\n initialViewport?: {\n width?: number;\n height?: number;\n };\n };\n}\n\nexport interface RecorderGenerationInput extends RecorderGenerationOptions {\n target: MidsceneRecorderTarget;\n events: MidsceneRecorderEvent[];\n}\n\nexport interface FilteredEvents {\n navigationEvents: ChromeRecordedEvent[];\n clickEvents: ChromeRecordedEvent[];\n inputEvents: ChromeRecordedEvent[];\n scrollEvents: ChromeRecordedEvent[];\n}\n\nfunction cleanRecorderSemanticField(value?: string) {\n return value?.trim() === 'AI is analyzing element...' ? undefined : value;\n}\n\nfunction truncateRecorderGenerationText(value: string, maxLength: number) {\n if (value.length <= maxLength) {\n return value;\n }\n return `${value.slice(0, maxLength)}... [truncated ${value.length - maxLength} chars]`;\n}\n\nfunction compactRecorderSemanticText(value?: string) {\n const cleaned = cleanRecorderSemanticField(value);\n return cleaned\n ? truncateRecorderGenerationText(\n cleaned,\n MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH,\n )\n : undefined;\n}\n\nfunction compactRecorderSemanticError(value?: string) {\n return value\n ? truncateRecorderGenerationText(\n value,\n MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH,\n )\n : undefined;\n}\n\nexport function compactRecorderSemanticForGeneration(\n semantic?: MidsceneRecorderSemantic,\n): MidsceneRecorderSemantic | undefined {\n if (!semantic) {\n return undefined;\n }\n\n return {\n source: semantic.source,\n status: semantic.status,\n confidence: semantic.confidence,\n elementDescription: compactRecorderSemanticText(\n semantic.elementDescription,\n ),\n replayInstruction: compactRecorderSemanticText(semantic.replayInstruction),\n actionSummary: compactRecorderSemanticText(semantic.actionSummary),\n error: compactRecorderSemanticError(semantic.error),\n ...(semantic.aiDescribe\n ? {\n aiDescribe: {\n verifyPrompt: semantic.aiDescribe.verifyPrompt,\n verifyPassed: semantic.aiDescribe.verifyPassed,\n deepLocate: semantic.aiDescribe.deepLocate,\n centerDistance: semantic.aiDescribe.centerDistance,\n expectedCenter: semantic.aiDescribe.expectedCenter,\n actualCenter: semantic.aiDescribe.actualCenter,\n },\n }\n : {}),\n };\n}\n\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n if (!events.length) {\n throw new Error('No events provided for test generation');\n }\n};\n\nexport const getScreenshotsForLLM = (\n events: ChromeRecordedEvent[],\n maxScreenshots = 1,\n): string[] => {\n return createMidsceneRecorderMarkdownScreenshotAssets(events, {\n baseDir: './screenshots',\n maxScreenshots,\n }).map((asset) => asset.dataUrl);\n};\n\nexport const filterEventsByType = (\n events: ChromeRecordedEvent[],\n): FilteredEvents => {\n return {\n navigationEvents: events.filter((event) => event.type === 'navigation'),\n clickEvents: events.filter((event) => event.type === 'click'),\n inputEvents: events.filter((event) => event.type === 'input'),\n scrollEvents: events.filter((event) => event.type === 'scroll'),\n };\n};\n\nexport const createEventCounts = (\n filteredEvents: FilteredEvents,\n totalEvents: number,\n): EventCounts => {\n return {\n navigation: filteredEvents.navigationEvents.length,\n click: filteredEvents.clickEvents.length,\n input: filteredEvents.inputEvents.length,\n scroll: filteredEvents.scrollEvents.length,\n total: totalEvents,\n };\n};\n\nexport const extractInputDescriptions = (\n inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n return inputEvents\n .map((event) => {\n const semantic = getMidsceneRecorderSemantic(event);\n return {\n description:\n cleanRecorderSemanticField(semantic?.elementDescription) || '',\n value: event.value || '',\n };\n })\n .filter((item) => item.description && item.value);\n};\n\nexport const processEventsForLLM = (\n events: ChromeRecordedEvent[],\n screenshotPathByEventHash: Map<string, string> = new Map(),\n): ProcessedEvent[] => {\n let inputIndex = 0;\n return events.map((event, index) => {\n const previousEvent = events[index - 1];\n const nextEvent = events[index + 1];\n const previousInput = events\n .slice(0, index)\n .reverse()\n .find((candidate) => candidate.type === 'input');\n const nextInput = events\n .slice(index + 1)\n .find((candidate) => candidate.type === 'input');\n const isInput = event.type === 'input';\n const inputSequenceIndex = isInput ? ++inputIndex : undefined;\n const hasNeighborInput = Boolean(previousInput || nextInput);\n const neighborInputValues = isInput\n ? [previousInput?.value, nextInput?.value].filter(\n (value): value is string => Boolean(value),\n )\n : undefined;\n const semantic = compactRecorderSemanticForGeneration(\n getMidsceneRecorderSemantic(event),\n );\n\n return {\n hashId: event.hashId,\n type: event.type,\n timestamp: event.timestamp,\n source: event.source,\n actionType: event.actionType,\n url: event.url,\n title: event.title,\n semantic,\n description: getMidsceneRecorderEventDescription(event),\n value: event.value,\n previousActionDescription: previousEvent\n ? getMidsceneRecorderEventDescription(previousEvent)\n : undefined,\n nextActionDescription: nextEvent\n ? getMidsceneRecorderEventDescription(nextEvent)\n : undefined,\n ...(isInput\n ? {\n typedText: event.value || '',\n inputIndex: inputSequenceIndex,\n isSequentialInput:\n previousEvent?.type === 'input' || nextEvent?.type === 'input',\n hasNeighborInput,\n previousInputDescription: previousInput\n ? getMidsceneRecorderEventDescription(previousInput)\n : undefined,\n neighborInputValues:\n neighborInputValues && neighborInputValues.length > 0\n ? neighborInputValues\n : undefined,\n }\n : {}),\n pageInfo: event.pageInfo,\n elementRect: event.elementRect,\n screenshotPath: screenshotPathByEventHash.get(event.hashId),\n };\n });\n};\n\nexport const prepareEventSummary = (\n events: ChromeRecordedEvent[],\n options: {\n testName?: string;\n maxScreenshots?: number;\n screenshotPathByEventHash?: Map<string, string>;\n } = {},\n): EventSummary => {\n const filteredEvents = filterEventsByType(events);\n const eventCounts = createEventCounts(filteredEvents, events.length);\n\n const startUrl =\n filteredEvents.navigationEvents.length > 0\n ? filteredEvents.navigationEvents[0].url || ''\n : '';\n\n const clickDescriptions = filteredEvents.clickEvents\n .map((event) => getMidsceneRecorderSemantic(event)?.elementDescription)\n .filter((desc): desc is string => Boolean(desc))\n .slice(0, 10);\n\n const inputDescriptions = extractInputDescriptions(\n filteredEvents.inputEvents,\n ).slice(0, 10);\n\n const urls = filteredEvents.navigationEvents\n .map((e) => e.url)\n .filter((url): url is string => Boolean(url))\n .slice(0, 5);\n\n const processedEvents = processEventsForLLM(\n events,\n options.screenshotPathByEventHash,\n );\n\n return {\n testName: options.testName || 'Automated test from recorded events',\n startUrl,\n eventCounts,\n urls,\n clickDescriptions,\n inputDescriptions,\n events: processedEvents,\n };\n};\n\nexport function prepareRecorderGenerationContext(\n input: RecorderGenerationInput,\n): RecorderGenerationContext {\n validateEvents(input.events);\n\n const maxScreenshots =\n input.maxScreenshots ?? DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS;\n const screenshotAssets = createMidsceneRecorderMarkdownScreenshotAssets(\n input.events,\n {\n baseDir: './screenshots',\n maxScreenshots,\n },\n );\n const screenshotPathByEventHash = new Map(\n screenshotAssets.map((asset) => [asset.eventHashId, asset.relativePath]),\n );\n\n return {\n summary: prepareEventSummary(input.events, {\n testName: input.testName,\n screenshotPathByEventHash,\n }),\n screenshotAssets,\n };\n}\n\nexport const createMessageContent = (\n promptText: string,\n screenshots: string[] = [],\n includeScreenshots = true,\n) => {\n const messageContent: MessageContent[] = [\n {\n type: 'text',\n text: promptText,\n },\n ];\n\n if (includeScreenshots && screenshots.length > 0) {\n messageContent.unshift({\n type: 'text',\n text: 'Here are screenshots from the recording session to help you understand the context:',\n });\n\n screenshots.forEach((screenshot) => {\n messageContent.push({\n type: 'image',\n url: screenshot,\n });\n });\n }\n\n return messageContent;\n};\n"],"names":["MAX_RECORDER_GENERATION_SEMANTIC_TEXT_LENGTH","MAX_RECORDER_GENERATION_SEMANTIC_ERROR_LENGTH","cleanRecorderSemanticField","value","undefined","truncateRecorderGenerationText","maxLength","compactRecorderSemanticText","cleaned","compactRecorderSemanticError","compactRecorderSemanticForGeneration","semantic","validateEvents","events","Error","getScreenshotsForLLM","maxScreenshots","createMidsceneRecorderMarkdownScreenshotAssets","asset","filterEventsByType","event","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","getMidsceneRecorderSemantic","item","processEventsForLLM","screenshotPathByEventHash","Map","inputIndex","index","previousEvent","nextEvent","previousInput","candidate","nextInput","isInput","inputSequenceIndex","hasNeighborInput","Boolean","neighborInputValues","getMidsceneRecorderEventDescription","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","inputDescriptions","urls","e","url","processedEvents","prepareRecorderGenerationContext","input","DEFAULT_MIDSCENE_RECORDER_MARKDOWN_MAX_SCREENSHOTS","screenshotAssets","createMessageContent","promptText","screenshots","includeScreenshots","messageContent","screenshot"],"mappings":";AAkEA,MAAMA,+CAA+C;AACrD,MAAMC,gDAAgD;AA+BtD,SAASC,2BAA2BC,KAAc;IAChD,OAAOA,OAAO,WAAW,+BAA+BC,SAAYD;AACtE;AAEA,SAASE,+BAA+BF,KAAa,EAAEG,SAAiB;IACtE,IAAIH,MAAM,MAAM,IAAIG,WAClB,OAAOH;IAET,OAAO,GAAGA,MAAM,KAAK,CAAC,GAAGG,WAAW,eAAe,EAAEH,MAAM,MAAM,GAAGG,UAAU,OAAO,CAAC;AACxF;AAEA,SAASC,4BAA4BJ,KAAc;IACjD,MAAMK,UAAUN,2BAA2BC;IAC3C,OAAOK,UACHH,+BACEG,SACAR,gDAEFI;AACN;AAEA,SAASK,6BAA6BN,KAAc;IAClD,OAAOA,QACHE,+BACEF,OACAF,iDAEFG;AACN;AAEO,SAASM,qCACdC,QAAmC;IAEnC,IAAI,CAACA,UACH;IAGF,OAAO;QACL,QAAQA,SAAS,MAAM;QACvB,QAAQA,SAAS,MAAM;QACvB,YAAYA,SAAS,UAAU;QAC/B,oBAAoBJ,4BAClBI,SAAS,kBAAkB;QAE7B,mBAAmBJ,4BAA4BI,SAAS,iBAAiB;QACzE,eAAeJ,4BAA4BI,SAAS,aAAa;QACjE,OAAOF,6BAA6BE,SAAS,KAAK;QAClD,GAAIA,SAAS,UAAU,GACnB;YACE,YAAY;gBACV,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,cAAcA,SAAS,UAAU,CAAC,YAAY;gBAC9C,YAAYA,SAAS,UAAU,CAAC,UAAU;gBAC1C,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,gBAAgBA,SAAS,UAAU,CAAC,cAAc;gBAClD,cAAcA,SAAS,UAAU,CAAC,YAAY;YAChD;QACF,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAMC,iBAAiB,CAACC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIC,MAAM;AAEpB;AAEO,MAAMC,uBAAuB,CAClCF,QACAG,iBAAiB,CAAC,GAEXC,+CAA+CJ,QAAQ;QAC5D,SAAS;QACTG;IACF,GAAG,GAAG,CAAC,CAACE,QAAUA,MAAM,OAAO;AAG1B,MAAMC,qBAAqB,CAChCN,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcP,OAAO,MAAM,CAAC,CAACO,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAGK,MAAMC,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAGK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACL;QACJ,MAAMT,WAAWe,4BAA4BN;QAC7C,OAAO;YACL,aACElB,2BAA2BS,UAAU,uBAAuB;YAC9D,OAAOS,MAAM,KAAK,IAAI;QACxB;IACF,GACC,MAAM,CAAC,CAACO,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAG7C,MAAMC,sBAAsB,CACjCf,QACAgB,4BAAiD,IAAIC,KAAK;IAE1D,IAAIC,aAAa;IACjB,OAAOlB,OAAO,GAAG,CAAC,CAACO,OAAOY;QACxB,MAAMC,gBAAgBpB,MAAM,CAACmB,QAAQ,EAAE;QACvC,MAAME,YAAYrB,MAAM,CAACmB,QAAQ,EAAE;QACnC,MAAMG,gBAAgBtB,OACnB,KAAK,CAAC,GAAGmB,OACT,OAAO,GACP,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAMC,YAAYxB,OACf,KAAK,CAACmB,QAAQ,GACd,IAAI,CAAC,CAACI,YAAcA,AAAmB,YAAnBA,UAAU,IAAI;QACrC,MAAME,UAAUlB,AAAe,YAAfA,MAAM,IAAI;QAC1B,MAAMmB,qBAAqBD,UAAU,EAAEP,aAAa3B;QACpD,MAAMoC,mBAAmBC,QAAQN,iBAAiBE;QAClD,MAAMK,sBAAsBJ,UACxB;YAACH,eAAe;YAAOE,WAAW;SAAM,CAAC,MAAM,CAC7C,CAAClC,QAA2BsC,QAAQtC,UAEtCC;QACJ,MAAMO,WAAWD,qCACfgB,4BAA4BN;QAG9B,OAAO;YACL,QAAQA,MAAM,MAAM;YACpB,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,QAAQA,MAAM,MAAM;YACpB,YAAYA,MAAM,UAAU;YAC5B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClBT;YACA,aAAagC,oCAAoCvB;YACjD,OAAOA,MAAM,KAAK;YAClB,2BAA2Ba,gBACvBU,oCAAoCV,iBACpC7B;YACJ,uBAAuB8B,YACnBS,oCAAoCT,aACpC9B;YACJ,GAAIkC,UACA;gBACE,WAAWlB,MAAM,KAAK,IAAI;gBAC1B,YAAYmB;gBACZ,mBACEN,eAAe,SAAS,WAAWC,WAAW,SAAS;gBACzDM;gBACA,0BAA0BL,gBACtBQ,oCAAoCR,iBACpC/B;gBACJ,qBACEsC,uBAAuBA,oBAAoB,MAAM,GAAG,IAChDA,sBACAtC;YACR,IACA,CAAC,CAAC;YACN,UAAUgB,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;YAC9B,gBAAgBS,0BAA0B,GAAG,CAACT,MAAM,MAAM;QAC5D;IACF;AACF;AAEO,MAAMwB,sBAAsB,CACjC/B,QACAgC,UAII,CAAC,CAAC;IAEN,MAAMvB,iBAAiBH,mBAAmBN;IAC1C,MAAMiC,cAAczB,kBAAkBC,gBAAgBT,OAAO,MAAM;IAEnE,MAAMkC,WACJzB,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAM0B,oBAAoB1B,eAAe,WAAW,CACjD,GAAG,CAAC,CAACF,QAAUM,4BAA4BN,QAAQ,oBACnD,MAAM,CAAC,CAAC6B,OAAyBR,QAAQQ,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAMC,oBAAoB1B,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAM6B,OAAO7B,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAAC8B,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBZ,QAAQY,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkB1B,oBACtBf,QACAgC,QAAQ,yBAAyB;IAGnC,OAAO;QACL,UAAUA,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAK;QACAH;QACAE;QACA,QAAQI;IACV;AACF;AAEO,SAASC,iCACdC,KAA8B;IAE9B5C,eAAe4C,MAAM,MAAM;IAE3B,MAAMxC,iBACJwC,MAAM,cAAc,IAAIC;IAC1B,MAAMC,mBAAmBzC,+CACvBuC,MAAM,MAAM,EACZ;QACE,SAAS;QACTxC;IACF;IAEF,MAAMa,4BAA4B,IAAIC,IACpC4B,iBAAiB,GAAG,CAAC,CAACxC,QAAU;YAACA,MAAM,WAAW;YAAEA,MAAM,YAAY;SAAC;IAGzE,OAAO;QACL,SAAS0B,oBAAoBY,MAAM,MAAM,EAAE;YACzC,UAAUA,MAAM,QAAQ;YACxB3B;QACF;QACA6B;IACF;AACF;AAEO,MAAMC,uBAAuB,CAClCC,YACAC,cAAwB,EAAE,EAC1BC,qBAAqB,IAAI;IAEzB,MAAMC,iBAAmC;QACvC;YACE,MAAM;YACN,MAAMH;QACR;KACD;IAED,IAAIE,sBAAsBD,YAAY,MAAM,GAAG,GAAG;QAChDE,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAF,YAAY,OAAO,CAAC,CAACG;YACnBD,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,KAAKC;YACP;QACF;IACF;IAEA,OAAOD;AACT"}
|
|
@@ -75,7 +75,7 @@ function getPromptShape(prompt) {
|
|
|
75
75
|
if (Array.isArray(content)) {
|
|
76
76
|
for (const part of content)if ('object' == typeof part && part && 'type' in part) {
|
|
77
77
|
if ('text' === part.type && 'text' in part) textChars += String(part.text).length;
|
|
78
|
-
if ('
|
|
78
|
+
if ('image' === part.type) imageCount += 1;
|
|
79
79
|
}
|
|
80
80
|
}
|
|
81
81
|
}
|
|
@@ -218,10 +218,8 @@ Important: Return ONLY raw Markdown. Do NOT wrap the response in markdown code b
|
|
|
218
218
|
text: `${screenshotRef} for event #${asset.eventIndex + 1}`
|
|
219
219
|
});
|
|
220
220
|
content.push({
|
|
221
|
-
type: '
|
|
222
|
-
|
|
223
|
-
url: asset.dataUrl
|
|
224
|
-
}
|
|
221
|
+
type: 'image',
|
|
222
|
+
url: asset.dataUrl
|
|
225
223
|
});
|
|
226
224
|
}
|
|
227
225
|
return [
|