@pheem49/mint 1.5.5 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex +0 -0
- package/.github/FUNDING.yml +2 -0
- package/.github/workflows/ci.yml +45 -0
- package/.github/workflows/release.yml +79 -0
- package/Cargo.lock +5792 -0
- package/Cargo.toml +32 -0
- package/README.md +387 -353
- package/assets/icon.png +0 -0
- package/bin/mint +0 -0
- package/crates/mint-cli/Cargo.toml +23 -0
- package/crates/mint-cli/src/agent.rs +851 -0
- package/crates/mint-cli/src/gmail.rs +216 -0
- package/crates/mint-cli/src/image.rs +142 -0
- package/crates/mint-cli/src/main.rs +2837 -0
- package/crates/mint-cli/src/mcp.rs +63 -0
- package/crates/mint-cli/src/onboard.rs +1149 -0
- package/crates/mint-cli/src/setup.rs +390 -0
- package/crates/mint-cli/src/skills.rs +8 -0
- package/crates/mint-cli/src/updater.rs +279 -0
- package/crates/mint-core/Cargo.toml +22 -0
- package/crates/mint-core/src/agent_loop.rs +94 -0
- package/crates/mint-core/src/api_server.rs +991 -0
- package/crates/mint-core/src/channels.rs +248 -0
- package/crates/mint-core/src/chat.rs +895 -0
- package/crates/mint-core/src/code_tools.rs +729 -0
- package/crates/mint-core/src/config.rs +368 -0
- package/crates/mint-core/src/files.rs +159 -0
- package/crates/mint-core/src/knowledge.rs +541 -0
- package/crates/mint-core/src/lib.rs +84 -0
- package/crates/mint-core/src/mcp.rs +273 -0
- package/crates/mint-core/src/memory.rs +673 -0
- package/crates/mint-core/src/orchestration.rs +2157 -0
- package/crates/mint-core/src/pictures.rs +314 -0
- package/crates/mint-core/src/plugins.rs +727 -0
- package/crates/mint-core/src/safety.rs +416 -0
- package/crates/mint-core/src/semantic.rs +254 -0
- package/crates/mint-core/src/shell.rs +317 -0
- package/crates/mint-core/src/skills.rs +71 -0
- package/crates/mint-core/src/symbols.rs +157 -0
- package/crates/mint-core/src/tasks.rs +308 -0
- package/crates/mint-core/src/tts.rs +92 -0
- package/crates/mint-core/src/weather.rs +93 -0
- package/crates/mint-core/src/web_search.rs +200 -0
- package/crates/mint-core/src/workflows.rs +81 -0
- package/crates/mint-core/tests/mcp_stdio.rs +45 -0
- package/crates/mint-core/tests/memory_persistence.rs +172 -0
- package/crates/mint-core/tests/pictures_storage.rs +14 -0
- package/crates/mint-core/tests/task_lifecycle.rs +87 -0
- package/package.json +35 -99
- package/src/bin/index.js +16 -0
- package/src/renderer/index-web.html +17 -0
- package/src/renderer/index.html +17 -0
- package/src/renderer/public/Live2DCubismCore.js +9 -0
- package/src/renderer/public/assets/icon.png +0 -0
- package/src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.model3.json +36 -0
- package/src/renderer/src/App.tsx +33 -0
- package/src/renderer/src/calculator.ts +47 -0
- package/src/renderer/src/components/ChatPanel.tsx +1598 -0
- package/src/renderer/src/components/DashboardSidebar.tsx +358 -0
- package/src/renderer/src/components/Live2DStage.tsx +374 -0
- package/src/renderer/src/components/MintDashboard.tsx +950 -0
- package/src/renderer/src/components/ModelPanel.tsx +154 -0
- package/src/renderer/src/components/PicturesLibrary.tsx +46 -0
- package/src/renderer/src/components/ProactiveGlow.tsx +19 -0
- package/src/renderer/src/components/ScreenPicker.tsx +579 -0
- package/src/renderer/src/components/SettingsWindow.tsx +1467 -0
- package/src/renderer/src/components/SpotlightWindow.tsx +280 -0
- package/src/renderer/src/components/WidgetWindow.tsx +36 -0
- package/src/renderer/src/components/WorkspacePanel.tsx +268 -0
- package/src/{UI → renderer/src/css}/settings.css +69 -16
- package/src/renderer/src/css/spotlight.css +113 -0
- package/src/renderer/src/css/styles.css +3722 -0
- package/src/renderer/src/css/widget.css +185 -0
- package/src/renderer/src/env.d.ts +116 -0
- package/src/renderer/src/index.css +379 -0
- package/src/renderer/src/main.tsx +13 -0
- package/src/renderer/src/tauri.ts +996 -0
- package/src/renderer/src-web/App.tsx +25 -0
- package/src/renderer/src-web/calculator.ts +47 -0
- package/src/renderer/src-web/components/ChatPanel.tsx +1662 -0
- package/src/renderer/src-web/components/DashboardSidebar.tsx +242 -0
- package/src/renderer/src-web/components/MintDashboard.tsx +763 -0
- package/src/renderer/src-web/components/PicturesLibrary.tsx +73 -0
- package/src/renderer/src-web/components/SettingsWindow.tsx +1500 -0
- package/src/renderer/src-web/css/settings.css +1100 -0
- package/src/{UI → renderer/src-web/css}/spotlight.css +4 -4
- package/src/{UI → renderer/src-web/css}/styles.css +1055 -159
- package/src/{UI → renderer/src-web/css}/widget.css +2 -2
- package/src/renderer/src-web/env.d.ts +107 -0
- package/src/renderer/src-web/index.css +379 -0
- package/src/renderer/src-web/main.tsx +13 -0
- package/src/renderer/src-web/tauri.ts +983 -0
- package/tsconfig.json +30 -0
- package/vite.config.ts +33 -0
- package/vite.config.web.ts +51 -0
- package/GUIDE_TH.md +0 -125
- package/assets/Agent_Mint.png +0 -0
- package/assets/CLI_Screen.png +0 -0
- package/assets/Settings.png +0 -0
- package/benchmark_ai.js +0 -71
- package/install.ps1 +0 -64
- package/install.sh +0 -54
- package/main.js +0 -139
- package/mint-cli-logic.js +0 -3
- package/mint-cli.js +0 -410
- package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.model3.json +0 -47
- package/models/Shiroko_Model/Shiroko//342/232/241/351/253/230/344/272/256/342/232/241/344/275/277/347/224/250/346/225/231/347/250/213/344/270/216/346/263/250/346/204/217/344/272/213/351/241/271.txt +0 -23
- package/preload-picker.js +0 -11
- package/preload-settings.js +0 -11
- package/preload.js +0 -41
- package/scripts/install_linux_desktop_entry.js +0 -48
- package/src/AI_Brain/Gemini_API.js +0 -813
- package/src/AI_Brain/agent_orchestrator.js +0 -73
- package/src/AI_Brain/autonomous_brain.js +0 -179
- package/src/AI_Brain/behavior_memory.js +0 -135
- package/src/AI_Brain/headless_agent.js +0 -143
- package/src/AI_Brain/knowledge_base.js +0 -349
- package/src/AI_Brain/memory_store.js +0 -662
- package/src/AI_Brain/proactive_engine.js +0 -172
- package/src/AI_Brain/provider_adapter.js +0 -365
- package/src/Automation_Layer/browser_automation.js +0 -149
- package/src/Automation_Layer/file_operations.js +0 -286
- package/src/Automation_Layer/open_app.js +0 -85
- package/src/Automation_Layer/open_website.js +0 -38
- package/src/CLI/approval_handler.js +0 -47
- package/src/CLI/chat_router.js +0 -247
- package/src/CLI/chat_ui.js +0 -1159
- package/src/CLI/cli_colors.js +0 -115
- package/src/CLI/cli_formatters.js +0 -94
- package/src/CLI/code_agent.js +0 -1667
- package/src/CLI/code_session_memory.js +0 -62
- package/src/CLI/gmail_auth.js +0 -210
- package/src/CLI/image_input.js +0 -90
- package/src/CLI/intent_detectors.js +0 -181
- package/src/CLI/interactive_chat.js +0 -658
- package/src/CLI/list_features.js +0 -64
- package/src/CLI/onboarding.js +0 -416
- package/src/CLI/repo_summarizer.js +0 -282
- package/src/CLI/semantic_code_search.js +0 -312
- package/src/CLI/skill_manager.js +0 -41
- package/src/CLI/slash_command_handler.js +0 -418
- package/src/CLI/symbol_indexer.js +0 -231
- package/src/CLI/updater.js +0 -230
- package/src/CLI/workspace_manager.js +0 -90
- package/src/Channels/brave_search_bridge.js +0 -35
- package/src/Channels/discord_bridge.js +0 -66
- package/src/Channels/google_search_bridge.js +0 -38
- package/src/Channels/line_bridge.js +0 -60
- package/src/Channels/slack_bridge.js +0 -48
- package/src/Channels/telegram_bridge.js +0 -41
- package/src/Channels/whatsapp_bridge.js +0 -57
- package/src/Command_Parser/parser.js +0 -45
- package/src/Plugins/dev_tools.js +0 -41
- package/src/Plugins/discord.js +0 -20
- package/src/Plugins/docker.js +0 -47
- package/src/Plugins/gmail.js +0 -251
- package/src/Plugins/google_calendar.js +0 -252
- package/src/Plugins/mcp_manager.js +0 -95
- package/src/Plugins/notion.js +0 -256
- package/src/Plugins/obsidian.js +0 -54
- package/src/Plugins/plugin_manager.js +0 -81
- package/src/Plugins/spotify.js +0 -173
- package/src/Plugins/system_metrics.js +0 -31
- package/src/Plugins/system_monitor.js +0 -72
- package/src/System/action_executor.js +0 -178
- package/src/System/bridge_manager.js +0 -76
- package/src/System/chat_history_manager.js +0 -83
- package/src/System/config_manager.js +0 -194
- package/src/System/custom_workflows.js +0 -163
- package/src/System/daemon_manager.js +0 -67
- package/src/System/google_tts_urls.js +0 -51
- package/src/System/granular_automation.js +0 -157
- package/src/System/ipc_handlers.js +0 -332
- package/src/System/notifications.js +0 -23
- package/src/System/optional_require.js +0 -23
- package/src/System/picture_store.js +0 -109
- package/src/System/proactive_loop.js +0 -153
- package/src/System/safety_manager.js +0 -273
- package/src/System/sandbox_runner.js +0 -182
- package/src/System/screen_capture.js +0 -175
- package/src/System/smart_context.js +0 -227
- package/src/System/system_automation.js +0 -162
- package/src/System/system_events.js +0 -79
- package/src/System/system_info.js +0 -125
- package/src/System/task_manager.js +0 -222
- package/src/System/tool_registry.js +0 -293
- package/src/System/window_manager.js +0 -220
- package/src/UI/floating.css +0 -80
- package/src/UI/floating.html +0 -17
- package/src/UI/floating.js +0 -67
- package/src/UI/live2d_manager.js +0 -600
- package/src/UI/preload-floating.js +0 -7
- package/src/UI/preload-spotlight.js +0 -11
- package/src/UI/preload-widget.js +0 -5
- package/src/UI/proactive-glow.html +0 -42
- package/src/UI/renderer.js +0 -2127
- package/src/UI/screenPicker.html +0 -214
- package/src/UI/screenPicker.js +0 -262
- package/src/UI/settings.html +0 -577
- package/src/UI/settings.js +0 -770
- package/src/UI/spotlight.html +0 -23
- package/src/UI/spotlight.js +0 -185
- package/src/UI/widget.html +0 -29
- package/src/UI/widget.js +0 -10
- /package/{models → src/renderer/public/models}/Shiroko_Model/Shiroko/Shiroko_Core/72d86db84cfa9730b894c241fd24c0db.png +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//345/233/264/350/243/231.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/apron.exp3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//347/214/253/345/222/252/346/273/244/351/225/234.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/catfilter.exp3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//347/202/271/344/270/200/344/270/213.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/click.exp3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//345/221/206/347/214/253.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/dazed.exp3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//345/221/206/347/214/253/347/234/274/347/217/240/346/221/207/346/231/203.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/dazedeyes.exp3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//347/234/274/351/225/234.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/glasses.exp3.json} +0 -0
- /package/{models → src/renderer/public/models}/Shiroko_Model/Shiroko/Shiroko_Core/items_pinned_to_model.json +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//346/213/277/347/254/224.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/pen.exp3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//346/213/215/347/205/247.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/photo.exp3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_00.png" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.4096/texture_00.png} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_01.png" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.4096/texture_01.png} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_02.png" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.4096/texture_02.png} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_03.png" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.4096/texture_03.png} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.cdi3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.cdi3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.moc3" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.moc3} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.physics3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.physics3.json} +0 -0
- /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.vtube.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.vtube.json} +0 -0
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
const { GoogleGenAI } = require('@google/genai');
|
|
2
|
-
const { readConfig } = require('../System/config_manager');
|
|
3
|
-
|
|
4
|
-
// ============================================================
|
|
5
|
-
// Proactive Engine — Smart Suggestion Engine (Multi-Choice)
|
|
6
|
-
// ============================================================
|
|
7
|
-
|
|
8
|
-
const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
|
|
9
|
-
let lastLoggedModel = '';
|
|
10
|
-
let _ai = null;
|
|
11
|
-
|
|
12
|
-
function getAi(apiKey) {
|
|
13
|
-
if (!_ai) _ai = new GoogleGenAI({ apiKey });
|
|
14
|
-
return _ai;
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
const PROACTIVE_SYSTEM_PROMPT = `You are a Smart Suggestion Engine built into a Desktop AI Agent called "Mint".
|
|
18
|
-
Your job: observe the user's screen + behavior, then offer MULTIPLE relevant quick-action options — NOT just one question.
|
|
19
|
-
|
|
20
|
-
CRITICAL RULES:
|
|
21
|
-
1. Respond ONLY with valid JSON, no markdown.
|
|
22
|
-
2. If nothing notable is on screen, return: {"message": null, "context": "", "suggestions": []}
|
|
23
|
-
3. Generate 2–4 SHORT suggestion chips that are genuinely useful based on what's visible.
|
|
24
|
-
4. Each suggestion must have a clear label (1–3 words) and an action.
|
|
25
|
-
5. Write "message" in Thai — short, friendly, observational (e.g. "พบว่าคุณเพิ่งเปิด Chrome").
|
|
26
|
-
6. Do NOT repeat suggestions from recent activities.
|
|
27
|
-
7. Suggestions should feel like smart shortcuts, not questions.
|
|
28
|
-
|
|
29
|
-
Response schema (STRICT):
|
|
30
|
-
{
|
|
31
|
-
"context": "short English description of what you see on screen",
|
|
32
|
-
"message": "สั้น ๆ เป็นภาษาไทย บอกว่า AI เห็นอะไร และเสนออะไร",
|
|
33
|
-
"suggestions": [
|
|
34
|
-
{ "label": "YouTube", "action": { "type": "open_url", "target": "https://youtube.com" } },
|
|
35
|
-
{ "label": "Gmail", "action": { "type": "open_url", "target": "https://mail.google.com" } },
|
|
36
|
-
{ "label": "GitHub", "action": { "type": "open_url", "target": "https://github.com" } }
|
|
37
|
-
]
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
Action types allowed: "open_url", "open_app", "search", "none"
|
|
41
|
-
|
|
42
|
-
Examples:
|
|
43
|
-
|
|
44
|
-
SCENARIO: User opened Chrome or Firefox
|
|
45
|
-
→ message: "เพิ่งเปิด Browser — ต้องการเข้าเว็บไหนคะ?"
|
|
46
|
-
→ suggestions: YouTube, Gmail, GitHub, Google Maps (based on behavior history)
|
|
47
|
-
|
|
48
|
-
SCENARIO: User is in VS Code / coding
|
|
49
|
-
→ message: "กำลัง Code อยู่ใช่ไหมคะ? มีอะไรช่วยได้บ้าง"
|
|
50
|
-
→ suggestions: Stack Overflow, MDN Docs, GitHub, ค้นหา Error
|
|
51
|
-
|
|
52
|
-
SCENARIO: User opened Spotify
|
|
53
|
-
→ message: "เปิด Spotify แล้ว ต้องการเล่นอะไรคะ?"
|
|
54
|
-
→ suggestions: เพลง Chill, เพลง Focus, Top Charts, Podcast
|
|
55
|
-
|
|
56
|
-
SCENARIO: User opened Terminal
|
|
57
|
-
→ message: "เปิด Terminal แล้ว ต้องการทำอะไรคะ?"
|
|
58
|
-
→ suggestions: GitHub, Stack Overflow, DevDocs, ค้นหา Command
|
|
59
|
-
|
|
60
|
-
BAD examples (return null):
|
|
61
|
-
- Nothing notable on screen
|
|
62
|
-
- User is actively typing
|
|
63
|
-
- Same context as before
|
|
64
|
-
`;
|
|
65
|
-
|
|
66
|
-
let lastSuggestionContext = '';
|
|
67
|
-
let lastSuggestionTime = 0;
|
|
68
|
-
|
|
69
|
-
function resolveGeminiModel() {
|
|
70
|
-
try {
|
|
71
|
-
const cfg = readConfig();
|
|
72
|
-
const model = (cfg.geminiModel || '').trim();
|
|
73
|
-
return model || DEFAULT_GEMINI_MODEL;
|
|
74
|
-
} catch {
|
|
75
|
-
return DEFAULT_GEMINI_MODEL;
|
|
76
|
-
}
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function getMinSuggestionIntervalMs() {
|
|
80
|
-
try {
|
|
81
|
-
const cfg = readConfig();
|
|
82
|
-
return (cfg.proactiveCooldown || 120) * 1000;
|
|
83
|
-
} catch {
|
|
84
|
-
// ignore
|
|
85
|
-
}
|
|
86
|
-
return 120_000;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/**
|
|
90
|
-
* Analyze screen and return a multi-choice suggestion object.
|
|
91
|
-
* @param {string} base64Image
|
|
92
|
-
* @param {string} behaviorSummary
|
|
93
|
-
* @returns {Promise<{message: string, context: string, suggestions: Array} | null>}
|
|
94
|
-
*/
|
|
95
|
-
async function analyzeAndSuggest(base64Image, behaviorSummary) {
|
|
96
|
-
try {
|
|
97
|
-
const cfg = readConfig();
|
|
98
|
-
const apiKey = cfg.apiKey || process.env.GEMINI_API_KEY;
|
|
99
|
-
if (!apiKey) return null; // silently skip if no API key configured
|
|
100
|
-
const model = (cfg.geminiModel || '').trim() || DEFAULT_GEMINI_MODEL;
|
|
101
|
-
if (model && model !== lastLoggedModel) {
|
|
102
|
-
console.log(`[Gemini] Proactive Engine model: ${model}`);
|
|
103
|
-
lastLoggedModel = model;
|
|
104
|
-
}
|
|
105
|
-
const ai = getAi(apiKey);
|
|
106
|
-
|
|
107
|
-
const now = Date.now();
|
|
108
|
-
const minInterval = getMinSuggestionIntervalMs();
|
|
109
|
-
|
|
110
|
-
if (now - lastSuggestionTime < minInterval) return null;
|
|
111
|
-
|
|
112
|
-
const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, '');
|
|
113
|
-
|
|
114
|
-
const userMessage = [
|
|
115
|
-
{
|
|
116
|
-
text: `Analyze the screen and generate smart multi-choice suggestions for the user.
|
|
117
|
-
|
|
118
|
-
User behavior context: ${behaviorSummary || 'No history yet.'}
|
|
119
|
-
|
|
120
|
-
Rules: Only suggest if you see a clear opportunity. Return 2–4 relevant chips. Return null message if nothing notable.`
|
|
121
|
-
},
|
|
122
|
-
{
|
|
123
|
-
inlineData: {
|
|
124
|
-
mimeType: 'image/png',
|
|
125
|
-
data: base64Data
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
];
|
|
129
|
-
|
|
130
|
-
const response = await ai.models.generateContent({
|
|
131
|
-
model,
|
|
132
|
-
config: {
|
|
133
|
-
systemInstruction: PROACTIVE_SYSTEM_PROMPT,
|
|
134
|
-
responseMimeType: 'application/json'
|
|
135
|
-
},
|
|
136
|
-
contents: [{ role: 'user', parts: userMessage }]
|
|
137
|
-
});
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
let parsed;
|
|
141
|
-
try {
|
|
142
|
-
parsed = JSON.parse(response.text);
|
|
143
|
-
} catch {
|
|
144
|
-
const jsonMatch = response.text.match(/\{[\s\S]*\}/);
|
|
145
|
-
if (jsonMatch) parsed = JSON.parse(jsonMatch[0]);
|
|
146
|
-
else return null;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
// Validate: must have message and at least 1 suggestion
|
|
150
|
-
if (!parsed || !parsed.message || !Array.isArray(parsed.suggestions) || parsed.suggestions.length === 0) {
|
|
151
|
-
return null;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
// Skip repeat context
|
|
155
|
-
if (parsed.context && parsed.context === lastSuggestionContext) {
|
|
156
|
-
console.log('[ProactiveEngine] Skipping repeat context.');
|
|
157
|
-
return null;
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
lastSuggestionContext = parsed.context || '';
|
|
161
|
-
lastSuggestionTime = now;
|
|
162
|
-
|
|
163
|
-
console.log(`[ProactiveEngine] ${parsed.suggestions.length} suggestions for: ${parsed.context}`);
|
|
164
|
-
return parsed;
|
|
165
|
-
|
|
166
|
-
} catch (err) {
|
|
167
|
-
console.error('[ProactiveEngine] Error:', err.message);
|
|
168
|
-
return null;
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
module.exports = { analyzeAndSuggest };
|
|
@@ -1,365 +0,0 @@
|
|
|
1
|
-
const axios = require('axios');
|
|
2
|
-
const { GoogleGenAI } = require('@google/genai');
|
|
3
|
-
const { getAvailableProviders } = require('../System/config_manager');
|
|
4
|
-
|
|
5
|
-
const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
|
|
6
|
-
const ALL_PROVIDERS = ['anthropic', 'openai', 'gemini', 'local_openai', 'ollama', 'huggingface'];
|
|
7
|
-
|
|
8
|
-
function splitDataUri(dataUri = '') {
|
|
9
|
-
const match = String(dataUri).match(/^data:([^;]+);base64,([\s\S]+)$/);
|
|
10
|
-
if (!match) return null;
|
|
11
|
-
return {
|
|
12
|
-
mimeType: match[1],
|
|
13
|
-
data: match[2]
|
|
14
|
-
};
|
|
15
|
-
}
|
|
16
|
-
|
|
17
|
-
function contentToText(content) {
|
|
18
|
-
if (content && typeof content === 'object' && !Array.isArray(content)) {
|
|
19
|
-
return String(content.text || '');
|
|
20
|
-
}
|
|
21
|
-
return String(content || '');
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
function contentToGeminiParts(content) {
|
|
25
|
-
const text = contentToText(content);
|
|
26
|
-
const parts = text ? [{ text }] : [];
|
|
27
|
-
if (content && typeof content === 'object') {
|
|
28
|
-
const images = Array.isArray(content.imageDataUris)
|
|
29
|
-
? content.imageDataUris
|
|
30
|
-
: (content.imageDataUri ? [content.imageDataUri] : []);
|
|
31
|
-
for (const item of images) {
|
|
32
|
-
const image = splitDataUri(item);
|
|
33
|
-
if (image) parts.push({ inlineData: { mimeType: image.mimeType, data: image.data } });
|
|
34
|
-
}
|
|
35
|
-
if (content.audioDataUri) {
|
|
36
|
-
const audio = splitDataUri(content.audioDataUri);
|
|
37
|
-
if (audio) parts.push({ inlineData: { mimeType: audio.mimeType, data: audio.data } });
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
return parts.length > 0 ? parts : [{ text: '' }];
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
function contentToOpenAIContent(content) {
|
|
44
|
-
const text = contentToText(content) || 'Analyze this input.';
|
|
45
|
-
if (content && typeof content === 'object') {
|
|
46
|
-
const images = Array.isArray(content.imageDataUris)
|
|
47
|
-
? content.imageDataUris
|
|
48
|
-
: (content.imageDataUri ? [content.imageDataUri] : []);
|
|
49
|
-
if (images.length > 0) {
|
|
50
|
-
return [
|
|
51
|
-
{ type: 'text', text },
|
|
52
|
-
...images.map(item => ({ type: 'image_url', image_url: { url: item } }))
|
|
53
|
-
];
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
return text;
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
function contentToAnthropicContent(content) {
|
|
60
|
-
const text = contentToText(content) || 'Analyze this input.';
|
|
61
|
-
if (content && typeof content === 'object') {
|
|
62
|
-
const images = Array.isArray(content.imageDataUris)
|
|
63
|
-
? content.imageDataUris
|
|
64
|
-
: (content.imageDataUri ? [content.imageDataUri] : []);
|
|
65
|
-
if (images.length > 0) {
|
|
66
|
-
const blocks = [];
|
|
67
|
-
for (const item of images) {
|
|
68
|
-
const image = splitDataUri(item);
|
|
69
|
-
if (image) {
|
|
70
|
-
blocks.push({ type: 'image', source: { type: 'base64', media_type: image.mimeType, data: image.data } });
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
blocks.push({ type: 'text', text });
|
|
74
|
-
return blocks;
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
return text;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
function contentToOllamaMessage(content) {
|
|
81
|
-
const text = contentToText(content) || 'Analyze this input.';
|
|
82
|
-
const message = { role: 'user', content: text };
|
|
83
|
-
if (content && typeof content === 'object') {
|
|
84
|
-
const images = Array.isArray(content.imageDataUris)
|
|
85
|
-
? content.imageDataUris
|
|
86
|
-
: (content.imageDataUri ? [content.imageDataUri] : []);
|
|
87
|
-
const imagePayloads = images
|
|
88
|
-
.map(item => splitDataUri(item))
|
|
89
|
-
.filter(Boolean)
|
|
90
|
-
.map(image => image.data);
|
|
91
|
-
if (imagePayloads.length > 0) message.images = imagePayloads;
|
|
92
|
-
}
|
|
93
|
-
return message;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function getProviderAttemptOrder(config = {}, options = {}) {
|
|
97
|
-
const supported = options.supported || ALL_PROVIDERS;
|
|
98
|
-
const available = (options.availableProviders || getAvailableProviders(config))
|
|
99
|
-
.filter(provider => supported.includes(provider));
|
|
100
|
-
const requested = options.requested || config.aiProvider || 'gemini';
|
|
101
|
-
const priority = (options.priority || ALL_PROVIDERS).filter(provider => supported.includes(provider));
|
|
102
|
-
const ordered = [];
|
|
103
|
-
|
|
104
|
-
if (supported.includes(requested) && available.includes(requested)) {
|
|
105
|
-
ordered.push(requested);
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
for (const provider of priority) {
|
|
109
|
-
if (available.includes(provider) && !ordered.includes(provider)) {
|
|
110
|
-
ordered.push(provider);
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
return ordered.length > 0 ? ordered : ['gemini'];
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
function getProviderModel(provider, config = {}) {
|
|
118
|
-
switch (provider) {
|
|
119
|
-
case 'anthropic':
|
|
120
|
-
return config.anthropicModel || 'claude-3-5-sonnet-latest';
|
|
121
|
-
case 'openai':
|
|
122
|
-
return config.openaiModel || 'gpt-4o';
|
|
123
|
-
case 'local_openai':
|
|
124
|
-
return config.localModelName || 'local-model';
|
|
125
|
-
case 'ollama':
|
|
126
|
-
return config.ollamaModel || 'llama3:latest';
|
|
127
|
-
case 'huggingface':
|
|
128
|
-
return config.hfModel || 'meta-llama/Meta-Llama-3-8B-Instruct';
|
|
129
|
-
case 'gemini':
|
|
130
|
-
default:
|
|
131
|
-
return config.geminiModel || DEFAULT_GEMINI_MODEL;
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
class AgentProviderClient {
|
|
136
|
-
constructor(options = {}) {
|
|
137
|
-
this.provider = options.provider || 'gemini';
|
|
138
|
-
this.providerOrder = options.providerOrder && options.providerOrder.length
|
|
139
|
-
? options.providerOrder
|
|
140
|
-
: [this.provider];
|
|
141
|
-
this.config = options.config || {};
|
|
142
|
-
this.history = options.history || [];
|
|
143
|
-
this.systemInstruction = options.systemInstruction || '';
|
|
144
|
-
this.responseMimeType = options.responseMimeType || 'application/json';
|
|
145
|
-
this.maxTokens = options.maxTokens || 8192;
|
|
146
|
-
this.lastSuccessfulProvider = null;
|
|
147
|
-
this.usageTotals = {};
|
|
148
|
-
}
|
|
149
|
-
|
|
150
|
-
recordUsage(provider, model, usage = {}) {
|
|
151
|
-
const key = `${provider}:${model || ''}`;
|
|
152
|
-
if (!this.usageTotals[key]) {
|
|
153
|
-
this.usageTotals[key] = {
|
|
154
|
-
provider,
|
|
155
|
-
model,
|
|
156
|
-
requests: 0,
|
|
157
|
-
inputTokens: 0,
|
|
158
|
-
cacheReads: 0,
|
|
159
|
-
outputTokens: 0,
|
|
160
|
-
reasoningTokens: 0,
|
|
161
|
-
totalTokens: 0
|
|
162
|
-
};
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
const row = this.usageTotals[key];
|
|
166
|
-
row.requests += 1;
|
|
167
|
-
row.inputTokens += Number(usage.inputTokens) || 0;
|
|
168
|
-
row.cacheReads += Number(usage.cacheReads) || 0;
|
|
169
|
-
row.outputTokens += Number(usage.outputTokens) || 0;
|
|
170
|
-
row.reasoningTokens += Number(usage.reasoningTokens) || 0;
|
|
171
|
-
row.totalTokens += Number(usage.totalTokens) || 0;
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
getUsageSummary() {
|
|
175
|
-
return Object.values(this.usageTotals);
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
async sendMessage(observation) {
|
|
179
|
-
this.history.push({ role: 'user', content: observation });
|
|
180
|
-
|
|
181
|
-
const failures = [];
|
|
182
|
-
for (const provider of this.providerOrder) {
|
|
183
|
-
this.provider = provider;
|
|
184
|
-
try {
|
|
185
|
-
let responseText = '';
|
|
186
|
-
if (provider === 'anthropic') responseText = await this.callAnthropic();
|
|
187
|
-
else if (provider === 'openai' || provider === 'local_openai') responseText = await this.callOpenAI();
|
|
188
|
-
else if (provider === 'ollama') responseText = await this.callOllama();
|
|
189
|
-
else if (provider === 'huggingface') responseText = await this.callHuggingFace();
|
|
190
|
-
else responseText = await this.callGemini();
|
|
191
|
-
|
|
192
|
-
this.history.push({ role: 'assistant', content: responseText });
|
|
193
|
-
this.lastSuccessfulProvider = provider;
|
|
194
|
-
return responseText;
|
|
195
|
-
} catch (error) {
|
|
196
|
-
const message = error.message || error.code || 'unknown error';
|
|
197
|
-
failures.push(`${provider}: ${message}`);
|
|
198
|
-
if (process.env.MINT_DEBUG === '1') {
|
|
199
|
-
console.error(`[ProviderAdapter] Provider '${provider}' failed: ${message}`);
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
throw new Error(`All providers failed. ${failures.join(' | ')}`);
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
async callAnthropic() {
|
|
208
|
-
const apiKey = this.config.anthropicApiKey || process.env.ANTHROPIC_API_KEY;
|
|
209
|
-
const model = getProviderModel('anthropic', this.config);
|
|
210
|
-
const messages = this.history.map(m => ({
|
|
211
|
-
role: m.role,
|
|
212
|
-
content: contentToAnthropicContent(m.content)
|
|
213
|
-
}));
|
|
214
|
-
|
|
215
|
-
const response = await axios.post('https://api.anthropic.com/v1/messages', {
|
|
216
|
-
model,
|
|
217
|
-
max_tokens: this.maxTokens,
|
|
218
|
-
system: this.systemInstruction,
|
|
219
|
-
messages
|
|
220
|
-
}, {
|
|
221
|
-
headers: {
|
|
222
|
-
'x-api-key': apiKey,
|
|
223
|
-
'anthropic-version': '2023-06-01',
|
|
224
|
-
'content-type': 'application/json'
|
|
225
|
-
}
|
|
226
|
-
});
|
|
227
|
-
const usage = response.data.usage || {};
|
|
228
|
-
this.recordUsage('anthropic', model, {
|
|
229
|
-
inputTokens: usage.input_tokens,
|
|
230
|
-
cacheReads: usage.cache_read_input_tokens,
|
|
231
|
-
outputTokens: usage.output_tokens,
|
|
232
|
-
totalTokens: (Number(usage.input_tokens) || 0) + (Number(usage.output_tokens) || 0)
|
|
233
|
-
});
|
|
234
|
-
return response.data.content[0].text;
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
async callOpenAI() {
|
|
238
|
-
const isLocal = this.provider === 'local_openai';
|
|
239
|
-
const apiKey = isLocal ? 'not-needed' : (this.config.openaiApiKey || process.env.OPENAI_API_KEY);
|
|
240
|
-
const baseUrl = isLocal ? (this.config.localApiBaseUrl || 'http://localhost:1234/v1') : 'https://api.openai.com/v1';
|
|
241
|
-
const model = getProviderModel(this.provider, this.config);
|
|
242
|
-
const messages = [
|
|
243
|
-
{ role: 'system', content: this.systemInstruction },
|
|
244
|
-
...this.history.map(m => ({
|
|
245
|
-
role: m.role,
|
|
246
|
-
content: contentToOpenAIContent(m.content)
|
|
247
|
-
}))
|
|
248
|
-
];
|
|
249
|
-
|
|
250
|
-
const response = await axios.post(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
|
|
251
|
-
model,
|
|
252
|
-
messages,
|
|
253
|
-
response_format: isLocal ? undefined : { type: 'json_object' }
|
|
254
|
-
}, {
|
|
255
|
-
headers: {
|
|
256
|
-
'Authorization': `Bearer ${apiKey}`,
|
|
257
|
-
'Content-Type': 'application/json'
|
|
258
|
-
}
|
|
259
|
-
});
|
|
260
|
-
const usage = response.data.usage || {};
|
|
261
|
-
this.recordUsage(this.provider, model, {
|
|
262
|
-
inputTokens: usage.prompt_tokens,
|
|
263
|
-
cacheReads: usage.prompt_tokens_details && usage.prompt_tokens_details.cached_tokens,
|
|
264
|
-
outputTokens: usage.completion_tokens,
|
|
265
|
-
reasoningTokens: usage.completion_tokens_details && usage.completion_tokens_details.reasoning_tokens,
|
|
266
|
-
totalTokens: usage.total_tokens
|
|
267
|
-
});
|
|
268
|
-
return response.data.choices[0].message.content;
|
|
269
|
-
}
|
|
270
|
-
|
|
271
|
-
async callGemini() {
|
|
272
|
-
const apiKey = this.config.apiKey || process.env.GEMINI_API_KEY;
|
|
273
|
-
const model = getProviderModel('gemini', this.config);
|
|
274
|
-
const ai = new GoogleGenAI({ apiKey });
|
|
275
|
-
const recentHistory = this.history.slice(-16);
|
|
276
|
-
const priorHistory = recentHistory.slice(0, -1);
|
|
277
|
-
const lastEntry = recentHistory[recentHistory.length - 1] || { content: '' };
|
|
278
|
-
const history = priorHistory.map(m => ({
|
|
279
|
-
role: m.role === 'assistant' ? 'model' : 'user',
|
|
280
|
-
parts: contentToGeminiParts(m.content)
|
|
281
|
-
}));
|
|
282
|
-
const chat = ai.chats.create({
|
|
283
|
-
model,
|
|
284
|
-
config: {
|
|
285
|
-
systemInstruction: this.systemInstruction,
|
|
286
|
-
responseMimeType: this.responseMimeType,
|
|
287
|
-
safetySettings: [
|
|
288
|
-
{ category: "HARM_CATEGORY_SEXUALLY_EXPLICIT", threshold: "BLOCK_NONE" },
|
|
289
|
-
{ category: "HARM_CATEGORY_HATE_SPEECH", threshold: "BLOCK_NONE" },
|
|
290
|
-
{ category: "HARM_CATEGORY_HARASSMENT", threshold: "BLOCK_NONE" },
|
|
291
|
-
{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", threshold: "BLOCK_NONE" },
|
|
292
|
-
{ category: "HARM_CATEGORY_CIVIC_INTEGRITY", threshold: "BLOCK_NONE" }
|
|
293
|
-
]
|
|
294
|
-
},
|
|
295
|
-
history
|
|
296
|
-
});
|
|
297
|
-
|
|
298
|
-
const response = await chat.sendMessage({ message: contentToGeminiParts(lastEntry.content) });
|
|
299
|
-
const usage = response.usageMetadata || {};
|
|
300
|
-
this.recordUsage('gemini', model, {
|
|
301
|
-
inputTokens: usage.promptTokenCount,
|
|
302
|
-
cacheReads: usage.cachedContentTokenCount,
|
|
303
|
-
outputTokens: usage.candidatesTokenCount,
|
|
304
|
-
reasoningTokens: usage.thoughtsTokenCount,
|
|
305
|
-
totalTokens: usage.totalTokenCount
|
|
306
|
-
});
|
|
307
|
-
return typeof response.text === 'function' ? response.text() : response.text;
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
async callOllama() {
|
|
311
|
-
const model = getProviderModel('ollama', this.config);
|
|
312
|
-
const baseUrl = (this.config.ollamaHost || 'http://localhost:11434').replace(/\/$/, '');
|
|
313
|
-
const messages = [
|
|
314
|
-
{ role: 'system', content: this.systemInstruction },
|
|
315
|
-
...this.history.map(m => m.role === 'assistant'
|
|
316
|
-
? { role: 'assistant', content: contentToText(m.content) }
|
|
317
|
-
: contentToOllamaMessage(m.content))
|
|
318
|
-
];
|
|
319
|
-
const response = await axios.post(`${baseUrl}/api/chat`, {
|
|
320
|
-
model,
|
|
321
|
-
messages,
|
|
322
|
-
format: this.responseMimeType === 'application/json' ? 'json' : undefined,
|
|
323
|
-
stream: false
|
|
324
|
-
});
|
|
325
|
-
return response.data.message.content;
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
async callHuggingFace() {
|
|
329
|
-
const apiKey = this.config.hfApiKey || process.env.HF_API_KEY;
|
|
330
|
-
const model = getProviderModel('huggingface', this.config);
|
|
331
|
-
const messages = [
|
|
332
|
-
{ role: 'system', content: this.systemInstruction },
|
|
333
|
-
...this.history.map(m => ({
|
|
334
|
-
role: m.role,
|
|
335
|
-
content: contentToOpenAIContent(m.content)
|
|
336
|
-
}))
|
|
337
|
-
];
|
|
338
|
-
const response = await axios.post(`https://api-inference.huggingface.co/models/${model}/v1/chat/completions`, {
|
|
339
|
-
model,
|
|
340
|
-
messages,
|
|
341
|
-
max_tokens: this.maxTokens
|
|
342
|
-
}, {
|
|
343
|
-
headers: {
|
|
344
|
-
'Authorization': `Bearer ${apiKey}`,
|
|
345
|
-
'Content-Type': 'application/json'
|
|
346
|
-
}
|
|
347
|
-
});
|
|
348
|
-
return response.data.choices[0].message.content;
|
|
349
|
-
}
|
|
350
|
-
}
|
|
351
|
-
|
|
352
|
-
module.exports = {
|
|
353
|
-
DEFAULT_GEMINI_MODEL,
|
|
354
|
-
ALL_PROVIDERS,
|
|
355
|
-
AgentProviderClient,
|
|
356
|
-
getProviderAttemptOrder,
|
|
357
|
-
getProviderModel,
|
|
358
|
-
_helpers: {
|
|
359
|
-
splitDataUri,
|
|
360
|
-
contentToGeminiParts,
|
|
361
|
-
contentToOpenAIContent,
|
|
362
|
-
contentToAnthropicContent,
|
|
363
|
-
contentToOllamaMessage
|
|
364
|
-
}
|
|
365
|
-
};
|
|
@@ -1,149 +0,0 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const { requireOptional } = require('../System/optional_require');
|
|
4
|
-
const { GoogleGenAI } = require('@google/genai');
|
|
5
|
-
const { readConfig } = require('../System/config_manager');
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
const BROWSER_SYSTEM_PROMPT = `You are an Autonomous Browser Agent. Your goal is to fulfill the user's web instruction by driving a headless browser.
|
|
10
|
-
|
|
11
|
-
CRITICAL INSTRUCTIONS:
|
|
12
|
-
Always respond EXACTLY with valid JSON. NO MARKDOWN. NO CODE BLOCKS (\`\`\`json). The JSON must have this exact structure:
|
|
13
|
-
{
|
|
14
|
-
"thought": "Reasoning about what to do next based on the current page content and goal.",
|
|
15
|
-
"action": "goto" | "click" | "eval" | "done",
|
|
16
|
-
"target": "URL for goto | CSS selector for click | JavaScript expression for eval | Final answer for done"
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
Actions:
|
|
20
|
-
- "goto": Navigate to the specified URL. Target MUST be a full URL (e.g. "https://www.google.com/search?q=AI+news")
|
|
21
|
-
- "click": Click an element. Target MUST be a valid CSS selector.
|
|
22
|
-
- "eval": Evaluate JavaScript to extract text. Target MUST be JS code returning a string (e.g. "document.body.innerText.substring(0, 1000)").
|
|
23
|
-
- "done": Task finished. Target MUST be the final summary or answer to present to the user.
|
|
24
|
-
|
|
25
|
-
You will receive the result of your previous action in the next message. If you get stuck or fail, try another approach or use "done" to report the failure.`;
|
|
26
|
-
|
|
27
|
-
const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
|
|
28
|
-
let lastLoggedModel = '';
|
|
29
|
-
|
|
30
|
-
function resolveGeminiModel() {
|
|
31
|
-
try {
|
|
32
|
-
const cfg = readConfig();
|
|
33
|
-
const model = (cfg.geminiModel || '').trim();
|
|
34
|
-
return model || DEFAULT_GEMINI_MODEL;
|
|
35
|
-
} catch {
|
|
36
|
-
return DEFAULT_GEMINI_MODEL;
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
async function performWebAutomation(query) {
|
|
41
|
-
if (!query) return 'No query provided.';
|
|
42
|
-
|
|
43
|
-
// Dynamic require — user must install puppeteer separately
|
|
44
|
-
const puppeteer = requireOptional('puppeteer', 'npm install puppeteer');
|
|
45
|
-
|
|
46
|
-
const config = readConfig();
|
|
47
|
-
const apiKey = config.apiKey || process.env.GEMINI_API_KEY;
|
|
48
|
-
const ai = new GoogleGenAI({ apiKey });
|
|
49
|
-
const browserPath = config.automationBrowser;
|
|
50
|
-
|
|
51
|
-
let browser;
|
|
52
|
-
try {
|
|
53
|
-
const launchOptions = {
|
|
54
|
-
headless: false,
|
|
55
|
-
defaultViewport: null,
|
|
56
|
-
args: ['--start-maximized']
|
|
57
|
-
};
|
|
58
|
-
|
|
59
|
-
if (browserPath && browserPath !== 'chromium') {
|
|
60
|
-
launchOptions.executablePath = browserPath;
|
|
61
|
-
if (browserPath.toLowerCase().includes('firefox')) {
|
|
62
|
-
launchOptions.browser = 'firefox';
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
browser = await puppeteer.launch(launchOptions);
|
|
67
|
-
const page = await browser.newPage();
|
|
68
|
-
|
|
69
|
-
const model = resolveGeminiModel();
|
|
70
|
-
if (model && model !== lastLoggedModel) {
|
|
71
|
-
console.log(`[Gemini] Web Automation model: ${model}`);
|
|
72
|
-
lastLoggedModel = model;
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
const chat = ai.chats.create({
|
|
76
|
-
model,
|
|
77
|
-
config: {
|
|
78
|
-
systemInstruction: BROWSER_SYSTEM_PROMPT,
|
|
79
|
-
responseMimeType: 'application/json'
|
|
80
|
-
}
|
|
81
|
-
});
|
|
82
|
-
|
|
83
|
-
let currentObservation = `Goal: ${query}\nSystem Note: You have a blank browser page. What is your first action? Start by using "goto" to navigate to a relevant search engine or website.`;
|
|
84
|
-
|
|
85
|
-
let maxSteps = 10;
|
|
86
|
-
let step = 0;
|
|
87
|
-
|
|
88
|
-
while (step < maxSteps) {
|
|
89
|
-
step++;
|
|
90
|
-
console.log(`\n--- Agent Step ${step} ---`);
|
|
91
|
-
console.log(`Observation:`, currentObservation.substring(0, 150) + (currentObservation.length > 150 ? '...' : ''));
|
|
92
|
-
|
|
93
|
-
const response = await chat.sendMessage({ message: currentObservation });
|
|
94
|
-
|
|
95
|
-
let parsed;
|
|
96
|
-
try {
|
|
97
|
-
const text = response.text;
|
|
98
|
-
const cleanText = text.replace(/^```json\n/, '').replace(/\n```$/, '').trim();
|
|
99
|
-
parsed = JSON.parse(cleanText);
|
|
100
|
-
} catch (e) {
|
|
101
|
-
console.error('Agent failed to return valid JSON:', response.text);
|
|
102
|
-
currentObservation = 'Error: Invalid JSON returned. Please reply with ONLY valid JSON matching the schema.';
|
|
103
|
-
continue;
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
console.log('Agent Thought:', parsed.thought);
|
|
107
|
-
console.log('Agent Action:', parsed.action);
|
|
108
|
-
console.log('Agent Target:', parsed.target);
|
|
109
|
-
|
|
110
|
-
const { action, target } = parsed;
|
|
111
|
-
|
|
112
|
-
if (action === 'done') {
|
|
113
|
-
console.log('Agent finished with answer:', target);
|
|
114
|
-
return `🤖 Web Automation Result: ${target}`;
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
try {
|
|
118
|
-
if (action === 'goto') {
|
|
119
|
-
await page.goto(target, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
120
|
-
const pageTitle = await page.title();
|
|
121
|
-
currentObservation = `Successfully navigated to ${pageTitle}. ` + await page.evaluate(() => document.body.innerText.substring(0, 1500));
|
|
122
|
-
} else if (action === 'click') {
|
|
123
|
-
await page.waitForSelector(target, { timeout: 5000 });
|
|
124
|
-
await page.click(target);
|
|
125
|
-
await new Promise(r => setTimeout(r, 2000));
|
|
126
|
-
const pageTitle = await page.title();
|
|
127
|
-
currentObservation = `Clicked element. Current page: ${pageTitle}. ` + await page.evaluate(() => document.body.innerText.substring(0, 1500));
|
|
128
|
-
} else if (action === 'eval') {
|
|
129
|
-
const evalResult = await page.evaluate(target);
|
|
130
|
-
currentObservation = `Eval result: ` + String(evalResult).substring(0, 1500);
|
|
131
|
-
} else {
|
|
132
|
-
currentObservation = `Error: Unknown action type "${action}".`;
|
|
133
|
-
}
|
|
134
|
-
} catch (actionError) {
|
|
135
|
-
console.error('Action execution failed:', actionError);
|
|
136
|
-
currentObservation = `Action failed: ${actionError.message}. Please try again or use another method (for instance, try a different CSS selector or just read the current page).`;
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
return 'Agent reached maximum steps (10) without finding a final answer.';
|
|
141
|
-
|
|
142
|
-
} catch (error) {
|
|
143
|
-
console.error('Web Automation Error:', error);
|
|
144
|
-
if (browser) browser.close();
|
|
145
|
-
return `I encountered an overall error while automating the browser: ${error.message}`;
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
module.exports = { performWebAutomation };
|