@pheem49/mint 1.5.5 → 1.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (222) hide show
  1. package/.codex +0 -0
  2. package/.github/FUNDING.yml +2 -0
  3. package/.github/workflows/ci.yml +45 -0
  4. package/.github/workflows/release.yml +79 -0
  5. package/Cargo.lock +5792 -0
  6. package/Cargo.toml +32 -0
  7. package/README.md +387 -353
  8. package/assets/icon.png +0 -0
  9. package/bin/mint +0 -0
  10. package/crates/mint-cli/Cargo.toml +23 -0
  11. package/crates/mint-cli/src/agent.rs +851 -0
  12. package/crates/mint-cli/src/gmail.rs +216 -0
  13. package/crates/mint-cli/src/image.rs +142 -0
  14. package/crates/mint-cli/src/main.rs +2837 -0
  15. package/crates/mint-cli/src/mcp.rs +63 -0
  16. package/crates/mint-cli/src/onboard.rs +1149 -0
  17. package/crates/mint-cli/src/setup.rs +390 -0
  18. package/crates/mint-cli/src/skills.rs +8 -0
  19. package/crates/mint-cli/src/updater.rs +279 -0
  20. package/crates/mint-core/Cargo.toml +22 -0
  21. package/crates/mint-core/src/agent_loop.rs +94 -0
  22. package/crates/mint-core/src/api_server.rs +991 -0
  23. package/crates/mint-core/src/channels.rs +248 -0
  24. package/crates/mint-core/src/chat.rs +895 -0
  25. package/crates/mint-core/src/code_tools.rs +729 -0
  26. package/crates/mint-core/src/config.rs +368 -0
  27. package/crates/mint-core/src/files.rs +159 -0
  28. package/crates/mint-core/src/knowledge.rs +541 -0
  29. package/crates/mint-core/src/lib.rs +84 -0
  30. package/crates/mint-core/src/mcp.rs +273 -0
  31. package/crates/mint-core/src/memory.rs +673 -0
  32. package/crates/mint-core/src/orchestration.rs +2157 -0
  33. package/crates/mint-core/src/pictures.rs +314 -0
  34. package/crates/mint-core/src/plugins.rs +727 -0
  35. package/crates/mint-core/src/safety.rs +416 -0
  36. package/crates/mint-core/src/semantic.rs +254 -0
  37. package/crates/mint-core/src/shell.rs +317 -0
  38. package/crates/mint-core/src/skills.rs +71 -0
  39. package/crates/mint-core/src/symbols.rs +157 -0
  40. package/crates/mint-core/src/tasks.rs +308 -0
  41. package/crates/mint-core/src/tts.rs +92 -0
  42. package/crates/mint-core/src/weather.rs +93 -0
  43. package/crates/mint-core/src/web_search.rs +200 -0
  44. package/crates/mint-core/src/workflows.rs +81 -0
  45. package/crates/mint-core/tests/mcp_stdio.rs +45 -0
  46. package/crates/mint-core/tests/memory_persistence.rs +172 -0
  47. package/crates/mint-core/tests/pictures_storage.rs +14 -0
  48. package/crates/mint-core/tests/task_lifecycle.rs +87 -0
  49. package/package.json +35 -99
  50. package/src/bin/index.js +16 -0
  51. package/src/renderer/index-web.html +17 -0
  52. package/src/renderer/index.html +17 -0
  53. package/src/renderer/public/Live2DCubismCore.js +9 -0
  54. package/src/renderer/public/assets/icon.png +0 -0
  55. package/src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.model3.json +36 -0
  56. package/src/renderer/src/App.tsx +33 -0
  57. package/src/renderer/src/calculator.ts +47 -0
  58. package/src/renderer/src/components/ChatPanel.tsx +1598 -0
  59. package/src/renderer/src/components/DashboardSidebar.tsx +358 -0
  60. package/src/renderer/src/components/Live2DStage.tsx +374 -0
  61. package/src/renderer/src/components/MintDashboard.tsx +950 -0
  62. package/src/renderer/src/components/ModelPanel.tsx +154 -0
  63. package/src/renderer/src/components/PicturesLibrary.tsx +46 -0
  64. package/src/renderer/src/components/ProactiveGlow.tsx +19 -0
  65. package/src/renderer/src/components/ScreenPicker.tsx +579 -0
  66. package/src/renderer/src/components/SettingsWindow.tsx +1467 -0
  67. package/src/renderer/src/components/SpotlightWindow.tsx +280 -0
  68. package/src/renderer/src/components/WidgetWindow.tsx +36 -0
  69. package/src/renderer/src/components/WorkspacePanel.tsx +268 -0
  70. package/src/{UI → renderer/src/css}/settings.css +69 -16
  71. package/src/renderer/src/css/spotlight.css +113 -0
  72. package/src/renderer/src/css/styles.css +3722 -0
  73. package/src/renderer/src/css/widget.css +185 -0
  74. package/src/renderer/src/env.d.ts +116 -0
  75. package/src/renderer/src/index.css +379 -0
  76. package/src/renderer/src/main.tsx +13 -0
  77. package/src/renderer/src/tauri.ts +996 -0
  78. package/src/renderer/src-web/App.tsx +25 -0
  79. package/src/renderer/src-web/calculator.ts +47 -0
  80. package/src/renderer/src-web/components/ChatPanel.tsx +1662 -0
  81. package/src/renderer/src-web/components/DashboardSidebar.tsx +242 -0
  82. package/src/renderer/src-web/components/MintDashboard.tsx +763 -0
  83. package/src/renderer/src-web/components/PicturesLibrary.tsx +73 -0
  84. package/src/renderer/src-web/components/SettingsWindow.tsx +1500 -0
  85. package/src/renderer/src-web/css/settings.css +1100 -0
  86. package/src/{UI → renderer/src-web/css}/spotlight.css +4 -4
  87. package/src/{UI → renderer/src-web/css}/styles.css +1055 -159
  88. package/src/{UI → renderer/src-web/css}/widget.css +2 -2
  89. package/src/renderer/src-web/env.d.ts +107 -0
  90. package/src/renderer/src-web/index.css +379 -0
  91. package/src/renderer/src-web/main.tsx +13 -0
  92. package/src/renderer/src-web/tauri.ts +983 -0
  93. package/tsconfig.json +30 -0
  94. package/vite.config.ts +33 -0
  95. package/vite.config.web.ts +51 -0
  96. package/GUIDE_TH.md +0 -125
  97. package/assets/Agent_Mint.png +0 -0
  98. package/assets/CLI_Screen.png +0 -0
  99. package/assets/Settings.png +0 -0
  100. package/benchmark_ai.js +0 -71
  101. package/install.ps1 +0 -64
  102. package/install.sh +0 -54
  103. package/main.js +0 -139
  104. package/mint-cli-logic.js +0 -3
  105. package/mint-cli.js +0 -410
  106. package/models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.model3.json +0 -47
  107. package/models/Shiroko_Model/Shiroko//342/232/241/351/253/230/344/272/256/342/232/241/344/275/277/347/224/250/346/225/231/347/250/213/344/270/216/346/263/250/346/204/217/344/272/213/351/241/271.txt +0 -23
  108. package/preload-picker.js +0 -11
  109. package/preload-settings.js +0 -11
  110. package/preload.js +0 -41
  111. package/scripts/install_linux_desktop_entry.js +0 -48
  112. package/src/AI_Brain/Gemini_API.js +0 -813
  113. package/src/AI_Brain/agent_orchestrator.js +0 -73
  114. package/src/AI_Brain/autonomous_brain.js +0 -179
  115. package/src/AI_Brain/behavior_memory.js +0 -135
  116. package/src/AI_Brain/headless_agent.js +0 -143
  117. package/src/AI_Brain/knowledge_base.js +0 -349
  118. package/src/AI_Brain/memory_store.js +0 -662
  119. package/src/AI_Brain/proactive_engine.js +0 -172
  120. package/src/AI_Brain/provider_adapter.js +0 -365
  121. package/src/Automation_Layer/browser_automation.js +0 -149
  122. package/src/Automation_Layer/file_operations.js +0 -286
  123. package/src/Automation_Layer/open_app.js +0 -85
  124. package/src/Automation_Layer/open_website.js +0 -38
  125. package/src/CLI/approval_handler.js +0 -47
  126. package/src/CLI/chat_router.js +0 -247
  127. package/src/CLI/chat_ui.js +0 -1159
  128. package/src/CLI/cli_colors.js +0 -115
  129. package/src/CLI/cli_formatters.js +0 -94
  130. package/src/CLI/code_agent.js +0 -1667
  131. package/src/CLI/code_session_memory.js +0 -62
  132. package/src/CLI/gmail_auth.js +0 -210
  133. package/src/CLI/image_input.js +0 -90
  134. package/src/CLI/intent_detectors.js +0 -181
  135. package/src/CLI/interactive_chat.js +0 -658
  136. package/src/CLI/list_features.js +0 -64
  137. package/src/CLI/onboarding.js +0 -416
  138. package/src/CLI/repo_summarizer.js +0 -282
  139. package/src/CLI/semantic_code_search.js +0 -312
  140. package/src/CLI/skill_manager.js +0 -41
  141. package/src/CLI/slash_command_handler.js +0 -418
  142. package/src/CLI/symbol_indexer.js +0 -231
  143. package/src/CLI/updater.js +0 -230
  144. package/src/CLI/workspace_manager.js +0 -90
  145. package/src/Channels/brave_search_bridge.js +0 -35
  146. package/src/Channels/discord_bridge.js +0 -66
  147. package/src/Channels/google_search_bridge.js +0 -38
  148. package/src/Channels/line_bridge.js +0 -60
  149. package/src/Channels/slack_bridge.js +0 -48
  150. package/src/Channels/telegram_bridge.js +0 -41
  151. package/src/Channels/whatsapp_bridge.js +0 -57
  152. package/src/Command_Parser/parser.js +0 -45
  153. package/src/Plugins/dev_tools.js +0 -41
  154. package/src/Plugins/discord.js +0 -20
  155. package/src/Plugins/docker.js +0 -47
  156. package/src/Plugins/gmail.js +0 -251
  157. package/src/Plugins/google_calendar.js +0 -252
  158. package/src/Plugins/mcp_manager.js +0 -95
  159. package/src/Plugins/notion.js +0 -256
  160. package/src/Plugins/obsidian.js +0 -54
  161. package/src/Plugins/plugin_manager.js +0 -81
  162. package/src/Plugins/spotify.js +0 -173
  163. package/src/Plugins/system_metrics.js +0 -31
  164. package/src/Plugins/system_monitor.js +0 -72
  165. package/src/System/action_executor.js +0 -178
  166. package/src/System/bridge_manager.js +0 -76
  167. package/src/System/chat_history_manager.js +0 -83
  168. package/src/System/config_manager.js +0 -194
  169. package/src/System/custom_workflows.js +0 -163
  170. package/src/System/daemon_manager.js +0 -67
  171. package/src/System/google_tts_urls.js +0 -51
  172. package/src/System/granular_automation.js +0 -157
  173. package/src/System/ipc_handlers.js +0 -332
  174. package/src/System/notifications.js +0 -23
  175. package/src/System/optional_require.js +0 -23
  176. package/src/System/picture_store.js +0 -109
  177. package/src/System/proactive_loop.js +0 -153
  178. package/src/System/safety_manager.js +0 -273
  179. package/src/System/sandbox_runner.js +0 -182
  180. package/src/System/screen_capture.js +0 -175
  181. package/src/System/smart_context.js +0 -227
  182. package/src/System/system_automation.js +0 -162
  183. package/src/System/system_events.js +0 -79
  184. package/src/System/system_info.js +0 -125
  185. package/src/System/task_manager.js +0 -222
  186. package/src/System/tool_registry.js +0 -293
  187. package/src/System/window_manager.js +0 -220
  188. package/src/UI/floating.css +0 -80
  189. package/src/UI/floating.html +0 -17
  190. package/src/UI/floating.js +0 -67
  191. package/src/UI/live2d_manager.js +0 -600
  192. package/src/UI/preload-floating.js +0 -7
  193. package/src/UI/preload-spotlight.js +0 -11
  194. package/src/UI/preload-widget.js +0 -5
  195. package/src/UI/proactive-glow.html +0 -42
  196. package/src/UI/renderer.js +0 -2127
  197. package/src/UI/screenPicker.html +0 -214
  198. package/src/UI/screenPicker.js +0 -262
  199. package/src/UI/settings.html +0 -577
  200. package/src/UI/settings.js +0 -770
  201. package/src/UI/spotlight.html +0 -23
  202. package/src/UI/spotlight.js +0 -185
  203. package/src/UI/widget.html +0 -29
  204. package/src/UI/widget.js +0 -10
  205. /package/{models → src/renderer/public/models}/Shiroko_Model/Shiroko/Shiroko_Core/72d86db84cfa9730b894c241fd24c0db.png +0 -0
  206. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//345/233/264/350/243/231.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/apron.exp3.json} +0 -0
  207. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//347/214/253/345/222/252/346/273/244/351/225/234.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/catfilter.exp3.json} +0 -0
  208. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//347/202/271/344/270/200/344/270/213.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/click.exp3.json} +0 -0
  209. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//345/221/206/347/214/253.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/dazed.exp3.json} +0 -0
  210. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//345/221/206/347/214/253/347/234/274/347/217/240/346/221/207/346/231/203.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/dazedeyes.exp3.json} +0 -0
  211. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//347/234/274/351/225/234.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/glasses.exp3.json} +0 -0
  212. /package/{models → src/renderer/public/models}/Shiroko_Model/Shiroko/Shiroko_Core/items_pinned_to_model.json +0 -0
  213. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//346/213/277/347/254/224.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/pen.exp3.json} +0 -0
  214. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//346/213/215/347/205/247.exp3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/photo.exp3.json} +0 -0
  215. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_00.png" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.4096/texture_00.png} +0 -0
  216. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_01.png" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.4096/texture_01.png} +0 -0
  217. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_02.png" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.4096/texture_02.png} +0 -0
  218. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.4096/texture_03.png" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.4096/texture_03.png} +0 -0
  219. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.cdi3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.cdi3.json} +0 -0
  220. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.moc3" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.moc3} +0 -0
  221. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.physics3.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.physics3.json} +0 -0
  222. /package/{models/Shiroko_Model/Shiroko/Shiroko_Core//351/235/242/351/245/2740.vtube.json" → src/renderer/public/models/Shiroko_Model/Shiroko/Shiroko_Core/shiroko.vtube.json} +0 -0
@@ -1,172 +0,0 @@
1
- const { GoogleGenAI } = require('@google/genai');
2
- const { readConfig } = require('../System/config_manager');
3
-
4
- // ============================================================
5
- // Proactive Engine — Smart Suggestion Engine (Multi-Choice)
6
- // ============================================================
7
-
8
- const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
9
- let lastLoggedModel = '';
10
- let _ai = null;
11
-
12
- function getAi(apiKey) {
13
- if (!_ai) _ai = new GoogleGenAI({ apiKey });
14
- return _ai;
15
- }
16
-
17
- const PROACTIVE_SYSTEM_PROMPT = `You are a Smart Suggestion Engine built into a Desktop AI Agent called "Mint".
18
- Your job: observe the user's screen + behavior, then offer MULTIPLE relevant quick-action options — NOT just one question.
19
-
20
- CRITICAL RULES:
21
- 1. Respond ONLY with valid JSON, no markdown.
22
- 2. If nothing notable is on screen, return: {"message": null, "context": "", "suggestions": []}
23
- 3. Generate 2–4 SHORT suggestion chips that are genuinely useful based on what's visible.
24
- 4. Each suggestion must have a clear label (1–3 words) and an action.
25
- 5. Write "message" in Thai — short, friendly, observational (e.g. "พบว่าคุณเพิ่งเปิด Chrome").
26
- 6. Do NOT repeat suggestions from recent activities.
27
- 7. Suggestions should feel like smart shortcuts, not questions.
28
-
29
- Response schema (STRICT):
30
- {
31
- "context": "short English description of what you see on screen",
32
- "message": "สั้น ๆ เป็นภาษาไทย บอกว่า AI เห็นอะไร และเสนออะไร",
33
- "suggestions": [
34
- { "label": "YouTube", "action": { "type": "open_url", "target": "https://youtube.com" } },
35
- { "label": "Gmail", "action": { "type": "open_url", "target": "https://mail.google.com" } },
36
- { "label": "GitHub", "action": { "type": "open_url", "target": "https://github.com" } }
37
- ]
38
- }
39
-
40
- Action types allowed: "open_url", "open_app", "search", "none"
41
-
42
- Examples:
43
-
44
- SCENARIO: User opened Chrome or Firefox
45
- → message: "เพิ่งเปิด Browser — ต้องการเข้าเว็บไหนคะ?"
46
- → suggestions: YouTube, Gmail, GitHub, Google Maps (based on behavior history)
47
-
48
- SCENARIO: User is in VS Code / coding
49
- → message: "กำลัง Code อยู่ใช่ไหมคะ? มีอะไรช่วยได้บ้าง"
50
- → suggestions: Stack Overflow, MDN Docs, GitHub, ค้นหา Error
51
-
52
- SCENARIO: User opened Spotify
53
- → message: "เปิด Spotify แล้ว ต้องการเล่นอะไรคะ?"
54
- → suggestions: เพลง Chill, เพลง Focus, Top Charts, Podcast
55
-
56
- SCENARIO: User opened Terminal
57
- → message: "เปิด Terminal แล้ว ต้องการทำอะไรคะ?"
58
- → suggestions: GitHub, Stack Overflow, DevDocs, ค้นหา Command
59
-
60
- BAD examples (return null):
61
- - Nothing notable on screen
62
- - User is actively typing
63
- - Same context as before
64
- `;
65
-
66
- let lastSuggestionContext = '';
67
- let lastSuggestionTime = 0;
68
-
69
- function resolveGeminiModel() {
70
- try {
71
- const cfg = readConfig();
72
- const model = (cfg.geminiModel || '').trim();
73
- return model || DEFAULT_GEMINI_MODEL;
74
- } catch {
75
- return DEFAULT_GEMINI_MODEL;
76
- }
77
- }
78
-
79
- function getMinSuggestionIntervalMs() {
80
- try {
81
- const cfg = readConfig();
82
- return (cfg.proactiveCooldown || 120) * 1000;
83
- } catch {
84
- // ignore
85
- }
86
- return 120_000;
87
- }
88
-
89
- /**
90
- * Analyze screen and return a multi-choice suggestion object.
91
- * @param {string} base64Image
92
- * @param {string} behaviorSummary
93
- * @returns {Promise<{message: string, context: string, suggestions: Array} | null>}
94
- */
95
- async function analyzeAndSuggest(base64Image, behaviorSummary) {
96
- try {
97
- const cfg = readConfig();
98
- const apiKey = cfg.apiKey || process.env.GEMINI_API_KEY;
99
- if (!apiKey) return null; // silently skip if no API key configured
100
- const model = (cfg.geminiModel || '').trim() || DEFAULT_GEMINI_MODEL;
101
- if (model && model !== lastLoggedModel) {
102
- console.log(`[Gemini] Proactive Engine model: ${model}`);
103
- lastLoggedModel = model;
104
- }
105
- const ai = getAi(apiKey);
106
-
107
- const now = Date.now();
108
- const minInterval = getMinSuggestionIntervalMs();
109
-
110
- if (now - lastSuggestionTime < minInterval) return null;
111
-
112
- const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, '');
113
-
114
- const userMessage = [
115
- {
116
- text: `Analyze the screen and generate smart multi-choice suggestions for the user.
117
-
118
- User behavior context: ${behaviorSummary || 'No history yet.'}
119
-
120
- Rules: Only suggest if you see a clear opportunity. Return 2–4 relevant chips. Return null message if nothing notable.`
121
- },
122
- {
123
- inlineData: {
124
- mimeType: 'image/png',
125
- data: base64Data
126
- }
127
- }
128
- ];
129
-
130
- const response = await ai.models.generateContent({
131
- model,
132
- config: {
133
- systemInstruction: PROACTIVE_SYSTEM_PROMPT,
134
- responseMimeType: 'application/json'
135
- },
136
- contents: [{ role: 'user', parts: userMessage }]
137
- });
138
-
139
-
140
- let parsed;
141
- try {
142
- parsed = JSON.parse(response.text);
143
- } catch {
144
- const jsonMatch = response.text.match(/\{[\s\S]*\}/);
145
- if (jsonMatch) parsed = JSON.parse(jsonMatch[0]);
146
- else return null;
147
- }
148
-
149
- // Validate: must have message and at least 1 suggestion
150
- if (!parsed || !parsed.message || !Array.isArray(parsed.suggestions) || parsed.suggestions.length === 0) {
151
- return null;
152
- }
153
-
154
- // Skip repeat context
155
- if (parsed.context && parsed.context === lastSuggestionContext) {
156
- console.log('[ProactiveEngine] Skipping repeat context.');
157
- return null;
158
- }
159
-
160
- lastSuggestionContext = parsed.context || '';
161
- lastSuggestionTime = now;
162
-
163
- console.log(`[ProactiveEngine] ${parsed.suggestions.length} suggestions for: ${parsed.context}`);
164
- return parsed;
165
-
166
- } catch (err) {
167
- console.error('[ProactiveEngine] Error:', err.message);
168
- return null;
169
- }
170
- }
171
-
172
- module.exports = { analyzeAndSuggest };
@@ -1,365 +0,0 @@
1
- const axios = require('axios');
2
- const { GoogleGenAI } = require('@google/genai');
3
- const { getAvailableProviders } = require('../System/config_manager');
4
-
5
- const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
6
- const ALL_PROVIDERS = ['anthropic', 'openai', 'gemini', 'local_openai', 'ollama', 'huggingface'];
7
-
8
- function splitDataUri(dataUri = '') {
9
- const match = String(dataUri).match(/^data:([^;]+);base64,([\s\S]+)$/);
10
- if (!match) return null;
11
- return {
12
- mimeType: match[1],
13
- data: match[2]
14
- };
15
- }
16
-
17
- function contentToText(content) {
18
- if (content && typeof content === 'object' && !Array.isArray(content)) {
19
- return String(content.text || '');
20
- }
21
- return String(content || '');
22
- }
23
-
24
- function contentToGeminiParts(content) {
25
- const text = contentToText(content);
26
- const parts = text ? [{ text }] : [];
27
- if (content && typeof content === 'object') {
28
- const images = Array.isArray(content.imageDataUris)
29
- ? content.imageDataUris
30
- : (content.imageDataUri ? [content.imageDataUri] : []);
31
- for (const item of images) {
32
- const image = splitDataUri(item);
33
- if (image) parts.push({ inlineData: { mimeType: image.mimeType, data: image.data } });
34
- }
35
- if (content.audioDataUri) {
36
- const audio = splitDataUri(content.audioDataUri);
37
- if (audio) parts.push({ inlineData: { mimeType: audio.mimeType, data: audio.data } });
38
- }
39
- }
40
- return parts.length > 0 ? parts : [{ text: '' }];
41
- }
42
-
43
- function contentToOpenAIContent(content) {
44
- const text = contentToText(content) || 'Analyze this input.';
45
- if (content && typeof content === 'object') {
46
- const images = Array.isArray(content.imageDataUris)
47
- ? content.imageDataUris
48
- : (content.imageDataUri ? [content.imageDataUri] : []);
49
- if (images.length > 0) {
50
- return [
51
- { type: 'text', text },
52
- ...images.map(item => ({ type: 'image_url', image_url: { url: item } }))
53
- ];
54
- }
55
- }
56
- return text;
57
- }
58
-
59
- function contentToAnthropicContent(content) {
60
- const text = contentToText(content) || 'Analyze this input.';
61
- if (content && typeof content === 'object') {
62
- const images = Array.isArray(content.imageDataUris)
63
- ? content.imageDataUris
64
- : (content.imageDataUri ? [content.imageDataUri] : []);
65
- if (images.length > 0) {
66
- const blocks = [];
67
- for (const item of images) {
68
- const image = splitDataUri(item);
69
- if (image) {
70
- blocks.push({ type: 'image', source: { type: 'base64', media_type: image.mimeType, data: image.data } });
71
- }
72
- }
73
- blocks.push({ type: 'text', text });
74
- return blocks;
75
- }
76
- }
77
- return text;
78
- }
79
-
80
- function contentToOllamaMessage(content) {
81
- const text = contentToText(content) || 'Analyze this input.';
82
- const message = { role: 'user', content: text };
83
- if (content && typeof content === 'object') {
84
- const images = Array.isArray(content.imageDataUris)
85
- ? content.imageDataUris
86
- : (content.imageDataUri ? [content.imageDataUri] : []);
87
- const imagePayloads = images
88
- .map(item => splitDataUri(item))
89
- .filter(Boolean)
90
- .map(image => image.data);
91
- if (imagePayloads.length > 0) message.images = imagePayloads;
92
- }
93
- return message;
94
- }
95
-
96
- function getProviderAttemptOrder(config = {}, options = {}) {
97
- const supported = options.supported || ALL_PROVIDERS;
98
- const available = (options.availableProviders || getAvailableProviders(config))
99
- .filter(provider => supported.includes(provider));
100
- const requested = options.requested || config.aiProvider || 'gemini';
101
- const priority = (options.priority || ALL_PROVIDERS).filter(provider => supported.includes(provider));
102
- const ordered = [];
103
-
104
- if (supported.includes(requested) && available.includes(requested)) {
105
- ordered.push(requested);
106
- }
107
-
108
- for (const provider of priority) {
109
- if (available.includes(provider) && !ordered.includes(provider)) {
110
- ordered.push(provider);
111
- }
112
- }
113
-
114
- return ordered.length > 0 ? ordered : ['gemini'];
115
- }
116
-
117
- function getProviderModel(provider, config = {}) {
118
- switch (provider) {
119
- case 'anthropic':
120
- return config.anthropicModel || 'claude-3-5-sonnet-latest';
121
- case 'openai':
122
- return config.openaiModel || 'gpt-4o';
123
- case 'local_openai':
124
- return config.localModelName || 'local-model';
125
- case 'ollama':
126
- return config.ollamaModel || 'llama3:latest';
127
- case 'huggingface':
128
- return config.hfModel || 'meta-llama/Meta-Llama-3-8B-Instruct';
129
- case 'gemini':
130
- default:
131
- return config.geminiModel || DEFAULT_GEMINI_MODEL;
132
- }
133
- }
134
-
135
- class AgentProviderClient {
136
- constructor(options = {}) {
137
- this.provider = options.provider || 'gemini';
138
- this.providerOrder = options.providerOrder && options.providerOrder.length
139
- ? options.providerOrder
140
- : [this.provider];
141
- this.config = options.config || {};
142
- this.history = options.history || [];
143
- this.systemInstruction = options.systemInstruction || '';
144
- this.responseMimeType = options.responseMimeType || 'application/json';
145
- this.maxTokens = options.maxTokens || 8192;
146
- this.lastSuccessfulProvider = null;
147
- this.usageTotals = {};
148
- }
149
-
150
- recordUsage(provider, model, usage = {}) {
151
- const key = `${provider}:${model || ''}`;
152
- if (!this.usageTotals[key]) {
153
- this.usageTotals[key] = {
154
- provider,
155
- model,
156
- requests: 0,
157
- inputTokens: 0,
158
- cacheReads: 0,
159
- outputTokens: 0,
160
- reasoningTokens: 0,
161
- totalTokens: 0
162
- };
163
- }
164
-
165
- const row = this.usageTotals[key];
166
- row.requests += 1;
167
- row.inputTokens += Number(usage.inputTokens) || 0;
168
- row.cacheReads += Number(usage.cacheReads) || 0;
169
- row.outputTokens += Number(usage.outputTokens) || 0;
170
- row.reasoningTokens += Number(usage.reasoningTokens) || 0;
171
- row.totalTokens += Number(usage.totalTokens) || 0;
172
- }
173
-
174
- getUsageSummary() {
175
- return Object.values(this.usageTotals);
176
- }
177
-
178
- async sendMessage(observation) {
179
- this.history.push({ role: 'user', content: observation });
180
-
181
- const failures = [];
182
- for (const provider of this.providerOrder) {
183
- this.provider = provider;
184
- try {
185
- let responseText = '';
186
- if (provider === 'anthropic') responseText = await this.callAnthropic();
187
- else if (provider === 'openai' || provider === 'local_openai') responseText = await this.callOpenAI();
188
- else if (provider === 'ollama') responseText = await this.callOllama();
189
- else if (provider === 'huggingface') responseText = await this.callHuggingFace();
190
- else responseText = await this.callGemini();
191
-
192
- this.history.push({ role: 'assistant', content: responseText });
193
- this.lastSuccessfulProvider = provider;
194
- return responseText;
195
- } catch (error) {
196
- const message = error.message || error.code || 'unknown error';
197
- failures.push(`${provider}: ${message}`);
198
- if (process.env.MINT_DEBUG === '1') {
199
- console.error(`[ProviderAdapter] Provider '${provider}' failed: ${message}`);
200
- }
201
- }
202
- }
203
-
204
- throw new Error(`All providers failed. ${failures.join(' | ')}`);
205
- }
206
-
207
- async callAnthropic() {
208
- const apiKey = this.config.anthropicApiKey || process.env.ANTHROPIC_API_KEY;
209
- const model = getProviderModel('anthropic', this.config);
210
- const messages = this.history.map(m => ({
211
- role: m.role,
212
- content: contentToAnthropicContent(m.content)
213
- }));
214
-
215
- const response = await axios.post('https://api.anthropic.com/v1/messages', {
216
- model,
217
- max_tokens: this.maxTokens,
218
- system: this.systemInstruction,
219
- messages
220
- }, {
221
- headers: {
222
- 'x-api-key': apiKey,
223
- 'anthropic-version': '2023-06-01',
224
- 'content-type': 'application/json'
225
- }
226
- });
227
- const usage = response.data.usage || {};
228
- this.recordUsage('anthropic', model, {
229
- inputTokens: usage.input_tokens,
230
- cacheReads: usage.cache_read_input_tokens,
231
- outputTokens: usage.output_tokens,
232
- totalTokens: (Number(usage.input_tokens) || 0) + (Number(usage.output_tokens) || 0)
233
- });
234
- return response.data.content[0].text;
235
- }
236
-
237
- async callOpenAI() {
238
- const isLocal = this.provider === 'local_openai';
239
- const apiKey = isLocal ? 'not-needed' : (this.config.openaiApiKey || process.env.OPENAI_API_KEY);
240
- const baseUrl = isLocal ? (this.config.localApiBaseUrl || 'http://localhost:1234/v1') : 'https://api.openai.com/v1';
241
- const model = getProviderModel(this.provider, this.config);
242
- const messages = [
243
- { role: 'system', content: this.systemInstruction },
244
- ...this.history.map(m => ({
245
- role: m.role,
246
- content: contentToOpenAIContent(m.content)
247
- }))
248
- ];
249
-
250
- const response = await axios.post(`${baseUrl.replace(/\/$/, '')}/chat/completions`, {
251
- model,
252
- messages,
253
- response_format: isLocal ? undefined : { type: 'json_object' }
254
- }, {
255
- headers: {
256
- 'Authorization': `Bearer ${apiKey}`,
257
- 'Content-Type': 'application/json'
258
- }
259
- });
260
- const usage = response.data.usage || {};
261
- this.recordUsage(this.provider, model, {
262
- inputTokens: usage.prompt_tokens,
263
- cacheReads: usage.prompt_tokens_details && usage.prompt_tokens_details.cached_tokens,
264
- outputTokens: usage.completion_tokens,
265
- reasoningTokens: usage.completion_tokens_details && usage.completion_tokens_details.reasoning_tokens,
266
- totalTokens: usage.total_tokens
267
- });
268
- return response.data.choices[0].message.content;
269
- }
270
-
271
- async callGemini() {
272
- const apiKey = this.config.apiKey || process.env.GEMINI_API_KEY;
273
- const model = getProviderModel('gemini', this.config);
274
- const ai = new GoogleGenAI({ apiKey });
275
- const recentHistory = this.history.slice(-16);
276
- const priorHistory = recentHistory.slice(0, -1);
277
- const lastEntry = recentHistory[recentHistory.length - 1] || { content: '' };
278
- const history = priorHistory.map(m => ({
279
- role: m.role === 'assistant' ? 'model' : 'user',
280
- parts: contentToGeminiParts(m.content)
281
- }));
282
- const chat = ai.chats.create({
283
- model,
284
- config: {
285
- systemInstruction: this.systemInstruction,
286
- responseMimeType: this.responseMimeType,
287
- safetySettings: [
288
- { category: "HARM_CATEGORY_SEXUALLY_EXPLICIT", threshold: "BLOCK_NONE" },
289
- { category: "HARM_CATEGORY_HATE_SPEECH", threshold: "BLOCK_NONE" },
290
- { category: "HARM_CATEGORY_HARASSMENT", threshold: "BLOCK_NONE" },
291
- { category: "HARM_CATEGORY_DANGEROUS_CONTENT", threshold: "BLOCK_NONE" },
292
- { category: "HARM_CATEGORY_CIVIC_INTEGRITY", threshold: "BLOCK_NONE" }
293
- ]
294
- },
295
- history
296
- });
297
-
298
- const response = await chat.sendMessage({ message: contentToGeminiParts(lastEntry.content) });
299
- const usage = response.usageMetadata || {};
300
- this.recordUsage('gemini', model, {
301
- inputTokens: usage.promptTokenCount,
302
- cacheReads: usage.cachedContentTokenCount,
303
- outputTokens: usage.candidatesTokenCount,
304
- reasoningTokens: usage.thoughtsTokenCount,
305
- totalTokens: usage.totalTokenCount
306
- });
307
- return typeof response.text === 'function' ? response.text() : response.text;
308
- }
309
-
310
- async callOllama() {
311
- const model = getProviderModel('ollama', this.config);
312
- const baseUrl = (this.config.ollamaHost || 'http://localhost:11434').replace(/\/$/, '');
313
- const messages = [
314
- { role: 'system', content: this.systemInstruction },
315
- ...this.history.map(m => m.role === 'assistant'
316
- ? { role: 'assistant', content: contentToText(m.content) }
317
- : contentToOllamaMessage(m.content))
318
- ];
319
- const response = await axios.post(`${baseUrl}/api/chat`, {
320
- model,
321
- messages,
322
- format: this.responseMimeType === 'application/json' ? 'json' : undefined,
323
- stream: false
324
- });
325
- return response.data.message.content;
326
- }
327
-
328
- async callHuggingFace() {
329
- const apiKey = this.config.hfApiKey || process.env.HF_API_KEY;
330
- const model = getProviderModel('huggingface', this.config);
331
- const messages = [
332
- { role: 'system', content: this.systemInstruction },
333
- ...this.history.map(m => ({
334
- role: m.role,
335
- content: contentToOpenAIContent(m.content)
336
- }))
337
- ];
338
- const response = await axios.post(`https://api-inference.huggingface.co/models/${model}/v1/chat/completions`, {
339
- model,
340
- messages,
341
- max_tokens: this.maxTokens
342
- }, {
343
- headers: {
344
- 'Authorization': `Bearer ${apiKey}`,
345
- 'Content-Type': 'application/json'
346
- }
347
- });
348
- return response.data.choices[0].message.content;
349
- }
350
- }
351
-
352
- module.exports = {
353
- DEFAULT_GEMINI_MODEL,
354
- ALL_PROVIDERS,
355
- AgentProviderClient,
356
- getProviderAttemptOrder,
357
- getProviderModel,
358
- _helpers: {
359
- splitDataUri,
360
- contentToGeminiParts,
361
- contentToOpenAIContent,
362
- contentToAnthropicContent,
363
- contentToOllamaMessage
364
- }
365
- };
@@ -1,149 +0,0 @@
1
- 'use strict';
2
-
3
- const { requireOptional } = require('../System/optional_require');
4
- const { GoogleGenAI } = require('@google/genai');
5
- const { readConfig } = require('../System/config_manager');
6
-
7
-
8
-
9
- const BROWSER_SYSTEM_PROMPT = `You are an Autonomous Browser Agent. Your goal is to fulfill the user's web instruction by driving a headless browser.
10
-
11
- CRITICAL INSTRUCTIONS:
12
- Always respond EXACTLY with valid JSON. NO MARKDOWN. NO CODE BLOCKS (\`\`\`json). The JSON must have this exact structure:
13
- {
14
- "thought": "Reasoning about what to do next based on the current page content and goal.",
15
- "action": "goto" | "click" | "eval" | "done",
16
- "target": "URL for goto | CSS selector for click | JavaScript expression for eval | Final answer for done"
17
- }
18
-
19
- Actions:
20
- - "goto": Navigate to the specified URL. Target MUST be a full URL (e.g. "https://www.google.com/search?q=AI+news")
21
- - "click": Click an element. Target MUST be a valid CSS selector.
22
- - "eval": Evaluate JavaScript to extract text. Target MUST be JS code returning a string (e.g. "document.body.innerText.substring(0, 1000)").
23
- - "done": Task finished. Target MUST be the final summary or answer to present to the user.
24
-
25
- You will receive the result of your previous action in the next message. If you get stuck or fail, try another approach or use "done" to report the failure.`;
26
-
27
- const DEFAULT_GEMINI_MODEL = 'gemini-2.5-flash';
28
- let lastLoggedModel = '';
29
-
30
- function resolveGeminiModel() {
31
- try {
32
- const cfg = readConfig();
33
- const model = (cfg.geminiModel || '').trim();
34
- return model || DEFAULT_GEMINI_MODEL;
35
- } catch {
36
- return DEFAULT_GEMINI_MODEL;
37
- }
38
- }
39
-
40
- async function performWebAutomation(query) {
41
- if (!query) return 'No query provided.';
42
-
43
- // Dynamic require — user must install puppeteer separately
44
- const puppeteer = requireOptional('puppeteer', 'npm install puppeteer');
45
-
46
- const config = readConfig();
47
- const apiKey = config.apiKey || process.env.GEMINI_API_KEY;
48
- const ai = new GoogleGenAI({ apiKey });
49
- const browserPath = config.automationBrowser;
50
-
51
- let browser;
52
- try {
53
- const launchOptions = {
54
- headless: false,
55
- defaultViewport: null,
56
- args: ['--start-maximized']
57
- };
58
-
59
- if (browserPath && browserPath !== 'chromium') {
60
- launchOptions.executablePath = browserPath;
61
- if (browserPath.toLowerCase().includes('firefox')) {
62
- launchOptions.browser = 'firefox';
63
- }
64
- }
65
-
66
- browser = await puppeteer.launch(launchOptions);
67
- const page = await browser.newPage();
68
-
69
- const model = resolveGeminiModel();
70
- if (model && model !== lastLoggedModel) {
71
- console.log(`[Gemini] Web Automation model: ${model}`);
72
- lastLoggedModel = model;
73
- }
74
-
75
- const chat = ai.chats.create({
76
- model,
77
- config: {
78
- systemInstruction: BROWSER_SYSTEM_PROMPT,
79
- responseMimeType: 'application/json'
80
- }
81
- });
82
-
83
- let currentObservation = `Goal: ${query}\nSystem Note: You have a blank browser page. What is your first action? Start by using "goto" to navigate to a relevant search engine or website.`;
84
-
85
- let maxSteps = 10;
86
- let step = 0;
87
-
88
- while (step < maxSteps) {
89
- step++;
90
- console.log(`\n--- Agent Step ${step} ---`);
91
- console.log(`Observation:`, currentObservation.substring(0, 150) + (currentObservation.length > 150 ? '...' : ''));
92
-
93
- const response = await chat.sendMessage({ message: currentObservation });
94
-
95
- let parsed;
96
- try {
97
- const text = response.text;
98
- const cleanText = text.replace(/^```json\n/, '').replace(/\n```$/, '').trim();
99
- parsed = JSON.parse(cleanText);
100
- } catch (e) {
101
- console.error('Agent failed to return valid JSON:', response.text);
102
- currentObservation = 'Error: Invalid JSON returned. Please reply with ONLY valid JSON matching the schema.';
103
- continue;
104
- }
105
-
106
- console.log('Agent Thought:', parsed.thought);
107
- console.log('Agent Action:', parsed.action);
108
- console.log('Agent Target:', parsed.target);
109
-
110
- const { action, target } = parsed;
111
-
112
- if (action === 'done') {
113
- console.log('Agent finished with answer:', target);
114
- return `🤖 Web Automation Result: ${target}`;
115
- }
116
-
117
- try {
118
- if (action === 'goto') {
119
- await page.goto(target, { waitUntil: 'domcontentloaded', timeout: 30000 });
120
- const pageTitle = await page.title();
121
- currentObservation = `Successfully navigated to ${pageTitle}. ` + await page.evaluate(() => document.body.innerText.substring(0, 1500));
122
- } else if (action === 'click') {
123
- await page.waitForSelector(target, { timeout: 5000 });
124
- await page.click(target);
125
- await new Promise(r => setTimeout(r, 2000));
126
- const pageTitle = await page.title();
127
- currentObservation = `Clicked element. Current page: ${pageTitle}. ` + await page.evaluate(() => document.body.innerText.substring(0, 1500));
128
- } else if (action === 'eval') {
129
- const evalResult = await page.evaluate(target);
130
- currentObservation = `Eval result: ` + String(evalResult).substring(0, 1500);
131
- } else {
132
- currentObservation = `Error: Unknown action type "${action}".`;
133
- }
134
- } catch (actionError) {
135
- console.error('Action execution failed:', actionError);
136
- currentObservation = `Action failed: ${actionError.message}. Please try again or use another method (for instance, try a different CSS selector or just read the current page).`;
137
- }
138
- }
139
-
140
- return 'Agent reached maximum steps (10) without finding a final answer.';
141
-
142
- } catch (error) {
143
- console.error('Web Automation Error:', error);
144
- if (browser) browser.close();
145
- return `I encountered an overall error while automating the browser: ${error.message}`;
146
- }
147
- }
148
-
149
- module.exports = { performWebAutomation };