@pheem49/mint 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/BUILD_AND_RELEASE.md +75 -0
- package/LICENSE +654 -0
- package/README.md +165 -0
- package/assets/Agent_Mint.png +0 -0
- package/assets/CLI_Screen.png +0 -0
- package/assets/Settings.png +0 -0
- package/assets/icon.png +0 -0
- package/benchmark_ai.js +71 -0
- package/main.js +968 -0
- package/mint-cli-logic.js +71 -0
- package/mint-cli.js +239 -0
- package/package.json +60 -0
- package/preload-picker.js +11 -0
- package/preload-settings.js +11 -0
- package/preload.js +37 -0
- package/privacy.txt +1 -0
- package/src/AI_Brain/Gemini_API.js +419 -0
- package/src/AI_Brain/autonomous_brain.js +139 -0
- package/src/AI_Brain/behavior_memory.js +114 -0
- package/src/AI_Brain/headless_agent.js +120 -0
- package/src/AI_Brain/knowledge_base.js +222 -0
- package/src/AI_Brain/proactive_engine.js +168 -0
- package/src/Automation_Layer/browser_automation.js +147 -0
- package/src/Automation_Layer/file_operations.js +80 -0
- package/src/Automation_Layer/open_app.js +56 -0
- package/src/Automation_Layer/open_website.js +38 -0
- package/src/CLI/chat_ui.js +468 -0
- package/src/CLI/list_features.js +56 -0
- package/src/CLI/onboarding.js +60 -0
- package/src/Command_Parser/parser.js +34 -0
- package/src/Plugins/dev_tools.js +41 -0
- package/src/Plugins/discord.js +20 -0
- package/src/Plugins/docker.js +45 -0
- package/src/Plugins/google_calendar.js +26 -0
- package/src/Plugins/obsidian.js +54 -0
- package/src/Plugins/plugin_manager.js +81 -0
- package/src/Plugins/spotify.js +45 -0
- package/src/Plugins/system_metrics.js +31 -0
- package/src/System/chat_history_manager.js +57 -0
- package/src/System/config_manager.js +73 -0
- package/src/System/custom_workflows.js +127 -0
- package/src/System/daemon_manager.js +67 -0
- package/src/System/system_automation.js +88 -0
- package/src/System/system_events.js +79 -0
- package/src/System/system_info.js +55 -0
- package/src/System/task_manager.js +85 -0
- package/src/UI/floating.css +80 -0
- package/src/UI/floating.html +17 -0
- package/src/UI/floating.js +67 -0
- package/src/UI/index.html +126 -0
- package/src/UI/preload-floating.js +7 -0
- package/src/UI/preload-spotlight.js +10 -0
- package/src/UI/preload-widget.js +5 -0
- package/src/UI/proactive-glow.html +42 -0
- package/src/UI/renderer.js +978 -0
- package/src/UI/screenPicker.html +214 -0
- package/src/UI/screenPicker.js +262 -0
- package/src/UI/settings.css +705 -0
- package/src/UI/settings.html +396 -0
- package/src/UI/settings.js +514 -0
- package/src/UI/spotlight.css +119 -0
- package/src/UI/spotlight.html +23 -0
- package/src/UI/spotlight.js +181 -0
- package/src/UI/styles.css +627 -0
- package/src/UI/widget.css +218 -0
- package/src/UI/widget.html +29 -0
- package/src/UI/widget.js +10 -0
- package/tech_news.txt +3 -0
- package/test_knowledge.txt +3 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mint Headless Agent
|
|
3
|
+
* Runs Mint's background features (like Proactive Suggestions) without a GUI.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
const { exec } = require('child_process');
|
|
7
|
+
const { getSystemInfo } = require('../System/system_info');
|
|
8
|
+
const { readConfig } = require('../System/config_manager');
|
|
9
|
+
const systemEvents = require('../System/system_events');
|
|
10
|
+
const taskManager = require('../System/task_manager');
|
|
11
|
+
const { executeAutonomousTask } = require('./autonomous_brain');
|
|
12
|
+
|
|
13
|
+
// ANSI Colors for console
|
|
14
|
+
const colors = {
|
|
15
|
+
reset: "\x1b[0m",
|
|
16
|
+
bright: "\x1b[1m",
|
|
17
|
+
mint: "\x1b[38;5;121m",
|
|
18
|
+
gray: "\x1b[90m"
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
let isProcessingTask = false;
|
|
22
|
+
|
|
23
|
+
async function startAgent() {
|
|
24
|
+
console.log(`\n${colors.mint}${colors.bright}[Mint-Agent] Background agent started.${colors.reset}`);
|
|
25
|
+
console.log(`${colors.gray}[Mint-Agent] Monitoring system events and task queue...${colors.reset}\n`);
|
|
26
|
+
|
|
27
|
+
// Initialize System Monitoring
|
|
28
|
+
systemEvents.startMonitoring();
|
|
29
|
+
|
|
30
|
+
// Listen for Battery Events
|
|
31
|
+
systemEvents.on('low-battery', (level) => {
|
|
32
|
+
sendNotification(
|
|
33
|
+
"⚠️ แบตเตอรี่ใกล้หมดแล้วนะคะ",
|
|
34
|
+
`ตอนนี้แบตเตอรี่เหลือ ${level}% แล้วค่ะ อย่าลืมชาร์จแบตนะค๊า ✨`
|
|
35
|
+
);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// Listen for Network Events
|
|
39
|
+
systemEvents.on('connection-change', (isOnline) => {
|
|
40
|
+
const title = isOnline ? "✅ เชื่อมต่อสำเร็จ" : "❌ การเชื่อมต่อขาดหาย";
|
|
41
|
+
const msg = isOnline ? "มิ้นท์เชื่อมต่ออินเทอร์เน็ตได้แล้วค่ะ! ✨" : "มิ้นท์ไม่เห็นสัญญาณอินเทอร์เน็ตเลยนะคะ";
|
|
42
|
+
sendNotification(title, msg);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
// Send a startup notification to let the user know the agent is alive
|
|
46
|
+
sendNotification("Mint Agent", "มิ้นท์ประจำการอยู่เบื้องหลังแล้วนะค๊า! 🛡️✨");
|
|
47
|
+
|
|
48
|
+
// Polling Loop for Tasks and Health
|
|
49
|
+
setInterval(async () => {
|
|
50
|
+
await checkTaskQueue();
|
|
51
|
+
|
|
52
|
+
try {
|
|
53
|
+
const info = await getSystemInfo();
|
|
54
|
+
// Heartbeat logic
|
|
55
|
+
} catch (err) {}
|
|
56
|
+
}, 15000); // Check every 15 seconds
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async function checkTaskQueue() {
|
|
60
|
+
if (isProcessingTask) return;
|
|
61
|
+
|
|
62
|
+
const task = taskManager.getPendingTask();
|
|
63
|
+
if (!task) return;
|
|
64
|
+
|
|
65
|
+
isProcessingTask = true;
|
|
66
|
+
console.log(`\n${colors.mint}[Agent] Picking up task: ${task.description}${colors.reset}`);
|
|
67
|
+
|
|
68
|
+
taskManager.updateTask(task.id, { status: 'running' });
|
|
69
|
+
sendNotification("🚀 เริ่มทำงานให้แล้วนะคะ", `กำลังดำเนินการ: ${task.description}`);
|
|
70
|
+
|
|
71
|
+
try {
|
|
72
|
+
const result = await executeAutonomousTask(task.description, (progress) => {
|
|
73
|
+
console.log(`${colors.gray}[Progress] ${progress}${colors.reset}`);
|
|
74
|
+
// Send periodic progress notifications if important
|
|
75
|
+
if (progress.includes('เสนอให้รันคำสั่ง')) {
|
|
76
|
+
sendNotification("💡 มิ้นท์มีข้อแนะนำค่ะ", progress);
|
|
77
|
+
}
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
taskManager.updateTask(task.id, { status: 'completed', result });
|
|
81
|
+
sendNotification("✅ งานเสร็จเรียบร้อยแล้วค่ะ!", result);
|
|
82
|
+
console.log(`\n${colors.mint}[Agent] Task completed successfully.${colors.reset}`);
|
|
83
|
+
|
|
84
|
+
} catch (err) {
|
|
85
|
+
console.error('[Agent] Task execution failed:', err);
|
|
86
|
+
taskManager.updateTask(task.id, { status: 'failed', result: err.message });
|
|
87
|
+
sendNotification("❌ เกิดข้อผิดพลาดในการทำงาน", err.message);
|
|
88
|
+
} finally {
|
|
89
|
+
isProcessingTask = false;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Sends a system-level notification using notify-send (Linux Pop!_OS)
|
|
95
|
+
*/
|
|
96
|
+
async function sendNotification(title, message) {
|
|
97
|
+
// Check if notify-send exists before trying to use it
|
|
98
|
+
const hasNotifySend = await new Promise(resolve => {
|
|
99
|
+
exec('which notify-send', (err) => resolve(!err));
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
if (!hasNotifySend) {
|
|
103
|
+
console.log(`${colors.gray}[Agent Info]${colors.reset} Notification suppressed (notify-send not found). Install with: sudo apt install libnotify-bin`);
|
|
104
|
+
console.log(`${colors.mint}[Agent Noti]${colors.reset} ${title}: ${message}`);
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const iconPath = require('path').join(__dirname, '../../assets/icon.png');
|
|
109
|
+
const cmd = `notify-send "${title}" "${message}" -i "${iconPath}" -a "Mint AI"`;
|
|
110
|
+
|
|
111
|
+
exec(cmd, (err) => {
|
|
112
|
+
if (err) {
|
|
113
|
+
console.error('[Mint-Agent] Failed to send notification:', err.message);
|
|
114
|
+
} else {
|
|
115
|
+
console.log(`${colors.mint}[Agent Noti]${colors.reset} ${title}: ${message}`);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
module.exports = { startAgent };
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
const fs = require('fs');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const os = require('os');
|
|
4
|
+
const { GoogleGenAI } = require('@google/genai');
|
|
5
|
+
const pdf = require('pdf-parse');
|
|
6
|
+
const mammoth = require('mammoth');
|
|
7
|
+
const xlsx = require('xlsx');
|
|
8
|
+
const axios = require('axios');
|
|
9
|
+
const cheerio = require('cheerio');
|
|
10
|
+
const { readConfig } = require('../System/config_manager');
|
|
11
|
+
|
|
12
|
+
// Handle electron dependency safely for benchmarks/tests
|
|
13
|
+
let app;
|
|
14
|
+
try {
|
|
15
|
+
const electron = require('electron');
|
|
16
|
+
app = electron.app;
|
|
17
|
+
} catch (e) {
|
|
18
|
+
app = null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let ai = null;
|
|
22
|
+
let activeApiKey = '';
|
|
23
|
+
const initialEnvKey = (process.env.GEMINI_API_KEY || '').trim();
|
|
24
|
+
|
|
25
|
+
function resolveApiKey() {
|
|
26
|
+
let settingsKey = '';
|
|
27
|
+
try {
|
|
28
|
+
const cfg = readConfig();
|
|
29
|
+
settingsKey = (cfg.apiKey || '').trim();
|
|
30
|
+
} catch (e) {
|
|
31
|
+
settingsKey = '';
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const envKey = initialEnvKey;
|
|
35
|
+
const selectedKey = settingsKey || envKey || '';
|
|
36
|
+
|
|
37
|
+
if (selectedKey !== (process.env.GEMINI_API_KEY || '')) {
|
|
38
|
+
process.env.GEMINI_API_KEY = selectedKey;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
activeApiKey = selectedKey;
|
|
42
|
+
return selectedKey;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function getAiClient() {
|
|
46
|
+
const prevKey = activeApiKey;
|
|
47
|
+
const nextKey = resolveApiKey();
|
|
48
|
+
if (!ai || nextKey !== prevKey) {
|
|
49
|
+
ai = new GoogleGenAI({ apiKey: nextKey });
|
|
50
|
+
}
|
|
51
|
+
return ai;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function getDbPath() {
|
|
55
|
+
if (app && app.getPath) {
|
|
56
|
+
return path.join(app.getPath('userData'), 'mint-knowledge.json');
|
|
57
|
+
}
|
|
58
|
+
// Use global .mint directory for CLI/Benchmarking
|
|
59
|
+
const mintDir = path.join(os.homedir(), '.mint');
|
|
60
|
+
if (!fs.existsSync(mintDir)) {
|
|
61
|
+
fs.mkdirSync(mintDir, { recursive: true });
|
|
62
|
+
}
|
|
63
|
+
return path.join(mintDir, 'mint-knowledge.json');
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function loadDb() {
|
|
67
|
+
try {
|
|
68
|
+
const p = getDbPath();
|
|
69
|
+
if (fs.existsSync(p)) {
|
|
70
|
+
return JSON.parse(fs.readFileSync(p, 'utf8'));
|
|
71
|
+
}
|
|
72
|
+
} catch (err) {
|
|
73
|
+
console.error('[KnowledgeBase] Load Error:', err);
|
|
74
|
+
}
|
|
75
|
+
return { documents: [] };
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function saveDb(db) {
|
|
79
|
+
fs.writeFileSync(getDbPath(), JSON.stringify(db, null, 2));
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function generateEmbedding(text) {
|
|
83
|
+
const client = getAiClient();
|
|
84
|
+
const response = await client.models.embedContent({
|
|
85
|
+
model: 'gemini-embedding-001',
|
|
86
|
+
contents: text,
|
|
87
|
+
});
|
|
88
|
+
// The google/genai package returns an array of embeddings
|
|
89
|
+
return response.embeddings[0].values;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function cosineSimilarity(vecA, vecB) {
|
|
93
|
+
let dotProduct = 0.0;
|
|
94
|
+
let normA = 0.0;
|
|
95
|
+
let normB = 0.0;
|
|
96
|
+
for (let i = 0; i < vecA.length; i++) {
|
|
97
|
+
dotProduct += vecA[i] * vecB[i];
|
|
98
|
+
normA += vecA[i] * vecA[i];
|
|
99
|
+
normB += vecB[i] * vecB[i];
|
|
100
|
+
}
|
|
101
|
+
if (normA === 0 || normB === 0) return 0;
|
|
102
|
+
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function chunkText(text, maxChars = 1000, overlap = 200) {
|
|
106
|
+
const chunks = [];
|
|
107
|
+
let current = 0;
|
|
108
|
+
const step = maxChars - overlap;
|
|
109
|
+
while (current < text.length) {
|
|
110
|
+
chunks.push(text.slice(current, current + maxChars));
|
|
111
|
+
current += step;
|
|
112
|
+
}
|
|
113
|
+
return chunks;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* Reads a local file or URL, chunks its text, generates embeddings, and saves to knowledge base.
|
|
118
|
+
*/
|
|
119
|
+
async function indexFile(resourcePath) {
|
|
120
|
+
try {
|
|
121
|
+
if (!resourcePath || resourcePath.trim() === '') return "ไม่พบข้อมูล กรุณาระบุ Path หรือ URL ค่ะ";
|
|
122
|
+
|
|
123
|
+
let content = '';
|
|
124
|
+
let sourceName = '';
|
|
125
|
+
let resourceId = '';
|
|
126
|
+
|
|
127
|
+
// Handle Web URLs
|
|
128
|
+
if (resourcePath.startsWith('http://') || resourcePath.startsWith('https://')) {
|
|
129
|
+
sourceName = resourcePath;
|
|
130
|
+
resourceId = resourcePath;
|
|
131
|
+
try {
|
|
132
|
+
const response = await axios.get(resourcePath);
|
|
133
|
+
const $ = cheerio.load(response.data);
|
|
134
|
+
$('script, style, noscript, nav, footer, header').remove();
|
|
135
|
+
content = $('body').text().replace(/\s+/g, ' ').trim();
|
|
136
|
+
} catch (e) {
|
|
137
|
+
return `ไม่สามารถดึงข้อมูลจากเว็บไซต์ได้ค่ะ: ${e.message}`;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// Handle Local Files
|
|
141
|
+
else {
|
|
142
|
+
const filePath = resourcePath;
|
|
143
|
+
if (!fs.existsSync(filePath)) return `ไม่พบไฟล์: ${filePath}`;
|
|
144
|
+
|
|
145
|
+
const stats = fs.statSync(filePath);
|
|
146
|
+
if (stats.size > 5 * 1024 * 1024) return `ขนาดไฟล์ใหญ่เกินไป (> 5MB): ${filePath}`;
|
|
147
|
+
|
|
148
|
+
sourceName = path.basename(filePath);
|
|
149
|
+
resourceId = filePath;
|
|
150
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
151
|
+
|
|
152
|
+
if (ext === '.pdf') {
|
|
153
|
+
const dataBuffer = fs.readFileSync(filePath);
|
|
154
|
+
const data = await pdf(dataBuffer);
|
|
155
|
+
content = data.text;
|
|
156
|
+
} else if (ext === '.docx') {
|
|
157
|
+
const result = await mammoth.extractRawText({path: filePath});
|
|
158
|
+
content = result.value;
|
|
159
|
+
} else if (ext === '.xlsx') {
|
|
160
|
+
const workbook = xlsx.readFile(filePath);
|
|
161
|
+
content = '';
|
|
162
|
+
for (const sheetName of workbook.SheetNames) {
|
|
163
|
+
const sheet = workbook.Sheets[sheetName];
|
|
164
|
+
const csv = xlsx.utils.sheet_to_csv(sheet);
|
|
165
|
+
content += `\n--- Sheet: ${sheetName} ---\n` + csv;
|
|
166
|
+
}
|
|
167
|
+
} else {
|
|
168
|
+
content = fs.readFileSync(filePath, 'utf8');
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
if (!content || content.trim().length === 0) return `ข้อมูลว่างเปล่าหรือไม่มีข้อความ: ${resourcePath}`;
|
|
173
|
+
|
|
174
|
+
const chunks = chunkText(content);
|
|
175
|
+
const db = loadDb();
|
|
176
|
+
|
|
177
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
178
|
+
const embedding = await generateEmbedding(chunks[i]);
|
|
179
|
+
db.documents.push({
|
|
180
|
+
id: `${resourceId}#${i}-${Date.now()}`,
|
|
181
|
+
source: sourceName,
|
|
182
|
+
path: resourcePath,
|
|
183
|
+
text: chunks[i],
|
|
184
|
+
embedding
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
saveDb(db);
|
|
189
|
+
return `✅ เรียนรู้ข้อมูลจาก ${sourceName} เรียบร้อยแล้ว (แบ่งเป็น ${chunks.length} ส่วน)`;
|
|
190
|
+
} catch (err) {
|
|
191
|
+
console.error('[KnowledgeBase] Indexing error:', err);
|
|
192
|
+
return `❌ เกิดข้อผิดพลาดในการเรียนรู้ไฟล์: ${err.message}`;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Searches the local knowledge base for relevant chunks.
|
|
198
|
+
*/
|
|
199
|
+
async function searchKnowledge(query, topK = 3) {
|
|
200
|
+
const db = loadDb();
|
|
201
|
+
if (!db.documents || db.documents.length === 0) return null;
|
|
202
|
+
|
|
203
|
+
try {
|
|
204
|
+
const queryVector = await generateEmbedding(query);
|
|
205
|
+
const results = db.documents.map(doc => ({
|
|
206
|
+
...doc,
|
|
207
|
+
score: cosineSimilarity(queryVector, doc.embedding)
|
|
208
|
+
})).sort((a, b) => b.score - a.score);
|
|
209
|
+
|
|
210
|
+
// Return top results above a threshold
|
|
211
|
+
const top = results.slice(0, topK).filter(r => r.score > 0.65);
|
|
212
|
+
if (top.length > 0) {
|
|
213
|
+
console.log(`[KnowledgeBase] Found ${top.length} matches for query.`);
|
|
214
|
+
return top;
|
|
215
|
+
}
|
|
216
|
+
} catch(err) {
|
|
217
|
+
console.error("[KnowledgeBase] Search error:", err);
|
|
218
|
+
}
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
module.exports = { indexFile, searchKnowledge };
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
const { GoogleGenAI } = require('@google/genai');
|
|
2
|
+
const path = require('path');
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const { app } = require('electron');
|
|
5
|
+
const { readConfig } = require('../System/config_manager');
|
|
6
|
+
|
|
7
|
+
// ============================================================
|
|
8
|
+
// Proactive Engine — Smart Suggestion Engine (Multi-Choice)
|
|
9
|
+
// ============================================================
|
|
10
|
+
|
|
11
|
+
const ai = new GoogleGenAI({});
|
|
12
|
+
const DEFAULT_GEMINI_MODEL = 'gemini-3.1-flash-lite-preview';
|
|
13
|
+
let lastLoggedModel = '';
|
|
14
|
+
|
|
15
|
+
const PROACTIVE_SYSTEM_PROMPT = `You are a Smart Suggestion Engine built into a Desktop AI Agent called "Mint".
|
|
16
|
+
Your job: observe the user's screen + behavior, then offer MULTIPLE relevant quick-action options — NOT just one question.
|
|
17
|
+
|
|
18
|
+
CRITICAL RULES:
|
|
19
|
+
1. Respond ONLY with valid JSON, no markdown.
|
|
20
|
+
2. If nothing notable is on screen, return: {"message": null, "context": "", "suggestions": []}
|
|
21
|
+
3. Generate 2–4 SHORT suggestion chips that are genuinely useful based on what's visible.
|
|
22
|
+
4. Each suggestion must have a clear label (1–3 words) and an action.
|
|
23
|
+
5. Write "message" in Thai — short, friendly, observational (e.g. "พบว่าคุณเพิ่งเปิด Chrome").
|
|
24
|
+
6. Do NOT repeat suggestions from recent activities.
|
|
25
|
+
7. Suggestions should feel like smart shortcuts, not questions.
|
|
26
|
+
|
|
27
|
+
Response schema (STRICT):
|
|
28
|
+
{
|
|
29
|
+
"context": "short English description of what you see on screen",
|
|
30
|
+
"message": "สั้น ๆ เป็นภาษาไทย บอกว่า AI เห็นอะไร และเสนออะไร",
|
|
31
|
+
"suggestions": [
|
|
32
|
+
{ "label": "YouTube", "action": { "type": "open_url", "target": "https://youtube.com" } },
|
|
33
|
+
{ "label": "Gmail", "action": { "type": "open_url", "target": "https://mail.google.com" } },
|
|
34
|
+
{ "label": "GitHub", "action": { "type": "open_url", "target": "https://github.com" } }
|
|
35
|
+
]
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
Action types allowed: "open_url", "open_app", "search", "none"
|
|
39
|
+
|
|
40
|
+
Examples:
|
|
41
|
+
|
|
42
|
+
SCENARIO: User opened Chrome or Firefox
|
|
43
|
+
→ message: "เพิ่งเปิด Browser — ต้องการเข้าเว็บไหนคะ?"
|
|
44
|
+
→ suggestions: YouTube, Gmail, GitHub, Google Maps (based on behavior history)
|
|
45
|
+
|
|
46
|
+
SCENARIO: User is in VS Code / coding
|
|
47
|
+
→ message: "กำลัง Code อยู่ใช่ไหมคะ? มีอะไรช่วยได้บ้าง"
|
|
48
|
+
→ suggestions: Stack Overflow, MDN Docs, GitHub, ค้นหา Error
|
|
49
|
+
|
|
50
|
+
SCENARIO: User opened Spotify
|
|
51
|
+
→ message: "เปิด Spotify แล้ว ต้องการเล่นอะไรคะ?"
|
|
52
|
+
→ suggestions: เพลง Chill, เพลง Focus, Top Charts, Podcast
|
|
53
|
+
|
|
54
|
+
SCENARIO: User opened Terminal
|
|
55
|
+
→ message: "เปิด Terminal แล้ว ต้องการทำอะไรคะ?"
|
|
56
|
+
→ suggestions: GitHub, Stack Overflow, DevDocs, ค้นหา Command
|
|
57
|
+
|
|
58
|
+
BAD examples (return null):
|
|
59
|
+
- Nothing notable on screen
|
|
60
|
+
- User is actively typing
|
|
61
|
+
- Same context as before
|
|
62
|
+
`;
|
|
63
|
+
|
|
64
|
+
let lastSuggestionContext = '';
|
|
65
|
+
let lastSuggestionTime = 0;
|
|
66
|
+
|
|
67
|
+
function resolveGeminiModel() {
|
|
68
|
+
try {
|
|
69
|
+
const cfg = readConfig();
|
|
70
|
+
const model = (cfg.geminiModel || '').trim();
|
|
71
|
+
return model || DEFAULT_GEMINI_MODEL;
|
|
72
|
+
} catch {
|
|
73
|
+
return DEFAULT_GEMINI_MODEL;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function getMinSuggestionIntervalMs() {
|
|
78
|
+
try {
|
|
79
|
+
const CONFIG_PATH = path.join(app.getPath('userData'), 'mint-config.json');
|
|
80
|
+
if (fs.existsSync(CONFIG_PATH)) {
|
|
81
|
+
const cfg = JSON.parse(fs.readFileSync(CONFIG_PATH, 'utf8'));
|
|
82
|
+
return (cfg.proactiveCooldown || 120) * 1000;
|
|
83
|
+
}
|
|
84
|
+
} catch {
|
|
85
|
+
// ignore
|
|
86
|
+
}
|
|
87
|
+
return 120_000;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Analyze screen and return a multi-choice suggestion object.
|
|
92
|
+
* @param {string} base64Image
|
|
93
|
+
* @param {string} behaviorSummary
|
|
94
|
+
* @returns {Promise<{message: string, context: string, suggestions: Array} | null>}
|
|
95
|
+
*/
|
|
96
|
+
async function analyzeAndSuggest(base64Image, behaviorSummary) {
|
|
97
|
+
try {
|
|
98
|
+
const model = resolveGeminiModel();
|
|
99
|
+
if (model && model !== lastLoggedModel) {
|
|
100
|
+
console.log(`[Gemini] Proactive Engine model: ${model}`);
|
|
101
|
+
lastLoggedModel = model;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const now = Date.now();
|
|
105
|
+
const minInterval = getMinSuggestionIntervalMs();
|
|
106
|
+
|
|
107
|
+
if (now - lastSuggestionTime < minInterval) return null;
|
|
108
|
+
|
|
109
|
+
const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, '');
|
|
110
|
+
|
|
111
|
+
const userMessage = [
|
|
112
|
+
{
|
|
113
|
+
text: `Analyze the screen and generate smart multi-choice suggestions for the user.
|
|
114
|
+
|
|
115
|
+
User behavior context: ${behaviorSummary || 'No history yet.'}
|
|
116
|
+
|
|
117
|
+
Rules: Only suggest if you see a clear opportunity. Return 2–4 relevant chips. Return null message if nothing notable.`
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
inlineData: {
|
|
121
|
+
mimeType: 'image/png',
|
|
122
|
+
data: base64Data
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
];
|
|
126
|
+
|
|
127
|
+
const response = await ai.models.generateContent({
|
|
128
|
+
model,
|
|
129
|
+
config: {
|
|
130
|
+
systemInstruction: PROACTIVE_SYSTEM_PROMPT,
|
|
131
|
+
responseMimeType: 'application/json'
|
|
132
|
+
},
|
|
133
|
+
contents: [{ role: 'user', parts: userMessage }]
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
let parsed;
|
|
137
|
+
try {
|
|
138
|
+
parsed = JSON.parse(response.text);
|
|
139
|
+
} catch {
|
|
140
|
+
const jsonMatch = response.text.match(/\{[\s\S]*\}/);
|
|
141
|
+
if (jsonMatch) parsed = JSON.parse(jsonMatch[0]);
|
|
142
|
+
else return null;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Validate: must have message and at least 1 suggestion
|
|
146
|
+
if (!parsed || !parsed.message || !Array.isArray(parsed.suggestions) || parsed.suggestions.length === 0) {
|
|
147
|
+
return null;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Skip repeat context
|
|
151
|
+
if (parsed.context && parsed.context === lastSuggestionContext) {
|
|
152
|
+
console.log('[ProactiveEngine] Skipping repeat context.');
|
|
153
|
+
return null;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
lastSuggestionContext = parsed.context || '';
|
|
157
|
+
lastSuggestionTime = now;
|
|
158
|
+
|
|
159
|
+
console.log(`[ProactiveEngine] ${parsed.suggestions.length} suggestions for: ${parsed.context}`);
|
|
160
|
+
return parsed;
|
|
161
|
+
|
|
162
|
+
} catch (err) {
|
|
163
|
+
console.error('[ProactiveEngine] Error:', err.message);
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
module.exports = { analyzeAndSuggest };
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
const puppeteer = require('puppeteer');
|
|
2
|
+
const { GoogleGenAI } = require('@google/genai');
|
|
3
|
+
const { readConfig } = require('../System/config_manager');
|
|
4
|
+
|
|
5
|
+
const ai = new GoogleGenAI({});
|
|
6
|
+
const DEFAULT_GEMINI_MODEL = 'gemini-3.1-flash-lite-preview';
|
|
7
|
+
let lastLoggedModel = '';
|
|
8
|
+
|
|
9
|
+
const BROWSER_SYSTEM_PROMPT = `You are an Autonomous Browser Agent. Your goal is to fulfill the user's web instruction by driving a headless browser.
|
|
10
|
+
|
|
11
|
+
CRITICAL INSTRUCTIONS:
|
|
12
|
+
Always respond EXACTLY with valid JSON. NO MARKDOWN. NO CODE BLOCKS (\`\`\`json). The JSON must have this exact structure:
|
|
13
|
+
{
|
|
14
|
+
"thought": "Reasoning about what to do next based on the current page content and goal.",
|
|
15
|
+
"action": "goto" | "click" | "eval" | "done",
|
|
16
|
+
"target": "URL for goto | CSS selector for click | JavaScript expression for eval | Final answer for done"
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
Actions:
|
|
20
|
+
- "goto": Navigate to the specified URL. Target MUST be a full URL (e.g. "https://www.google.com/search?q=AI+news")
|
|
21
|
+
- "click": Click an element. Target MUST be a valid CSS selector.
|
|
22
|
+
- "eval": Evaluate JavaScript to extract text. Target MUST be JS code returning a string (e.g. "document.body.innerText.substring(0, 1000)").
|
|
23
|
+
- "done": Task finished. Target MUST be the final summary or answer to present to the user.
|
|
24
|
+
|
|
25
|
+
You will receive the result of your previous action in the next message. If you get stuck or fail, try another approach or use "done" to report the failure.`;
|
|
26
|
+
|
|
27
|
+
function resolveGeminiModel() {
|
|
28
|
+
try {
|
|
29
|
+
const cfg = readConfig();
|
|
30
|
+
const model = (cfg.geminiModel || '').trim();
|
|
31
|
+
return model || DEFAULT_GEMINI_MODEL;
|
|
32
|
+
} catch {
|
|
33
|
+
return DEFAULT_GEMINI_MODEL;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function performWebAutomation(query) {
|
|
38
|
+
if (!query) return "No query provided.";
|
|
39
|
+
|
|
40
|
+
console.log("Starting web automation for:", query);
|
|
41
|
+
|
|
42
|
+
const config = readConfig();
|
|
43
|
+
const browserPath = config.automationBrowser;
|
|
44
|
+
|
|
45
|
+
let browser;
|
|
46
|
+
try {
|
|
47
|
+
const launchOptions = {
|
|
48
|
+
headless: false,
|
|
49
|
+
defaultViewport: null,
|
|
50
|
+
args: ['--start-maximized']
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
// If it's a specific path (like /usr/bin/firefox), set executablePath
|
|
54
|
+
if (browserPath && browserPath !== 'chromium') {
|
|
55
|
+
launchOptions.executablePath = browserPath;
|
|
56
|
+
if (browserPath.toLowerCase().includes('firefox')) {
|
|
57
|
+
launchOptions.browser = 'firefox';
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
browser = await puppeteer.launch(launchOptions);
|
|
62
|
+
|
|
63
|
+
const page = await browser.newPage();
|
|
64
|
+
|
|
65
|
+
const model = resolveGeminiModel();
|
|
66
|
+
if (model && model !== lastLoggedModel) {
|
|
67
|
+
console.log(`[Gemini] Web Automation model: ${model}`);
|
|
68
|
+
lastLoggedModel = model;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const chat = ai.chats.create({
|
|
72
|
+
model,
|
|
73
|
+
config: {
|
|
74
|
+
systemInstruction: BROWSER_SYSTEM_PROMPT,
|
|
75
|
+
responseMimeType: "application/json"
|
|
76
|
+
}
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
let currentObservation = `Goal: ${query}\nSystem Note: You have a blank browser page. What is your first action? Start by using "goto" to navigate to a relevant search engine or website.`;
|
|
80
|
+
|
|
81
|
+
let maxSteps = 10;
|
|
82
|
+
let step = 0;
|
|
83
|
+
|
|
84
|
+
while (step < maxSteps) {
|
|
85
|
+
step++;
|
|
86
|
+
console.log(`\n--- Agent Step ${step} ---`);
|
|
87
|
+
console.log(`Observation:`, currentObservation.substring(0, 150) + (currentObservation.length > 150 ? '...' : ''));
|
|
88
|
+
|
|
89
|
+
const response = await chat.sendMessage({ message: currentObservation });
|
|
90
|
+
|
|
91
|
+
let parsed;
|
|
92
|
+
try {
|
|
93
|
+
const text = response.text;
|
|
94
|
+
const cleanText = text.replace(/^```json\n/, '').replace(/\n```$/, '').trim();
|
|
95
|
+
parsed = JSON.parse(cleanText);
|
|
96
|
+
} catch (e) {
|
|
97
|
+
console.error("Agent failed to return valid JSON:", response.text);
|
|
98
|
+
currentObservation = "Error: Invalid JSON returned. Please reply with ONLY valid JSON matching the schema.";
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
console.log("Agent Thought:", parsed.thought);
|
|
103
|
+
console.log("Agent Action:", parsed.action);
|
|
104
|
+
console.log("Agent Target:", parsed.target);
|
|
105
|
+
|
|
106
|
+
const { action, target } = parsed;
|
|
107
|
+
|
|
108
|
+
if (action === 'done') {
|
|
109
|
+
console.log("Agent finished with answer:", target);
|
|
110
|
+
// Intentionally keeping the browser open so the user can see the page.
|
|
111
|
+
return `🤖 Web Automation Result: ${target}`;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
try {
|
|
115
|
+
if (action === 'goto') {
|
|
116
|
+
await page.goto(target, { waitUntil: 'domcontentloaded', timeout: 30000 });
|
|
117
|
+
const pageTitle = await page.title();
|
|
118
|
+
currentObservation = `Successfully navigated to ${pageTitle}. ` + await page.evaluate(() => document.body.innerText.substring(0, 1500));
|
|
119
|
+
} else if (action === 'click') {
|
|
120
|
+
await page.waitForSelector(target, { timeout: 5000 });
|
|
121
|
+
await page.click(target);
|
|
122
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
123
|
+
const pageTitle = await page.title();
|
|
124
|
+
currentObservation = `Clicked element. Current page: ${pageTitle}. ` + await page.evaluate(() => document.body.innerText.substring(0, 1500));
|
|
125
|
+
} else if (action === 'eval') {
|
|
126
|
+
const evalResult = await page.evaluate(target);
|
|
127
|
+
currentObservation = `Eval result: ` + String(evalResult).substring(0, 1500);
|
|
128
|
+
} else {
|
|
129
|
+
currentObservation = `Error: Unknown action type "${action}".`;
|
|
130
|
+
}
|
|
131
|
+
} catch (actionError) {
|
|
132
|
+
console.error("Action execution failed:", actionError);
|
|
133
|
+
currentObservation = `Action failed: ${actionError.message}. Please try again or use another method (for instance, try a different CSS selector or just read the current page).`;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Intentionally keeping the browser open
|
|
138
|
+
return "Agent reached maximum steps (10) without finding a final answer.";
|
|
139
|
+
|
|
140
|
+
} catch (error) {
|
|
141
|
+
console.error("Web Automation Error:", error);
|
|
142
|
+
if (browser) browser.close();
|
|
143
|
+
return `I encountered an overall error while automating the browser: ${error.message}`;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
module.exports = { performWebAutomation };
|