neoagent 2.3.0 → 2.3.1-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -0
- package/README.md +3 -1
- package/docs/automation.md +1 -1
- package/docs/capabilities.md +2 -2
- package/docs/configuration.md +14 -1
- package/docs/integrations.md +6 -1
- package/lib/manager.js +127 -1
- package/package.json +2 -1
- package/server/db/database.js +68 -0
- package/server/http/middleware.js +50 -0
- package/server/http/routes.js +3 -1
- package/server/index.js +1 -0
- package/server/public/.last_build_id +1 -1
- package/server/public/assets/NOTICES +61 -0
- package/server/public/assets/fonts/MaterialIcons-Regular.otf +0 -0
- package/server/public/flutter_bootstrap.js +1 -1
- package/server/public/main.dart.js +61049 -60444
- package/server/routes/integrations.js +97 -8
- package/server/routes/memory.js +11 -2
- package/server/routes/screenHistory.js +46 -0
- package/server/routes/triggers.js +81 -0
- package/server/services/ai/engine.js +9 -0
- package/server/services/ai/models.js +30 -0
- package/server/services/ai/providers/githubCopilot.js +97 -0
- package/server/services/ai/providers/openaiCodex.js +26 -0
- package/server/services/ai/settings.js +20 -0
- package/server/services/ai/systemPrompt.js +1 -1
- package/server/services/ai/tools.js +150 -11
- package/server/services/browser/controller.js +47 -3
- package/server/services/desktop/screenRecorder.js +126 -0
- package/server/services/integrations/env.js +19 -0
- package/server/services/integrations/github/common.js +106 -0
- package/server/services/integrations/github/provider.js +499 -0
- package/server/services/integrations/github/repos.js +1124 -0
- package/server/services/integrations/home_assistant/provider.js +630 -0
- package/server/services/integrations/manager.js +63 -7
- package/server/services/integrations/oauth_provider.js +13 -6
- package/server/services/integrations/provider_config_store.js +76 -0
- package/server/services/integrations/registry.js +10 -0
- package/server/services/integrations/spotify/provider.js +487 -0
- package/server/services/integrations/weather/provider.js +559 -0
- package/server/services/integrations/whatsapp/provider.js +6 -2
- package/server/services/manager.js +22 -0
- package/server/services/memory/manager.js +39 -2
- package/server/services/messaging/manager.js +29 -7
- package/server/services/skills/base_catalog.js +33 -0
- package/server/services/tasks/adapters/index.js +2 -0
- package/server/services/tasks/adapters/manual.js +12 -0
- package/server/services/tasks/adapters/schedule.js +33 -5
- package/server/services/tasks/adapters/weather_event.js +84 -0
- package/server/services/tasks/integration_runtime.js +85 -0
- package/server/services/tasks/runtime.js +2 -2
- package/server/services/voice/agentBridge.js +20 -4
- package/server/services/voice/message.js +3 -0
- package/server/services/voice/openaiClient.js +4 -1
- package/server/services/voice/providers.js +2 -1
- package/server/services/voice/runtimeManager.js +136 -1
- package/server/services/widgets/service.js +49 -4
- package/server/utils/local_secrets.js +56 -0
- package/server/utils/logger.js +37 -9
|
@@ -91,6 +91,103 @@ function compactToolDefinition(tool, options = {}) {
|
|
|
91
91
|
return compact;
|
|
92
92
|
}
|
|
93
93
|
|
|
94
|
+
function normalizeScheduleTriggerConfig(inputConfig = {}) {
|
|
95
|
+
if (!inputConfig || typeof inputConfig !== 'object' || Array.isArray(inputConfig)) {
|
|
96
|
+
return inputConfig;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const normalized = { ...inputConfig };
|
|
100
|
+
const schedule = (normalized.schedule && typeof normalized.schedule === 'object' && !Array.isArray(normalized.schedule))
|
|
101
|
+
? normalized.schedule
|
|
102
|
+
: null;
|
|
103
|
+
|
|
104
|
+
const modeCandidate = String(
|
|
105
|
+
normalized.mode
|
|
106
|
+
|| normalized.type
|
|
107
|
+
|| normalized.scheduleType
|
|
108
|
+
|| normalized.schedule_type
|
|
109
|
+
|| (normalized.oneTime || normalized.one_time ? 'one_time' : '')
|
|
110
|
+
|| ''
|
|
111
|
+
).trim().toLowerCase();
|
|
112
|
+
if (modeCandidate === 'once') normalized.mode = 'one_time';
|
|
113
|
+
else if (modeCandidate === 'one_time' || modeCandidate === 'recurring') normalized.mode = modeCandidate;
|
|
114
|
+
|
|
115
|
+
const cronCandidate = normalized.cronExpression
|
|
116
|
+
|| normalized.cron_expression
|
|
117
|
+
|| normalized.cron
|
|
118
|
+
|| schedule?.cronExpression
|
|
119
|
+
|| schedule?.cron_expression
|
|
120
|
+
|| schedule?.cron;
|
|
121
|
+
if (cronCandidate != null) {
|
|
122
|
+
normalized.cronExpression = String(cronCandidate).trim();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const runAtCandidate = normalized.runAt
|
|
126
|
+
|| normalized.run_at
|
|
127
|
+
|| normalized.at
|
|
128
|
+
|| normalized.when
|
|
129
|
+
|| schedule?.runAt
|
|
130
|
+
|| schedule?.run_at
|
|
131
|
+
|| schedule?.at
|
|
132
|
+
|| schedule?.when;
|
|
133
|
+
if (runAtCandidate != null) {
|
|
134
|
+
normalized.runAt = String(runAtCandidate).trim();
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (!normalized.mode) {
|
|
138
|
+
normalized.mode = normalized.runAt ? 'one_time' : 'recurring';
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
delete normalized.schedule;
|
|
142
|
+
return normalized;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function normalizeTaskTriggerInput(triggerType, triggerConfig) {
|
|
146
|
+
if (String(triggerType || '').trim() !== 'schedule') {
|
|
147
|
+
return triggerConfig;
|
|
148
|
+
}
|
|
149
|
+
return normalizeScheduleTriggerConfig(triggerConfig || {});
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function resolveTaskTriggerArgs(args = {}, fallbackTriggerType = null) {
|
|
153
|
+
let triggerType = args.trigger_type !== undefined
|
|
154
|
+
? String(args.trigger_type || '').trim()
|
|
155
|
+
: String(fallbackTriggerType || '').trim();
|
|
156
|
+
let triggerConfig = args.trigger_config;
|
|
157
|
+
let hasType = args.trigger_type !== undefined;
|
|
158
|
+
let hasConfig = args.trigger_config !== undefined;
|
|
159
|
+
|
|
160
|
+
if (args.trigger && typeof args.trigger === 'object' && !Array.isArray(args.trigger)) {
|
|
161
|
+
const trigger = args.trigger;
|
|
162
|
+
const unifiedType = trigger.type ?? trigger.triggerType ?? trigger.trigger_type;
|
|
163
|
+
const unifiedHasType = unifiedType !== undefined && unifiedType !== null && String(unifiedType).trim().length > 0;
|
|
164
|
+
if (unifiedHasType) {
|
|
165
|
+
triggerType = String(unifiedType).trim();
|
|
166
|
+
hasType = true;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const hasUnifiedConfig = Object.prototype.hasOwnProperty.call(trigger, 'config')
|
|
170
|
+
|| Object.prototype.hasOwnProperty.call(trigger, 'triggerConfig')
|
|
171
|
+
|| Object.prototype.hasOwnProperty.call(trigger, 'trigger_config');
|
|
172
|
+
if (hasUnifiedConfig) {
|
|
173
|
+
triggerConfig = trigger.config ?? trigger.triggerConfig ?? trigger.trigger_config;
|
|
174
|
+
hasConfig = true;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const normalizedType = String(triggerType || '').trim() || null;
|
|
179
|
+
const normalizedConfig = hasConfig
|
|
180
|
+
? normalizeTaskTriggerInput(normalizedType || fallbackTriggerType || 'schedule', triggerConfig)
|
|
181
|
+
: undefined;
|
|
182
|
+
|
|
183
|
+
return {
|
|
184
|
+
triggerType: normalizedType,
|
|
185
|
+
triggerConfig: normalizedConfig,
|
|
186
|
+
hasType,
|
|
187
|
+
hasConfig,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
|
|
94
191
|
function isProactiveTrigger(triggerSource) {
|
|
95
192
|
return triggerSource === 'schedule' || triggerSource === 'tasks';
|
|
96
193
|
}
|
|
@@ -876,15 +973,16 @@ function getAvailableTools(app, options = {}) {
|
|
|
876
973
|
type: 'object',
|
|
877
974
|
properties: {
|
|
878
975
|
name: { type: 'string', description: 'Short descriptive name for the task.' },
|
|
879
|
-
|
|
880
|
-
|
|
976
|
+
trigger: { type: 'object', description: 'Unified trigger object. Prefer { type: "manual" | "schedule" | integration_trigger_type, config: {...} }.' },
|
|
977
|
+
trigger_type: { type: 'string', description: 'Trigger type such as manual, schedule, gmail_message_received, outlook_email_received, slack_message_received, teams_message_received, weather_event, or whatsapp_personal_message_received.' },
|
|
978
|
+
trigger_config: { type: 'object', description: 'Trigger-specific configuration object. For schedule triggers prefer { mode: "recurring", cronExpression: "m h dom mon dow" } or { mode: "one_time", runAt: ISO datetime }. 5-field cron only (seconds unsupported).' },
|
|
881
979
|
prompt: { type: 'string', description: 'The instructions the agent will run when the trigger fires.' },
|
|
882
980
|
enabled: { type: 'boolean', description: 'Whether to activate immediately.' },
|
|
883
981
|
model: { type: 'string', description: 'Optional model override.' },
|
|
884
982
|
call_to: { type: 'string', description: 'Optional E.164 phone number to call via Telnyx when this task fires.' },
|
|
885
983
|
call_greeting: { type: 'string', description: 'Optional spoken greeting hint for make_call.' }
|
|
886
984
|
},
|
|
887
|
-
required: ['name', '
|
|
985
|
+
required: ['name', 'prompt']
|
|
888
986
|
}
|
|
889
987
|
},
|
|
890
988
|
{
|
|
@@ -911,8 +1009,9 @@ function getAvailableTools(app, options = {}) {
|
|
|
911
1009
|
properties: {
|
|
912
1010
|
task_id: { type: 'number', description: 'The numeric ID of the task to update.' },
|
|
913
1011
|
name: { type: 'string', description: 'New name for the task.' },
|
|
914
|
-
|
|
915
|
-
|
|
1012
|
+
trigger: { type: 'object', description: 'Unified trigger object. Use { type, config } to update trigger in one section.' },
|
|
1013
|
+
trigger_type: { type: 'string', description: 'Updated trigger type, e.g. manual, schedule, or integration trigger type.' },
|
|
1014
|
+
trigger_config: { type: 'object', description: 'Updated trigger-specific configuration. For schedule triggers use mode+cronExpression (recurring) or mode+runAt (one_time).' },
|
|
916
1015
|
prompt: { type: 'string', description: 'Updated task prompt.' },
|
|
917
1016
|
enabled: { type: 'boolean', description: 'Enable or disable the task.' },
|
|
918
1017
|
model: { type: 'string', description: 'Specific AI model ID for this task. Set to empty string to clear the override.' },
|
|
@@ -1053,6 +1152,17 @@ function getAvailableTools(app, options = {}) {
|
|
|
1053
1152
|
required: ['image_path']
|
|
1054
1153
|
}
|
|
1055
1154
|
},
|
|
1155
|
+
{
|
|
1156
|
+
name: 'ocr_extract',
|
|
1157
|
+
description: 'Extract raw text from an image locally using Tesseract OCR. This is faster and completely offline compared to analyze_image.',
|
|
1158
|
+
parameters: {
|
|
1159
|
+
type: 'object',
|
|
1160
|
+
properties: {
|
|
1161
|
+
image_path: { type: 'string', description: 'Absolute path to the image file' }
|
|
1162
|
+
},
|
|
1163
|
+
required: ['image_path']
|
|
1164
|
+
}
|
|
1165
|
+
},
|
|
1056
1166
|
{
|
|
1057
1167
|
name: 'read_health_data',
|
|
1058
1168
|
description: 'Read the user\'s synced mobile health data. Omit metric_type for a summary of all available metrics. With metric_type, returns an aggregate summary (total, avg, min, max over all stored data) plus the most recent individual records. Always report the summary figures — avoid listing every raw record.',
|
|
@@ -1123,7 +1233,8 @@ function getAvailableTools(app, options = {}) {
|
|
|
1123
1233
|
properties: {
|
|
1124
1234
|
content: { type: 'string', description: 'Natural assistant message derived from the current task state.' },
|
|
1125
1235
|
kind: { type: 'string', enum: Array.from(INTERIM_KINDS), description: 'ack, progress, question, or blocker' },
|
|
1126
|
-
expects_reply: { type: 'boolean', description: 'Set true only when the current run should pause for the user to answer.' }
|
|
1236
|
+
expects_reply: { type: 'boolean', description: 'Set true only when the current run should pause for the user to answer.' },
|
|
1237
|
+
defer_follow_up: { type: 'boolean', description: 'Set true when you choose to deliver the final result later via the user\'s last connected chat target.' }
|
|
1127
1238
|
},
|
|
1128
1239
|
required: ['content', 'kind']
|
|
1129
1240
|
}
|
|
@@ -1847,6 +1958,7 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1847
1958
|
}
|
|
1848
1959
|
const interimContent = typeof args.content === 'string' ? args.content : '';
|
|
1849
1960
|
const expectsReply = args.expects_reply === true;
|
|
1961
|
+
const deferFollowUp = args.defer_follow_up === true;
|
|
1850
1962
|
return engine.publishInterimUpdate({
|
|
1851
1963
|
userId,
|
|
1852
1964
|
runId,
|
|
@@ -1858,6 +1970,7 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
1858
1970
|
content: interimContent,
|
|
1859
1971
|
kind: normalizeInterimKind(args.kind),
|
|
1860
1972
|
expectsReply,
|
|
1973
|
+
deferFollowUp,
|
|
1861
1974
|
});
|
|
1862
1975
|
}
|
|
1863
1976
|
|
|
@@ -2205,10 +2318,21 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2205
2318
|
const s = taskRuntime();
|
|
2206
2319
|
if (!s) return { error: 'Task runtime not available' };
|
|
2207
2320
|
try {
|
|
2321
|
+
const resolvedTrigger = resolveTaskTriggerArgs(args, 'schedule');
|
|
2322
|
+
if (!resolvedTrigger.hasType || !resolvedTrigger.triggerType) {
|
|
2323
|
+
return { error: 'Task trigger type is required (use trigger.type or trigger_type).' };
|
|
2324
|
+
}
|
|
2325
|
+
const normalizedTriggerType = String(resolvedTrigger.triggerType || '').trim();
|
|
2326
|
+
const triggerConfig = (!resolvedTrigger.hasConfig || resolvedTrigger.triggerConfig === undefined)
|
|
2327
|
+
? (normalizedTriggerType === 'manual' ? {} : undefined)
|
|
2328
|
+
: resolvedTrigger.triggerConfig;
|
|
2329
|
+
if (triggerConfig === undefined) {
|
|
2330
|
+
return { error: 'Task trigger config is required (use trigger.config or trigger_config).' };
|
|
2331
|
+
}
|
|
2208
2332
|
const task = await s.createTask(userId, {
|
|
2209
2333
|
name: args.name,
|
|
2210
|
-
triggerType:
|
|
2211
|
-
triggerConfig
|
|
2334
|
+
triggerType: normalizedTriggerType,
|
|
2335
|
+
triggerConfig,
|
|
2212
2336
|
prompt: args.prompt,
|
|
2213
2337
|
enabled: args.enabled !== false,
|
|
2214
2338
|
model: args.model || null,
|
|
@@ -2246,12 +2370,13 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2246
2370
|
const s = taskRuntime();
|
|
2247
2371
|
if (!s) return { error: 'Task runtime not available' };
|
|
2248
2372
|
try {
|
|
2249
|
-
const existing = db.prepare('SELECT agent_id FROM scheduled_tasks WHERE id = ? AND user_id = ?').get(args.task_id, userId);
|
|
2373
|
+
const existing = db.prepare('SELECT agent_id, trigger_type FROM scheduled_tasks WHERE id = ? AND user_id = ?').get(args.task_id, userId);
|
|
2250
2374
|
if (!existing || existing.agent_id !== agentId) return { error: 'Task not found for this agent.' };
|
|
2251
2375
|
const updates = {};
|
|
2376
|
+
const resolvedTrigger = resolveTaskTriggerArgs(args, existing.trigger_type || 'schedule');
|
|
2252
2377
|
if (args.name !== undefined) updates.name = args.name;
|
|
2253
|
-
if (
|
|
2254
|
-
if (
|
|
2378
|
+
if (resolvedTrigger.hasType && resolvedTrigger.triggerType) updates.triggerType = resolvedTrigger.triggerType;
|
|
2379
|
+
if (resolvedTrigger.hasConfig && resolvedTrigger.triggerConfig !== undefined) updates.triggerConfig = resolvedTrigger.triggerConfig;
|
|
2255
2380
|
if (args.prompt !== undefined) updates.prompt = args.prompt;
|
|
2256
2381
|
if (args.enabled !== undefined) updates.enabled = args.enabled;
|
|
2257
2382
|
if (args.model !== undefined) updates.model = args.model || null;
|
|
@@ -2470,6 +2595,20 @@ async function executeTool(toolName, args, context, engine) {
|
|
|
2470
2595
|
}
|
|
2471
2596
|
}
|
|
2472
2597
|
|
|
2598
|
+
case 'ocr_extract': {
|
|
2599
|
+
try {
|
|
2600
|
+
const fs = require('fs');
|
|
2601
|
+
if (!fs.existsSync(args.image_path)) {
|
|
2602
|
+
return { error: 'File not found: ' + args.image_path };
|
|
2603
|
+
}
|
|
2604
|
+
const Tesseract = require('tesseract.js');
|
|
2605
|
+
const result = await Tesseract.recognize(args.image_path, 'eng');
|
|
2606
|
+
return { text: result.data.text, confidence: result.data.confidence };
|
|
2607
|
+
} catch (err) {
|
|
2608
|
+
return { error: err.message };
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
|
|
2473
2612
|
case 'spawn_subagent': {
|
|
2474
2613
|
try {
|
|
2475
2614
|
const task = args.context ? `${args.task}\n\nContext: ${args.context}` : args.task;
|
|
@@ -6,11 +6,11 @@ const SCREENSHOTS_DIR = path.join(DATA_DIR, 'screenshots');
|
|
|
6
6
|
if (!fs.existsSync(SCREENSHOTS_DIR)) fs.mkdirSync(SCREENSHOTS_DIR, { recursive: true });
|
|
7
7
|
|
|
8
8
|
const USER_AGENTS = [
|
|
9
|
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/
|
|
10
|
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/
|
|
9
|
+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
|
10
|
+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
|
11
11
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
|
|
12
12
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36',
|
|
13
|
-
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/
|
|
13
|
+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
|
|
14
14
|
];
|
|
15
15
|
|
|
16
16
|
const VIEWPORTS = [
|
|
@@ -148,6 +148,50 @@ class BrowserController {
|
|
|
148
148
|
return arr;
|
|
149
149
|
}
|
|
150
150
|
});
|
|
151
|
+
|
|
152
|
+
// WebGL Spoofing
|
|
153
|
+
const getParameterProxyHandler = {
|
|
154
|
+
apply: function(target, ctx, args) {
|
|
155
|
+
const param = args[0];
|
|
156
|
+
// UNMASKED_VENDOR_WEBGL
|
|
157
|
+
if (param === 37445) return 'Google Inc. (Apple)';
|
|
158
|
+
// UNMASKED_RENDERER_WEBGL
|
|
159
|
+
if (param === 37446) return 'ANGLE (Apple, Apple M2, OpenGL 4.1)';
|
|
160
|
+
return Reflect.apply(target, ctx, args);
|
|
161
|
+
}
|
|
162
|
+
};
|
|
163
|
+
const getParam = WebGLRenderingContext.prototype.getParameter;
|
|
164
|
+
WebGLRenderingContext.prototype.getParameter = new Proxy(getParam, getParameterProxyHandler);
|
|
165
|
+
if (typeof WebGL2RenderingContext !== 'undefined') {
|
|
166
|
+
const getParam2 = WebGL2RenderingContext.prototype.getParameter;
|
|
167
|
+
WebGL2RenderingContext.prototype.getParameter = new Proxy(getParam2, getParameterProxyHandler);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// Canvas Spoofing (slight noise)
|
|
171
|
+
const originalFillText = CanvasRenderingContext2D.prototype.fillText;
|
|
172
|
+
CanvasRenderingContext2D.prototype.fillText = function(...args) {
|
|
173
|
+
if (!this._spoofing_applied) {
|
|
174
|
+
this._spoofing_applied = true;
|
|
175
|
+
const r = Math.random() * 0.0001;
|
|
176
|
+
const g = Math.random() * 0.0001;
|
|
177
|
+
const b = Math.random() * 0.0001;
|
|
178
|
+
this.fillStyle = \`rgba(\${Math.floor(r * 255)}, \${Math.floor(g * 255)}, \${Math.floor(b * 255)}, 0.01)\`;
|
|
179
|
+
originalFillText.call(this, "spoof", 0, 0);
|
|
180
|
+
}
|
|
181
|
+
return originalFillText.apply(this, args);
|
|
182
|
+
};
|
|
183
|
+
|
|
184
|
+
// Media Devices Spoofing
|
|
185
|
+
if (navigator.mediaDevices && navigator.mediaDevices.enumerateDevices) {
|
|
186
|
+
const originalEnumerateDevices = navigator.mediaDevices.enumerateDevices.bind(navigator.mediaDevices);
|
|
187
|
+
navigator.mediaDevices.enumerateDevices = async () => {
|
|
188
|
+
return [
|
|
189
|
+
{ kind: 'audioinput', deviceId: 'default', groupId: 'a', label: 'MacBook Pro Microphone' },
|
|
190
|
+
{ kind: 'audiooutput', deviceId: 'default', groupId: 'b', label: 'MacBook Pro Speakers' },
|
|
191
|
+
{ kind: 'videoinput', deviceId: 'default', groupId: 'c', label: 'FaceTime HD Camera' }
|
|
192
|
+
];
|
|
193
|
+
};
|
|
194
|
+
}
|
|
151
195
|
})();
|
|
152
196
|
`);
|
|
153
197
|
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs/promises');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const os = require('os');
|
|
6
|
+
const { exec } = require('child_process');
|
|
7
|
+
const { promisify } = require('util');
|
|
8
|
+
const tesseract = require('tesseract.js');
|
|
9
|
+
const db = require('../../db/database');
|
|
10
|
+
const { getErrorMessage } = require('../bootstrap_helpers');
|
|
11
|
+
|
|
12
|
+
const execAsync = promisify(exec);
|
|
13
|
+
|
|
14
|
+
class ScreenRecorder {
|
|
15
|
+
constructor() {
|
|
16
|
+
this.intervalMs = 10000; // 10 seconds
|
|
17
|
+
this.intervalId = null;
|
|
18
|
+
this.cleanupIntervalId = null;
|
|
19
|
+
this.isRecording = false;
|
|
20
|
+
this.isProcessing = false;
|
|
21
|
+
this.tempFilePath = path.join(os.tmpdir(), `neoagent-screen-${Date.now()}.png`);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
start() {
|
|
25
|
+
if (process.platform !== 'darwin') {
|
|
26
|
+
console.log('[ScreenRecorder] Not starting: Screen recording is currently macOS only.');
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
if (this.isRecording) return;
|
|
31
|
+
this.isRecording = true;
|
|
32
|
+
|
|
33
|
+
console.log('[ScreenRecorder] Starting continuous screen recording (10s interval)');
|
|
34
|
+
|
|
35
|
+
// Start the recording loop
|
|
36
|
+
this.intervalId = setInterval(() => this.captureAndProcess(), this.intervalMs);
|
|
37
|
+
|
|
38
|
+
// Run an initial capture
|
|
39
|
+
this.captureAndProcess();
|
|
40
|
+
|
|
41
|
+
// Start daily cleanup of old records (7 days)
|
|
42
|
+
this.cleanupIntervalId = setInterval(() => this.cleanupOldRecords(), 24 * 60 * 60 * 1000);
|
|
43
|
+
this.cleanupOldRecords();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
stop() {
|
|
47
|
+
this.isRecording = false;
|
|
48
|
+
if (this.intervalId) {
|
|
49
|
+
clearInterval(this.intervalId);
|
|
50
|
+
this.intervalId = null;
|
|
51
|
+
}
|
|
52
|
+
if (this.cleanupIntervalId) {
|
|
53
|
+
clearInterval(this.cleanupIntervalId);
|
|
54
|
+
this.cleanupIntervalId = null;
|
|
55
|
+
}
|
|
56
|
+
console.log('[ScreenRecorder] Stopped continuous screen recording');
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
async captureAndProcess() {
|
|
60
|
+
if (this.isProcessing || !this.isRecording) return;
|
|
61
|
+
this.isProcessing = true;
|
|
62
|
+
|
|
63
|
+
try {
|
|
64
|
+
// Capture screen silently (-x) to file
|
|
65
|
+
await execAsync(`screencapture -x "${this.tempFilePath}"`);
|
|
66
|
+
|
|
67
|
+
// Verify file exists
|
|
68
|
+
await fs.access(this.tempFilePath);
|
|
69
|
+
|
|
70
|
+
// Extract text via local OCR
|
|
71
|
+
const { data } = await tesseract.recognize(this.tempFilePath, 'eng+deu', {
|
|
72
|
+
logger: () => {} // Silence verbose OCR logs
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
const textContent = data.text.trim();
|
|
76
|
+
|
|
77
|
+
// Only store if meaningful text was found
|
|
78
|
+
if (textContent.length > 5) {
|
|
79
|
+
// We need a user ID. For the local desktop agent, usually user 1 or we query the active user.
|
|
80
|
+
const userRow = db.prepare('SELECT id FROM users ORDER BY id ASC LIMIT 1').get();
|
|
81
|
+
if (userRow) {
|
|
82
|
+
// Identify the active foreground app via AppleScript
|
|
83
|
+
let appName = 'Unknown';
|
|
84
|
+
try {
|
|
85
|
+
const { stdout } = await execAsync(`osascript -e 'tell application "System Events" to get name of first application process whose frontmost is true'`);
|
|
86
|
+
appName = stdout.trim();
|
|
87
|
+
} catch (e) {
|
|
88
|
+
// Ignore AppleScript errors
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
db.prepare(`
|
|
92
|
+
INSERT INTO screen_history (user_id, app_name, text_content)
|
|
93
|
+
VALUES (?, ?, ?)
|
|
94
|
+
`).run(userRow.id, appName, textContent);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
} catch (err) {
|
|
99
|
+
console.error('[ScreenRecorder] Capture/OCR failed:', getErrorMessage(err));
|
|
100
|
+
} finally {
|
|
101
|
+
// Always cleanup the screenshot image immediately
|
|
102
|
+
try {
|
|
103
|
+
await fs.unlink(this.tempFilePath);
|
|
104
|
+
} catch (e) {
|
|
105
|
+
// Ignore unlink errors if file didn't exist
|
|
106
|
+
}
|
|
107
|
+
this.isProcessing = false;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
cleanupOldRecords() {
|
|
112
|
+
try {
|
|
113
|
+
const result = db.prepare(`
|
|
114
|
+
DELETE FROM screen_history
|
|
115
|
+
WHERE timestamp < datetime('now', '-7 days')
|
|
116
|
+
`).run();
|
|
117
|
+
if (result.changes > 0) {
|
|
118
|
+
console.log(`[ScreenRecorder] Purged ${result.changes} old screen history records.`);
|
|
119
|
+
}
|
|
120
|
+
} catch (err) {
|
|
121
|
+
console.error('[ScreenRecorder] Cleanup failed:', getErrorMessage(err));
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
module.exports = { ScreenRecorder };
|
|
@@ -56,6 +56,22 @@ function resolveMicrosoftOAuthConfig() {
|
|
|
56
56
|
};
|
|
57
57
|
}
|
|
58
58
|
|
|
59
|
+
function resolveHomeAssistantOAuthConfig() {
|
|
60
|
+
const base = resolveOAuthConfig('HOME_ASSISTANT');
|
|
61
|
+
return {
|
|
62
|
+
...base,
|
|
63
|
+
baseUrl: trimEnv('HOME_ASSISTANT_BASE_URL').replace(/\/$/, ''),
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function resolveSpotifyOAuthConfig() {
|
|
68
|
+
return resolveOAuthConfig('SPOTIFY');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function resolveGithubOAuthConfig() {
|
|
72
|
+
return resolveOAuthConfig('GITHUB');
|
|
73
|
+
}
|
|
74
|
+
|
|
59
75
|
function describeEnvStatus(config, options = {}) {
|
|
60
76
|
const label = String(options.label || 'This integration').trim() || 'This integration';
|
|
61
77
|
if (config.configured) {
|
|
@@ -76,10 +92,13 @@ function describeEnvStatus(config, options = {}) {
|
|
|
76
92
|
module.exports = {
|
|
77
93
|
describeEnvStatus,
|
|
78
94
|
resolveFigmaOAuthConfig,
|
|
95
|
+
resolveHomeAssistantOAuthConfig,
|
|
79
96
|
resolveMicrosoftOAuthConfig,
|
|
80
97
|
resolveNotionOAuthConfig,
|
|
81
98
|
resolveOAuthConfig,
|
|
82
99
|
resolveGoogleOAuthConfig,
|
|
83
100
|
resolvePublicBaseUrl,
|
|
101
|
+
resolveSpotifyOAuthConfig,
|
|
84
102
|
resolveSlackOAuthConfig,
|
|
103
|
+
resolveGithubOAuthConfig,
|
|
85
104
|
};
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const crypto = require('crypto');
|
|
4
|
+
|
|
5
|
+
function base64UrlSha256(value) {
|
|
6
|
+
return crypto
|
|
7
|
+
.createHash('sha256')
|
|
8
|
+
.update(String(value || ''))
|
|
9
|
+
.digest('base64')
|
|
10
|
+
.replace(/\+/g, '-')
|
|
11
|
+
.replace(/\//g, '_')
|
|
12
|
+
.replace(/=+$/g, '');
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async function githubApiRequest(auth, options = {}) {
|
|
16
|
+
const {
|
|
17
|
+
method = 'GET',
|
|
18
|
+
path,
|
|
19
|
+
query = null,
|
|
20
|
+
body = null,
|
|
21
|
+
baseUrl = 'https://api.github.com',
|
|
22
|
+
token: overrideToken = '',
|
|
23
|
+
} = options;
|
|
24
|
+
|
|
25
|
+
const token = String(overrideToken || auth?.token || '').trim();
|
|
26
|
+
if (!token) {
|
|
27
|
+
throw new Error('GitHub authentication token is required for GitHub API requests.');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const url = new URL(path, baseUrl);
|
|
31
|
+
if (query && typeof query === 'object') {
|
|
32
|
+
for (const [key, value] of Object.entries(query)) {
|
|
33
|
+
if (value !== undefined && value !== null) {
|
|
34
|
+
url.searchParams.set(key, String(value));
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const headers = {
|
|
40
|
+
'Accept': 'application/vnd.github.v3+json',
|
|
41
|
+
'Authorization': `Bearer ${token}`,
|
|
42
|
+
'X-GitHub-Api-Version': '2022-11-28',
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
if (body && (method === 'POST' || method === 'PUT' || method === 'PATCH')) {
|
|
46
|
+
headers['Content-Type'] = 'application/json';
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const response = await fetch(url.toString(), {
|
|
50
|
+
method,
|
|
51
|
+
headers,
|
|
52
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
let data = null;
|
|
56
|
+
if (response.status !== 204 && response.status !== 205) {
|
|
57
|
+
const rawBody = await response.text();
|
|
58
|
+
if (rawBody.trim()) {
|
|
59
|
+
try {
|
|
60
|
+
data = JSON.parse(rawBody);
|
|
61
|
+
} catch {
|
|
62
|
+
if (!response.ok) {
|
|
63
|
+
const error = new Error(`GitHub API error ${response.status}: ${rawBody}`);
|
|
64
|
+
error.status = response.status;
|
|
65
|
+
error.data = rawBody;
|
|
66
|
+
throw error;
|
|
67
|
+
}
|
|
68
|
+
data = rawBody;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (!response.ok) {
|
|
74
|
+
const errorMessage =
|
|
75
|
+
(data && typeof data === 'object' ? data.message : null) ||
|
|
76
|
+
`GitHub API error: ${response.status}`;
|
|
77
|
+
const error = new Error(errorMessage);
|
|
78
|
+
error.status = response.status;
|
|
79
|
+
error.data = data;
|
|
80
|
+
throw error;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
return data;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function buildPaginationParams(options = {}) {
|
|
87
|
+
const params = {};
|
|
88
|
+
if (options.page) params.page = Number(options.page);
|
|
89
|
+
if (options.per_page) params.per_page = Math.min(Number(options.per_page) || 30, 100);
|
|
90
|
+
return params;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function parseOwnerRepo(ownerRepo) {
|
|
94
|
+
const parts = String(ownerRepo || '').split('/');
|
|
95
|
+
if (parts.length !== 2) {
|
|
96
|
+
throw new Error('owner_repo must be in format "owner/repo"');
|
|
97
|
+
}
|
|
98
|
+
return { owner: parts[0], repo: parts[1] };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
module.exports = {
|
|
102
|
+
base64UrlSha256,
|
|
103
|
+
buildPaginationParams,
|
|
104
|
+
githubApiRequest,
|
|
105
|
+
parseOwnerRepo,
|
|
106
|
+
};
|