@projectservan8n/cnapse 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ProviderSelector-MXRZFAOB.js +6 -0
- package/dist/chunk-OPX7FFL6.js +391 -0
- package/dist/index.js +733 -702
- package/package.json +17 -16
- package/src/agents/executor.ts +20 -13
- package/src/index.tsx +32 -6
- package/src/lib/tasks.ts +307 -323
- package/src/services/browser.ts +669 -0
- package/src/tools/index.ts +0 -1
- package/dist/ConfigUI-I2CJVODT.js +0 -305
- package/dist/Setup-KGYXCA7Y.js +0 -177
- package/dist/chunk-COKO6V5J.js +0 -50
- package/src/components/ConfigUI.tsx +0 -352
- package/src/components/Setup.tsx +0 -202
- package/src/lib/screen.ts +0 -118
- package/src/tools/vision.ts +0 -65
package/dist/index.js
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
|
+
ProviderSelector,
|
|
3
4
|
getApiKey,
|
|
4
5
|
getConfig,
|
|
5
6
|
setApiKey,
|
|
6
7
|
setModel,
|
|
7
8
|
setProvider
|
|
8
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-OPX7FFL6.js";
|
|
9
10
|
|
|
10
11
|
// src/index.tsx
|
|
11
12
|
import { render } from "ink";
|
|
12
13
|
|
|
13
14
|
// src/components/App.tsx
|
|
14
|
-
import { useState as
|
|
15
|
-
import { Box as
|
|
15
|
+
import { useState as useState6, useCallback as useCallback5 } from "react";
|
|
16
|
+
import { Box as Box6, Text as Text6, useApp, useInput as useInput2 } from "ink";
|
|
16
17
|
|
|
17
18
|
// src/components/Header.tsx
|
|
18
19
|
import { Box, Text } from "ink";
|
|
@@ -191,354 +192,14 @@ function HelpMenu({ onClose, onSelect }) {
|
|
|
191
192
|
);
|
|
192
193
|
}
|
|
193
194
|
|
|
194
|
-
// src/components/ProviderSelector.tsx
|
|
195
|
-
import { useState as useState2, useEffect } from "react";
|
|
196
|
-
import { Box as Box6, Text as Text6, useInput as useInput2 } from "ink";
|
|
197
|
-
import TextInput2 from "ink-text-input";
|
|
198
|
-
import Spinner from "ink-spinner";
|
|
199
|
-
|
|
200
|
-
// src/lib/ollama.ts
|
|
201
|
-
import { exec } from "child_process";
|
|
202
|
-
import { promisify } from "util";
|
|
203
|
-
var execAsync = promisify(exec);
|
|
204
|
-
async function checkOllamaStatus() {
|
|
205
|
-
try {
|
|
206
|
-
const { stdout } = await execAsync("ollama list", { timeout: 1e4 });
|
|
207
|
-
const lines = stdout.trim().split("\n");
|
|
208
|
-
const models = [];
|
|
209
|
-
for (let i = 1; i < lines.length; i++) {
|
|
210
|
-
const line = lines[i];
|
|
211
|
-
if (!line?.trim()) continue;
|
|
212
|
-
const parts = line.split(/\s{2,}/);
|
|
213
|
-
if (parts.length >= 3) {
|
|
214
|
-
models.push({
|
|
215
|
-
name: parts[0]?.trim() || "",
|
|
216
|
-
size: parts[2]?.trim() || "",
|
|
217
|
-
modified: parts[3]?.trim() || ""
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
return {
|
|
222
|
-
installed: true,
|
|
223
|
-
running: true,
|
|
224
|
-
models
|
|
225
|
-
};
|
|
226
|
-
} catch (err2) {
|
|
227
|
-
const errorMsg = err2 instanceof Error ? err2.message : "Unknown error";
|
|
228
|
-
if (errorMsg.includes("connect") || errorMsg.includes("refused")) {
|
|
229
|
-
return {
|
|
230
|
-
installed: true,
|
|
231
|
-
running: false,
|
|
232
|
-
models: [],
|
|
233
|
-
error: "Ollama is not running. Start it with: ollama serve"
|
|
234
|
-
};
|
|
235
|
-
}
|
|
236
|
-
if (errorMsg.includes("not found") || errorMsg.includes("not recognized")) {
|
|
237
|
-
return {
|
|
238
|
-
installed: false,
|
|
239
|
-
running: false,
|
|
240
|
-
models: [],
|
|
241
|
-
error: "Ollama not installed. Get it at: https://ollama.ai"
|
|
242
|
-
};
|
|
243
|
-
}
|
|
244
|
-
return {
|
|
245
|
-
installed: false,
|
|
246
|
-
running: false,
|
|
247
|
-
models: [],
|
|
248
|
-
error: errorMsg
|
|
249
|
-
};
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
function hasModel(status, modelId) {
|
|
253
|
-
const modelName = modelId.split(":")[0]?.toLowerCase() || "";
|
|
254
|
-
return status.models.some((m) => m.name.toLowerCase().startsWith(modelName));
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
// src/components/ProviderSelector.tsx
|
|
258
|
-
import { Fragment, jsx as jsx6, jsxs as jsxs5 } from "react/jsx-runtime";
|
|
259
|
-
var PROVIDERS = [
|
|
260
|
-
{
|
|
261
|
-
id: "ollama",
|
|
262
|
-
name: "Ollama",
|
|
263
|
-
description: "Local AI - Free, private",
|
|
264
|
-
needsApiKey: false,
|
|
265
|
-
models: [
|
|
266
|
-
{ id: "qwen2.5:0.5b", name: "Qwen 2.5 0.5B (fast)", recommended: true },
|
|
267
|
-
{ id: "qwen2.5:1.5b", name: "Qwen 2.5 1.5B" },
|
|
268
|
-
{ id: "qwen2.5:7b", name: "Qwen 2.5 7B (quality)" },
|
|
269
|
-
{ id: "llama3.2:1b", name: "Llama 3.2 1B" },
|
|
270
|
-
{ id: "llama3.2:3b", name: "Llama 3.2 3B" },
|
|
271
|
-
{ id: "codellama:7b", name: "Code Llama 7B" },
|
|
272
|
-
{ id: "llava:7b", name: "LLaVA 7B (vision)" }
|
|
273
|
-
]
|
|
274
|
-
},
|
|
275
|
-
{
|
|
276
|
-
id: "openrouter",
|
|
277
|
-
name: "OpenRouter",
|
|
278
|
-
description: "Many models, budget-friendly",
|
|
279
|
-
needsApiKey: true,
|
|
280
|
-
models: [
|
|
281
|
-
{ id: "openai/gpt-5-nano", name: "GPT-5 Nano ($0.05/1M) + Vision", recommended: true },
|
|
282
|
-
{ id: "google/gemini-2.5-flash-lite", name: "Gemini 2.5 Flash Lite ($0.10/1M)" },
|
|
283
|
-
{ id: "qwen/qwen-2.5-coder-32b-instruct", name: "Qwen Coder 32B ($0.07/1M)" },
|
|
284
|
-
{ id: "meta-llama/llama-3.3-70b-instruct", name: "Llama 3.3 70B ($0.10/1M)" },
|
|
285
|
-
{ id: "deepseek/deepseek-chat", name: "DeepSeek V3 ($0.14/1M)" }
|
|
286
|
-
]
|
|
287
|
-
},
|
|
288
|
-
{
|
|
289
|
-
id: "anthropic",
|
|
290
|
-
name: "Anthropic",
|
|
291
|
-
description: "Claude - Best reasoning",
|
|
292
|
-
needsApiKey: true,
|
|
293
|
-
models: [
|
|
294
|
-
{ id: "claude-3-5-sonnet-20241022", name: "Claude 3.5 Sonnet", recommended: true },
|
|
295
|
-
{ id: "claude-3-opus-20240229", name: "Claude 3 Opus" },
|
|
296
|
-
{ id: "claude-3-haiku-20240307", name: "Claude 3 Haiku" }
|
|
297
|
-
]
|
|
298
|
-
},
|
|
299
|
-
{
|
|
300
|
-
id: "openai",
|
|
301
|
-
name: "OpenAI",
|
|
302
|
-
description: "GPT models",
|
|
303
|
-
needsApiKey: true,
|
|
304
|
-
models: [
|
|
305
|
-
{ id: "gpt-4o", name: "GPT-4o", recommended: true },
|
|
306
|
-
{ id: "gpt-4o-mini", name: "GPT-4o Mini" },
|
|
307
|
-
{ id: "gpt-4-turbo", name: "GPT-4 Turbo" }
|
|
308
|
-
]
|
|
309
|
-
}
|
|
310
|
-
];
|
|
311
|
-
function ProviderSelector({ onClose, onSelect }) {
|
|
312
|
-
const config = getConfig();
|
|
313
|
-
const [step, setStep] = useState2("provider");
|
|
314
|
-
const [providerIndex, setProviderIndex] = useState2(() => {
|
|
315
|
-
const idx = PROVIDERS.findIndex((p) => p.id === config.provider);
|
|
316
|
-
return idx >= 0 ? idx : 0;
|
|
317
|
-
});
|
|
318
|
-
const [modelIndex, setModelIndex] = useState2(0);
|
|
319
|
-
const [apiKeyInput, setApiKeyInput] = useState2("");
|
|
320
|
-
const [selectedProvider, setSelectedProvider] = useState2(null);
|
|
321
|
-
const [ollamaStatus, setOllamaStatus] = useState2(null);
|
|
322
|
-
const [checkingOllama, setCheckingOllama] = useState2(false);
|
|
323
|
-
useEffect(() => {
|
|
324
|
-
if (step === "model" && selectedProvider?.id === "ollama" && !ollamaStatus) {
|
|
325
|
-
setCheckingOllama(true);
|
|
326
|
-
checkOllamaStatus().then((status) => {
|
|
327
|
-
setOllamaStatus(status);
|
|
328
|
-
setCheckingOllama(false);
|
|
329
|
-
if (!status.running) {
|
|
330
|
-
setStep("ollamaError");
|
|
331
|
-
}
|
|
332
|
-
});
|
|
333
|
-
}
|
|
334
|
-
}, [step, selectedProvider, ollamaStatus]);
|
|
335
|
-
useInput2((input, key) => {
|
|
336
|
-
if (key.escape) {
|
|
337
|
-
onClose();
|
|
338
|
-
return;
|
|
339
|
-
}
|
|
340
|
-
if (step === "provider") {
|
|
341
|
-
if (key.upArrow) {
|
|
342
|
-
setProviderIndex((prev) => prev > 0 ? prev - 1 : PROVIDERS.length - 1);
|
|
343
|
-
} else if (key.downArrow) {
|
|
344
|
-
setProviderIndex((prev) => prev < PROVIDERS.length - 1 ? prev + 1 : 0);
|
|
345
|
-
} else if (key.return) {
|
|
346
|
-
const provider = PROVIDERS[providerIndex];
|
|
347
|
-
setSelectedProvider(provider);
|
|
348
|
-
const currentIdx = provider.models.findIndex((m) => m.id === config.model);
|
|
349
|
-
const recommendedIdx = provider.models.findIndex((m) => m.recommended);
|
|
350
|
-
setModelIndex(currentIdx >= 0 ? currentIdx : recommendedIdx >= 0 ? recommendedIdx : 0);
|
|
351
|
-
if (provider.needsApiKey) {
|
|
352
|
-
const apiKeyProvider = provider.id;
|
|
353
|
-
if (!config.apiKeys[apiKeyProvider]) {
|
|
354
|
-
setStep("apiKey");
|
|
355
|
-
} else {
|
|
356
|
-
setStep("model");
|
|
357
|
-
}
|
|
358
|
-
} else {
|
|
359
|
-
setStep("model");
|
|
360
|
-
}
|
|
361
|
-
}
|
|
362
|
-
} else if (step === "model" && selectedProvider) {
|
|
363
|
-
if (key.upArrow) {
|
|
364
|
-
setModelIndex((prev) => prev > 0 ? prev - 1 : selectedProvider.models.length - 1);
|
|
365
|
-
} else if (key.downArrow) {
|
|
366
|
-
setModelIndex((prev) => prev < selectedProvider.models.length - 1 ? prev + 1 : 0);
|
|
367
|
-
} else if (key.return) {
|
|
368
|
-
const model = selectedProvider.models[modelIndex];
|
|
369
|
-
if (selectedProvider.id === "ollama" && ollamaStatus && !hasModel(ollamaStatus, model.id)) {
|
|
370
|
-
}
|
|
371
|
-
setProvider(selectedProvider.id);
|
|
372
|
-
setModel(model.id);
|
|
373
|
-
setStep("done");
|
|
374
|
-
onSelect(selectedProvider.id, model.id);
|
|
375
|
-
setTimeout(() => onClose(), 1500);
|
|
376
|
-
} else if (key.leftArrow || input === "b") {
|
|
377
|
-
setStep("provider");
|
|
378
|
-
setOllamaStatus(null);
|
|
379
|
-
}
|
|
380
|
-
} else if (step === "ollamaError") {
|
|
381
|
-
if (key.return || input === "b") {
|
|
382
|
-
setStep("provider");
|
|
383
|
-
setOllamaStatus(null);
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
});
|
|
387
|
-
const handleApiKeySubmit = (value) => {
|
|
388
|
-
if (value.trim() && selectedProvider) {
|
|
389
|
-
setApiKey(selectedProvider.id, value.trim());
|
|
390
|
-
setStep("model");
|
|
391
|
-
}
|
|
392
|
-
};
|
|
393
|
-
if (step === "provider") {
|
|
394
|
-
return /* @__PURE__ */ jsxs5(Box6, { flexDirection: "column", borderStyle: "round", borderColor: "cyan", padding: 1, width: 60, children: [
|
|
395
|
-
/* @__PURE__ */ jsx6(Box6, { marginBottom: 1, children: /* @__PURE__ */ jsx6(Text6, { bold: true, color: "cyan", children: "Select Provider" }) }),
|
|
396
|
-
/* @__PURE__ */ jsx6(Box6, { marginBottom: 1, children: /* @__PURE__ */ jsx6(Text6, { color: "gray", dimColor: true, children: "Arrows to navigate, Enter to select" }) }),
|
|
397
|
-
PROVIDERS.map((provider, index) => {
|
|
398
|
-
const isSelected = index === providerIndex;
|
|
399
|
-
const isCurrent = provider.id === config.provider;
|
|
400
|
-
const hasKey = provider.needsApiKey && provider.id !== "ollama" ? !!config.apiKeys[provider.id] : true;
|
|
401
|
-
return /* @__PURE__ */ jsxs5(Box6, { flexDirection: "column", children: [
|
|
402
|
-
/* @__PURE__ */ jsxs5(Text6, { color: isSelected ? "cyan" : "white", children: [
|
|
403
|
-
isSelected ? "\u276F " : " ",
|
|
404
|
-
provider.name,
|
|
405
|
-
isCurrent && /* @__PURE__ */ jsx6(Text6, { color: "green", children: " (current)" }),
|
|
406
|
-
provider.needsApiKey && !hasKey && /* @__PURE__ */ jsx6(Text6, { color: "red", children: " (needs key)" }),
|
|
407
|
-
provider.needsApiKey && hasKey && !isCurrent && /* @__PURE__ */ jsx6(Text6, { color: "yellow", children: " (key saved)" })
|
|
408
|
-
] }),
|
|
409
|
-
isSelected && /* @__PURE__ */ jsxs5(Text6, { color: "gray", children: [
|
|
410
|
-
" ",
|
|
411
|
-
provider.description
|
|
412
|
-
] })
|
|
413
|
-
] }, provider.id);
|
|
414
|
-
}),
|
|
415
|
-
/* @__PURE__ */ jsx6(Box6, { marginTop: 1, children: /* @__PURE__ */ jsx6(Text6, { color: "gray", dimColor: true, children: "Press Esc to cancel" }) })
|
|
416
|
-
] });
|
|
417
|
-
}
|
|
418
|
-
if (step === "apiKey" && selectedProvider) {
|
|
419
|
-
return /* @__PURE__ */ jsxs5(Box6, { flexDirection: "column", borderStyle: "round", borderColor: "cyan", padding: 1, width: 60, children: [
|
|
420
|
-
/* @__PURE__ */ jsx6(Box6, { marginBottom: 1, children: /* @__PURE__ */ jsx6(Text6, { bold: true, color: "cyan", children: "Enter API Key" }) }),
|
|
421
|
-
/* @__PURE__ */ jsxs5(Text6, { children: [
|
|
422
|
-
/* @__PURE__ */ jsx6(Text6, { color: "green", children: "\u2713" }),
|
|
423
|
-
" Provider: ",
|
|
424
|
-
selectedProvider.name
|
|
425
|
-
] }),
|
|
426
|
-
/* @__PURE__ */ jsxs5(Box6, { marginTop: 1, flexDirection: "column", children: [
|
|
427
|
-
/* @__PURE__ */ jsxs5(Text6, { color: "gray", dimColor: true, children: [
|
|
428
|
-
selectedProvider.id === "openrouter" && "Get key: openrouter.ai/keys",
|
|
429
|
-
selectedProvider.id === "anthropic" && "Get key: console.anthropic.com",
|
|
430
|
-
selectedProvider.id === "openai" && "Get key: platform.openai.com/api-keys"
|
|
431
|
-
] }),
|
|
432
|
-
/* @__PURE__ */ jsxs5(Box6, { marginTop: 1, children: [
|
|
433
|
-
/* @__PURE__ */ jsx6(Text6, { color: "cyan", children: "\u276F " }),
|
|
434
|
-
/* @__PURE__ */ jsx6(
|
|
435
|
-
TextInput2,
|
|
436
|
-
{
|
|
437
|
-
value: apiKeyInput,
|
|
438
|
-
onChange: setApiKeyInput,
|
|
439
|
-
onSubmit: handleApiKeySubmit,
|
|
440
|
-
mask: "*"
|
|
441
|
-
}
|
|
442
|
-
)
|
|
443
|
-
] })
|
|
444
|
-
] }),
|
|
445
|
-
/* @__PURE__ */ jsx6(Box6, { marginTop: 1, children: /* @__PURE__ */ jsx6(Text6, { color: "gray", dimColor: true, children: "Press Esc to cancel" }) })
|
|
446
|
-
] });
|
|
447
|
-
}
|
|
448
|
-
if (step === "ollamaError" && ollamaStatus) {
|
|
449
|
-
return /* @__PURE__ */ jsxs5(Box6, { flexDirection: "column", borderStyle: "round", borderColor: "red", padding: 1, width: 60, children: [
|
|
450
|
-
/* @__PURE__ */ jsx6(Box6, { marginBottom: 1, children: /* @__PURE__ */ jsx6(Text6, { bold: true, color: "red", children: "Ollama Not Available" }) }),
|
|
451
|
-
/* @__PURE__ */ jsx6(Text6, { color: "red", children: ollamaStatus.error }),
|
|
452
|
-
/* @__PURE__ */ jsxs5(Box6, { marginTop: 1, flexDirection: "column", children: [
|
|
453
|
-
!ollamaStatus.installed && /* @__PURE__ */ jsxs5(Fragment, { children: [
|
|
454
|
-
/* @__PURE__ */ jsx6(Text6, { children: "1. Install Ollama from https://ollama.ai" }),
|
|
455
|
-
/* @__PURE__ */ jsx6(Text6, { children: "2. Run: ollama pull qwen2.5:0.5b" }),
|
|
456
|
-
/* @__PURE__ */ jsx6(Text6, { children: "3. Try again" })
|
|
457
|
-
] }),
|
|
458
|
-
ollamaStatus.installed && !ollamaStatus.running && /* @__PURE__ */ jsxs5(Fragment, { children: [
|
|
459
|
-
/* @__PURE__ */ jsx6(Text6, { children: "1. Start Ollama: ollama serve" }),
|
|
460
|
-
/* @__PURE__ */ jsx6(Text6, { children: "2. Or run any model: ollama run qwen2.5:0.5b" }),
|
|
461
|
-
/* @__PURE__ */ jsx6(Text6, { children: "3. Try again" })
|
|
462
|
-
] })
|
|
463
|
-
] }),
|
|
464
|
-
/* @__PURE__ */ jsx6(Box6, { marginTop: 1, children: /* @__PURE__ */ jsx6(Text6, { color: "gray", dimColor: true, children: "Press Enter or B to go back" }) })
|
|
465
|
-
] });
|
|
466
|
-
}
|
|
467
|
-
if (step === "model" && selectedProvider) {
|
|
468
|
-
const isOllama = selectedProvider.id === "ollama";
|
|
469
|
-
return /* @__PURE__ */ jsxs5(Box6, { flexDirection: "column", borderStyle: "round", borderColor: "cyan", padding: 1, width: 60, children: [
|
|
470
|
-
/* @__PURE__ */ jsx6(Box6, { marginBottom: 1, children: /* @__PURE__ */ jsx6(Text6, { bold: true, color: "cyan", children: "Select Model" }) }),
|
|
471
|
-
/* @__PURE__ */ jsxs5(Text6, { children: [
|
|
472
|
-
/* @__PURE__ */ jsx6(Text6, { color: "green", children: "\u2713" }),
|
|
473
|
-
" Provider: ",
|
|
474
|
-
selectedProvider.name
|
|
475
|
-
] }),
|
|
476
|
-
isOllama && checkingOllama && /* @__PURE__ */ jsxs5(Box6, { marginY: 1, children: [
|
|
477
|
-
/* @__PURE__ */ jsx6(Text6, { color: "cyan", children: /* @__PURE__ */ jsx6(Spinner, { type: "dots" }) }),
|
|
478
|
-
/* @__PURE__ */ jsx6(Text6, { children: " Checking Ollama status..." })
|
|
479
|
-
] }),
|
|
480
|
-
isOllama && ollamaStatus && ollamaStatus.running && /* @__PURE__ */ jsxs5(Text6, { color: "green", children: [
|
|
481
|
-
"\u2713 Ollama running (",
|
|
482
|
-
ollamaStatus.models.length,
|
|
483
|
-
" models installed)"
|
|
484
|
-
] }),
|
|
485
|
-
/* @__PURE__ */ jsx6(Box6, { marginTop: 1, marginBottom: 1, children: /* @__PURE__ */ jsx6(Text6, { color: "gray", dimColor: true, children: "Arrows to navigate, Enter to select, B to go back" }) }),
|
|
486
|
-
selectedProvider.models.map((model, index) => {
|
|
487
|
-
const isSelected = index === modelIndex;
|
|
488
|
-
const isCurrent = model.id === config.model && selectedProvider.id === config.provider;
|
|
489
|
-
let modelStatus = "";
|
|
490
|
-
if (isOllama && ollamaStatus) {
|
|
491
|
-
const available = hasModel(ollamaStatus, model.id);
|
|
492
|
-
modelStatus = available ? " (installed)" : " (not installed)";
|
|
493
|
-
}
|
|
494
|
-
return /* @__PURE__ */ jsxs5(Text6, { color: isSelected ? "cyan" : "white", children: [
|
|
495
|
-
isSelected ? "\u276F " : " ",
|
|
496
|
-
model.name,
|
|
497
|
-
model.recommended && /* @__PURE__ */ jsx6(Text6, { color: "yellow", children: " *" }),
|
|
498
|
-
isCurrent && /* @__PURE__ */ jsx6(Text6, { color: "green", children: " (current)" }),
|
|
499
|
-
isOllama && ollamaStatus && (hasModel(ollamaStatus, model.id) ? /* @__PURE__ */ jsx6(Text6, { color: "green", children: modelStatus }) : /* @__PURE__ */ jsx6(Text6, { color: "red", children: modelStatus }))
|
|
500
|
-
] }, model.id);
|
|
501
|
-
}),
|
|
502
|
-
isOllama && /* @__PURE__ */ jsxs5(Box6, { marginTop: 1, flexDirection: "column", children: [
|
|
503
|
-
/* @__PURE__ */ jsx6(Text6, { color: "gray", dimColor: true, children: "* = Recommended" }),
|
|
504
|
-
ollamaStatus && !hasModel(ollamaStatus, selectedProvider.models[modelIndex]?.id || "") && /* @__PURE__ */ jsxs5(Text6, { color: "yellow", children: [
|
|
505
|
-
"Run: ollama pull ",
|
|
506
|
-
selectedProvider.models[modelIndex]?.id
|
|
507
|
-
] })
|
|
508
|
-
] }),
|
|
509
|
-
/* @__PURE__ */ jsx6(Box6, { marginTop: 1, children: /* @__PURE__ */ jsx6(Text6, { color: "gray", dimColor: true, children: "Press Esc to cancel" }) })
|
|
510
|
-
] });
|
|
511
|
-
}
|
|
512
|
-
if (step === "done" && selectedProvider) {
|
|
513
|
-
return /* @__PURE__ */ jsxs5(Box6, { flexDirection: "column", borderStyle: "round", borderColor: "green", padding: 1, width: 60, children: [
|
|
514
|
-
/* @__PURE__ */ jsx6(Text6, { color: "green", bold: true, children: "Configuration Updated!" }),
|
|
515
|
-
/* @__PURE__ */ jsxs5(Text6, { children: [
|
|
516
|
-
/* @__PURE__ */ jsx6(Text6, { color: "green", children: "\u2713" }),
|
|
517
|
-
" Provider: ",
|
|
518
|
-
selectedProvider.name
|
|
519
|
-
] }),
|
|
520
|
-
/* @__PURE__ */ jsxs5(Text6, { children: [
|
|
521
|
-
/* @__PURE__ */ jsx6(Text6, { color: "green", children: "\u2713" }),
|
|
522
|
-
" Model: ",
|
|
523
|
-
selectedProvider.models[modelIndex]?.name
|
|
524
|
-
] }),
|
|
525
|
-
selectedProvider.id === "ollama" && ollamaStatus && !hasModel(ollamaStatus, selectedProvider.models[modelIndex]?.id || "") && /* @__PURE__ */ jsxs5(Text6, { color: "yellow", children: [
|
|
526
|
-
"Remember to run: ollama pull ",
|
|
527
|
-
selectedProvider.models[modelIndex]?.id
|
|
528
|
-
] })
|
|
529
|
-
] });
|
|
530
|
-
}
|
|
531
|
-
return null;
|
|
532
|
-
}
|
|
533
|
-
|
|
534
195
|
// src/hooks/useChat.ts
|
|
535
|
-
import { useState as
|
|
196
|
+
import { useState as useState2, useCallback, useRef, useEffect } from "react";
|
|
536
197
|
|
|
537
198
|
// src/lib/system.ts
|
|
538
199
|
import os from "os";
|
|
539
|
-
import { exec
|
|
540
|
-
import { promisify
|
|
541
|
-
var
|
|
200
|
+
import { exec } from "child_process";
|
|
201
|
+
import { promisify } from "util";
|
|
202
|
+
var execAsync = promisify(exec);
|
|
542
203
|
var cachedSystemInfo = null;
|
|
543
204
|
async function getSystemInfo() {
|
|
544
205
|
if (cachedSystemInfo) return cachedSystemInfo;
|
|
@@ -548,7 +209,7 @@ async function getSystemInfo() {
|
|
|
548
209
|
const osVersion = os.release();
|
|
549
210
|
if (platform === "win32") {
|
|
550
211
|
try {
|
|
551
|
-
const { stdout } = await
|
|
212
|
+
const { stdout } = await execAsync("wmic os get Caption /value", { timeout: 5e3 });
|
|
552
213
|
const match = stdout.match(/Caption=(.+)/);
|
|
553
214
|
if (match) osName = match[1].trim();
|
|
554
215
|
} catch {
|
|
@@ -556,7 +217,7 @@ async function getSystemInfo() {
|
|
|
556
217
|
}
|
|
557
218
|
} else if (platform === "darwin") {
|
|
558
219
|
try {
|
|
559
|
-
const { stdout } = await
|
|
220
|
+
const { stdout } = await execAsync("sw_vers -productName && sw_vers -productVersion", { timeout: 5e3 });
|
|
560
221
|
const lines = stdout.trim().split("\n");
|
|
561
222
|
osName = `${lines[0]} ${lines[1]}`;
|
|
562
223
|
} catch {
|
|
@@ -564,7 +225,7 @@ async function getSystemInfo() {
|
|
|
564
225
|
}
|
|
565
226
|
} else if (platform === "linux") {
|
|
566
227
|
try {
|
|
567
|
-
const { stdout } = await
|
|
228
|
+
const { stdout } = await execAsync("cat /etc/os-release | grep PRETTY_NAME", { timeout: 5e3 });
|
|
568
229
|
const match = stdout.match(/PRETTY_NAME="(.+)"/);
|
|
569
230
|
if (match) osName = match[1];
|
|
570
231
|
} catch {
|
|
@@ -922,17 +583,17 @@ async function captureScreenshot() {
|
|
|
922
583
|
}
|
|
923
584
|
}
|
|
924
585
|
async function captureScreenFallback() {
|
|
925
|
-
const { exec:
|
|
926
|
-
const { promisify:
|
|
586
|
+
const { exec: exec5 } = await import("child_process");
|
|
587
|
+
const { promisify: promisify5 } = await import("util");
|
|
927
588
|
const { tmpdir } = await import("os");
|
|
928
589
|
const { join: join3 } = await import("path");
|
|
929
590
|
const { readFile: readFile2, unlink } = await import("fs/promises");
|
|
930
|
-
const
|
|
591
|
+
const execAsync5 = promisify5(exec5);
|
|
931
592
|
const tempFile = join3(tmpdir(), `cnapse-screen-${Date.now()}.png`);
|
|
932
593
|
try {
|
|
933
594
|
const platform = process.platform;
|
|
934
595
|
if (platform === "win32") {
|
|
935
|
-
await
|
|
596
|
+
await execAsync5(`
|
|
936
597
|
Add-Type -AssemblyName System.Windows.Forms
|
|
937
598
|
$screen = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds
|
|
938
599
|
$bitmap = New-Object System.Drawing.Bitmap($screen.Width, $screen.Height)
|
|
@@ -943,9 +604,9 @@ async function captureScreenFallback() {
|
|
|
943
604
|
$bitmap.Dispose()
|
|
944
605
|
`, { shell: "powershell.exe" });
|
|
945
606
|
} else if (platform === "darwin") {
|
|
946
|
-
await
|
|
607
|
+
await execAsync5(`screencapture -x "${tempFile}"`);
|
|
947
608
|
} else {
|
|
948
|
-
await
|
|
609
|
+
await execAsync5(`gnome-screenshot -f "${tempFile}" 2>/dev/null || scrot "${tempFile}" 2>/dev/null || import -window root "${tempFile}"`);
|
|
949
610
|
}
|
|
950
611
|
const imageBuffer = await readFile2(tempFile);
|
|
951
612
|
await unlink(tempFile).catch(() => {
|
|
@@ -1114,11 +775,11 @@ var WELCOME_MESSAGE = {
|
|
|
1114
775
|
timestamp: /* @__PURE__ */ new Date()
|
|
1115
776
|
};
|
|
1116
777
|
function useChat(screenWatch = false) {
|
|
1117
|
-
const [messages, setMessages] =
|
|
1118
|
-
const [isProcessing, setIsProcessing] =
|
|
1119
|
-
const [error, setError] =
|
|
778
|
+
const [messages, setMessages] = useState2([WELCOME_MESSAGE]);
|
|
779
|
+
const [isProcessing, setIsProcessing] = useState2(false);
|
|
780
|
+
const [error, setError] = useState2(null);
|
|
1120
781
|
const screenWatchRef = useRef(screenWatch);
|
|
1121
|
-
|
|
782
|
+
useEffect(() => {
|
|
1122
783
|
screenWatchRef.current = screenWatch;
|
|
1123
784
|
}, [screenWatch]);
|
|
1124
785
|
const addSystemMessage = useCallback((content) => {
|
|
@@ -1197,12 +858,12 @@ function useChat(screenWatch = false) {
|
|
|
1197
858
|
}
|
|
1198
859
|
|
|
1199
860
|
// src/hooks/useVision.ts
|
|
1200
|
-
import { useState as
|
|
861
|
+
import { useState as useState3, useCallback as useCallback2 } from "react";
|
|
1201
862
|
function useVision() {
|
|
1202
|
-
const [isAnalyzing, setIsAnalyzing] =
|
|
1203
|
-
const [lastDescription, setLastDescription] =
|
|
1204
|
-
const [lastScreenshot, setLastScreenshot] =
|
|
1205
|
-
const [error, setError] =
|
|
863
|
+
const [isAnalyzing, setIsAnalyzing] = useState3(false);
|
|
864
|
+
const [lastDescription, setLastDescription] = useState3(null);
|
|
865
|
+
const [lastScreenshot, setLastScreenshot] = useState3(null);
|
|
866
|
+
const [error, setError] = useState3(null);
|
|
1206
867
|
const analyze = useCallback2(async () => {
|
|
1207
868
|
setIsAnalyzing(true);
|
|
1208
869
|
setError(null);
|
|
@@ -1229,14 +890,14 @@ function useVision() {
|
|
|
1229
890
|
}
|
|
1230
891
|
|
|
1231
892
|
// src/hooks/useTelegram.ts
|
|
1232
|
-
import { useState as
|
|
893
|
+
import { useState as useState4, useCallback as useCallback3, useEffect as useEffect2, useRef as useRef2 } from "react";
|
|
1233
894
|
|
|
1234
895
|
// src/services/telegram.ts
|
|
1235
896
|
import { EventEmitter } from "events";
|
|
1236
897
|
|
|
1237
898
|
// src/tools/shell.ts
|
|
1238
|
-
import { exec as
|
|
1239
|
-
import { promisify as
|
|
899
|
+
import { exec as exec4 } from "child_process";
|
|
900
|
+
import { promisify as promisify4 } from "util";
|
|
1240
901
|
|
|
1241
902
|
// src/tools/filesystem.ts
|
|
1242
903
|
import { promises as fs } from "fs";
|
|
@@ -1292,22 +953,22 @@ async function listDir(path2, recursive = false) {
|
|
|
1292
953
|
import clipboardy from "clipboardy";
|
|
1293
954
|
|
|
1294
955
|
// src/tools/process.ts
|
|
956
|
+
import { exec as exec2 } from "child_process";
|
|
957
|
+
import { promisify as promisify2 } from "util";
|
|
958
|
+
var execAsync2 = promisify2(exec2);
|
|
959
|
+
|
|
960
|
+
// src/tools/computer.ts
|
|
1295
961
|
import { exec as exec3 } from "child_process";
|
|
1296
962
|
import { promisify as promisify3 } from "util";
|
|
1297
963
|
var execAsync3 = promisify3(exec3);
|
|
1298
|
-
|
|
1299
|
-
// src/tools/computer.ts
|
|
1300
|
-
import { exec as exec4 } from "child_process";
|
|
1301
|
-
import { promisify as promisify4 } from "util";
|
|
1302
|
-
var execAsync4 = promisify4(exec4);
|
|
1303
964
|
async function moveMouse(x, y) {
|
|
1304
965
|
try {
|
|
1305
966
|
if (process.platform === "win32") {
|
|
1306
|
-
await
|
|
967
|
+
await execAsync3(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Cursor]::Position = New-Object System.Drawing.Point(${x}, ${y})"`, { shell: "cmd.exe" });
|
|
1307
968
|
} else if (process.platform === "darwin") {
|
|
1308
|
-
await
|
|
969
|
+
await execAsync3(`cliclick m:${x},${y}`);
|
|
1309
970
|
} else {
|
|
1310
|
-
await
|
|
971
|
+
await execAsync3(`xdotool mousemove ${x} ${y}`);
|
|
1311
972
|
}
|
|
1312
973
|
return ok(`Mouse moved to (${x}, ${y})`);
|
|
1313
974
|
} catch (error) {
|
|
@@ -1323,12 +984,12 @@ Add-Type -MemberDefinition @"
|
|
|
1323
984
|
public static extern void mouse_event(long dwFlags, long dx, long dy, long cButtons, long dwExtraInfo);
|
|
1324
985
|
"@ -Name Mouse -Namespace Win32
|
|
1325
986
|
${button === "left" ? "[Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x04, 0, 0, 0, 0)" : button === "right" ? "[Win32.Mouse]::mouse_event(0x08, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x10, 0, 0, 0, 0)" : "[Win32.Mouse]::mouse_event(0x20, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x40, 0, 0, 0, 0)"}`;
|
|
1326
|
-
await
|
|
987
|
+
await execAsync3(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1327
988
|
} else if (process.platform === "darwin") {
|
|
1328
|
-
await
|
|
989
|
+
await execAsync3(`cliclick c:.`);
|
|
1329
990
|
} else {
|
|
1330
991
|
const btn = button === "left" ? "1" : button === "right" ? "3" : "2";
|
|
1331
|
-
await
|
|
992
|
+
await execAsync3(`xdotool click ${btn}`);
|
|
1332
993
|
}
|
|
1333
994
|
return ok(`Clicked ${button} button`);
|
|
1334
995
|
} catch (error) {
|
|
@@ -1346,11 +1007,11 @@ public static extern void mouse_event(long dwFlags, long dx, long dy, long cButt
|
|
|
1346
1007
|
[Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x04, 0, 0, 0, 0)
|
|
1347
1008
|
Start-Sleep -Milliseconds 50
|
|
1348
1009
|
[Win32.Mouse]::mouse_event(0x02, 0, 0, 0, 0); [Win32.Mouse]::mouse_event(0x04, 0, 0, 0, 0)`;
|
|
1349
|
-
await
|
|
1010
|
+
await execAsync3(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1350
1011
|
} else if (process.platform === "darwin") {
|
|
1351
|
-
await
|
|
1012
|
+
await execAsync3(`cliclick dc:.`);
|
|
1352
1013
|
} else {
|
|
1353
|
-
await
|
|
1014
|
+
await execAsync3(`xdotool click --repeat 2 --delay 50 1`);
|
|
1354
1015
|
}
|
|
1355
1016
|
return ok("Double clicked");
|
|
1356
1017
|
} catch (error) {
|
|
@@ -1361,13 +1022,13 @@ async function typeText(text) {
|
|
|
1361
1022
|
try {
|
|
1362
1023
|
if (process.platform === "win32") {
|
|
1363
1024
|
const escapedText = text.replace(/'/g, "''").replace(/[+^%~(){}[\]]/g, "{$&}");
|
|
1364
|
-
await
|
|
1025
|
+
await execAsync3(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${escapedText}')"`, { shell: "cmd.exe" });
|
|
1365
1026
|
} else if (process.platform === "darwin") {
|
|
1366
1027
|
const escaped = text.replace(/'/g, "'\\''");
|
|
1367
|
-
await
|
|
1028
|
+
await execAsync3(`osascript -e 'tell application "System Events" to keystroke "${escaped}"'`);
|
|
1368
1029
|
} else {
|
|
1369
1030
|
const escaped = text.replace(/'/g, "'\\''");
|
|
1370
|
-
await
|
|
1031
|
+
await execAsync3(`xdotool type '${escaped}'`);
|
|
1371
1032
|
}
|
|
1372
1033
|
return ok(`Typed: ${text}`);
|
|
1373
1034
|
} catch (error) {
|
|
@@ -1408,7 +1069,7 @@ async function pressKey(key) {
|
|
|
1408
1069
|
"f12": "{F12}"
|
|
1409
1070
|
};
|
|
1410
1071
|
const winKey = winKeyMap[key.toLowerCase()] || key;
|
|
1411
|
-
await
|
|
1072
|
+
await execAsync3(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${winKey}')"`, { shell: "cmd.exe" });
|
|
1412
1073
|
} else if (process.platform === "darwin") {
|
|
1413
1074
|
const macKeyMap = {
|
|
1414
1075
|
"return": 36,
|
|
@@ -1426,12 +1087,12 @@ async function pressKey(key) {
|
|
|
1426
1087
|
};
|
|
1427
1088
|
const keyCode = macKeyMap[key.toLowerCase()];
|
|
1428
1089
|
if (keyCode) {
|
|
1429
|
-
await
|
|
1090
|
+
await execAsync3(`osascript -e 'tell application "System Events" to key code ${keyCode}'`);
|
|
1430
1091
|
} else {
|
|
1431
|
-
await
|
|
1092
|
+
await execAsync3(`osascript -e 'tell application "System Events" to keystroke "${key}"'`);
|
|
1432
1093
|
}
|
|
1433
1094
|
} else {
|
|
1434
|
-
await
|
|
1095
|
+
await execAsync3(`xdotool key ${key}`);
|
|
1435
1096
|
}
|
|
1436
1097
|
return ok(`Pressed: ${key}`);
|
|
1437
1098
|
} catch (error) {
|
|
@@ -1444,7 +1105,7 @@ async function keyCombo(keys) {
|
|
|
1444
1105
|
const hasWin = keys.some((k) => k.toLowerCase() === "meta" || k.toLowerCase() === "win");
|
|
1445
1106
|
const hasR = keys.some((k) => k.toLowerCase() === "r");
|
|
1446
1107
|
if (hasWin && hasR) {
|
|
1447
|
-
await
|
|
1108
|
+
await execAsync3(`powershell -Command "$shell = New-Object -ComObject WScript.Shell; $shell.Run('explorer shell:::{2559a1f3-21d7-11d4-bdaf-00c04f60b9f0}')"`, { shell: "cmd.exe" });
|
|
1448
1109
|
return ok(`Pressed: ${keys.join("+")}`);
|
|
1449
1110
|
}
|
|
1450
1111
|
const modifierMap = {
|
|
@@ -1464,7 +1125,7 @@ async function keyCombo(keys) {
|
|
|
1464
1125
|
}
|
|
1465
1126
|
}
|
|
1466
1127
|
combo += regularKeys.join("");
|
|
1467
|
-
await
|
|
1128
|
+
await execAsync3(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('${combo}')"`, { shell: "cmd.exe" });
|
|
1468
1129
|
} else if (process.platform === "darwin") {
|
|
1469
1130
|
const modifiers = keys.filter((k) => ["control", "ctrl", "alt", "shift", "command", "meta"].includes(k.toLowerCase()));
|
|
1470
1131
|
const regular = keys.filter((k) => !["control", "ctrl", "alt", "shift", "command", "meta"].includes(k.toLowerCase()));
|
|
@@ -1480,9 +1141,9 @@ async function keyCombo(keys) {
|
|
|
1480
1141
|
};
|
|
1481
1142
|
cmd += " using {" + modifiers.map((m) => modMap[m.toLowerCase()]).join(", ") + "}";
|
|
1482
1143
|
}
|
|
1483
|
-
await
|
|
1144
|
+
await execAsync3(`osascript -e '${cmd}'`);
|
|
1484
1145
|
} else {
|
|
1485
|
-
await
|
|
1146
|
+
await execAsync3(`xdotool key ${keys.join("+")}`);
|
|
1486
1147
|
}
|
|
1487
1148
|
return ok(`Pressed: ${keys.join("+")}`);
|
|
1488
1149
|
} catch (error) {
|
|
@@ -1508,13 +1169,13 @@ $hwnd = [Win32]::GetForegroundWindow()
|
|
|
1508
1169
|
$sb = New-Object System.Text.StringBuilder 256
|
|
1509
1170
|
[Win32]::GetWindowText($hwnd, $sb, 256)
|
|
1510
1171
|
$sb.ToString()`;
|
|
1511
|
-
const { stdout } = await
|
|
1172
|
+
const { stdout } = await execAsync3(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1512
1173
|
return ok(stdout.trim() || "Unknown window");
|
|
1513
1174
|
} else if (process.platform === "darwin") {
|
|
1514
|
-
const { stdout } = await
|
|
1175
|
+
const { stdout } = await execAsync3(`osascript -e 'tell application "System Events" to get name of first application process whose frontmost is true'`);
|
|
1515
1176
|
return ok(stdout.trim());
|
|
1516
1177
|
} else {
|
|
1517
|
-
const { stdout } = await
|
|
1178
|
+
const { stdout } = await execAsync3(`xdotool getactivewindow getwindowname`);
|
|
1518
1179
|
return ok(stdout.trim());
|
|
1519
1180
|
}
|
|
1520
1181
|
} catch (error) {
|
|
@@ -1524,13 +1185,13 @@ $sb.ToString()`;
|
|
|
1524
1185
|
async function listWindows() {
|
|
1525
1186
|
try {
|
|
1526
1187
|
if (process.platform === "win32") {
|
|
1527
|
-
const { stdout } = await
|
|
1188
|
+
const { stdout } = await execAsync3(`powershell -Command "Get-Process | Where-Object {$_.MainWindowTitle} | Select-Object ProcessName, MainWindowTitle | Format-Table -AutoSize"`, { shell: "cmd.exe" });
|
|
1528
1189
|
return ok(stdout);
|
|
1529
1190
|
} else if (process.platform === "darwin") {
|
|
1530
|
-
const { stdout } = await
|
|
1191
|
+
const { stdout } = await execAsync3(`osascript -e 'tell application "System Events" to get name of every application process whose visible is true'`);
|
|
1531
1192
|
return ok(stdout);
|
|
1532
1193
|
} else {
|
|
1533
|
-
const { stdout } = await
|
|
1194
|
+
const { stdout } = await execAsync3(`wmctrl -l`);
|
|
1534
1195
|
return ok(stdout);
|
|
1535
1196
|
}
|
|
1536
1197
|
} catch (error) {
|
|
@@ -1541,11 +1202,11 @@ async function focusWindow(title) {
|
|
|
1541
1202
|
try {
|
|
1542
1203
|
if (process.platform === "win32") {
|
|
1543
1204
|
const escaped = title.replace(/'/g, "''");
|
|
1544
|
-
await
|
|
1205
|
+
await execAsync3(`powershell -Command "$wshell = New-Object -ComObject wscript.shell; $wshell.AppActivate('${escaped}')"`, { shell: "cmd.exe" });
|
|
1545
1206
|
} else if (process.platform === "darwin") {
|
|
1546
|
-
await
|
|
1207
|
+
await execAsync3(`osascript -e 'tell application "${title}" to activate'`);
|
|
1547
1208
|
} else {
|
|
1548
|
-
await
|
|
1209
|
+
await execAsync3(`wmctrl -a "${title}"`);
|
|
1549
1210
|
}
|
|
1550
1211
|
return ok(`Focused window: ${title}`);
|
|
1551
1212
|
} catch (error) {
|
|
@@ -1573,27 +1234,27 @@ if ($proc) {
|
|
|
1573
1234
|
} else {
|
|
1574
1235
|
Write-Output "NOT_FOUND"
|
|
1575
1236
|
}`;
|
|
1576
|
-
const { stdout } = await
|
|
1237
|
+
const { stdout } = await execAsync3(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1577
1238
|
if (stdout.includes("NOT_FOUND")) {
|
|
1578
1239
|
return err(`Window containing "${title}" not found`);
|
|
1579
1240
|
}
|
|
1580
1241
|
return ok(stdout.trim());
|
|
1581
1242
|
} else {
|
|
1582
|
-
await
|
|
1243
|
+
await execAsync3(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('% n')"`, { shell: "cmd.exe" });
|
|
1583
1244
|
return ok("Minimized active window");
|
|
1584
1245
|
}
|
|
1585
1246
|
} else if (process.platform === "darwin") {
|
|
1586
1247
|
if (title) {
|
|
1587
|
-
await
|
|
1248
|
+
await execAsync3(`osascript -e 'tell application "${title}" to set miniaturized of window 1 to true'`);
|
|
1588
1249
|
} else {
|
|
1589
|
-
await
|
|
1250
|
+
await execAsync3(`osascript -e 'tell application "System Events" to keystroke "m" using command down'`);
|
|
1590
1251
|
}
|
|
1591
1252
|
return ok(`Minimized window${title ? `: ${title}` : ""}`);
|
|
1592
1253
|
} else {
|
|
1593
1254
|
if (title) {
|
|
1594
|
-
await
|
|
1255
|
+
await execAsync3(`wmctrl -r "${title}" -b add,hidden`);
|
|
1595
1256
|
} else {
|
|
1596
|
-
await
|
|
1257
|
+
await execAsync3(`xdotool getactivewindow windowminimize`);
|
|
1597
1258
|
}
|
|
1598
1259
|
return ok(`Minimized window${title ? `: ${title}` : ""}`);
|
|
1599
1260
|
}
|
|
@@ -1622,27 +1283,27 @@ if ($proc) {
|
|
|
1622
1283
|
} else {
|
|
1623
1284
|
Write-Output "NOT_FOUND"
|
|
1624
1285
|
}`;
|
|
1625
|
-
const { stdout } = await
|
|
1286
|
+
const { stdout } = await execAsync3(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1626
1287
|
if (stdout.includes("NOT_FOUND")) {
|
|
1627
1288
|
return err(`Window containing "${title}" not found`);
|
|
1628
1289
|
}
|
|
1629
1290
|
return ok(stdout.trim());
|
|
1630
1291
|
} else {
|
|
1631
|
-
await
|
|
1292
|
+
await execAsync3(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('% x')"`, { shell: "cmd.exe" });
|
|
1632
1293
|
return ok("Maximized active window");
|
|
1633
1294
|
}
|
|
1634
1295
|
} else if (process.platform === "darwin") {
|
|
1635
1296
|
if (title) {
|
|
1636
|
-
await
|
|
1297
|
+
await execAsync3(`osascript -e 'tell application "${title}" to set zoomed of window 1 to true'`);
|
|
1637
1298
|
} else {
|
|
1638
|
-
await
|
|
1299
|
+
await execAsync3(`osascript -e 'tell application "System Events" to keystroke "f" using {control down, command down}'`);
|
|
1639
1300
|
}
|
|
1640
1301
|
return ok(`Maximized window${title ? `: ${title}` : ""}`);
|
|
1641
1302
|
} else {
|
|
1642
1303
|
if (title) {
|
|
1643
|
-
await
|
|
1304
|
+
await execAsync3(`wmctrl -r "${title}" -b add,maximized_vert,maximized_horz`);
|
|
1644
1305
|
} else {
|
|
1645
|
-
await
|
|
1306
|
+
await execAsync3(`wmctrl -r :ACTIVE: -b add,maximized_vert,maximized_horz`);
|
|
1646
1307
|
}
|
|
1647
1308
|
return ok(`Maximized window${title ? `: ${title}` : ""}`);
|
|
1648
1309
|
}
|
|
@@ -1655,24 +1316,24 @@ async function closeWindow(title) {
|
|
|
1655
1316
|
if (process.platform === "win32") {
|
|
1656
1317
|
if (title) {
|
|
1657
1318
|
const escaped = title.replace(/'/g, "''");
|
|
1658
|
-
await
|
|
1319
|
+
await execAsync3(`powershell -Command "Get-Process | Where-Object { $_.MainWindowTitle -like '*${escaped}*' } | ForEach-Object { $_.CloseMainWindow() }"`, { shell: "cmd.exe" });
|
|
1659
1320
|
return ok(`Closed window: ${title}`);
|
|
1660
1321
|
} else {
|
|
1661
|
-
await
|
|
1322
|
+
await execAsync3(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait('%{F4}')"`, { shell: "cmd.exe" });
|
|
1662
1323
|
return ok("Closed active window");
|
|
1663
1324
|
}
|
|
1664
1325
|
} else if (process.platform === "darwin") {
|
|
1665
1326
|
if (title) {
|
|
1666
|
-
await
|
|
1327
|
+
await execAsync3(`osascript -e 'tell application "${title}" to close window 1'`);
|
|
1667
1328
|
} else {
|
|
1668
|
-
await
|
|
1329
|
+
await execAsync3(`osascript -e 'tell application "System Events" to keystroke "w" using command down'`);
|
|
1669
1330
|
}
|
|
1670
1331
|
return ok(`Closed window${title ? `: ${title}` : ""}`);
|
|
1671
1332
|
} else {
|
|
1672
1333
|
if (title) {
|
|
1673
|
-
await
|
|
1334
|
+
await execAsync3(`wmctrl -c "${title}"`);
|
|
1674
1335
|
} else {
|
|
1675
|
-
await
|
|
1336
|
+
await execAsync3(`xdotool getactivewindow windowclose`);
|
|
1676
1337
|
}
|
|
1677
1338
|
return ok(`Closed window${title ? `: ${title}` : ""}`);
|
|
1678
1339
|
}
|
|
@@ -1700,16 +1361,16 @@ if ($proc) {
|
|
|
1700
1361
|
} else {
|
|
1701
1362
|
Write-Output "NOT_FOUND"
|
|
1702
1363
|
}`;
|
|
1703
|
-
const { stdout } = await
|
|
1364
|
+
const { stdout } = await execAsync3(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1704
1365
|
if (stdout.includes("NOT_FOUND")) {
|
|
1705
1366
|
return err(`Window containing "${title}" not found`);
|
|
1706
1367
|
}
|
|
1707
1368
|
return ok(stdout.trim());
|
|
1708
1369
|
} else if (process.platform === "darwin") {
|
|
1709
|
-
await
|
|
1370
|
+
await execAsync3(`osascript -e 'tell application "${title}" to set miniaturized of window 1 to false'`);
|
|
1710
1371
|
return ok(`Restored window: ${title}`);
|
|
1711
1372
|
} else {
|
|
1712
|
-
await
|
|
1373
|
+
await execAsync3(`wmctrl -r "${title}" -b remove,hidden`);
|
|
1713
1374
|
return ok(`Restored window: ${title}`);
|
|
1714
1375
|
}
|
|
1715
1376
|
} catch (error) {
|
|
@@ -1726,13 +1387,13 @@ Add-Type -MemberDefinition @"
|
|
|
1726
1387
|
public static extern void mouse_event(long dwFlags, long dx, long dy, long cButtons, long dwExtraInfo);
|
|
1727
1388
|
"@ -Name Mouse -Namespace Win32
|
|
1728
1389
|
[Win32.Mouse]::mouse_event(0x0800, 0, 0, ${direction}, 0)`;
|
|
1729
|
-
await
|
|
1390
|
+
await execAsync3(`powershell -Command "${script.replace(/\n/g, " ")}"`, { shell: "cmd.exe" });
|
|
1730
1391
|
} else if (process.platform === "darwin") {
|
|
1731
1392
|
const dir = amount > 0 ? "u" : "d";
|
|
1732
|
-
await
|
|
1393
|
+
await execAsync3(`cliclick -r ${dir}:${Math.abs(amount)}`);
|
|
1733
1394
|
} else {
|
|
1734
1395
|
const btn = amount > 0 ? "4" : "5";
|
|
1735
|
-
await
|
|
1396
|
+
await execAsync3(`xdotool click --repeat ${Math.abs(amount)} ${btn}`);
|
|
1736
1397
|
}
|
|
1737
1398
|
return ok(`Scrolled ${amount > 0 ? "up" : "down"} by ${Math.abs(amount)}`);
|
|
1738
1399
|
} catch (error) {
|
|
@@ -1742,13 +1403,13 @@ public static extern void mouse_event(long dwFlags, long dx, long dy, long cButt
|
|
|
1742
1403
|
async function getMousePosition() {
|
|
1743
1404
|
try {
|
|
1744
1405
|
if (process.platform === "win32") {
|
|
1745
|
-
const { stdout } = await
|
|
1406
|
+
const { stdout } = await execAsync3(`powershell -Command "Add-Type -AssemblyName System.Windows.Forms; $p = [System.Windows.Forms.Cursor]::Position; Write-Output \\"$($p.X),$($p.Y)\\""`, { shell: "cmd.exe" });
|
|
1746
1407
|
return ok(`Mouse position: ${stdout.trim()}`);
|
|
1747
1408
|
} else if (process.platform === "darwin") {
|
|
1748
|
-
const { stdout } = await
|
|
1409
|
+
const { stdout } = await execAsync3(`cliclick p`);
|
|
1749
1410
|
return ok(`Mouse position: ${stdout.trim()}`);
|
|
1750
1411
|
} else {
|
|
1751
|
-
const { stdout } = await
|
|
1412
|
+
const { stdout } = await execAsync3(`xdotool getmouselocation --shell`);
|
|
1752
1413
|
return ok(stdout);
|
|
1753
1414
|
}
|
|
1754
1415
|
} catch (error) {
|
|
@@ -1765,13 +1426,13 @@ function err(error) {
|
|
|
1765
1426
|
}
|
|
1766
1427
|
|
|
1767
1428
|
// src/tools/shell.ts
|
|
1768
|
-
var
|
|
1429
|
+
var execAsync4 = promisify4(exec4);
|
|
1769
1430
|
async function runCommand(cmd, timeout = 3e4) {
|
|
1770
1431
|
try {
|
|
1771
1432
|
const isWindows = process.platform === "win32";
|
|
1772
1433
|
const shell = isWindows ? "cmd.exe" : "/bin/sh";
|
|
1773
1434
|
const shellArg = isWindows ? "/C" : "-c";
|
|
1774
|
-
const { stdout, stderr } = await
|
|
1435
|
+
const { stdout, stderr } = await execAsync4(cmd, {
|
|
1775
1436
|
shell,
|
|
1776
1437
|
timeout,
|
|
1777
1438
|
maxBuffer: 10 * 1024 * 1024
|
|
@@ -2194,12 +1855,12 @@ function getTelegramBot() {
|
|
|
2194
1855
|
|
|
2195
1856
|
// src/hooks/useTelegram.ts
|
|
2196
1857
|
function useTelegram(onMessage) {
|
|
2197
|
-
const [isEnabled, setIsEnabled] =
|
|
2198
|
-
const [isStarting, setIsStarting] =
|
|
2199
|
-
const [error, setError] =
|
|
2200
|
-
const [lastMessage, setLastMessage] =
|
|
1858
|
+
const [isEnabled, setIsEnabled] = useState4(false);
|
|
1859
|
+
const [isStarting, setIsStarting] = useState4(false);
|
|
1860
|
+
const [error, setError] = useState4(null);
|
|
1861
|
+
const [lastMessage, setLastMessage] = useState4(null);
|
|
2201
1862
|
const onMessageRef = useRef2(onMessage);
|
|
2202
|
-
|
|
1863
|
+
useEffect2(() => {
|
|
2203
1864
|
onMessageRef.current = onMessage;
|
|
2204
1865
|
}, [onMessage]);
|
|
2205
1866
|
const start = useCallback3(async () => {
|
|
@@ -2256,7 +1917,324 @@ function useTelegram(onMessage) {
|
|
|
2256
1917
|
}
|
|
2257
1918
|
|
|
2258
1919
|
// src/hooks/useTasks.ts
|
|
2259
|
-
import { useState as
|
|
1920
|
+
import { useState as useState5, useCallback as useCallback4 } from "react";
|
|
1921
|
+
|
|
1922
|
+
// src/services/browser.ts
|
|
1923
|
+
import { chromium } from "playwright";
|
|
1924
|
+
var browser = null;
|
|
1925
|
+
var context = null;
|
|
1926
|
+
var activePage = null;
|
|
1927
|
+
var defaultConfig = {
|
|
1928
|
+
headless: false,
|
|
1929
|
+
// Show browser so user can see what's happening
|
|
1930
|
+
slowMo: 50,
|
|
1931
|
+
// Slight delay for visibility
|
|
1932
|
+
viewport: { width: 1280, height: 800 }
|
|
1933
|
+
};
|
|
1934
|
+
async function initBrowser(config = {}) {
|
|
1935
|
+
const cfg = { ...defaultConfig, ...config };
|
|
1936
|
+
if (!browser) {
|
|
1937
|
+
browser = await chromium.launch({
|
|
1938
|
+
headless: cfg.headless,
|
|
1939
|
+
slowMo: cfg.slowMo
|
|
1940
|
+
});
|
|
1941
|
+
}
|
|
1942
|
+
if (!context) {
|
|
1943
|
+
context = await browser.newContext({
|
|
1944
|
+
viewport: cfg.viewport,
|
|
1945
|
+
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
|
1946
|
+
});
|
|
1947
|
+
}
|
|
1948
|
+
if (!activePage) {
|
|
1949
|
+
activePage = await context.newPage();
|
|
1950
|
+
}
|
|
1951
|
+
return activePage;
|
|
1952
|
+
}
|
|
1953
|
+
async function getPage() {
|
|
1954
|
+
if (!activePage) {
|
|
1955
|
+
return initBrowser();
|
|
1956
|
+
}
|
|
1957
|
+
return activePage;
|
|
1958
|
+
}
|
|
1959
|
+
async function navigateTo(url) {
|
|
1960
|
+
const page = await getPage();
|
|
1961
|
+
await page.goto(url, { waitUntil: "domcontentloaded" });
|
|
1962
|
+
}
|
|
1963
|
+
async function takeScreenshot() {
|
|
1964
|
+
const page = await getPage();
|
|
1965
|
+
const buffer = await page.screenshot({ type: "png" });
|
|
1966
|
+
return buffer.toString("base64");
|
|
1967
|
+
}
|
|
1968
|
+
async function clickElement(selector, timeout = 1e4) {
|
|
1969
|
+
const page = await getPage();
|
|
1970
|
+
try {
|
|
1971
|
+
await page.click(selector, { timeout });
|
|
1972
|
+
return true;
|
|
1973
|
+
} catch {
|
|
1974
|
+
return false;
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1977
|
+
async function typeInElement(selector, text, timeout = 1e4) {
|
|
1978
|
+
const page = await getPage();
|
|
1979
|
+
try {
|
|
1980
|
+
await page.fill(selector, text, { timeout });
|
|
1981
|
+
return true;
|
|
1982
|
+
} catch {
|
|
1983
|
+
return false;
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
async function pressKey2(key) {
|
|
1987
|
+
const page = await getPage();
|
|
1988
|
+
await page.keyboard.press(key);
|
|
1989
|
+
}
|
|
1990
|
+
async function scroll(direction, amount = 500) {
|
|
1991
|
+
const page = await getPage();
|
|
1992
|
+
await page.mouse.wheel(0, direction === "down" ? amount : -amount);
|
|
1993
|
+
}
|
|
1994
|
+
async function getPageText() {
|
|
1995
|
+
const page = await getPage();
|
|
1996
|
+
return await page.evaluate(() => document.body.innerText);
|
|
1997
|
+
}
|
|
1998
|
+
async function elementExists(selector) {
|
|
1999
|
+
const page = await getPage();
|
|
2000
|
+
try {
|
|
2001
|
+
const element = await page.$(selector);
|
|
2002
|
+
return element !== null;
|
|
2003
|
+
} catch {
|
|
2004
|
+
return false;
|
|
2005
|
+
}
|
|
2006
|
+
}
|
|
2007
|
+
var aiChatConfigs = {
|
|
2008
|
+
perplexity: {
|
|
2009
|
+
url: "https://www.perplexity.ai",
|
|
2010
|
+
inputSelector: 'textarea[placeholder*="Ask"]',
|
|
2011
|
+
submitKey: "Enter",
|
|
2012
|
+
responseSelector: '.prose, [class*="answer"], [class*="response"]',
|
|
2013
|
+
waitForResponse: 15e3
|
|
2014
|
+
},
|
|
2015
|
+
chatgpt: {
|
|
2016
|
+
url: "https://chat.openai.com",
|
|
2017
|
+
inputSelector: 'textarea[id="prompt-textarea"], textarea[data-id="root"]',
|
|
2018
|
+
submitSelector: 'button[data-testid="send-button"]',
|
|
2019
|
+
responseSelector: '[data-message-author-role="assistant"]',
|
|
2020
|
+
waitForResponse: 2e4
|
|
2021
|
+
},
|
|
2022
|
+
claude: {
|
|
2023
|
+
url: "https://claude.ai",
|
|
2024
|
+
inputSelector: '[contenteditable="true"], textarea',
|
|
2025
|
+
submitKey: "Enter",
|
|
2026
|
+
responseSelector: '[data-testid="message-content"]',
|
|
2027
|
+
waitForResponse: 2e4
|
|
2028
|
+
},
|
|
2029
|
+
copilot: {
|
|
2030
|
+
url: "https://copilot.microsoft.com",
|
|
2031
|
+
inputSelector: 'textarea, [contenteditable="true"]',
|
|
2032
|
+
submitKey: "Enter",
|
|
2033
|
+
responseSelector: '[class*="response"], [class*="message"]',
|
|
2034
|
+
waitForResponse: 15e3
|
|
2035
|
+
},
|
|
2036
|
+
google: {
|
|
2037
|
+
url: "https://www.google.com",
|
|
2038
|
+
inputSelector: 'textarea[name="q"], input[name="q"]',
|
|
2039
|
+
submitKey: "Enter",
|
|
2040
|
+
responseSelector: "#search",
|
|
2041
|
+
waitForResponse: 5e3
|
|
2042
|
+
}
|
|
2043
|
+
};
|
|
2044
|
+
async function askAI(site, question, includeScreenshot = false) {
|
|
2045
|
+
const config = aiChatConfigs[site];
|
|
2046
|
+
if (!config) {
|
|
2047
|
+
throw new Error(`Unknown AI site: ${site}`);
|
|
2048
|
+
}
|
|
2049
|
+
const page = await getPage();
|
|
2050
|
+
await page.goto(config.url, { waitUntil: "domcontentloaded" });
|
|
2051
|
+
await page.waitForTimeout(2e3);
|
|
2052
|
+
try {
|
|
2053
|
+
await page.waitForSelector(config.inputSelector, { timeout: 1e4 });
|
|
2054
|
+
await page.fill(config.inputSelector, question);
|
|
2055
|
+
} catch {
|
|
2056
|
+
await page.click(config.inputSelector);
|
|
2057
|
+
await page.type(config.inputSelector, question, { delay: 30 });
|
|
2058
|
+
}
|
|
2059
|
+
if (config.submitSelector) {
|
|
2060
|
+
await page.click(config.submitSelector);
|
|
2061
|
+
} else if (config.submitKey) {
|
|
2062
|
+
await page.keyboard.press(config.submitKey);
|
|
2063
|
+
}
|
|
2064
|
+
await page.waitForTimeout(config.waitForResponse);
|
|
2065
|
+
let response = "";
|
|
2066
|
+
try {
|
|
2067
|
+
const elements = await page.$$(config.responseSelector);
|
|
2068
|
+
if (elements.length > 0) {
|
|
2069
|
+
const lastElement = elements[elements.length - 1];
|
|
2070
|
+
response = await lastElement.textContent() || "";
|
|
2071
|
+
}
|
|
2072
|
+
} catch {
|
|
2073
|
+
response = await getPageText();
|
|
2074
|
+
}
|
|
2075
|
+
let screenshot;
|
|
2076
|
+
if (includeScreenshot) {
|
|
2077
|
+
screenshot = await takeScreenshot();
|
|
2078
|
+
}
|
|
2079
|
+
return { response: response.trim(), screenshot };
|
|
2080
|
+
}
|
|
2081
|
+
async function getFullAIResponse(site, maxScrolls = 5) {
|
|
2082
|
+
const config = aiChatConfigs[site];
|
|
2083
|
+
const page = await getPage();
|
|
2084
|
+
const responseParts = [];
|
|
2085
|
+
for (let i = 0; i < maxScrolls; i++) {
|
|
2086
|
+
try {
|
|
2087
|
+
const elements = await page.$$(config.responseSelector);
|
|
2088
|
+
if (elements.length > 0) {
|
|
2089
|
+
const lastElement = elements[elements.length - 1];
|
|
2090
|
+
const text = await lastElement.textContent();
|
|
2091
|
+
if (text) {
|
|
2092
|
+
responseParts.push(text.trim());
|
|
2093
|
+
}
|
|
2094
|
+
}
|
|
2095
|
+
await page.mouse.wheel(0, 500);
|
|
2096
|
+
await page.waitForTimeout(1e3);
|
|
2097
|
+
const atBottom = await page.evaluate(() => {
|
|
2098
|
+
return window.innerHeight + window.scrollY >= document.body.scrollHeight - 100;
|
|
2099
|
+
});
|
|
2100
|
+
if (atBottom) break;
|
|
2101
|
+
} catch {
|
|
2102
|
+
break;
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
return responseParts;
|
|
2106
|
+
}
|
|
2107
|
+
async function sendGmail(email) {
|
|
2108
|
+
const page = await getPage();
|
|
2109
|
+
try {
|
|
2110
|
+
await page.goto("https://mail.google.com/mail/u/0/#inbox?compose=new");
|
|
2111
|
+
await page.waitForTimeout(3e3);
|
|
2112
|
+
await page.waitForSelector('input[aria-label*="To"]', { timeout: 1e4 });
|
|
2113
|
+
await page.fill('input[aria-label*="To"]', email.to);
|
|
2114
|
+
await page.keyboard.press("Tab");
|
|
2115
|
+
await page.fill('input[name="subjectbox"]', email.subject);
|
|
2116
|
+
await page.keyboard.press("Tab");
|
|
2117
|
+
await page.fill('[aria-label*="Message Body"], [role="textbox"]', email.body);
|
|
2118
|
+
await page.keyboard.press("Control+Enter");
|
|
2119
|
+
await page.waitForTimeout(2e3);
|
|
2120
|
+
return true;
|
|
2121
|
+
} catch {
|
|
2122
|
+
return false;
|
|
2123
|
+
}
|
|
2124
|
+
}
|
|
2125
|
+
async function sendOutlook(email) {
|
|
2126
|
+
const page = await getPage();
|
|
2127
|
+
try {
|
|
2128
|
+
await page.goto("https://outlook.office.com/mail/0/inbox");
|
|
2129
|
+
await page.waitForTimeout(3e3);
|
|
2130
|
+
await page.click('button[aria-label*="New mail"], button[title*="New mail"]');
|
|
2131
|
+
await page.waitForTimeout(2e3);
|
|
2132
|
+
await page.fill('input[aria-label*="To"]', email.to);
|
|
2133
|
+
await page.keyboard.press("Tab");
|
|
2134
|
+
await page.fill('input[aria-label*="Subject"], input[placeholder*="Subject"]', email.subject);
|
|
2135
|
+
await page.keyboard.press("Tab");
|
|
2136
|
+
await page.fill('[aria-label*="Message body"], [role="textbox"]', email.body);
|
|
2137
|
+
await page.click('button[aria-label*="Send"], button[title*="Send"]');
|
|
2138
|
+
await page.waitForTimeout(2e3);
|
|
2139
|
+
return true;
|
|
2140
|
+
} catch {
|
|
2141
|
+
return false;
|
|
2142
|
+
}
|
|
2143
|
+
}
|
|
2144
|
+
async function googleSheetsType(cellData) {
|
|
2145
|
+
const page = await getPage();
|
|
2146
|
+
try {
|
|
2147
|
+
await page.goto("https://docs.google.com/spreadsheets/create");
|
|
2148
|
+
await page.waitForTimeout(5e3);
|
|
2149
|
+
for (const { cell, value } of cellData) {
|
|
2150
|
+
await page.click("input#t-name-box");
|
|
2151
|
+
await page.fill("input#t-name-box", cell);
|
|
2152
|
+
await page.keyboard.press("Enter");
|
|
2153
|
+
await page.waitForTimeout(500);
|
|
2154
|
+
await page.keyboard.type(value);
|
|
2155
|
+
await page.keyboard.press("Enter");
|
|
2156
|
+
await page.waitForTimeout(300);
|
|
2157
|
+
}
|
|
2158
|
+
return true;
|
|
2159
|
+
} catch {
|
|
2160
|
+
return false;
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
async function googleDocsType(text) {
|
|
2164
|
+
const page = await getPage();
|
|
2165
|
+
try {
|
|
2166
|
+
await page.goto("https://docs.google.com/document/create");
|
|
2167
|
+
await page.waitForTimeout(5e3);
|
|
2168
|
+
await page.click(".kix-appview-editor");
|
|
2169
|
+
await page.waitForTimeout(500);
|
|
2170
|
+
await page.keyboard.type(text, { delay: 20 });
|
|
2171
|
+
return true;
|
|
2172
|
+
} catch {
|
|
2173
|
+
return false;
|
|
2174
|
+
}
|
|
2175
|
+
}
|
|
2176
|
+
async function webSearch(query, engine = "google") {
|
|
2177
|
+
const page = await getPage();
|
|
2178
|
+
const results = [];
|
|
2179
|
+
const urls = {
|
|
2180
|
+
google: "https://www.google.com",
|
|
2181
|
+
bing: "https://www.bing.com",
|
|
2182
|
+
duckduckgo: "https://duckduckgo.com"
|
|
2183
|
+
};
|
|
2184
|
+
const selectors = {
|
|
2185
|
+
google: { input: 'textarea[name="q"]', results: "#search .g h3" },
|
|
2186
|
+
bing: { input: 'input[name="q"]', results: "#b_results h2 a" },
|
|
2187
|
+
duckduckgo: { input: 'input[name="q"]', results: "[data-result] h2" }
|
|
2188
|
+
};
|
|
2189
|
+
try {
|
|
2190
|
+
await page.goto(urls[engine]);
|
|
2191
|
+
await page.waitForTimeout(2e3);
|
|
2192
|
+
await page.fill(selectors[engine].input, query);
|
|
2193
|
+
await page.keyboard.press("Enter");
|
|
2194
|
+
await page.waitForTimeout(3e3);
|
|
2195
|
+
const elements = await page.$$(selectors[engine].results);
|
|
2196
|
+
for (const el of elements.slice(0, 10)) {
|
|
2197
|
+
const text = await el.textContent();
|
|
2198
|
+
if (text) results.push(text);
|
|
2199
|
+
}
|
|
2200
|
+
} catch {
|
|
2201
|
+
}
|
|
2202
|
+
return results;
|
|
2203
|
+
}
|
|
2204
|
+
async function research(topic, maxSources = 3) {
|
|
2205
|
+
const page = await getPage();
|
|
2206
|
+
const sources = [];
|
|
2207
|
+
await webSearch(topic);
|
|
2208
|
+
await page.waitForTimeout(2e3);
|
|
2209
|
+
for (let i = 0; i < maxSources; i++) {
|
|
2210
|
+
try {
|
|
2211
|
+
const results = await page.$$("#search .g");
|
|
2212
|
+
if (results[i]) {
|
|
2213
|
+
const titleEl = await results[i].$("h3");
|
|
2214
|
+
const linkEl = await results[i].$("a");
|
|
2215
|
+
const title = await titleEl?.textContent() || "Unknown";
|
|
2216
|
+
const url = await linkEl?.getAttribute("href") || "";
|
|
2217
|
+
await titleEl?.click();
|
|
2218
|
+
await page.waitForTimeout(3e3);
|
|
2219
|
+
const content = await page.evaluate(() => {
|
|
2220
|
+
const article = document.querySelector("article, main, .content, #content");
|
|
2221
|
+
return article?.textContent?.slice(0, 2e3) || document.body.innerText.slice(0, 2e3);
|
|
2222
|
+
});
|
|
2223
|
+
sources.push({ title, url, content: content.trim() });
|
|
2224
|
+
await page.goBack();
|
|
2225
|
+
await page.waitForTimeout(1500);
|
|
2226
|
+
}
|
|
2227
|
+
} catch {
|
|
2228
|
+
continue;
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
return {
|
|
2232
|
+
query: topic,
|
|
2233
|
+
sources,
|
|
2234
|
+
summary: ""
|
|
2235
|
+
// To be filled by AI
|
|
2236
|
+
};
|
|
2237
|
+
}
|
|
2260
2238
|
|
|
2261
2239
|
// src/lib/tasks.ts
|
|
2262
2240
|
import * as fs2 from "fs";
|
|
@@ -2396,6 +2374,12 @@ Before outputting steps, THINK through these questions:
|
|
|
2396
2374
|
### Research
|
|
2397
2375
|
- research: Multi-step web research - searches, gathers info, summarizes (e.g., "research:What are the latest AI trends in 2024?")
|
|
2398
2376
|
|
|
2377
|
+
### Adaptive/Learning
|
|
2378
|
+
- ask_llm: Ask another LLM for help with a screenshot (e.g., "ask_llm:perplexity|How do I do X in this app?")
|
|
2379
|
+
- ask_llm: Supports: perplexity, chatgpt, claude, copilot - sends screenshot + question, gets answer
|
|
2380
|
+
- adaptive_do: Try to accomplish something, if stuck ask LLMs for help (e.g., "adaptive_do:book a flight to NYC on kayak.com")
|
|
2381
|
+
- learn_ui: Take screenshot and learn how to interact with current UI (e.g., "learn_ui:What buttons can I click here?")
|
|
2382
|
+
|
|
2399
2383
|
### Utility
|
|
2400
2384
|
- wait: Wait N seconds (e.g., "wait:2" - use 1-3s for app loads)
|
|
2401
2385
|
- screenshot: Capture and describe screen
|
|
@@ -2552,6 +2536,39 @@ Next Steps:
|
|
|
2552
2536
|
- Deploy to production" }
|
|
2553
2537
|
]
|
|
2554
2538
|
|
|
2539
|
+
### Example 12: "I don't know how to use this app, can you figure it out?"
|
|
2540
|
+
Thinking:
|
|
2541
|
+
- Goal: Learn the current UI and understand how to use it
|
|
2542
|
+
- How: Use learn_ui to take screenshot and analyze
|
|
2543
|
+
- Sequence: Screenshot -> AI analysis -> report back
|
|
2544
|
+
|
|
2545
|
+
Output:
|
|
2546
|
+
[
|
|
2547
|
+
{ "description": "Analyze current UI", "action": "learn_ui:What are all the buttons, menus, and interactive elements I can use?" }
|
|
2548
|
+
]
|
|
2549
|
+
|
|
2550
|
+
### Example 13: "book a hotel on booking.com for next weekend"
|
|
2551
|
+
Thinking:
|
|
2552
|
+
- Goal: Complex task on unfamiliar website - need adaptive approach
|
|
2553
|
+
- How: Use adaptive_do which will try, and if stuck ask LLMs for help
|
|
2554
|
+
- Sequence: Single adaptive action handles the complexity
|
|
2555
|
+
|
|
2556
|
+
Output:
|
|
2557
|
+
[
|
|
2558
|
+
{ "description": "Adaptively book hotel", "action": "adaptive_do:Go to booking.com and book a hotel for next weekend" }
|
|
2559
|
+
]
|
|
2560
|
+
|
|
2561
|
+
### Example 14: "I'm stuck, ask Claude how to proceed"
|
|
2562
|
+
Thinking:
|
|
2563
|
+
- Goal: Get help from another LLM with current screen context
|
|
2564
|
+
- How: Use ask_llm with claude and send screenshot
|
|
2565
|
+
- Sequence: Screenshot + question -> Get answer
|
|
2566
|
+
|
|
2567
|
+
Output:
|
|
2568
|
+
[
|
|
2569
|
+
{ "description": "Ask Claude for help with screenshot", "action": "ask_llm:claude|I'm stuck on this screen. What should I do next to accomplish my task?" }
|
|
2570
|
+
]
|
|
2571
|
+
|
|
2555
2572
|
## YOUR TASK
|
|
2556
2573
|
Now parse this request: "${input}"
|
|
2557
2574
|
|
|
@@ -2741,66 +2758,40 @@ ${existingResult.output}`;
|
|
|
2741
2758
|
case "browse_and_ask": {
|
|
2742
2759
|
const [site, ...questionParts] = params.split("|");
|
|
2743
2760
|
const question = questionParts.join("|");
|
|
2744
|
-
const
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
|
|
2748
|
-
|
|
2749
|
-
|
|
2750
|
-
|
|
2751
|
-
|
|
2752
|
-
|
|
2753
|
-
|
|
2754
|
-
|
|
2755
|
-
|
|
2756
|
-
} else if (process.platform === "darwin") {
|
|
2757
|
-
await runCommand(`open "${siteConfig.url}"`, 5e3);
|
|
2758
|
-
} else {
|
|
2759
|
-
await runCommand(`xdg-open "${siteConfig.url}"`, 5e3);
|
|
2760
|
-
}
|
|
2761
|
-
await sleep(siteConfig.loadTime * 1e3);
|
|
2762
|
-
await typeText(question);
|
|
2763
|
-
await sleep(300);
|
|
2764
|
-
await pressKey("Return");
|
|
2765
|
-
await sleep(siteConfig.responseTime * 1e3);
|
|
2766
|
-
const extractedParts = [];
|
|
2767
|
-
const maxScrolls = 5;
|
|
2768
|
-
for (let scrollIndex = 0; scrollIndex < maxScrolls; scrollIndex++) {
|
|
2769
|
-
const screenResult = await describeScreen();
|
|
2770
|
-
const extractPrompt = `You are looking at screenshot ${scrollIndex + 1} of ${site}. The user asked: "${question}"
|
|
2771
|
-
|
|
2772
|
-
Extract ONLY the AI's response/answer text visible on screen. Do NOT include:
|
|
2773
|
-
- The user's question
|
|
2774
|
-
- Any UI elements, buttons, navigation, or headers
|
|
2775
|
-
- Any disclaimers, suggestions, or "related questions"
|
|
2776
|
-
- Any "Sources" or citation links
|
|
2777
|
-
- Any text you already extracted (avoid duplicates)
|
|
2778
|
-
|
|
2779
|
-
${scrollIndex > 0 ? `Previous parts already extracted:
|
|
2780
|
-
${extractedParts.join("\n---\n")}
|
|
2781
|
-
|
|
2782
|
-
Only extract NEW text that continues from where we left off.` : ""}
|
|
2783
|
-
|
|
2784
|
-
Just give me the actual answer text, word for word as it appears. If there's no more response text visible, respond with exactly: "END_OF_RESPONSE"`;
|
|
2785
|
-
const extractResponse = await chat([{ role: "user", content: extractPrompt }]);
|
|
2786
|
-
const extracted = extractResponse.content.trim();
|
|
2787
|
-
if (extracted === "END_OF_RESPONSE" || extracted.includes("END_OF_RESPONSE")) {
|
|
2788
|
-
break;
|
|
2761
|
+
const supportedSites = ["perplexity", "chatgpt", "claude", "copilot", "google"];
|
|
2762
|
+
const siteLower = site.toLowerCase();
|
|
2763
|
+
if (supportedSites.includes(siteLower)) {
|
|
2764
|
+
const result = await askAI(siteLower, question, true);
|
|
2765
|
+
if (result.response.length < 500) {
|
|
2766
|
+
const fullParts = await getFullAIResponse(siteLower, 5);
|
|
2767
|
+
if (fullParts.length > 0) {
|
|
2768
|
+
step.result = `\u{1F4DD} ${site.charAt(0).toUpperCase() + site.slice(1)} says:
|
|
2769
|
+
|
|
2770
|
+
${fullParts.join("\n\n")}`;
|
|
2771
|
+
break;
|
|
2772
|
+
}
|
|
2789
2773
|
}
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2774
|
+
step.result = `\u{1F4DD} ${site.charAt(0).toUpperCase() + site.slice(1)} says:
|
|
2775
|
+
|
|
2776
|
+
${result.response}`;
|
|
2777
|
+
} else {
|
|
2778
|
+
await navigateTo(`https://${site}`);
|
|
2779
|
+
await sleep(2e3);
|
|
2780
|
+
const page = await getPage();
|
|
2781
|
+
const inputs = ["textarea", 'input[type="text"]', 'input[type="search"]', '[contenteditable="true"]'];
|
|
2782
|
+
for (const selector of inputs) {
|
|
2783
|
+
if (await elementExists(selector)) {
|
|
2784
|
+
await typeInElement(selector, question);
|
|
2785
|
+
await pressKey2("Enter");
|
|
2786
|
+
break;
|
|
2793
2787
|
}
|
|
2794
|
-
break;
|
|
2795
2788
|
}
|
|
2796
|
-
|
|
2797
|
-
await
|
|
2798
|
-
|
|
2799
|
-
}
|
|
2800
|
-
const fullResponse = extractedParts.join("\n\n");
|
|
2801
|
-
step.result = `\u{1F4DD} ${site.charAt(0).toUpperCase() + site.slice(1)} says:
|
|
2789
|
+
await sleep(5e3);
|
|
2790
|
+
const pageText = await getPageText();
|
|
2791
|
+
step.result = `\u{1F4DD} Response from ${site}:
|
|
2802
2792
|
|
|
2803
|
-
${
|
|
2793
|
+
${pageText.slice(0, 3e3)}`;
|
|
2794
|
+
}
|
|
2804
2795
|
break;
|
|
2805
2796
|
}
|
|
2806
2797
|
case "screenshot":
|
|
@@ -2808,84 +2799,36 @@ ${fullResponse}`;
|
|
|
2808
2799
|
step.result = vision.description;
|
|
2809
2800
|
break;
|
|
2810
2801
|
case "web_search": {
|
|
2811
|
-
await
|
|
2812
|
-
|
|
2813
|
-
|
|
2814
|
-
await pressKey("Return");
|
|
2815
|
-
await sleep(2e3);
|
|
2816
|
-
await keyCombo(["control", "l"]);
|
|
2817
|
-
await sleep(300);
|
|
2818
|
-
await typeText("google.com");
|
|
2819
|
-
await pressKey("Return");
|
|
2820
|
-
await sleep(2e3);
|
|
2821
|
-
await typeText(params);
|
|
2822
|
-
await sleep(300);
|
|
2823
|
-
await pressKey("Return");
|
|
2824
|
-
await sleep(3e3);
|
|
2825
|
-
const searchScreen = await describeScreen();
|
|
2826
|
-
const searchExtract = await chat([{
|
|
2827
|
-
role: "user",
|
|
2828
|
-
content: `Extract the top search results from this Google search page. For each result, include:
|
|
2829
|
-
- Title
|
|
2830
|
-
- Brief snippet/description
|
|
2831
|
-
- URL if visible
|
|
2802
|
+
const searchResults = await webSearch(params, "google");
|
|
2803
|
+
if (searchResults.length > 0) {
|
|
2804
|
+
step.result = `\u{1F50D} Search results for "${params}":
|
|
2832
2805
|
|
|
2833
|
-
|
|
2834
|
-
}
|
|
2835
|
-
|
|
2806
|
+
${searchResults.map((r, i) => `${i + 1}. ${r}`).join("\n")}`;
|
|
2807
|
+
} else {
|
|
2808
|
+
const pageText = await getPageText();
|
|
2809
|
+
step.result = `\u{1F50D} Search results for "${params}":
|
|
2836
2810
|
|
|
2837
|
-
${
|
|
2811
|
+
${pageText.slice(0, 2e3)}`;
|
|
2812
|
+
}
|
|
2838
2813
|
break;
|
|
2839
2814
|
}
|
|
2840
2815
|
case "send_email": {
|
|
2841
2816
|
const [provider, to, subject, ...bodyParts] = params.split("|");
|
|
2842
2817
|
const body = bodyParts.join("|");
|
|
2843
|
-
|
|
2844
|
-
|
|
2845
|
-
await typeText("chrome");
|
|
2846
|
-
await pressKey("Return");
|
|
2847
|
-
await sleep(2e3);
|
|
2848
|
-
await keyCombo(["control", "l"]);
|
|
2849
|
-
await sleep(300);
|
|
2818
|
+
const emailData = { to, subject, body };
|
|
2819
|
+
let success = false;
|
|
2850
2820
|
if (provider.toLowerCase() === "gmail") {
|
|
2851
|
-
await
|
|
2852
|
-
await pressKey("Return");
|
|
2853
|
-
await sleep(4e3);
|
|
2854
|
-
await typeText("c");
|
|
2855
|
-
await sleep(2e3);
|
|
2856
|
-
await typeText(to);
|
|
2857
|
-
await sleep(300);
|
|
2858
|
-
await pressKey("Tab");
|
|
2859
|
-
await sleep(200);
|
|
2860
|
-
await typeText(subject);
|
|
2861
|
-
await sleep(300);
|
|
2862
|
-
await pressKey("Tab");
|
|
2863
|
-
await sleep(200);
|
|
2864
|
-
await typeText(body);
|
|
2865
|
-
await sleep(500);
|
|
2866
|
-
await keyCombo(["control", "Return"]);
|
|
2821
|
+
success = await sendGmail(emailData);
|
|
2867
2822
|
} else if (provider.toLowerCase() === "outlook") {
|
|
2868
|
-
await
|
|
2869
|
-
await pressKey("Return");
|
|
2870
|
-
await sleep(4e3);
|
|
2871
|
-
await typeText("n");
|
|
2872
|
-
await sleep(2e3);
|
|
2873
|
-
await typeText(to);
|
|
2874
|
-
await sleep(300);
|
|
2875
|
-
await pressKey("Tab");
|
|
2876
|
-
await sleep(200);
|
|
2877
|
-
await typeText(subject);
|
|
2878
|
-
await sleep(300);
|
|
2879
|
-
await pressKey("Tab");
|
|
2880
|
-
await sleep(200);
|
|
2881
|
-
await typeText(body);
|
|
2882
|
-
await sleep(500);
|
|
2883
|
-
await keyCombo(["control", "Return"]);
|
|
2823
|
+
success = await sendOutlook(emailData);
|
|
2884
2824
|
} else {
|
|
2885
2825
|
throw new Error(`Unsupported email provider: ${provider}. Use gmail or outlook.`);
|
|
2886
2826
|
}
|
|
2887
|
-
|
|
2888
|
-
|
|
2827
|
+
if (success) {
|
|
2828
|
+
step.result = `\u{1F4E7} Email sent via ${provider} to ${to}`;
|
|
2829
|
+
} else {
|
|
2830
|
+
throw new Error(`Failed to send email via ${provider}. Make sure you're logged in.`);
|
|
2831
|
+
}
|
|
2889
2832
|
break;
|
|
2890
2833
|
}
|
|
2891
2834
|
case "google_sheets": {
|
|
@@ -2893,50 +2836,26 @@ ${searchExtract.content}`;
|
|
|
2893
2836
|
switch (sheetCmd.toLowerCase()) {
|
|
2894
2837
|
case "new": {
|
|
2895
2838
|
const sheetName = sheetArgs[0] || "Untitled spreadsheet";
|
|
2896
|
-
await
|
|
2897
|
-
await sleep(
|
|
2898
|
-
await typeText("chrome");
|
|
2899
|
-
await pressKey("Return");
|
|
2900
|
-
await sleep(2e3);
|
|
2901
|
-
await keyCombo(["control", "l"]);
|
|
2902
|
-
await sleep(300);
|
|
2903
|
-
await typeText("sheets.google.com");
|
|
2904
|
-
await pressKey("Return");
|
|
2905
|
-
await sleep(3e3);
|
|
2906
|
-
await pressKey("Tab");
|
|
2907
|
-
await pressKey("Tab");
|
|
2908
|
-
await pressKey("Return");
|
|
2909
|
-
await sleep(3e3);
|
|
2910
|
-
await keyCombo(["alt", "f"]);
|
|
2911
|
-
await sleep(500);
|
|
2912
|
-
await typeText("r");
|
|
2913
|
-
await sleep(500);
|
|
2914
|
-
await keyCombo(["control", "a"]);
|
|
2915
|
-
await typeText(sheetName);
|
|
2916
|
-
await pressKey("Return");
|
|
2917
|
-
await sleep(500);
|
|
2918
|
-
await pressKey("Escape");
|
|
2839
|
+
await navigateTo("https://docs.google.com/spreadsheets/create");
|
|
2840
|
+
await sleep(5e3);
|
|
2919
2841
|
step.result = `\u{1F4CA} Created Google Sheet: ${sheetName}`;
|
|
2920
2842
|
break;
|
|
2921
2843
|
}
|
|
2922
2844
|
case "type": {
|
|
2923
2845
|
const cell = sheetArgs[0] || "A1";
|
|
2924
2846
|
const cellValue = sheetArgs.slice(1).join("|");
|
|
2925
|
-
await
|
|
2926
|
-
|
|
2927
|
-
await typeText(cell);
|
|
2928
|
-
await pressKey("Return");
|
|
2929
|
-
await sleep(300);
|
|
2930
|
-
await typeText(cellValue);
|
|
2931
|
-
await pressKey("Return");
|
|
2932
|
-
await sleep(200);
|
|
2933
|
-
step.result = `\u{1F4CA} Typed "${cellValue}" in cell ${cell}`;
|
|
2847
|
+
const success = await googleSheetsType([{ cell, value: cellValue }]);
|
|
2848
|
+
step.result = success ? `\u{1F4CA} Typed "${cellValue}" in cell ${cell}` : `\u{1F4CA} Could not type in cell ${cell}`;
|
|
2934
2849
|
break;
|
|
2935
2850
|
}
|
|
2936
2851
|
case "read": {
|
|
2937
|
-
const
|
|
2852
|
+
const screenshot = await takeScreenshot();
|
|
2853
|
+
const analysis = await chat([{
|
|
2854
|
+
role: "user",
|
|
2855
|
+
content: "Describe the contents of this Google Sheet. List visible data in the cells."
|
|
2856
|
+
}]);
|
|
2938
2857
|
step.result = `\u{1F4CA} Current sheet view:
|
|
2939
|
-
${
|
|
2858
|
+
${analysis.content}`;
|
|
2940
2859
|
break;
|
|
2941
2860
|
}
|
|
2942
2861
|
default:
|
|
@@ -2949,36 +2868,14 @@ ${readScreen.description}`;
|
|
|
2949
2868
|
switch (docCmd.toLowerCase()) {
|
|
2950
2869
|
case "new": {
|
|
2951
2870
|
const docName = docArgs[0] || "Untitled document";
|
|
2952
|
-
await
|
|
2953
|
-
|
|
2954
|
-
await typeText("chrome");
|
|
2955
|
-
await pressKey("Return");
|
|
2956
|
-
await sleep(2e3);
|
|
2957
|
-
await keyCombo(["control", "l"]);
|
|
2958
|
-
await sleep(300);
|
|
2959
|
-
await typeText("docs.google.com");
|
|
2960
|
-
await pressKey("Return");
|
|
2961
|
-
await sleep(3e3);
|
|
2962
|
-
await pressKey("Tab");
|
|
2963
|
-
await pressKey("Tab");
|
|
2964
|
-
await pressKey("Return");
|
|
2965
|
-
await sleep(3e3);
|
|
2966
|
-
await keyCombo(["alt", "f"]);
|
|
2967
|
-
await sleep(500);
|
|
2968
|
-
await typeText("r");
|
|
2969
|
-
await sleep(500);
|
|
2970
|
-
await keyCombo(["control", "a"]);
|
|
2971
|
-
await typeText(docName);
|
|
2972
|
-
await pressKey("Return");
|
|
2973
|
-
await sleep(500);
|
|
2974
|
-
await pressKey("Escape");
|
|
2975
|
-
step.result = `\u{1F4C4} Created Google Doc: ${docName}`;
|
|
2871
|
+
const success = await googleDocsType("");
|
|
2872
|
+
step.result = success ? `\u{1F4C4} Created Google Doc: ${docName}` : `\u{1F4C4} Could not create Google Doc`;
|
|
2976
2873
|
break;
|
|
2977
2874
|
}
|
|
2978
2875
|
case "type": {
|
|
2979
2876
|
const docText = docArgs.join("|");
|
|
2980
|
-
await
|
|
2981
|
-
step.result = `\u{1F4C4} Typed content in Google Doc`;
|
|
2877
|
+
const success = await googleDocsType(docText);
|
|
2878
|
+
step.result = success ? `\u{1F4C4} Typed content in Google Doc` : `\u{1F4C4} Could not type in Google Doc`;
|
|
2982
2879
|
break;
|
|
2983
2880
|
}
|
|
2984
2881
|
default:
|
|
@@ -2988,66 +2885,16 @@ ${readScreen.description}`;
|
|
|
2988
2885
|
}
|
|
2989
2886
|
case "research": {
|
|
2990
2887
|
const researchQuery = params;
|
|
2991
|
-
const
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
await sleep(2e3);
|
|
2997
|
-
await keyCombo(["control", "l"]);
|
|
2998
|
-
await sleep(300);
|
|
2999
|
-
await typeText("google.com");
|
|
3000
|
-
await pressKey("Return");
|
|
3001
|
-
await sleep(2e3);
|
|
3002
|
-
await typeText(researchQuery);
|
|
3003
|
-
await pressKey("Return");
|
|
3004
|
-
await sleep(3e3);
|
|
3005
|
-
let searchScreen = await describeScreen();
|
|
3006
|
-
const initialResults = await chat([{
|
|
3007
|
-
role: "user",
|
|
3008
|
-
content: `Extract the key information from these Google search results about: "${researchQuery}"
|
|
3009
|
-
Include any relevant facts, numbers, dates, or key points visible. Be thorough but concise.`
|
|
3010
|
-
}]);
|
|
3011
|
-
researchResults.push(`Search Results:
|
|
3012
|
-
${initialResults.content}`);
|
|
3013
|
-
await pressKey("Tab");
|
|
3014
|
-
await sleep(200);
|
|
3015
|
-
await pressKey("Tab");
|
|
3016
|
-
await sleep(200);
|
|
3017
|
-
await pressKey("Return");
|
|
3018
|
-
await sleep(4e3);
|
|
3019
|
-
searchScreen = await describeScreen();
|
|
3020
|
-
const pageContent = await chat([{
|
|
3021
|
-
role: "user",
|
|
3022
|
-
content: `Extract the main content and key information from this webpage about: "${researchQuery}"
|
|
3023
|
-
Ignore ads, navigation, footers. Focus on the actual article/content.`
|
|
3024
|
-
}]);
|
|
3025
|
-
researchResults.push(`
|
|
3026
|
-
Source 1 Content:
|
|
3027
|
-
${pageContent.content}`);
|
|
3028
|
-
await keyCombo(["alt", "Left"]);
|
|
3029
|
-
await sleep(2e3);
|
|
3030
|
-
await scrollMouse(-3);
|
|
3031
|
-
await sleep(500);
|
|
3032
|
-
await pressKey("Tab");
|
|
3033
|
-
await pressKey("Tab");
|
|
3034
|
-
await pressKey("Tab");
|
|
3035
|
-
await pressKey("Return");
|
|
3036
|
-
await sleep(4e3);
|
|
3037
|
-
searchScreen = await describeScreen();
|
|
3038
|
-
const pageContent2 = await chat([{
|
|
3039
|
-
role: "user",
|
|
3040
|
-
content: `Extract additional information from this webpage about: "${researchQuery}"
|
|
3041
|
-
Look for details not covered in the previous source.`
|
|
3042
|
-
}]);
|
|
3043
|
-
researchResults.push(`
|
|
3044
|
-
Source 2 Content:
|
|
3045
|
-
${pageContent2.content}`);
|
|
2888
|
+
const researchData = await research(researchQuery, 3);
|
|
2889
|
+
const sourceSummaries = researchData.sources.map(
|
|
2890
|
+
(s, i) => `Source ${i + 1}: ${s.title}
|
|
2891
|
+
${s.content.slice(0, 500)}...`
|
|
2892
|
+
).join("\n\n");
|
|
3046
2893
|
const synthesis = await chat([{
|
|
3047
2894
|
role: "user",
|
|
3048
2895
|
content: `Based on the following research gathered about "${researchQuery}", provide a comprehensive summary:
|
|
3049
2896
|
|
|
3050
|
-
${
|
|
2897
|
+
${sourceSummaries}
|
|
3051
2898
|
|
|
3052
2899
|
Create a well-organized summary with:
|
|
3053
2900
|
1. Key findings
|
|
@@ -3062,6 +2909,160 @@ Be thorough but concise.`
|
|
|
3062
2909
|
${synthesis.content}`;
|
|
3063
2910
|
break;
|
|
3064
2911
|
}
|
|
2912
|
+
case "ask_llm": {
|
|
2913
|
+
const [llmName, ...questionParts] = params.split("|");
|
|
2914
|
+
const question = questionParts.join("|");
|
|
2915
|
+
const currentScreen = await describeScreen();
|
|
2916
|
+
const fullQuestion = `I'm looking at my screen and I need help. ${question}
|
|
2917
|
+
|
|
2918
|
+
Here's what I see on my screen: ${currentScreen.description}`;
|
|
2919
|
+
const supportedLLMs = ["perplexity", "chatgpt", "claude", "copilot"];
|
|
2920
|
+
const llmLower = llmName.toLowerCase();
|
|
2921
|
+
if (!supportedLLMs.includes(llmLower)) {
|
|
2922
|
+
throw new Error(`Unknown LLM: ${llmName}. Supported: ${supportedLLMs.join(", ")}`);
|
|
2923
|
+
}
|
|
2924
|
+
const result = await askAI(llmLower, fullQuestion, false);
|
|
2925
|
+
const fullParts = await getFullAIResponse(llmLower, 3);
|
|
2926
|
+
const finalResponse = fullParts.length > 0 ? fullParts.join("\n\n") : result.response;
|
|
2927
|
+
step.result = `\u{1F916} ${llmName} says:
|
|
2928
|
+
|
|
2929
|
+
${finalResponse}`;
|
|
2930
|
+
break;
|
|
2931
|
+
}
|
|
2932
|
+
case "learn_ui": {
|
|
2933
|
+
const uiScreen = await describeScreen();
|
|
2934
|
+
const uiAnalysis = await chat([{
|
|
2935
|
+
role: "user",
|
|
2936
|
+
content: `Analyze this screenshot and identify all interactive UI elements. List:
|
|
2937
|
+
1. All clickable buttons and their likely functions
|
|
2938
|
+
2. Text input fields
|
|
2939
|
+
3. Menus and dropdowns
|
|
2940
|
+
4. Links
|
|
2941
|
+
5. Any keyboard shortcuts visible
|
|
2942
|
+
6. The main actions available in this interface
|
|
2943
|
+
|
|
2944
|
+
Question: ${params}
|
|
2945
|
+
|
|
2946
|
+
Be specific about locations (top-left, center, etc.) and what each element does.`
|
|
2947
|
+
}]);
|
|
2948
|
+
step.result = `\u{1F50D} UI Analysis:
|
|
2949
|
+
|
|
2950
|
+
${uiAnalysis.content}`;
|
|
2951
|
+
break;
|
|
2952
|
+
}
|
|
2953
|
+
case "adaptive_do": {
|
|
2954
|
+
const goal = params;
|
|
2955
|
+
const maxAttempts = 5;
|
|
2956
|
+
const actionHistory = [];
|
|
2957
|
+
let accomplished = false;
|
|
2958
|
+
const page = await getPage();
|
|
2959
|
+
for (let attempt = 0; attempt < maxAttempts && !accomplished; attempt++) {
|
|
2960
|
+
const screenshot = await takeScreenshot();
|
|
2961
|
+
const currentState = await chat([{
|
|
2962
|
+
role: "user",
|
|
2963
|
+
content: `Describe what you see on this screen. What app/website is it? What elements are visible?`
|
|
2964
|
+
}]);
|
|
2965
|
+
const nextAction = await chat([{
|
|
2966
|
+
role: "user",
|
|
2967
|
+
content: `GOAL: ${goal}
|
|
2968
|
+
|
|
2969
|
+
CURRENT SCREEN: ${currentState.content}
|
|
2970
|
+
|
|
2971
|
+
PREVIOUS ACTIONS TAKEN:
|
|
2972
|
+
${actionHistory.length > 0 ? actionHistory.join("\n") : "None yet"}
|
|
2973
|
+
|
|
2974
|
+
Based on what you see, what's the SINGLE next action to take?
|
|
2975
|
+
Options:
|
|
2976
|
+
- click: Click element (describe CSS selector or visible text)
|
|
2977
|
+
- type: Type something (specify selector and text)
|
|
2978
|
+
- press: Press a key (specify key)
|
|
2979
|
+
- scroll: Scroll up/down
|
|
2980
|
+
- navigate: Go to URL
|
|
2981
|
+
- done: Goal is accomplished
|
|
2982
|
+
- stuck: Can't figure out what to do
|
|
2983
|
+
|
|
2984
|
+
Respond in format:
|
|
2985
|
+
ACTION: <action_type>
|
|
2986
|
+
SELECTOR: <css selector or text to find>
|
|
2987
|
+
VALUE: <text to type or URL>
|
|
2988
|
+
REASONING: <why>`
|
|
2989
|
+
}]);
|
|
2990
|
+
const actionContent = nextAction.content;
|
|
2991
|
+
const actionMatch = actionContent.match(/ACTION:\s*(\w+)/i);
|
|
2992
|
+
const selectorMatch = actionContent.match(/SELECTOR:\s*(.+?)(?:\n|$)/i);
|
|
2993
|
+
const valueMatch = actionContent.match(/VALUE:\s*(.+?)(?:\n|$)/i);
|
|
2994
|
+
if (!actionMatch) {
|
|
2995
|
+
actionHistory.push(`Attempt ${attempt + 1}: Couldn't parse action`);
|
|
2996
|
+
continue;
|
|
2997
|
+
}
|
|
2998
|
+
const action = actionMatch[1].toLowerCase();
|
|
2999
|
+
const selector = selectorMatch?.[1]?.trim() || "";
|
|
3000
|
+
const value = valueMatch?.[1]?.trim() || "";
|
|
3001
|
+
if (action === "done") {
|
|
3002
|
+
accomplished = true;
|
|
3003
|
+
actionHistory.push(`Attempt ${attempt + 1}: Goal accomplished!`);
|
|
3004
|
+
break;
|
|
3005
|
+
}
|
|
3006
|
+
if (action === "stuck") {
|
|
3007
|
+
actionHistory.push(`Attempt ${attempt + 1}: Got stuck, asking Perplexity for help...`);
|
|
3008
|
+
const helpRequest = `I'm trying to: ${goal}
|
|
3009
|
+
|
|
3010
|
+
I'm stuck. What should I do next? Be specific about what to click or type.`;
|
|
3011
|
+
const advice = await askAI("perplexity", helpRequest, false);
|
|
3012
|
+
actionHistory.push(`Got advice: ${advice.response.slice(0, 200)}...`);
|
|
3013
|
+
await navigateTo(page.url());
|
|
3014
|
+
continue;
|
|
3015
|
+
}
|
|
3016
|
+
try {
|
|
3017
|
+
switch (action) {
|
|
3018
|
+
case "click":
|
|
3019
|
+
if (selector) {
|
|
3020
|
+
const clicked = await clickElement(selector);
|
|
3021
|
+
if (!clicked) {
|
|
3022
|
+
await page.getByText(selector).first().click({ timeout: 5e3 });
|
|
3023
|
+
}
|
|
3024
|
+
}
|
|
3025
|
+
actionHistory.push(`Attempt ${attempt + 1}: Clicked "${selector}"`);
|
|
3026
|
+
break;
|
|
3027
|
+
case "type":
|
|
3028
|
+
if (selector && value) {
|
|
3029
|
+
const typed = await typeInElement(selector, value);
|
|
3030
|
+
if (!typed) {
|
|
3031
|
+
await page.getByPlaceholder(selector).first().fill(value);
|
|
3032
|
+
}
|
|
3033
|
+
}
|
|
3034
|
+
actionHistory.push(`Attempt ${attempt + 1}: Typed "${value}" in "${selector}"`);
|
|
3035
|
+
break;
|
|
3036
|
+
case "press":
|
|
3037
|
+
await pressKey2(value || selector);
|
|
3038
|
+
actionHistory.push(`Attempt ${attempt + 1}: Pressed ${value || selector}`);
|
|
3039
|
+
break;
|
|
3040
|
+
case "scroll":
|
|
3041
|
+
await scroll(value.toLowerCase().includes("up") ? "up" : "down");
|
|
3042
|
+
actionHistory.push(`Attempt ${attempt + 1}: Scrolled ${value || "down"}`);
|
|
3043
|
+
break;
|
|
3044
|
+
case "navigate":
|
|
3045
|
+
const url = value.startsWith("http") ? value : `https://${value}`;
|
|
3046
|
+
await navigateTo(url);
|
|
3047
|
+
actionHistory.push(`Attempt ${attempt + 1}: Navigated to ${url}`);
|
|
3048
|
+
break;
|
|
3049
|
+
default:
|
|
3050
|
+
actionHistory.push(`Attempt ${attempt + 1}: Unknown action ${action}`);
|
|
3051
|
+
}
|
|
3052
|
+
} catch (e) {
|
|
3053
|
+
actionHistory.push(`Attempt ${attempt + 1}: Action failed - ${e}`);
|
|
3054
|
+
}
|
|
3055
|
+
await sleep(2e3);
|
|
3056
|
+
}
|
|
3057
|
+
step.result = `\u{1F3AF} Adaptive Agent Result:
|
|
3058
|
+
|
|
3059
|
+
Goal: ${goal}
|
|
3060
|
+
Accomplished: ${accomplished ? "Yes \u2705" : "Partial/No \u274C"}
|
|
3061
|
+
|
|
3062
|
+
Action Log:
|
|
3063
|
+
${actionHistory.join("\n")}`;
|
|
3064
|
+
break;
|
|
3065
|
+
}
|
|
3065
3066
|
case "chat":
|
|
3066
3067
|
step.result = `Task noted: ${params}`;
|
|
3067
3068
|
break;
|
|
@@ -3152,10 +3153,10 @@ function formatTask(task) {
|
|
|
3152
3153
|
|
|
3153
3154
|
// src/hooks/useTasks.ts
|
|
3154
3155
|
function useTasks(onProgress) {
|
|
3155
|
-
const [isRunning, setIsRunning] =
|
|
3156
|
-
const [currentTask, setCurrentTask] =
|
|
3157
|
-
const [currentStep, setCurrentStep] =
|
|
3158
|
-
const [error, setError] =
|
|
3156
|
+
const [isRunning, setIsRunning] = useState5(false);
|
|
3157
|
+
const [currentTask, setCurrentTask] = useState5(null);
|
|
3158
|
+
const [currentStep, setCurrentStep] = useState5(null);
|
|
3159
|
+
const [error, setError] = useState5(null);
|
|
3159
3160
|
const run = useCallback4(async (description) => {
|
|
3160
3161
|
setIsRunning(true);
|
|
3161
3162
|
setError(null);
|
|
@@ -3191,13 +3192,13 @@ function useTasks(onProgress) {
|
|
|
3191
3192
|
}
|
|
3192
3193
|
|
|
3193
3194
|
// src/components/App.tsx
|
|
3194
|
-
import { jsx as
|
|
3195
|
+
import { jsx as jsx6, jsxs as jsxs5 } from "react/jsx-runtime";
|
|
3195
3196
|
function App() {
|
|
3196
3197
|
const { exit } = useApp();
|
|
3197
|
-
const [overlay, setOverlay] =
|
|
3198
|
-
const [screenWatch, setScreenWatch] =
|
|
3199
|
-
const [status, setStatus] =
|
|
3200
|
-
const [inputValue, setInputValue] =
|
|
3198
|
+
const [overlay, setOverlay] = useState6("none");
|
|
3199
|
+
const [screenWatch, setScreenWatch] = useState6(false);
|
|
3200
|
+
const [status, setStatus] = useState6("Ready");
|
|
3201
|
+
const [inputValue, setInputValue] = useState6("");
|
|
3201
3202
|
const chat2 = useChat(screenWatch);
|
|
3202
3203
|
const vision = useVision();
|
|
3203
3204
|
const telegram = useTelegram((msg) => {
|
|
@@ -3208,7 +3209,7 @@ function App() {
|
|
|
3208
3209
|
setStatus(`Running: ${step.description}`);
|
|
3209
3210
|
}
|
|
3210
3211
|
});
|
|
3211
|
-
|
|
3212
|
+
useInput2((inputChar, key) => {
|
|
3212
3213
|
if (overlay !== "none") return;
|
|
3213
3214
|
if (key.ctrl && inputChar === "c") exit();
|
|
3214
3215
|
if (key.ctrl && inputChar === "l") chat2.clearMessages();
|
|
@@ -3386,7 +3387,7 @@ ${tasks.format(task)}`);
|
|
|
3386
3387
|
chat2.addSystemMessage(`\u2705 Updated: ${provider} / ${model}`);
|
|
3387
3388
|
}, [chat2]);
|
|
3388
3389
|
if (overlay === "help") {
|
|
3389
|
-
return /* @__PURE__ */
|
|
3390
|
+
return /* @__PURE__ */ jsx6(Box6, { flexDirection: "column", height: "100%", alignItems: "center", justifyContent: "center", children: /* @__PURE__ */ jsx6(
|
|
3390
3391
|
HelpMenu,
|
|
3391
3392
|
{
|
|
3392
3393
|
onClose: () => setOverlay("none"),
|
|
@@ -3398,7 +3399,7 @@ ${tasks.format(task)}`);
|
|
|
3398
3399
|
) });
|
|
3399
3400
|
}
|
|
3400
3401
|
if (overlay === "provider") {
|
|
3401
|
-
return /* @__PURE__ */
|
|
3402
|
+
return /* @__PURE__ */ jsx6(Box6, { flexDirection: "column", height: "100%", alignItems: "center", justifyContent: "center", children: /* @__PURE__ */ jsx6(
|
|
3402
3403
|
ProviderSelector,
|
|
3403
3404
|
{
|
|
3404
3405
|
onClose: () => setOverlay("none"),
|
|
@@ -3408,11 +3409,11 @@ ${tasks.format(task)}`);
|
|
|
3408
3409
|
}
|
|
3409
3410
|
const visibleMessages = chat2.messages.slice(-20);
|
|
3410
3411
|
const isProcessing = chat2.isProcessing || vision.isAnalyzing || tasks.isRunning || telegram.isStarting;
|
|
3411
|
-
return /* @__PURE__ */
|
|
3412
|
-
/* @__PURE__ */
|
|
3413
|
-
/* @__PURE__ */
|
|
3414
|
-
/* @__PURE__ */
|
|
3415
|
-
visibleMessages.map((msg) => /* @__PURE__ */
|
|
3412
|
+
return /* @__PURE__ */ jsxs5(Box6, { flexDirection: "column", height: "100%", children: [
|
|
3413
|
+
/* @__PURE__ */ jsx6(Header, { screenWatch, telegramEnabled: telegram.isEnabled }),
|
|
3414
|
+
/* @__PURE__ */ jsxs5(Box6, { flexDirection: "column", flexGrow: 1, borderStyle: "round", borderColor: "gray", padding: 1, children: [
|
|
3415
|
+
/* @__PURE__ */ jsx6(Text6, { bold: true, color: "gray", children: " Chat " }),
|
|
3416
|
+
visibleMessages.map((msg) => /* @__PURE__ */ jsx6(
|
|
3416
3417
|
ChatMessage,
|
|
3417
3418
|
{
|
|
3418
3419
|
role: msg.role,
|
|
@@ -3423,11 +3424,11 @@ ${tasks.format(task)}`);
|
|
|
3423
3424
|
msg.id
|
|
3424
3425
|
))
|
|
3425
3426
|
] }),
|
|
3426
|
-
chat2.error && /* @__PURE__ */
|
|
3427
|
+
chat2.error && /* @__PURE__ */ jsx6(Box6, { marginY: 1, children: /* @__PURE__ */ jsxs5(Text6, { color: "red", children: [
|
|
3427
3428
|
"Error: ",
|
|
3428
3429
|
chat2.error
|
|
3429
3430
|
] }) }),
|
|
3430
|
-
/* @__PURE__ */
|
|
3431
|
+
/* @__PURE__ */ jsx6(
|
|
3431
3432
|
ChatInput,
|
|
3432
3433
|
{
|
|
3433
3434
|
value: inputValue,
|
|
@@ -3436,12 +3437,12 @@ ${tasks.format(task)}`);
|
|
|
3436
3437
|
isProcessing
|
|
3437
3438
|
}
|
|
3438
3439
|
),
|
|
3439
|
-
/* @__PURE__ */
|
|
3440
|
+
/* @__PURE__ */ jsx6(StatusBar, { status })
|
|
3440
3441
|
] });
|
|
3441
3442
|
}
|
|
3442
3443
|
|
|
3443
3444
|
// src/index.tsx
|
|
3444
|
-
import { jsx as
|
|
3445
|
+
import { jsx as jsx7, jsxs as jsxs6 } from "react/jsx-runtime";
|
|
3445
3446
|
var args = process.argv.slice(2);
|
|
3446
3447
|
async function main() {
|
|
3447
3448
|
if (args.length > 0) {
|
|
@@ -3470,8 +3471,21 @@ async function main() {
|
|
|
3470
3471
|
case "config": {
|
|
3471
3472
|
const subcommand = args[1];
|
|
3472
3473
|
if (!subcommand) {
|
|
3473
|
-
const {
|
|
3474
|
-
|
|
3474
|
+
const { ProviderSelector: ProviderSelector2 } = await import("./ProviderSelector-MXRZFAOB.js");
|
|
3475
|
+
const { Box: Box7 } = await import("ink");
|
|
3476
|
+
render(
|
|
3477
|
+
/* @__PURE__ */ jsx7(Box7, { flexDirection: "column", padding: 1, children: /* @__PURE__ */ jsx7(
|
|
3478
|
+
ProviderSelector2,
|
|
3479
|
+
{
|
|
3480
|
+
onClose: () => process.exit(0),
|
|
3481
|
+
onSelect: (provider, model) => {
|
|
3482
|
+
console.log(`
|
|
3483
|
+
\u2713 Set to ${provider} / ${model}`);
|
|
3484
|
+
process.exit(0);
|
|
3485
|
+
}
|
|
3486
|
+
}
|
|
3487
|
+
) })
|
|
3488
|
+
);
|
|
3475
3489
|
return;
|
|
3476
3490
|
}
|
|
3477
3491
|
if (subcommand === "set") {
|
|
@@ -3569,12 +3583,29 @@ ${dim}GitHub: https://github.com/projectservan8n/C-napse${reset}
|
|
|
3569
3583
|
case "version":
|
|
3570
3584
|
case "--version":
|
|
3571
3585
|
case "-v": {
|
|
3572
|
-
console.log("cnapse v0.
|
|
3586
|
+
console.log("cnapse v0.8.0");
|
|
3573
3587
|
process.exit(0);
|
|
3574
3588
|
}
|
|
3575
3589
|
case "init": {
|
|
3576
|
-
const {
|
|
3577
|
-
|
|
3590
|
+
const { ProviderSelector: ProviderSelector2 } = await import("./ProviderSelector-MXRZFAOB.js");
|
|
3591
|
+
const { Box: Box7, Text: Text7 } = await import("ink");
|
|
3592
|
+
render(
|
|
3593
|
+
/* @__PURE__ */ jsxs6(Box7, { flexDirection: "column", padding: 1, children: [
|
|
3594
|
+
/* @__PURE__ */ jsx7(Box7, { marginBottom: 1, children: /* @__PURE__ */ jsx7(Text7, { bold: true, color: "cyan", children: "C-napse Setup" }) }),
|
|
3595
|
+
/* @__PURE__ */ jsx7(
|
|
3596
|
+
ProviderSelector2,
|
|
3597
|
+
{
|
|
3598
|
+
onClose: () => process.exit(0),
|
|
3599
|
+
onSelect: (provider, model) => {
|
|
3600
|
+
console.log(`
|
|
3601
|
+
\u2713 Setup complete! Provider: ${provider}, Model: ${model}`);
|
|
3602
|
+
console.log("Run `cnapse` to start chatting.");
|
|
3603
|
+
process.exit(0);
|
|
3604
|
+
}
|
|
3605
|
+
}
|
|
3606
|
+
)
|
|
3607
|
+
] })
|
|
3608
|
+
);
|
|
3578
3609
|
return;
|
|
3579
3610
|
}
|
|
3580
3611
|
default: {
|
|
@@ -3582,6 +3613,6 @@ ${dim}GitHub: https://github.com/projectservan8n/C-napse${reset}
|
|
|
3582
3613
|
}
|
|
3583
3614
|
}
|
|
3584
3615
|
}
|
|
3585
|
-
render(/* @__PURE__ */
|
|
3616
|
+
render(/* @__PURE__ */ jsx7(App, {}));
|
|
3586
3617
|
}
|
|
3587
3618
|
main();
|