aiden-runtime 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +465 -0
- package/config/devos.config.json +186 -0
- package/config/hardware.json +9 -0
- package/config/model-selection.json +7 -0
- package/config/setup-complete.json +20 -0
- package/dist/api/routes/computerUse.js +112 -0
- package/dist/api/server.js +6870 -0
- package/dist/bin/npx-init.js +71 -0
- package/dist/coordination/commandGate.js +115 -0
- package/dist/coordination/livePulse.js +127 -0
- package/dist/core/agentLoop.js +2718 -0
- package/dist/core/agentShield.js +231 -0
- package/dist/core/aidenIdentity.js +215 -0
- package/dist/core/aidenPersonality.js +166 -0
- package/dist/core/aidenSdk.js +374 -0
- package/dist/core/asyncTasks.js +82 -0
- package/dist/core/auditTrail.js +61 -0
- package/dist/core/auxiliaryClient.js +114 -0
- package/dist/core/bgLLM.js +108 -0
- package/dist/core/bm25.js +68 -0
- package/dist/core/callbackSystem.js +64 -0
- package/dist/core/channels/adapter.js +6 -0
- package/dist/core/channels/discord.js +173 -0
- package/dist/core/channels/email.js +253 -0
- package/dist/core/channels/imessage.js +164 -0
- package/dist/core/channels/manager.js +96 -0
- package/dist/core/channels/signal.js +140 -0
- package/dist/core/channels/slack.js +139 -0
- package/dist/core/channels/twilio.js +144 -0
- package/dist/core/channels/webhook.js +186 -0
- package/dist/core/channels/whatsapp.js +185 -0
- package/dist/core/clarifyBus.js +75 -0
- package/dist/core/codeInterpreter.js +82 -0
- package/dist/core/computerControl.js +439 -0
- package/dist/core/conversationMemory.js +334 -0
- package/dist/core/costTracker.js +221 -0
- package/dist/core/cronManager.js +217 -0
- package/dist/core/deepKB.js +77 -0
- package/dist/core/doctor.js +279 -0
- package/dist/core/dreamEngine.js +334 -0
- package/dist/core/entityGraph.js +169 -0
- package/dist/core/eventBus.js +16 -0
- package/dist/core/evolutionAnalyzer.js +153 -0
- package/dist/core/executionLoop.js +309 -0
- package/dist/core/executor.js +224 -0
- package/dist/core/failureAnalyzer.js +166 -0
- package/dist/core/fastPathExpansion.js +82 -0
- package/dist/core/faultEngine.js +106 -0
- package/dist/core/featureGates.js +70 -0
- package/dist/core/fileIngestion.js +113 -0
- package/dist/core/gateway.js +97 -0
- package/dist/core/goalTracker.js +75 -0
- package/dist/core/growthEngine.js +168 -0
- package/dist/core/hardwareDetector.js +98 -0
- package/dist/core/hooks.js +45 -0
- package/dist/core/httpKeepalive.js +46 -0
- package/dist/core/hybridSearch.js +101 -0
- package/dist/core/importers.js +164 -0
- package/dist/core/instinctSystem.js +223 -0
- package/dist/core/knowledgeBase.js +351 -0
- package/dist/core/learningMemory.js +121 -0
- package/dist/core/lessonsBrowser.js +125 -0
- package/dist/core/licenseManager.js +399 -0
- package/dist/core/logBuffer.js +85 -0
- package/dist/core/machineId.js +87 -0
- package/dist/core/mcpClient.js +442 -0
- package/dist/core/memoryDistiller.js +165 -0
- package/dist/core/memoryExtractor.js +212 -0
- package/dist/core/memoryIds.js +213 -0
- package/dist/core/memoryPreamble.js +113 -0
- package/dist/core/memoryQuery.js +136 -0
- package/dist/core/memoryRecall.js +140 -0
- package/dist/core/memoryStrategy.js +201 -0
- package/dist/core/messageValidator.js +85 -0
- package/dist/core/modelDiscovery.js +108 -0
- package/dist/core/modelRouter.js +118 -0
- package/dist/core/morningBriefing.js +203 -0
- package/dist/core/multiGoalValidator.js +51 -0
- package/dist/core/parallelExecutor.js +43 -0
- package/dist/core/passiveSkillObserver.js +204 -0
- package/dist/core/paths.js +57 -0
- package/dist/core/patternDetector.js +83 -0
- package/dist/core/planResponseRepair.js +64 -0
- package/dist/core/planTool.js +111 -0
- package/dist/core/playwrightBridge.js +356 -0
- package/dist/core/pluginSystem.js +121 -0
- package/dist/core/privateMode.js +85 -0
- package/dist/core/reactLoop.js +156 -0
- package/dist/core/recipeEngine.js +166 -0
- package/dist/core/responseCache.js +128 -0
- package/dist/core/runSandbox.js +132 -0
- package/dist/core/sandboxRunner.js +200 -0
- package/dist/core/scheduler.js +543 -0
- package/dist/core/secretScanner.js +49 -0
- package/dist/core/semanticMemory.js +223 -0
- package/dist/core/sessionMemory.js +259 -0
- package/dist/core/sessionRouter.js +91 -0
- package/dist/core/sessionSearch.js +163 -0
- package/dist/core/setupWizard.js +225 -0
- package/dist/core/skillImporter.js +303 -0
- package/dist/core/skillLibrary.js +144 -0
- package/dist/core/skillLoader.js +471 -0
- package/dist/core/skillTeacher.js +352 -0
- package/dist/core/skillValidator.js +210 -0
- package/dist/core/skillWriter.js +384 -0
- package/dist/core/slashAsTool.js +226 -0
- package/dist/core/spawnManager.js +197 -0
- package/dist/core/statusVerbs.js +43 -0
- package/dist/core/swarmManager.js +109 -0
- package/dist/core/taskQueue.js +119 -0
- package/dist/core/taskRecovery.js +128 -0
- package/dist/core/taskState.js +168 -0
- package/dist/core/telegramBot.js +152 -0
- package/dist/core/todoManager.js +70 -0
- package/dist/core/toolNameRepair.js +71 -0
- package/dist/core/toolRegistry.js +2730 -0
- package/dist/core/tools/calendarTool.js +98 -0
- package/dist/core/tools/companyFilingsTool.js +98 -0
- package/dist/core/tools/gmailTool.js +87 -0
- package/dist/core/tools/marketDataTool.js +135 -0
- package/dist/core/tools/socialResearchTool.js +121 -0
- package/dist/core/truthCheck.js +57 -0
- package/dist/core/updateChecker.js +74 -0
- package/dist/core/userCognitionProfile.js +238 -0
- package/dist/core/userProfile.js +341 -0
- package/dist/core/version.js +5 -0
- package/dist/core/visionAnalyze.js +161 -0
- package/dist/core/voice/audio.js +187 -0
- package/dist/core/voice/stt.js +226 -0
- package/dist/core/voice/tts.js +310 -0
- package/dist/core/voiceInput.js +118 -0
- package/dist/core/voiceOutput.js +130 -0
- package/dist/core/webSearch.js +326 -0
- package/dist/core/workflowTracker.js +72 -0
- package/dist/core/workspaceMemory.js +54 -0
- package/dist/core/youtubeTranscript.js +224 -0
- package/dist/integrations/computerUse/apiRegistry.js +113 -0
- package/dist/integrations/computerUse/screenAgent.js +203 -0
- package/dist/integrations/computerUse/visionLoop.js +296 -0
- package/dist/memory/memoryLayers.js +143 -0
- package/dist/providers/boa.js +93 -0
- package/dist/providers/cerebras.js +70 -0
- package/dist/providers/custom.js +89 -0
- package/dist/providers/gemini.js +82 -0
- package/dist/providers/groq.js +92 -0
- package/dist/providers/index.js +149 -0
- package/dist/providers/nvidia.js +70 -0
- package/dist/providers/ollama.js +99 -0
- package/dist/providers/openrouter.js +74 -0
- package/dist/providers/router.js +497 -0
- package/dist/providers/types.js +6 -0
- package/dist/security/browserVault.js +129 -0
- package/dist/security/dataGuard.js +89 -0
- package/dist/tools/eonetTool.js +72 -0
- package/dist/types/computerUse.js +2 -0
- package/dist/types/executor.js +2 -0
- package/dist-bundle/cli.js +357859 -0
- package/package.json +256 -0
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ============================================================
|
|
3
|
+
// DevOS — Autonomous AI Execution System
|
|
4
|
+
// Copyright (c) 2026 Shiva Deore. All rights reserved.
|
|
5
|
+
// ============================================================
|
|
6
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
7
|
+
if (k2 === undefined) k2 = k;
|
|
8
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
9
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
10
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
11
|
+
}
|
|
12
|
+
Object.defineProperty(o, k2, desc);
|
|
13
|
+
}) : (function(o, m, k, k2) {
|
|
14
|
+
if (k2 === undefined) k2 = k;
|
|
15
|
+
o[k2] = m[k];
|
|
16
|
+
}));
|
|
17
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
18
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
19
|
+
}) : function(o, v) {
|
|
20
|
+
o["default"] = v;
|
|
21
|
+
});
|
|
22
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
23
|
+
var ownKeys = function(o) {
|
|
24
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
25
|
+
var ar = [];
|
|
26
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
27
|
+
return ar;
|
|
28
|
+
};
|
|
29
|
+
return ownKeys(o);
|
|
30
|
+
};
|
|
31
|
+
return function (mod) {
|
|
32
|
+
if (mod && mod.__esModule) return mod;
|
|
33
|
+
var result = {};
|
|
34
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
35
|
+
__setModuleDefault(result, mod);
|
|
36
|
+
return result;
|
|
37
|
+
};
|
|
38
|
+
})();
|
|
39
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
40
|
+
exports.apiRegistry = void 0;
|
|
41
|
+
class APIRegistry {
|
|
42
|
+
constructor() {
|
|
43
|
+
this.registry = new Map();
|
|
44
|
+
}
|
|
45
|
+
// ── Registration ─────────────────────────────────────────────
|
|
46
|
+
register(serviceName, handler) {
|
|
47
|
+
this.registry.set(serviceName.toLowerCase(), handler);
|
|
48
|
+
}
|
|
49
|
+
hasAPI(service) {
|
|
50
|
+
return this.registry.has(service.toLowerCase());
|
|
51
|
+
}
|
|
52
|
+
// ── Execute ──────────────────────────────────────────────────
|
|
53
|
+
/**
|
|
54
|
+
* Execute a service call.
|
|
55
|
+
*
|
|
56
|
+
* Resolution order:
|
|
57
|
+
* 1. Registered API handler → fast, structured, no UI
|
|
58
|
+
* 2. BrowserVault UI fallback → slow, visual, always works
|
|
59
|
+
*
|
|
60
|
+
* Returns `{ result, usedAPI: true }` when a handler succeeded,
|
|
61
|
+
* or `{ result: { status: 'ui_fallback', ... }, usedAPI: false }` for fallback.
|
|
62
|
+
*/
|
|
63
|
+
async execute(service, params) {
|
|
64
|
+
const handler = this.registry.get(service.toLowerCase());
|
|
65
|
+
if (handler) {
|
|
66
|
+
try {
|
|
67
|
+
const result = await handler(params);
|
|
68
|
+
return { result, usedAPI: true };
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
console.warn(`[APIRegistry] ${service} API handler failed — falling back to UI: ${err?.message}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
// UI fallback — open BrowserVault on the service's web app
|
|
75
|
+
const { browserVault } = await Promise.resolve().then(() => __importStar(require('../../security/browserVault')));
|
|
76
|
+
const SERVICE_URLS = {
|
|
77
|
+
gmail: 'https://mail.google.com',
|
|
78
|
+
notion: 'https://notion.so',
|
|
79
|
+
github: 'https://github.com',
|
|
80
|
+
slack: 'https://app.slack.com',
|
|
81
|
+
sheets: 'https://docs.google.com/spreadsheets',
|
|
82
|
+
linear: 'https://linear.app',
|
|
83
|
+
jira: 'https://id.atlassian.com',
|
|
84
|
+
figma: 'https://www.figma.com',
|
|
85
|
+
trello: 'https://trello.com',
|
|
86
|
+
asana: 'https://app.asana.com',
|
|
87
|
+
};
|
|
88
|
+
const url = SERVICE_URLS[service.toLowerCase()];
|
|
89
|
+
if (!url)
|
|
90
|
+
throw new Error(`[APIRegistry] No API handler and no UI fallback for service: ${service}`);
|
|
91
|
+
const taskId = `api-fallback-${Date.now()}`;
|
|
92
|
+
const vault = await browserVault.createBrowserVault(taskId);
|
|
93
|
+
const liveViewUrl = browserVault.getLiveViewUrl(taskId);
|
|
94
|
+
console.log(`[APIRegistry] 🌐 UI fallback → ${url} (vault: ${vault.containerName})`);
|
|
95
|
+
return {
|
|
96
|
+
result: {
|
|
97
|
+
status: 'ui_fallback',
|
|
98
|
+
url,
|
|
99
|
+
vaultId: vault.taskId,
|
|
100
|
+
liveViewUrl,
|
|
101
|
+
},
|
|
102
|
+
usedAPI: false,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
// ── Introspection ─────────────────────────────────────────────
|
|
106
|
+
listServices() {
|
|
107
|
+
return Array.from(this.registry.keys());
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
exports.apiRegistry = new APIRegistry();
|
|
111
|
+
// ── Built-in registrations ────────────────────────────────────
|
|
112
|
+
// Extend via config/api-keys.json or register() calls at startup.
|
|
113
|
+
// Example: apiRegistry.register('github', githubHandler)
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ============================================================
|
|
3
|
+
// DevOS — Autonomous AI Execution System
|
|
4
|
+
// Copyright (c) 2026 Shiva Deore. All rights reserved.
|
|
5
|
+
// ============================================================
|
|
6
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
7
|
+
if (k2 === undefined) k2 = k;
|
|
8
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
9
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
10
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
11
|
+
}
|
|
12
|
+
Object.defineProperty(o, k2, desc);
|
|
13
|
+
}) : (function(o, m, k, k2) {
|
|
14
|
+
if (k2 === undefined) k2 = k;
|
|
15
|
+
o[k2] = m[k];
|
|
16
|
+
}));
|
|
17
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
18
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
19
|
+
}) : function(o, v) {
|
|
20
|
+
o["default"] = v;
|
|
21
|
+
});
|
|
22
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
23
|
+
var ownKeys = function(o) {
|
|
24
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
25
|
+
var ar = [];
|
|
26
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
27
|
+
return ar;
|
|
28
|
+
};
|
|
29
|
+
return ownKeys(o);
|
|
30
|
+
};
|
|
31
|
+
return function (mod) {
|
|
32
|
+
if (mod && mod.__esModule) return mod;
|
|
33
|
+
var result = {};
|
|
34
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
35
|
+
__setModuleDefault(result, mod);
|
|
36
|
+
return result;
|
|
37
|
+
};
|
|
38
|
+
})();
|
|
39
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
40
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
41
|
+
};
|
|
42
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
43
|
+
exports.screenAgent = void 0;
|
|
44
|
+
// integrations/computerUse/screenAgent.ts
|
|
45
|
+
// Low-level computer control: screenshot, mouse, keyboard.
|
|
46
|
+
// Every action is gated by CommandGate (confidence < 0.65)
|
|
47
|
+
// and verified by TruthChecker after execution.
|
|
48
|
+
const screenshot_desktop_1 = __importDefault(require("screenshot-desktop"));
|
|
49
|
+
const commandGate_1 = require("../../coordination/commandGate");
|
|
50
|
+
const truthCheck_1 = require("../../core/truthCheck");
|
|
51
|
+
const faultEngine_1 = require("../../core/faultEngine");
|
|
52
|
+
// ── Lazy nut-js loader ────────────────────────────────────────
|
|
53
|
+
// @nut-tree-fork/nut-js requires native build tools. We load it
|
|
54
|
+
// lazily so the API server can start even when the native module
|
|
55
|
+
// is not installed / built.
|
|
56
|
+
let _nutjs = null;
|
|
57
|
+
async function getNut() {
|
|
58
|
+
if (!_nutjs) {
|
|
59
|
+
try {
|
|
60
|
+
_nutjs = await Promise.resolve().then(() => __importStar(require('@nut-tree-fork/nut-js')));
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
throw new Error(`@nut-tree-fork/nut-js is not installed or failed to build. ` +
|
|
64
|
+
`Run: npm install @nut-tree-fork/nut-js --build-from-source\n` +
|
|
65
|
+
`Original error: ${err?.message ?? err}`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return _nutjs;
|
|
69
|
+
}
|
|
70
|
+
// ── ScreenAgent ───────────────────────────────────────────────
|
|
71
|
+
class ScreenAgent {
|
|
72
|
+
constructor() {
|
|
73
|
+
this.actionLog = [];
|
|
74
|
+
}
|
|
75
|
+
// ── Approval ─────────────────────────────────────────────────
|
|
76
|
+
/**
|
|
77
|
+
* Request CommandGate approval before executing a sensitive action.
|
|
78
|
+
* Always triggered for actions with confidence < 0.65 and for the
|
|
79
|
+
* first action of a new session.
|
|
80
|
+
*/
|
|
81
|
+
async requestApproval(action) {
|
|
82
|
+
return commandGate_1.commandGate.requestApproval(`Computer control: ${action.type} — ${action.description ?? 'no description'}`, `Action ID: ${action.id}, Confidence: ${action.confidence}`);
|
|
83
|
+
}
|
|
84
|
+
// ── Screenshot ───────────────────────────────────────────────
|
|
85
|
+
/**
|
|
86
|
+
* Capture the current screen and return a base64-encoded PNG string.
|
|
87
|
+
*/
|
|
88
|
+
async takeScreenshot() {
|
|
89
|
+
const img = await (0, screenshot_desktop_1.default)({ format: 'png' });
|
|
90
|
+
return img.toString('base64');
|
|
91
|
+
}
|
|
92
|
+
// ── Execute ──────────────────────────────────────────────────
|
|
93
|
+
/**
|
|
94
|
+
* Execute a single ComputerUseAction.
|
|
95
|
+
* - Appends to actionLog unconditionally
|
|
96
|
+
* - Confidence < 0.65 → requests CommandGate approval first
|
|
97
|
+
* - After execution, does a lightweight postcondition check:
|
|
98
|
+
* takes a new screenshot and verifies the screen changed
|
|
99
|
+
* (non-blocking best-effort; never blocks success on indeterminate result)
|
|
100
|
+
*/
|
|
101
|
+
async execute(action) {
|
|
102
|
+
this.actionLog.push(action);
|
|
103
|
+
// Confidence gate
|
|
104
|
+
if (action.confidence < 0.65) {
|
|
105
|
+
const approved = await this.requestApproval(action);
|
|
106
|
+
if (!approved)
|
|
107
|
+
return { success: false, error: 'Rejected by CommandGate' };
|
|
108
|
+
}
|
|
109
|
+
try {
|
|
110
|
+
const { mouse, keyboard, Button, Key } = await getNut();
|
|
111
|
+
switch (action.type) {
|
|
112
|
+
case 'click': {
|
|
113
|
+
const a = action;
|
|
114
|
+
await mouse.move([{ x: a.x, y: a.y }]);
|
|
115
|
+
await mouse.click(a.button === 'right' ? Button.RIGHT : Button.LEFT);
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
case 'type': {
|
|
119
|
+
const a = action;
|
|
120
|
+
await keyboard.type(a.text);
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
case 'scroll': {
|
|
124
|
+
const a = action;
|
|
125
|
+
if (a.deltaY && a.deltaY > 0)
|
|
126
|
+
await mouse.scrollDown(Math.abs(a.deltaY));
|
|
127
|
+
if (a.deltaY && a.deltaY < 0)
|
|
128
|
+
await mouse.scrollUp(Math.abs(a.deltaY));
|
|
129
|
+
break;
|
|
130
|
+
}
|
|
131
|
+
case 'keypress': {
|
|
132
|
+
const a = action;
|
|
133
|
+
const mapped = a.keys
|
|
134
|
+
.map((k) => Key[k])
|
|
135
|
+
.filter((v) => v !== undefined);
|
|
136
|
+
if (mapped.length === 1)
|
|
137
|
+
await keyboard.pressKey(mapped[0]);
|
|
138
|
+
else if (mapped.length > 1)
|
|
139
|
+
await keyboard.pressKey(...mapped);
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
case 'screenshot':
|
|
143
|
+
await this.takeScreenshot();
|
|
144
|
+
break;
|
|
145
|
+
// api_call is handled upstream by APIRegistry — not executed here
|
|
146
|
+
default:
|
|
147
|
+
break;
|
|
148
|
+
}
|
|
149
|
+
// Lightweight postcondition: verify via TruthChecker that the action type
|
|
150
|
+
// is considered complete. We build a minimal single-node TaskGraph to
|
|
151
|
+
// re-use the existing TruthChecker.verify() logic.
|
|
152
|
+
const verified = await this.verifyAction(action);
|
|
153
|
+
if (!verified) {
|
|
154
|
+
const fault = faultEngine_1.faultEngine.classify(`Action ${action.type} produced no visible change`, {
|
|
155
|
+
actionType: action.type,
|
|
156
|
+
workspacePath: process.cwd(),
|
|
157
|
+
});
|
|
158
|
+
return { success: false, error: fault.manualFix };
|
|
159
|
+
}
|
|
160
|
+
return { success: true };
|
|
161
|
+
}
|
|
162
|
+
catch (err) {
|
|
163
|
+
return { success: false, error: err?.message ?? String(err) };
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
// ── Verify ───────────────────────────────────────────────────
|
|
167
|
+
/**
|
|
168
|
+
* Best-effort postcondition check for computer-use actions.
|
|
169
|
+
* TruthChecker.verify() is graph-oriented; for computer actions we
|
|
170
|
+
* use the 'notify' fallback (fire-and-forget → always passes) so we
|
|
171
|
+
* don't block on unverifiable state, while still running the
|
|
172
|
+
* TruthChecker code path for auditability.
|
|
173
|
+
*/
|
|
174
|
+
async verifyAction(action) {
|
|
175
|
+
try {
|
|
176
|
+
// Build a minimal synthetic TaskGraph node for the action
|
|
177
|
+
const fakeGraph = {
|
|
178
|
+
nodes: new Map([
|
|
179
|
+
[
|
|
180
|
+
action.id,
|
|
181
|
+
{
|
|
182
|
+
id: action.id,
|
|
183
|
+
description: action.description ?? action.type,
|
|
184
|
+
status: 'done',
|
|
185
|
+
action: { type: 'notify' }, // maps to pass-through postcondition
|
|
186
|
+
result: { status: 'completed' },
|
|
187
|
+
},
|
|
188
|
+
],
|
|
189
|
+
]),
|
|
190
|
+
};
|
|
191
|
+
const result = truthCheck_1.truthChecker.verify(fakeGraph, process.cwd());
|
|
192
|
+
return result.passed;
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
// Non-fatal — don't block execution on verify failure
|
|
196
|
+
return true;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
// ── Log ──────────────────────────────────────────────────────
|
|
200
|
+
getLog() { return this.actionLog; }
|
|
201
|
+
clearLog() { this.actionLog = []; }
|
|
202
|
+
}
|
|
203
|
+
exports.screenAgent = new ScreenAgent();
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ============================================================
|
|
3
|
+
// DevOS — Autonomous AI Execution System
|
|
4
|
+
// Copyright (c) 2026 Shiva Deore. All rights reserved.
|
|
5
|
+
// ============================================================
|
|
6
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
7
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
8
|
+
};
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.visionLoop = void 0;
|
|
11
|
+
// integrations/computerUse/visionLoop.ts
|
|
12
|
+
// Agentic vision-action loop: screenshot → vision LLM → action → verify → repeat.
|
|
13
|
+
//
|
|
14
|
+
// Model routing (DataGuard):
|
|
15
|
+
// local → llava:13b via Ollama REST (/api/chat with images[])
|
|
16
|
+
// claude → Anthropic Messages API (ANTHROPIC_API_KEY required)
|
|
17
|
+
// auto → DataGuard decides per screenshot: sensitive → local, otherwise cloud
|
|
18
|
+
//
|
|
19
|
+
// CommandGate approval before session start + low-confidence (< 0.65) escalation.
|
|
20
|
+
const axios_1 = __importDefault(require("axios"));
|
|
21
|
+
const screenAgent_1 = require("./screenAgent");
|
|
22
|
+
const apiRegistry_1 = require("./apiRegistry");
|
|
23
|
+
const commandGate_1 = require("../../coordination/commandGate");
|
|
24
|
+
const memoryLayers_1 = require("../../memory/memoryLayers");
|
|
25
|
+
const dataGuard_1 = require("../../security/dataGuard");
|
|
26
|
+
const executor_1 = require("../../core/executor");
|
|
27
|
+
const memoryStrategy_1 = require("../../core/memoryStrategy");
|
|
28
|
+
// ── Constants ──────────────────────────────────────────────────
|
|
29
|
+
const OLLAMA_BASE = process.env.OLLAMA_HOST ?? 'http://localhost:11434';
|
|
30
|
+
const VISION_MODEL = process.env.DEVOS_VISION_MODEL ?? 'llava:13b';
|
|
31
|
+
// ── Vision API helpers ────────────────────────────────────────
|
|
32
|
+
/**
|
|
33
|
+
* Call llava (or any vision-capable Ollama model) with a base64 image.
|
|
34
|
+
*/
|
|
35
|
+
async function callOllamaVision(prompt, imageB64) {
|
|
36
|
+
const res = await axios_1.default.post(`${OLLAMA_BASE}/api/chat`, {
|
|
37
|
+
model: VISION_MODEL,
|
|
38
|
+
messages: [
|
|
39
|
+
{
|
|
40
|
+
role: 'user',
|
|
41
|
+
content: prompt,
|
|
42
|
+
images: [imageB64],
|
|
43
|
+
},
|
|
44
|
+
],
|
|
45
|
+
stream: false,
|
|
46
|
+
options: { temperature: 0.2, num_predict: 512 },
|
|
47
|
+
}, { timeout: 90000 });
|
|
48
|
+
return res.data?.message?.content ?? '';
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Call Claude via Anthropic Messages API with a base64 PNG image.
|
|
52
|
+
* Requires ANTHROPIC_API_KEY env var.
|
|
53
|
+
*/
|
|
54
|
+
async function callClaudeVision(prompt, imageB64) {
|
|
55
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
56
|
+
if (!apiKey)
|
|
57
|
+
throw new Error('[VisionLoop] ANTHROPIC_API_KEY not set — use visionModel: "local"');
|
|
58
|
+
const res = await axios_1.default.post('https://api.anthropic.com/v1/messages', {
|
|
59
|
+
model: process.env.DEVOS_CLAUDE_VISION_MODEL ?? 'claude-sonnet-4-20250514',
|
|
60
|
+
max_tokens: 512,
|
|
61
|
+
messages: [
|
|
62
|
+
{
|
|
63
|
+
role: 'user',
|
|
64
|
+
content: [
|
|
65
|
+
{
|
|
66
|
+
type: 'image',
|
|
67
|
+
source: { type: 'base64', media_type: 'image/png', data: imageB64 },
|
|
68
|
+
},
|
|
69
|
+
{ type: 'text', text: prompt },
|
|
70
|
+
],
|
|
71
|
+
},
|
|
72
|
+
],
|
|
73
|
+
}, {
|
|
74
|
+
headers: {
|
|
75
|
+
'x-api-key': apiKey,
|
|
76
|
+
'anthropic-version': '2023-06-01',
|
|
77
|
+
'content-type': 'application/json',
|
|
78
|
+
},
|
|
79
|
+
timeout: 30000,
|
|
80
|
+
});
|
|
81
|
+
return res.data?.content?.[0]?.text ?? '';
|
|
82
|
+
}
|
|
83
|
+
// ── VisionLoop ────────────────────────────────────────────────
|
|
84
|
+
class VisionLoop {
|
|
85
|
+
constructor() {
|
|
86
|
+
this.aborted = false;
|
|
87
|
+
}
|
|
88
|
+
// ── Run ──────────────────────────────────────────────────────
|
|
89
|
+
async run(goal, options = {}) {
|
|
90
|
+
const maxIterations = options.maxIterations ?? 20;
|
|
91
|
+
const timeoutMs = options.timeoutMs ?? 120000;
|
|
92
|
+
const actionsExecuted = [];
|
|
93
|
+
const start = Date.now();
|
|
94
|
+
this.aborted = false;
|
|
95
|
+
// Memory lookup — replay known-good action sequence if available
|
|
96
|
+
const knownActions = memoryStrategy_1.memoryStrategy.retrieveActions(goal);
|
|
97
|
+
if (knownActions) {
|
|
98
|
+
console.log(`[VisionLoop] Memory hit for goal: "${goal}" — replaying ${knownActions.length} known action(s)`);
|
|
99
|
+
executor_1.executor.startSession(goal);
|
|
100
|
+
for (const action of knownActions) {
|
|
101
|
+
if (this.aborted)
|
|
102
|
+
break;
|
|
103
|
+
await executor_1.executor.execute(action);
|
|
104
|
+
actionsExecuted.push(action);
|
|
105
|
+
}
|
|
106
|
+
// Take a final screenshot to verify goal completion
|
|
107
|
+
const verifyScreenshot = await screenAgent_1.screenAgent.takeScreenshot();
|
|
108
|
+
const useLocalForVerify = options.visionModel === 'local'
|
|
109
|
+
? true
|
|
110
|
+
: options.visionModel === 'claude'
|
|
111
|
+
? false
|
|
112
|
+
: await dataGuard_1.dataGuard.isSensitive(verifyScreenshot.slice(0, 200));
|
|
113
|
+
const done = await this.checkGoalComplete(goal, verifyScreenshot, useLocalForVerify);
|
|
114
|
+
if (done) {
|
|
115
|
+
await memoryStrategy_1.memoryStrategy.storeSuccess(goal, actionsExecuted);
|
|
116
|
+
memoryLayers_1.memoryLayers.write(`ComputerUse success (memory replay): ${goal}`, ['computer_use', 'success']);
|
|
117
|
+
const session = executor_1.executor.endSession();
|
|
118
|
+
if (session)
|
|
119
|
+
memoryLayers_1.memoryLayers.write(JSON.stringify(session), ['computer_use', 'session']);
|
|
120
|
+
return { success: true, iterations: 1, actionsExecuted };
|
|
121
|
+
}
|
|
122
|
+
// Memory replay didn't complete goal — fall through to full vision loop
|
|
123
|
+
executor_1.executor.endSession();
|
|
124
|
+
actionsExecuted.length = 0;
|
|
125
|
+
this.aborted = false;
|
|
126
|
+
}
|
|
127
|
+
// CommandGate approval before taking control of the computer
|
|
128
|
+
if (options.requireApproval !== false) {
|
|
129
|
+
const approved = await commandGate_1.commandGate.requestApproval(`Computer control session: ${goal}`, 'VisionLoop will take screenshots and control mouse/keyboard');
|
|
130
|
+
if (!approved) {
|
|
131
|
+
executor_1.executor.endSession();
|
|
132
|
+
await memoryStrategy_1.memoryStrategy.storeFailure(goal);
|
|
133
|
+
return {
|
|
134
|
+
success: false,
|
|
135
|
+
iterations: 0,
|
|
136
|
+
actionsExecuted: [],
|
|
137
|
+
failureReason: 'Rejected by user via CommandGate',
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
executor_1.executor.startSession(goal);
|
|
142
|
+
for (let i = 0; i < maxIterations; i++) {
|
|
143
|
+
if (this.aborted)
|
|
144
|
+
break;
|
|
145
|
+
if (Date.now() - start > timeoutMs) {
|
|
146
|
+
await memoryStrategy_1.memoryStrategy.storeFailure(goal);
|
|
147
|
+
return { success: false, iterations: i, actionsExecuted, failureReason: 'Timeout' };
|
|
148
|
+
}
|
|
149
|
+
// 1. Take screenshot
|
|
150
|
+
const screenshotB64 = await screenAgent_1.screenAgent.takeScreenshot();
|
|
151
|
+
// 2. DataGuard: choose vision model
|
|
152
|
+
let useLocal;
|
|
153
|
+
if (options.visionModel === 'local') {
|
|
154
|
+
useLocal = true;
|
|
155
|
+
}
|
|
156
|
+
else if (options.visionModel === 'claude') {
|
|
157
|
+
useLocal = false;
|
|
158
|
+
}
|
|
159
|
+
else {
|
|
160
|
+
// 'auto' — scan a prefix of the base64 for sensitive text (e.g. from OCR metadata)
|
|
161
|
+
useLocal = await dataGuard_1.dataGuard.isSensitive(screenshotB64.slice(0, 200));
|
|
162
|
+
}
|
|
163
|
+
// 3. Vision LLM → next action (null = goal complete)
|
|
164
|
+
const action = await this.callVisionLLM(goal, screenshotB64, useLocal, actionsExecuted);
|
|
165
|
+
if (!action) {
|
|
166
|
+
await memoryStrategy_1.memoryStrategy.storeSuccess(goal, actionsExecuted);
|
|
167
|
+
memoryLayers_1.memoryLayers.write(`ComputerUse success: ${goal}`, ['computer_use', 'success']);
|
|
168
|
+
const session = executor_1.executor.endSession();
|
|
169
|
+
if (session)
|
|
170
|
+
memoryLayers_1.memoryLayers.write(JSON.stringify(session), ['computer_use', 'session']);
|
|
171
|
+
return { success: true, iterations: i + 1, actionsExecuted };
|
|
172
|
+
}
|
|
173
|
+
// 4. Low-confidence escalation
|
|
174
|
+
if (action.confidence < 0.65) {
|
|
175
|
+
const approved = await commandGate_1.commandGate.requestApproval(`Low-confidence action: ${action.type} (${Math.round(action.confidence * 100)}%)`, action.description ?? 'No description');
|
|
176
|
+
if (!approved)
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
// 5. API-first routing for api_call actions
|
|
180
|
+
if (action.type === 'api_call') {
|
|
181
|
+
const apiAction = action;
|
|
182
|
+
const r = await apiRegistry_1.apiRegistry.execute(apiAction.service, {
|
|
183
|
+
endpoint: apiAction.endpoint,
|
|
184
|
+
method: apiAction.method,
|
|
185
|
+
payload: apiAction.payload,
|
|
186
|
+
headers: apiAction.headers,
|
|
187
|
+
});
|
|
188
|
+
actionsExecuted.push(action);
|
|
189
|
+
if (r.usedAPI) {
|
|
190
|
+
await memoryStrategy_1.memoryStrategy.storeSuccess(goal, actionsExecuted);
|
|
191
|
+
memoryLayers_1.memoryLayers.write(`ComputerUse success: ${goal}`, ['computer_use', 'success']);
|
|
192
|
+
const session = executor_1.executor.endSession();
|
|
193
|
+
if (session)
|
|
194
|
+
memoryLayers_1.memoryLayers.write(JSON.stringify(session), ['computer_use', 'session']);
|
|
195
|
+
return { success: true, iterations: i + 1, actionsExecuted };
|
|
196
|
+
}
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
// 6. Execute via unified Executor (handles retry, fallback, TruthCheck)
|
|
200
|
+
const result = await executor_1.executor.execute(action);
|
|
201
|
+
actionsExecuted.push(action);
|
|
202
|
+
if (result.status === 'failed' && action.fallback) {
|
|
203
|
+
// Executor already tried the fallback internally; log the outcome
|
|
204
|
+
console.warn(`[VisionLoop] Action ${action.id} and its fallback both failed`);
|
|
205
|
+
}
|
|
206
|
+
// 7. Goal-complete check on fresh screenshot
|
|
207
|
+
const newScreenshot = await screenAgent_1.screenAgent.takeScreenshot();
|
|
208
|
+
const done = await this.checkGoalComplete(goal, newScreenshot, useLocal);
|
|
209
|
+
if (done) {
|
|
210
|
+
await memoryStrategy_1.memoryStrategy.storeSuccess(goal, actionsExecuted);
|
|
211
|
+
memoryLayers_1.memoryLayers.write(`ComputerUse success: ${goal}`, ['computer_use', 'success']);
|
|
212
|
+
const session = executor_1.executor.endSession();
|
|
213
|
+
if (session)
|
|
214
|
+
memoryLayers_1.memoryLayers.write(JSON.stringify(session), ['computer_use', 'session']);
|
|
215
|
+
return { success: true, iterations: i + 1, actionsExecuted };
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
await memoryStrategy_1.memoryStrategy.storeFailure(goal);
|
|
219
|
+
const session = executor_1.executor.endSession();
|
|
220
|
+
if (session)
|
|
221
|
+
memoryLayers_1.memoryLayers.write(JSON.stringify(session), ['computer_use', 'session']);
|
|
222
|
+
return {
|
|
223
|
+
success: false,
|
|
224
|
+
iterations: maxIterations,
|
|
225
|
+
actionsExecuted,
|
|
226
|
+
failureReason: this.aborted ? 'Aborted by user' : 'Max iterations reached',
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
// ── Vision LLM call ──────────────────────────────────────────
|
|
230
|
+
async callVisionLLM(goal, screenshotB64, useLocal, history) {
|
|
231
|
+
const historyStr = history
|
|
232
|
+
.slice(-5)
|
|
233
|
+
.map(a => `${a.type}: ${a.description ?? ''}`)
|
|
234
|
+
.join('\n');
|
|
235
|
+
const prompt = `You are controlling a computer to achieve this goal: "${goal}"
|
|
236
|
+
|
|
237
|
+
Recent actions taken:
|
|
238
|
+
${historyStr || 'None yet'}
|
|
239
|
+
|
|
240
|
+
Look at the screenshot and decide the SINGLE next action to take.
|
|
241
|
+
If the goal is already complete, respond with: { "done": true }
|
|
242
|
+
|
|
243
|
+
Respond ONLY with valid JSON:
|
|
244
|
+
{
|
|
245
|
+
"id": "action_<timestamp>",
|
|
246
|
+
"type": "click|type|scroll|keypress|screenshot|api_call",
|
|
247
|
+
"confidence": 0.0-1.0,
|
|
248
|
+
"description": "what this action does"
|
|
249
|
+
}`;
|
|
250
|
+
try {
|
|
251
|
+
const raw = useLocal
|
|
252
|
+
? await callOllamaVision(prompt, screenshotB64)
|
|
253
|
+
: await callClaudeVision(prompt, screenshotB64);
|
|
254
|
+
return this.parseAction(raw);
|
|
255
|
+
}
|
|
256
|
+
catch (err) {
|
|
257
|
+
console.warn(`[VisionLoop] LLM call failed (${useLocal ? 'local' : 'claude'}): ${err?.message}`);
|
|
258
|
+
return null;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
// ── Goal-complete check ───────────────────────────────────────
|
|
262
|
+
async checkGoalComplete(goal, screenshotB64, useLocal) {
|
|
263
|
+
const prompt = `Is this goal complete based on the screenshot? Goal: "${goal}". ` +
|
|
264
|
+
`Respond ONLY with JSON: {"complete": true} or {"complete": false}`;
|
|
265
|
+
try {
|
|
266
|
+
const raw = useLocal
|
|
267
|
+
? await callOllamaVision(prompt, screenshotB64)
|
|
268
|
+
: await callClaudeVision(prompt, screenshotB64);
|
|
269
|
+
const cleaned = raw.replace(/```json|```/g, '').trim();
|
|
270
|
+
return JSON.parse(cleaned).complete === true;
|
|
271
|
+
}
|
|
272
|
+
catch {
|
|
273
|
+
return false;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
// ── Parse LLM response → action ──────────────────────────────
|
|
277
|
+
parseAction(raw) {
|
|
278
|
+
try {
|
|
279
|
+
const cleaned = raw.replace(/```json|```/g, '').trim();
|
|
280
|
+
const parsed = JSON.parse(cleaned);
|
|
281
|
+
if (parsed.done)
|
|
282
|
+
return null;
|
|
283
|
+
if (!parsed.id)
|
|
284
|
+
parsed.id = `action_${Date.now()}`;
|
|
285
|
+
if (typeof parsed.confidence !== 'number')
|
|
286
|
+
parsed.confidence = 0.7;
|
|
287
|
+
return parsed;
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
return null;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
// ── Abort ─────────────────────────────────────────────────────
|
|
294
|
+
abort() { this.aborted = true; }
|
|
295
|
+
}
|
|
296
|
+
exports.visionLoop = new VisionLoop();
|