@elizaos/plugin-vision 1.2.1 → 2.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/LICENSE +21 -0
  2. package/build.config.ts +53 -53
  3. package/dist/index.js +6716 -67
  4. package/dist/index.js.map +33 -1
  5. package/dist/workers/florence2-worker.js +111763 -307
  6. package/dist/workers/florence2-worker.js.map +92 -1
  7. package/dist/workers/ocr-worker.js +119177 -339
  8. package/dist/workers/ocr-worker.js.map +137 -1
  9. package/dist/workers/screen-capture-worker.js +350 -418
  10. package/dist/workers/screen-capture-worker.js.map +11 -1
  11. package/package.json +15 -20
  12. package/README.md +0 -270
  13. package/dist/action.d.ts +0 -8
  14. package/dist/action.js +0 -1212
  15. package/dist/action.js.map +0 -1
  16. package/dist/audio-capture-stream.d.ts +0 -42
  17. package/dist/audio-capture-stream.js +0 -516
  18. package/dist/audio-capture-stream.js.map +0 -1
  19. package/dist/audio-capture.d.ts +0 -25
  20. package/dist/audio-capture.js +0 -412
  21. package/dist/audio-capture.js.map +0 -1
  22. package/dist/basic.test.d.ts +0 -1
  23. package/dist/basic.test.js +0 -97
  24. package/dist/basic.test.js.map +0 -1
  25. package/dist/config.d.ts +0 -73
  26. package/dist/config.js +0 -254
  27. package/dist/config.js.map +0 -1
  28. package/dist/entity-tracker.d.ts +0 -32
  29. package/dist/entity-tracker.js +0 -361
  30. package/dist/entity-tracker.js.map +0 -1
  31. package/dist/errors.d.ts +0 -67
  32. package/dist/errors.js +0 -395
  33. package/dist/errors.js.map +0 -1
  34. package/dist/face-recognition.d.ts +0 -31
  35. package/dist/face-recognition.js +0 -332
  36. package/dist/face-recognition.js.map +0 -1
  37. package/dist/florence2-local.d.ts +0 -25
  38. package/dist/florence2-local.js +0 -280
  39. package/dist/florence2-local.js.map +0 -1
  40. package/dist/florence2-model.d.ts +0 -36
  41. package/dist/florence2-model.js +0 -503
  42. package/dist/florence2-model.js.map +0 -1
  43. package/dist/index.d.ts +0 -3
  44. package/dist/ocr-service-real.d.ts +0 -32
  45. package/dist/ocr-service-real.js +0 -396
  46. package/dist/ocr-service-real.js.map +0 -1
  47. package/dist/ocr-service.d.ts +0 -28
  48. package/dist/ocr-service.js +0 -216
  49. package/dist/ocr-service.js.map +0 -1
  50. package/dist/provider.d.ts +0 -2
  51. package/dist/provider.js +0 -285
  52. package/dist/provider.js.map +0 -1
  53. package/dist/screen-capture.d.ts +0 -16
  54. package/dist/screen-capture.js +0 -302
  55. package/dist/screen-capture.js.map +0 -1
  56. package/dist/service.d.ts +0 -73
  57. package/dist/service.js +0 -1662
  58. package/dist/service.js.map +0 -1
  59. package/dist/tests/e2e/index.d.ts +0 -8
  60. package/dist/tests/e2e/index.js +0 -33
  61. package/dist/tests/e2e/index.js.map +0 -1
  62. package/dist/tests/e2e/run-local.d.ts +0 -2
  63. package/dist/tests/e2e/run-local.js +0 -166
  64. package/dist/tests/e2e/run-local.js.map +0 -1
  65. package/dist/tests/e2e/screen-vision.d.ts +0 -11
  66. package/dist/tests/e2e/screen-vision.js +0 -384
  67. package/dist/tests/e2e/screen-vision.js.map +0 -1
  68. package/dist/tests/e2e/vision-autonomy.d.ts +0 -11
  69. package/dist/tests/e2e/vision-autonomy.js +0 -375
  70. package/dist/tests/e2e/vision-autonomy.js.map +0 -1
  71. package/dist/tests/e2e/vision-basic.d.ts +0 -11
  72. package/dist/tests/e2e/vision-basic.js +0 -434
  73. package/dist/tests/e2e/vision-basic.js.map +0 -1
  74. package/dist/tests/e2e/vision-capture-log.d.ts +0 -11
  75. package/dist/tests/e2e/vision-capture-log.js +0 -302
  76. package/dist/tests/e2e/vision-capture-log.js.map +0 -1
  77. package/dist/tests/e2e/vision-runtime.d.ts +0 -11
  78. package/dist/tests/e2e/vision-runtime.js +0 -357
  79. package/dist/tests/e2e/vision-runtime.js.map +0 -1
  80. package/dist/tests/e2e/vision-worker-tests.d.ts +0 -11
  81. package/dist/tests/e2e/vision-worker-tests.js +0 -466
  82. package/dist/tests/e2e/vision-worker-tests.js.map +0 -1
  83. package/dist/tests/test-pattern-generator.d.ts +0 -40
  84. package/dist/tests/test-pattern-generator.js +0 -191
  85. package/dist/tests/test-pattern-generator.js.map +0 -1
  86. package/dist/tests.d.ts +0 -3
  87. package/dist/tests.js +0 -11
  88. package/dist/tests.js.map +0 -1
  89. package/dist/types.d.ts +0 -222
  90. package/dist/types.js +0 -16
  91. package/dist/types.js.map +0 -1
  92. package/dist/vision-models.d.ts +0 -47
  93. package/dist/vision-models.js +0 -501
  94. package/dist/vision-models.js.map +0 -1
  95. package/dist/vision-worker-manager.d.ts +0 -61
  96. package/dist/vision-worker-manager.js +0 -668
  97. package/dist/vision-worker-manager.js.map +0 -1
  98. package/dist/workers/florence2-worker-simple.d.ts +0 -13
  99. package/dist/workers/florence2-worker-simple.js +0 -121
  100. package/dist/workers/florence2-worker-simple.js.map +0 -1
  101. package/dist/workers/florence2-worker.d.ts +0 -1
  102. package/dist/workers/ocr-worker.d.ts +0 -1
  103. package/dist/workers/screen-capture-worker.d.ts +0 -1
  104. package/dist/workers/worker-logger.d.ts +0 -9
  105. package/dist/workers/worker-logger.js +0 -95
  106. package/dist/workers/worker-logger.js.map +0 -1
@@ -1,375 +0,0 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __generator = (this && this.__generator) || function (thisArg, body) {
12
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
13
- return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14
- function verb(n) { return function (v) { return step([n, v]); }; }
15
- function step(op) {
16
- if (f) throw new TypeError("Generator is already executing.");
17
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
18
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19
- if (y = 0, t) op = [op[0] & 2, t.value];
20
- switch (op[0]) {
21
- case 0: case 1: t = op; break;
22
- case 4: _.label++; return { value: op[1], done: false };
23
- case 5: _.label++; y = op[1]; op = [0]; continue;
24
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
25
- default:
26
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30
- if (t[2]) _.ops.pop();
31
- _.trys.pop(); continue;
32
- }
33
- op = body.call(thisArg, _);
34
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36
- }
37
- };
38
- Object.defineProperty(exports, "__esModule", { value: true });
39
- exports.VisionAutonomyE2ETestSuite = void 0;
40
- var core_1 = require("@elizaos/core");
41
- var action_1 = require("../../action");
42
- var VisionAutonomyE2ETestSuite = /** @class */ (function () {
43
- function VisionAutonomyE2ETestSuite() {
44
- var _this = this;
45
- this.name = 'plugin-vision-autonomy-e2e';
46
- this.description = 'Tests for vision plugin integration with autonomy plugin';
47
- this.tests = [
48
- {
49
- name: 'Should stop autonomous loop with kill command',
50
- fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
51
- var roomId, message, callbackCalled, callbackResponse, isValid;
52
- var _this = this;
53
- return __generator(this, function (_a) {
54
- switch (_a.label) {
55
- case 0:
56
- console.log('Testing kill autonomous action...');
57
- roomId = (0, core_1.createUniqueUuid)(runtime, 'test-room');
58
- message = {
59
- id: (0, core_1.createUniqueUuid)(runtime, 'test-msg-kill'),
60
- entityId: runtime.agentId,
61
- content: { text: 'kill the autonomous loop' },
62
- agentId: runtime.agentId,
63
- roomId: roomId,
64
- createdAt: Date.now(),
65
- };
66
- callbackCalled = false;
67
- callbackResponse = null;
68
- return [4 /*yield*/, action_1.killAutonomousAction.validate(runtime, message, {
69
- values: {},
70
- data: {},
71
- text: '',
72
- })];
73
- case 1:
74
- isValid = _a.sent();
75
- if (!isValid) {
76
- throw new Error('killAutonomousAction validation failed');
77
- }
78
- return [4 /*yield*/, action_1.killAutonomousAction.handler(runtime, message, { values: {}, data: {}, text: '' }, {}, function (response) { return __awaiter(_this, void 0, void 0, function () {
79
- return __generator(this, function (_a) {
80
- callbackCalled = true;
81
- callbackResponse = response;
82
- return [2 /*return*/, []];
83
- });
84
- }); })];
85
- case 2:
86
- _a.sent();
87
- if (!callbackCalled) {
88
- throw new Error('Callback was not called - action handler failed');
89
- }
90
- if (!callbackResponse || !callbackResponse.text) {
91
- throw new Error('No response text returned from kill action');
92
- }
93
- console.log('✓ Kill autonomous action executed');
94
- console.log(" Response: ".concat(callbackResponse.text));
95
- if (callbackResponse.thought) {
96
- console.log(" Thought: ".concat(callbackResponse.thought));
97
- }
98
- // Verify action was included
99
- if (!callbackResponse.actions || !callbackResponse.actions.includes('KILL_AUTONOMOUS')) {
100
- throw new Error('Response does not include KILL_AUTONOMOUS action');
101
- }
102
- return [2 /*return*/];
103
- }
104
- });
105
- }); },
106
- },
107
- {
108
- name: 'Should provide continuous vision updates for autonomous agent',
109
- fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
110
- var visionService, updateIntervals, lastTimestamp, updateCount, testDuration, startTime, scene, avgInterval;
111
- return __generator(this, function (_a) {
112
- switch (_a.label) {
113
- case 0:
114
- console.log('Testing vision updates for autonomous behavior...');
115
- visionService = runtime.getService('VISION');
116
- if (!visionService) {
117
- throw new Error('Vision service not available');
118
- }
119
- if (!visionService.isActive()) {
120
- console.warn('⚠️ Vision service not active - skipping continuous update test');
121
- console.log(' This is acceptable in environments without cameras');
122
- return [2 /*return*/];
123
- }
124
- updateIntervals = [];
125
- lastTimestamp = 0;
126
- updateCount = 0;
127
- testDuration = 5000;
128
- startTime = Date.now();
129
- console.log(' Monitoring scene updates for 5 seconds...');
130
- _a.label = 1;
131
- case 1:
132
- if (!(Date.now() - startTime < testDuration)) return [3 /*break*/, 4];
133
- return [4 /*yield*/, visionService.getSceneDescription()];
134
- case 2:
135
- scene = _a.sent();
136
- if (scene && scene.timestamp !== lastTimestamp) {
137
- if (lastTimestamp > 0) {
138
- updateIntervals.push(scene.timestamp - lastTimestamp);
139
- }
140
- lastTimestamp = scene.timestamp;
141
- updateCount++;
142
- console.log(" Scene update ".concat(updateCount, ": ").concat(scene.description.substring(0, 50), "..."));
143
- }
144
- return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 100); })];
145
- case 3:
146
- _a.sent();
147
- return [3 /*break*/, 1];
148
- case 4:
149
- console.log('✓ Vision monitoring complete');
150
- console.log(" Total updates: ".concat(updateCount));
151
- if (updateCount === 0) {
152
- throw new Error('No scene updates detected during 5 second monitoring period');
153
- }
154
- if (updateIntervals.length > 0) {
155
- avgInterval = updateIntervals.reduce(function (a, b) { return a + b; }, 0) / updateIntervals.length;
156
- console.log(" Average update interval: ".concat(Math.round(avgInterval), "ms"));
157
- // Verify updates are happening at reasonable intervals
158
- if (avgInterval > 5000) {
159
- throw new Error('Scene updates too infrequent for autonomous operation');
160
- }
161
- }
162
- return [2 /*return*/];
163
- }
164
- });
165
- }); },
166
- },
167
- {
168
- name: 'Should maintain vision memory across interactions',
169
- fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
170
- var visionService, roomId, firstMessage, firstResponse, secondMessage, secondResponse, messages;
171
- return __generator(this, function (_a) {
172
- switch (_a.label) {
173
- case 0:
174
- console.log('Testing vision memory persistence...');
175
- visionService = runtime.getService('VISION');
176
- if (!visionService) {
177
- throw new Error('Vision service not available');
178
- }
179
- roomId = (0, core_1.createUniqueUuid)(runtime, 'test-room');
180
- firstMessage = {
181
- id: (0, core_1.createUniqueUuid)(runtime, 'test-msg-1'),
182
- entityId: runtime.agentId,
183
- content: { text: 'what do you see?' },
184
- agentId: runtime.agentId,
185
- roomId: roomId,
186
- createdAt: Date.now(),
187
- };
188
- // Store first message
189
- return [4 /*yield*/, runtime.createMemory(firstMessage, 'messages')];
190
- case 1:
191
- // Store first message
192
- _a.sent();
193
- firstResponse = {
194
- id: (0, core_1.createUniqueUuid)(runtime, 'test-response-1'),
195
- entityId: runtime.agentId,
196
- content: {
197
- text: 'I see a test scene',
198
- actions: ['DESCRIBE_SCENE'],
199
- },
200
- agentId: runtime.agentId,
201
- roomId: roomId,
202
- createdAt: Date.now() + 100,
203
- };
204
- return [4 /*yield*/, runtime.createMemory(firstResponse, 'messages')];
205
- case 2:
206
- _a.sent();
207
- // Wait for processing
208
- return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 500); })];
209
- case 3:
210
- // Wait for processing
211
- _a.sent();
212
- secondMessage = {
213
- id: (0, core_1.createUniqueUuid)(runtime, 'test-msg-2'),
214
- entityId: runtime.agentId,
215
- content: { text: 'what did you see before?' },
216
- agentId: runtime.agentId,
217
- roomId: roomId,
218
- createdAt: Date.now() + 1000,
219
- };
220
- return [4 /*yield*/, runtime.createMemory(secondMessage, 'messages')];
221
- case 4:
222
- _a.sent();
223
- secondResponse = {
224
- id: (0, core_1.createUniqueUuid)(runtime, 'test-response-2'),
225
- entityId: runtime.agentId,
226
- content: {
227
- text: 'Previously, I saw a test scene',
228
- actions: ['DESCRIBE_SCENE'],
229
- },
230
- agentId: runtime.agentId,
231
- roomId: roomId,
232
- createdAt: Date.now() + 1100,
233
- };
234
- return [4 /*yield*/, runtime.createMemory(secondResponse, 'messages')];
235
- case 5:
236
- _a.sent();
237
- // Wait for processing
238
- return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 500); })];
239
- case 6:
240
- // Wait for processing
241
- _a.sent();
242
- return [4 /*yield*/, runtime.getMemories({
243
- roomId: roomId,
244
- count: 10,
245
- tableName: 'messages',
246
- })];
247
- case 7:
248
- messages = _a.sent();
249
- console.log('✓ Vision memory test complete');
250
- console.log(" Total messages: ".concat(messages ? messages.length : 0));
251
- // In the minimal test runtime, we just verify memory creation works
252
- // Real runtime would have actual message processing
253
- if (!visionService.isActive()) {
254
- console.log(' Note: Vision service not active, simulated memory test only');
255
- }
256
- return [2 /*return*/];
257
- }
258
- });
259
- }); },
260
- },
261
- {
262
- name: 'Should integrate vision data with agent decision making',
263
- fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
264
- var roomId, scenarios, scenariosWithVision, _i, scenarios_1, scenario, message, state, hasVisionContext;
265
- return __generator(this, function (_a) {
266
- switch (_a.label) {
267
- case 0:
268
- console.log('Testing vision-based decision making...');
269
- roomId = (0, core_1.createUniqueUuid)(runtime, 'test-room');
270
- scenarios = [
271
- { text: 'Is anyone in the room?', expectedContext: 'people' },
272
- { text: 'Should I turn on the lights?', expectedContext: 'scene' },
273
- { text: 'What objects are nearby?', expectedContext: 'objects' },
274
- ];
275
- scenariosWithVision = 0;
276
- _i = 0, scenarios_1 = scenarios;
277
- _a.label = 1;
278
- case 1:
279
- if (!(_i < scenarios_1.length)) return [3 /*break*/, 4];
280
- scenario = scenarios_1[_i];
281
- message = {
282
- id: (0, core_1.createUniqueUuid)(runtime, "test-msg-".concat(Date.now())),
283
- entityId: runtime.agentId,
284
- content: { text: scenario.text },
285
- agentId: runtime.agentId,
286
- roomId: roomId,
287
- createdAt: Date.now(),
288
- };
289
- return [4 /*yield*/, runtime.composeState(message)];
290
- case 2:
291
- state = _a.sent();
292
- hasVisionContext = state.text.includes('Visual Perception') || state.values.visionAvailable !== undefined;
293
- console.log(" Scenario: \"".concat(scenario.text, "\""));
294
- console.log(" Has vision context: ".concat(hasVisionContext));
295
- if (hasVisionContext) {
296
- scenariosWithVision++;
297
- if (state.values.sceneDescription) {
298
- console.log(" Scene info available: ".concat(state.values.sceneDescription.substring(0, 50), "..."));
299
- }
300
- }
301
- _a.label = 3;
302
- case 3:
303
- _i++;
304
- return [3 /*break*/, 1];
305
- case 4:
306
- console.log('✓ Vision-based decision making test complete');
307
- // All scenarios should have vision context from the provider
308
- if (scenariosWithVision !== scenarios.length) {
309
- throw new Error("Vision context missing in ".concat(scenarios.length - scenariosWithVision, " scenarios"));
310
- }
311
- return [2 /*return*/];
312
- }
313
- });
314
- }); },
315
- },
316
- {
317
- name: 'Should handle autonomy gracefully when vision is unavailable',
318
- fn: function (runtime) { return __awaiter(_this, void 0, void 0, function () {
319
- var visionService, isActive, message, state;
320
- return __generator(this, function (_a) {
321
- switch (_a.label) {
322
- case 0:
323
- console.log('Testing autonomy behavior without vision...');
324
- visionService = runtime.getService('VISION');
325
- if (!visionService) {
326
- throw new Error('Vision service not registered - cannot test graceful handling');
327
- }
328
- isActive = visionService.isActive();
329
- console.log(" Vision service active: ".concat(isActive));
330
- message = {
331
- id: (0, core_1.createUniqueUuid)(runtime, 'test-msg-no-vision'),
332
- entityId: runtime.agentId,
333
- content: { text: 'test without vision' },
334
- agentId: runtime.agentId,
335
- roomId: (0, core_1.createUniqueUuid)(runtime, 'test-room'),
336
- createdAt: Date.now(),
337
- };
338
- return [4 /*yield*/, runtime.composeState(message)];
339
- case 1:
340
- state = _a.sent();
341
- // Vision provider should always provide status
342
- if (state.values.visionAvailable === undefined) {
343
- throw new Error('Vision provider did not report availability status');
344
- }
345
- if (!isActive) {
346
- // Vision should indicate it's not available
347
- if (state.values.visionAvailable !== false) {
348
- throw new Error('Vision incorrectly reports as available when service is not active');
349
- }
350
- if (!state.values.cameraStatus || !state.values.cameraStatus.includes('not connected')) {
351
- throw new Error('Camera status does not indicate disconnection');
352
- }
353
- console.log('✓ Vision correctly reports unavailable state');
354
- console.log(" Status: ".concat(state.values.cameraStatus));
355
- }
356
- else {
357
- // Vision is active
358
- if (state.values.visionAvailable !== true) {
359
- throw new Error('Vision incorrectly reports as unavailable when service is active');
360
- }
361
- console.log('✓ Vision correctly reports available state');
362
- console.log(" Status: ".concat(state.values.cameraStatus));
363
- }
364
- return [2 /*return*/];
365
- }
366
- });
367
- }); },
368
- },
369
- ];
370
- }
371
- return VisionAutonomyE2ETestSuite;
372
- }());
373
- exports.VisionAutonomyE2ETestSuite = VisionAutonomyE2ETestSuite;
374
- exports.default = new VisionAutonomyE2ETestSuite();
375
- //# sourceMappingURL=vision-autonomy.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"vision-autonomy.js","sourceRoot":"","sources":["../../../src/tests/e2e/vision-autonomy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,sCAAiD;AAEjD,uCAAoD;AAEpD;IAAA;QAAA,iBAwUC;QAvUC,SAAI,GAAG,4BAA4B,CAAC;QACpC,gBAAW,GAAG,0DAA0D,CAAC;QAEzE,UAAK,GAAG;YACN;gBACE,IAAI,EAAE,+CAA+C;gBACrD,EAAE,EAAE,UAAO,OAAsB;;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;gCAE3C,MAAM,GAAG,IAAA,uBAAgB,EAAC,OAAO,EAAE,WAAW,CAAC,CAAC;gCAChD,OAAO,GAAW;oCACtB,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,eAAe,CAAC;oCAC9C,QAAQ,EAAE,OAAO,CAAC,OAAO;oCACzB,OAAO,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE;oCAC7C,OAAO,EAAE,OAAO,CAAC,OAAO;oCACxB,MAAM,QAAA;oCACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iCACtB,CAAC;gCAEE,cAAc,GAAG,KAAK,CAAC;gCACvB,gBAAgB,GAAQ,IAAI,CAAC;gCAGjB,qBAAM,6BAAoB,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE;wCACpE,MAAM,EAAE,EAAE;wCACV,IAAI,EAAE,EAAE;wCACR,IAAI,EAAE,EAAE;qCACT,CAAC,EAAA;;gCAJI,OAAO,GAAG,SAId;gCACF,IAAI,CAAC,OAAO,EAAE,CAAC;oCACb,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;gCAC5D,CAAC;gCAED,qBAAM,6BAAoB,CAAC,OAAO,CAChC,OAAO,EACP,OAAO,EACP,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAClC,EAAE,EACF,UAAO,QAAQ;;4CACb,cAAc,GAAG,IAAI,CAAC;4CACtB,gBAAgB,GAAG,QAAQ,CAAC;4CAC5B,sBAAO,EAAE,EAAC;;yCACX,CACF,EAAA;;gCAVD,SAUC,CAAC;gCAEF,IAAI,CAAC,cAAc,EAAE,CAAC;oCACpB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;gCACrE,CAAC;gCAED,IAAI,CAAC,gBAAgB,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;oCAChD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;gCAChE,CAAC;gCAED,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;gCACjD,OAAO,CAAC,GAAG,CAAC,sBAAe,gBAAgB,CAAC,IAAI,CAAE,CAAC,CAAC;gCAEpD,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;oCAC7B,OAAO,CAAC,GAAG,CAAC,qBAAc,gBAAgB,CAAC,OAAO,CAAE,CAAC,CAAC;gCACxD,CAAC;gCAED,6BAA6B;gCAC7B,IAAI,CAAC,gBAAgB,CAAC,OAAO,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;oCACvF,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;gCACtE,CAAC;;;;qBACF;aACF;YAED;gBACE,IAAI,EAAE,+DAA+D;gBACrE,EAAE,EAAE,UAAO,OAAsB;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,mDAAmD,CAAC,CAAC;gCAE3D,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAED,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;oCAC9B,OAAO,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;oCAChF,OAAO,CAAC,GAAG,CAAC,uDAAuD,CAAC,CAAC;oCACrE,sBAAO;gCACT,CAAC;gCAGK,eAAe,GAAa,EAAE,CAAC;gCACjC,aAAa,GAAG,CAAC,CAAC;gCAClB,WAAW,GAAG,CAAC,CAAC;gCACd,YAAY,GAAG,IAAI,CAAC;gCACpB,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gCAE7B,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;;;qCAEpD,CAAA,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,YAAY,CAAA;gCAC5B,qBAAM,aAAa,CAAC,mBAAmB,EAAE,EAAA;;gCAAjD,KAAK,GAAG,SAAyC;gCAEvD,IAAI,KAAK,IAAI,KAAK,CAAC,SAAS,KAAK,aAAa,EAAE,CAAC;oCAC/C,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;wCACtB,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,aAAa,CAAC,CAAC;oCACxD,CAAC;oCACD,aAAa,GAAG,KAAK,CAAC,SAAS,CAAC;oCAChC,WAAW,EAAE,CAAC;oCACd,OAAO,CAAC,GAAG,CAAC,yBAAkB,WAAW,eAAK,KAAK,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,QAAK,CAAC,CAAC;gCACzF,CAAC;gCAED,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,EAAxB,CAAwB,CAAC,EAAA;;gCAAxD,SAAwD,CAAC;;;gCAG3D,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;gCAC5C,OAAO,CAAC,GAAG,CAAC,2BAAoB,WAAW,CAAE,CAAC,CAAC;gCAE/C,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;oCACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;gCACjF,CAAC;gCAED,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oCACzB,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,UAAC,CAAC,EAAE,CAAC,IAAK,OAAA,CAAC,GAAG,CAAC,EAAL,CAAK,EAAE,CAAC,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC;oCACxF,OAAO,CAAC,GAAG,CAAC,qCAA8B,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,OAAI,CAAC,CAAC;oCAEvE,uDAAuD;oCACvD,IAAI,WAAW,GAAG,IAAI,EAAE,CAAC;wCACvB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;oCAC3E,CAAC;gCACH,CAAC;;;;qBACF;aACF;YAED;gBACE,IAAI,EAAE,mDAAmD;gBACzD,EAAE,EAAE,UAAO,OAAsB;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,sCAAsC,CAAC,CAAC;gCAE9C,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAClE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;gCAClD,CAAC;gCAGK,MAAM,GAAG,IAAA,uBAAgB,EAAC,OAAO,EAAE,WAAW,CAAC,CAAC;gCAChD,YAAY,GAAW;oCAC3B,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,YAAY,CAAC;oCAC3C,QAAQ,EAAE,OAAO,CAAC,OAAO;oCACzB,OAAO,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;oCACrC,OAAO,EAAE,OAAO,CAAC,OAAO;oCACxB,MAAM,QAAA;oCACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iCACtB,CAAC;gCAEF,sBAAsB;gCACtB,qBAAM,OAAO,CAAC,YAAY,CAAC,YAAY,EAAE,UAAU,CAAC,EAAA;;gCADpD,sBAAsB;gCACtB,SAAoD,CAAC;gCAG/C,aAAa,GAAW;oCAC5B,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,iBAAiB,CAAC;oCAChD,QAAQ,EAAE,OAAO,CAAC,OAAO;oCACzB,OAAO,EAAE;wCACP,IAAI,EAAE,oBAAoB;wCAC1B,OAAO,EAAE,CAAC,gBAAgB,CAAC;qCAC5B;oCACD,OAAO,EAAE,OAAO,CAAC,OAAO;oCACxB,MAAM,QAAA;oCACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG;iCAC5B,CAAC;gCACF,qBAAM,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,UAAU,CAAC,EAAA;;gCAArD,SAAqD,CAAC;gCAEtD,sBAAsB;gCACtB,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,EAAxB,CAAwB,CAAC,EAAA;;gCADxD,sBAAsB;gCACtB,SAAwD,CAAC;gCAGnD,aAAa,GAAW;oCAC5B,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,YAAY,CAAC;oCAC3C,QAAQ,EAAE,OAAO,CAAC,OAAO;oCACzB,OAAO,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE;oCAC7C,OAAO,EAAE,OAAO,CAAC,OAAO;oCACxB,MAAM,QAAA;oCACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;iCAC7B,CAAC;gCAEF,qBAAM,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,UAAU,CAAC,EAAA;;gCAArD,SAAqD,CAAC;gCAGhD,cAAc,GAAW;oCAC7B,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,iBAAiB,CAAC;oCAChD,QAAQ,EAAE,OAAO,CAAC,OAAO;oCACzB,OAAO,EAAE;wCACP,IAAI,EAAE,gCAAgC;wCACtC,OAAO,EAAE,CAAC,gBAAgB,CAAC;qCAC5B;oCACD,OAAO,EAAE,OAAO,CAAC,OAAO;oCACxB,MAAM,QAAA;oCACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;iCAC7B,CAAC;gCACF,qBAAM,OAAO,CAAC,YAAY,CAAC,cAAc,EAAE,UAAU,CAAC,EAAA;;gCAAtD,SAAsD,CAAC;gCAEvD,sBAAsB;gCACtB,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,EAAxB,CAAwB,CAAC,EAAA;;gCADxD,sBAAsB;gCACtB,SAAwD,CAAC;gCAGxC,qBAAM,OAAO,CAAC,WAAW,CAAC;wCACzC,MAAM,QAAA;wCACN,KAAK,EAAE,EAAE;wCACT,SAAS,EAAE,UAAU;qCACtB,CAAC,EAAA;;gCAJI,QAAQ,GAAG,SAIf;gCAEF,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;gCAC7C,OAAO,CAAC,GAAG,CAAC,4BAAqB,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;gCAEnE,oEAAoE;gCACpE,oDAAoD;gCACpD,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC;oCAC9B,OAAO,CAAC,GAAG,CAAC,+DAA+D,CAAC,CAAC;gCAC/E,CAAC;;;;qBACF;aACF;YAED;gBACE,IAAI,EAAE,yDAAyD;gBAC/D,EAAE,EAAE,UAAO,OAAsB;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;gCAGjD,MAAM,GAAG,IAAA,uBAAgB,EAAC,OAAO,EAAE,WAAW,CAAC,CAAC;gCAGhD,SAAS,GAAG;oCAChB,EAAE,IAAI,EAAE,wBAAwB,EAAE,eAAe,EAAE,QAAQ,EAAE;oCAC7D,EAAE,IAAI,EAAE,8BAA8B,EAAE,eAAe,EAAE,OAAO,EAAE;oCAClE,EAAE,IAAI,EAAE,0BAA0B,EAAE,eAAe,EAAE,SAAS,EAAE;iCACjE,CAAC;gCAEE,mBAAmB,GAAG,CAAC,CAAC;sCAEI,EAAT,uBAAS;;;qCAAT,CAAA,uBAAS,CAAA;gCAArB,QAAQ;gCACX,OAAO,GAAW;oCACtB,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,mBAAY,IAAI,CAAC,GAAG,EAAE,CAAE,CAAC;oCACvD,QAAQ,EAAE,OAAO,CAAC,OAAO;oCACzB,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE;oCAChC,OAAO,EAAE,OAAO,CAAC,OAAO;oCACxB,MAAM,QAAA;oCACN,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iCACtB,CAAC;gCAGY,qBAAM,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,EAAA;;gCAA3C,KAAK,GAAG,SAAmC;gCAG3C,gBAAgB,GACpB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,mBAAmB,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,eAAe,KAAK,SAAS,CAAC;gCAEzF,OAAO,CAAC,GAAG,CAAC,wBAAgB,QAAQ,CAAC,IAAI,OAAG,CAAC,CAAC;gCAC9C,OAAO,CAAC,GAAG,CAAC,kCAA2B,gBAAgB,CAAE,CAAC,CAAC;gCAE3D,IAAI,gBAAgB,EAAE,CAAC;oCACrB,mBAAmB,EAAE,CAAC;oCAEtB,IAAI,KAAK,CAAC,MAAM,CAAC,gBAAgB,EAAE,CAAC;wCAClC,OAAO,CAAC,GAAG,CACT,oCAA6B,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,QAAK,CACjF,CAAC;oCACJ,CAAC;gCACH,CAAC;;;gCA5BoB,IAAS,CAAA;;;gCA+BhC,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;gCAE5D,6DAA6D;gCAC7D,IAAI,mBAAmB,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;oCAC7C,MAAM,IAAI,KAAK,CACb,oCAA6B,SAAS,CAAC,MAAM,GAAG,mBAAmB,eAAY,CAChF,CAAC;gCACJ,CAAC;;;;qBACF;aACF;YAED;gBACE,IAAI,EAAE,8DAA8D;gBACpE,EAAE,EAAE,UAAO,OAAsB;;;;;gCAC/B,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;gCAErD,aAAa,GAAG,OAAO,CAAC,UAAU,CAAgB,QAAQ,CAAC,CAAC;gCAElE,IAAI,CAAC,aAAa,EAAE,CAAC;oCACnB,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;gCACnF,CAAC;gCAEK,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,CAAC;gCAC1C,OAAO,CAAC,GAAG,CAAC,mCAA4B,QAAQ,CAAE,CAAC,CAAC;gCAG9C,OAAO,GAAW;oCACtB,EAAE,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,oBAAoB,CAAC;oCACnD,QAAQ,EAAE,OAAO,CAAC,OAAO;oCACzB,OAAO,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;oCACxC,OAAO,EAAE,OAAO,CAAC,OAAO;oCACxB,MAAM,EAAE,IAAA,uBAAgB,EAAC,OAAO,EAAE,WAAW,CAAC;oCAC9C,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iCACtB,CAAC;gCAEY,qBAAM,OAAO,CAAC,YAAY,CAAC,OAAO,CAAC,EAAA;;gCAA3C,KAAK,GAAG,SAAmC;gCAEjD,+CAA+C;gCAC/C,IAAI,KAAK,CAAC,MAAM,CAAC,eAAe,KAAK,SAAS,EAAE,CAAC;oCAC/C,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;gCACxE,CAAC;gCAED,IAAI,CAAC,QAAQ,EAAE,CAAC;oCACd,4CAA4C;oCAC5C,IAAI,KAAK,CAAC,MAAM,CAAC,eAAe,KAAK,KAAK,EAAE,CAAC;wCAC3C,MAAM,IAAI,KAAK,CAAC,oEAAoE,CAAC,CAAC;oCACxF,CAAC;oCAED,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;wCACvF,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;oCACnE,CAAC;oCAED,OAAO,CAAC,GAAG,CAAC,8CAA8C,CAAC,CAAC;oCAC5D,OAAO,CAAC,GAAG,CAAC,oBAAa,KAAK,CAAC,MAAM,CAAC,YAAY,CAAE,CAAC,CAAC;gCACxD,CAAC;qCAAM,CAAC;oCACN,mBAAmB;oCACnB,IAAI,KAAK,CAAC,MAAM,CAAC,eAAe,KAAK,IAAI,EAAE,CAAC;wCAC1C,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;oCACtF,CAAC;oCAED,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;oCAC1D,OAAO,CAAC,GAAG,CAAC,oBAAa,KAAK,CAAC,MAAM,CAAC,YAAY,CAAE,CAAC,CAAC;gCACxD,CAAC;;;;qBACF;aACF;SACF,CAAC;IACJ,CAAC;IAAD,iCAAC;AAAD,CAAC,AAxUD,IAwUC;AAxUY,gEAA0B;AA0UvC,kBAAe,IAAI,0BAA0B,EAAE,CAAC","sourcesContent":["import type { TestSuite, IAgentRuntime, Memory } from '@elizaos/core';\nimport { createUniqueUuid } from '@elizaos/core';\nimport { VisionService } from '../../service';\nimport { killAutonomousAction } from '../../action';\n\nexport class VisionAutonomyE2ETestSuite implements TestSuite {\n name = 'plugin-vision-autonomy-e2e';\n description = 'Tests for vision plugin integration with autonomy plugin';\n\n tests = [\n {\n name: 'Should stop autonomous loop with kill command',\n fn: async (runtime: IAgentRuntime) => {\n console.log('Testing kill autonomous action...');\n\n const roomId = createUniqueUuid(runtime, 'test-room');\n const message: Memory = {\n id: createUniqueUuid(runtime, 'test-msg-kill'),\n entityId: runtime.agentId,\n content: { text: 'kill the autonomous loop' },\n agentId: runtime.agentId,\n roomId,\n createdAt: Date.now(),\n };\n\n let callbackCalled = false;\n let callbackResponse: any = null;\n\n // Validate the action\n const isValid = await killAutonomousAction.validate(runtime, message, {\n values: {},\n data: {},\n text: '',\n });\n if (!isValid) {\n throw new Error('killAutonomousAction validation failed');\n }\n\n await killAutonomousAction.handler(\n runtime,\n message,\n { values: {}, data: {}, text: '' },\n {},\n async (response) => {\n callbackCalled = true;\n callbackResponse = response;\n return [];\n }\n );\n\n if (!callbackCalled) {\n throw new Error('Callback was not called - action handler failed');\n }\n\n if (!callbackResponse || !callbackResponse.text) {\n throw new Error('No response text returned from kill action');\n }\n\n console.log('✓ Kill autonomous action executed');\n console.log(` Response: ${callbackResponse.text}`);\n\n if (callbackResponse.thought) {\n console.log(` Thought: ${callbackResponse.thought}`);\n }\n\n // Verify action was included\n if (!callbackResponse.actions || !callbackResponse.actions.includes('KILL_AUTONOMOUS')) {\n throw new Error('Response does not include KILL_AUTONOMOUS action');\n }\n },\n },\n\n {\n name: 'Should provide continuous vision updates for autonomous agent',\n fn: async (runtime: IAgentRuntime) => {\n console.log('Testing vision updates for autonomous behavior...');\n\n const visionService = runtime.getService<VisionService>('VISION');\n if (!visionService) {\n throw new Error('Vision service not available');\n }\n\n if (!visionService.isActive()) {\n console.warn('⚠️ Vision service not active - skipping continuous update test');\n console.log(' This is acceptable in environments without cameras');\n return;\n }\n\n // Track scene updates over time\n const updateIntervals: number[] = [];\n let lastTimestamp = 0;\n let updateCount = 0;\n const testDuration = 5000; // 5 seconds\n const startTime = Date.now();\n\n console.log(' Monitoring scene updates for 5 seconds...');\n\n while (Date.now() - startTime < testDuration) {\n const scene = await visionService.getSceneDescription();\n\n if (scene && scene.timestamp !== lastTimestamp) {\n if (lastTimestamp > 0) {\n updateIntervals.push(scene.timestamp - lastTimestamp);\n }\n lastTimestamp = scene.timestamp;\n updateCount++;\n console.log(` Scene update ${updateCount}: ${scene.description.substring(0, 50)}...`);\n }\n\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n console.log('✓ Vision monitoring complete');\n console.log(` Total updates: ${updateCount}`);\n\n if (updateCount === 0) {\n throw new Error('No scene updates detected during 5 second monitoring period');\n }\n\n if (updateIntervals.length > 0) {\n const avgInterval = updateIntervals.reduce((a, b) => a + b, 0) / updateIntervals.length;\n console.log(` Average update interval: ${Math.round(avgInterval)}ms`);\n\n // Verify updates are happening at reasonable intervals\n if (avgInterval > 5000) {\n throw new Error('Scene updates too infrequent for autonomous operation');\n }\n }\n },\n },\n\n {\n name: 'Should maintain vision memory across interactions',\n fn: async (runtime: IAgentRuntime) => {\n console.log('Testing vision memory persistence...');\n\n const visionService = runtime.getService<VisionService>('VISION');\n if (!visionService) {\n throw new Error('Vision service not available');\n }\n\n // First interaction - describe scene\n const roomId = createUniqueUuid(runtime, 'test-room');\n const firstMessage: Memory = {\n id: createUniqueUuid(runtime, 'test-msg-1'),\n entityId: runtime.agentId,\n content: { text: 'what do you see?' },\n agentId: runtime.agentId,\n roomId,\n createdAt: Date.now(),\n };\n\n // Store first message\n await runtime.createMemory(firstMessage, 'messages');\n\n // Simulate agent response\n const firstResponse: Memory = {\n id: createUniqueUuid(runtime, 'test-response-1'),\n entityId: runtime.agentId,\n content: {\n text: 'I see a test scene',\n actions: ['DESCRIBE_SCENE'],\n },\n agentId: runtime.agentId,\n roomId,\n createdAt: Date.now() + 100,\n };\n await runtime.createMemory(firstResponse, 'messages');\n\n // Wait for processing\n await new Promise((resolve) => setTimeout(resolve, 500));\n\n // Second interaction - ask about previous observation\n const secondMessage: Memory = {\n id: createUniqueUuid(runtime, 'test-msg-2'),\n entityId: runtime.agentId,\n content: { text: 'what did you see before?' },\n agentId: runtime.agentId,\n roomId,\n createdAt: Date.now() + 1000,\n };\n\n await runtime.createMemory(secondMessage, 'messages');\n\n // Simulate agent response recalling previous observation\n const secondResponse: Memory = {\n id: createUniqueUuid(runtime, 'test-response-2'),\n entityId: runtime.agentId,\n content: {\n text: 'Previously, I saw a test scene',\n actions: ['DESCRIBE_SCENE'],\n },\n agentId: runtime.agentId,\n roomId,\n createdAt: Date.now() + 1100,\n };\n await runtime.createMemory(secondResponse, 'messages');\n\n // Wait for processing\n await new Promise((resolve) => setTimeout(resolve, 500));\n\n // Check if messages were created\n const messages = await runtime.getMemories({\n roomId,\n count: 10,\n tableName: 'messages',\n });\n\n console.log('✓ Vision memory test complete');\n console.log(` Total messages: ${messages ? messages.length : 0}`);\n\n // In the minimal test runtime, we just verify memory creation works\n // Real runtime would have actual message processing\n if (!visionService.isActive()) {\n console.log(' Note: Vision service not active, simulated memory test only');\n }\n },\n },\n\n {\n name: 'Should integrate vision data with agent decision making',\n fn: async (runtime: IAgentRuntime) => {\n console.log('Testing vision-based decision making...');\n\n // Create a test scenario where vision affects agent behavior\n const roomId = createUniqueUuid(runtime, 'test-room');\n\n // Simulate different vision states\n const scenarios = [\n { text: 'Is anyone in the room?', expectedContext: 'people' },\n { text: 'Should I turn on the lights?', expectedContext: 'scene' },\n { text: 'What objects are nearby?', expectedContext: 'objects' },\n ];\n\n let scenariosWithVision = 0;\n\n for (const scenario of scenarios) {\n const message: Memory = {\n id: createUniqueUuid(runtime, `test-msg-${Date.now()}`),\n entityId: runtime.agentId,\n content: { text: scenario.text },\n agentId: runtime.agentId,\n roomId,\n createdAt: Date.now(),\n };\n\n // Compose state to see if vision data is included\n const state = await runtime.composeState(message);\n\n // Check if vision context is available for decision\n const hasVisionContext =\n state.text.includes('Visual Perception') || state.values.visionAvailable !== undefined;\n\n console.log(` Scenario: \"${scenario.text}\"`);\n console.log(` Has vision context: ${hasVisionContext}`);\n\n if (hasVisionContext) {\n scenariosWithVision++;\n\n if (state.values.sceneDescription) {\n console.log(\n ` Scene info available: ${state.values.sceneDescription.substring(0, 50)}...`\n );\n }\n }\n }\n\n console.log('✓ Vision-based decision making test complete');\n\n // All scenarios should have vision context from the provider\n if (scenariosWithVision !== scenarios.length) {\n throw new Error(\n `Vision context missing in ${scenarios.length - scenariosWithVision} scenarios`\n );\n }\n },\n },\n\n {\n name: 'Should handle autonomy gracefully when vision is unavailable',\n fn: async (runtime: IAgentRuntime) => {\n console.log('Testing autonomy behavior without vision...');\n\n const visionService = runtime.getService<VisionService>('VISION');\n\n if (!visionService) {\n throw new Error('Vision service not registered - cannot test graceful handling');\n }\n\n const isActive = visionService.isActive();\n console.log(` Vision service active: ${isActive}`);\n\n // Compose state to check provider behavior\n const message: Memory = {\n id: createUniqueUuid(runtime, 'test-msg-no-vision'),\n entityId: runtime.agentId,\n content: { text: 'test without vision' },\n agentId: runtime.agentId,\n roomId: createUniqueUuid(runtime, 'test-room'),\n createdAt: Date.now(),\n };\n\n const state = await runtime.composeState(message);\n\n // Vision provider should always provide status\n if (state.values.visionAvailable === undefined) {\n throw new Error('Vision provider did not report availability status');\n }\n\n if (!isActive) {\n // Vision should indicate it's not available\n if (state.values.visionAvailable !== false) {\n throw new Error('Vision incorrectly reports as available when service is not active');\n }\n\n if (!state.values.cameraStatus || !state.values.cameraStatus.includes('not connected')) {\n throw new Error('Camera status does not indicate disconnection');\n }\n\n console.log('✓ Vision correctly reports unavailable state');\n console.log(` Status: ${state.values.cameraStatus}`);\n } else {\n // Vision is active\n if (state.values.visionAvailable !== true) {\n throw new Error('Vision incorrectly reports as unavailable when service is active');\n }\n\n console.log('✓ Vision correctly reports available state');\n console.log(` Status: ${state.values.cameraStatus}`);\n }\n },\n },\n ];\n}\n\nexport default new VisionAutonomyE2ETestSuite();\n"]}
@@ -1,11 +0,0 @@
1
- import type { TestSuite, IAgentRuntime } from '@elizaos/core';
2
- export declare class VisionBasicE2ETestSuite implements TestSuite {
3
- name: string;
4
- description: string;
5
- tests: {
6
- name: string;
7
- fn: (runtime: IAgentRuntime) => Promise<void>;
8
- }[];
9
- }
10
- declare const _default: VisionBasicE2ETestSuite;
11
- export default _default;