@elizaos/plugin-vision 2.0.0-alpha.5 → 2.0.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +32 -31
- package/dist/index.js.map +3 -3
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1852,7 +1852,7 @@ class StreamingAudioCaptureService extends EventEmitter {
|
|
|
1852
1852
|
}
|
|
1853
1853
|
|
|
1854
1854
|
// src/entity-tracker.ts
|
|
1855
|
-
import { logger as logger5 } from "@elizaos/core";
|
|
1855
|
+
import { createUniqueUuid as createUniqueUuid2, logger as logger5 } from "@elizaos/core";
|
|
1856
1856
|
|
|
1857
1857
|
class EntityTracker {
|
|
1858
1858
|
worldState;
|
|
@@ -2030,8 +2030,9 @@ class EntityTracker {
|
|
|
2030
2030
|
async syncWithRuntime(runtime, frameEntities) {
|
|
2031
2031
|
try {
|
|
2032
2032
|
for (const entity of frameEntities) {
|
|
2033
|
+
const entityUuid = createUniqueUuid2(runtime, entity.id);
|
|
2033
2034
|
const elizaEntity = {
|
|
2034
|
-
id:
|
|
2035
|
+
id: entityUuid,
|
|
2035
2036
|
names: [entity.attributes.name || entity.id],
|
|
2036
2037
|
metadata: {
|
|
2037
2038
|
type: entity.entityType,
|
|
@@ -2043,16 +2044,16 @@ class EntityTracker {
|
|
|
2043
2044
|
agentId: runtime.agentId
|
|
2044
2045
|
};
|
|
2045
2046
|
try {
|
|
2046
|
-
const existing = await runtime.getEntityById(
|
|
2047
|
+
const existing = await runtime.getEntityById(entityUuid);
|
|
2047
2048
|
if (!existing) {
|
|
2048
2049
|
await runtime.createEntity(elizaEntity);
|
|
2049
|
-
logger5.debug(`[EntityTracker] Created entity ${entity.id} in runtime`);
|
|
2050
|
+
logger5.debug(`[EntityTracker] Created entity ${entity.id} (${entityUuid}) in runtime`);
|
|
2050
2051
|
} else {
|
|
2051
2052
|
await runtime.updateEntity({
|
|
2052
2053
|
...elizaEntity,
|
|
2053
|
-
id:
|
|
2054
|
+
id: entityUuid
|
|
2054
2055
|
});
|
|
2055
|
-
logger5.debug(`[EntityTracker] Updated entity ${entity.id} in runtime`);
|
|
2056
|
+
logger5.debug(`[EntityTracker] Updated entity ${entity.id} (${entityUuid}) in runtime`);
|
|
2056
2057
|
}
|
|
2057
2058
|
} catch (err) {
|
|
2058
2059
|
logger5.debug(`[EntityTracker] Could not sync entity ${entity.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -5183,7 +5184,7 @@ class VisionService extends Service {
|
|
|
5183
5184
|
}
|
|
5184
5185
|
|
|
5185
5186
|
// src/tests/e2e/screen-vision.ts
|
|
5186
|
-
import { createUniqueUuid as
|
|
5187
|
+
import { createUniqueUuid as createUniqueUuid3 } from "@elizaos/core";
|
|
5187
5188
|
class ScreenVisionE2ETestSuite {
|
|
5188
5189
|
name = "plugin-vision-screen-e2e";
|
|
5189
5190
|
description = "E2E tests for screen vision functionality including Florence-2 and OCR";
|
|
@@ -5302,11 +5303,11 @@ class ScreenVisionE2ETestSuite {
|
|
|
5302
5303
|
}
|
|
5303
5304
|
console.log(" Testing SET_VISION_MODE action...");
|
|
5304
5305
|
const message = {
|
|
5305
|
-
id:
|
|
5306
|
+
id: createUniqueUuid3(runtime, "test-msg"),
|
|
5306
5307
|
entityId: runtime.agentId,
|
|
5307
5308
|
content: { text: "set vision mode to both" },
|
|
5308
5309
|
agentId: runtime.agentId,
|
|
5309
|
-
roomId:
|
|
5310
|
+
roomId: createUniqueUuid3(runtime, "test-room"),
|
|
5310
5311
|
createdAt: Date.now()
|
|
5311
5312
|
};
|
|
5312
5313
|
let callbackCalled = false;
|
|
@@ -5355,11 +5356,11 @@ class ScreenVisionE2ETestSuite {
|
|
|
5355
5356
|
}
|
|
5356
5357
|
}
|
|
5357
5358
|
const state = await runtime.composeState({
|
|
5358
|
-
id:
|
|
5359
|
+
id: createUniqueUuid3(runtime, "test-msg"),
|
|
5359
5360
|
entityId: runtime.agentId,
|
|
5360
5361
|
content: { text: "test" },
|
|
5361
5362
|
agentId: runtime.agentId,
|
|
5362
|
-
roomId:
|
|
5363
|
+
roomId: createUniqueUuid3(runtime, "test-room"),
|
|
5363
5364
|
createdAt: Date.now()
|
|
5364
5365
|
});
|
|
5365
5366
|
if (state.text.includes("Vision mode: BOTH")) {
|
|
@@ -5395,7 +5396,7 @@ class ScreenVisionE2ETestSuite {
|
|
|
5395
5396
|
}
|
|
5396
5397
|
var screen_vision_default = new ScreenVisionE2ETestSuite;
|
|
5397
5398
|
// src/tests/e2e/vision-autonomy.ts
|
|
5398
|
-
import { createUniqueUuid as
|
|
5399
|
+
import { createUniqueUuid as createUniqueUuid4 } from "@elizaos/core";
|
|
5399
5400
|
class VisionAutonomyE2ETestSuite {
|
|
5400
5401
|
name = "plugin-vision-autonomy-e2e";
|
|
5401
5402
|
description = "Tests for vision plugin integration with autonomy plugin";
|
|
@@ -5404,9 +5405,9 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5404
5405
|
name: "Should stop autonomous loop with kill command",
|
|
5405
5406
|
fn: async (runtime) => {
|
|
5406
5407
|
console.log("Testing kill autonomous action...");
|
|
5407
|
-
const roomId =
|
|
5408
|
+
const roomId = createUniqueUuid4(runtime, "test-room");
|
|
5408
5409
|
const message = {
|
|
5409
|
-
id:
|
|
5410
|
+
id: createUniqueUuid4(runtime, "test-msg-kill"),
|
|
5410
5411
|
entityId: runtime.agentId,
|
|
5411
5412
|
content: { text: "kill the autonomous loop" },
|
|
5412
5413
|
agentId: runtime.agentId,
|
|
@@ -5497,9 +5498,9 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5497
5498
|
if (!visionService) {
|
|
5498
5499
|
throw new Error("Vision service not available");
|
|
5499
5500
|
}
|
|
5500
|
-
const roomId =
|
|
5501
|
+
const roomId = createUniqueUuid4(runtime, "test-room");
|
|
5501
5502
|
const firstMessage = {
|
|
5502
|
-
id:
|
|
5503
|
+
id: createUniqueUuid4(runtime, "test-msg-1"),
|
|
5503
5504
|
entityId: runtime.agentId,
|
|
5504
5505
|
content: { text: "what do you see?" },
|
|
5505
5506
|
agentId: runtime.agentId,
|
|
@@ -5508,7 +5509,7 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5508
5509
|
};
|
|
5509
5510
|
await runtime.createMemory(firstMessage, "messages");
|
|
5510
5511
|
const firstResponse = {
|
|
5511
|
-
id:
|
|
5512
|
+
id: createUniqueUuid4(runtime, "test-response-1"),
|
|
5512
5513
|
entityId: runtime.agentId,
|
|
5513
5514
|
content: {
|
|
5514
5515
|
text: "I see a test scene",
|
|
@@ -5521,7 +5522,7 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5521
5522
|
await runtime.createMemory(firstResponse, "messages");
|
|
5522
5523
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
5523
5524
|
const secondMessage = {
|
|
5524
|
-
id:
|
|
5525
|
+
id: createUniqueUuid4(runtime, "test-msg-2"),
|
|
5525
5526
|
entityId: runtime.agentId,
|
|
5526
5527
|
content: { text: "what did you see before?" },
|
|
5527
5528
|
agentId: runtime.agentId,
|
|
@@ -5530,7 +5531,7 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5530
5531
|
};
|
|
5531
5532
|
await runtime.createMemory(secondMessage, "messages");
|
|
5532
5533
|
const secondResponse = {
|
|
5533
|
-
id:
|
|
5534
|
+
id: createUniqueUuid4(runtime, "test-response-2"),
|
|
5534
5535
|
entityId: runtime.agentId,
|
|
5535
5536
|
content: {
|
|
5536
5537
|
text: "Previously, I saw a test scene",
|
|
@@ -5558,7 +5559,7 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5558
5559
|
name: "Should integrate vision data with agent decision making",
|
|
5559
5560
|
fn: async (runtime) => {
|
|
5560
5561
|
console.log("Testing vision-based decision making...");
|
|
5561
|
-
const roomId =
|
|
5562
|
+
const roomId = createUniqueUuid4(runtime, "test-room");
|
|
5562
5563
|
const scenarios = [
|
|
5563
5564
|
{ text: "Is anyone in the room?", expectedContext: "people" },
|
|
5564
5565
|
{ text: "Should I turn on the lights?", expectedContext: "scene" },
|
|
@@ -5567,7 +5568,7 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5567
5568
|
let scenariosWithVision = 0;
|
|
5568
5569
|
for (const scenario of scenarios) {
|
|
5569
5570
|
const message = {
|
|
5570
|
-
id:
|
|
5571
|
+
id: createUniqueUuid4(runtime, `test-msg-${Date.now()}`),
|
|
5571
5572
|
entityId: runtime.agentId,
|
|
5572
5573
|
content: { text: scenario.text },
|
|
5573
5574
|
agentId: runtime.agentId,
|
|
@@ -5602,11 +5603,11 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5602
5603
|
const isActive = visionService.isActive();
|
|
5603
5604
|
console.log(` Vision service active: ${isActive}`);
|
|
5604
5605
|
const message = {
|
|
5605
|
-
id:
|
|
5606
|
+
id: createUniqueUuid4(runtime, "test-msg-no-vision"),
|
|
5606
5607
|
entityId: runtime.agentId,
|
|
5607
5608
|
content: { text: "test without vision" },
|
|
5608
5609
|
agentId: runtime.agentId,
|
|
5609
|
-
roomId:
|
|
5610
|
+
roomId: createUniqueUuid4(runtime, "test-room"),
|
|
5610
5611
|
createdAt: Date.now()
|
|
5611
5612
|
};
|
|
5612
5613
|
const state = await runtime.composeState(message);
|
|
@@ -5635,7 +5636,7 @@ class VisionAutonomyE2ETestSuite {
|
|
|
5635
5636
|
}
|
|
5636
5637
|
var vision_autonomy_default = new VisionAutonomyE2ETestSuite;
|
|
5637
5638
|
// src/tests/e2e/vision-basic.ts
|
|
5638
|
-
import { createUniqueUuid as
|
|
5639
|
+
import { createUniqueUuid as createUniqueUuid5 } from "@elizaos/core";
|
|
5639
5640
|
class VisionBasicE2ETestSuite {
|
|
5640
5641
|
name = "plugin-vision-basic-e2e";
|
|
5641
5642
|
description = "Basic end-to-end tests for vision plugin functionality";
|
|
@@ -5663,9 +5664,9 @@ class VisionBasicE2ETestSuite {
|
|
|
5663
5664
|
name: "Should describe scene when requested",
|
|
5664
5665
|
fn: async (runtime) => {
|
|
5665
5666
|
console.log("Testing scene description action...");
|
|
5666
|
-
const roomId =
|
|
5667
|
+
const roomId = createUniqueUuid5(runtime, "test-room");
|
|
5667
5668
|
const message = {
|
|
5668
|
-
id:
|
|
5669
|
+
id: createUniqueUuid5(runtime, "test-msg-describe"),
|
|
5669
5670
|
entityId: runtime.agentId,
|
|
5670
5671
|
content: { text: "what do you see?" },
|
|
5671
5672
|
agentId: runtime.agentId,
|
|
@@ -5729,9 +5730,9 @@ class VisionBasicE2ETestSuite {
|
|
|
5729
5730
|
name: "Should capture image when requested",
|
|
5730
5731
|
fn: async (runtime) => {
|
|
5731
5732
|
console.log("Testing image capture action...");
|
|
5732
|
-
const roomId =
|
|
5733
|
+
const roomId = createUniqueUuid5(runtime, "test-room");
|
|
5733
5734
|
const message = {
|
|
5734
|
-
id:
|
|
5735
|
+
id: createUniqueUuid5(runtime, "test-msg-capture"),
|
|
5735
5736
|
entityId: runtime.agentId,
|
|
5736
5737
|
content: { text: "take a photo" },
|
|
5737
5738
|
agentId: runtime.agentId,
|
|
@@ -5801,11 +5802,11 @@ class VisionBasicE2ETestSuite {
|
|
|
5801
5802
|
fn: async (runtime) => {
|
|
5802
5803
|
console.log("Testing vision provider...");
|
|
5803
5804
|
const message = {
|
|
5804
|
-
id:
|
|
5805
|
+
id: createUniqueUuid5(runtime, "test-msg-provider"),
|
|
5805
5806
|
entityId: runtime.agentId,
|
|
5806
5807
|
content: { text: "test provider" },
|
|
5807
5808
|
agentId: runtime.agentId,
|
|
5808
|
-
roomId:
|
|
5809
|
+
roomId: createUniqueUuid5(runtime, "test-room"),
|
|
5809
5810
|
createdAt: Date.now()
|
|
5810
5811
|
};
|
|
5811
5812
|
const state = await runtime.composeState(message);
|
|
@@ -6759,4 +6760,4 @@ export {
|
|
|
6759
6760
|
src_default as default
|
|
6760
6761
|
};
|
|
6761
6762
|
|
|
6762
|
-
//# debugId=
|
|
6763
|
+
//# debugId=8F996A8EAA2C894364756E2164756E21
|