@elizaos/plugin-vision 1.2.1 → 2.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. package/LICENSE +21 -0
  2. package/build.config.ts +53 -53
  3. package/dist/index.js +6716 -67
  4. package/dist/index.js.map +33 -1
  5. package/dist/workers/florence2-worker.js +111763 -307
  6. package/dist/workers/florence2-worker.js.map +92 -1
  7. package/dist/workers/ocr-worker.js +119177 -339
  8. package/dist/workers/ocr-worker.js.map +137 -1
  9. package/dist/workers/screen-capture-worker.js +350 -418
  10. package/dist/workers/screen-capture-worker.js.map +11 -1
  11. package/package.json +15 -20
  12. package/README.md +0 -270
  13. package/dist/action.d.ts +0 -8
  14. package/dist/action.js +0 -1212
  15. package/dist/action.js.map +0 -1
  16. package/dist/audio-capture-stream.d.ts +0 -42
  17. package/dist/audio-capture-stream.js +0 -516
  18. package/dist/audio-capture-stream.js.map +0 -1
  19. package/dist/audio-capture.d.ts +0 -25
  20. package/dist/audio-capture.js +0 -412
  21. package/dist/audio-capture.js.map +0 -1
  22. package/dist/basic.test.d.ts +0 -1
  23. package/dist/basic.test.js +0 -97
  24. package/dist/basic.test.js.map +0 -1
  25. package/dist/config.d.ts +0 -73
  26. package/dist/config.js +0 -254
  27. package/dist/config.js.map +0 -1
  28. package/dist/entity-tracker.d.ts +0 -32
  29. package/dist/entity-tracker.js +0 -361
  30. package/dist/entity-tracker.js.map +0 -1
  31. package/dist/errors.d.ts +0 -67
  32. package/dist/errors.js +0 -395
  33. package/dist/errors.js.map +0 -1
  34. package/dist/face-recognition.d.ts +0 -31
  35. package/dist/face-recognition.js +0 -332
  36. package/dist/face-recognition.js.map +0 -1
  37. package/dist/florence2-local.d.ts +0 -25
  38. package/dist/florence2-local.js +0 -280
  39. package/dist/florence2-local.js.map +0 -1
  40. package/dist/florence2-model.d.ts +0 -36
  41. package/dist/florence2-model.js +0 -503
  42. package/dist/florence2-model.js.map +0 -1
  43. package/dist/index.d.ts +0 -3
  44. package/dist/ocr-service-real.d.ts +0 -32
  45. package/dist/ocr-service-real.js +0 -396
  46. package/dist/ocr-service-real.js.map +0 -1
  47. package/dist/ocr-service.d.ts +0 -28
  48. package/dist/ocr-service.js +0 -216
  49. package/dist/ocr-service.js.map +0 -1
  50. package/dist/provider.d.ts +0 -2
  51. package/dist/provider.js +0 -285
  52. package/dist/provider.js.map +0 -1
  53. package/dist/screen-capture.d.ts +0 -16
  54. package/dist/screen-capture.js +0 -302
  55. package/dist/screen-capture.js.map +0 -1
  56. package/dist/service.d.ts +0 -73
  57. package/dist/service.js +0 -1662
  58. package/dist/service.js.map +0 -1
  59. package/dist/tests/e2e/index.d.ts +0 -8
  60. package/dist/tests/e2e/index.js +0 -33
  61. package/dist/tests/e2e/index.js.map +0 -1
  62. package/dist/tests/e2e/run-local.d.ts +0 -2
  63. package/dist/tests/e2e/run-local.js +0 -166
  64. package/dist/tests/e2e/run-local.js.map +0 -1
  65. package/dist/tests/e2e/screen-vision.d.ts +0 -11
  66. package/dist/tests/e2e/screen-vision.js +0 -384
  67. package/dist/tests/e2e/screen-vision.js.map +0 -1
  68. package/dist/tests/e2e/vision-autonomy.d.ts +0 -11
  69. package/dist/tests/e2e/vision-autonomy.js +0 -375
  70. package/dist/tests/e2e/vision-autonomy.js.map +0 -1
  71. package/dist/tests/e2e/vision-basic.d.ts +0 -11
  72. package/dist/tests/e2e/vision-basic.js +0 -434
  73. package/dist/tests/e2e/vision-basic.js.map +0 -1
  74. package/dist/tests/e2e/vision-capture-log.d.ts +0 -11
  75. package/dist/tests/e2e/vision-capture-log.js +0 -302
  76. package/dist/tests/e2e/vision-capture-log.js.map +0 -1
  77. package/dist/tests/e2e/vision-runtime.d.ts +0 -11
  78. package/dist/tests/e2e/vision-runtime.js +0 -357
  79. package/dist/tests/e2e/vision-runtime.js.map +0 -1
  80. package/dist/tests/e2e/vision-worker-tests.d.ts +0 -11
  81. package/dist/tests/e2e/vision-worker-tests.js +0 -466
  82. package/dist/tests/e2e/vision-worker-tests.js.map +0 -1
  83. package/dist/tests/test-pattern-generator.d.ts +0 -40
  84. package/dist/tests/test-pattern-generator.js +0 -191
  85. package/dist/tests/test-pattern-generator.js.map +0 -1
  86. package/dist/tests.d.ts +0 -3
  87. package/dist/tests.js +0 -11
  88. package/dist/tests.js.map +0 -1
  89. package/dist/types.d.ts +0 -222
  90. package/dist/types.js +0 -16
  91. package/dist/types.js.map +0 -1
  92. package/dist/vision-models.d.ts +0 -47
  93. package/dist/vision-models.js +0 -501
  94. package/dist/vision-models.js.map +0 -1
  95. package/dist/vision-worker-manager.d.ts +0 -61
  96. package/dist/vision-worker-manager.js +0 -668
  97. package/dist/vision-worker-manager.js.map +0 -1
  98. package/dist/workers/florence2-worker-simple.d.ts +0 -13
  99. package/dist/workers/florence2-worker-simple.js +0 -121
  100. package/dist/workers/florence2-worker-simple.js.map +0 -1
  101. package/dist/workers/florence2-worker.d.ts +0 -1
  102. package/dist/workers/ocr-worker.d.ts +0 -1
  103. package/dist/workers/screen-capture-worker.d.ts +0 -1
  104. package/dist/workers/worker-logger.d.ts +0 -9
  105. package/dist/workers/worker-logger.js +0 -95
  106. package/dist/workers/worker-logger.js.map +0 -1
@@ -1,412 +0,0 @@
1
- "use strict";
2
- var __assign = (this && this.__assign) || function () {
3
- __assign = Object.assign || function(t) {
4
- for (var s, i = 1, n = arguments.length; i < n; i++) {
5
- s = arguments[i];
6
- for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
7
- t[p] = s[p];
8
- }
9
- return t;
10
- };
11
- return __assign.apply(this, arguments);
12
- };
13
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
14
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
15
- return new (P || (P = Promise))(function (resolve, reject) {
16
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
17
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
18
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
19
- step((generator = generator.apply(thisArg, _arguments || [])).next());
20
- });
21
- };
22
- var __generator = (this && this.__generator) || function (thisArg, body) {
23
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
24
- return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
25
- function verb(n) { return function (v) { return step([n, v]); }; }
26
- function step(op) {
27
- if (f) throw new TypeError("Generator is already executing.");
28
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
29
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
30
- if (y = 0, t) op = [op[0] & 2, t.value];
31
- switch (op[0]) {
32
- case 0: case 1: t = op; break;
33
- case 4: _.label++; return { value: op[1], done: false };
34
- case 5: _.label++; y = op[1]; op = [0]; continue;
35
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
36
- default:
37
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
38
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
39
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
40
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
41
- if (t[2]) _.ops.pop();
42
- _.trys.pop(); continue;
43
- }
44
- op = body.call(thisArg, _);
45
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
46
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
47
- }
48
- };
49
- Object.defineProperty(exports, "__esModule", { value: true });
50
- exports.AudioCaptureService = void 0;
51
- var core_1 = require("@elizaos/core");
52
- var child_process_1 = require("child_process");
53
- var util_1 = require("util");
54
- var fs = require("fs/promises");
55
- var path = require("path");
56
- var execAsync = (0, util_1.promisify)(child_process_1.exec);
57
- var AudioCaptureService = /** @class */ (function () {
58
- function AudioCaptureService(runtime, config) {
59
- this.isRecording = false;
60
- this.recordingInterval = null;
61
- this.currentRecordingPath = null;
62
- this.runtime = runtime;
63
- this.config = __assign({ sampleRate: 16000, channels: 1 }, config);
64
- }
65
- AudioCaptureService.prototype.initialize = function () {
66
- return __awaiter(this, void 0, void 0, function () {
67
- var tool, error_1;
68
- return __generator(this, function (_a) {
69
- switch (_a.label) {
70
- case 0:
71
- if (!this.config.enabled) {
72
- core_1.logger.info('[AudioCapture] Audio capture disabled');
73
- return [2 /*return*/];
74
- }
75
- _a.label = 1;
76
- case 1:
77
- _a.trys.push([1, 3, , 4]);
78
- core_1.logger.info('[AudioCapture] Initializing audio capture...');
79
- return [4 /*yield*/, this.checkAudioTools()];
80
- case 2:
81
- tool = _a.sent();
82
- if (!tool.available) {
83
- throw new Error("Audio recording tool not available. ".concat(tool.message));
84
- }
85
- core_1.logger.info("[AudioCapture] Using ".concat(tool.tool, " for audio capture"));
86
- core_1.logger.info("[AudioCapture] Transcription interval: ".concat(this.config.transcriptionInterval, "ms"));
87
- // Start recording loop if interval is set
88
- if (this.config.transcriptionInterval > 0) {
89
- this.startTranscriptionLoop();
90
- }
91
- core_1.logger.info('[AudioCapture] Audio capture initialized');
92
- return [3 /*break*/, 4];
93
- case 3:
94
- error_1 = _a.sent();
95
- core_1.logger.error('[AudioCapture] Failed to initialize:', error_1);
96
- throw error_1;
97
- case 4: return [2 /*return*/];
98
- }
99
- });
100
- });
101
- };
102
- AudioCaptureService.prototype.checkAudioTools = function () {
103
- return __awaiter(this, void 0, void 0, function () {
104
- var platform, _error_1, toolName, installCmd;
105
- return __generator(this, function (_a) {
106
- switch (_a.label) {
107
- case 0:
108
- platform = process.platform;
109
- _a.label = 1;
110
- case 1:
111
- _a.trys.push([1, 8, , 9]);
112
- if (!(platform === 'darwin')) return [3 /*break*/, 3];
113
- // macOS: Use sox
114
- return [4 /*yield*/, execAsync('which sox')];
115
- case 2:
116
- // macOS: Use sox
117
- _a.sent();
118
- return [2 /*return*/, { available: true, tool: 'sox' }];
119
- case 3:
120
- if (!(platform === 'linux')) return [3 /*break*/, 5];
121
- // Linux: Use arecord (ALSA)
122
- return [4 /*yield*/, execAsync('which arecord')];
123
- case 4:
124
- // Linux: Use arecord (ALSA)
125
- _a.sent();
126
- return [2 /*return*/, { available: true, tool: 'arecord' }];
127
- case 5:
128
- if (!(platform === 'win32')) return [3 /*break*/, 7];
129
- // Windows: Use ffmpeg
130
- return [4 /*yield*/, execAsync('where ffmpeg')];
131
- case 6:
132
- // Windows: Use ffmpeg
133
- _a.sent();
134
- return [2 /*return*/, { available: true, tool: 'ffmpeg' }];
135
- case 7: return [2 /*return*/, { available: false, tool: 'none', message: 'Unsupported platform' }];
136
- case 8:
137
- _error_1 = _a.sent();
138
- toolName = platform === 'darwin' ? 'sox' : platform === 'linux' ? 'arecord' : 'ffmpeg';
139
- installCmd = platform === 'darwin'
140
- ? 'brew install sox'
141
- : platform === 'linux'
142
- ? 'sudo apt-get install alsa-utils'
143
- : 'Download ffmpeg from ffmpeg.org';
144
- return [2 /*return*/, {
145
- available: false,
146
- tool: toolName,
147
- message: "Install with: ".concat(installCmd),
148
- }];
149
- case 9: return [2 /*return*/];
150
- }
151
- });
152
- });
153
- };
154
- AudioCaptureService.prototype.startTranscriptionLoop = function () {
155
- var _this = this;
156
- if (this.recordingInterval) {
157
- return;
158
- }
159
- // Start continuous recording with periodic transcription
160
- this.recordingInterval = setInterval(function () { return __awaiter(_this, void 0, void 0, function () {
161
- return __generator(this, function (_a) {
162
- switch (_a.label) {
163
- case 0:
164
- if (!!this.isRecording) return [3 /*break*/, 2];
165
- return [4 /*yield*/, this.recordAndTranscribe()];
166
- case 1:
167
- _a.sent();
168
- _a.label = 2;
169
- case 2: return [2 /*return*/];
170
- }
171
- });
172
- }); }, this.config.transcriptionInterval);
173
- core_1.logger.info('[AudioCapture] Started transcription loop');
174
- };
175
- AudioCaptureService.prototype.recordAndTranscribe = function () {
176
- return __awaiter(this, void 0, void 0, function () {
177
- var audioFile, audioBuffer, transcription, error_2;
178
- return __generator(this, function (_a) {
179
- switch (_a.label) {
180
- case 0:
181
- if (this.isRecording) {
182
- core_1.logger.warn('[AudioCapture] Already recording');
183
- return [2 /*return*/, null];
184
- }
185
- this.isRecording = true;
186
- audioFile = path.join(process.cwd(), "audio_".concat(Date.now(), ".wav"));
187
- _a.label = 1;
188
- case 1:
189
- _a.trys.push([1, 8, 10, 11]);
190
- core_1.logger.debug('[AudioCapture] Starting audio recording...');
191
- // Record audio for the specified duration
192
- return [4 /*yield*/, this.recordAudio(audioFile, this.config.transcriptionInterval / 1000)];
193
- case 2:
194
- // Record audio for the specified duration
195
- _a.sent();
196
- core_1.logger.debug('[AudioCapture] Recording complete, transcribing...');
197
- return [4 /*yield*/, fs.readFile(audioFile)];
198
- case 3:
199
- audioBuffer = _a.sent();
200
- return [4 /*yield*/, this.runtime.useModel(core_1.ModelType.TRANSCRIPTION, {
201
- audio: audioBuffer,
202
- language: 'en',
203
- })];
204
- case 4:
205
- transcription = _a.sent();
206
- // Clean up audio file
207
- return [4 /*yield*/, fs.unlink(audioFile).catch(function () { })];
208
- case 5:
209
- // Clean up audio file
210
- _a.sent();
211
- if (!(transcription && typeof transcription === 'string' && transcription.trim())) return [3 /*break*/, 7];
212
- core_1.logger.info("[AudioCapture] Transcribed: \"".concat(transcription, "\""));
213
- // Create a memory of what was heard
214
- return [4 /*yield*/, this.createAudioMemory(transcription)];
215
- case 6:
216
- // Create a memory of what was heard
217
- _a.sent();
218
- return [2 /*return*/, transcription];
219
- case 7: return [2 /*return*/, null];
220
- case 8:
221
- error_2 = _a.sent();
222
- core_1.logger.error('[AudioCapture] Recording/transcription failed:', error_2);
223
- return [4 /*yield*/, fs.unlink(audioFile).catch(function () { })];
224
- case 9:
225
- _a.sent();
226
- return [2 /*return*/, null];
227
- case 10:
228
- this.isRecording = false;
229
- return [7 /*endfinally*/];
230
- case 11: return [2 /*return*/];
231
- }
232
- });
233
- });
234
- };
235
- AudioCaptureService.prototype.recordAudio = function (outputPath, duration) {
236
- return __awaiter(this, void 0, void 0, function () {
237
- var platform, _device, device, device, error_3;
238
- return __generator(this, function (_a) {
239
- switch (_a.label) {
240
- case 0:
241
- platform = process.platform;
242
- _a.label = 1;
243
- case 1:
244
- _a.trys.push([1, 9, , 10]);
245
- if (!(platform === 'darwin')) return [3 /*break*/, 3];
246
- _device = this.config.device || 'default';
247
- return [4 /*yield*/, execAsync("sox -d -r ".concat(this.config.sampleRate, " -c ").concat(this.config.channels, " -b 16 \"").concat(outputPath, "\" trim 0 ").concat(duration))];
248
- case 2:
249
- _a.sent();
250
- return [3 /*break*/, 8];
251
- case 3:
252
- if (!(platform === 'linux')) return [3 /*break*/, 5];
253
- device = this.config.device || 'default';
254
- return [4 /*yield*/, execAsync("arecord -D ".concat(device, " -f S16_LE -r ").concat(this.config.sampleRate, " -c ").concat(this.config.channels, " -d ").concat(duration, " \"").concat(outputPath, "\""))];
255
- case 4:
256
- _a.sent();
257
- return [3 /*break*/, 8];
258
- case 5:
259
- if (!(platform === 'win32')) return [3 /*break*/, 7];
260
- device = this.config.device || 'Microphone';
261
- return [4 /*yield*/, execAsync("ffmpeg -f dshow -i audio=\"".concat(device, "\" -t ").concat(duration, " -acodec pcm_s16le -ar ").concat(this.config.sampleRate, " -ac ").concat(this.config.channels, " \"").concat(outputPath, "\" -y"))];
262
- case 6:
263
- _a.sent();
264
- return [3 /*break*/, 8];
265
- case 7: throw new Error("Unsupported platform: ".concat(platform));
266
- case 8: return [3 /*break*/, 10];
267
- case 9:
268
- error_3 = _a.sent();
269
- core_1.logger.error('[AudioCapture] Audio recording failed:', error_3);
270
- throw error_3;
271
- case 10: return [2 /*return*/];
272
- }
273
- });
274
- });
275
- };
276
- AudioCaptureService.prototype.createAudioMemory = function (transcription) {
277
- return __awaiter(this, void 0, void 0, function () {
278
- var _memory;
279
- return __generator(this, function (_a) {
280
- try {
281
- _memory = {
282
- content: {
283
- text: "[Audio Transcription] ".concat(transcription),
284
- type: 'audio_transcription',
285
- source: 'microphone',
286
- timestamp: Date.now(),
287
- },
288
- metadata: {
289
- isAudioTranscription: true,
290
- duration: this.config.transcriptionInterval,
291
- },
292
- };
293
- // Store in agent's context
294
- // Note: createMemory requires runtime implementation specific to the database adapter
295
- // For now, we'll just log it
296
- core_1.logger.info('[AudioCapture] Audio transcription stored in context');
297
- }
298
- catch (error) {
299
- core_1.logger.error('[AudioCapture] Failed to create audio memory:', error);
300
- }
301
- return [2 /*return*/];
302
- });
303
- });
304
- };
305
- AudioCaptureService.prototype.listAudioDevices = function () {
306
- return __awaiter(this, void 0, void 0, function () {
307
- var platform, devices, stdout, data, _i, _a, device, stdout, lines, _b, lines_1, line, match, stdout, lines, isAudioSection, _c, lines_2, line, match, _error_2, error_4;
308
- return __generator(this, function (_d) {
309
- switch (_d.label) {
310
- case 0:
311
- platform = process.platform;
312
- devices = [];
313
- _d.label = 1;
314
- case 1:
315
- _d.trys.push([1, 10, , 11]);
316
- if (!(platform === 'darwin')) return [3 /*break*/, 3];
317
- return [4 /*yield*/, execAsync('system_profiler SPAudioDataType -json')];
318
- case 2:
319
- stdout = (_d.sent()).stdout;
320
- data = JSON.parse(stdout);
321
- if (data.SPAudioDataType) {
322
- for (_i = 0, _a = data.SPAudioDataType; _i < _a.length; _i++) {
323
- device = _a[_i];
324
- if (device._name && device._name.includes('Input')) {
325
- devices.push(device._name);
326
- }
327
- }
328
- }
329
- return [3 /*break*/, 9];
330
- case 3:
331
- if (!(platform === 'linux')) return [3 /*break*/, 5];
332
- return [4 /*yield*/, execAsync('arecord -l')];
333
- case 4:
334
- stdout = (_d.sent()).stdout;
335
- lines = stdout.split('\n');
336
- for (_b = 0, lines_1 = lines; _b < lines_1.length; _b++) {
337
- line = lines_1[_b];
338
- if (line.includes('card')) {
339
- match = line.match(/card (\d+):.*\[(.*?)\]/);
340
- if (match) {
341
- devices.push("hw:".concat(match[1]));
342
- }
343
- }
344
- }
345
- return [3 /*break*/, 9];
346
- case 5:
347
- if (!(platform === 'win32')) return [3 /*break*/, 9];
348
- _d.label = 6;
349
- case 6:
350
- _d.trys.push([6, 8, , 9]);
351
- return [4 /*yield*/, execAsync('ffmpeg -list_devices true -f dshow -i dummy 2>&1')];
352
- case 7:
353
- stdout = (_d.sent()).stdout;
354
- lines = stdout.split('\n');
355
- isAudioSection = false;
356
- for (_c = 0, lines_2 = lines; _c < lines_2.length; _c++) {
357
- line = lines_2[_c];
358
- if (line.includes('DirectShow audio devices')) {
359
- isAudioSection = true;
360
- }
361
- else if (isAudioSection && line.includes('"')) {
362
- match = line.match(/"([^"]+)"/);
363
- if (match) {
364
- devices.push(match[1]);
365
- }
366
- }
367
- }
368
- return [3 /*break*/, 9];
369
- case 8:
370
- _error_2 = _d.sent();
371
- return [3 /*break*/, 9];
372
- case 9: return [3 /*break*/, 11];
373
- case 10:
374
- error_4 = _d.sent();
375
- core_1.logger.error('[AudioCapture] Failed to list audio devices:', error_4);
376
- return [3 /*break*/, 11];
377
- case 11: return [2 /*return*/, devices];
378
- }
379
- });
380
- });
381
- };
382
- AudioCaptureService.prototype.isActive = function () {
383
- return this.config.enabled && this.recordingInterval !== null;
384
- };
385
- AudioCaptureService.prototype.stop = function () {
386
- return __awaiter(this, void 0, void 0, function () {
387
- return __generator(this, function (_a) {
388
- switch (_a.label) {
389
- case 0:
390
- core_1.logger.info('[AudioCapture] Stopping audio capture...');
391
- if (this.recordingInterval) {
392
- clearInterval(this.recordingInterval);
393
- this.recordingInterval = null;
394
- }
395
- _a.label = 1;
396
- case 1:
397
- if (!this.isRecording) return [3 /*break*/, 3];
398
- return [4 /*yield*/, new Promise(function (resolve) { return setTimeout(resolve, 100); })];
399
- case 2:
400
- _a.sent();
401
- return [3 /*break*/, 1];
402
- case 3:
403
- core_1.logger.info('[AudioCapture] Audio capture stopped');
404
- return [2 /*return*/];
405
- }
406
- });
407
- });
408
- };
409
- return AudioCaptureService;
410
- }());
411
- exports.AudioCaptureService = AudioCaptureService;
412
- //# sourceMappingURL=audio-capture.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"audio-capture.js","sourceRoot":"","sources":["../src/audio-capture.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sCAAsE;AACtE,+CAAqC;AACrC,6BAAiC;AACjC,gCAAkC;AAClC,2BAA6B;AAE7B,IAAM,SAAS,GAAG,IAAA,gBAAS,EAAC,oBAAI,CAAC,CAAC;AAUlC;IAOE,6BAAY,OAAsB,EAAE,MAAmB;QAJ/C,gBAAW,GAAG,KAAK,CAAC;QACpB,sBAAiB,GAA0B,IAAI,CAAC;QAChD,yBAAoB,GAAkB,IAAI,CAAC;QAGjD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,MAAM,cACT,UAAU,EAAE,KAAK,EACjB,QAAQ,EAAE,CAAC,IACR,MAAM,CACV,CAAC;IACJ,CAAC;IAEK,wCAAU,GAAhB;;;;;;wBACE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;4BACzB,aAAM,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;4BACrD,sBAAO;wBACT,CAAC;;;;wBAGC,aAAM,CAAC,IAAI,CAAC,8CAA8C,CAAC,CAAC;wBAG/C,qBAAM,IAAI,CAAC,eAAe,EAAE,EAAA;;wBAAnC,IAAI,GAAG,SAA4B;wBACzC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;4BACpB,MAAM,IAAI,KAAK,CAAC,8CAAuC,IAAI,CAAC,OAAO,CAAE,CAAC,CAAC;wBACzE,CAAC;wBAED,aAAM,CAAC,IAAI,CAAC,+BAAwB,IAAI,CAAC,IAAI,uBAAoB,CAAC,CAAC;wBACnE,aAAM,CAAC,IAAI,CAAC,iDAA0C,IAAI,CAAC,MAAM,CAAC,qBAAqB,OAAI,CAAC,CAAC;wBAE7F,0CAA0C;wBAC1C,IAAI,IAAI,CAAC,MAAM,CAAC,qBAAqB,GAAG,CAAC,EAAE,CAAC;4BAC1C,IAAI,CAAC,sBAAsB,EAAE,CAAC;wBAChC,CAAC;wBAED,aAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;;;;wBAExD,aAAM,CAAC,KAAK,CAAC,sCAAsC,EAAE,OAAK,CAAC,CAAC;wBAC5D,MAAM,OAAK,CAAC;;;;;KAEf;IAEa,6CAAe,GAA7B;;;;;;wBACQ,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;;;;6BAG5B,CAAA,QAAQ,KAAK,QAAQ,CAAA,EAArB,wBAAqB;wBACvB,iBAAiB;wBACjB,qBAAM,SAAS,CAAC,WAAW,CAAC,EAAA;;wBAD5B,iBAAiB;wBACjB,SAA4B,CAAC;wBAC7B,sBAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,EAAC;;6BAC/B,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;wBAC7B,4BAA4B;wBAC5B,qBAAM,SAAS,CAAC,eAAe,CAAC,EAAA;;wBADhC,4BAA4B;wBAC5B,SAAgC,CAAC;wBACjC,sBAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,EAAC;;6BACnC,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;wBAC7B,sBAAsB;wBACtB,qBAAM,SAAS,CAAC,cAAc,CAAC,EAAA;;wBAD/B,sBAAsB;wBACtB,SAA+B,CAAC;wBAChC,sBAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAC;4BAE7C,sBAAO,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,sBAAsB,EAAE,EAAC;;;wBAGrE,QAAQ,GAAG,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;wBACvF,UAAU,GACd,QAAQ,KAAK,QAAQ;4BACnB,CAAC,CAAC,kBAAkB;4BACpB,CAAC,CAAC,QAAQ,KAAK,OAAO;gCACpB,CAAC,CAAC,iCAAiC;gCACnC,CAAC,CAAC,iCAAiC,CAAC;wBAC1C,sBAAO;gCACL,SAAS,EAAE,KAAK;gCAChB,IAAI,EAAE,QAAQ;gCACd,OAAO,EAAE,wBAAiB,UAAU,CAAE;6BACvC,EAAC;;;;;KAEL;IAEO,oDAAsB,GAA9B;QAAA,iBAaC;QAZC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,OAAO;QACT,CAAC;QAED,yDAAyD;QACzD,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC;;;;6BAC/B,CAAC,IAAI,CAAC,WAAW,EAAjB,wBAAiB;wBACnB,qBAAM,IAAI,CAAC,mBAAmB,EAAE,EAAA;;wBAAhC,SAAgC,CAAC;;;;;aAEpC,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QAEtC,aAAM,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAC;IAC3D,CAAC;IAEK,iDAAmB,GAAzB;;;;;;wBACE,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrB,aAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;4BAChD,sBAAO,IAAI,EAAC;wBACd,CAAC;wBAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;wBAClB,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,gBAAS,IAAI,CAAC,GAAG,EAAE,SAAM,CAAC,CAAC;;;;wBAGpE,aAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;wBAE3D,0CAA0C;wBAC1C,qBAAM,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB,GAAG,IAAI,CAAC,EAAA;;wBAD3E,0CAA0C;wBAC1C,SAA2E,CAAC;wBAE5E,aAAM,CAAC,KAAK,CAAC,oDAAoD,CAAC,CAAC;wBAG/C,qBAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAA;;wBAA1C,WAAW,GAAG,SAA4B;wBAC1B,qBAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAS,CAAC,aAAa,EAAE;gCACzE,KAAK,EAAE,WAAW;gCAClB,QAAQ,EAAE,IAAI;6BACf,CAAC,EAAA;;wBAHI,aAAa,GAAG,SAGpB;wBAEF,sBAAsB;wBACtB,qBAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,cAAO,CAAC,CAAC,EAAA;;wBAD1C,sBAAsB;wBACtB,SAA0C,CAAC;6BAEvC,CAAA,aAAa,IAAI,OAAO,aAAa,KAAK,QAAQ,IAAI,aAAa,CAAC,IAAI,EAAE,CAAA,EAA1E,wBAA0E;wBAC5E,aAAM,CAAC,IAAI,CAAC,wCAAgC,aAAa,OAAG,CAAC,CAAC;wBAE9D,oCAAoC;wBACpC,qBAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,EAAA;;wBAD3C,oCAAoC;wBACpC,SAA2C,CAAC;wBAE5C,sBAAO,aAAa,EAAC;4BAGvB,sBAAO,IAAI,EAAC;;;wBAEZ,aAAM,CAAC,KAAK,CAAC,gDAAgD,EAAE,OAAK,CAAC,CAAC;wBACtE,qBAAM,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,cAAO,CAAC,CAAC,EAAA;;wBAA1C,SAA0C,CAAC;wBAC3C,sBAAO,IAAI,EAAC;;wBAEZ,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;;;;;;KAE5B;IAEa,yCAAW,GAAzB,UAA0B,UAAkB,EAAE,QAAgB;;;;;;wBACtD,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;;;;6BAG5B,CAAA,QAAQ,KAAK,QAAQ,CAAA,EAArB,wBAAqB;wBAEjB,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC;wBAChD,qBAAM,SAAS,CACb,oBAAa,IAAI,CAAC,MAAM,CAAC,UAAU,iBAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,sBAAW,UAAU,uBAAY,QAAQ,CAAE,CAC1G,EAAA;;wBAFD,SAEC,CAAC;;;6BACO,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;wBAEvB,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,SAAS,CAAC;wBAC/C,qBAAM,SAAS,CACb,qBAAc,MAAM,2BAAiB,IAAI,CAAC,MAAM,CAAC,UAAU,iBAAO,IAAI,CAAC,MAAM,CAAC,QAAQ,iBAAO,QAAQ,gBAAK,UAAU,OAAG,CACxH,EAAA;;wBAFD,SAEC,CAAC;;;6BACO,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;wBAEvB,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,YAAY,CAAC;wBAClD,qBAAM,SAAS,CACb,qCAA6B,MAAM,mBAAQ,QAAQ,oCAA0B,IAAI,CAAC,MAAM,CAAC,UAAU,kBAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,gBAAK,UAAU,UAAM,CACrJ,EAAA;;wBAFD,SAEC,CAAC;;4BAEF,MAAM,IAAI,KAAK,CAAC,gCAAyB,QAAQ,CAAE,CAAC,CAAC;;;;wBAGvD,aAAM,CAAC,KAAK,CAAC,wCAAwC,EAAE,OAAK,CAAC,CAAC;wBAC9D,MAAM,OAAK,CAAC;;;;;KAEf;IAEa,+CAAiB,GAA/B,UAAgC,aAAqB;;;;gBACnD,IAAI,CAAC;oBACG,OAAO,GAAG;wBACd,OAAO,EAAE;4BACP,IAAI,EAAE,gCAAyB,aAAa,CAAE;4BAC9C,IAAI,EAAE,qBAAqB;4BAC3B,MAAM,EAAE,YAAY;4BACpB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;yBACtB;wBACD,QAAQ,EAAE;4BACR,oBAAoB,EAAE,IAAI;4BAC1B,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,qBAAqB;yBAC5C;qBACF,CAAC;oBAEF,2BAA2B;oBAC3B,sFAAsF;oBACtF,6BAA6B;oBAC7B,aAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;gBACtE,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,aAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,KAAK,CAAC,CAAC;gBACvE,CAAC;;;;KACF;IAEK,8CAAgB,GAAtB;;;;;;wBACQ,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;wBAC5B,OAAO,GAAa,EAAE,CAAC;;;;6BAGvB,CAAA,QAAQ,KAAK,QAAQ,CAAA,EAArB,wBAAqB;wBAEJ,qBAAM,SAAS,CAAC,uCAAuC,CAAC,EAAA;;wBAAnE,MAAM,GAAK,CAAA,SAAwD,CAAA,OAA7D;wBACR,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;wBAEhC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;4BACzB,WAAyC,EAApB,KAAA,IAAI,CAAC,eAAe,EAApB,cAAoB,EAApB,IAAoB,EAAE,CAAC;gCAAjC,MAAM;gCACf,IAAI,MAAM,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oCACnD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gCAC7B,CAAC;4BACH,CAAC;wBACH,CAAC;;;6BACQ,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;wBAEV,qBAAM,SAAS,CAAC,YAAY,CAAC,EAAA;;wBAAxC,MAAM,GAAK,CAAA,SAA6B,CAAA,OAAlC;wBACR,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAEjC,WAAwB,EAAL,eAAK,EAAL,mBAAK,EAAL,IAAK,EAAE,CAAC;4BAAhB,IAAI;4BACb,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gCACpB,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;gCACnD,IAAI,KAAK,EAAE,CAAC;oCACV,OAAO,CAAC,IAAI,CAAC,aAAM,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;gCACjC,CAAC;4BACH,CAAC;wBACH,CAAC;;;6BACQ,CAAA,QAAQ,KAAK,OAAO,CAAA,EAApB,wBAAoB;;;;wBAGR,qBAAM,SAAS,CAAC,kDAAkD,CAAC,EAAA;;wBAA9E,MAAM,GAAK,CAAA,SAAmE,CAAA,OAAxE;wBACR,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAC7B,cAAc,GAAG,KAAK,CAAC;wBAE3B,WAAwB,EAAL,eAAK,EAAL,mBAAK,EAAL,IAAK,EAAE,CAAC;4BAAhB,IAAI;4BACb,IAAI,IAAI,CAAC,QAAQ,CAAC,0BAA0B,CAAC,EAAE,CAAC;gCAC9C,cAAc,GAAG,IAAI,CAAC;4BACxB,CAAC;iCAAM,IAAI,cAAc,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gCAC1C,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gCACtC,IAAI,KAAK,EAAE,CAAC;oCACV,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gCACzB,CAAC;4BACH,CAAC;wBACH,CAAC;;;;;;;;wBAOL,aAAM,CAAC,KAAK,CAAC,8CAA8C,EAAE,OAAK,CAAC,CAAC;;6BAGtE,sBAAO,OAAO,EAAC;;;;KAChB;IAED,sCAAQ,GAAR;QACE,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,IAAI,CAAC,iBAAiB,KAAK,IAAI,CAAC;IAChE,CAAC;IAEK,kCAAI,GAAV;;;;;wBACE,aAAM,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAC;wBAExD,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;4BAC3B,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;4BACtC,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;wBAChC,CAAC;;;6BAGM,IAAI,CAAC,WAAW;wBACrB,qBAAM,IAAI,OAAO,CAAC,UAAC,OAAO,IAAK,OAAA,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,EAAxB,CAAwB,CAAC,EAAA;;wBAAxD,SAAwD,CAAC;;;wBAG3D,aAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;;;;;KACrD;IACH,0BAAC;AAAD,CAAC,AAnRD,IAmRC;AAnRY,kDAAmB","sourcesContent":["import { logger, ModelType, type IAgentRuntime } from '@elizaos/core';\nimport { exec } from 'child_process';\nimport { promisify } from 'util';\nimport * as fs from 'fs/promises';\nimport * as path from 'path';\n\nconst execAsync = promisify(exec);\n\nexport interface AudioConfig {\n enabled: boolean;\n transcriptionInterval: number; // milliseconds\n device?: string;\n sampleRate?: number;\n channels?: number;\n}\n\nexport class AudioCaptureService {\n private runtime: IAgentRuntime;\n private config: AudioConfig;\n private isRecording = false;\n private recordingInterval: NodeJS.Timeout | null = null;\n private currentRecordingPath: string | null = null;\n\n constructor(runtime: IAgentRuntime, config: AudioConfig) {\n this.runtime = runtime;\n this.config = {\n sampleRate: 16000,\n channels: 1,\n ...config,\n };\n }\n\n async initialize(): Promise<void> {\n if (!this.config.enabled) {\n logger.info('[AudioCapture] Audio capture disabled');\n return;\n }\n\n try {\n logger.info('[AudioCapture] Initializing audio capture...');\n\n // Check for audio recording tools\n const tool = await this.checkAudioTools();\n if (!tool.available) {\n throw new Error(`Audio recording tool not available. ${tool.message}`);\n }\n\n logger.info(`[AudioCapture] Using ${tool.tool} for audio capture`);\n logger.info(`[AudioCapture] Transcription interval: ${this.config.transcriptionInterval}ms`);\n\n // Start recording loop if interval is set\n if (this.config.transcriptionInterval > 0) {\n this.startTranscriptionLoop();\n }\n\n logger.info('[AudioCapture] Audio capture initialized');\n } catch (error) {\n logger.error('[AudioCapture] Failed to initialize:', error);\n throw error;\n }\n }\n\n private async checkAudioTools(): Promise<{ available: boolean; tool: string; message?: string }> {\n const platform = process.platform;\n\n try {\n if (platform === 'darwin') {\n // macOS: Use sox\n await execAsync('which sox');\n return { available: true, tool: 'sox' };\n } else if (platform === 'linux') {\n // Linux: Use arecord (ALSA)\n await execAsync('which arecord');\n return { available: true, tool: 'arecord' };\n } else if (platform === 'win32') {\n // Windows: Use ffmpeg\n await execAsync('where ffmpeg');\n return { available: true, tool: 'ffmpeg' };\n }\n return { available: false, tool: 'none', message: 'Unsupported platform' };\n } catch (_error) {\n // Tool not found\n const toolName = platform === 'darwin' ? 'sox' : platform === 'linux' ? 'arecord' : 'ffmpeg';\n const installCmd =\n platform === 'darwin'\n ? 'brew install sox'\n : platform === 'linux'\n ? 'sudo apt-get install alsa-utils'\n : 'Download ffmpeg from ffmpeg.org';\n return {\n available: false,\n tool: toolName,\n message: `Install with: ${installCmd}`,\n };\n }\n }\n\n private startTranscriptionLoop(): void {\n if (this.recordingInterval) {\n return;\n }\n\n // Start continuous recording with periodic transcription\n this.recordingInterval = setInterval(async () => {\n if (!this.isRecording) {\n await this.recordAndTranscribe();\n }\n }, this.config.transcriptionInterval);\n\n logger.info('[AudioCapture] Started transcription loop');\n }\n\n async recordAndTranscribe(): Promise<string | null> {\n if (this.isRecording) {\n logger.warn('[AudioCapture] Already recording');\n return null;\n }\n\n this.isRecording = true;\n const audioFile = path.join(process.cwd(), `audio_${Date.now()}.wav`);\n\n try {\n logger.debug('[AudioCapture] Starting audio recording...');\n\n // Record audio for the specified duration\n await this.recordAudio(audioFile, this.config.transcriptionInterval / 1000);\n\n logger.debug('[AudioCapture] Recording complete, transcribing...');\n\n // Transcribe using runtime model\n const audioBuffer = await fs.readFile(audioFile);\n const transcription = await this.runtime.useModel(ModelType.TRANSCRIPTION, {\n audio: audioBuffer,\n language: 'en',\n });\n\n // Clean up audio file\n await fs.unlink(audioFile).catch(() => {});\n\n if (transcription && typeof transcription === 'string' && transcription.trim()) {\n logger.info(`[AudioCapture] Transcribed: \"${transcription}\"`);\n\n // Create a memory of what was heard\n await this.createAudioMemory(transcription);\n\n return transcription;\n }\n\n return null;\n } catch (error) {\n logger.error('[AudioCapture] Recording/transcription failed:', error);\n await fs.unlink(audioFile).catch(() => {});\n return null;\n } finally {\n this.isRecording = false;\n }\n }\n\n private async recordAudio(outputPath: string, duration: number): Promise<void> {\n const platform = process.platform;\n\n try {\n if (platform === 'darwin') {\n // macOS: Use sox\n const _device = this.config.device || 'default';\n await execAsync(\n `sox -d -r ${this.config.sampleRate} -c ${this.config.channels} -b 16 \"${outputPath}\" trim 0 ${duration}`\n );\n } else if (platform === 'linux') {\n // Linux: Use arecord\n const device = this.config.device || 'default';\n await execAsync(\n `arecord -D ${device} -f S16_LE -r ${this.config.sampleRate} -c ${this.config.channels} -d ${duration} \"${outputPath}\"`\n );\n } else if (platform === 'win32') {\n // Windows: Use ffmpeg with DirectShow\n const device = this.config.device || 'Microphone';\n await execAsync(\n `ffmpeg -f dshow -i audio=\"${device}\" -t ${duration} -acodec pcm_s16le -ar ${this.config.sampleRate} -ac ${this.config.channels} \"${outputPath}\" -y`\n );\n } else {\n throw new Error(`Unsupported platform: ${platform}`);\n }\n } catch (error: any) {\n logger.error('[AudioCapture] Audio recording failed:', error);\n throw error;\n }\n }\n\n private async createAudioMemory(transcription: string): Promise<void> {\n try {\n const _memory = {\n content: {\n text: `[Audio Transcription] ${transcription}`,\n type: 'audio_transcription',\n source: 'microphone',\n timestamp: Date.now(),\n },\n metadata: {\n isAudioTranscription: true,\n duration: this.config.transcriptionInterval,\n },\n };\n\n // Store in agent's context\n // Note: createMemory requires runtime implementation specific to the database adapter\n // For now, we'll just log it\n logger.info('[AudioCapture] Audio transcription stored in context');\n } catch (error) {\n logger.error('[AudioCapture] Failed to create audio memory:', error);\n }\n }\n\n async listAudioDevices(): Promise<string[]> {\n const platform = process.platform;\n const devices: string[] = [];\n\n try {\n if (platform === 'darwin') {\n // macOS: List audio devices using system_profiler\n const { stdout } = await execAsync('system_profiler SPAudioDataType -json');\n const data = JSON.parse(stdout);\n\n if (data.SPAudioDataType) {\n for (const device of data.SPAudioDataType) {\n if (device._name && device._name.includes('Input')) {\n devices.push(device._name);\n }\n }\n }\n } else if (platform === 'linux') {\n // Linux: List ALSA devices\n const { stdout } = await execAsync('arecord -l');\n const lines = stdout.split('\\n');\n\n for (const line of lines) {\n if (line.includes('card')) {\n const match = line.match(/card (\\d+):.*\\[(.*?)\\]/);\n if (match) {\n devices.push(`hw:${match[1]}`);\n }\n }\n }\n } else if (platform === 'win32') {\n // Windows: List audio devices using ffmpeg\n try {\n const { stdout } = await execAsync('ffmpeg -list_devices true -f dshow -i dummy 2>&1');\n const lines = stdout.split('\\n');\n let isAudioSection = false;\n\n for (const line of lines) {\n if (line.includes('DirectShow audio devices')) {\n isAudioSection = true;\n } else if (isAudioSection && line.includes('\"')) {\n const match = line.match(/\"([^\"]+)\"/);\n if (match) {\n devices.push(match[1]);\n }\n }\n }\n } catch (_error) {\n // ffmpeg returns non-zero exit code when listing devices\n // but we can still parse the output\n }\n }\n } catch (error) {\n logger.error('[AudioCapture] Failed to list audio devices:', error);\n }\n\n return devices;\n }\n\n isActive(): boolean {\n return this.config.enabled && this.recordingInterval !== null;\n }\n\n async stop(): Promise<void> {\n logger.info('[AudioCapture] Stopping audio capture...');\n\n if (this.recordingInterval) {\n clearInterval(this.recordingInterval);\n this.recordingInterval = null;\n }\n\n // Wait for any ongoing recording to complete\n while (this.isRecording) {\n await new Promise((resolve) => setTimeout(resolve, 100));\n }\n\n logger.info('[AudioCapture] Audio capture stopped');\n }\n}\n"]}
@@ -1 +0,0 @@
1
- export {};
@@ -1,97 +0,0 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- var __generator = (this && this.__generator) || function (thisArg, body) {
12
- var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
13
- return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
14
- function verb(n) { return function (v) { return step([n, v]); }; }
15
- function step(op) {
16
- if (f) throw new TypeError("Generator is already executing.");
17
- while (g && (g = 0, op[0] && (_ = 0)), _) try {
18
- if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
19
- if (y = 0, t) op = [op[0] & 2, t.value];
20
- switch (op[0]) {
21
- case 0: case 1: t = op; break;
22
- case 4: _.label++; return { value: op[1], done: false };
23
- case 5: _.label++; y = op[1]; op = [0]; continue;
24
- case 7: op = _.ops.pop(); _.trys.pop(); continue;
25
- default:
26
- if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
27
- if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
28
- if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
29
- if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
30
- if (t[2]) _.ops.pop();
31
- _.trys.pop(); continue;
32
- }
33
- op = body.call(thisArg, _);
34
- } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
35
- if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
36
- }
37
- };
38
- Object.defineProperty(exports, "__esModule", { value: true });
39
- var bun_test_1 = require("bun:test");
40
- var index_1 = require("./index");
41
- // Simplified TestSuite implementation for local use
42
- var TestSuite = /** @class */ (function () {
43
- function TestSuite(name, config) {
44
- this.name = name;
45
- this.config = config;
46
- }
47
- TestSuite.prototype.addTest = function (test) {
48
- var _this = this;
49
- (0, bun_test_1.it)(test.name, function () { return __awaiter(_this, void 0, void 0, function () {
50
- var context;
51
- return __generator(this, function (_a) {
52
- switch (_a.label) {
53
- case 0:
54
- context = this.config.beforeEach ? this.config.beforeEach() : {};
55
- return [4 /*yield*/, test.fn(context)];
56
- case 1:
57
- _a.sent();
58
- return [2 /*return*/];
59
- }
60
- });
61
- }); });
62
- };
63
- TestSuite.prototype.run = function () {
64
- // No-op, bun:test handles execution
65
- };
66
- return TestSuite;
67
- }());
68
- var createUnitTest = function (config) {
69
- return config;
70
- };
71
- (0, bun_test_1.describe)('Vision Plugin', function () {
72
- var visionPluginSuite = new TestSuite('Vision Plugin', {});
73
- visionPluginSuite.addTest(createUnitTest({
74
- name: 'should export a valid plugin',
75
- fn: function () {
76
- (0, bun_test_1.expect)(index_1.visionPlugin).toBeDefined();
77
- (0, bun_test_1.expect)(index_1.visionPlugin.name).toBe('vision');
78
- (0, bun_test_1.expect)(index_1.visionPlugin.description).toBeDefined();
79
- },
80
- }));
81
- visionPluginSuite.addTest(createUnitTest({
82
- name: 'should have actions',
83
- fn: function () {
84
- (0, bun_test_1.expect)(index_1.visionPlugin.actions).toBeDefined();
85
- (0, bun_test_1.expect)(Array.isArray(index_1.visionPlugin.actions)).toBe(true);
86
- },
87
- }));
88
- visionPluginSuite.addTest(createUnitTest({
89
- name: 'should have providers',
90
- fn: function () {
91
- (0, bun_test_1.expect)(index_1.visionPlugin.providers).toBeDefined();
92
- (0, bun_test_1.expect)(Array.isArray(index_1.visionPlugin.providers)).toBe(true);
93
- },
94
- }));
95
- visionPluginSuite.run();
96
- });
97
- //# sourceMappingURL=basic.test.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"basic.test.js","sourceRoot":"","sources":["../src/basic.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,qCAAgD;AAChD,iCAAuC;AAEvC,oDAAoD;AACpD;IACE,mBACU,IAAY,EACZ,MAAW;QADX,SAAI,GAAJ,IAAI,CAAQ;QACZ,WAAM,GAAN,MAAM,CAAK;IAClB,CAAC;IAEJ,2BAAO,GAAP,UAAQ,IAAS;QAAjB,iBAKC;QAJC,IAAA,aAAE,EAAC,IAAI,CAAC,IAAI,EAAE;;;;;wBACN,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACvE,qBAAM,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,EAAA;;wBAAtB,SAAsB,CAAC;;;;aACxB,CAAC,CAAC;IACL,CAAC;IAED,uBAAG,GAAH;QACE,oCAAoC;IACtC,CAAC;IACH,gBAAC;AAAD,CAAC,AAhBD,IAgBC;AAED,IAAM,cAAc,GAAG,UAAC,MAAqE;IAC3F,OAAA,MAAM;AAAN,CAAM,CAAC;AAET,IAAA,mBAAQ,EAAC,eAAe,EAAE;IACxB,IAAM,iBAAiB,GAAG,IAAI,SAAS,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IAE7D,iBAAiB,CAAC,OAAO,CACvB,cAAc,CAAC;QACb,IAAI,EAAE,8BAA8B;QACpC,EAAE,EAAE;YACF,IAAA,iBAAM,EAAC,oBAAY,CAAC,CAAC,WAAW,EAAE,CAAC;YACnC,IAAA,iBAAM,EAAC,oBAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACzC,IAAA,iBAAM,EAAC,oBAAY,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE,CAAC;QACjD,CAAC;KACF,CAAC,CACH,CAAC;IAEF,iBAAiB,CAAC,OAAO,CACvB,cAAc,CAAC;QACb,IAAI,EAAE,qBAAqB;QAC3B,EAAE,EAAE;YACF,IAAA,iBAAM,EAAC,oBAAY,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC3C,IAAA,iBAAM,EAAC,KAAK,CAAC,OAAO,CAAC,oBAAY,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC;KACF,CAAC,CACH,CAAC;IAEF,iBAAiB,CAAC,OAAO,CACvB,cAAc,CAAC;QACb,IAAI,EAAE,uBAAuB;QAC7B,EAAE,EAAE;YACF,IAAA,iBAAM,EAAC,oBAAY,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;YAC7C,IAAA,iBAAM,EAAC,KAAK,CAAC,OAAO,CAAC,oBAAY,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,CAAC;KACF,CAAC,CACH,CAAC;IAEF,iBAAiB,CAAC,GAAG,EAAE,CAAC;AAC1B,CAAC,CAAC,CAAC","sourcesContent":["import { describe, it, expect } from 'bun:test';\nimport { visionPlugin } from './index';\n\n// Simplified TestSuite implementation for local use\nclass TestSuite {\n constructor(\n private name: string,\n private config: any\n ) {}\n\n addTest(test: any) {\n it(test.name, async () => {\n const context = this.config.beforeEach ? this.config.beforeEach() : {};\n await test.fn(context);\n });\n }\n\n run() {\n // No-op, bun:test handles execution\n }\n}\n\nconst createUnitTest = (config: { name: string; fn: (context?: any) => Promise<void> | void }) =>\n config;\n\ndescribe('Vision Plugin', () => {\n const visionPluginSuite = new TestSuite('Vision Plugin', {});\n\n visionPluginSuite.addTest(\n createUnitTest({\n name: 'should export a valid plugin',\n fn: () => {\n expect(visionPlugin).toBeDefined();\n expect(visionPlugin.name).toBe('vision');\n expect(visionPlugin.description).toBeDefined();\n },\n })\n );\n\n visionPluginSuite.addTest(\n createUnitTest({\n name: 'should have actions',\n fn: () => {\n expect(visionPlugin.actions).toBeDefined();\n expect(Array.isArray(visionPlugin.actions)).toBe(true);\n },\n })\n );\n\n visionPluginSuite.addTest(\n createUnitTest({\n name: 'should have providers',\n fn: () => {\n expect(visionPlugin.providers).toBeDefined();\n expect(Array.isArray(visionPlugin.providers)).toBe(true);\n },\n })\n );\n\n visionPluginSuite.run();\n});\n"]}
package/dist/config.d.ts DELETED
@@ -1,73 +0,0 @@
1
- import { z } from 'zod';
2
- import type { VisionConfig } from './types';
3
- export declare const defaultVisionConfig: VisionConfig;
4
- export declare const VisionConfigSchema: z.ZodObject<{
5
- cameraName: z.ZodOptional<z.ZodString>;
6
- enableCamera: z.ZodDefault<z.ZodBoolean>;
7
- pixelChangeThreshold: z.ZodDefault<z.ZodNumber>;
8
- updateInterval: z.ZodDefault<z.ZodNumber>;
9
- enableObjectDetection: z.ZodDefault<z.ZodBoolean>;
10
- objectConfidenceThreshold: z.ZodDefault<z.ZodNumber>;
11
- enablePoseDetection: z.ZodDefault<z.ZodBoolean>;
12
- poseConfidenceThreshold: z.ZodDefault<z.ZodNumber>;
13
- tfUpdateInterval: z.ZodDefault<z.ZodNumber>;
14
- vlmUpdateInterval: z.ZodDefault<z.ZodNumber>;
15
- tfChangeThreshold: z.ZodDefault<z.ZodNumber>;
16
- vlmChangeThreshold: z.ZodDefault<z.ZodNumber>;
17
- visionMode: z.ZodDefault<z.ZodEnum<{
18
- OFF: "OFF";
19
- CAMERA: "CAMERA";
20
- SCREEN: "SCREEN";
21
- BOTH: "BOTH";
22
- }>>;
23
- screenCaptureInterval: z.ZodDefault<z.ZodNumber>;
24
- tileSize: z.ZodDefault<z.ZodNumber>;
25
- tileProcessingOrder: z.ZodDefault<z.ZodEnum<{
26
- priority: "priority";
27
- sequential: "sequential";
28
- random: "random";
29
- }>>;
30
- maxConcurrentTiles: z.ZodDefault<z.ZodNumber>;
31
- ocrEnabled: z.ZodDefault<z.ZodBoolean>;
32
- ocrLanguage: z.ZodDefault<z.ZodString>;
33
- ocrConfidenceThreshold: z.ZodDefault<z.ZodNumber>;
34
- florence2Enabled: z.ZodDefault<z.ZodBoolean>;
35
- florence2Provider: z.ZodOptional<z.ZodEnum<{
36
- local: "local";
37
- azure: "azure";
38
- huggingface: "huggingface";
39
- replicate: "replicate";
40
- }>>;
41
- florence2Endpoint: z.ZodOptional<z.ZodString>;
42
- florence2ApiKey: z.ZodOptional<z.ZodString>;
43
- florence2Timeout: z.ZodDefault<z.ZodNumber>;
44
- enableFaceRecognition: z.ZodDefault<z.ZodBoolean>;
45
- faceMatchThreshold: z.ZodDefault<z.ZodNumber>;
46
- maxFaceProfiles: z.ZodDefault<z.ZodNumber>;
47
- entityTimeout: z.ZodDefault<z.ZodNumber>;
48
- maxTrackedEntities: z.ZodDefault<z.ZodNumber>;
49
- enableGPUAcceleration: z.ZodDefault<z.ZodBoolean>;
50
- maxMemoryUsageMB: z.ZodDefault<z.ZodNumber>;
51
- debugMode: z.ZodDefault<z.ZodBoolean>;
52
- logLevel: z.ZodDefault<z.ZodEnum<{
53
- error: "error";
54
- debug: "debug";
55
- info: "info";
56
- warn: "warn";
57
- }>>;
58
- }, z.core.$strip>;
59
- export type VisionConfigInput = z.input<typeof VisionConfigSchema>;
60
- export type VisionConfigOutput = z.output<typeof VisionConfigSchema>;
61
- export declare class ConfigurationManager {
62
- private config;
63
- private runtime;
64
- constructor(runtime: any);
65
- private loadConfiguration;
66
- private getSetting;
67
- private getBooleanSetting;
68
- private getNumberSetting;
69
- get(): VisionConfigOutput;
70
- update(updates: Partial<VisionConfigInput>): void;
71
- toLegacyFormat(): VisionConfig;
72
- static getPreset(name: string): Partial<VisionConfigInput>;
73
- }