@elizaos/plugin-vision 1.2.1

package/dist/audio-capture-stream.js

@@ -0,0 +1,516 @@
+"use strict";
+var __extends = (this && this.__extends) || (function () {
+    var extendStatics = function (d, b) {
+        extendStatics = Object.setPrototypeOf ||
+            ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
+            function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
+        return extendStatics(d, b);
+    };
+    return function (d, b) {
+        if (typeof b !== "function" && b !== null)
+            throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
+        extendStatics(d, b);
+        function __() { this.constructor = d; }
+        d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
+    };
+})();
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamingAudioCaptureService = void 0;
+var core_1 = require("@elizaos/core");
+var child_process_1 = require("child_process");
+var events_1 = require("events");
+var StreamingAudioCaptureService = /** @class */ (function (_super) {
+    __extends(StreamingAudioCaptureService, _super);
+    function StreamingAudioCaptureService(runtime, config) {
+        var _this = _super.call(this) || this;
+        _this.captureProcess = null;
+        _this.isCapturing = false;
+        _this.audioBuffer = [];
+        _this.isSpeaking = false;
+        _this.lastSpeechTime = 0;
+        _this.silenceTimer = null;
+        _this.transcriptionInProgress = false;
+        _this.currentTranscription = '';
+        _this.responseTimer = null;
+        _this.runtime = runtime;
+        _this.config = __assign({ sampleRate: 16000, channels: 1, vadThreshold: 0.01, silenceTimeout: 1500, responseDelay: 3000, chunkSize: 4096 }, config);
+        return _this;
+    }
+    StreamingAudioCaptureService.prototype.initialize = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var error_1;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (!this.config.enabled) {
+                            core_1.logger.info('[StreamingAudio] Audio capture disabled');
+                            return [2 /*return*/];
+                        }
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        core_1.logger.info('[StreamingAudio] Initializing streaming audio capture...');
+                        // Start continuous audio capture
+                        return [4 /*yield*/, this.startContinuousCapture()];
+                    case 2:
+                        // Start continuous audio capture
+                        _a.sent();
+                        core_1.logger.info('[StreamingAudio] Streaming audio capture initialized');
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_1 = _a.sent();
+                        core_1.logger.error('[StreamingAudio] Failed to initialize:', error_1);
+                        throw error_1;
+                    case 4: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    StreamingAudioCaptureService.prototype.startContinuousCapture = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var platform, command, args;
+            var _this = this;
+            var _a, _b;
+            return __generator(this, function (_c) {
+                platform = process.platform;
+                if (platform === 'darwin') {
+                    // macOS: Use sox for continuous capture
+                    command = 'sox';
+                    args = [
+                        '-d', // default input device
+                        '-r',
+                        this.config.sampleRate.toString(),
+                        '-c',
+                        this.config.channels.toString(),
+                        '-b',
+                        '16',
+                        '-e',
+                        'signed',
+                        '-t',
+                        'raw',
+                        '-', // output to stdout
+                    ];
+                }
+                else if (platform === 'linux') {
+                    // Linux: Use arecord
+                    command = 'arecord';
+                    args = [
+                        '-D',
+                        this.config.device || 'default',
+                        '-f',
+                        'S16_LE',
+                        '-r',
+                        this.config.sampleRate.toString(),
+                        '-c',
+                        this.config.channels.toString(),
+                        '-t',
+                        'raw',
+                        '-', // output to stdout
+                    ];
+                }
+                else if (platform === 'win32') {
+                    // Windows: Use ffmpeg
+                    command = 'ffmpeg';
+                    args = [
+                        '-f',
+                        'dshow',
+                        '-i',
+                        "audio=\"".concat(this.config.device || 'Microphone', "\""),
+                        '-acodec',
+                        'pcm_s16le',
+                        '-ar',
+                        this.config.sampleRate.toString(),
+                        '-ac',
+                        this.config.channels.toString(),
+                        '-f',
+                        's16le',
+                        'pipe:1', // output to stdout
+                    ];
+                }
+                else {
+                    throw new Error("Unsupported platform: ".concat(platform));
+                }
+                this.captureProcess = (0, child_process_1.spawn)(command, args);
+                this.isCapturing = true;
+                // Handle audio data stream
+                (_a = this.captureProcess.stdout) === null || _a === void 0 ? void 0 : _a.on('data', function (chunk) {
+                    _this.processAudioChunk(chunk);
+                });
+                (_b = this.captureProcess.stderr) === null || _b === void 0 ? void 0 : _b.on('data', function (data) {
+                    core_1.logger.debug('[StreamingAudio] Capture stderr:', data.toString());
+                });
+                this.captureProcess.on('error', function (error) {
+                    core_1.logger.error('[StreamingAudio] Capture process error:', error);
+                    _this.isCapturing = false;
+                });
+                this.captureProcess.on('exit', function (code) {
+                    core_1.logger.info('[StreamingAudio] Capture process exited with code:', code);
+                    _this.isCapturing = false;
+                });
+                return [2 /*return*/];
+            });
+        });
+    };
+    StreamingAudioCaptureService.prototype.processAudioChunk = function (chunk) {
+        var _this = this;
+        // Calculate audio energy for VAD
+        var energy = this.calculateEnergy(chunk);
+        var timestamp = Date.now();
+        // Store chunk
+        var audioChunk = { data: chunk, timestamp: timestamp, energy: energy };
+        // Voice Activity Detection
+        if (energy > this.config.vadThreshold) {
+            if (!this.isSpeaking) {
+                // Speech started
+                this.isSpeaking = true;
+                this.lastSpeechTime = timestamp;
+                core_1.logger.debug('[StreamingAudio] Speech detected, starting recording');
+                this.emit('speechStart');
+                // Clear any pending response
+                if (this.responseTimer) {
+                    clearTimeout(this.responseTimer);
+                    this.responseTimer = null;
+                    core_1.logger.debug('[StreamingAudio] Cancelled pending response due to new speech');
+                }
+            }
+            // Add to buffer
+            this.audioBuffer.push(audioChunk);
+            this.lastSpeechTime = timestamp;
+            // Reset silence timer
+            if (this.silenceTimer) {
+                clearTimeout(this.silenceTimer);
+            }
+            // Start streaming transcription if not already running
+            if (!this.transcriptionInProgress) {
+                this.startStreamingTranscription();
+            }
+        }
+        else if (this.isSpeaking) {
+            // Currently in speech but detected silence
+            this.audioBuffer.push(audioChunk);
+            // Set timer for end of speech
+            if (!this.silenceTimer) {
+                this.silenceTimer = setTimeout(function () {
+                    _this.endSpeech();
+                }, this.config.silenceTimeout);
+            }
+        }
+        // Clean up old chunks (keep last 30 seconds)
+        var cutoffTime = timestamp - 30000;
+        this.audioBuffer = this.audioBuffer.filter(function (c) { return c.timestamp > cutoffTime; });
+    };
+    StreamingAudioCaptureService.prototype.calculateEnergy = function (chunk) {
+        // Calculate RMS energy of audio chunk
+        var sum = 0;
+        var samples = chunk.length / 2; // 16-bit samples
+        for (var i = 0; i < chunk.length; i += 2) {
+            var sample = chunk.readInt16LE(i);
+            sum += sample * sample;
+        }
+        var rms = Math.sqrt(sum / samples);
+        return rms / 32768; // Normalize to 0-1
+    };
+    StreamingAudioCaptureService.prototype.startStreamingTranscription = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var audioData, result, error_2;
+            var _this = this;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        if (this.transcriptionInProgress) {
+                            return [2 /*return*/];
+                        }
+                        this.transcriptionInProgress = true;
+                        core_1.logger.debug('[StreamingAudio] Starting streaming transcription');
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        audioData = this.getRecentAudioData();
+                        if (audioData.length === 0) {
+                            this.transcriptionInProgress = false;
+                            return [2 /*return*/];
+                        }
+                        return [4 /*yield*/, this.transcribeAudio(audioData)];
+                    case 2:
+                        result = _a.sent();
+                        if (result && result.trim()) {
+                            this.currentTranscription = result;
+                            core_1.logger.info("[StreamingAudio] Partial transcription: \"".concat(result, "\""));
+                            this.emit('transcription', { text: result, isFinal: false });
+                        }
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_2 = _a.sent();
+                        core_1.logger.error('[StreamingAudio] Transcription error:', error_2);
+                        return [3 /*break*/, 4];
+                    case 4:
+                        this.transcriptionInProgress = false;
+                        // Continue transcription if still speaking
+                        if (this.isSpeaking) {
+                            setTimeout(function () { return _this.startStreamingTranscription(); }, 500);
+                        }
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
+    StreamingAudioCaptureService.prototype.endSpeech = function () {
+        if (!this.isSpeaking) {
+            return;
+        }
+        this.isSpeaking = false;
+        this.silenceTimer = null;
+        core_1.logger.debug('[StreamingAudio] Speech ended');
+        this.emit('speechEnd');
+        // Get final transcription
+        this.processFinalTranscription();
+    };
+    StreamingAudioCaptureService.prototype.processFinalTranscription = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            var audioData, finalText_1, error_3;
+            var _this = this;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        audioData = this.getRecentAudioData();
+                        if (audioData.length === 0) {
+                            return [2 /*return*/];
+                        }
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, 4, 5]);
+                        return [4 /*yield*/, this.transcribeAudio(audioData)];
+                    case 2:
+                        finalText_1 = _a.sent();
+                        if (finalText_1 && finalText_1.trim()) {
+                            this.currentTranscription = finalText_1;
+                            core_1.logger.info("[StreamingAudio] Final transcription: \"".concat(finalText_1, "\""));
+                            this.emit('transcription', { text: finalText_1, isFinal: true });
+                            // Set timer for response generation
+                            this.responseTimer = setTimeout(function () {
+                                _this.generateResponse(finalText_1);
+                            }, this.config.responseDelay);
+                        }
+                        return [3 /*break*/, 5];
+                    case 3:
+                        error_3 = _a.sent();
+                        core_1.logger.error('[StreamingAudio] Final transcription error:', error_3);
+                        return [3 /*break*/, 5];
+                    case 4:
+                        // Clear audio buffer
+                        this.audioBuffer = [];
+                        this.currentTranscription = '';
+                        return [7 /*endfinally*/];
+                    case 5: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    StreamingAudioCaptureService.prototype.getRecentAudioData = function () {
+        if (this.audioBuffer.length === 0) {
+            return Buffer.alloc(0);
+        }
+        // Get audio from start of speech to now
+        var startTime = this.audioBuffer[0].timestamp;
+        var relevantChunks = this.audioBuffer.filter(function (c) { return c.timestamp >= startTime; });
+        // Combine chunks
+        var totalLength = relevantChunks.reduce(function (sum, c) { return sum + c.data.length; }, 0);
+        var combined = Buffer.alloc(totalLength);
+        var offset = 0;
+        for (var _i = 0, relevantChunks_1 = relevantChunks; _i < relevantChunks_1.length; _i++) {
+            var chunk = relevantChunks_1[_i];
+            chunk.data.copy(combined, offset);
+            offset += chunk.data.length;
+        }
+        return combined;
+    };
+    StreamingAudioCaptureService.prototype.transcribeAudio = function (audioData) {
+        return __awaiter(this, void 0, void 0, function () {
+            var wavBuffer, result, error_4;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        _a.trys.push([0, 2, , 3]);
+                        wavBuffer = this.rawToWav(audioData);
+                        return [4 /*yield*/, this.runtime.useModel(core_1.ModelType.TRANSCRIPTION, {
+                                audio: wavBuffer,
+                                language: 'en',
+                                stream: true, // Request streaming if supported
+                            })];
+                    case 1:
+                        result = _a.sent();
+                        return [2 /*return*/, result];
+                    case 2:
+                        error_4 = _a.sent();
+                        core_1.logger.error('[StreamingAudio] Transcription failed:', error_4);
+                        return [2 /*return*/, null];
+                    case 3: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    StreamingAudioCaptureService.prototype.rawToWav = function (rawData) {
+        // Create WAV header
+        var sampleRate = this.config.sampleRate;
+        var channels = this.config.channels;
+        var bitsPerSample = 16;
+        var byteRate = sampleRate * channels * (bitsPerSample / 8);
+        var blockAlign = channels * (bitsPerSample / 8);
+        var dataSize = rawData.length;
+        var fileSize = 36 + dataSize;
+        var header = Buffer.alloc(44);
+        // RIFF chunk
+        header.write('RIFF', 0);
+        header.writeUInt32LE(fileSize, 4);
+        header.write('WAVE', 8);
+        // fmt chunk
+        header.write('fmt ', 12);
+        header.writeUInt32LE(16, 16); // fmt chunk size
+        header.writeUInt16LE(1, 20); // PCM format
+        header.writeUInt16LE(channels, 22);
+        header.writeUInt32LE(sampleRate, 24);
+        header.writeUInt32LE(byteRate, 28);
+        header.writeUInt16LE(blockAlign, 32);
+        header.writeUInt16LE(bitsPerSample, 34);
+        // data chunk
+        header.write('data', 36);
+        header.writeUInt32LE(dataSize, 40);
+        return Buffer.concat([header, rawData]);
+    };
+    StreamingAudioCaptureService.prototype.generateResponse = function (transcription) {
+        return __awaiter(this, void 0, void 0, function () {
+            var error_5;
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        this.responseTimer = null;
+                        _a.label = 1;
+                    case 1:
+                        _a.trys.push([1, 3, , 4]);
+                        // Create audio memory
+                        return [4 /*yield*/, this.createAudioMemory(transcription)];
+                    case 2:
+                        // Create audio memory
+                        _a.sent();
+                        // Emit event for response generation
+                        this.emit('utteranceComplete', transcription);
+                        return [3 /*break*/, 4];
+                    case 3:
+                        error_5 = _a.sent();
+                        core_1.logger.error('[StreamingAudio] Response generation error:', error_5);
+                        return [3 /*break*/, 4];
+                    case 4: return [2 /*return*/];
+                }
+            });
+        });
+    };
+    StreamingAudioCaptureService.prototype.createAudioMemory = function (transcription) {
+        return __awaiter(this, void 0, void 0, function () {
+            var _memory;
+            return __generator(this, function (_a) {
+                try {
+                    _memory = {
+                        content: {
+                            text: "[Audio] ".concat(transcription),
+                            type: 'audio_transcription',
+                            source: 'microphone_streaming',
+                            timestamp: Date.now(),
+                        },
+                        metadata: {
+                            isAudioTranscription: true,
+                            streaming: true,
+                        },
+                    };
+                    core_1.logger.info('[StreamingAudio] Audio transcription stored in context');
+                }
+                catch (error) {
+                    core_1.logger.error('[StreamingAudio] Failed to create audio memory:', error);
+                }
+                return [2 /*return*/];
+            });
+        });
+    };
+    StreamingAudioCaptureService.prototype.stop = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                core_1.logger.info('[StreamingAudio] Stopping audio capture...');
+                if (this.captureProcess) {
+                    this.captureProcess.kill();
+                    this.captureProcess = null;
+                }
+                if (this.silenceTimer) {
+                    clearTimeout(this.silenceTimer);
+                    this.silenceTimer = null;
+                }
+                if (this.responseTimer) {
+                    clearTimeout(this.responseTimer);
+                    this.responseTimer = null;
+                }
+                this.isCapturing = false;
+                this.isSpeaking = false;
+                this.audioBuffer = [];
+                core_1.logger.info('[StreamingAudio] Audio capture stopped');
+                return [2 /*return*/];
+            });
+        });
+    };
+    StreamingAudioCaptureService.prototype.isActive = function () {
+        return this.isCapturing;
+    };
+    StreamingAudioCaptureService.prototype.getCurrentTranscription = function () {
+        return this.currentTranscription;
+    };
+    StreamingAudioCaptureService.prototype.isSpeechActive = function () {
+        return this.isSpeaking;
+    };
+    return StreamingAudioCaptureService;
+}(events_1.EventEmitter));
+exports.StreamingAudioCaptureService = StreamingAudioCaptureService;
+//# sourceMappingURL=audio-capture-stream.js.map

package/dist/audio-capture-stream.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio-capture-stream.js","sourceRoot":"","sources":["../src/audio-capture-stream.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,sCAAsE;AACtE,+CAAoD;AACpD,iCAAsC;AAmBtC;IAAkD,gDAAY;IAa5D,sCAAY,OAAsB,EAAE,MAA4B;QAC9D,YAAA,MAAK,WAAE,SAAC;QAXF,oBAAc,GAAwB,IAAI,CAAC;QAC3C,iBAAW,GAAG,KAAK,CAAC;QACpB,iBAAW,GAAiB,EAAE,CAAC;QAC/B,gBAAU,GAAG,KAAK,CAAC;QACnB,oBAAc,GAAG,CAAC,CAAC;QACnB,kBAAY,GAA0B,IAAI,CAAC;QAC3C,6BAAuB,GAAG,KAAK,CAAC;QAChC,0BAAoB,GAAG,EAAE,CAAC;QAC1B,mBAAa,GAA0B,IAAI,CAAC;QAIlD,KAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,KAAI,CAAC,MAAM,cACT,UAAU,EAAE,KAAK,EACjB,QAAQ,EAAE,CAAC,EACX,YAAY,EAAE,IAAI,EAClB,cAAc,EAAE,IAAI,EACpB,aAAa,EAAE,IAAI,EACnB,SAAS,EAAE,IAAI,IACZ,MAAM,CACV,CAAC;;IACJ,CAAC;IAEK,iDAAU,GAAhB;;;;;;wBACE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;4BACzB,aAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;4BACvD,sBAAO;wBACT,CAAC;;;;wBAGC,aAAM,CAAC,IAAI,CAAC,0DAA0D,CAAC,CAAC;wBAExE,iCAAiC;wBACjC,qBAAM,IAAI,CAAC,sBAAsB,EAAE,EAAA;;wBADnC,iCAAiC;wBACjC,SAAmC,CAAC;wBAEpC,aAAM,CAAC,IAAI,CAAC,sDAAsD,CAAC,CAAC;;;;wBAEpE,aAAM,CAAC,KAAK,CAAC,wCAAwC,EAAE,OAAK,CAAC,CAAC;wBAC9D,MAAM,OAAK,CAAC;;;;;KAEf;IAEa,6DAAsB,GAApC;;;;;;gBACQ,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;gBAIlC,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;oBAC1B,wCAAwC;oBACxC,OAAO,GAAG,KAAK,CAAC;oBAChB,IAAI,GAAG;wBACL,IAAI,EAAE,uBAAuB;wBAC7B,IAAI;wBACJ,IAAI,CAAC,MAAM,CAAC,UAAW,CAAC,QAAQ,EAAE;wBAClC,IAAI;wBACJ,IAAI,CAAC,MAAM,CAAC,QAAS,CAAC,QAAQ,EAAE;wBAChC,IAAI;wBACJ,IAAI;wBACJ,IAAI;wBACJ,QAAQ;wBACR,IAAI;wBACJ,KAAK;wBACL,GAAG,EAAE,mBAAmB;qBACzB,CAAC;gBACJ,CAAC;qBAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;oBAChC,qBAAqB;oBACrB,OAAO,GAAG,SAAS,CAAC;oBACpB,IAAI,GAAG;wBACL,IAAI;wBACJ,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,SAAS;wBAC/B,IAAI;wBACJ,QAAQ;wBACR,IAAI;wBACJ,IAAI,CAAC,MAAM,CAAC,UAAW,CAAC,QAAQ,EAAE;wBAClC,IAAI;wBACJ,IAAI,CAAC,MAAM,CAAC,QAAS,CAAC,QAAQ,EAAE;wBAChC,IAAI;wBACJ,KAAK;wBACL,GAAG,EAAE,mBAAmB;qBACzB,CAAC;gBACJ,CAAC;qBAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;oBAChC,sBAAsB;oBACtB,OAAO,GAAG,QAAQ,CAAC;oBACnB,IAAI,GAAG;wBACL,IAAI;wBACJ,OAAO;wBACP,IAAI;wBACJ,kBAAU,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,YAAY,OAAG;wBAC/C,SAAS;wBACT,WAAW;wBACX,KAAK;wBACL,IAAI,CAAC,MAAM,CAAC,UAAW,CAAC,QAAQ,EAAE;wBAClC,KAAK;wBACL,IAAI,CAAC,MAAM,CAAC,QAAS,CAAC,QAAQ,EAAE;wBAChC,IAAI;wBACJ,OAAO;wBACP,QAAQ,EAAE,mBAAmB;qBAC9B,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CAAC,gCAAyB,QAAQ,CAAE,CAAC,CAAC;gBACvD,CAAC;gBAED,IAAI,CAAC,cAAc,GAAG,IAAA,qBAAK,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC3C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;gBAExB,2BAA2B;gBAC3B,MAAA,IAAI,CAAC,cAAc,CAAC,MAAM,0CAAE,EAAE,CAAC,MAAM,EAAE,UAAC,KAAa;oBACnD,KAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBAChC,CAAC,CAAC,CAAC;gBAEH,MAAA,IAAI,CAAC,cAAc,CAAC,MAAM,0CAAE,EAAE,CAAC,MAAM,EAAE,UAAC,IAAI;oBAC1C,aAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACpE,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,UAAC,KAAK;oBACpC,aAAM,CAAC,KAAK,CAAC,yCAAyC,EAAE,KAAK,CAAC,CAAC;oBAC/D,KAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBAEH,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,UAAC,IAAI;oBAClC,aAAM,CAAC,IAAI,CAAC,oDAAoD,EAAE,IAAI,CAAC,CAAC;oBACxE,KAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBAC3B,CAAC,CAAC,CAAC;;;;KACJ;IAEO,wDAAiB,GAAzB,UAA0B,KAAa;QAAvC,iBAqDC;QApDC,iCAAiC;QACjC,IAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,cAAc;QACd,IAAM,UAAU,GAAe,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,WAAA,EAAE,MAAM,QAAA,EAAE,CAAC;QAElE,2BAA2B;QAC3B,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,YAAa,EAAE,CAAC;YACvC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrB,iBAAiB;gBACjB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;gBACvB,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;gBAChC,aAAM,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBACrE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAEzB,6BAA6B;gBAC7B,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACvB,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBACjC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;oBAC1B,aAAM,CAAC,KAAK,CAAC,+DAA+D,CAAC,CAAC;gBAChF,CAAC;YACH,CAAC;YAED,gBAAgB;YAChB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAClC,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;YAEhC,sBAAsB;YACtB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACtB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAClC,CAAC;YAED,uDAAuD;YACvD,IAAI,CAAC,IAAI,CAAC,uBAAuB,EAAE,CAAC;gBAClC,IAAI,CAAC,2BAA2B,EAAE,CAAC;YACrC,CAAC;QACH,CAAC;aAAM,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YAC3B,2CAA2C;YAC3C,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAElC,8BAA8B;YAC9B,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;gBACvB,IAAI,CAAC,YAAY,GAAG,UAAU,CAAC;oBAC7B,KAAI,CAAC,SAAS,EAAE,CAAC;gBACnB,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,cAAe,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAED,6CAA6C;QAC7C,IAAM,UAAU,GAAG,SAAS,GAAG,KAAK,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,SAAS,GAAG,UAAU,EAAxB,CAAwB,CAAC,CAAC;IAC9E,CAAC;IAEO,sDAAe,GAAvB,UAAwB,KAAa;QACnC,sCAAsC;QACtC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAM,OAAO,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,iBAAiB;QAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,IAAM,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YACpC,GAAG,IAAI,MAAM,GAAG,MAAM,CAAC;QACzB,CAAC;QAED,IAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC;QACrC,OAAO,GAAG,GAAG,KAAK,CAAC,CAAC,mBAAmB;IACzC,CAAC;IAEa,kEAA2B,GAAzC;;;;;;;wBACE,IAAI,IAAI,CAAC,uBAAuB,EAAE,CAAC;4BACjC,sBAAO;wBACT,CAAC;wBAED,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;wBACpC,aAAM,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;;;;wBAI1D,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBAE5C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC3B,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;4BACrC,sBAAO;wBACT,CAAC;wBAGc,qBAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,EAAA;;wBAA9C,MAAM,GAAG,SAAqC;wBAEpD,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;4BAC5B,IAAI,CAAC,oBAAoB,GAAG,MAAM,CAAC;4BACnC,aAAM,CAAC,IAAI,CAAC,oDAA4C,MAAM,OAAG,CAAC,CAAC;4BACnE,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;wBAC/D,CAAC;;;;wBAED,aAAM,CAAC,KAAK,CAAC,uCAAuC,EAAE,OAAK,CAAC,CAAC;;;wBAG/D,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;wBAErC,2CAA2C;wBAC3C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;4BACpB,UAAU,CAAC,cAAM,OAAA,KAAI,CAAC,2BAA2B,EAAE,EAAlC,CAAkC,EAAE,GAAG,CAAC,CAAC;wBAC5D,CAAC;;;;;KACF;IAEO,gDAAS,GAAjB;QACE,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,aAAM,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAC9C,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAEvB,0BAA0B;QAC1B,IAAI,CAAC,yBAAyB,EAAE,CAAC;IACnC,CAAC;IAEa,gEAAyB,GAAvC;;;;;;;wBACQ,SAAS,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;wBAE5C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;4BAC3B,sBAAO;wBACT,CAAC;;;;wBAImB,qBAAM,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,EAAA;;wBAAjD,cAAY,SAAqC;wBAEvD,IAAI,WAAS,IAAI,WAAS,CAAC,IAAI,EAAE,EAAE,CAAC;4BAClC,IAAI,CAAC,oBAAoB,GAAG,WAAS,CAAC;4BACtC,aAAM,CAAC,IAAI,CAAC,kDAA0C,WAAS,OAAG,CAAC,CAAC;4BACpE,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,IAAI,EAAE,WAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;4BAE/D,oCAAoC;4BACpC,IAAI,CAAC,aAAa,GAAG,UAAU,CAAC;gCAC9B,KAAI,CAAC,gBAAgB,CAAC,WAAS,CAAC,CAAC;4BACnC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,aAAc,CAAC,CAAC;wBACjC,CAAC;;;;wBAED,aAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE,OAAK,CAAC,CAAC;;;wBAEnE,qBAAqB;wBACrB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;wBACtB,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;;;;;;KAElC;IAEO,yDAAkB,GAA1B;QACE,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzB,CAAC;QAED,wCAAwC;QACxC,IAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAChD,IAAM,cAAc,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAC,CAAC,IAAK,OAAA,CAAC,CAAC,SAAS,IAAI,SAAS,EAAxB,CAAwB,CAAC,CAAC;QAEhF,iBAAiB;QACjB,IAAM,WAAW,GAAG,cAAc,CAAC,MAAM,CAAC,UAAC,GAAG,EAAE,CAAC,IAAK,OAAA,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,EAAnB,CAAmB,EAAE,CAAC,CAAC,CAAC;QAC9E,IAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,KAAoB,UAAc,EAAd,iCAAc,EAAd,4BAAc,EAAd,IAAc,EAAE,CAAC;YAAhC,IAAM,KAAK,uBAAA;YACd,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAC9B,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEa,sDAAe,GAA7B,UAA8B,SAAiB;;;;;;;wBAGrC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;wBAG5B,qBAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAS,CAAC,aAAa,EAAE;gCAClE,KAAK,EAAE,SAAS;gCAChB,QAAQ,EAAE,IAAI;gCACd,MAAM,EAAE,IAAI,EAAE,iCAAiC;6BAChD,CAAC,EAAA;;wBAJI,MAAM,GAAG,SAIb;wBAEF,sBAAO,MAAgB,EAAC;;;wBAExB,aAAM,CAAC,KAAK,CAAC,wCAAwC,EAAE,OAAK,CAAC,CAAC;wBAC9D,sBAAO,IAAI,EAAC;;;;;KAEf;IAEO,+CAAQ,GAAhB,UAAiB,OAAe;QAC9B,oBAAoB;QACpB,IAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,UAAW,CAAC;QAC3C,IAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAS,CAAC;QACvC,IAAM,aAAa,GAAG,EAAE,CAAC;QACzB,IAAM,QAAQ,GAAG,UAAU,GAAG,QAAQ,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;QAC7D,IAAM,UAAU,GAAG,QAAQ,GAAG,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;QAClD,IAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC;QAChC,IAAM,QAAQ,GAAG,EAAE,GAAG,QAAQ,CAAC;QAE/B,IAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAEhC,aAAa;QACb,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACxB,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAExB,YAAY;QACZ,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzB,MAAM,CAAC,aAAa,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB;QAC/C,MAAM,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa;QAC1C,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACnC,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACrC,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACnC,MAAM,CAAC,aAAa,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QACrC,MAAM,CAAC,aAAa,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAExC,aAAa;QACb,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzB,MAAM,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAEnC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1C,CAAC;IAEa,uDAAgB,GAA9B,UAA+B,aAAqB;;;;;;wBAClD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;;;;wBAGxB,sBAAsB;wBACtB,qBAAM,IAAI,CAAC,iBAAiB,CAAC,aAAa,CAAC,EAAA;;wBAD3C,sBAAsB;wBACtB,SAA2C,CAAC;wBAE5C,qCAAqC;wBACrC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;;;;wBAE9C,aAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE,OAAK,CAAC,CAAC;;;;;;KAEtE;IAEa,wDAAiB,GAA/B,UAAgC,aAAqB;;;;gBACnD,IAAI,CAAC;oBACG,OAAO,GAAG;wBACd,OAAO,EAAE;4BACP,IAAI,EAAE,kBAAW,aAAa,CAAE;4BAChC,IAAI,EAAE,qBAAqB;4BAC3B,MAAM,EAAE,sBAAsB;4BAC9B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;yBACtB;wBACD,QAAQ,EAAE;4BACR,oBAAoB,EAAE,IAAI;4BAC1B,SAAS,EAAE,IAAI;yBAChB;qBACF,CAAC;oBAEF,aAAM,CAAC,IAAI,CAAC,wDAAwD,CAAC,CAAC;gBACxE,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,aAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,KAAK,CAAC,CAAC;gBACzE,CAAC;;;;KACF;IAEK,2CAAI,GAAV;;;gBACE,aAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAC;gBAE1D,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBACxB,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;oBAC3B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;gBAC7B,CAAC;gBAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtB,YAAY,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;oBAChC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;gBAC3B,CAAC;gBAED,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;oBACvB,YAAY,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;oBACjC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAC5B,CAAC;gBAED,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;gBACzB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;gBACxB,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;gBAEtB,aAAM,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;;;;KACvD;IAED,+CAAQ,GAAR;QACE,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED,8DAAuB,GAAvB;QACE,OAAO,IAAI,CAAC,oBAAoB,CAAC;IACnC,CAAC;IAED,qDAAc,GAAd;QACE,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IACH,mCAAC;AAAD,CAAC,AAzaD,CAAkD,qBAAY,GAya7D;AAzaY,oEAA4B","sourcesContent":["import { logger, ModelType, type IAgentRuntime } from '@elizaos/core';\nimport { spawn, ChildProcess } from 'child_process';\nimport { EventEmitter } from 'events';\n\nexport interface StreamingAudioConfig {\n  enabled: boolean;\n  device?: string;\n  sampleRate?: number;\n  channels?: number;\n  vadThreshold?: number; // 0-1, energy threshold for speech detection\n  silenceTimeout?: number; // ms to wait before considering speech ended\n  responseDelay?: number; // ms to wait before processing (for interruption detection)\n  chunkSize?: number; // bytes per chunk for streaming\n}\n\ninterface AudioChunk {\n  data: Buffer;\n  timestamp: number;\n  energy: number;\n}\n\nexport class StreamingAudioCaptureService extends EventEmitter {\n  private runtime: IAgentRuntime;\n  private config: StreamingAudioConfig;\n  private captureProcess: ChildProcess | null = null;\n  private isCapturing = false;\n  private audioBuffer: AudioChunk[] = [];\n  private isSpeaking = false;\n  private lastSpeechTime = 0;\n  private silenceTimer: NodeJS.Timeout | null = null;\n  private transcriptionInProgress = false;\n  private currentTranscription = '';\n  private responseTimer: NodeJS.Timeout | null = null;\n\n  constructor(runtime: IAgentRuntime, config: StreamingAudioConfig) {\n    super();\n    this.runtime = runtime;\n    this.config = {\n      sampleRate: 16000,\n      channels: 1,\n      vadThreshold: 0.01,\n      silenceTimeout: 1500, // 1.5 seconds of silence to end speech\n      responseDelay: 3000, // 3 seconds before response (allows for interruption)\n      chunkSize: 4096,\n      ...config,\n    };\n  }\n\n  async initialize(): Promise<void> {\n    if (!this.config.enabled) {\n      logger.info('[StreamingAudio] Audio capture disabled');\n      return;\n    }\n\n    try {\n      logger.info('[StreamingAudio] Initializing streaming audio capture...');\n\n      // Start continuous audio capture\n      await this.startContinuousCapture();\n\n      logger.info('[StreamingAudio] Streaming audio capture initialized');\n    } catch (error) {\n      logger.error('[StreamingAudio] Failed to initialize:', error);\n      throw error;\n    }\n  }\n\n  private async startContinuousCapture(): Promise<void> {\n    const platform = process.platform;\n    let command: string;\n    let args: string[];\n\n    if (platform === 'darwin') {\n      // macOS: Use sox for continuous capture\n      command = 'sox';\n      args = [\n        '-d', // default input device\n        '-r',\n        this.config.sampleRate!.toString(),\n        '-c',\n        this.config.channels!.toString(),\n        '-b',\n        '16',\n        '-e',\n        'signed',\n        '-t',\n        'raw',\n        '-', // output to stdout\n      ];\n    } else if (platform === 'linux') {\n      // Linux: Use arecord\n      command = 'arecord';\n      args = [\n        '-D',\n        this.config.device || 'default',\n        '-f',\n        'S16_LE',\n        '-r',\n        this.config.sampleRate!.toString(),\n        '-c',\n        this.config.channels!.toString(),\n        '-t',\n        'raw',\n        '-', // output to stdout\n      ];\n    } else if (platform === 'win32') {\n      // Windows: Use ffmpeg\n      command = 'ffmpeg';\n      args = [\n        '-f',\n        'dshow',\n        '-i',\n        `audio=\"${this.config.device || 'Microphone'}\"`,\n        '-acodec',\n        'pcm_s16le',\n        '-ar',\n        this.config.sampleRate!.toString(),\n        '-ac',\n        this.config.channels!.toString(),\n        '-f',\n        's16le',\n        'pipe:1', // output to stdout\n      ];\n    } else {\n      throw new Error(`Unsupported platform: ${platform}`);\n    }\n\n    this.captureProcess = spawn(command, args);\n    this.isCapturing = true;\n\n    // Handle audio data stream\n    this.captureProcess.stdout?.on('data', (chunk: Buffer) => {\n      this.processAudioChunk(chunk);\n    });\n\n    this.captureProcess.stderr?.on('data', (data) => {\n      logger.debug('[StreamingAudio] Capture stderr:', data.toString());\n    });\n\n    this.captureProcess.on('error', (error) => {\n      logger.error('[StreamingAudio] Capture process error:', error);\n      this.isCapturing = false;\n    });\n\n    this.captureProcess.on('exit', (code) => {\n      logger.info('[StreamingAudio] Capture process exited with code:', code);\n      this.isCapturing = false;\n    });\n  }\n\n  private processAudioChunk(chunk: Buffer): void {\n    // Calculate audio energy for VAD\n    const energy = this.calculateEnergy(chunk);\n    const timestamp = Date.now();\n\n    // Store chunk\n    const audioChunk: AudioChunk = { data: chunk, timestamp, energy };\n\n    // Voice Activity Detection\n    if (energy > this.config.vadThreshold!) {\n      if (!this.isSpeaking) {\n        // Speech started\n        this.isSpeaking = true;\n        this.lastSpeechTime = timestamp;\n        logger.debug('[StreamingAudio] Speech detected, starting recording');\n        this.emit('speechStart');\n\n        // Clear any pending response\n        if (this.responseTimer) {\n          clearTimeout(this.responseTimer);\n          this.responseTimer = null;\n          logger.debug('[StreamingAudio] Cancelled pending response due to new speech');\n        }\n      }\n\n      // Add to buffer\n      this.audioBuffer.push(audioChunk);\n      this.lastSpeechTime = timestamp;\n\n      // Reset silence timer\n      if (this.silenceTimer) {\n        clearTimeout(this.silenceTimer);\n      }\n\n      // Start streaming transcription if not already running\n      if (!this.transcriptionInProgress) {\n        this.startStreamingTranscription();\n      }\n    } else if (this.isSpeaking) {\n      // Currently in speech but detected silence\n      this.audioBuffer.push(audioChunk);\n\n      // Set timer for end of speech\n      if (!this.silenceTimer) {\n        this.silenceTimer = setTimeout(() => {\n          this.endSpeech();\n        }, this.config.silenceTimeout!);\n      }\n    }\n\n    // Clean up old chunks (keep last 30 seconds)\n    const cutoffTime = timestamp - 30000;\n    this.audioBuffer = this.audioBuffer.filter((c) => c.timestamp > cutoffTime);\n  }\n\n  private calculateEnergy(chunk: Buffer): number {\n    // Calculate RMS energy of audio chunk\n    let sum = 0;\n    const samples = chunk.length / 2; // 16-bit samples\n\n    for (let i = 0; i < chunk.length; i += 2) {\n      const sample = chunk.readInt16LE(i);\n      sum += sample * sample;\n    }\n\n    const rms = Math.sqrt(sum / samples);\n    return rms / 32768; // Normalize to 0-1\n  }\n\n  private async startStreamingTranscription(): Promise<void> {\n    if (this.transcriptionInProgress) {\n      return;\n    }\n\n    this.transcriptionInProgress = true;\n    logger.debug('[StreamingAudio] Starting streaming transcription');\n\n    try {\n      // Get audio data from buffer\n      const audioData = this.getRecentAudioData();\n\n      if (audioData.length === 0) {\n        this.transcriptionInProgress = false;\n        return;\n      }\n\n      // Use streaming transcription if available, otherwise batch\n      const result = await this.transcribeAudio(audioData);\n\n      if (result && result.trim()) {\n        this.currentTranscription = result;\n        logger.info(`[StreamingAudio] Partial transcription: \"${result}\"`);\n        this.emit('transcription', { text: result, isFinal: false });\n      }\n    } catch (error) {\n      logger.error('[StreamingAudio] Transcription error:', error);\n    }\n\n    this.transcriptionInProgress = false;\n\n    // Continue transcription if still speaking\n    if (this.isSpeaking) {\n      setTimeout(() => this.startStreamingTranscription(), 500);\n    }\n  }\n\n  private endSpeech(): void {\n    if (!this.isSpeaking) {\n      return;\n    }\n\n    this.isSpeaking = false;\n    this.silenceTimer = null;\n    logger.debug('[StreamingAudio] Speech ended');\n    this.emit('speechEnd');\n\n    // Get final transcription\n    this.processFinalTranscription();\n  }\n\n  private async processFinalTranscription(): Promise<void> {\n    const audioData = this.getRecentAudioData();\n\n    if (audioData.length === 0) {\n      return;\n    }\n\n    try {\n      // Get final transcription\n      const finalText = await this.transcribeAudio(audioData);\n\n      if (finalText && finalText.trim()) {\n        this.currentTranscription = finalText;\n        logger.info(`[StreamingAudio] Final transcription: \"${finalText}\"`);\n        this.emit('transcription', { text: finalText, isFinal: true });\n\n        // Set timer for response generation\n        this.responseTimer = setTimeout(() => {\n          this.generateResponse(finalText);\n        }, this.config.responseDelay!);\n      }\n    } catch (error) {\n      logger.error('[StreamingAudio] Final transcription error:', error);\n    } finally {\n      // Clear audio buffer\n      this.audioBuffer = [];\n      this.currentTranscription = '';\n    }\n  }\n\n  private getRecentAudioData(): Buffer {\n    if (this.audioBuffer.length === 0) {\n      return Buffer.alloc(0);\n    }\n\n    // Get audio from start of speech to now\n    const startTime = this.audioBuffer[0].timestamp;\n    const relevantChunks = this.audioBuffer.filter((c) => c.timestamp >= startTime);\n\n    // Combine chunks\n    const totalLength = relevantChunks.reduce((sum, c) => sum + c.data.length, 0);\n    const combined = Buffer.alloc(totalLength);\n    let offset = 0;\n\n    for (const chunk of relevantChunks) {\n      chunk.data.copy(combined, offset);\n      offset += chunk.data.length;\n    }\n\n    return combined;\n  }\n\n  private async transcribeAudio(audioData: Buffer): Promise<string | null> {\n    try {\n      // Convert raw audio to WAV format\n      const wavBuffer = this.rawToWav(audioData);\n\n      // Use runtime transcription model\n      const result = await this.runtime.useModel(ModelType.TRANSCRIPTION, {\n        audio: wavBuffer,\n        language: 'en',\n        stream: true, // Request streaming if supported\n      });\n\n      return result as string;\n    } catch (error) {\n      logger.error('[StreamingAudio] Transcription failed:', error);\n      return null;\n    }\n  }\n\n  private rawToWav(rawData: Buffer): Buffer {\n    // Create WAV header\n    const sampleRate = this.config.sampleRate!;\n    const channels = this.config.channels!;\n    const bitsPerSample = 16;\n    const byteRate = sampleRate * channels * (bitsPerSample / 8);\n    const blockAlign = channels * (bitsPerSample / 8);\n    const dataSize = rawData.length;\n    const fileSize = 36 + dataSize;\n\n    const header = Buffer.alloc(44);\n\n    // RIFF chunk\n    header.write('RIFF', 0);\n    header.writeUInt32LE(fileSize, 4);\n    header.write('WAVE', 8);\n\n    // fmt chunk\n    header.write('fmt ', 12);\n    header.writeUInt32LE(16, 16); // fmt chunk size\n    header.writeUInt16LE(1, 20); // PCM format\n    header.writeUInt16LE(channels, 22);\n    header.writeUInt32LE(sampleRate, 24);\n    header.writeUInt32LE(byteRate, 28);\n    header.writeUInt16LE(blockAlign, 32);\n    header.writeUInt16LE(bitsPerSample, 34);\n\n    // data chunk\n    header.write('data', 36);\n    header.writeUInt32LE(dataSize, 40);\n\n    return Buffer.concat([header, rawData]);\n  }\n\n  private async generateResponse(transcription: string): Promise<void> {\n    this.responseTimer = null;\n\n    try {\n      // Create audio memory\n      await this.createAudioMemory(transcription);\n\n      // Emit event for response generation\n      this.emit('utteranceComplete', transcription);\n    } catch (error) {\n      logger.error('[StreamingAudio] Response generation error:', error);\n    }\n  }\n\n  private async createAudioMemory(transcription: string): Promise<void> {\n    try {\n      const _memory = {\n        content: {\n          text: `[Audio] ${transcription}`,\n          type: 'audio_transcription',\n          source: 'microphone_streaming',\n          timestamp: Date.now(),\n        },\n        metadata: {\n          isAudioTranscription: true,\n          streaming: true,\n        },\n      };\n\n      logger.info('[StreamingAudio] Audio transcription stored in context');\n    } catch (error) {\n      logger.error('[StreamingAudio] Failed to create audio memory:', error);\n    }\n  }\n\n  async stop(): Promise<void> {\n    logger.info('[StreamingAudio] Stopping audio capture...');\n\n    if (this.captureProcess) {\n      this.captureProcess.kill();\n      this.captureProcess = null;\n    }\n\n    if (this.silenceTimer) {\n      clearTimeout(this.silenceTimer);\n      this.silenceTimer = null;\n    }\n\n    if (this.responseTimer) {\n      clearTimeout(this.responseTimer);\n      this.responseTimer = null;\n    }\n\n    this.isCapturing = false;\n    this.isSpeaking = false;\n    this.audioBuffer = [];\n\n    logger.info('[StreamingAudio] Audio capture stopped');\n  }\n\n  isActive(): boolean {\n    return this.isCapturing;\n  }\n\n  getCurrentTranscription(): string {\n    return this.currentTranscription;\n  }\n\n  isSpeechActive(): boolean {\n    return this.isSpeaking;\n  }\n}\n"]}

package/dist/audio-capture.d.ts

@@ -0,0 +1,25 @@
+import { type IAgentRuntime } from '@elizaos/core';
+export interface AudioConfig {
+    enabled: boolean;
+    transcriptionInterval: number;
+    device?: string;
+    sampleRate?: number;
+    channels?: number;
+}
+export declare class AudioCaptureService {
+    private runtime;
+    private config;
+    private isRecording;
+    private recordingInterval;
+    private currentRecordingPath;
+    constructor(runtime: IAgentRuntime, config: AudioConfig);
+    initialize(): Promise<void>;
+    private checkAudioTools;
+    private startTranscriptionLoop;
+    recordAndTranscribe(): Promise<string | null>;
+    private recordAudio;
+    private createAudioMemory;
+    listAudioDevices(): Promise<string[]>;
+    isActive(): boolean;
+    stop(): Promise<void>;
+}