aiden-runtime 3.16.0

package/dist/core/visionAnalyze.js

@@ -0,0 +1,161 @@
+"use strict";
+// ============================================================
+// DevOS — Autonomous AI Execution System
+// Copyright (c) 2026 Shiva Deore. All rights reserved.
+// ============================================================
+//
+// core/visionAnalyze.ts — Image analysis via vision-capable providers.
+//
+// Provider chain (first available wins):
+//   1. Anthropic claude-3-5-sonnet  (ANTHROPIC_API_KEY)
+//   2. OpenAI gpt-4o                (OPENAI_API_KEY)
+//   3. Ollama llava                 (local, no key needed)
+//
+// Accepts local file paths (→ base64) or HTTP/HTTPS URLs.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.analyzeImage = analyzeImage;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const axios_1 = __importDefault(require("axios"));
+// ── Media type resolver ───────────────────────────────────────────────────────
+function extToMediaType(ext) {
+    const map = {
+        jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png',
+        gif: 'image/gif', webp: 'image/webp', bmp: 'image/bmp',
+    };
+    return map[ext.toLowerCase().replace(/^\./, '')] ?? 'image/jpeg';
+}
+// ── Core function ─────────────────────────────────────────────────────────────
+/**
+ * Analyze an image using the first available vision-capable provider.
+ *
+ * @param imageSource  File path (absolute or relative) or HTTP(S) URL.
+ * @param prompt       Instruction prompt (default: describe the image).
+ * @returns            VisionResult with description, provider, model, timing.
+ */
+async function analyzeImage(imageSource, prompt = 'Describe this image in detail.') {
+    const start = Date.now();
+    // Resolve image data
+    const isUrl = imageSource.startsWith('http://') || imageSource.startsWith('https://');
+    let base64Data = '';
+    let mediaType = 'image/jpeg';
+    if (!isUrl) {
+        const absPath = path.isAbsolute(imageSource)
+            ? imageSource
+            : path.resolve(process.cwd(), imageSource);
+        const buf = fs.readFileSync(absPath);
+        base64Data = buf.toString('base64');
+        mediaType = extToMediaType(path.extname(absPath));
+    }
+    // ── Provider 1: Anthropic ─────────────────────────────────────────────────
+    const anthropicKey = process.env.ANTHROPIC_API_KEY;
+    if (anthropicKey) {
+        try {
+            const imageBlock = isUrl
+                ? { type: 'image', source: { type: 'url', url: imageSource } }
+                : { type: 'image', source: { type: 'base64', media_type: mediaType, data: base64Data } };
+            const res = await axios_1.default.post('https://api.anthropic.com/v1/messages', {
+                model: 'claude-3-5-sonnet-20241022',
+                max_tokens: 1024,
+                messages: [{ role: 'user', content: [imageBlock, { type: 'text', text: prompt }] }],
+            }, {
+                headers: {
+                    'x-api-key': anthropicKey,
+                    'anthropic-version': '2023-06-01',
+                    'content-type': 'application/json',
+                },
+                timeout: 30000,
+            });
+            const description = (res.data?.content?.[0]?.text ?? '').trim();
+            if (description) {
+                return { description, provider: 'anthropic', modelUsed: 'claude-3-5-sonnet-20241022', durationMs: Date.now() - start };
+            }
+        }
+        catch { /* fall through */ }
+    }
+    // ── Provider 2: OpenAI ────────────────────────────────────────────────────
+    const openaiKey = process.env.OPENAI_API_KEY;
+    if (openaiKey) {
+        try {
+            const imageUrl = isUrl
+                ? imageSource
+                : `data:${mediaType};base64,${base64Data}`;
+            const res = await axios_1.default.post('https://api.openai.com/v1/chat/completions', {
+                model: 'gpt-4o',
+                max_tokens: 1024,
+                messages: [{
+                        role: 'user',
+                        content: [
+                            { type: 'image_url', image_url: { url: imageUrl } },
+                            { type: 'text', text: prompt },
+                        ],
+                    }],
+            }, {
+                headers: { Authorization: `Bearer ${openaiKey}`, 'content-type': 'application/json' },
+                timeout: 30000,
+            });
+            const description = (res.data?.choices?.[0]?.message?.content ?? '').trim();
+            if (description) {
+                return { description, provider: 'openai', modelUsed: 'gpt-4o', durationMs: Date.now() - start };
+            }
+        }
+        catch { /* fall through */ }
+    }
+    // ── Provider 3: Ollama llava ──────────────────────────────────────────────
+    const ollamaBase = (process.env.OLLAMA_BASE_URL ?? 'http://127.0.0.1:11434').replace(/\/$/, '');
+    // For URLs we need to download first so Ollama can receive base64
+    let ollamaBase64 = base64Data;
+    if (isUrl) {
+        try {
+            const imgRes = await axios_1.default.get(imageSource, { responseType: 'arraybuffer', timeout: 15000 });
+            ollamaBase64 = Buffer.from(imgRes.data).toString('base64');
+        }
+        catch (e) {
+            throw new Error(`vision_analyze: all providers failed (could not download URL for Ollama). ${e.message}`);
+        }
+    }
+    try {
+        const res = await axios_1.default.post(`${ollamaBase}/api/generate`, { model: 'llava', prompt, images: [ollamaBase64], stream: false }, { timeout: 60000 });
+        const description = (res.data?.response ?? '').trim();
+        return { description, provider: 'ollama', modelUsed: 'llava', durationMs: Date.now() - start };
+    }
+    catch (e) {
+        throw new Error(`vision_analyze: all providers exhausted. ${e.message}`);
+    }
+}

package/dist/core/voice/audio.js

@@ -0,0 +1,187 @@
+"use strict";
+// ============================================================
+// DevOS — Autonomous AI Execution System
+// Copyright (c) 2026 Shiva Deore. All rights reserved.
+// ============================================================
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.recordAudio = recordAudio;
+exports.playAudio = playAudio;
+exports.checkAudioAvailable = checkAudioAvailable;
+// core/voice/audio.ts — Platform audio I/O: recording + playback.
+//
+// Recording:   Windows MCI (P/Invoke via PowerShell, no third-party dep)
+// Playback:    Windows Media Player (presentationCore) → Start-Process fallback
+//
+// Cross-platform note: recording falls back to arecord/sox on Linux/macOS.
+// Playback falls back to afplay (macOS) / paplay (Linux).
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const execAsync = (0, util_1.promisify)(child_process_1.exec);
+const WORKSPACE = path_1.default.join(process.cwd(), 'workspace');
+function ensureWorkspace() {
+    if (!fs_1.default.existsSync(WORKSPACE))
+        fs_1.default.mkdirSync(WORKSPACE, { recursive: true });
+}
+// ── Record audio from microphone ──────────────────────────────────────────────
+/**
+ * Record audio from the default microphone.
+ *
+ * @param durationSeconds  Recording length in seconds (default 5).
+ * @param outputPath       Where to save the .wav file. Defaults to a temp file in workspace/.
+ * @returns                Resolved path to the recorded file.
+ */
+async function recordAudio(durationSeconds = 5, outputPath) {
+    ensureWorkspace();
+    const outPath = outputPath ?? path_1.default.join(WORKSPACE, `recording_${Date.now()}.wav`);
+    const durationMs = Math.round(durationSeconds * 1000);
+    if (process.platform === 'win32') {
+        return _recordWindows(outPath, durationMs);
+    }
+    else {
+        return _recordUnix(outPath, durationMs);
+    }
+}
+async function _recordWindows(outputPath, durationMs) {
+    const escapedPath = outputPath.replace(/\\/g, '\\\\');
+    const psScript = `
+Add-Type -TypeDefinition @"
+using System;
+using System.Threading;
+using System.Runtime.InteropServices;
+
+public class AudioRecorder {
+  [DllImport("winmm.dll")]
+  private static extern int mciSendString(
+    string command,
+    System.Text.StringBuilder returnValue,
+    int returnLength,
+    IntPtr winHandle
+  );
+
+  public static void Record(string outputPath, int durationMs) {
+    mciSendString("open new Type waveaudio Alias recsound", null, 0, IntPtr.Zero);
+    mciSendString("set recsound channels 1 bitspersample 16 samplespersec 16000", null, 0, IntPtr.Zero);
+    mciSendString("record recsound", null, 0, IntPtr.Zero);
+    Thread.Sleep(durationMs);
+    mciSendString("stop recsound", null, 0, IntPtr.Zero);
+    mciSendString("save recsound " + outputPath, null, 0, IntPtr.Zero);
+    mciSendString("close recsound", null, 0, IntPtr.Zero);
+  }
+}
+"@
+[AudioRecorder]::Record("${escapedPath}", ${durationMs})
+Write-Output "${outputPath}"
+`.trim();
+    const psFile = path_1.default.join(WORKSPACE, `record_${Date.now()}.ps1`);
+    fs_1.default.writeFileSync(psFile, psScript);
+    try {
+        await execAsync(`powershell.exe -ExecutionPolicy Bypass -File "${psFile}"`, { timeout: durationMs + 8000 });
+        return outputPath;
+    }
+    catch (e) {
+        throw new Error(`[Audio] Recording failed: ${e.message}`);
+    }
+    finally {
+        try {
+            fs_1.default.unlinkSync(psFile);
+        }
+        catch { /* ignore */ }
+    }
+}
+async function _recordUnix(outputPath, durationMs) {
+    const seconds = Math.ceil(durationMs / 1000);
+    // Try sox first, then arecord
+    try {
+        await execAsync(`sox -d -t wav "${outputPath}" trim 0 ${seconds}`, { timeout: durationMs + 5000 });
+    }
+    catch {
+        await execAsync(`arecord -d ${seconds} -f S16_LE -r 16000 -c 1 "${outputPath}"`, { timeout: durationMs + 5000 });
+    }
+    return outputPath;
+}
+// ── Play audio ────────────────────────────────────────────────────────────────
+/**
+ * Play an audio file (wav / mp3 / ogg).
+ * Non-blocking on Windows (fires MediaPlayer async); blocking on Unix.
+ *
+ * @param audioSource  Path to audio file, or raw audio Buffer.
+ */
+async function playAudio(audioSource) {
+    ensureWorkspace();
+    let filePath;
+    let isTmp = false;
+    if (Buffer.isBuffer(audioSource)) {
+        filePath = path_1.default.join(WORKSPACE, `playback_${Date.now()}.wav`);
+        fs_1.default.writeFileSync(filePath, audioSource);
+        isTmp = true;
+    }
+    else {
+        filePath = audioSource;
+    }
+    if (!fs_1.default.existsSync(filePath)) {
+        throw new Error(`[Audio] File not found: ${filePath}`);
+    }
+    try {
+        if (process.platform === 'win32') {
+            await _playWindows(filePath);
+        }
+        else {
+            await _playUnix(filePath);
+        }
+    }
+    finally {
+        if (isTmp) {
+            setTimeout(() => { try {
+                fs_1.default.unlinkSync(filePath);
+            }
+            catch { /* ignore */ } }, 10000);
+        }
+    }
+}
+async function _playWindows(filePath) {
+    const escaped = filePath.replace(/\\/g, '\\\\');
+    await execAsync(`powershell -Command "Add-Type -AssemblyName presentationCore; $mp = New-Object System.Windows.Media.MediaPlayer; $mp.Open([uri]'${escaped}'); $mp.Play(); Start-Sleep -Seconds 10; $mp.Stop(); $mp.Close()"`, { timeout: 30000 }).catch(async () => {
+        // Fallback: system default media player
+        await execAsync(`powershell -Command "Start-Process '${escaped}'"`, { timeout: 5000 })
+            .catch(() => { });
+    });
+}
+async function _playUnix(filePath) {
+    if (process.platform === 'darwin') {
+        await execAsync(`afplay "${filePath}"`, { timeout: 30000 });
+    }
+    else {
+        try {
+            await execAsync(`paplay "${filePath}"`, { timeout: 30000 });
+        }
+        catch {
+            await execAsync(`aplay "${filePath}"`, { timeout: 30000 });
+        }
+    }
+}
+// ── Availability check ────────────────────────────────────────────────────────
+/** Returns true if audio recording is likely possible on this system. */
+async function checkAudioAvailable() {
+    if (process.platform === 'win32') {
+        try {
+            await execAsync('powershell -Command "Add-Type -AssemblyName System.Speech; Write-Output ok"', { timeout: 3000 });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    // Unix: check for arecord or sox
+    try {
+        await execAsync('which arecord || which sox', { timeout: 2000 });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/core/voice/stt.js

@@ -0,0 +1,226 @@
+"use strict";
+// ============================================================
+// DevOS — Autonomous AI Execution System
+// Copyright (c) 2026 Shiva Deore. All rights reserved.
+// ============================================================
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.transcribe = transcribe;
+exports.getSttProviders = getSttProviders;
+// core/voice/stt.ts — Speech-to-Text with three-provider fallback chain.
+//
+// Priority order (auto-selected at runtime):
+//   1. Groq Whisper API   (GROQ_API_KEY)         — fastest, cloud
+//   2. OpenAI Whisper API (OPENAI_API_KEY)        — reliable, cloud
+//   3. Local Whisper.cpp  (WHISPER_MODEL_PATH)    — offline, no API key
+//
+// If all providers fail: returns { text: '', provider: 'none', error }
+// — never throws; callers check result.text.
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const child_process_1 = require("child_process");
+const util_1 = require("util");
+const axios_1 = __importDefault(require("axios"));
+const execAsync = (0, util_1.promisify)(child_process_1.exec);
+// ── Internal helpers ──────────────────────────────────────────────────────────
+const WORKSPACE = path_1.default.join(process.cwd(), 'workspace');
+function ensureWorkspace() {
+    if (!fs_1.default.existsSync(WORKSPACE))
+        fs_1.default.mkdirSync(WORKSPACE, { recursive: true });
+}
+/** Resolves the audio file path, writing buffer to a temp file if needed. */
+function resolveAudioPath(opts) {
+    if (opts.audioFilePath)
+        return opts.audioFilePath;
+    if (opts.audioBuffer) {
+        ensureWorkspace();
+        const tmp = path_1.default.join(WORKSPACE, `stt_input_${Date.now()}.wav`);
+        fs_1.default.writeFileSync(tmp, opts.audioBuffer);
+        return tmp;
+    }
+    throw new Error('SttOptions: provide audioFilePath or audioBuffer');
+}
+// ── Provider 1 — Groq Whisper ─────────────────────────────────────────────────
+async function transcribeGroq(audioPath, opts) {
+    const apiKey = process.env.GROQ_API_KEY;
+    if (!apiKey)
+        throw new Error('GROQ_API_KEY not set');
+    const timeout = opts.timeoutMs ?? 30000;
+    const t0 = Date.now();
+    const FormData = (await Promise.resolve().then(() => __importStar(require('form-data')))).default;
+    const form = new FormData();
+    form.append('file', fs_1.default.createReadStream(audioPath), path_1.default.basename(audioPath));
+    form.append('model', 'whisper-large-v3');
+    if (opts.language)
+        form.append('language', opts.language);
+    form.append('response_format', 'json');
+    const res = await axios_1.default.post('https://api.groq.com/openai/v1/audio/transcriptions', form, {
+        headers: { ...form.getHeaders(), Authorization: `Bearer ${apiKey}` },
+        timeout,
+    });
+    return {
+        text: (res.data.text ?? '').trim(),
+        provider: 'groq',
+        durationMs: Date.now() - t0,
+    };
+}
+// ── Provider 2 — OpenAI Whisper ───────────────────────────────────────────────
+async function transcribeOpenAI(audioPath, opts) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey)
+        throw new Error('OPENAI_API_KEY not set');
+    const timeout = opts.timeoutMs ?? 30000;
+    const t0 = Date.now();
+    const FormData = (await Promise.resolve().then(() => __importStar(require('form-data')))).default;
+    const form = new FormData();
+    form.append('file', fs_1.default.createReadStream(audioPath), path_1.default.basename(audioPath));
+    form.append('model', 'whisper-1');
+    if (opts.language)
+        form.append('language', opts.language);
+    form.append('response_format', 'json');
+    const res = await axios_1.default.post('https://api.openai.com/v1/audio/transcriptions', form, {
+        headers: { ...form.getHeaders(), Authorization: `Bearer ${apiKey}` },
+        timeout,
+    });
+    return {
+        text: (res.data.text ?? '').trim(),
+        provider: 'openai',
+        durationMs: Date.now() - t0,
+    };
+}
+// ── Provider 3 — Local Whisper.cpp ────────────────────────────────────────────
+async function transcribeLocal(audioPath, opts) {
+    const modelPath = process.env.WHISPER_MODEL_PATH;
+    const t0 = Date.now();
+    const timeout = opts.timeoutMs ?? 60000;
+    // whisper-cli binary: try PATH first, then common install locations
+    const binaryName = process.platform === 'win32' ? 'whisper-cli.exe' : 'whisper-cli';
+    const binaryCandidates = [
+        binaryName,
+        path_1.default.join(process.cwd(), 'bin', binaryName),
+        path_1.default.join(process.cwd(), binaryName),
+    ];
+    let binary = binaryName;
+    for (const candidate of binaryCandidates) {
+        try {
+            await execAsync(`"${candidate}" --version`, { timeout: 3000 });
+            binary = candidate;
+            break;
+        }
+        catch { /* try next */ }
+    }
+    const modelArg = modelPath ? `-m "${modelPath}"` : '';
+    const langArg = opts.language ? `-l ${opts.language}` : '';
+    const cmd = `"${binary}" ${modelArg} ${langArg} -f "${audioPath}" --output-txt`.trim();
+    await execAsync(cmd, { timeout });
+    // whisper-cli writes <audioPath>.txt
+    const txtPath = audioPath + '.txt';
+    if (!fs_1.default.existsSync(txtPath))
+        throw new Error('whisper-cli produced no output file');
+    const text = fs_1.default.readFileSync(txtPath, 'utf-8').trim();
+    try {
+        fs_1.default.unlinkSync(txtPath);
+    }
+    catch { /* ignore */ }
+    return { text, provider: 'local', durationMs: Date.now() - t0 };
+}
+// ── Main exported function ────────────────────────────────────────────────────
+/**
+ * Transcribe audio using the first available provider.
+ * Never throws — always returns an SttResult; check result.error on failure.
+ */
+async function transcribe(options) {
+    const t0 = Date.now();
+    let tmpFile = '';
+    const errors = [];
+    try {
+        const audioPath = resolveAudioPath(options);
+        if (!options.audioFilePath && options.audioBuffer)
+            tmpFile = audioPath;
+        // Provider 1 — Groq
+        try {
+            const r = await transcribeGroq(audioPath, options);
+            console.log(`[STT] Groq Whisper: "${r.text.slice(0, 60)}" (${r.durationMs}ms)`);
+            return r;
+        }
+        catch (e) {
+            errors.push(`groq: ${e.message}`);
+        }
+        // Provider 2 — OpenAI
+        try {
+            const r = await transcribeOpenAI(audioPath, options);
+            console.log(`[STT] OpenAI Whisper: "${r.text.slice(0, 60)}" (${r.durationMs}ms)`);
+            return r;
+        }
+        catch (e) {
+            errors.push(`openai: ${e.message}`);
+        }
+        // Provider 3 — Local Whisper.cpp
+        try {
+            const r = await transcribeLocal(audioPath, options);
+            console.log(`[STT] Local Whisper.cpp: "${r.text.slice(0, 60)}" (${r.durationMs}ms)`);
+            return r;
+        }
+        catch (e) {
+            errors.push(`local: ${e.message}`);
+        }
+        // All failed
+        const errorMsg = errors.join(' | ');
+        console.warn(`[STT] All providers failed: ${errorMsg}`);
+        return { text: '', provider: 'none', durationMs: Date.now() - t0, error: errorMsg };
+    }
+    catch (outer) {
+        return { text: '', provider: 'none', durationMs: Date.now() - t0, error: outer.message };
+    }
+    finally {
+        if (tmpFile) {
+            try {
+                fs_1.default.unlinkSync(tmpFile);
+            }
+            catch { /* ignore */ }
+        }
+    }
+}
+/** Returns which STT providers are likely available (env-key check only). */
+function getSttProviders() {
+    return [
+        { name: 'groq', available: !!process.env.GROQ_API_KEY },
+        { name: 'openai', available: !!process.env.OPENAI_API_KEY },
+        { name: 'local', available: !!process.env.WHISPER_MODEL_PATH },
+    ];
+}