iobroker.assistant 1.4.6 → 1.4.7

package/build/main.js

@@ -37,6 +37,7 @@ const os = __importStar(require("node:os"));
 const fs = __importStar(require("node:fs"));
 const node_child_process_1 = require("node:child_process");
 const node_async_hooks_1 = require("node:async_hooks");
+const node_util_1 = require("node:util");
 const adapter_core_1 = require("@iobroker/adapter-core");
 const path = __importStar(require("node:path"));
 const mcp_server_1 = require("@iobroker/mcp-server");
@@ -69,7 +70,9 @@ const engines_1 = require("./lib/voice/engines");
 const openrouter_1 = require("./lib/voice/openrouter");
 const ttsCache_1 = require("./lib/voice/ttsCache");
 const captures_1 = require("./lib/voice/captures");
+const ffmpegStatic_1 = require("./lib/voice/ffmpegStatic");
 const piAgent_1 = require("./lib/piAgent");
+const media_1 = require("./lib/media");
 const systemAi_1 = require("./lib/systemAi");
 /**
  * How long a rendered weather context line stays valid. Weather adapters refresh every few minutes at best,
@@ -89,6 +92,7 @@ const NLU_DEVICES_TTL_MS = 60_000;
  * confirmation by text.
  */
 const SCRIPT_TOPIC_MS = 2 * 60_000;
+const NO_ATTACHMENTS = { images: [], video: null };
 /** Default for `localLlmTimeoutSec`: after this the local model is stopped and the next tier answers. */
 const LOCAL_LLM_TIMEOUT_SEC = 15;
 /**
@@ -103,6 +107,8 @@ const LOCAL_LLM_MAX_TOKENS = 256;
 /** Fields of the Voice tab the voice test takes from the unsaved form (everything an STT or TTS engine is built from). */
 /** Transports whose answer is spoken by a satellite (and therefore cut at `ttsMaxChars`). */
 const VOICE_TRANSPORTS = new Set(['esphome', 'udp', 'native', 'wyoming']);
+/** Appended to a failed ffmpeg start: the usual cause is a missing binary or a wrong path. */
+const FFMPEG_HINT = ' — install ffmpeg, or download it or set its path on the Voice tab';
 const VOICE_FORM_FIELDS = [
     'voiceLanguage',
     'sttProvider',
@@ -318,6 +324,8 @@ class Assistant extends adapter_core_1.Adapter {
     localLlmLoading = null;
     /** Last reported local-LLM download percent (to throttle state writes / avoid log spam). */
     lastLlmPct = -1;
+    /** A running "Download ffmpeg" - a second click waits for the same download. */
+    ffmpegDownload = null;
     /** UDP voice server for satellites (only when voiceEnabled); null otherwise. */
     voice = null;
     /** Wyoming TCP endpoint (only when wyomingEnabled); null otherwise. */
@@ -652,6 +660,7 @@ class Assistant extends adapter_core_1.Adapter {
                 sttMs: origin.sttMs,
                 audioMs: origin.audioMs,
                 vad: origin.vad,
+                button: origin.button,
             }),
             onSpoken: (device, info) => this.reportSpoken(device, info),
             getHints: () => this.buildSttHints(),
@@ -816,11 +825,10 @@ class Assistant extends adapter_core_1.Adapter {
     }
     /**
      * True when the assistant's reply asks the user something, so a satellite should re-open its mic to
-     * capture the answer without a wake word (→ the `listen` flag of the voice response). Detects a question
-     * mark anywhere, not just at the end, because a clarifying reply often reads "Welches Gerät …? Sag mir …".
+     * capture the answer without a wake word (→ the `listen` flag of the voice response). See {@link endsWithQuestion}.
      */
     expectsFollowUp(answer) {
-        return /[?？]/.test(answer || '');
+        return (0, ask_1.endsWithQuestion)(answer);
     }
     /** A transport synthesised the reply to `source`'s latest request — the journal shows it with that request. */
     reportSpoken(source, info) {
@@ -1995,21 +2003,57 @@ class Assistant extends adapter_core_1.Adapter {
     decodeAudioToPcm(src) {
         return this.runFfmpegDecode(src);
     }
-    /** Decode an in-memory audio buffer (uploaded mp3/wav) to mono 16-bit PCM via ffmpeg (stdin pipe). */
-    decodeAudioBufferToPcm(buf) {
-        return this.runFfmpegDecode('pipe:0', buf);
+    /**
+     * Decode an in-memory audio buffer (uploaded mp3/wav, a Telegram voice message) to mono 16-bit PCM via ffmpeg
+     * (stdin pipe). `sampleRate` 16 kHz is what speech recognition wants; playback keeps 24 kHz.
+     */
+    decodeAudioBufferToPcm(buf, sampleRate = 24000) {
+        return this.runFfmpegDecode('pipe:0', buf, sampleRate);
+    }
+    /** The ffmpeg binary: the configured path, else the downloaded one, else `ffmpeg` from the search path. */
+    ffmpegBin() {
+        return this.config.ffmpegPath?.trim() || this.downloadedFfmpeg() || 'ffmpeg';
+    }
+    /**
+     * The static ffmpeg from "Download ffmpeg" on the Voice tab, '' if there is none. Looked up every time (one
+     * `stat`), so deleting the file falls back to the search path without a restart.
+     */
+    downloadedFfmpeg() {
+        const file = (0, ffmpegStatic_1.ffmpegStaticPath)(this.instanceDataDir());
+        return fs.existsSync(file) ? file : '';
+    }
+    /**
+     * "Download ffmpeg" on the Voice tab: the static build of ffmpeg-static into the data dir, then the same check
+     * as "Check ffmpeg". It is used from now on, unless a path is entered.
+     *
+     * @param configuredPath the path from the (unsaved) form
+     */
+    async downloadFfmpeg(configuredPath) {
+        let file;
+        try {
+            this.ffmpegDownload ??= (0, ffmpegStatic_1.downloadFfmpegStatic)(this.instanceDataDir(), { log: this.log }).finally(() => (this.ffmpegDownload = null));
+            file = await this.ffmpegDownload;
+        }
+        catch (e) {
+            this.log.warn(`Download of ffmpeg failed: ${e.message}`);
+            return { error: `Download failed: ${e.message}` };
+        }
+        const check = await this.testFfmpeg(file);
+        if (check.error || !configuredPath.trim()) {
+            return check;
+        }
+        return { result: `${check.result} — not used while a path to ffmpeg is entered` };
     }
     /** Run ffmpeg to decode `input` (a path/URL, or `pipe:0` with `stdinData`) to mono 16-bit PCM. */
-    runFfmpegDecode(input, stdinData) {
+    runFfmpegDecode(input, stdinData, sampleRate = 24000) {
         return new Promise((resolve, reject) => {
-            const sampleRate = 24000;
             // prettier-ignore
             const args = ['-hide_banner', '-loglevel', 'error', '-i', input, '-ac', '1', '-ar', String(sampleRate), '-f', 's16le', '-'];
-            const proc = (0, node_child_process_1.spawn)('ffmpeg', args);
+            const proc = (0, node_child_process_1.spawn)(this.ffmpegBin(), args);
             const chunks = [];
             proc.stdout.on('data', (d) => chunks.push(d));
             proc.stderr.on('data', (d) => this.log.debug(`ffmpeg: ${String(d).trim()}`));
-            proc.on('error', e => reject(new Error(`ffmpeg failed: ${e.message} — install ffmpeg to play audio files`)));
+            proc.on('error', e => reject(new Error(`ffmpeg failed: ${e.message}${FFMPEG_HINT}`)));
             proc.on('close', code => code === 0
                 ? resolve({ pcm: Buffer.concat(chunks), sampleRate })
                 : reject(new Error(`ffmpeg exited with code ${code}`)));
@@ -2036,11 +2080,11 @@ class Assistant extends adapter_core_1.Adapter {
                 ? 48000
                 : target.sampleRate;
             const args = [...input, '-ar', String(rate), '-ac', String(target.channels), ...output, '-'];
-            const proc = (0, node_child_process_1.spawn)('ffmpeg', args);
+            const proc = (0, node_child_process_1.spawn)(this.ffmpegBin(), args);
             const chunks = [];
             proc.stdout.on('data', (d) => chunks.push(d));
             proc.stderr.on('data', (d) => this.log.debug(`ffmpeg: ${String(d).trim()}`));
-            proc.on('error', e => reject(new Error(`ffmpeg failed: ${e.message}`)));
+            proc.on('error', e => reject(new Error(`ffmpeg failed: ${e.message}${FFMPEG_HINT}`)));
             proc.on('close', code => code === 0 && chunks.length
                 ? resolve(Buffer.concat(chunks))
                 : reject(new Error(`ffmpeg exited with code ${code}`)));
@@ -2048,6 +2092,32 @@ class Assistant extends adapter_core_1.Adapter {
             proc.stdin.end(pcm);
         });
     }
+    /**
+     * "Check ffmpeg" on the Voice tab: start the binary at `path` ('' = from the search path), report its version and
+     * whether it can write MP3 and Opus (ESPHome devices that play nothing else). Decoding needs no extra library.
+     */
+    async testFfmpeg(path) {
+        const bin = path.trim() || this.downloadedFfmpeg() || 'ffmpeg';
+        const run = async (args) => (await (0, node_util_1.promisify)(node_child_process_1.execFile)(bin, args, { timeout: 10_000, maxBuffer: 4 * 1024 * 1024 })).stdout;
+        try {
+            const firstLine = (await run(['-version'])).split(/\r?\n/)[0];
+            const version = firstLine.replace(/\s+Copyright.*$/i, '').trim();
+            const encoders = await run(['-hide_banner', '-encoders']).catch(() => '');
+            const has = (name) => new RegExp(`\\s${name}\\s`).test(encoders);
+            const missing = ['libmp3lame', 'libopus'].filter(e => !has(e));
+            return {
+                result: `${version} (${bin})${missing.length ? ` — without ${missing.join(', ')}: MP3/Opus replies for ESPHome devices are not possible` : ''}`,
+            };
+        }
+        catch (e) {
+            const err = e;
+            return {
+                error: err.code === 'ENOENT'
+                    ? `${bin} not found${path.trim() ? '' : ' in the search path'} — install ffmpeg, download it or enter the full path`
+                    : `${bin} failed: ${err.message}`,
+            };
+        }
+    }
     /** Vendor and key of the cloud LLM: speech of the same vendor (OpenAI, OpenRouter) reuses the key. */
     async speechLlmKey() {
         const provider = this.config.provider || 'openai';
@@ -2652,6 +2722,24 @@ class Assistant extends adapter_core_1.Adapter {
             }
             return;
         }
+        // Voice tab: "Check ffmpeg" with the (unsaved) path from the form. message = { path }.
+        if (obj.command === 'testFfmpeg') {
+            const raw = obj.message?.path;
+            const result = await this.testFfmpeg(typeof raw === 'string' ? raw : this.config.ffmpegPath);
+            if (obj.callback) {
+                this.sendTo(obj.from, obj.command, result, obj.callback);
+            }
+            return;
+        }
+        // Voice tab: "Download ffmpeg". message = { path } (the unsaved path, only for the result text).
+        if (obj.command === 'downloadFfmpeg') {
+            const raw = obj.message?.path;
+            const result = await this.downloadFfmpeg(typeof raw === 'string' ? raw : this.config.ffmpegPath || '');
+            if (obj.callback) {
+                this.sendTo(obj.from, obj.command, result, obj.callback);
+            }
+            return;
+        }
         // Satellites tab: forget a satellite that is gone. message = { id } (the state id under `satellites.`).
         if (obj.command === 'deleteSatellite') {
             const raw = obj.message?.id;
@@ -2879,8 +2967,49 @@ class Assistant extends adapter_core_1.Adapter {
         const source = (typeof message === 'string' ? '' : message?.source?.trim()) || 'chat';
         // Who sent it (javascript.0, telegram.0, admin.0 for the chat tab) — the way to find a script gone wrong.
         const caller = String(obj.from || '').replace(/^system\.adapter\./, '') || 'unknown';
-        const question = String(text ?? '').trim();
-        if (!question) {
+        let question = String(text ?? '').trim();
+        // A recording instead of text (a Telegram voice message): recognise it first, then answer as usual.
+        const audio = typeof message === 'string' ? undefined : message?.audio;
+        let heard;
+        if (!question && typeof audio === 'string' && audio) {
+            if (!this.ready) {
+                if (obj.callback) {
+                    this.sendTo(obj.from, obj.command, { error: 'agent not ready' }, obj.callback);
+                }
+                return;
+            }
+            try {
+                const res = await this.transcribeAsk(message);
+                if (!res.text) {
+                    this.log.info(`Voice Q (ask from ${caller}, ${source}): (empty) — speech recognition returned no text for ${res.audioMs} ms of audio`);
+                    if (obj.callback) {
+                        this.sendTo(obj.from, obj.command, { heard: '', error: this.nothingHeardText() }, obj.callback);
+                    }
+                    return;
+                }
+                question = res.text;
+                heard = { sttMs: res.sttMs, audioMs: res.audioMs };
+            }
+            catch (e) {
+                this.log.warn(`Voice message from ${caller} (${source}) not recognised: ${e.message}`);
+                if (obj.callback) {
+                    this.sendTo(obj.from, obj.command, { error: e.message }, obj.callback);
+                }
+                return;
+            }
+        }
+        // Pictures (a Telegram photo or album) go to the cloud model; a video only as the place it was stored.
+        const attachments = this.askAttachments(message, caller, source);
+        if (typeof attachments === 'string') {
+            if (obj.callback) {
+                this.sendTo(obj.from, obj.command, { error: attachments }, obj.callback);
+            }
+            return;
+        }
+        if (!question && attachments.images.length) {
+            question = (0, media_1.imageOnlyQuestion)(this.answerLanguage(), attachments.images.length);
+        }
+        if (!question && !attachments.video) {
             this.reportEmptyAsk(caller);
             if (obj.callback) {
                 this.sendTo(obj.from, obj.command, { error: 'empty question' }, obj.callback);
@@ -2893,25 +3022,101 @@ class Assistant extends adapter_core_1.Adapter {
             }
             return;
         }
-        this.log.info(`Q (${obj.command} from ${caller}): ${question}`);
-        this.setStateAsync('text.request', { val: question, ack: true }).catch(() => { });
+        const shown = (0, media_1.historyText)(question, attachments.images.length, !!attachments.video);
+        this.log.info(`${heard ? 'Voice Q' : 'Q'} (${obj.command} from ${caller}): ${shown}${attachments.video ? ` (video ${attachments.video.adapter}/${attachments.video.path})` : ''}`);
+        this.setStateAsync('text.request', { val: shown, ack: true }).catch(() => { });
         this.setQuerySource(source);
         try {
-            const answer = await this.answer(question, source);
+            const answer = await this.answer(question, source, heard ?? {}, attachments);
             this.log.info(`A: ${answer}`);
             this.setStateAsync('text.response', { val: answer, ack: true }).catch(() => { });
             if (obj.callback) {
                 // `journaled`: the admin chat gets this exchange from the journal and must not add it twice.
-                this.sendTo(obj.from, obj.command, { answer, journaled: !!this.journal?.enabled }, obj.callback);
+                // `heard`: what was recognised in a recording, so the caller can show it with the answer.
+                this.sendTo(obj.from, obj.command, { answer, journaled: !!this.journal?.enabled, ...(heard ? { heard: question } : {}) }, obj.callback);
             }
         }
         catch (e) {
             this.log.error(`Assistant error: ${e.message}`);
             if (obj.callback) {
-                this.sendTo(obj.from, obj.command, { error: e.message, journaled: !!this.journal?.enabled }, obj.callback);
+                this.sendTo(obj.from, obj.command, {
+                    error: e.message,
+                    journaled: !!this.journal?.enabled,
+                    ...(heard ? { heard: question } : {}),
+                }, obj.callback);
             }
         }
     }
+    /**
+     * The pictures and the video of an `ask`, or why they are refused (then nothing is answered: a question about a
+     * picture without the picture would be answered with a guess).
+     */
+    askAttachments(message, caller, source) {
+        if (typeof message === 'string' || !message) {
+            return NO_ATTACHMENTS;
+        }
+        const { images, error } = (0, media_1.parseAskImages)(message.images);
+        if (error) {
+            this.log.warn(`Pictures from ${caller} (${source}) refused: ${error}`);
+            return `pictures: ${error}`;
+        }
+        const video = (0, media_1.parseAskVideo)(message.video);
+        if ((0, media_1.isVideoError)(video)) {
+            this.log.warn(`Video from ${caller} (${source}) refused: ${video.error}`);
+            return video.error;
+        }
+        return images.length || video ? { images, video } : NO_ATTACHMENTS;
+    }
+    /** Pictures and video of the request being answered */
+    attachments() {
+        return this.requestOrigin.getStore()?.attachments ?? NO_ATTACHMENTS;
+    }
+    /** Language of fixed replies: the voice language, else the system language. */
+    answerLanguage() {
+        return String(this.config.voiceLanguage || this.language || 'en');
+    }
+    /**
+     * Speech recognition for an `ask` that carries a recording instead of text. `audio` is base64; `format` says what
+     * it is: `wav` and `pcm` (16-bit mono, `sampleRate`, default 16 kHz) are read directly, anything else (`ogg` from
+     * Telegram, mp3, m4a …) goes through ffmpeg. Uses the STT engine of the Voice tab, so voice must be enabled.
+     */
+    async transcribeAsk(msg) {
+        if (!this.config.voiceEnabled) {
+            throw new Error('speech recognition is off — enable voice on the Voice tab of the assistant');
+        }
+        const engines = await this.getSpeechEngines();
+        if (!engines) {
+            throw new Error('speech recognition is not configured — check the Voice tab of the assistant');
+        }
+        const data = Buffer.from(msg.audio || '', 'base64');
+        const format = (msg.format || '').toLowerCase().replace(/^\./, '');
+        let pcm;
+        let sampleRate;
+        if (format === 'pcm') {
+            pcm = data;
+            sampleRate = Number(msg.sampleRate) || 16000;
+        }
+        else if (data.toString('latin1', 0, 4) === 'RIFF') {
+            ({ pcm, sampleRate } = (0, openrouter_1.pcmFromResponse)(data, ''));
+        }
+        else {
+            ({ pcm, sampleRate } = await this.decodeAudioBufferToPcm(data, 16000));
+        }
+        const audioMs = Math.round((pcm.length / 2 / sampleRate) * 1000);
+        const language = msg.language || engines.language;
+        const start = Date.now();
+        const text = (await engines.stt.transcribe(pcm, sampleRate, language, await this.buildSttHints())).trim();
+        return { text, sttMs: Date.now() - start, audioMs };
+    }
+    /** Reply to a voice message in which speech recognition found nothing. */
+    nothingHeardText() {
+        const lang = String(this.config.voiceLanguage || this.language || 'en');
+        return lang === 'ru'
+            ? 'В голосовом сообщении я ничего не разобрала.'
+            : lang === 'de'
+                ? 'In der Sprachnachricht habe ich nichts verstanden.'
+                : "I couldn't make out anything in the voice message.";
+    }
     /**
      * Answer a question. Tier 0: the built-in rule-based NLU handles simple device commands offline/free;
      * anything it can't resolve falls through to the LLM (Tier 2, and later the local model as Tier 1a).
@@ -2921,19 +3126,38 @@ class Assistant extends adapter_core_1.Adapter {
      * is applied to the cloud LLM tier; every completed exchange (whichever tier answered) is recorded so
      * later turns have the full thread.
      */
-    answer(question, source = '', origin = {}) {
-        if (!question.trim()) {
+    answer(question, source = '', origin = {}, attachments = NO_ATTACHMENTS) {
+        if (!question.trim() && !attachments.video) {
             // Nothing to answer — and a model asked with nothing but the time context still costs a request.
             return Promise.resolve('');
         }
         const voice = VOICE_TRANSPORTS.has(origin.transport || '');
-        return this.requestOrigin.run({ source, room: origin.room || '', voice }, () => this.journal
-            ? this.journal.run(question, source, origin, () => {
+        // The journal shows "[picture] What is in this picture?", like the conversation history
+        const shown = (0, media_1.historyText)(question, attachments.images.length, !!attachments.video);
+        return this.requestOrigin.run({ source, room: origin.room || '', voice, attachments }, () => this.journal
+            ? this.journal.run(shown, source, origin, () => {
                 this.traceSpeechInput(question, origin);
+                this.traceAttachments(attachments);
                 return this.answerTracked(question, source);
             })
             : this.answerTracked(question, source));
     }
+    /** Pictures and video of the request at the head of its trace (sizes only, not the base64). */
+    traceAttachments({ images, video }) {
+        if (!images.length && !video) {
+            return;
+        }
+        const kb = (base64) => Math.round((base64.length * 3) / 4 / 1024);
+        const parts = [
+            images.length ? `${images.length} picture(s), ${images.reduce((n, i) => n + kb(i.data), 0)} KB` : '',
+            video ? `video ${video.adapter}/${video.path}` : '',
+        ].filter(Boolean);
+        this.journal?.step({
+            stage: 'request',
+            title: `Attachments: ${parts.join(', ')}`,
+            data: { images: images.map(i => ({ mimeType: i.mimeType, kb: kb(i.data) })), video: video || undefined },
+        });
+    }
     /** Speech recognition happened before the request: put it at the head of its trace. */
     traceSpeechInput(question, origin) {
         if (origin.sttMs === undefined) {
@@ -2978,7 +3202,7 @@ class Assistant extends adapter_core_1.Adapter {
         const followUp = /\[\[LISTEN\]\]/i.test(raw);
         this.followUps.set(source, followUp);
         // A script conversation goes on while the cloud model asks back: the reply goes to it as well.
-        if (cloudOnly && followUp) {
+        if ((cloudOnly === 'script' || cloudOnly === 'scriptReply') && followUp) {
             this.scriptTopics.set(source, Date.now());
         }
         else {
@@ -3007,7 +3231,9 @@ class Assistant extends adapter_core_1.Adapter {
             data: { answer: result, raw: raw !== result ? raw : undefined, followUp, tone: toned || undefined },
         });
         if (useCtx && (result || toned)) {
-            this.context.add(source, question, result || this.doneText());
+            // The pictures are not sent again in later turns; a marker says there were some
+            const { images, video } = this.attachments();
+            this.context.add(source, (0, media_1.historyText)(question, images.length, !!video), result || this.doneText());
         }
         return result;
     }
@@ -3063,7 +3289,11 @@ class Assistant extends adapter_core_1.Adapter {
             return 'script';
         }
         const since = this.scriptTopics.get(source);
-        return since !== undefined && Date.now() - since < SCRIPT_TOPIC_MS ? 'scriptReply' : null;
+        if (since !== undefined && Date.now() - since < SCRIPT_TOPIC_MS) {
+            return 'scriptReply';
+        }
+        // Rules and the local model are blind: the caption of a photo is about the photo, not a command
+        return this.attachments().images.length ? 'image' : null;
     }
     /**
      * Should the satellite behind `source` re-open its mic after this reply? Primarily the LLM's own
@@ -3222,6 +3452,14 @@ class Assistant extends adapter_core_1.Adapter {
     }
     async produceAnswer(question, history, source = '', cloudOnly = null) {
         this.log.debug(`answer: source='${source}', context=${history.length} turn(s)`);
+        const { images, video } = this.attachments();
+        if (!question.trim() && video) {
+            // A video without a question: nothing looks at videos yet
+            this.log.info(`Video from '${source}' (${video.adapter}/${video.path}) not looked at: not supported yet.`);
+            this.journal?.note({ tier: 'none' });
+            this.journal?.step({ stage: 'none', title: 'A video without a question: videos are not looked at yet' });
+            return (0, media_1.videoNotSupportedText)(this.answerLanguage());
+        }
         // A ringing timer/alarm is silenced by a stop word ("stop"/"halt"/"aufhören"/…) — checked first so
         // it always wins over the normal pipeline while something is ringing.
         if (this.rings.length && (0, nlu_1.isStopCommand)(question)) {
@@ -3248,10 +3486,22 @@ class Assistant extends adapter_core_1.Adapter {
                 return '';
             }
         }
+        if (images.length && !this.agent && !this.piAgent) {
+            this.log.info(`Pictures from '${source}' not looked at: no cloud model is configured.`);
+            this.journal?.note({ tier: 'none' });
+            this.journal?.step({
+                stage: 'none',
+                level: 'warn',
+                title: 'Pictures came with the request, but no cloud model is configured to look at them',
+            });
+            return (0, media_1.noVisionText)(this.answerLanguage());
+        }
         if (cloudOnly) {
             const why = cloudOnly === 'script'
                 ? 'it is about a script'
-                : 'it answers a question of the cloud model about a script';
+                : cloudOnly === 'scriptReply'
+                    ? 'it answers a question of the cloud model about a script'
+                    : 'pictures came with it, only the cloud model can see them';
             this.log.info(`Routines, NLU and local LLM skipped (source='${source}'): ${why}.`);
             this.journal?.step({
                 stage: 'nlu',
@@ -3331,7 +3581,10 @@ class Assistant extends adapter_core_1.Adapter {
         // every time voice and text requests alternate.
         const time = this.buildTimeContext();
         const lengthHint = this.spokenLengthHint();
-        const questionForLlm = [time, weather, question, lengthHint].filter(Boolean).join('\n\n');
+        // The pictures themselves go along; only a video needs a word, the model gets nothing of it
+        const questionForLlm = [time, weather, question, (0, media_1.attachmentNote)(0, !!video), lengthHint]
+            .filter(Boolean)
+            .join('\n\n');
         this.journal?.note({
             tier: 'cloud',
             detail: `${this.llm?.provider || this.config.provider || 'openai'} · ${this.agent.model || 'auto'}`,
@@ -3351,10 +3604,12 @@ class Assistant extends adapter_core_1.Adapter {
                 devices: deviceLines,
                 time,
                 weather: weather || undefined,
+                images: images.length || undefined,
+                video: video ? `${video.adapter}/${video.path}` : undefined,
             },
         });
         try {
-            return await this.agent.ask(questionForLlm, sys, history);
+            return await this.agent.ask(questionForLlm, sys, history, images);
         }
         catch (e) {
             // Offline, quota, outage: the local model is the fallback — unless it was already asked above.
@@ -3387,7 +3642,15 @@ class Assistant extends adapter_core_1.Adapter {
         if (dropped) {
             this.log.debug(`pi-agent: ${dropped} context block(s) left out, the instructions would be too long.`);
         }
-        const text = [this.buildTimeContext(), weather, this.buildPresenceContext(), question]
+        // An older pi-agent drops the pictures: the note at least tells the model that there were some
+        const { images, video } = this.attachments();
+        const text = [
+            this.buildTimeContext(),
+            weather,
+            this.buildPresenceContext(),
+            question,
+            (0, media_1.attachmentNote)(images.length, !!video),
+        ]
             .filter(Boolean)
             .join('\n\n');
         this.log.info(`Cloud tier ${harness.instance} (source='${source}') — asking pi-agent…`);
@@ -3396,7 +3659,7 @@ class Assistant extends adapter_core_1.Adapter {
             // pi-agent asks its questions to confirm a write ("… Say yes or no.") in this language, not in the system
             // language: a satellite listening in Russian does not hear a German "Ja"
             const language = String(this.config.voiceLanguage || this.language || '');
-            return await harness.ask({ text, conversation: source, instructions, language });
+            return await harness.ask({ text, conversation: source, instructions, language, images });
         }
         catch (e) {
             if (localFallback) {