iobroker.assistant 1.4.6 → 1.4.7

package/build/lib/media.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"media.js","sourceRoot":"","sources":["../../src/lib/media.ts"],"names":[],"mappings":";AAAA;;;;;;GAMG;;;AAkCH,wCAcC;AASD,wCA6BC;AAQD,sCA0BC;AAGD,oCAEC;AASD,oCAsBC;AAiBD,8CAQC;AAOD,sDAQC;AAOD,oCAQC;AASD,wCASC;AAUD,kCAGC;AA7ND,kFAAkF;AACrE,QAAA,UAAU,GAAG,CAAC,CAAC;AAE5B,yEAAyE;AAC5D,QAAA,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEhD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;AAEpF;;;;GAIG;AACH,SAAgB,cAAc,CAAC,GAAW;IACtC,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC3E,OAAO,YAAY,CAAC;IACxB,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9G,OAAO,WAAW,CAAC;IACvB,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;QAC/E,OAAO,WAAW,CAAC;IACvB,CAAC;IACD,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,MAAM,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,EAAE,CAAC;QAC1G,OAAO,YAAY,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,GAAY;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QAClD,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAC1B,CAAC;IACD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAU,EAAE,CAAC;QAC3B,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,sBAAsB,IAAI,CAAC,MAAM,aAAa,kBAAU,GAAG,EAAE,CAAC;IAC9F,CAAC;IACD,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,IAAkC,EAAE,IAAI,CAAC;QACzF,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YACpC,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;QAC3D,CAAC;QACD,6BAA6B;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,sBAAsB,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC5E,IAAI,MAAM,CAAC,MAAM,GAAG,wBAAgB,EAAE,CAAC;YACnC,OAAO;gBACH,MAAM,EAAE,EAAE;gBACV,KAAK,EAAE,oBAAoB,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,wBAAgB,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,MAAM;aAC/H,CAAC;QACN,CAAC;QACD,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC1C,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,sCAAsC,EAAE,CAAC;QACzE,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,CAAC;AACtB,CAAC;AAED;;;;;GAKG;AACH,SAAgB,aAAa,CAAC,GAAY;IACtC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,KAAK,EAAE,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAChD,OAAO,EAAE,KAAK,EAAE,8DAA8D,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QACd,OAAO,EAAE,KAAK,EAAE,sEAAsE,EAAE,CAAC;IAC7F,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5E,MAAM,GAAG,GAAG,CAAC,CAAU,EAAsB,EAAE,CAC3C,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1E,MAAM,GAAG,GAAa;QAClB,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC;QACvB,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACrC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,SAAS;QACtC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;QAC/B,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;QACjB,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,SAAS;KACnC,CAAC;IACF,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QAC5C,OAAO,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC;IACvD,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,sDAAsD;AACtD,SAAgB,YAAY,CAAC,CAAsC;IAC/D,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,YAAY,CAAI,KAAQ;IACpC,MAAM,IAAI,GAAG,CAAC,CAAU,EAAW,EAAE;QACjC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACnB,OAAO,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC9B,OAAO,CAAC,CAAC;QACb,CAAC;QACD,MAAM,CAAC,GAAG,CAAwB,CAAC;QACnC,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,QAAQ,IAAI,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxG,OAAO,EAAE,GAAG,CAAC,EAAE,SAAS,EAAE,EAAE,GAAG,CAAC,CAAC,SAAS,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;QACpF,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,IAAI,CAAC,CAAC,MAAM,EAAE,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzF,OAAO,EAAE,GAAG,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QAClG,CAAC;QACD,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACrC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;QACD,OAAO,GAAG,CAAC;IACf,CAAC,CAAC;IACF,OAAO,IAAI,CAAC,KAAK,CAAM,CAAC;AAC5B,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,QAAiB;IAC9C,MAAM,CAAC,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAChD,OAAO,IAAI,QAAQ,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;AAClG,CAAC;AAKD;;;;;GAKG;AACH,SAAgB,iBAAiB,CAAC,IAAU,EAAE,KAAK,GAAG,CAAC;IACnD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,uBAAuB,CAAC;IAC1E,CAAC;IACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,6BAA6B,CAAC,CAAC,CAAC,0BAA0B,CAAC;IAClF,CAAC;IACD,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,CAAC,CAAC,0BAA0B,CAAC;AACjF,CAAC;AAED;;;;GAIG;AACH,SAAgB,qBAAqB,CAAC,IAAU;IAC5C,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,OAAO,gCAAgC,CAAC;IAC5C,CAAC;IACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,OAAO,yCAAyC,CAAC;IACrD,CAAC;IACD,OAAO,2BAA2B,CAAC;AACvC,CAAC;AAED;;;;GAIG;AACH,SAAgB,YAAY,CAAC,IAAU;IACnC,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,OAAO,uDAAuD,CAAC;IACnE,CAAC;IACD,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAChB,OAAO,yDAAyD,CAAC;IACrE,CAAC;IACD,OAAO,iDAAiD,CAAC;AAC7D,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,cAAc,CAAC,MAAc,EAAE,KAAc;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,EAAE,CAAC;QACT,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,yBAAyB,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAC3F,CAAC;IACD,IAAI,KAAK,EAAE,CAAC;QACR,KAAK,CAAC,IAAI,CAAC,8FAA8F,CAAC,CAAC;IAC/G,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;AACtD,CAAC;AAED;;;;;;;GAOG;AACH,SAAgB,WAAW,CAAC,QAAgB,EAAE,MAAc,EAAE,KAAc;IACxE,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3G,OAAO,CAAC,GAAG,KAAK,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC","sourcesContent":["/**\n * Pictures and videos that come with a question (`ask` with `images` / `video`, e.g. from Telegram).\n *\n * Pictures go to the cloud model as they are (base64, the vendors take JPEG/PNG/GIF/WebP). Videos are not looked at\n * yet: the sender keeps them in the ioBroker file storage and only says where, so a video never travels through the\n * message bus as base64.\n */\n\n/** A picture for the model: base64 without a `data:` prefix and its type. */\nexport interface ImageInput {\n    data: string;\n    mimeType: string;\n}\n\n/** Where the sender stored a video (ioBroker file storage), with what it knows about it. */\nexport interface VideoRef {\n    /** File storage namespace, e.g. `telegram.0` */\n    adapter: string;\n    /** Path in it, e.g. `assistant/video_2026-11-04_10-00-00.mp4` */\n    path: string;\n    mimeType?: string;\n    durationSec?: number;\n    size?: number;\n    /** Set when the sender could not store it (e.g. too big to download) */\n    error?: string;\n}\n\n/** At most this many pictures per question (an album of Telegram has up to 10) */\nexport const MAX_IMAGES = 4;\n\n/** Largest picture as base64 text: Anthropic refuses images over 5 MB */\nexport const MAX_IMAGE_BASE64 = 5 * 1024 * 1024;\n\nconst IMAGE_TYPES = new Set(['image/jpeg', 'image/png', 'image/gif', 'image/webp']);\n\n/**\n * The type of a picture from its first bytes, `null` if it is none of the four the vendors take.\n *\n * @param buf the start of the file (16 bytes are enough)\n */\nexport function sniffImageType(buf: Buffer): string | null {\n    if (buf.length >= 3 && buf[0] === 0xff && buf[1] === 0xd8 && buf[2] === 0xff) {\n        return 'image/jpeg';\n    }\n    if (buf.length >= 8 && buf.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) {\n        return 'image/png';\n    }\n    if (buf.length >= 6 && /^GIF8[79]a$/.test(buf.subarray(0, 6).toString('latin1'))) {\n        return 'image/gif';\n    }\n    if (buf.length >= 12 && buf.toString('latin1', 0, 4) === 'RIFF' && buf.toString('latin1', 8, 12) === 'WEBP') {\n        return 'image/webp';\n    }\n    return null;\n}\n\n/**\n * Check the `images` of an `ask`: base64 of JPEG/PNG/GIF/WebP, at most {@link MAX_IMAGES} and\n * {@link MAX_IMAGE_BASE64} each. The type is taken from the content, not from what the sender says.\n *\n * @param raw the `images` field: a list of `{ data, mimeType? }` (one object or a bare base64 string work too)\n * @returns the pictures, or the reason they are refused\n */\nexport function parseAskImages(raw: unknown): { images: ImageInput[]; error?: string } {\n    if (raw === undefined || raw === null || raw === '') {\n        return { images: [] };\n    }\n    const list = Array.isArray(raw) ? raw : [raw];\n    if (list.length > MAX_IMAGES) {\n        return { images: [], error: `too many pictures (${list.length}, at most ${MAX_IMAGES})` };\n    }\n    const images: ImageInput[] = [];\n    for (const item of list) {\n        const data = typeof item === 'string' ? item : (item as { data?: unknown } | null)?.data;\n        if (typeof data !== 'string' || !data) {\n            return { images: [], error: 'a picture without data' };\n        }\n        // a data URL is fine as well\n        const base64 = data.replace(/^data:[^;,]*;base64,/, '').replace(/\\s+/g, '');\n        if (base64.length > MAX_IMAGE_BASE64) {\n            return {\n                images: [],\n                error: `picture too big (${Math.round((base64.length * 3) / 4 / 1024)} KB, at most ${(MAX_IMAGE_BASE64 * 3) / 4 / 1024} KB)`,\n            };\n        }\n        const mimeType = sniffImageType(Buffer.from(base64.slice(0, 32), 'base64'));\n        if (!mimeType || !IMAGE_TYPES.has(mimeType)) {\n            return { images: [], error: 'not a JPEG, PNG, GIF or WebP picture' };\n        }\n        images.push({ data: base64, mimeType });\n    }\n    return { images };\n}\n\n/**\n * Check the `video` of an `ask`: only a reference into the ioBroker file storage, never the content.\n *\n * @param raw the `video` field\n * @returns the reference, `null` without one, or the reason it is refused (e.g. base64 content)\n */\nexport function parseAskVideo(raw: unknown): VideoRef | null | { error: string } {\n    if (raw === undefined || raw === null || raw === '') {\n        return null;\n    }\n    if (typeof raw !== 'object' || Array.isArray(raw)) {\n        return { error: 'video must be { adapter, path } of the ioBroker file storage' };\n    }\n    const v = raw as Record<string, unknown>;\n    if ('data' in v) {\n        return { error: 'a video is not sent as content - store it and send { adapter, path }' };\n    }\n    const str = (x: unknown): string => (typeof x === 'string' ? x.trim() : '');\n    const num = (x: unknown): number | undefined =>\n        typeof x === 'number' && Number.isFinite(x) && x >= 0 ? x : undefined;\n    const ref: VideoRef = {\n        adapter: str(v.adapter),\n        path: str(v.path).replace(/^\\/+/, ''),\n        mimeType: str(v.mimeType) || undefined,\n        durationSec: num(v.durationSec),\n        size: num(v.size),\n        error: str(v.error) || undefined,\n    };\n    if ((!ref.adapter || !ref.path) && !ref.error) {\n        return { error: 'video without adapter and path' };\n    }\n    return ref;\n}\n\n/** Whether {@link parseAskVideo} refused the video */\nexport function isVideoError(v: VideoRef | null | { error: string }): v is { error: string } {\n    return !!v && !('adapter' in v);\n}\n\n/**\n * Copy of a request / message list with the pictures replaced by a short note — for the trace and the debug log,\n * where hundreds of KB of base64 per round help nobody. Understands the OpenAI (`image_url` with a data URL) and the\n * Anthropic form (`image` with a base64 `source`).\n *\n * @param value what to copy\n */\nexport function redactImages<T>(value: T): T {\n    const walk = (v: unknown): unknown => {\n        if (Array.isArray(v)) {\n            return v.map(walk);\n        }\n        if (!v || typeof v !== 'object') {\n            return v;\n        }\n        const o = v as Record<string, any>;\n        if (o.type === 'image_url' && typeof o.image_url?.url === 'string' && o.image_url.url.startsWith('data:')) {\n            return { ...o, image_url: { ...o.image_url, url: imageNote(o.image_url.url) } };\n        }\n        if (o.type === 'image' && o.source?.type === 'base64' && typeof o.source.data === 'string') {\n            return { ...o, source: { ...o.source, data: imageNote(o.source.data, o.source.media_type) } };\n        }\n        const out: Record<string, unknown> = {};\n        for (const [k, x] of Object.entries(o)) {\n            out[k] = walk(x);\n        }\n        return out;\n    };\n    return walk(value) as T;\n}\n\nfunction imageNote(data: string, mimeType?: string): string {\n    const m = /^data:([^;,]+)/.exec(data);\n    const base64 = data.replace(/^data:[^,]*,/, '');\n    return `<${mimeType || m?.[1] || 'image'}, ${Math.round((base64.length * 3) / 4 / 1024)} KB>`;\n}\n\n/** Language of the texts below: de, ru, else en */\ntype Lang = string;\n\n/**\n * The question for pictures that came without one (a photo without caption).\n *\n * @param lang language of the answer\n * @param count number of pictures\n */\nexport function imageOnlyQuestion(lang: Lang, count = 1): string {\n    if (lang === 'ru') {\n        return count > 1 ? 'Что на этих картинках?' : 'Что на этой картинке?';\n    }\n    if (lang === 'de') {\n        return count > 1 ? 'Was ist auf diesen Bildern?' : 'Was ist auf diesem Bild?';\n    }\n    return count > 1 ? 'What is in these pictures?' : 'What is in this picture?';\n}\n\n/**\n * The answer to a video that came without a question.\n *\n * @param lang language of the answer\n */\nexport function videoNotSupportedText(lang: Lang): string {\n    if (lang === 'ru') {\n        return 'Видео я пока смотреть не умею.';\n    }\n    if (lang === 'de') {\n        return 'Videos kann ich mir noch nicht ansehen.';\n    }\n    return \"I can't watch videos yet.\";\n}\n\n/**\n * The answer to pictures when there is no cloud model to look at them.\n *\n * @param lang language of the answer\n */\nexport function noVisionText(lang: Lang): string {\n    if (lang === 'ru') {\n        return 'Картинки я могу посмотреть только с облачной моделью.';\n    }\n    if (lang === 'de') {\n        return 'Bilder kann ich mir nur mit einem Cloud-Modell ansehen.';\n    }\n    return 'I can only look at pictures with a cloud model.';\n}\n\n/**\n * Note for the model in the user turn: what came along besides the text. A model that does not get the pictures\n * (an older pi-agent) at least knows that there were some.\n *\n * @param images number of pictures attached\n * @param video whether a video came along\n */\nexport function attachmentNote(images: number, video: boolean): string {\n    const parts: string[] = [];\n    if (images) {\n        parts.push(images > 1 ? `${images} pictures are attached.` : 'A picture is attached.');\n    }\n    if (video) {\n        parts.push('The user also sent a video, which you cannot watch yet - say so if the question is about it.');\n    }\n    return parts.length ? `(${parts.join(' ')})` : '';\n}\n\n/**\n * How the question is kept in the conversation history: the pictures are not repeated in later turns, a marker says\n * there were some.\n *\n * @param question the question\n * @param images number of pictures\n * @param video whether a video came along\n */\nexport function historyText(question: string, images: number, video: boolean): string {\n    const marks = [images ? `[${images > 1 ? `${images} pictures` : 'picture'}]` : '', video ? '[video]' : ''];\n    return [...marks, question].filter(Boolean).join(' ');\n}\n"]}

package/build/lib/piAgent.js

@@ -77,6 +77,7 @@ class PiAgentHarness {
             conversation: conversationOf(req.conversation),
             instructions: req.instructions,
             ...(req.language ? { language: req.language } : {}),
+            ...(req.images?.length ? { images: req.images } : {}),
             requestId,
         }, { timeout: this.timeoutMs }));
         if (result?.error) {

package/build/lib/piAgent.js.map

@@ -1 +1 @@
-{"version":3,"file":"piAgent.js","sourceRoot":"","sources":["../../src/lib/piAgent.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;;AAmCH,kDAaC;AAGD,wCAEC;AAnDD,8FAA8F;AACjF,QAAA,yBAAyB,GAAG,KAAK,CAAC;AAE/C,wFAAwF;AAC3E,QAAA,oBAAoB,GAAG,EAAE,CAAC;AAsBvC;;;;;;GAMG;AACH,SAAgB,mBAAmB,CAAC,QAAkB,EAAE,QAAkB;IACtE,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACvC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,MAAM,IAAI,iCAAyB,EAAE,CAAC;YAC3C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,KAAK,CAAC,GAAG,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;IACd,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,iCAAyB,CAAC,EAAE,OAAO,EAAE,CAAC;AACtF,CAAC;AAED,0FAA0F;AAC1F,SAAgB,cAAc,CAAC,MAAc;IACzC,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACnE,CAAC;AAED,MAAa,cAAc;IAIF;IAED;IACC;IANb,GAAG,GAAG,CAAC,CAAC;IAEhB,YACqB,IAAiB;IAClC,wBAAwB;IACR,QAAgB,EACf,YAAoB,4BAAoB,GAAG,IAAI;QAH/C,SAAI,GAAJ,IAAI,CAAa;QAElB,aAAQ,GAAR,QAAQ,CAAQ;QACf,cAAS,GAAT,SAAS,CAAsC;IACjE,CAAC;IAEJ;;;OAGG;IACI,KAAK,CAAC,GAAG,CAAC,GAAmB;QAChC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,kBAAkB,IAAI,CAAC,QAAQ,QAAQ,CAAC,CAAC;QAC5F,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,iBAAiB,CAAC,CAAC;QACvD,CAAC;QACD,uEAAuE;QACvE,MAAM,SAAS,GAAG,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QACtF,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CACvC,IAAI,CAAC,QAAQ,EACb,KAAK,EACL;YACI,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC;YAC9C,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,SAAS;SACZ,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAC9B,CAA6D,CAAC;QAC/D,IAAI,MAAM,EAAE,KAAK,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7F,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,MAAM,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,iBAAiB,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AA3CD,wCA2CC","sourcesContent":["/**\n * The adapter `pi-agent` as the cloud tier (\"harness\"): instead of running its own tool loop, the assistant\n * hands the question to a pi-agent instance via `sendTo(<instance>, 'ask', …)`. pi-agent stores every model\n * and tool call as a checkpoint before it runs (an interrupted request continues after a restart, a write is\n * never repeated), enforces its own write permissions and confirmations, and can use the system-wide AI\n * settings of admin — none of which the built-in `LlmAgent` does.\n *\n * Each request source (satellite, `chat`, `telegram:<user>`) gets a conversation of its own in pi-agent\n * (`conversation`), so two rooms do not see each other's follow-ups. The source-independent part of the\n * prompt (style, follow-up marker, memory, device list) goes along as the `instructions` of that\n * conversation; what changes every turn (time, weather, presence) is prepended to the question.\n * pi-agent 0.3.0 and older ignore both fields and answer everything in their root conversation.\n *\n * With \"Confirmation by text in sendTo conversations\", pi-agent answers a write it may not do unasked with a\n * question ending in `[[LISTEN]]`, and the next `ask` of the conversation is the yes or no. `language` makes it\n * ask that question in the language the satellite listens in.\n */\n\n/** Longest instructions pi-agent accepts for a conversation; anything beyond is cut there. */\nexport const PI_AGENT_MAX_INSTRUCTIONS = 20000;\n\n/** Default for `piAgentTimeoutSec`: a tool loop with several reads can take a while. */\nexport const PI_AGENT_TIMEOUT_SEC = 60;\n\n/** The part of the adapter this module needs (kept small so tests can fake it). */\nexport interface PiAgentHost {\n    sendToAsync(instance: string, command: string, message: unknown, options?: { timeout?: number }): Promise<unknown>;\n    getForeignStateAsync(id: string): Promise<ioBroker.State | null | undefined>;\n}\n\nexport interface PiAgentRequest {\n    /** The text pi-agent answers: context lines + the question. */\n    text: string;\n    /** Conversation of the caller, e.g. the satellite name. */\n    conversation: string;\n    /** Instructions of that conversation. */\n    instructions: string;\n    /**\n     * Language the user speaks, e.g. `ru`. pi-agent asks its questions to confirm in it; older versions ask in the\n     * system language, and a satellite listening in Russian does not hear a German \"Ja\" (the STT returns nothing).\n     */\n    language?: string;\n}\n\n/**\n * Join instruction blocks, leaving out optional ones from the end until the result fits into pi-agent's\n * limit — pi-agent would cut the text in the middle, and a half device list is worse than none.\n *\n * @param required blocks that always go along (style, follow-up marker)\n * @param optional blocks in order of importance (memory before the device list)\n */\nexport function composeInstructions(required: string[], optional: string[]): { text: string; dropped: number } {\n    const blocks = [...required.filter(Boolean)];\n    const extra = optional.filter(Boolean);\n    let dropped = 0;\n    while (extra.length) {\n        const text = [...blocks, ...extra].join('\\n\\n');\n        if (text.length <= PI_AGENT_MAX_INSTRUCTIONS) {\n            return { text, dropped };\n        }\n        extra.pop();\n        dropped++;\n    }\n    return { text: blocks.join('\\n\\n').slice(0, PI_AGENT_MAX_INSTRUCTIONS), dropped };\n}\n\n/** Conversation name for a request source; pi-agent uses \"/\" as separator in its keys. */\nexport function conversationOf(source: string): string {\n    return (source || 'default').replace(/\\//g, '_').slice(0, 100);\n}\n\nexport class PiAgentHarness {\n    private seq = 0;\n\n    public constructor(\n        private readonly host: PiAgentHost,\n        /** e.g. `pi-agent.0` */\n        public readonly instance: string,\n        private readonly timeoutMs: number = PI_AGENT_TIMEOUT_SEC * 1000,\n    ) {}\n\n    /**\n     * Ask pi-agent. Fails fast when the instance is not running (otherwise the caller would wait for the whole\n     * timeout), and turns pi-agent's `{error}` into an exception so the caller can fall back to the local model.\n     */\n    public async ask(req: PiAgentRequest): Promise<string> {\n        const alive = await this.host.getForeignStateAsync(`system.adapter.${this.instance}.alive`);\n        if (!alive?.val) {\n            throw new Error(`${this.instance} is not running`);\n        }\n        // Unique per request: pi-agent answers a repeated requestId only once.\n        const requestId = `assistant-${Date.now().toString(36)}-${(++this.seq).toString(36)}`;\n        const result = (await this.host.sendToAsync(\n            this.instance,\n            'ask',\n            {\n                text: req.text,\n                conversation: conversationOf(req.conversation),\n                instructions: req.instructions,\n                ...(req.language ? { language: req.language } : {}),\n                requestId,\n            },\n            { timeout: this.timeoutMs },\n        )) as { answer?: unknown; error?: unknown } | null | undefined;\n        if (result?.error) {\n            const error = typeof result.error === 'string' ? result.error : JSON.stringify(result.error);\n            throw new Error(`${this.instance}: ${error}`);\n        }\n        const answer = typeof result?.answer === 'string' ? result.answer.trim() : '';\n        if (!answer) {\n            throw new Error(`${this.instance} gave no answer`);\n        }\n        return answer;\n    }\n}\n"]}
+{"version":3,"file":"piAgent.js","sourceRoot":"","sources":["../../src/lib/piAgent.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;GAgBG;;;AA0CH,kDAaC;AAGD,wCAEC;AAxDD,8FAA8F;AACjF,QAAA,yBAAyB,GAAG,KAAK,CAAC;AAE/C,wFAAwF;AAC3E,QAAA,oBAAoB,GAAG,EAAE,CAAC;AA2BvC;;;;;;GAMG;AACH,SAAgB,mBAAmB,CAAC,QAAkB,EAAE,QAAkB;IACtE,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACvC,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,MAAM,IAAI,iCAAyB,EAAE,CAAC;YAC3C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,KAAK,CAAC,GAAG,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;IACd,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,iCAAyB,CAAC,EAAE,OAAO,EAAE,CAAC;AACtF,CAAC;AAED,0FAA0F;AAC1F,SAAgB,cAAc,CAAC,MAAc;IACzC,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACnE,CAAC;AAED,MAAa,cAAc;IAIF;IAED;IACC;IANb,GAAG,GAAG,CAAC,CAAC;IAEhB,YACqB,IAAiB;IAClC,wBAAwB;IACR,QAAgB,EACf,YAAoB,4BAAoB,GAAG,IAAI;QAH/C,SAAI,GAAJ,IAAI,CAAa;QAElB,aAAQ,GAAR,QAAQ,CAAQ;QACf,cAAS,GAAT,SAAS,CAAsC;IACjE,CAAC;IAEJ;;;OAGG;IACI,KAAK,CAAC,GAAG,CAAC,GAAmB;QAChC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,kBAAkB,IAAI,CAAC,QAAQ,QAAQ,CAAC,CAAC;QAC5F,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,iBAAiB,CAAC,CAAC;QACvD,CAAC;QACD,uEAAuE;QACvE,MAAM,SAAS,GAAG,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;QACtF,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CACvC,IAAI,CAAC,QAAQ,EACb,KAAK,EACL;YACI,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC;YAC9C,YAAY,EAAE,GAAG,CAAC,YAAY;YAC9B,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnD,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,SAAS;SACZ,EACD,EAAE,OAAO,EAAE,IAAI,CAAC,SAAS,EAAE,CAC9B,CAA6D,CAAC;QAC/D,IAAI,MAAM,EAAE,KAAK,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7F,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,MAAM,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,QAAQ,iBAAiB,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,MAAM,CAAC;IAClB,CAAC;CACJ;AA5CD,wCA4CC","sourcesContent":["/**\n * The adapter `pi-agent` as the cloud tier (\"harness\"): instead of running its own tool loop, the assistant\n * hands the question to a pi-agent instance via `sendTo(<instance>, 'ask', …)`. pi-agent stores every model\n * and tool call as a checkpoint before it runs (an interrupted request continues after a restart, a write is\n * never repeated), enforces its own write permissions and confirmations, and can use the system-wide AI\n * settings of admin — none of which the built-in `LlmAgent` does.\n *\n * Each request source (satellite, `chat`, `telegram:<user>`) gets a conversation of its own in pi-agent\n * (`conversation`), so two rooms do not see each other's follow-ups. The source-independent part of the\n * prompt (style, follow-up marker, memory, device list) goes along as the `instructions` of that\n * conversation; what changes every turn (time, weather, presence) is prepended to the question.\n * pi-agent 0.3.0 and older ignore both fields and answer everything in their root conversation.\n *\n * With \"Confirmation by text in sendTo conversations\", pi-agent answers a write it may not do unasked with a\n * question ending in `[[LISTEN]]`, and the next `ask` of the conversation is the yes or no. `language` makes it\n * ask that question in the language the satellite listens in.\n */\n\nimport type { ImageInput } from './media';\n\n/** Longest instructions pi-agent accepts for a conversation; anything beyond is cut there. */\nexport const PI_AGENT_MAX_INSTRUCTIONS = 20000;\n\n/** Default for `piAgentTimeoutSec`: a tool loop with several reads can take a while. */\nexport const PI_AGENT_TIMEOUT_SEC = 60;\n\n/** The part of the adapter this module needs (kept small so tests can fake it). */\nexport interface PiAgentHost {\n    sendToAsync(instance: string, command: string, message: unknown, options?: { timeout?: number }): Promise<unknown>;\n    getForeignStateAsync(id: string): Promise<ioBroker.State | null | undefined>;\n}\n\nexport interface PiAgentRequest {\n    /** The text pi-agent answers: context lines + the question. */\n    text: string;\n    /** Conversation of the caller, e.g. the satellite name. */\n    conversation: string;\n    /** Instructions of that conversation. */\n    instructions: string;\n    /**\n     * Language the user speaks, e.g. `ru`. pi-agent asks its questions to confirm in it; older versions ask in the\n     * system language, and a satellite listening in Russian does not hear a German \"Ja\" (the STT returns nothing).\n     */\n    language?: string;\n    /**\n     * Pictures that came with the question (`{ data: base64, mimeType }`, the form of pi-ai's `ImageContent`).\n     * Versions of pi-agent before image support ignore them; the text then says that pictures came along.\n     */\n    images?: ImageInput[];\n}\n\n/**\n * Join instruction blocks, leaving out optional ones from the end until the result fits into pi-agent's\n * limit — pi-agent would cut the text in the middle, and a half device list is worse than none.\n *\n * @param required blocks that always go along (style, follow-up marker)\n * @param optional blocks in order of importance (memory before the device list)\n */\nexport function composeInstructions(required: string[], optional: string[]): { text: string; dropped: number } {\n    const blocks = [...required.filter(Boolean)];\n    const extra = optional.filter(Boolean);\n    let dropped = 0;\n    while (extra.length) {\n        const text = [...blocks, ...extra].join('\\n\\n');\n        if (text.length <= PI_AGENT_MAX_INSTRUCTIONS) {\n            return { text, dropped };\n        }\n        extra.pop();\n        dropped++;\n    }\n    return { text: blocks.join('\\n\\n').slice(0, PI_AGENT_MAX_INSTRUCTIONS), dropped };\n}\n\n/** Conversation name for a request source; pi-agent uses \"/\" as separator in its keys. */\nexport function conversationOf(source: string): string {\n    return (source || 'default').replace(/\\//g, '_').slice(0, 100);\n}\n\nexport class PiAgentHarness {\n    private seq = 0;\n\n    public constructor(\n        private readonly host: PiAgentHost,\n        /** e.g. `pi-agent.0` */\n        public readonly instance: string,\n        private readonly timeoutMs: number = PI_AGENT_TIMEOUT_SEC * 1000,\n    ) {}\n\n    /**\n     * Ask pi-agent. Fails fast when the instance is not running (otherwise the caller would wait for the whole\n     * timeout), and turns pi-agent's `{error}` into an exception so the caller can fall back to the local model.\n     */\n    public async ask(req: PiAgentRequest): Promise<string> {\n        const alive = await this.host.getForeignStateAsync(`system.adapter.${this.instance}.alive`);\n        if (!alive?.val) {\n            throw new Error(`${this.instance} is not running`);\n        }\n        // Unique per request: pi-agent answers a repeated requestId only once.\n        const requestId = `assistant-${Date.now().toString(36)}-${(++this.seq).toString(36)}`;\n        const result = (await this.host.sendToAsync(\n            this.instance,\n            'ask',\n            {\n                text: req.text,\n                conversation: conversationOf(req.conversation),\n                instructions: req.instructions,\n                ...(req.language ? { language: req.language } : {}),\n                ...(req.images?.length ? { images: req.images } : {}),\n                requestId,\n            },\n            { timeout: this.timeoutMs },\n        )) as { answer?: unknown; error?: unknown } | null | undefined;\n        if (result?.error) {\n            const error = typeof result.error === 'string' ? result.error : JSON.stringify(result.error);\n            throw new Error(`${this.instance}: ${error}`);\n        }\n        const answer = typeof result?.answer === 'string' ? result.answer.trim() : '';\n        if (!answer) {\n            throw new Error(`${this.instance} gave no answer`);\n        }\n        return answer;\n    }\n}\n"]}

package/build/lib/voice/esphome.js

@@ -34,6 +34,9 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.EsphomeSatellites = exports.TIMER_EVENT = exports.DEFAULT_SPEECH_LEVEL = void 0;
+exports.yesNoWord = yesNoWord;
+exports.confirmAnswerOf = confirmAnswerOf;
+exports.displayLanguageOption = displayLanguageOption;
 exports.speechLevelOf = speechLevelOf;
 exports.esphomeAddress = esphomeAddress;
 /**
@@ -170,6 +173,106 @@ const NO_TEXT_ERROR = { code: 'stt-no-text-recognized', message: 'No text recogn
  * that was too quiet, not an empty room — worth telling the user where to lower the threshold.
  */
 const QUIET_VOICE_SHARE = 0.4;
+/**
+ * Event entity of a satellite with a screen: the user answered the open question by touch, event type `yes` or `no`.
+ * Pressing it is the same as saying "yes"/"no" into the open microphone — no speech recognition to get it wrong
+ * (a German "Ja" came back as "Я" once).
+ */
+const CONFIRM_ENTITY = 'confirm_answer';
+/** Select entity of a satellite with a screen: the language of the date and the hints it shows. */
+const DISPLAY_LANGUAGE_ENTITY = 'display_language';
+/**
+ * After a reply that asks something: how long, beyond the reply's own length, the device may take to play it and
+ * open the microphone. Until then a button press answers the question.
+ */
+const FOLLOW_UP_START_MS = 10_000;
+/**
+ * A button press still answers the question this long after the follow-up microphone closed: the firmware may stop
+ * the microphone on the press before it reports which button it was.
+ */
+const CONFIRM_GRACE_MS = 3000;
+/**
+ * After a button answer: a follow-up microphone the device opens this soon was asked for by the question the button
+ * already answered, and is closed at once (unless our reply asked something again).
+ */
+const IGNORE_FOLLOW_UP_MS = 15_000;
+/** "Yes" and "no" as a button answer is passed on: the words of the satellite's language, which pi-agent also takes. */
+const YES_NO = {
+    en: ['Yes', 'No'],
+    de: ['Ja', 'Nein'],
+    ru: ['Да', 'Нет'],
+    pt: ['Sim', 'Não'],
+    nl: ['Ja', 'Nee'],
+    fr: ['Oui', 'Non'],
+    it: ['Sì', 'No'],
+    es: ['Sí', 'No'],
+    pl: ['Tak', 'Nie'],
+    uk: ['Так', 'Ні'],
+    zh: ['是', '不'],
+};
+/** The word a yes/no button stands for in `language` (ISO 639-1, `zh-cn` too); English for an unknown one. */
+function yesNoWord(language, yes) {
+    const base = (language || 'en').trim().toLowerCase().split(/[-_]/)[0];
+    return (YES_NO[base] ?? YES_NO.en)[yes ? 0 : 1];
+}
+/** The event type of a {@link CONFIRM_ENTITY} as yes (true) or no (false); null for anything else. */
+function confirmAnswerOf(eventType) {
+    const type = typeof eventType === 'string' ? eventType.trim().toLowerCase() : '';
+    if (type === 'yes' || type === 'confirm') {
+        return true;
+    }
+    if (type === 'no' || type === 'cancel') {
+        return false;
+    }
+    return null;
+}
+/**
+ * Names of the adapter's languages in that language, for a display-language select. `Intl.DisplayNames` covers the
+ * rest, but a Node.js built with only English ICU data knows no "Deutsch" or "Русский".
+ */
+const NATIVE_LANGUAGE_NAMES = {
+    en: 'english',
+    de: 'deutsch',
+    ru: 'русский',
+    pt: 'português',
+    nl: 'nederlands',
+    fr: 'français',
+    it: 'italiano',
+    es: 'español',
+    pl: 'polski',
+    uk: 'українська',
+    zh: '中文',
+};
+/**
+ * The option of a display-language select that means `language`: its name in that language ("Deutsch"), in English
+ * ("German") or its code ("de", "de-DE"), case ignored. Null when the device offers none of them.
+ */
+function displayLanguageOption(options, language) {
+    const code = (language || '').trim().toLowerCase();
+    const base = code.split(/[-_]/)[0];
+    if (!base) {
+        return null;
+    }
+    const names = new Set([code, base]);
+    if (NATIVE_LANGUAGE_NAMES[base]) {
+        names.add(NATIVE_LANGUAGE_NAMES[base]);
+    }
+    for (const locale of [base, 'en']) {
+        try {
+            const name = new Intl.DisplayNames([locale], { type: 'language' }).of(base);
+            if (name) {
+                names.add(name.toLocaleLowerCase(locale));
+            }
+        }
+        catch {
+            /* not a language code Intl knows */
+        }
+    }
+    const norm = (option) => option.trim().toLocaleLowerCase();
+    return (options.find(o => names.has(norm(o))) ??
+        options.find(o => norm(o).split(/[-_]/)[0] === base && /^[a-z]{2,3}([-_][a-z0-9]+)?$/.test(norm(o))) ??
+        null);
+}
 /** The configured speech threshold of a device, or 0 for the default. Table cells may arrive as strings. */
 function speechLevelOf(device) {
     const level = Math.round(Number(device.speechLevel));
@@ -325,6 +428,15 @@ class EsphomeConnection {
     errorSentAt = 0;
     /** The reply format last reported in the log, to log it again only when it changes. */
     loggedFormat = '';
+    /**
+     * The last reply asked something (`continue_conversation`, or an announcement with `start_conversation`): until
+     * when a yes/no button press answers it while no follow-up recording is open. 0 = no open question.
+     */
+    questionUntil = 0;
+    /** After a button answer: until when a follow-up microphone is the stale one of the answered question. */
+    ignoreFollowUpUntil = 0;
+    /** The display language was compared with ours since the last connect. */
+    displayLanguageChecked = false;
     constructor(device, opts) {
         this.opts = opts;
         this.host = device.ip.trim();
@@ -367,6 +479,7 @@ class EsphomeConnection {
         // reboots, so starting from a clean slate is the only safe option.
         this.entities.clear();
         this.configChecks = [];
+        this.displayLanguageChecked = false;
         const socket = net.createConnection({ host: this.host, port: this.port }, () => {
             this.opts.log.debug(`ESPHome ${this.host}:${this.port} connected`);
             this.handshake();
@@ -391,6 +504,8 @@ class EsphomeConnection {
         // A reply still being worked out must not land on the next connection, in the middle of another run.
         this.run++;
         this.runOpen = false;
+        this.questionUntil = 0;
+        this.ignoreFollowUpUntil = 0;
         if (this.online) {
             this.opts.log.info(`ESPHome satellite offline: ${this.deviceName}`);
         }
@@ -599,11 +714,110 @@ class EsphomeConnection {
                     const entity = this.entities.applyState(name, message);
                     if (entity) {
                         this.opts.onEntityState?.(this.deviceName, this.room, entity);
+                        this.onEntityChange(entity);
                     }
                 }
                 return;
         }
     }
+    /** Entities of a satellite with a screen that take part in the conversation (see {@link CONFIRM_ENTITY}). */
+    onEntityChange(entity) {
+        if (entity.objectId === CONFIRM_ENTITY && entity.kind === 'event') {
+            this.onConfirmButton(entity.value);
+        }
+        else if (entity.objectId === DISPLAY_LANGUAGE_ENTITY && entity.kind === 'select') {
+            // Once per connect, with the state the device reports on subscribing: a language the user picks on the
+            // device afterwards stays until the next connect (a change of the voice language restarts the adapter).
+            if (!this.displayLanguageChecked) {
+                this.displayLanguageChecked = true;
+                this.syncDisplayLanguage(entity);
+            }
+        }
+    }
+    /** Show the device's screen in the voice language, if it offers that language. */
+    syncDisplayLanguage(entity) {
+        const language = this.opts.language;
+        if (!language) {
+            return;
+        }
+        const option = displayLanguageOption(entity.options ?? [], language);
+        if (!option) {
+            this.opts.log.debug(`ESPHome ${this.deviceName}: ${DISPLAY_LANGUAGE_ENTITY} has no option for "${language}" ` +
+                `(${(entity.options ?? []).join(', ')})`);
+            return;
+        }
+        if (entity.value === option) {
+            return;
+        }
+        const shown = typeof entity.value === 'string' ? entity.value : '?';
+        this.opts.log.info(`ESPHome ${this.deviceName}: display language ${shown} → ${option}`);
+        this.setEntity(DISPLAY_LANGUAGE_ENTITY, option);
+    }
+    /** Whether a yes/no button press now answers a question the last reply asked. */
+    questionOpen() {
+        return !!this.capture?.answersQuestion || Date.now() < this.questionUntil;
+    }
+    /**
+     * The user pressed yes or no on the device's screen. While a reply waits for an answer that is the answer: the
+     * microphone opened for it is closed, the yes or no word goes on as the user's reply (pi-agent decides its
+     * confirmation on it like on a spoken one), and the reply to that is announced on this device.
+     */
+    onConfirmButton(eventType) {
+        const yes = confirmAnswerOf(eventType);
+        if (yes === null) {
+            this.opts.log.debug(`ESPHome ${this.deviceName}: ${CONFIRM_ENTITY} "${String(eventType)}" — not yes/no, ignored`);
+            return;
+        }
+        if (!this.questionOpen()) {
+            this.opts.log.debug(`ESPHome ${this.deviceName}: ${CONFIRM_ENTITY} "${yes ? 'yes' : 'no'}" without an open question — ignored`);
+            return;
+        }
+        this.questionUntil = 0;
+        if (this.capture) {
+            // A RUN_END while the microphone streams stops it and puts the device back to idle — without the error
+            // display an empty recognition would end with.
+            this.abortCapture();
+            this.event(EVENT.RUN_END);
+        }
+        // Whatever is still in flight for the question stays silent; so does the follow-up microphone the device may
+        // still open for it.
+        const run = ++this.run;
+        this.ignoreFollowUpUntil = Date.now() + IGNORE_FOLLOW_UP_MS;
+        void this.answerButton(yes, run);
+    }
+    /** Pass a button answer on and announce the reply (see {@link onConfirmButton}). */
+    async answerButton(yes, run) {
+        const text = yesNoWord(this.opts.language, yes);
+        this.opts.log.info(`ESPHome ${this.deviceName}: answered "${text}" with the button`);
+        this.status('processing');
+        try {
+            const answer = (await this.opts.answer(text, { device: this.deviceName, room: this.room, button: yes ? 'yes' : 'no' })).trim();
+            if (run !== this.run) {
+                return;
+            }
+            if (!answer) {
+                this.status('idle');
+                return;
+            }
+            const followUp = this.opts.expectsFollowUp?.(answer, this.deviceName) === true;
+            const ttsStart = Date.now();
+            const { pcm, sampleRate } = await this.opts.tts.synthesize(answer, this.opts.language);
+            if (run !== this.run) {
+                return;
+            }
+            this.opts.onSpoken?.(this.deviceName, {
+                ms: Date.now() - ttsStart,
+                audioMs: (pcm.length / 2 / sampleRate) * 1000,
+            });
+            await this.announce(pcm, sampleRate, followUp, answer);
+        }
+        catch (e) {
+            this.opts.log.warn(`ESPHome ${this.deviceName}: answer to the button failed — ${e.message}`);
+            if (run === this.run) {
+                this.status('idle');
+            }
+        }
+    }
     /** Write one of the device's entities. See {@link EsphomeSatellites.setEntity}. */
     setEntity(objectId, value) {
         const message = this.entities.command(objectId, value);
@@ -644,13 +858,27 @@ class EsphomeConnection {
             this.opts.log.debug(`ESPHome ${this.deviceName}: pipeline already running — ignoring wake`);
             return;
         }
+        const phrase = request.getWakeWordPhrase();
+        if (!phrase && Date.now() < this.ignoreFollowUpUntil) {
+            // The microphone for the question a button already answered: close it at once. In this state the device
+            // takes a RUN_END as "stop the microphone" and goes back to idle.
+            this.ignoreFollowUpUntil = 0;
+            this.opts.log.debug(`ESPHome ${this.deviceName}: follow-up of a question answered by button — closing it`);
+            this.event(EVENT.RUN_END);
+            return;
+        }
         if (this.runOpen) {
             // The device started over while the last request was still being answered: that answer is not
             // wanted any more, and its events would land in the middle of the new run.
             this.opts.log.debug(`ESPHome ${this.deviceName}: new wake while answering — dropping the old reply`);
         }
         this.run++;
-        const phrase = request.getWakeWordPhrase();
+        // Without a wake word the device re-opened the mic for the reply that asked something; with one, the user
+        // started over and the question is gone.
+        const answersQuestion = !phrase && Date.now() < this.questionUntil;
+        if (phrase) {
+            this.questionUntil = 0;
+        }
         this.opts.log.debug(`ESPHome ${this.deviceName}: wake "${phrase || '(follow-up)'}"`);
         const response = new esphomeProto_1.pb.VoiceAssistantResponse();
         response.setPort(0); // 0 = stream the audio over this socket
@@ -682,6 +910,7 @@ class EsphomeConnection {
             }),
             conversationId: request.getConversationId(),
             wakeWord: phrase || '',
+            answersQuestion,
             timer,
             stallCheck,
             startedAt: Date.now(),
@@ -704,6 +933,8 @@ class EsphomeConnection {
      */
     onStopRequest() {
         const running = !!this.capture || this.runOpen;
+        // A device with yes/no buttons may stop its microphone on the press before it reports which button it was.
+        this.questionUntil = this.questionOpen() ? Date.now() + CONFIRM_GRACE_MS : 0;
         this.abortCapture();
         if (running) {
             this.run++; // whatever is still being answered stays silent
@@ -803,6 +1034,11 @@ class EsphomeConnection {
         this.opts.log.debug(`ESPHome ${this.deviceName}: captured ${seconds.toFixed(2)} s ` +
             `(peak RMS ${vad.peak}, noise floor ${vad.noiseFloor}, speech/silence gate ${vad.startLevel}/${vad.endLevel}` +
             `${startedAt}, ${endedBy})`);
+        if (capture.answersQuestion) {
+            // Spoken: that is the answer, a button press now would be a second one. Silent: the buttons still answer
+            // for a moment, as the screen may still show them.
+            this.questionUntil = capture.detector.sawSpeech ? 0 : Date.now() + CONFIRM_GRACE_MS;
+        }
         if (!pcm.length || !capture.detector.sawSpeech) {
             const loudest = Math.round(capture.detector.analysedPeak);
             const gate = Math.round(capture.detector.startLevel);
@@ -846,6 +1082,9 @@ class EsphomeConnection {
                 // language than the STT listens in (a German "Ja" to a Russian STT comes back empty).
                 this.opts.log.info(`Voice Q (${this.deviceName}): (empty) — speech recognition returned no text for ${seconds.toFixed(1)} s of audio`);
                 this.discard(capture, pcm, 'empty-stt', vad);
+                if (capture.answersQuestion) {
+                    this.questionUntil = Date.now() + CONFIRM_GRACE_MS;
+                }
                 // No STT_END without text — like Home Assistant, which reports this case as an error only.
                 this.endWithoutRequest();
                 return;
@@ -865,7 +1104,11 @@ class EsphomeConnection {
                 return;
             }
             const followUp = !!answer && this.opts.expectsFollowUp?.(answer, this.deviceName) === true;
-            this.event(EVENT.INTENT_END, followUp ? { continue_conversation: '1' } : undefined);
+            // Always say it, "0" included: ESPHome keeps the flag of the last INTENT_END that carried it, so after
+            // one question every later reply re-opened the microphone (field report 2026-11-04). Home Assistant
+            // sends "0" just the same.
+            const conversation = { continue_conversation: followUp ? '1' : '0' };
+            this.event(EVENT.INTENT_END, conversation);
             if (!answer) {
                 this.event(EVENT.RUN_END);
                 this.status('idle');
@@ -887,7 +1130,11 @@ class EsphomeConnection {
             this.status('speaking');
             this.event(EVENT.TTS_START, { text: answer });
             this.event(EVENT.TTS_END, { url });
-            this.event(EVENT.RUN_END, followUp ? { continue_conversation: '1' } : undefined);
+            // linux-voice-assistant reads the flag here (ignoring "0"), ESPHome at INTENT_END
+            this.event(EVENT.RUN_END, conversation);
+            this.questionUntil = followUp
+                ? Date.now() + (ttsPcm.length / 2 / sampleRate) * 1000 + FOLLOW_UP_START_MS
+                : 0;
             this.armPlaybackTimeout();
         }
         catch (e) {
@@ -989,17 +1236,23 @@ class EsphomeConnection {
      * Speak a ready PCM buffer on this device (timers, alarms, `tts.text`, jingles). With
      * `startConversation` the microphone opens once the clip has played, to catch the answer to a question.
      */
-    async announce(pcm, sampleRate, startConversation = false) {
+    async announce(pcm, sampleRate, startConversation = false, text = '') {
         if (!this.online || !pcm.length) {
             return;
         }
         const request = new esphomeProto_1.pb.VoiceAssistantAnnounceRequest();
         request.setMediaId(await this.publish(pcm, sampleRate));
-        request.setText('');
+        // What a device with a screen shows while it plays (its on_tts_start)
+        request.setText(text);
         request.setPreannounceMediaId('');
         // Needs feature flag 32 (START_CONVERSATION); on a device without it the clip still plays and
         // only the microphone stays shut, so there is nothing to guard here.
         request.setStartConversation(startConversation);
+        if (startConversation) {
+            // A question: the microphone the device opens after it is wanted, and a button press answers it
+            this.ignoreFollowUpUntil = 0;
+            this.questionUntil = Date.now() + (pcm.length / 2 / sampleRate) * 1000 + FOLLOW_UP_START_MS;
+        }
         this.status('speaking');
         this.send(request);
         this.armPlaybackTimeout();