@sympoies/dsh-telegram 0.0.0-bootstrap.0 → 0.6.0

package/lib/media/collect.js

@@ -0,0 +1,235 @@
+/**
+ * Fetching a message's media and turning it into prompt content.
+ *
+ * Downloading is the part that can go wrong in ordinary ways — a file above
+ * the bot download limit, a network drop, an image the harness refuses — and
+ * none of them should cost the user their message. So every failure becomes a
+ * note in the prompt saying what could not be read, and the text the user
+ * typed alongside it still reaches the agent.
+ */
+import { SILENT_LOGGER } from '../harness/types.js';
+import { describeMedia, isStorableImage } from './intake.js';
+import { describeSize, isTooLarge, orderBySuitability } from './limits.js';
+export class MediaCollector {
+    options;
+    logger;
+    constructor(options) {
+        this.options = options;
+        this.logger = options.logger ?? SILENT_LOGGER;
+    }
+    /**
+     * Turn one message into the parts a prompt is built from.
+     *
+     * @param message - the incoming message.
+     * @param caption - the text the user typed, if any.
+     * @returns text and image parts, in the order the model should read them.
+     */
+    async collect(message, caption, context = {}) {
+        const item = describeMedia(message);
+        const said = caption?.trim();
+        if (!item) {
+            return { parts: said ? [{ type: 'text', text: said }] : [] };
+        }
+        if (item.kind === 'unsupported') {
+            const what = item.describedAs ?? 'a file';
+            return this.declined(said, `${what}, which cannot be read`, `I can't read ${what}.`);
+        }
+        if (item.size !== undefined && item.size > this.options.maxBytes) {
+            const limit = Math.floor(this.options.maxBytes / 1024 / 1024);
+            return this.declined(said, `a file of ${item.size} bytes, above the size limit`, `That file is too large — the limit is ${limit} MB.`);
+        }
+        if (item.kind === 'image') {
+            // Skipped when anything can read the picture: the conversation's own
+            // model, a vision model behind it, or OCR. The refusal exists to replace
+            // a failed turn with a useful sentence, and with any of those in place it
+            // would instead throw away an image that was going to be read.
+            const readable = context.modelSees === true || (await this.options.canReadWithoutModel?.()) === true;
+            const refusal = readable ? undefined : await this.imageRefusal();
+            if (refusal)
+                return this.declined(said, refusal.forAgent, refusal.forUser);
+        }
+        try {
+            if (item.kind === 'text') {
+                const bytes = await this.download(item.fileId);
+                const text = decodeText(bytes, this.options.maxTextChars);
+                const name = item.name ?? 'attachment';
+                return {
+                    parts: [
+                        { type: 'text', text: `${said ? `${said}\n\n` : ''}File \`${name}\`:\n\n${text}` },
+                    ],
+                };
+            }
+            if (!this.options.attachments) {
+                return this.declined(said, 'an image, but this deployment stores none', 'This deployment cannot store images.');
+            }
+            if (!isStorableImage(item.mediaType)) {
+                const type = item.mediaType ?? 'unknown';
+                return this.declined(said, `an image of type ${type}, which cannot be stored`, `Images of type ${type} cannot be stored. PNG, JPEG, WebP and GIF work.`);
+            }
+            const stored = await this.storeImage(item);
+            if (!stored) {
+                // Reached only when every size Telegram offered was still refused,
+                // which in practice means a file sent uncompressed: there is one size
+                // and it is the full-resolution original.
+                const limits = this.options.attachments.imageLimits;
+                const explanation = describeSize(largest(item.candidates), limits) ??
+                    'That image is larger than this harness stores.';
+                return this.declined(said, 'an image too large for this harness to store, so it was left out', `${explanation} Sending it as a photo rather than as a file lets Telegram offer a smaller copy.`);
+            }
+            // Text first: it is what the user asked, and the image is its subject.
+            const parts = [];
+            if (said)
+                parts.push({ type: 'text', text: said });
+            parts.push({ type: 'image', attachment: stored });
+            return { parts };
+        }
+        catch (error) {
+            this.logger.warn('[dsh-telegram] could not read an attachment', error);
+            const reason = error instanceof Error ? error.message : String(error);
+            return this.declined(said, `a file that could not be read: ${reason}`, `That file could not be read: ${reason}`);
+        }
+    }
+    /**
+     * Turn a whole album into one prompt.
+     *
+     * Each part is collected on its own — the same path a single photo takes, so
+     * refusals and size limits behave identically — and the results are then
+     * laid out as one message would have been: what the user said, then every
+     * image, then anything that could not be read.
+     *
+     * @param messages - the album's parts, in the order they were sent.
+     * @param caption - the one caption the album carries.
+     */
+    async collectAll(messages, caption, context = {}) {
+        if (messages.length === 0)
+            return { parts: [] };
+        if (messages.length === 1) {
+            return await this.collect(messages[0], caption, context);
+        }
+        // Sequential rather than parallel: each part downloads bytes and commits
+        // them, and a dozen at once is a burst of traffic and disk for no gain
+        // when the user is waiting on the whole set anyway.
+        const collected = [];
+        for (const message of messages) {
+            collected.push(await this.collect(message, undefined, context));
+        }
+        const said = caption?.trim();
+        const images = collected.flatMap((item) => item.parts.filter((part) => part.type === 'image'));
+        const notes = collected.flatMap((item) => item.parts.filter((part) => part.type === 'text'));
+        const notices = collected.map((item) => item.notice).filter((notice) => notice !== undefined);
+        const parts = [];
+        if (said)
+            parts.push({ type: 'text', text: said });
+        parts.push(...images);
+        // After the images, because a note explains what is missing from them.
+        parts.push(...notes);
+        return {
+            parts,
+            // One line however many parts failed the same way: three identical
+            // refusals in a row say nothing the first did not.
+            ...(notices.length > 0 ? { notice: [...new Set(notices)].join('\n') } : {}),
+        };
+    }
+    /**
+     * Store an image, stepping down through Telegram's smaller renderings.
+     *
+     * The seam refuses anything over its per-side limit, and the largest size of
+     * a phone screenshot is always over it — 1179×2556 against a limit of 2000.
+     * The largest size that fits is tried first, and a rejection falls through
+     * to the next, because the limit belongs to the harness and may not be the
+     * number read here.
+     *
+     * @returns the stored reference, or undefined when no size was accepted.
+     */
+    async storeImage(item) {
+        const store = this.options.attachments;
+        if (!store)
+            return undefined;
+        const candidates = item.candidates ?? [{ fileId: item.fileId }];
+        const ordered = orderBySuitability(candidates, store.imageLimits);
+        for (const candidate of ordered) {
+            try {
+                const bytes = await this.download(candidate.fileId);
+                return await store.saveImage({
+                    data: bytes,
+                    mediaType: item.mediaType,
+                    ...(item.name !== undefined ? { name: item.name } : {}),
+                });
+            }
+            catch (error) {
+                // Only a size refusal is worth another download; a malformed file or a
+                // failed write would fail identically however small the image was.
+                if (!isTooLarge(error))
+                    throw error;
+                this.logger.debug('[dsh-telegram] image refused as too large; trying a smaller size', error);
+            }
+        }
+        // Exhausted rather than thrown: the seam's own wording — "Image exceeds
+        // the configured per-side pixel limit" — tells the user nothing they can
+        // act on, and the caller has something better to say.
+        return undefined;
+    }
+    /**
+     * Why an image must not be sent on the current route, if it must not.
+     *
+     * A model with no image input rejects the entire request, so this refusal
+     * replaces a failed turn with a sentence naming what would have worked.
+     */
+    async imageRefusal() {
+        const vision = this.options.vision;
+        if (!vision)
+            return undefined;
+        if ((await vision.verdict()) !== 'no')
+            return undefined;
+        const model = vision.currentModel() ?? 'the current model';
+        const alternatives = await vision.alternatives();
+        const suggestion = alternatives.length > 0
+            ? ` These accept images: ${alternatives.join(', ')}.`
+            : ' No configured model accepts images.';
+        return {
+            forAgent: `an image, which the model ${model} cannot read, so it was left out`,
+            forUser: `${model} can't read images.${suggestion} Change it in Settings → Models.`,
+        };
+    }
+    /** Keep the caption, tell the agent what was left out, and tell the user why. */
+    declined(said, forAgent, forUser) {
+        // Applied here rather than at each call site: every refusal in this class
+        // funnels through this method, so one guard covers all of them and a
+        // refusal added later cannot forget it.
+        const clean = this.options.redact ?? ((text) => text);
+        return {
+            parts: [{ type: 'text', text: clean(note(said, `The user sent ${forAgent}.`)) }],
+            notice: clean(forUser),
+        };
+    }
+    /** Resolve a file id and pull its bytes, refusing anything oversized. */
+    async download(fileId) {
+        const file = await this.options.source.getFile(fileId);
+        if (!file.file_path)
+            throw new Error('telegram returned no download path');
+        if (file.file_size !== undefined && file.file_size > this.options.maxBytes) {
+            throw new Error(`file is ${file.file_size} bytes, above the size limit`);
+        }
+        return await this.options.source.downloadFile(file.file_path);
+    }
+}
+/** The biggest size offered, for a refusal that can name what was sent. */
+function largest(candidates) {
+    if (!candidates || candidates.length === 0)
+        return undefined;
+    return candidates.reduce((biggest, candidate) => (candidate.width ?? 0) * (candidate.height ?? 0) > (biggest.width ?? 0) * (biggest.height ?? 0)
+        ? candidate
+        : biggest);
+}
+/** Keep what the user said, and add why their file did not come through. */
+function note(said, explanation) {
+    return said ? `${said}\n\n(${explanation})` : `(${explanation})`;
+}
+/** Decode file bytes as text, replacing what is not valid UTF-8. */
+function decodeText(bytes, maxChars) {
+    const text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+    if (text.length <= maxChars)
+        return text;
+    return `${text.slice(0, maxChars)}\n\n… (truncated at ${maxChars} characters)`;
+}
+//# sourceMappingURL=collect.js.map

package/lib/media/collect.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"collect.js","sourceRoot":"","sources":["../../src/media/collect.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AAEnD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC5D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AA+F1E,MAAM,OAAO,cAAc;IAGI;IAFZ,MAAM,CAAQ;IAE/B,YAA6B,OAA8B;QAA9B,YAAO,GAAP,OAAO,CAAuB;QACzD,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAA;IAC/C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CACX,OAAwB,EACxB,OAA2B,EAC3B,UAA6B,EAAE;QAE/B,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,CAAA;QACnC,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,EAAE,CAAA;QAE5B,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAA;QAC9D,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,IAAI,QAAQ,CAAA;YACzC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,IAAI,wBAAwB,EAAE,gBAAgB,IAAI,GAAG,CAAC,CAAA;QACtF,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC,CAAA;YAC7D,OAAO,IAAI,CAAC,QAAQ,CAClB,IAAI,EACJ,aAAa,IAAI,CAAC,IAAI,8BAA8B,EACpD,yCAAyC,KAAK,MAAM,CACrD,CAAA;QACH,CAAC;QAED,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1B,qEAAqE;YACrE,yEAAyE;YACzE,0EAA0E;YAC1E,+DAA+D;YAC/D,MAAM,QAAQ,GACZ,OAAO,CAAC,SAAS,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,KAAK,IAAI,CAAA;YACrF,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAA;YAChE,IAAI,OAAO;gBAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;QAC5E,CAAC;QAED,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;gBACzD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,YAAY,CAAA;gBACtC,OAAO;oBACL,KAAK,EAAE;wBACL,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,UAAU,IAAI,EAAE,EAAE;qBACnF;iBACF,CAAA;YACH,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAC9B,OAAO,IAAI,CAAC,QAAQ,CAClB,IAAI,EACJ,2CAA2C,EAC3C,sCAAsC,CACvC,CAAA;YACH,CAAC;YACD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,IAAI,SAAS,CAAA;gBACxC,OAAO,IAAI,CAAC,QAAQ,CAClB,IAAI,EACJ,oBAAoB,IAAI,0BAA0B,EAClD,kBAAkB,IAAI,kDAAkD,CACzE,CAAA;YACH,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,mEAAmE;gBACnE,sEAAsE;gBACtE,0CAA0C;gBAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,WAAW,CAAA;gBACnD,MAAM,WAAW,GACf,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;oBAC9C,gDAAgD,CAAA;gBAClD,OAAO,IAAI,CAAC,QAAQ,CAClB,IAAI,EACJ,kEAAkE,EAClE,GAAG,WAAW,kFAAkF,CACjG,CAAA;YACH,CAAC;YAED,uEAAuE;YACvE,MAAM,KAAK,GAAiB,EAAE,CAAA;YAC9B,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;YAClD,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAA;YACjD,OAAO,EAAE,KAAK,EAAE,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,EAAE,KAAK,CAAC,CAAA;YACtE,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACrE,OAAO,IAAI,CAAC,QAAQ,CAClB,IAAI,EACJ,kCAAkC,MAAM,EAAE,EAC1C,gCAAgC,MAAM,EAAE,CACzC,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,UAAU,CACd,QAAoC,EACpC,OAA2B,EAC3B,UAA6B,EAAE;QAE/B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;QAC/C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAoB,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;QAC7E,CAAC;QAED,yEAAyE;QACzE,uEAAuE;QACvE,oDAAoD;QACpD,MAAM,SAAS,GAAqB,EAAE,CAAA;QACtC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAA;QACjE,CAAC;QAED,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,EAAE,CAAA;QAC5B,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAA;QAC9F,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CACvC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAA0C,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAC1F,CAAA;QACD,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAA;QAE7F,MAAM,KAAK,GAAiB,EAAE,CAAA;QAC9B,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QAClD,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;QACrB,uEAAuE;QACvE,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAA;QAEpB,OAAO;YACL,KAAK;YACL,mEAAmE;YACnE,mDAAmD;YACnD,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5E,CAAA;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACK,KAAK,CAAC,UAAU,CAAC,IAKxB;QACC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAA;QACtC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAA;QAE5B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QAC/D,MAAM,OAAO,GAAG,kBAAkB,CAAC,UAAU,EAAE,KAAK,CAAC,WAAW,CAAC,CAAA;QAEjE,KAAK,MAAM,SAAS,IAAI,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC;gBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;gBACnD,OAAO,MAAM,KAAK,CAAC,SAAS,CAAC;oBAC3B,IAAI,EAAE,KAAK;oBACX,SAAS,EAAE,IAAI,CAAC,SAAmB;oBACnC,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACxD,CAAC,CAAA;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,uEAAuE;gBACvE,mEAAmE;gBACnE,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;oBAAE,MAAM,KAAK,CAAA;gBACnC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,kEAAkE,EAAE,KAAK,CAAC,CAAA;YAC9F,CAAC;QACH,CAAC;QAED,wEAAwE;QACxE,yEAAyE;QACzE,sDAAsD;QACtD,OAAO,SAAS,CAAA;IAClB,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,YAAY;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QAClC,IAAI,CAAC,MAAM;YAAE,OAAO,SAAS,CAAA;QAC7B,IAAI,CAAC,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,IAAI;YAAE,OAAO,SAAS,CAAA;QAEvD,MAAM,KAAK,GAAG,MAAM,CAAC,YAAY,EAAE,IAAI,mBAAmB,CAAA;QAC1D,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,YAAY,EAAE,CAAA;QAChD,MAAM,UAAU,GACd,YAAY,CAAC,MAAM,GAAG,CAAC;YACrB,CAAC,CAAC,yBAAyB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG;YACrD,CAAC,CAAC,sCAAsC,CAAA;QAE5C,OAAO;YACL,QAAQ,EAAE,6BAA6B,KAAK,kCAAkC;YAC9E,OAAO,EAAE,GAAG,KAAK,sBAAsB,UAAU,kCAAkC;SACpF,CAAA;IACH,CAAC;IAED,iFAAiF;IACzE,QAAQ,CAAC,IAAwB,EAAE,QAAgB,EAAE,OAAe;QAC1E,0EAA0E;QAC1E,qEAAqE;QACrE,wCAAwC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;QAC7D,OAAO;YACL,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC;YAChF,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC;SACvB,CAAA;IACH,CAAC;IAED,yEAAyE;IACjE,KAAK,CAAC,QAAQ,CAAC,MAAc;QACnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACtD,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAA;QAC1E,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;YAC3E,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,8BAA8B,CAAC,CAAA;QAC1E,CAAC;QACD,OAAO,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAC/D,CAAC;CACF;AAED,2EAA2E;AAC3E,SAAS,OAAO,CAAC,UAAiD;IAChE,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAA;IAC5D,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,CAC9C,CAAC,SAAS,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;QAC7F,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,CACZ,CAAA;AACH,CAAC;AAED,4EAA4E;AAC5E,SAAS,IAAI,CAAC,IAAwB,EAAE,WAAmB;IACzD,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,QAAQ,WAAW,GAAG,CAAC,CAAC,CAAC,IAAI,WAAW,GAAG,CAAA;AAClE,CAAC;AAED,oEAAoE;AACpE,SAAS,UAAU,CAAC,KAAiB,EAAE,QAAgB;IACrD,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACrE,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ;QAAE,OAAO,IAAI,CAAA;IACxC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,uBAAuB,QAAQ,cAAc,CAAA;AAChF,CAAC"}

package/lib/media/extractor.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Reading an image with one model so another can use what it says.
+ *
+ * A provider inspects the whole request history for images, so an image left
+ * in a conversation binds that conversation to a model that can see — for
+ * good. That costs more than the picture is usually worth: the conversation
+ * loses the model it was chosen for, and the tools configured around it.
+ *
+ * So the image never enters the conversation. It goes to a throwaway session
+ * on the vision model, whose reply — the transcription, the description — is
+ * what the conversation receives, as ordinary text. The conversation's history
+ * stays free of images, so it keeps its own model and never becomes stuck.
+ *
+ * The throwaway session is disposed either way. It exists for one turn.
+ */
+import type { PromptPart } from './collect.js';
+import type { ModelRoute } from '../harness/model-selection.js';
+import type { Logger } from '../harness/types.js';
+/** A live agent driven for exactly one turn. */
+export interface ExtractionAgent {
+    readonly sessionId: string;
+    followup(content: readonly PromptPart[]): void;
+    dispose(): Promise<void>;
+}
+/** Creating the throwaway agent the extraction runs on. */
+export interface ExtractionHost {
+    create(sessionId: string, cwd: string, route: ModelRoute): Promise<ExtractionAgent>;
+}
+/** The session-event shapes an extraction watches for. */
+export interface ExtractionEvent {
+    readonly type: string;
+    readonly data?: unknown;
+}
+/** Reading an image's text without a model, when there is no model. */
+export interface FallbackReader {
+    available(): Promise<boolean>;
+    read(data: Uint8Array): Promise<string | undefined>;
+}
+/** Fetching the bytes back for a stored attachment. */
+export interface AttachmentReader {
+    readImage(ref: unknown): Promise<{
+        data: Uint8Array;
+    }>;
+}
+/** Construction options. */
+export interface VisionExtractorOptions {
+    readonly host: ExtractionHost;
+    readonly cwd: string;
+    /**
+     * Whether any vision model is configured anywhere.
+     *
+     * Only decides {@link VisionExtractor.available}; which model a particular
+     * conversation reads with is passed to `resolve`, because `/vision` makes
+     * that a per-conversation answer.
+     */
+    readonly visionModel: () => ModelRoute | undefined;
+    /**
+     * OCR, for when no vision model is configured or the one configured could
+     * not be reached. Strictly a fallback: it reads text and does not see, so a
+     * model that can look is always preferred.
+     */
+    readonly fallback?: FallbackReader;
+    /** Reads a stored image back, which the fallback needs and the model does not. */
+    readonly attachments?: AttachmentReader;
+    readonly newSessionId?: () => string;
+    readonly timeoutMs?: number;
+    readonly logger?: Logger;
+}
+export declare class VisionExtractor {
+    private readonly options;
+    private readonly pending;
+    private readonly logger;
+    private counter;
+    constructor(options: VisionExtractorOptions);
+    /**
+     * Whether an image can be read at all right now.
+     *
+     * Optimistic about the fallback: probing Tesseract spawns a process, and
+     * this is consulted on the path of every message. A fallback that turns out
+     * to be absent simply reads nothing, which is the same outcome as saying so
+     * here would have been.
+     */
+    get available(): boolean;
+    /**
+     * Replace image parts with what a vision model reads in them.
+     *
+     * @param content - the prompt as the user sent it.
+     * @returns the same prompt with each image replaced by its reading, or the
+     *   original content when there is nothing to read or no model to read it.
+     */
+    resolve(content: readonly PromptPart[], route?: ModelRoute | undefined): Promise<PromptPart[]>;
+    /**
+     * Read every image's text with the fallback.
+     *
+     * The bytes come back from the attachment store rather than being kept
+     * around: the model path never needs them, and holding every image in memory
+     * against the chance that a model call fails would be a strange thing to pay
+     * for on every message.
+     *
+     * @returns the readings joined, or undefined when there were none.
+     */
+    private scan;
+    /**
+     * Consume one session event.
+     *
+     * @returns whether it belonged to an extraction, so the caller knows not to
+     *   treat it as a conversation of its own.
+     */
+    handle(sessionId: string, event: ExtractionEvent): boolean;
+    /** Abandon every extraction in flight — the plugin is unloading. */
+    dispose(): void;
+    /** Run one throwaway turn and return what the model said. */
+    private read;
+    /** A session id no conversation can collide with. */
+    private nextSessionId;
+}
+//# sourceMappingURL=extractor.d.ts.map

package/lib/media/extractor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor.d.ts","sourceRoot":"","sources":["../../src/media/extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAA;AAC/D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAA;AAcjD,gDAAgD;AAChD,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,IAAI,CAAA;IAC9C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CACzB;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,eAAe,CAAC,CAAA;CACpF;AAED,0DAA0D;AAC1D,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,CAAA;CACxB;AAED,uEAAuE;AACvE,MAAM,WAAW,cAAc;IAC7B,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,CAAA;IAC7B,IAAI,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAA;CACpD;AAED,uDAAuD;AACvD,MAAM,WAAW,gBAAgB;IAC/B,SAAS,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,UAAU,CAAA;KAAE,CAAC,CAAA;CACvD;AAED,4BAA4B;AAC5B,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,IAAI,EAAE,cAAc,CAAA;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB;;;;;;OAMG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,UAAU,GAAG,SAAS,CAAA;IAClD;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,cAAc,CAAA;IAClC,kFAAkF;IAClF,QAAQ,CAAC,WAAW,CAAC,EAAE,gBAAgB,CAAA;IACvC,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,MAAM,CAAA;IACpC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AASD,qBAAa,eAAe;IAKd,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA6B;IACrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAQ;IAC/B,OAAO,CAAC,OAAO,CAAI;gBAEU,OAAO,EAAE,sBAAsB;IAI5D;;;;;;;OAOG;IACH,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED;;;;;;OAMG;IACG,OAAO,CACX,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,KAAK,GAAE,UAAU,GAAG,SAAsC,GACzD,OAAO,CAAC,UAAU,EAAE,CAAC;IA0CxB;;;;;;;;;OASG;YACW,IAAI;IA2BlB;;;;;OAKG;IACH,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,GAAG,OAAO;IAoC1D,oEAAoE;IACpE,OAAO,IAAI,IAAI;IAIf,6DAA6D;YAC/C,IAAI;IA2ClB,qDAAqD;IACrD,OAAO,CAAC,aAAa;CAKtB"}

package/lib/media/extractor.js

@@ -0,0 +1,207 @@
+/**
+ * Reading an image with one model so another can use what it says.
+ *
+ * A provider inspects the whole request history for images, so an image left
+ * in a conversation binds that conversation to a model that can see — for
+ * good. That costs more than the picture is usually worth: the conversation
+ * loses the model it was chosen for, and the tools configured around it.
+ *
+ * So the image never enters the conversation. It goes to a throwaway session
+ * on the vision model, whose reply — the transcription, the description — is
+ * what the conversation receives, as ordinary text. The conversation's history
+ * stays free of images, so it keeps its own model and never becomes stuck.
+ *
+ * The throwaway session is disposed either way. It exists for one turn.
+ */
+import { SILENT_LOGGER } from '../harness/types.js';
+import { labelOcr } from './ocr.js';
+/** What the vision model is asked to do with the picture. */
+const INSTRUCTION = 'Read this image for someone who cannot see it. Transcribe every piece of ' +
+    'text it contains, exactly, including numbers, dates, names and amounts. ' +
+    'Then describe briefly what the image is. Reply with only that — no ' +
+    'preamble, no offer of further help.';
+/** How long one extraction may take before the conversation moves on without it. */
+const DEFAULT_TIMEOUT_MS = 120_000;
+export class VisionExtractor {
+    options;
+    pending = new Map();
+    logger;
+    counter = 0;
+    constructor(options) {
+        this.options = options;
+        this.logger = options.logger ?? SILENT_LOGGER;
+    }
+    /**
+     * Whether an image can be read at all right now.
+     *
+     * Optimistic about the fallback: probing Tesseract spawns a process, and
+     * this is consulted on the path of every message. A fallback that turns out
+     * to be absent simply reads nothing, which is the same outcome as saying so
+     * here would have been.
+     */
+    get available() {
+        return this.options.visionModel() !== undefined || this.options.fallback !== undefined;
+    }
+    /**
+     * Replace image parts with what a vision model reads in them.
+     *
+     * @param content - the prompt as the user sent it.
+     * @returns the same prompt with each image replaced by its reading, or the
+     *   original content when there is nothing to read or no model to read it.
+     */
+    async resolve(content, route = this.options.visionModel()) {
+        const images = content.filter((part) => part.type === 'image');
+        if (images.length === 0)
+            return [...content];
+        // A route is no longer required: without one there is still OCR, and
+        // without either the prompt says so rather than carrying an image that
+        // nothing downstream can use.
+        if (route === undefined && this.options.fallback === undefined)
+            return [...content];
+        const said = content
+            .filter((part) => part.type === 'text')
+            .map((part) => part.text);
+        const read = route === undefined ? undefined : await this.read(images, route);
+        if (read !== undefined) {
+            // The user's own words first: the image is what they are asking about.
+            return [
+                ...said.map((text) => ({ type: 'text', text })),
+                { type: 'text', text: `Contents of the image the user sent:\n\n${read}` },
+            ];
+        }
+        // Reached either because nothing was configured to look, or because what
+        // was configured could not be reached. Both are cases where reading the
+        // text beats returning nothing at all.
+        const scanned = await this.scan(images);
+        if (scanned !== undefined) {
+            return [...said.map((text) => ({ type: 'text', text })), { type: 'text', text: scanned }];
+        }
+        return [
+            ...said.map((text) => ({ type: 'text', text })),
+            {
+                type: 'text',
+                text: route === undefined
+                    ? '(The user sent an image, but nothing here can read one. Configure a vision model in Settings → Telegram, or install tesseract.)'
+                    : `(The user sent an image, but it could not be read by ${route.model}.)`,
+            },
+        ];
+    }
+    /**
+     * Read every image's text with the fallback.
+     *
+     * The bytes come back from the attachment store rather than being kept
+     * around: the model path never needs them, and holding every image in memory
+     * against the chance that a model call fails would be a strange thing to pay
+     * for on every message.
+     *
+     * @returns the readings joined, or undefined when there were none.
+     */
+    async scan(images) {
+        const { fallback, attachments } = this.options;
+        if (!fallback || !attachments)
+            return undefined;
+        if (!(await fallback.available()))
+            return undefined;
+        const readings = [];
+        for (const image of images) {
+            if (image.type !== 'image')
+                continue;
+            try {
+                const stored = await attachments.readImage(image.attachment);
+                const text = await fallback.read(stored.data);
+                if (text !== undefined)
+                    readings.push(text);
+            }
+            catch (error) {
+                this.logger.warn('[dsh-telegram] could not read a stored image back', error);
+            }
+        }
+        if (readings.length === 0)
+            return undefined;
+        return labelOcr(readings.length === 1
+            ? readings[0]
+            : readings.map((text, index) => `--- image ${index + 1} ---\n${text}`).join('\n\n'));
+    }
+    /**
+     * Consume one session event.
+     *
+     * @returns whether it belonged to an extraction, so the caller knows not to
+     *   treat it as a conversation of its own.
+     */
+    handle(sessionId, event) {
+        const entry = this.pending.get(sessionId);
+        if (!entry)
+            return false;
+        if (event.type === 'assistant/message') {
+            const message = event.data?.message;
+            const text = (message?.content ?? [])
+                .filter((block) => block.type === 'text')
+                .map((block) => block.text ?? '')
+                .join('')
+                .trim();
+            if (text !== '')
+                entry.text = text;
+        }
+        if (event.type === 'turn/end') {
+            const reason = event.data?.reason;
+            if (reason?.kind === 'error') {
+                this.logger.warn(`[dsh-telegram] the vision model could not read the image: ${reason.error?.message ?? 'the turn failed'}`);
+                entry.settle(undefined);
+                return true;
+            }
+            entry.settle(entry.text === '' ? undefined : entry.text);
+        }
+        return true;
+    }
+    /** Abandon every extraction in flight — the plugin is unloading. */
+    dispose() {
+        for (const entry of [...this.pending.values()])
+            entry.settle(undefined);
+    }
+    /** Run one throwaway turn and return what the model said. */
+    async read(images, route) {
+        const sessionId = this.nextSessionId();
+        let agent;
+        try {
+            agent = await this.options.host.create(sessionId, this.options.cwd, route);
+        }
+        catch (error) {
+            this.logger.warn('[dsh-telegram] could not open a session to read the image', error);
+            return undefined;
+        }
+        try {
+            const settled = new Promise((resolve) => {
+                const timer = setTimeout(() => {
+                    this.pending.delete(sessionId);
+                    this.logger.warn('[dsh-telegram] reading the image timed out');
+                    resolve(undefined);
+                }, this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+                this.pending.set(sessionId, {
+                    text: '',
+                    settle: (text) => {
+                        clearTimeout(timer);
+                        this.pending.delete(sessionId);
+                        resolve(text);
+                    },
+                });
+            });
+            agent.followup([{ type: 'text', text: INSTRUCTION }, ...images]);
+            return await settled;
+        }
+        finally {
+            // One turn is its whole life; leaving it loaded would keep a second
+            // model warm for every picture anyone sends.
+            await agent.dispose().catch((error) => {
+                this.logger.warn('[dsh-telegram] could not dispose the reading session', error);
+            });
+        }
+    }
+    /** A session id no conversation can collide with. */
+    nextSessionId() {
+        if (this.options.newSessionId)
+            return this.options.newSessionId();
+        this.counter += 1;
+        return `tg-vision-${Date.now()}-${this.counter}`;
+    }
+}
+//# sourceMappingURL=extractor.js.map

package/lib/media/extractor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extractor.js","sourceRoot":"","sources":["../../src/media/extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AAEnC,6DAA6D;AAC7D,MAAM,WAAW,GACf,2EAA2E;IAC3E,0EAA0E;IAC1E,qEAAqE;IACrE,qCAAqC,CAAA;AAEvC,oFAAoF;AACpF,MAAM,kBAAkB,GAAG,OAAO,CAAA;AA+DlC,MAAM,OAAO,eAAe;IAKG;IAJZ,OAAO,GAAG,IAAI,GAAG,EAAmB,CAAA;IACpC,MAAM,CAAQ;IACvB,OAAO,GAAG,CAAC,CAAA;IAEnB,YAA6B,OAA+B;QAA/B,YAAO,GAAP,OAAO,CAAwB;QAC1D,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,aAAa,CAAA;IAC/C,CAAC;IAED;;;;;;;OAOG;IACH,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAA;IACxF,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,OAAO,CACX,OAA8B,EAC9B,QAAgC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE;QAE1D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,CAAA;QAC9D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,GAAG,OAAO,CAAC,CAAA;QAE5C,qEAAqE;QACrE,uEAAuE;QACvE,8BAA8B;QAC9B,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS;YAAE,OAAO,CAAC,GAAG,OAAO,CAAC,CAAA;QAEnF,MAAM,IAAI,GAAG,OAAO;aACjB,MAAM,CAAC,CAAC,IAAI,EAA0C,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;aAC9E,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAE3B,MAAM,IAAI,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;QAC7E,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,uEAAuE;YACvE,OAAO;gBACL,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,CAAC,CAAC;gBACxD,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,2CAA2C,IAAI,EAAE,EAAE;aACnF,CAAA;QACH,CAAC;QAED,yEAAyE;QACzE,wEAAwE;QACxE,uCAAuC;QACvC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACvC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAA;QAC7G,CAAC;QAED,OAAO;YACL,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,IAAI,EAAE,CAAC,CAAC;YACxD;gBACE,IAAI,EAAE,MAAe;gBACrB,IAAI,EACF,KAAK,KAAK,SAAS;oBACjB,CAAC,CAAC,iIAAiI;oBACnI,CAAC,CAAC,wDAAwD,KAAK,CAAC,KAAK,IAAI;aAC9E;SACF,CAAA;IACH,CAAC;IAED;;;;;;;;;OASG;IACK,KAAK,CAAC,IAAI,CAAC,MAA6B;QAC9C,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA;QAC9C,IAAI,CAAC,QAAQ,IAAI,CAAC,WAAW;YAAE,OAAO,SAAS,CAAA;QAC/C,IAAI,CAAC,CAAC,MAAM,QAAQ,CAAC,SAAS,EAAE,CAAC;YAAE,OAAO,SAAS,CAAA;QAEnD,MAAM,QAAQ,GAAa,EAAE,CAAA;QAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;gBAAE,SAAQ;YAEpC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC,CAAA;gBAC5D,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBAC7C,IAAI,IAAI,KAAK,SAAS;oBAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC7C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mDAAmD,EAAE,KAAK,CAAC,CAAA;YAC9E,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,SAAS,CAAA;QAE3C,OAAO,QAAQ,CACb,QAAQ,CAAC,MAAM,KAAK,CAAC;YACnB,CAAC,CAAE,QAAQ,CAAC,CAAC,CAAY;YACzB,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,aAAa,KAAK,GAAG,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CACtF,CAAA;IACH,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,SAAiB,EAAE,KAAsB;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;QACzC,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;QAExB,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB,EAAE,CAAC;YACvC,MAAM,OAAO,GACX,KAAK,CAAC,IACP,EAAE,OAAO,CAAA;YACV,MAAM,IAAI,GAAG,CAAC,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;iBAClC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;iBACxC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;iBAChC,IAAI,CAAC,EAAE,CAAC;iBACR,IAAI,EAAE,CAAA;YACT,IAAI,IAAI,KAAK,EAAE;gBAAE,KAAK,CAAC,IAAI,GAAG,IAAI,CAAA;QACpC,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC9B,MAAM,MAAM,GACV,KAAK,CAAC,IACP,EAAE,MAAM,CAAA;YACT,IAAI,MAAM,EAAE,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,6DACE,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,iBAC3B,EAAE,CACH,CAAA;gBACD,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBACvB,OAAO,IAAI,CAAA;YACb,CAAC;YAED,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC1D,CAAC;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED,oEAAoE;IACpE,OAAO;QACL,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YAAE,KAAK,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IACzE,CAAC;IAED,6DAA6D;IACrD,KAAK,CAAC,IAAI,CAChB,MAA6B,EAC7B,KAAiB;QAEjB,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,EAAE,CAAA;QAEtC,IAAI,KAAsB,CAAA;QAC1B,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QAC5E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2DAA2D,EAAE,KAAK,CAAC,CAAA;YACpF,OAAO,SAAS,CAAA;QAClB,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,IAAI,OAAO,CAAqB,CAAC,OAAO,EAAE,EAAE;gBAC1D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;oBAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4CAA4C,CAAC,CAAA;oBAC9D,OAAO,CAAC,SAAS,CAAC,CAAA;gBACpB,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAA;gBAEhD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE;oBAC1B,IAAI,EAAE,EAAE;oBACR,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;wBACf,YAAY,CAAC,KAAK,CAAC,CAAA;wBACnB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;wBAC9B,OAAO,CAAC,IAAI,CAAC,CAAA;oBACf,CAAC;iBACF,CAAC,CAAA;YACJ,CAAC,CAAC,CAAA;YAEF,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,GAAG,MAAM,CAAC,CAAC,CAAA;YAChE,OAAO,MAAM,OAAO,CAAA;QACtB,CAAC;gBAAS,CAAC;YACT,oEAAoE;YACpE,6CAA6C;YAC7C,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC7C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sDAAsD,EAAE,KAAK,CAAC,CAAA;YACjF,CAAC,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,qDAAqD;IAC7C,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,CAAC,YAAY;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAA;QACjE,IAAI,CAAC,OAAO,IAAI,CAAC,CAAA;QACjB,OAAO,aAAa,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAA;IAClD,CAAC;CACF"}

package/lib/media/intake.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Getting what the user sent into what the agent can read.
+ *
+ * Three kinds arrive and only two can go anywhere useful:
+ *
+ * - **Images** reach the model. The harness attachment seam accepts PNG,
+ *   JPEG, WebP and GIF, commits the bytes durably, and hands back a reference
+ *   the session log can carry — which is why a screenshot works at all.
+ * - **Text documents** — a log, a stack trace, a source file — have no seam,
+ *   but they do not need one: their content is text, so it is read and placed
+ *   in the prompt where the model already looks.
+ * - **Voice, audio and video** have neither. The harness explicitly defers
+ *   them, so the honest answer is to say so rather than to accept the message
+ *   and silently drop what it carried.
+ *
+ * Telegram sends an uncompressed image as a *document*, so kind is decided by
+ * media type rather than by which field it arrived in.
+ */
+import type { TelegramMessage } from '../telegram/types.js';
+import type { ImageCandidate } from './limits.js';
+/** One file a message carries, and what can be done with it. */
+export interface MediaItem {
+    /** Telegram's handle for the bytes. */
+    readonly fileId: string;
+    /** What this plugin can do with it. */
+    readonly kind: 'image' | 'text' | 'unsupported';
+    /** Media type, where Telegram supplied one. */
+    readonly mediaType?: string;
+    /** File name, where one was sent. */
+    readonly name?: string;
+    /** Size in bytes, where Telegram reported it. */
+    readonly size?: number;
+    /** For `unsupported`, what it was — so the refusal can name it. */
+    readonly describedAs?: string;
+    /**
+     * Every size this image is available in, for `image` items.
+     *
+     * Telegram renders a photo at several sizes and the largest routinely
+     * exceeds what the harness will store, so the choice is made against the
+     * seam's limits rather than here. A document arrives at one size only.
+     */
+    readonly candidates?: readonly ImageCandidate[];
+}
+/**
+ * Identify the media a message carries.
+ *
+ * @param message - the incoming Telegram message.
+ * @returns the item, or undefined for a message carrying none.
+ */
+export declare function describeMedia(message: TelegramMessage): MediaItem | undefined;
+/**
+ * Whether the harness can store this image.
+ *
+ * @param mediaType - the type Telegram reported.
+ */
+export declare function isStorableImage(mediaType: string | undefined): boolean;
+//# sourceMappingURL=intake.d.ts.map

package/lib/media/intake.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"intake.d.ts","sourceRoot":"","sources":["../../src/media/intake.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAC3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAyBjD,gEAAgE;AAChE,MAAM,WAAW,SAAS;IACxB,uCAAuC;IACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,uCAAuC;IACvC,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,GAAG,aAAa,CAAA;IAC/C,+CAA+C;IAC/C,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,qCAAqC;IACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,iDAAiD;IACjD,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,mEAAmE;IACnE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAA;IAC7B;;;;;;OAMG;IACH,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,cAAc,EAAE,CAAA;CAChD;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,eAAe,GAAG,SAAS,GAAG,SAAS,CAoD7E;AAeD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEtE"}