codsh-bundle 0.7.0 → 0.8.0

package/lib/index.js

@@ -6,7 +6,7 @@ import z from "@deepseek-ai/schemastery";
 import { installModelSelection } from "@deepseek-ai/dsh-agent";
 import { isUserInvocable } from "@deepseek-ai/dsh-skill";
 import { admitEncodedImages, isImageAdmissionError } from "@deepseek-ai/dsh-attachment";
-import { createUserMessage } from "@deepseek-ai/dsh-llm";
+import { BlockAssembler, ReasoningEffortId, createUserMessage } from "@deepseek-ai/dsh-llm";
 import { SessionId } from "@deepseek-ai/dsh-session";
 import { homedir, tmpdir } from "node:os";
 import stringWidth from "string-width";
@@ -4970,6 +4970,8 @@ var Spinner = class {
 //#region src/vision.ts
 /** How long one description may take before the paste falls back to file-only. */
 const VISION_TIMEOUT_MS = 3e4;
+/** The sibling route that lends sight to DeepSeek's text-only models. */
+const DEEPSEEK_VISION_MODEL = "deepseek-v4-flash-vision-exp";
 /**
 * The one instruction the sidecar gets.
 *
@@ -4978,6 +4980,8 @@ const VISION_TIMEOUT_MS = 3e4;
 * code is exactly the part the coding agent needed.
 */
 const VISION_PROMPT = "You are the eyes for a text-only coding agent. Describe this image precisely and completely. Transcribe ALL visible text, code, commands, error messages, numbers and labels verbatim. When it shows a UI, terminal, diagram or chart, describe its structure and layout so the agent can reason about it. Do not speculate beyond what is visible.";
+/** How the conversation model should consume a successful visual handoff. */
+const DESCRIPTION_HANDLING = "A vision model has already inspected this image. Treat the description as the image content available to you and answer the user directly. Do not say that you cannot see, read, access, or directly inspect the image, and do not mention the description, metadata, or vision handoff unless the user asks how image handling works.";
 /**
 * The sidecar from the environment, or undefined when none is configured.
 * @param env - the process environment.
@@ -5032,6 +5036,54 @@ async function describeImage(image, config, signal) {
 	return text;
 }
 /**
+* Ask DeepSeek's native vision sibling to describe one durable image.
+*
+* This is deliberately a one-shot auxiliary call: it receives no conversation
+* history, system prompt, or tools, and its answer is returned as plain text
+* for the selected text model's ordinary user turn.
+* @param image - the image reference already admitted to the durable store.
+* @param llm - the provider-neutral runtime serving the DeepSeek route.
+* @param options - optional caller cancellation and request attribution.
+* @returns the visible text emitted by the vision model.
+* @throws when the route cannot see, the provider fails, the call is aborted,
+*   or the response contains no visible text.
+*/
+async function describeImageWithLlm(image, llm, options = {}) {
+	const timeout = AbortSignal.timeout(VISION_TIMEOUT_MS);
+	const signal = options.signal === void 0 ? timeout : AbortSignal.any([options.signal, timeout]);
+	const prepared = await llm.prepareCall({
+		provider: "deepseek-official",
+		model: DEEPSEEK_VISION_MODEL,
+		reasoningEffort: ReasoningEffortId("off")
+	}, signal);
+	if (prepared.inputModalities?.includes("image") !== true) throw new Error(`${DEEPSEEK_VISION_MODEL} does not accept image input`);
+	const assembler = new BlockAssembler();
+	const message = createUserMessage({
+		content: [{
+			type: "image",
+			attachment: image
+		}, {
+			type: "text",
+			text: VISION_PROMPT
+		}],
+		source: {
+			kind: "plugin",
+			plugin: "coding-cli"
+		}
+	});
+	for await (const chunk of prepared.stream({
+		...prepared.config,
+		messages: [message],
+		signal,
+		...options.sessionId === void 0 ? {} : { sessionId: options.sessionId }
+	})) assembler.push(chunk);
+	const finish = assembler.finish;
+	if (finish.kind === "error" || finish.kind === "aborted") throw new Error(`vision model ${finish.kind}: ${finish.failure.message}`);
+	const text = assembler.blocks().filter((block) => block.type === "text").map((block) => block.text).join("\n").trim();
+	if (text === "") throw new Error("vision model answered without text");
+	return text;
+}
+/**
 * The upstream store's default admission limits, for use when no store is
 * mounted: the sidecar payload is bounded by the same line either way.
 */
@@ -5087,7 +5139,7 @@ async function savePastedImage(image) {
 */
 function pastedImageBlock(id, image, at) {
 	const size = at.width !== void 0 && at.height !== void 0 ? ` dimensions="${at.width}x${at.height}"` : "";
-	const body = at.description === void 0 ? "" : `\n<description>\n${at.description}\n</description>`;
+	const body = at.description === void 0 ? "" : `\n<description>\n${at.description}\n</description>\n<handling>\n${DESCRIPTION_HANDLING}\n</handling>`;
 	return `<pasted-image id="${id}" media="${image.mediaType}"${size} path="${at.path}">${body}\n</pasted-image>`;
 }
 /**
@@ -6163,17 +6215,19 @@ async function run(ctx, config, io) {
 	* modalities or a failed resolution get the text fallback, which degrades,
 	* where the block path would crash the turn.
 	*/
-	const routeAcceptsImages = async () => {
+	const routeImageCapability = async () => {
 		const current = selection.current;
-		if (current === void 0) return false;
+		if (current === void 0) return void 0;
 		const llm = ctx.get("llm");
-		if (llm === void 0) return false;
+		if (llm === void 0) return void 0;
 		try {
-			return (await llm.resolveModelInfo(current.provider, current.model)).inputModalities?.includes("image") === true;
+			const resolved = await llm.resolveModelInfo(current.provider, current.model);
+			return resolved.inputModalities === void 0 ? void 0 : resolved.inputModalities.includes("image");
 		} catch {
-			return false;
+			return;
 		}
 	};
+	const routeAcceptsImages = async () => await routeImageCapability() === true;
 	/**
 	* Resolve a /model argument to a selection.
 	* @param typed - a bare model id, or an explicit `provider/model`.
@@ -6845,20 +6899,20 @@ async function run(ctx, config, io) {
 	/**
 	* Turn pasted images into what this turn's message can carry.
 	*
-	* Three exits, decided by the route. An image-capable model gets the images
-	* as first-class blocks through the durable store — the runtime's own path.
-	* A text-only model gets each image saved as a file plus, when the vision
-	* sidecar is configured, a description standing in for sight; both ride the
-	* same message so they persist for `--resume`. A failure never loses the
-	* turn: it flashes, and the text still goes.
+	* An image-capable model gets first-class blocks through the durable store.
+	* A text-only model gets each image saved as a file plus a description from
+	* either the configured sidecar or DeepSeek's built-in Vision Exp bridge;
+	* both ride the same message so they persist for `--resume`. A recognition
+	* failure never loses the turn: it flashes, and the saved path still goes.
 	* @param images - the submission's pasted images, in token order.
 	* @returns blocks around the text, or undefined when there is nothing extra.
 	*/
-	const prepareImages = async (images) => {
+	const prepareImages = async (images, signal) => {
 		if (images.length === 0) return void 0;
 		const store = ctx.get("attachments");
 		const limits = store?.imageLimits ?? DEFAULT_IMAGE_LIMITS;
-		if (await routeAcceptsImages() && store !== void 0) try {
+		const currentAcceptsImages = await routeImageCapability();
+		if (currentAcceptsImages === true && store !== void 0) try {
 			return {
 				leading: (await admitEncodedImages(store, await Promise.all(images.map((pending) => fitWithinLimits(pending.image, limits))))).map((attachment) => ({
 					type: "image",
@@ -6872,6 +6926,9 @@ async function run(ctx, config, io) {
 			if (!isImageAdmissionError(error)) return void 0;
 		}
 		const vision = visionConfigFromEnv(process.env);
+		const current = selection.current;
+		const llm = ctx.get("llm");
+		const automaticVision = vision === void 0 && currentAcceptsImages === false && current?.provider === "deepseek-official" && llm !== void 0 && store !== void 0;
 		const trailing = [];
 		for (const pending of images) {
 			const at = { path: await savePastedImage(pending.image) };
@@ -6880,8 +6937,25 @@ async function run(ctx, config, io) {
 			if (vision !== void 0) {
 				prompt.setHint(theme.dim(`  ✻ describing image #${pending.id} with ${vision.model}…`));
 				try {
-					at.description = await describeImage(await fitWithinLimits(pending.image, limits), vision);
+					at.description = await describeImage(await fitWithinLimits(pending.image, limits), vision, signal);
 				} catch (error) {
+					if (signal?.aborted === true) throw error;
+					const reason = error instanceof Error ? error.message : String(error);
+					prompt.setFlash(theme.error(truncate(`  image #${pending.id}: description failed (${reason}) — attached as file only`, io.console.columns)));
+				} finally {
+					prompt.setHint(void 0);
+				}
+			} else if (automaticVision) {
+				prompt.setHint(theme.dim(`  ✻ describing image #${pending.id} with ${DEEPSEEK_VISION_MODEL}…`));
+				try {
+					const [attachment] = await admitEncodedImages(store, [await fitWithinLimits(pending.image, limits)]);
+					if (attachment === void 0) throw new Error("vision image was not admitted");
+					at.description = await describeImageWithLlm(attachment, llm, {
+						...signal === void 0 ? {} : { signal },
+						sessionId: live.agent.session.id
+					});
+				} catch (error) {
+					if (signal?.aborted === true) throw error;
 					const reason = error instanceof Error ? error.message : String(error);
 					prompt.setFlash(theme.error(truncate(`  image #${pending.id}: description failed (${reason}) — attached as file only`, io.console.columns)));
 				} finally {
@@ -6899,7 +6973,18 @@ async function run(ctx, config, io) {
 		};
 	};
 	const answer = async (text, source, images = []) => {
-		const extra = await prepareImages(images);
+		const preparing = images.length === 0 ? void 0 : new AbortController();
+		if (preparing !== void 0) running = preparing;
+		let extra;
+		try {
+			extra = await prepareImages(images, preparing?.signal);
+			if (preparing?.signal.aborted === true) return;
+		} catch (error) {
+			if (preparing?.signal.aborted === true) return;
+			throw error;
+		} finally {
+			if (running === preparing) running = void 0;
+		}
 		const before = totalTokens(facts(branch).usage) ?? 0;
 		turnBaseTokens = before;
 		const started = performance.now();

package/lib/types/vision.d.ts

@@ -4,14 +4,16 @@
  * DeepSeek Vision routes receive first-class image blocks before this module
  * is involved. For text-only routes such as Flash and Pro, codsh gives an
  * image two honest lives: it is always saved to a stable file the agent's
- * tools can touch — inspect, commit, embed. And when a vision sidecar is
- * configured (`CODSH_VISION_*`: any OpenAI-compatible multimodal endpoint),
- * the image is also described into text the model can actually read: everything in it
- * transcribed, structure narrated. Both ride the same message the person
+ * tools can touch — inspect, commit, embed. An explicit `CODSH_VISION_*`
+ * sidecar, or DeepSeek's built-in Vision Exp bridge when no sidecar is set,
+ * can also describe it into text the model can actually read: everything in
+ * it transcribed, structure narrated. Both ride the same message the person
  * sent, so they persist in durable history and survive `--resume`.
  * @module codsh-bundle/src/vision
  */
-import type { EncodedImageAttachment, ImageAttachmentLimits } from '@deepseek-ai/dsh-attachment/types';
+import type { EncodedImageAttachment, ImageAttachmentLimits, ImageAttachmentRef } from '@deepseek-ai/dsh-attachment/types';
+import type { GenerateOptions, LlmCallConfig, ModelModality, StreamChunk } from '@deepseek-ai/dsh-llm';
+import type { SessionId } from '@deepseek-ai/dsh-session';
 /** A vision sidecar: an OpenAI-compatible endpoint that can see. */
 export interface VisionConfig {
     /** The API base, e.g. `https://open.bigmodel.cn/api/paas/v4`. */
@@ -21,6 +23,16 @@ export interface VisionConfig {
     /** The multimodal model to ask. */
     model: string;
 }
+/** The sibling route that lends sight to DeepSeek's text-only models. */
+export declare const DEEPSEEK_VISION_MODEL = "deepseek-v4-flash-vision-exp";
+/** The provider-owned preparation seam needed for one auxiliary vision call. */
+export interface VisionLlm {
+    prepareCall(config: LlmCallConfig, signal?: AbortSignal): Promise<{
+        readonly config: LlmCallConfig;
+        readonly inputModalities?: readonly ModelModality[];
+        stream(options: GenerateOptions): AsyncIterable<StreamChunk>;
+    }>;
+}
 /**
  * The sidecar from the environment, or undefined when none is configured.
  * @param env - the process environment.
@@ -36,6 +48,23 @@ export declare function visionConfigFromEnv(env: Record<string, string | undefin
  * @throws on timeout, a non-2xx answer, or an answer with no text.
  */
 export declare function describeImage(image: EncodedImageAttachment, config: VisionConfig, signal?: AbortSignal): Promise<string>;
+/**
+ * Ask DeepSeek's native vision sibling to describe one durable image.
+ *
+ * This is deliberately a one-shot auxiliary call: it receives no conversation
+ * history, system prompt, or tools, and its answer is returned as plain text
+ * for the selected text model's ordinary user turn.
+ * @param image - the image reference already admitted to the durable store.
+ * @param llm - the provider-neutral runtime serving the DeepSeek route.
+ * @param options - optional caller cancellation and request attribution.
+ * @returns the visible text emitted by the vision model.
+ * @throws when the route cannot see, the provider fails, the call is aborted,
+ *   or the response contains no visible text.
+ */
+export declare function describeImageWithLlm(image: ImageAttachmentRef, llm: VisionLlm, options?: {
+    signal?: AbortSignal;
+    sessionId?: SessionId;
+}): Promise<string>;
 /**
  * The upstream store's default admission limits, for use when no store is
  * mounted: the sidecar payload is bounded by the same line either way.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "codsh-bundle",
   "description": "The codsh runtime: the interactive TTY surface and code-cli agent preset, installed into a dsh profile. Users install codsh-cli (the launcher) — this package is what it registers.",
-  "version": "0.7.0",
+  "version": "0.8.0",
   "type": "module",
   "license": "MIT",
   "main": "lib/index.js",