@charzhu/openjaw-agent 0.2.8 → 0.2.9

package/dist/main.js

@@ -2393,6 +2393,9 @@ var init_copilot = __esm({
       }
       async chat(options) {
         const modelInfo = await this.resolveModelInfo(this.config.model);
+        if (hasImageContent(options.messages) && modelInfo?.supportsVision === false) {
+          throw new Error(`model ${this.config.model} does not support image input; switch to a vision-capable model before submitting an image`);
+        }
         if (this.shouldRouteToResponses(modelInfo)) {
           return this.chatResponses(options, modelInfo);
         }
@@ -5091,7 +5094,9 @@ ${summary}
           if (imageData) {
             const contentBlocks = [
               { type: "image", source: { type: "base64", media_type: imageData.mimeType, data: imageData.base64 } },
-              { type: "text", text: taskText }
+              { type: "text", text: `${taskText}
+
+[ATTACHED IMAGE]: The image is attached as visual input in this message. Inspect it directly; do not search the filesystem for an uploaded image path unless the user explicitly asks for a file.` }
             ];
             this.conversationHistory.push({ role: "user", content: contentBlocks });
           } else {
@@ -48661,14 +48666,26 @@ ${helpMessage}` : field.label;
     bus.log("info", `session.steer ${String(params.text ?? "").slice(0, 200)}`);
     return { status: "queued", text: String(params.text ?? "") };
   });
+  let pendingImageSeq = 0;
   let pendingImage = null;
+  const nextImageAttachmentId = /* @__PURE__ */ __name(() => `img-${Date.now().toString(36)}-${++pendingImageSeq}`, "nextImageAttachmentId");
   bus.registerRpc("prompt.submit", async (params) => {
     const text = String(params.text ?? "");
     if (!text) return { ok: false };
     const systemPromptArr = await systemPromptFn();
     const systemPrompt = systemPromptArr.join("\n\n");
-    const imageData = pendingImage ? { base64: pendingImage.base64, mimeType: pendingImage.mimeType } : void 0;
-    pendingImage = null;
+    const requestedImageId = String(params.image_attachment_id ?? "");
+    const imageForSubmit = pendingImage && (!requestedImageId || pendingImage.attachmentId === requestedImageId) ? pendingImage : null;
+    if (imageForSubmit) {
+      const modelInfo = agentLoop.getActiveModelMetadata() ?? await agentLoop.listModels().then(() => agentLoop.getActiveModelMetadata()).catch(() => void 0);
+      if (modelInfo?.supportsVision === false) {
+        throw new Error(`model ${agentLoop.model} does not support image input; switch to a vision-capable model before submitting an image`);
+      }
+    }
+    const imageData = imageForSubmit ? { base64: imageForSubmit.base64, mimeType: imageForSubmit.mimeType } : void 0;
+    if (imageForSubmit) {
+      pendingImage = null;
+    }
     currentRun = { abort: /* @__PURE__ */ __name(() => agentLoop.abort(), "abort") };
     void streamAgentRun({ agentLoop, bus, systemPrompt, text, imageData }).finally(() => {
       currentRun = null;
@@ -48744,6 +48761,32 @@ ${helpMessage}` : field.label;
     });
   });
   bus.registerRpc("clipboard.paste", async () => {
+    try {
+      if (clipboardHasImage()) {
+        const img = readClipboardImage();
+        if (img) {
+          const attachmentId = nextImageAttachmentId();
+          pendingImage = {
+            attachmentId,
+            base64: img.base64,
+            mimeType: img.mimeType,
+            name: "clipboard.png"
+          };
+          const byteLength = Math.ceil(img.base64.length * 3 / 4);
+          const tokenEstimate = Math.max(1, Math.ceil(byteLength / 750));
+          return {
+            attached: true,
+            attachment_id: attachmentId,
+            count: 1,
+            height: img.height,
+            name: "clipboard.png",
+            token_estimate: tokenEstimate,
+            width: img.width
+          };
+        }
+      }
+    } catch {
+    }
     try {
       const text = await readClipboardText();
       return { attached: false, message: text ?? "" };
@@ -49368,13 +49411,16 @@ ${helpMessage}` : field.label;
     }
     const ext = extname4(path3).toLowerCase().replace(/^\./, "");
     const mimeType = ext === "png" ? "image/png" : ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "gif" ? "image/gif" : ext === "webp" ? "image/webp" : "image/png";
+    const attachmentId = nextImageAttachmentId();
     pendingImage = {
+      attachmentId,
       base64: buffer.toString("base64"),
       mimeType,
       name: basename3(path3)
     };
     const tokenEstimate = Math.max(1, Math.ceil(buffer.byteLength / 750));
     return {
+      attachment_id: attachmentId,
       height: 0,
       name: basename3(path3),
       remainder,
@@ -49382,6 +49428,13 @@ ${helpMessage}` : field.label;
       width: 0
     };
   });
+  bus.registerRpc("image.clear", (params) => {
+    const attachmentId = String(params.attachment_id ?? "");
+    if (!attachmentId || pendingImage?.attachmentId === attachmentId) {
+      pendingImage = null;
+    }
+    return { ok: true };
+  });
   bus.registerRpc("process.stop", () => {
     const wasRunning = agentLoop.isRunning;
     agentLoop.abort();
@@ -49581,6 +49634,7 @@ var init_rpcHandlers = __esm({
     init_uiStore();
     init_catalog();
     init_clipboard();
+    init_clipboard_image();
     init_models_static();
     init_providers();
     init_types();
@@ -52842,6 +52896,7 @@ function useComposerState({
 }) {
   const [input, setInput] = useState12("");
   const [inputBuf, setInputBuf] = useState12([]);
+  const [attachedImage, setAttachedImage] = useState12(null);
   const [pasteSnips, setPasteSnips] = useState12([]);
   const isBlocked = useStore($isBlocked);
   const { querier } = use_stdin_default();
@@ -52859,14 +52914,25 @@ function useComposerState({
   } = useQueue();
   const { historyRef, historyIdx, setHistoryIdx, historyDraftRef, pushHistory } = useInputHistory();
   const { completions, compIdx, setCompIdx, compReplace } = useCompletion(input, isBlocked, gw);
-  const clearIn = useCallback7(() => {
+  const clearAttachedImage = useCallback7(() => {
+    const attachmentId = attachedImage?.attachment_id;
+    setAttachedImage(null);
+    if (attachmentId) {
+      void gw.request("image.clear", { attachment_id: attachmentId }).catch(() => {
+      });
+    }
+  }, [attachedImage?.attachment_id, gw]);
+  const clearIn = useCallback7((options = {}) => {
     setInput("");
     setInputBuf([]);
     setPasteSnips([]);
+    if (!options.keepAttachedImage) {
+      clearAttachedImage();
+    }
     setQueueEdit(null);
     setHistoryIdx(null);
     historyDraftRef.current = "";
-  }, [historyDraftRef, setQueueEdit, setHistoryIdx]);
+  }, [clearAttachedImage, historyDraftRef, setQueueEdit, setHistoryIdx]);
   const handleResolvedPaste = useCallback7(
     async ({
       bracketed,
@@ -52889,6 +52955,13 @@ function useComposerState({
             session_id: sid
           });
           if (attached?.name) {
+            setAttachedImage({
+              attachment_id: attached.attachment_id,
+              height: attached.height,
+              name: attached.name,
+              token_estimate: attached.token_estimate,
+              width: attached.width
+            });
             onImageAttached?.(attached);
             const remainder = attached.remainder?.trim() ?? "";
             if (!remainder) {
@@ -52941,7 +53014,7 @@ function useComposerState({
     }) => {
       if (hotkey) {
         const preferOsc52 = isRemoteShellSession(process.env);
-        const readPreferredText = preferOsc52 ? readOsc52Clipboard(querier).then(async (osc52Text) => {
+        const readPreferredText = /* @__PURE__ */ __name(() => preferOsc52 ? readOsc52Clipboard(querier).then(async (osc52Text) => {
           if (isUsableClipboardText(osc52Text)) {
             return osc52Text;
           }
@@ -52951,8 +53024,12 @@ function useComposerState({
             return clipText;
           }
           return readOsc52Clipboard(querier);
-        });
-        return readPreferredText.then(async (preferredText) => {
+        }), "readPreferredText");
+        return Promise.resolve(onClipboardPaste(true)).then(async (imageAttached) => {
+          if (imageAttached) {
+            return null;
+          }
+          const preferredText = await readPreferredText();
           if (isUsableClipboardText(preferredText)) {
             return handleResolvedPaste({ bracketed: false, cursor, text: preferredText, value });
           }
@@ -52990,6 +53067,7 @@ function useComposerState({
   }, [input, inputBuf, submitRef]);
   const actions = useMemo6(
     () => ({
+      clearAttachedImage,
       clearIn,
       dequeue,
       enqueue,
@@ -53000,6 +53078,7 @@ function useComposerState({
       replaceQueue: replaceQ,
       setCompIdx,
       setHistoryIdx,
+      setAttachedImage,
       setInput,
       setInputBuf,
       setPasteSnips,
@@ -53007,6 +53086,7 @@ function useComposerState({
       syncQueue
     }),
     [
+      clearAttachedImage,
       clearIn,
       dequeue,
       enqueue,
@@ -53033,6 +53113,7 @@ function useComposerState({
   );
   const state = useMemo6(
     () => ({
+      attachedImage,
       compIdx,
       compReplace,
       completions,
@@ -53043,7 +53124,7 @@ function useComposerState({
       queueEditIdx,
       queuedDisplay
     }),
-    [compIdx, compReplace, completions, historyIdx, input, inputBuf, pasteSnips, queueEditIdx, queuedDisplay]
+    [attachedImage, compIdx, compReplace, completions, historyIdx, input, inputBuf, pasteSnips, queueEditIdx, queuedDisplay]
   );
   return {
     actions,
@@ -53632,6 +53713,9 @@ function useInputHandlers(ctx) {
     if (key.escape && terminal.hasSelection) {
       return clearSelection2();
     }
+    if (key.escape && cState.attachedImage) {
+      return cActions.clearAttachedImage();
+    }
     if (key.escape && live.focusedPane === "transcript") {
       patchUiState({ focusedPane: "composer" });
       return;
@@ -53680,7 +53764,7 @@ function useInputHandlers(ctx) {
           sys: actions.sys
         });
       }
-      if (cState.input || cState.inputBuf.length) {
+      if (cState.input || cState.inputBuf.length || cState.attachedImage) {
         return cActions.clearIn();
       }
       return actions.die();
@@ -53870,6 +53954,7 @@ function useSessionLifecycle(opts) {
     setHistoryItems([]);
     setLastUserMsg("");
     setStickyPrompt("");
+    composerActions.clearAttachedImage();
     composerActions.setPasteSnips([]);
     evictInkCaches("half");
   }, [composerActions, setHistoryItems, setLastUserMsg, setStickyPrompt, setVoiceProcessing, setVoiceRecording]);
@@ -53882,6 +53967,7 @@ function useSessionLifecycle(opts) {
       setHistoryItems(info ? [introMsg(info)] : []);
       setStickyPrompt("");
       setLastUserMsg("");
+      composerActions.clearAttachedImage();
       composerActions.setPasteSnips([]);
       patchTurnState({ activity: [] });
       patchUiState({ info, usage: usageFrom(info) });
@@ -54112,6 +54198,7 @@ function useSubmission(opts) {
       const expand = expandSnips(composerState.pasteSnips);
       const startSubmit = /* @__PURE__ */ __name((displayText, submitText, showUserMessage2 = true) => {
         const sid2 = getUiState().sid;
+        const imageAttachment = composerState.attachedImage;
         if (!sid2) {
           return sys("session not ready yet");
         }
@@ -54124,7 +54211,20 @@ function useSubmission(opts) {
         patchUiState({ busy: true, status: "running\u2026" });
         turnController.bufRef = "";
         turnController.interrupted = false;
-        gw.request("prompt.submit", { session_id: sid2, text: submitText }).catch((e) => {
+        const submitParams = { session_id: sid2, text: submitText };
+        if (imageAttachment?.attachment_id) {
+          submitParams.image_attachment_id = imageAttachment.attachment_id;
+        }
+        if (imageAttachment) {
+          composerActions.setAttachedImage(null);
+        }
+        gw.request("prompt.submit", submitParams).catch((e) => {
+          if (imageAttachment) {
+            composerActions.setAttachedImage(imageAttachment);
+            sys(`error: ${e.message}`);
+            patchUiState({ busy: false, status: "ready" });
+            return;
+          }
           if (isSessionBusyError(e)) {
             composerActions.enqueue(submitText);
             patchUiState({ busy: true, status: "queued for next turn" });
@@ -54150,7 +54250,7 @@ function useSubmission(opts) {
         startSubmit(r.text || text, expand(r.text || text), showUserMessage);
       }).catch(() => startSubmit(text, expand(text), showUserMessage));
     },
-    [appendMessage, composerActions, composerState.pasteSnips, gw, maybeGoodVibes, setLastUserMsg, sys]
+    [appendMessage, composerActions, composerState.attachedImage, composerState.pasteSnips, gw, maybeGoodVibes, setLastUserMsg, sys]
   );
   const shellExec = useCallback9(
     (cmd) => {
@@ -54213,6 +54313,16 @@ function useSubmission(opts) {
         }
         sys(note);
       }, "fallback");
+      if (composerState.attachedImage) {
+        if (live.sid) {
+          turnController.interruptTurn({ appendMessage, gw, sid: live.sid, sys });
+        }
+        if (hasInterpolation(full)) {
+          patchUiState({ busy: true });
+          return interpolate(full, send);
+        }
+        return send(full);
+      }
       if (mode === "queue") {
         return composerActions.enqueue(full);
       }
@@ -54234,7 +54344,7 @@ function useSubmission(opts) {
       }
       send(full);
     },
-    [appendMessage, composerActions, composerRefs, gw, interpolate, send, sys]
+    [appendMessage, composerActions, composerRefs, composerState.attachedImage, gw, interpolate, send, sys]
   );
   const dispatchSubmission = useCallback9(
     (full) => {
@@ -54259,8 +54369,8 @@ function useSubmission(opts) {
         return;
       }
       const editIdx = composerRefs.queueEditRef.current;
-      composerActions.clearIn();
       if (editIdx !== null) {
+        composerActions.clearIn();
         composerActions.replaceQueue(editIdx, full);
         const picked = composerRefs.queueRef.current.splice(editIdx, 1)[0];
         composerActions.syncQueue();
@@ -54277,6 +54387,7 @@ function useSubmission(opts) {
         }
         return sendQueued(picked);
       }
+      composerActions.clearIn({ keepAttachedImage: !!composerState.attachedImage });
       composerActions.pushHistory(full);
       if (getUiState().busy) {
         return handleBusyInput(full);
@@ -54287,7 +54398,7 @@ function useSubmission(opts) {
       }
       send(full);
     },
-    [appendMessage, composerActions, composerRefs, handleBusyInput, interpolate, send, sendQueued, shellExec, slashRef]
+    [appendMessage, composerActions, composerRefs, composerState.attachedImage, handleBusyInput, interpolate, send, sendQueued, shellExec, slashRef]
   );
   const submit = useCallback9(
     (value) => {
@@ -54405,8 +54516,7 @@ function useMainApp(gw) {
   const scrollRef = useRef13(null);
   const onEventRef = useRef13(() => {
   });
-  const clipboardPasteRef = useRef13(() => {
-  });
+  const clipboardPasteRef = useRef13(() => false);
   const submitRef = useRef13(() => {
   });
   const terminalHintsShownRef = useRef13(/* @__PURE__ */ new Set());
@@ -54459,9 +54569,7 @@ function useMainApp(gw) {
   const composer = useComposerState({
     gw,
     onClipboardPaste: /* @__PURE__ */ __name((quiet) => clipboardPasteRef.current(quiet), "onClipboardPaste"),
-    onImageAttached: /* @__PURE__ */ __name((info) => {
-      sys(attachedImageNotice(info));
-    }, "onImageAttached"),
+    onImageAttached: /* @__PURE__ */ __name(() => patchUiState({ status: "image attached" }), "onImageAttached"),
     submitRef
   });
   const { actions: composerActions, refs: composerRefs, state: composerState } = composer;
@@ -54669,17 +54777,25 @@ function useMainApp(gw) {
   const paste2 = useCallback10(
     (quiet = false) => rpc("clipboard.paste", { session_id: getUiState().sid }).then((r) => {
       if (!r) {
-        return;
+        return false;
       }
       if (r.attached) {
-        const meta = imageTokenMeta(r);
-        return sys(`\u{1F4CE} Image #${r.count} attached from clipboard${meta ? ` \xB7 ${meta}` : ""}`);
+        composerActions.setAttachedImage({
+          attachment_id: r.attachment_id,
+          height: r.height,
+          name: r.name ?? `Image #${r.count ?? 1}`,
+          token_estimate: r.token_estimate,
+          width: r.width
+        });
+        patchUiState({ status: "image attached" });
+        return true;
       }
       if (!quiet) {
         sys(r.message || "No image found in clipboard");
       }
+      return false;
     }),
-    [rpc, sys]
+    [composerActions, rpc, sys]
   );
   clipboardPasteRef.current = paste2;
   const { dispatchSubmission, send, sendQueued, submit } = useSubmission({
@@ -54919,6 +55035,7 @@ function useMainApp(gw) {
       cols,
       compIdx: composerState.compIdx,
       completions: composerState.completions,
+      attachedImage: composerState.attachedImage,
       empty,
       handleTextPaste: composerActions.handleTextPaste,
       input: composerState.input,
@@ -54975,7 +55092,6 @@ var init_useMainApp = __esm({
     init_env();
     init_limits();
     init_details();
-    init_messages();
     init_paths();
     init_useGitBranch();
     init_useVirtualHistory();
@@ -62280,6 +62396,7 @@ var init_appLayout = __esm({
     init_env();
     init_limits();
     init_placeholders();
+    init_messages();
     init_inputMetrics();
     init_perfPane();
     init_agentsOverlay();
@@ -62399,6 +62516,7 @@ var init_appLayout = __esm({
       const inputColumns = stableComposerColumns(composer.cols, promptWidth);
       const inputHeight = inputVisualHeight(composer.input, inputColumns);
       const inputMouseRef = useRef19(null);
+      const attachedImageMeta = composer.attachedImage ? imageTokenMeta(composer.attachedImage) : "";
       const captureInputDrag = /* @__PURE__ */ __name((e) => {
         if (e.button !== 0) {
           return;
@@ -62469,6 +62587,12 @@ var init_appLayout = __esm({
               ),
               composer.input === "?" && !composer.inputBuf.length && /* @__PURE__ */ jsx41(HelpHint, { t: ui.theme }),
               !isBlocked && /* @__PURE__ */ jsxs28(Fragment11, { children: [
+                composer.attachedImage && /* @__PURE__ */ jsx41(Box_default, { width: Math.max(1, composer.cols - 2), children: /* @__PURE__ */ jsxs28(Text9, { color: ui.theme.color.systemNote, wrap: "truncate-end", children: [
+                  "\u{1F4CE} ",
+                  composer.attachedImage.name,
+                  attachedImageMeta ? ` \xB7 ${attachedImageMeta}` : "",
+                  /* @__PURE__ */ jsx41(Text9, { color: ui.theme.color.muted, children: " \xB7 Esc to remove" })
+                ] }) }),
                 composer.inputBuf.map((line, i) => /* @__PURE__ */ jsxs28(Box_default, { children: [
                   /* @__PURE__ */ jsx41(Box_default, { width: promptWidth, children: i === 0 ? /* @__PURE__ */ jsx41(PromptPrefix, { color: ui.theme.color.muted, promptText, width: promptWidth }) : /* @__PURE__ */ jsx41(Text9, { color: ui.theme.color.muted, children: promptBlank }) }),
                   /* @__PURE__ */ jsx41(Text9, { color: ui.theme.color.composeText, children: line || " " })