@cubicecho/agent-core 2.10.0 → 2.11.0

package/README.md

@@ -202,6 +202,21 @@ mean every property required and `additionalProperties: false`; a looser schema
 `preselect` is its first user, answering `{ tools: [...] }`, and `preselection` still takes the
 bare array an older prompt produced.
 
+### Showing a side task an image
+
+`ask` and `askJson` take the user turn as a string or as content parts, because an image reaches an
+OpenAI-compatible server only as an `image_url` part beside the text. The parts are sent as given
+and nothing here checks that the model can see: whether a text-only model rejects the image or
+answers without it is up to the server, so pick a vision model.
+
+```ts
+const page: SideTaskInput = [
+  { type: "text", text: "Transcribe this page." },
+  { type: "image_url", image_url: { url: `data:image/png;base64,${png}` } },
+];
+const text = await ask(config, visionModel, "You are an OCR engine.", page);
+```
+
 ## Sizing a request before sending it
 
 `runTurn` will refuse a request that cannot fit rather than spending a round trip finding out:

package/dist/index.d.ts

@@ -22,7 +22,7 @@ export { resetAll } from "./reset.ts";
 export { backoffMs, ContextOverflow, compact, EndpointSilent, isModelLoading, isOverflow, isTransient, LOADING_POLL_MS, LOADING_TIMEOUT_MS, messageTokens, requestTokens, SMALLEST_LIKELY_WINDOW, sleep, } from "./retry.ts";
 export { type RunTurnOptions, runTurn } from "./run-turn.ts";
 export { isGrammarError, relaxTools, sanitizeTools } from "./schema-compat.ts";
-export { type AskJsonOptions, ask, askJson, clean, listLines, parseJson, resetHints, type SideTaskOptions, tryAsk, } from "./side-task.ts";
+export { type AskJsonOptions, ask, askJson, clean, listLines, parseJson, resetHints, type SideTaskInput, type SideTaskOptions, tryAsk, } from "./side-task.ts";
 export { CAPABILITY_SNAPSHOT_VERSION, type CapabilitySnapshot, type EndpointSnapshot, exportCapabilities, importCapabilities, type ModelSnapshot, } from "./snapshot.ts";
 export { type Produced, type StreamTurnOptions, streamTurn, type Turn, type TurnUsage, } from "./stream.ts";
 export { ALL_FENCES, DEFAULT_FENCES, type Fence, FenceSplitter, type FenceSplitterOptions, type Split, stripThinking, THINK_FENCE, } from "./thinking.ts";

package/dist/side-task.d.ts

@@ -1,3 +1,4 @@
+import OpenAI from "openai";
 import type { Endpoint } from "./config.ts";
 /** An (endpoint, model) pair as `noHints` holds it: `[endpointId, model]`, stringified. */
 export declare const hintKey: (endpoint: string, model: string) => string;
@@ -5,6 +6,18 @@ export declare const hintKey: (endpoint: string, model: string) => string;
 export declare const refusedHints: () => Set<string>;
 /** Test seam, alongside `resetClients` and `resetAll`: forget which models refused the hints. */
 export declare const resetHints: () => void;
+/**
+ * The input a side task applies its instruction to: text, or the content parts a vision model
+ * reads.
+ *
+ * A string is the ordinary case and stays the cheapest thing to write. The array is what an
+ * image needs, because a page, a screenshot or a photo reaches an OpenAI-compatible server only
+ * as an `image_url` part alongside the text — there is no other spelling for it, and a caller
+ * with one otherwise has to leave this module and build the request itself. Nothing here reads
+ * the parts: they are handed to the SDK as given, and whether a model that cannot see rejects
+ * the image or answers without it is the server's decision, not this module's.
+ */
+export type SideTaskInput = string | OpenAI.ChatCompletionContentPart[];
 /** What a side task may be given. All optional — one given none of them still runs. */
 export interface SideTaskOptions {
     /** Ceiling on the reply, default 512. These answers are meant to be short. */
@@ -33,10 +46,10 @@ export interface SideTaskOptions {
  * @param config Where to send it and how long to wait.
  * @param model The model to ask, usually smaller than the one running the work.
  * @param system The instruction.
- * @param user The input it applies to.
+ * @param user The input it applies to. Content parts where the model is being shown an image.
  * @param options Reply ceiling, temperature, cancellation, notices.
  */
-export declare function ask(config: Endpoint, model: string, system: string, user: string, options?: SideTaskOptions): Promise<string>;
+export declare function ask(config: Endpoint, model: string, system: string, user: SideTaskInput, options?: SideTaskOptions): Promise<string>;
 /** What `askJson` takes besides a side task's options. */
 export interface AskJsonOptions extends SideTaskOptions {
     /** What the schema is called in the request, `answer` by default. Letters, digits, `_` and `-`. */
@@ -62,11 +75,11 @@ export interface AskJsonOptions extends SideTaskOptions {
  * @param config Where to send it and how long to wait.
  * @param model The model to ask.
  * @param system The instruction. The schema is appended to it.
- * @param user The input it applies to.
+ * @param user The input it applies to. Content parts where the model is being shown an image.
  * @param schema The JSON Schema of the answer. Its root is held to an object, as a tool's is.
  * @param options A side task's options, plus the schema's `name` and whether it is `strict`.
  */
-export declare function askJson<T>(config: Endpoint, model: string, system: string, user: string, schema: Record<string, unknown>, { name, strict, ...options }?: AskJsonOptions): Promise<T | undefined>;
+export declare function askJson<T>(config: Endpoint, model: string, system: string, user: SideTaskInput, schema: Record<string, unknown>, { name, strict, ...options }?: AskJsonOptions): Promise<T | undefined>;
 /**
  * A side task is never worth failing the work it supports. Callers that can carry on without
  * an answer use this and get `undefined` instead of an exception.

package/dist/side-task.js

@@ -71,7 +71,7 @@ function rejectedTheRequest(error) {
  * @param config Where to send it and how long to wait.
  * @param model The model to ask, usually smaller than the one running the work.
  * @param system The instruction.
- * @param user The input it applies to.
+ * @param user The input it applies to. Content parts where the model is being shown an image.
  * @param options Reply ceiling, temperature, cancellation, notices.
  */
 export function ask(config, model, system, user, options = {}) {
@@ -168,7 +168,7 @@ async function complete(config, model, system, user, { maxTokens = 512, temperat
  * @param config Where to send it and how long to wait.
  * @param model The model to ask.
  * @param system The instruction. The schema is appended to it.
- * @param user The input it applies to.
+ * @param user The input it applies to. Content parts where the model is being shown an image.
  * @param schema The JSON Schema of the answer. Its root is held to an object, as a tool's is.
  * @param options A side task's options, plus the schema's `name` and whether it is `strict`.
  */

package/llms.txt

@@ -182,6 +182,7 @@ One-shot calls that support a run without being one: picking tools, naming a ses
 - `listLines` — A list-shaped reply, one item per line, cleaned of the bullets and quotes models decorate them with.
 - `parseJson` — Models are asked for JSON and often answer with prose around it, or a fenced block.
 - `resetHints` — Test seam, alongside `resetClients` and `resetAll`: forget which models refused the hints.
+- `SideTaskInput` (type) — The input a side task applies its instruction to: text, or the content parts a vision model reads.
 - `SideTaskOptions` (type) — What a side task may be given.
 - `tryAsk` — A side task is never worth failing the work it supports.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cubicecho/agent-core",
-  "version": "2.10.0",
+  "version": "2.11.0",
   "description": "The endpoint-agnostic half of an OpenAI-compatible agent loop: tool-schema compatibility, on-demand tool loading, one-shot side tasks, run events, and a pooled client.",
   "keywords": [
     "openai",