@alia.onl/sdk 22.6.0 → 22.7.0

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @alia.onl/sdk
 
+## 22.7.0
+
+- **Image results for awaited host tools.** An awaited tool's `execute` may answer
+  `result: { type: "image", mediaType, data, text? }` (`AliaHostImageResult`: `image/jpeg`,
+  `image/png` or `image/webp`, `data` in standard base64 with no `data:` prefix, an optional
+  caption of at most `HOST_IMAGE_TEXT_MAX_CHARS`, 4,000). The model sees the image itself in that
+  turn; the stored conversation and the `alia.tool_result` frame keep
+  `{ type, mediaType, text?, omitted: true }` instead. The whole result still counts toward
+  `HOST_RESULT_MAX_CHARS` (100,000 characters as JSON), so size a screenshot to fit.
+- A result that claims `type: "image"` but is malformed is answered `failed` with the reason
+  (`hostImageResultProblem`), instead of being sent for the API to refuse while the model waits.
+- Needs the Alia API that ships with this release (`docs/agent-embedding.md` §2); an older API
+  passes an image result to the model as JSON text.
+
 ## 22.6.0
 
 - **`@alia.onl/sdk/dictation`: dictation's request on its own.** `requestTranscription`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@alia.onl/sdk",
-  "version": "22.6.0",
+  "version": "22.7.0",
   "description": "Alia AI chat SDK — the conversation runtime, chat UI, voice calls and dictation for Oxy apps",
   "main": "src/index.ts",
   "react-native": "src/index.ts",

package/src/runtime/host-tools.ts

@@ -9,7 +9,8 @@
  *
  * A tool declared with `awaitsResult: true` is read in the SAME turn instead ("this tab's text",
  * "the open tabs"): the server waits, and this client answers it at once on the turn's
- * `host-results` route with the outcome and the `result` its `execute` returned.
+ * `host-results` route with the outcome and the `result` its `execute` returned — text, JSON or
+ * an image ({@link AliaHostImageResult}) the model sees as one.
  *
  * Three rules hold here, in a queue that knows nothing of React:
  *  - an action is performed at most once, whatever the stream repeats;
@@ -45,11 +46,61 @@ export interface AliaHostOutcome {
   detail?: string;
   /**
    * What an awaited tool read, for the model: a string or any JSON value, at most
-   * {@link HOST_RESULT_MAX_CHARS} characters as JSON. Ignored for a tool that does not await.
+   * {@link HOST_RESULT_MAX_CHARS} characters as JSON, or an {@link AliaHostImageResult} the model
+   * sees as an image. Ignored for a tool that does not await.
    */
   result?: unknown;
 }
 
+export const HOST_IMAGE_MEDIA_TYPES = ['image/jpeg', 'image/png', 'image/webp'] as const;
+export type AliaHostImageMediaType = (typeof HOST_IMAGE_MEDIA_TYPES)[number];
+
+/** The longest caption an image result may carry. */
+export const HOST_IMAGE_TEXT_MAX_CHARS = 4_000;
+
+/**
+ * An awaited tool's image (a tab's screenshot): the model sees the image itself in that turn, and
+ * the stored conversation keeps `{ type, mediaType, text?, omitted: true }` in its place.
+ * `data` is the image's bytes in standard base64, with no `data:` prefix; the whole result counts
+ * toward {@link HOST_RESULT_MAX_CHARS}, so size the image to fit (a JPEG of the viewport).
+ */
+export interface AliaHostImageResult {
+  type: 'image';
+  mediaType: AliaHostImageMediaType;
+  data: string;
+  /** A caption for the model (the tab's URL and title), at most {@link HOST_IMAGE_TEXT_MAX_CHARS}. */
+  text?: string;
+}
+
+const BASE64 = /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/;
+const IMAGE_KEYS = ['type', 'mediaType', 'data', 'text'];
+
+/**
+ * Why a result that claims to be an image (`type: "image"`) is not a valid one, or `null`. The
+ * server refuses such an answer, which would leave the model waiting until the action goes stale;
+ * the runtime answers `failed` with this reason instead.
+ */
+export function hostImageResultProblem(result: unknown): string | null {
+  if (typeof result !== 'object' || result === null || Array.isArray(result)) return null;
+  const image = result as Record<string, unknown>;
+  if (image.type !== 'image') return null;
+  const extra = Object.keys(image).find((key) => !IMAGE_KEYS.includes(key));
+  if (extra !== undefined) return `An image result has no "${extra}".`;
+  if (!HOST_IMAGE_MEDIA_TYPES.includes(image.mediaType as AliaHostImageMediaType)) {
+    return `An image result's mediaType is one of ${HOST_IMAGE_MEDIA_TYPES.join(', ')}.`;
+  }
+  if (typeof image.data !== 'string' || image.data === '' || !BASE64.test(image.data)) {
+    return "An image result's data is standard base64 with no data: prefix.";
+  }
+  if (
+    image.text !== undefined &&
+    (typeof image.text !== 'string' || image.text.length > HOST_IMAGE_TEXT_MAX_CHARS)
+  ) {
+    return `An image result's text is a string of at most ${HOST_IMAGE_TEXT_MAX_CHARS} characters.`;
+  }
+  return null;
+}
+
 /**
  * The host tools a surface declares (`hostTools` on the chat). They are the tools of the chat's
  * `app`, which a surface with host tools must name: the app is said once, on the chat, and is what
@@ -96,8 +147,10 @@ const DETAIL_LIMIT = 500;
 /** The server's bound on an awaited action's `result`, as JSON. */
 export const HOST_RESULT_MAX_CHARS = 100_000;
 
-function resultOf(result: unknown): { result?: unknown } | 'too_large' {
+function resultOf(result: unknown): { result?: unknown } | 'too_large' | { invalid: string } {
   if (result === undefined || result === null) return {};
+  const problem = hostImageResultProblem(result);
+  if (problem !== null) return { invalid: problem };
   let text: string | undefined;
   try {
     text = JSON.stringify(result);
@@ -158,7 +211,9 @@ export class HostOutcomeQueue {
             outcome: 'failed',
             detail: 'The result was too large to send.',
           })
-        : { ...record(action.toolCallId, outcome), ...result };
+        : 'invalid' in result
+          ? record(action.toolCallId, { outcome: 'failed', detail: result.invalid })
+          : { ...record(action.toolCallId, outcome), ...result };
     try {
       await send(answer);
     } catch (error) {

package/src/runtime/index.ts

@@ -33,9 +33,19 @@ export {
   useAgentProfile,
 } from './agents';
 export type { AliaAgentFeature, AliaAgentProfile, AliaAgentStarter, FeatureOffer } from './agents';
-export { HOST_OUTCOMES, HOST_RESULT_MAX_CHARS, HostOutcomeQueue, hostRequest } from './host-tools';
+export {
+  HOST_IMAGE_MEDIA_TYPES,
+  HOST_IMAGE_TEXT_MAX_CHARS,
+  HOST_OUTCOMES,
+  HOST_RESULT_MAX_CHARS,
+  HostOutcomeQueue,
+  hostImageResultProblem,
+  hostRequest,
+} from './host-tools';
 export type {
   AliaHostAction,
+  AliaHostImageMediaType,
+  AliaHostImageResult,
   AliaHostOutcome,
   AliaHostOutcomeKind,
   AliaHostToolDefinition,