@niscorp/signal 0.1.6 → 0.1.7

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @niscorp/signal
 
+## 0.1.7
+
+### Patch Changes
+
+- f4a3f04: `decide()` can ask about images: `images` maps a uuid to an `ImageSource`, and a string in `state` equal to a uuid points at that image.
+
+  On `typesafe` the images go as the request's `assets` with the `TypeSafe-Danger-Zone-Images: true` header, so Jev answers about them with calibrated probabilities. TypeSafe takes PNG or JPEG data; a `url` image or another `mediaType` fails with `E_VALIDATION_FAILED` before the request. On a chat provider each image follows the state in the emulation message, after a label with its uuid; a model whose registry row says `multimodal: false` is refused before the request. A key that is not a uuid is refused on every provider.
+
+  A request without `images`, or with `images: {}`, is unchanged: TypeSafe gets the same body and headers, and a chat provider the same single text message.
+
+  **What to change:** nothing. A caller that posts images to TypeSafe itself can pass them to `decide()` instead.
+
 ## 0.1.6
 
 ### Patch Changes

package/DESIGN.md

@@ -119,6 +119,10 @@ sides is two changes.
     to repair. The questions are the schema: the result type and the
     gate (`decide/gate.ts`) both derive from them, which is the package's
     one sentence applied to a model that answers instead of writing.
+    Images ride beside `state`, keyed by the uuid `state` uses for them —
+    TypeSafe's own wire structure — so the adapter translates fields,
+    refuses what TypeSafe's wire cannot carry, and never walks or
+    rewrites the caller's state.
 12. **Emulation is honest or absent.** On a chat adapter `decide()`
     runs the derived schema through `complete()` and returns picks with
     `calibrated: false` — the discriminant of `DecideResult`, so the

package/DOCS.md

@@ -543,6 +543,24 @@ On a chat provider `decide()` still works: the same questions run through struct
 if (signal.describe().kind === 'decisions') { /* sub-second */ }
 ```
 
+### Images
+
+`images` maps a uuid to an `ImageSource`, the same type `complete()` takes in content parts. A string in `state` equal to a uuid points at that image:
+
+```typescript
+const duck = randomUUID();
+const { decisions } = await jev.decide({
+  state: { instruction: 'Select all squares containing a duck', tiles: { tile1: duck } },
+  images: { [duck]: { type: 'base64', mediaType: 'image/png', data: pngBase64 } },
+  questions: { tile1: { type: 'noul', instructions: 'Does the picture in `tiles.tile1` show the requested object?' } },
+});
+```
+
+- **On `typesafe`** the images go as the request's `assets`, with the `TypeSafe-Danger-Zone-Images: true` header. TypeSafe takes PNG or JPEG data: a `url` image or another `mediaType` fails with `E_VALIDATION_FAILED` before the request. TypeSafe's image documentation gives a minimum of 280 pixels a side; signal does not check sizes.
+- **On a chat provider** each image follows the state in the message, after a label with its uuid. A model whose registry row says `multimodal: false` fails with `E_VALIDATION_FAILED` before the request; a model with no row is not checked. Emulation's correction retries send the images again.
+
+A key that is not a uuid fails with `E_VALIDATION_FAILED` on every provider. A request with no images, or `images: {}`, is sent exactly as one without the field.
+
 ### Abort
 
 ```typescript

package/dist/index.cjs

@@ -484,17 +484,46 @@ var jsonOrText = (text) => {
   }
 };
 var providerError = (message, raw) => new SignalError(`Provider error: ${message}`, ErrorCode.PROVIDER_ERROR, { raw });
+var FORMATS = { "image/png": "png", "image/jpeg": "jpeg" };
+var assetOf = (id, source) => {
+  if (source.type === "url") {
+    throw new SignalError(`Image ${id}: TypeSafe takes image data, not a URL`, ErrorCode.VALIDATION_FAILED, { image: id });
+  }
+  const format = FORMATS[source.mediaType];
+  if (format === void 0) {
+    throw new SignalError(
+      `Image ${id}: TypeSafe takes image/png or image/jpeg, not ${source.mediaType}`,
+      ErrorCode.VALIDATION_FAILED,
+      { image: id }
+    );
+  }
+  return { type: "image", format, data_base64: source.data };
+};
+var assetsOf = (images) => {
+  const entries = Object.entries(images ?? {});
+  return entries.length === 0 ? void 0 : Object.fromEntries(entries.map(([id, source]) => [id, assetOf(id, source)]));
+};
 var createSystemOneAdapter = async (config) => {
   const send = sendOf(config.client);
   const url = `${config.baseUrl.replace(/\/+$/, "")}/systemone`;
   const decide = async (request, options) => {
+    const assets = assetsOf(request.images);
     let status;
     let body;
     try {
       const response = await send(url, {
         method: "POST",
-        headers: { authorization: `Bearer ${config.apiKey}`, "content-type": "application/json" },
-        body: JSON.stringify({ model: request.model, state: request.state, questions: request.questions }),
+        headers: {
+          authorization: `Bearer ${config.apiKey}`,
+          "content-type": "application/json",
+          ...assets !== void 0 && { "TypeSafe-Danger-Zone-Images": "true" }
+        },
+        body: JSON.stringify({
+          model: request.model,
+          state: request.state,
+          questions: request.questions,
+          ...assets !== void 0 && { assets }
+        }),
         ...options?.signal !== void 0 && { signal: options.signal }
       });
       status = response.status;
@@ -550,6 +579,7 @@ var bareSchemaCorrection = (issues) => `Your output was invalid: ${issues}. Repl
 var decisionEmulationPrompt = (state) => `Answer every field of the OUTPUT SCHEMA about the STATE below. Each field's description is the question and lists everything you may answer with.
 STATE:
 ${state}`;
+var decisionImageLabel = (id) => `Image ${id}:`;
 var decisionQuestionDescription = (question) => {
   if (question.type === "choice") return `${question.instructions} Options: ${JSON.stringify(question.criteria)}`;
   if (question.type === "score") {
@@ -1340,6 +1370,35 @@ var assertEffort = (config, effort) => {
     ErrorCode.VALIDATION_FAILED
   );
 };
+var imageId = zod.z.uuid();
+var assertImageIds = (images) => {
+  for (const id of Object.keys(images ?? {})) {
+    if (!imageId.safeParse(id).success) {
+      throw new SignalError(`Image key "${id}" is not a uuid`, ErrorCode.VALIDATION_FAILED, { image: id });
+    }
+  }
+};
+var assertTakesImages = (config) => {
+  const resolved = modelRowOf(config);
+  const row = resolved?.row;
+  if (resolved === void 0 || row === void 0 || row.capabilities.multimodal) return;
+  throw new SignalError(
+    `${resolved.provider} ${resolved.model} takes no images (measured ${row.verified})`,
+    ErrorCode.VALIDATION_FAILED
+  );
+};
+var emulationContent = (state, images) => {
+  const prompt = decisionEmulationPrompt(state);
+  const entries = Object.entries(images ?? {});
+  if (entries.length === 0) return prompt;
+  return [
+    { type: "text", text: prompt },
+    ...entries.flatMap(([id, source]) => [
+      { type: "text", text: decisionImageLabel(id) },
+      { type: "image", source }
+    ])
+  ];
+};
 var wireIdsOf = (config) => {
   const source = typeof config.provider === "string" ? providerRegistry[config.provider] : config.provider;
   return source === void 0 || source.adapter === "systemone" ? [] : source.wire ?? [];
@@ -1517,7 +1576,12 @@ var createSignalFromConfig = (config) => {
   const stepStream = stepStreamAs("stepStream");
   const decideCalibrated = async (adapter, model, request, started) => {
     const response = await adapter.decide(
-      { model, state: request.state, questions: request.questions },
+      {
+        model,
+        state: request.state,
+        questions: request.questions,
+        ...request.images !== void 0 && { images: request.images }
+      },
       request.options?.signal !== void 0 ? { signal: request.options.signal } : void 0
     );
     const decisions = acceptCalibrated(request.questions, response.answers, response.raw);
@@ -1535,10 +1599,11 @@ var createSignalFromConfig = (config) => {
     };
   };
   const decideEmulated = async (model, request, started) => {
+    if (Object.keys(request.images ?? {}).length > 0) assertTakesImages(config);
     const state = typeof request.state === "string" ? request.state : JSON.stringify(request.state);
     const result = await runComplete(
       {
-        messages: [{ role: "user", content: decisionEmulationPrompt(state) }],
+        messages: [{ role: "user", content: emulationContent(state, request.images) }],
         schema: uncalibratedSchemaOf(request.questions),
         tools: void 0,
         retries: config.retries ?? 2,
@@ -1637,6 +1702,7 @@ var createSignalFromConfig = (config) => {
     }),
     decide: async (request) => {
       assertQuestions(request.questions);
+      assertImageIds(request.images);
       const started = Date.now();
       const adapter = await getAdapter();
       const model = resolveProvider(config).model;