@lampdotdev/lamp 0.36.2-nightly.40 → 0.36.2-nightly.43

package/cli/lamps/web-starter/lamp.js

@@ -138,8 +138,20 @@
       delete asked[m.id];
       waiting({ status: m.status, body: m.body });
     };
-    bridge = { postMessage: function (m) {
-      return new Promise(function (resolve) { var id = ++askId; asked[id] = resolve; gate.postMessage(JSON.stringify({ id: id, path: m.path, body: m.body, timeout: m.timeout })); });
+    bridge = { postMessage: function (m, signal) {
+      return new Promise(function (resolve) {
+        var id = ++askId;
+        asked[id] = resolve;
+        gate.postMessage(JSON.stringify({ id: id, path: m.path, body: m.body, timeout: m.timeout }));
+        // A page that stops waiting tells the host, which can stop the answer (lamp dev does), and hears nothing after.
+        function stop() {
+          if (!asked[id]) return;
+          delete asked[id];
+          gate.postMessage(JSON.stringify({ id: id, cancel: true }));
+          resolve({ status: 499, body: '{}' });
+        }
+        if (signal) { if (signal.aborted) stop(); else signal.addEventListener('abort', stop); }
+      });
     } };
   }
   var preferred = null;
@@ -305,14 +317,14 @@
   // with a little more for the answer to come back.
   function AI_WAIT(seconds) { return (typeof seconds === 'number' && seconds > 0 ? Math.min(seconds, 600) : 600) * 1000 + 15000; }
 
-  function post(path, body, timeout, deadline) {
+  function post(path, body, timeout, deadline, signal) {
     timeout = timeout || 240000;
     var done;
     if (!bridge && shared()) {
       done = nearby(path, body, timeout, deadline);
     } else if (bridge) {
       // The TV preview on the Mac answers for Lamp itself.
-      done = bridge.postMessage({ path: path, body: JSON.stringify(body), timeout: timeout }).then(function (r) {
+      done = bridge.postMessage({ path: path, body: JSON.stringify(body), timeout: timeout }, signal).then(function (r) {
         var reply = {};
         try { reply = JSON.parse(r.body) || {}; } catch (e) {}
         return { status: r.status, reply: reply };
@@ -328,10 +340,15 @@
       });
     }
     return done.then(function (answer) {
+      // Stopped by the page: no failure of the AI's, so nothing is said about it.
+      if (signal && signal.aborted) throw stoppedFailure();
       if (answer.status === 200) return answer.reply;
       var message = answer.reply.error || 'The AI request failed.';
       if (window.console) console.error('LampAI ' + path + ' failed: ' + message);
-      throw new Failure(message, answer.reply.needs || null);
+      var failed = new Failure(message, answer.reply.needs || null);
+      // What was billed before it failed (a retry, a timeout), for a turn's `cost`.
+      if (typeof answer.reply.cost === 'number' && answer.reply.cost > 0) failed.cost = answer.reply.cost;
+      throw failed;
     });
   }
 
@@ -346,14 +363,14 @@
     });
   }
 
-  /** Downscales to at most 1568px on the long side (the most the model uses) and JPEG-encodes. Takes an <img>, a
-   *  <canvas>, or a picture's address (a data: URL or a file of the app's). */
-  function encode(images) {
+  /** Downscales to at most 1568px on the long side (the most the model uses), or `side`, and JPEG-encodes. Takes an
+   *  <img>, a <canvas>, or a picture's address (a data: URL or a file of the app's). */
+  function encode(images, side) {
     return Promise.all((images || []).map(function (image) {
       var ready = typeof image === 'string' ? loadImage(image) : Promise.resolve(image);
       return ready.then(function (source) {
         var w = source.naturalWidth || source.width, h = source.naturalHeight || source.height;
-        var scale = Math.min(1, 1568 / Math.max(w, h, 1));
+        var scale = Math.min(1, (side || 1568) / Math.max(w, h, 1));
         var canvas = document.createElement('canvas');
         canvas.width = Math.max(1, Math.round(w * scale));
         canvas.height = Math.max(1, Math.round(h * scale));
@@ -483,21 +500,44 @@
     score: function (question, levels) { return { type: 'score', instructions: question, criteria: levels }; }
   };
 
+  function stoppedFailure() { var f = new Failure('Stopped.'); f.stopped = true; return f; }
+
+  // A promise the page can stop waiting for with an AbortSignal: it fails at once with `stopped`, whatever comes after.
+  function stoppable(promise, signal) {
+    if (!signal) return promise;
+    return new Promise(function (resolve, reject) {
+      function stop() { reject(stoppedFailure()); }
+      // Settled either way, so a stopped ask's own failure is never left unhandled.
+      if (signal.aborted) { promise.then(null, function () {}); return stop(); }
+      signal.addEventListener('abort', stop);
+      promise.then(function (v) { signal.removeEventListener('abort', stop); if (!signal.aborted) resolve(v); },
+                   function (e) { signal.removeEventListener('abort', stop); if (!signal.aborted) reject(e); });
+    });
+  }
+
   function call(prompt, options) {
-    var o = options || {}, text = prompt, rules = [o.system || '', present()];
+    var o = options || {}, text = prompt, rules = [o.system || '', present()], started = Date.now(), cost = 0;
+    var schema = o.schema && typeof o.schema === 'object' ? o.schema : null, json = !!o.json || !!schema;
     if (o.data) { var f = fenced(prompt, o.data); text = f.prompt; rules.push(f.rules); }
-    return encode(o.images).then(function (images) {
-      var body = { prompt: text, system: rules.filter(function (r) { return r; }).join('\n\n'), json: !!o.json, images: images };
+    return stoppable(encode(o.images).then(function (images) {
+      // Stopped before or while its pictures were read: nothing is asked.
+      if (o.signal && o.signal.aborted) throw stoppedFailure();
+      var body = { prompt: text, system: rules.filter(function (r) { return r; }).join('\n\n'), json: json, images: images };
+      if (schema) body.schema = schema;
       if (o.model) body.model = o.model;
       // No limit of the page's own unless it asks for one: Lamp's guard ends only a call that stopped answering.
       if (typeof o.seconds === 'number' && o.seconds > 0) body.seconds = o.seconds;
-      return post('/ai', body, AI_WAIT(body.seconds));
+      return post('/ai', body, AI_WAIT(body.seconds), null, o.signal);
     }).then(function (reply) {
       if (typeof reply.text !== 'string') throw new Failure('The AI request failed.');
-      if (!o.json) return reply.text;
-      try { return JSON.parse(reply.text.replace(/^\s*```(?:json)?\s*|\s*```\s*$/g, '')); }
-      catch (e) { throw new Failure('The AI\'s answer wasn\'t in the expected format. Try again.'); }
-    });
+      cost = typeof reply.cost === 'number' && reply.cost > 0 ? reply.cost : 0;
+      var answer = reply.text;
+      if (json) {
+        try { answer = JSON.parse(reply.text.replace(/^\s*```(?:json)?\s*|\s*```\s*$/g, '')); }
+        catch (e) { throw new Failure('The AI\'s answer wasn\'t in the expected format. Try again.'); }
+      }
+      return o.details ? { answer: answer, seconds: Math.round((Date.now() - started) / 100) / 10, cost: cost } : answer;
+    }), o.signal);
   }
 
   function image(prompt, images, aspectRatio) {
@@ -538,6 +578,12 @@
   // What a turn's reads give back in all, which every later step carries again: kept under what a step may carry
   // (`MAX_BODY` in lamp's app-ai.mjs), with room for the request, the tools and the steps' own words.
   var ACT_TURN_RESULTS = 1200000;
+  // The pictures a step carries from its tools' results (`LampAI.picture`): the newest few, each at most this many
+  // pixels on its long side, together at most this many bytes, beside the text results under the step's limit
+  // (`MAX_PICTURES`, `MAX_BODY` in lamp's app-ai.mjs). Older ones the model has seen go as no longer shown.
+  var ACT_PICTURES = 6, ACT_PICTURE_SIDE = 1024, ACT_PICTURE_BYTES = 1600000;
+  /** A picture in a tool's result, for the AI to look at (`LampAI.picture`). */
+  function ActPicture(source) { this.source = source; }
   var ACT_STEP_SECONDS = 600;
   var actRan = {};      // turn id -> call key -> the result a write gave, while this copy of the page is open
   function actId() { return 't' + Date.now().toString(36) + Math.random().toString(36).slice(2, 8); }
@@ -596,8 +642,50 @@
     }
     var ran = actRan[id];
     var started = Date.now(), deadline = asked ? started + Math.max(1, asked) * 1000 : Infinity;
-    var out = { turn: id, reply: '', ask: false, choices: [], done: [], failed: [], skipped: [], reads: [], stopped: null, error: '' };
-    var steps = [], route = null, writes = 0, over = false, settle;
+    var out = { turn: id, reply: '', ask: false, choices: [], done: [], failed: [], skipped: [], reads: [], stopped: null, error: '', seconds: 0, cost: 0 };
+    var steps = [], route = null, writes = 0, over = false, settle, pictureCount = 0;
+    // A result with its pictures taken out: each one sent beside it, scaled down, and named in its place
+    // ("[picture 2]", numbered across the turn), so the model sees what the app drew in the middle of the turn.
+    function pictured(value) {
+      var found = [];
+      function walk(v, depth) {
+        if (v instanceof ActPicture) { found.push({ source: v.source, number: ++pictureCount }); return '[picture ' + pictureCount + ']'; }
+        if (!v || typeof v !== 'object' || depth > 8) return v;
+        if (isList(v)) return v.map(function (x) { return walk(x, depth + 1); });
+        var proto = Object.getPrototypeOf(v);
+        if (proto !== Object.prototype && proto !== null) return v;
+        var copy = {};
+        for (var k in v) if (Object.prototype.hasOwnProperty.call(v, k)) copy[k] = walk(v[k], depth + 1);
+        return copy;
+      }
+      var kept = walk(value, 0);
+      if (!found.length) return Promise.resolve({ value: value });
+      // A picture that never loads costs the turn nothing: after a few seconds the result goes without it.
+      var slow = new Promise(function (resolve, reject) { browserTimeout(function () { reject(new Error('slow')); }, 10000); });
+      return Promise.race([encode(found.map(function (f) { return f.source; }), ACT_PICTURE_SIDE), slow]).then(function (encoded) {
+        return { value: kept, pictures: encoded.map(function (e, i) { return { media_type: e.media_type, data: e.data, number: found[i].number }; }) };
+      }, function () {
+        return { value: { result: kept, note: 'Its pictures couldn\'t be read, so they aren\'t shown.' } };
+      });
+    }
+    // The steps as sent: only the newest pictures go with their data, within the step's room for them.
+    function sentSteps() {
+      // Chosen newest first by number across every result, whatever order the results are in.
+      var room = ACT_PICTURE_BYTES, chosen = {}, all = [];
+      steps.forEach(function (s) { s.results.forEach(function (r) { (r.pictures || []).forEach(function (p) { all.push(p); }); }); });
+      all.sort(function (a, b) { return b.number - a.number; }).forEach(function (p) {
+        if (Object.keys(chosen).length < ACT_PICTURES && p.data.length <= room) { chosen[p.number] = true; room -= p.data.length; }
+      });
+      return steps.map(function (s) {
+        return { calls: s.calls, results: s.results.map(function (r) {
+          if (!r.pictures) return r;
+          var copy = {};
+          for (var k in r) if (Object.prototype.hasOwnProperty.call(r, k)) copy[k] = r[k];
+          copy.pictures = r.pictures.map(function (p) { return { media_type: p.media_type, number: p.number, data: chosen[p.number] ? p.data : null }; });
+          return copy;
+        }) };
+      });
+    }
     var declared = Object.keys(tools).map(function (name) {
       var t = tools[name];
       return { name: name, about: t.about || '', reads: !!t.reads, args: t.args || {} };
@@ -608,6 +696,7 @@
       over = true;
       out.stopped = reason || null;
       if (error) out.error = error;
+      out.seconds = Math.round((Date.now() - started) / 100) / 10;
       settle(out);
     }
     // Over, or past the deadline by the clock even if its timer hasn't run yet (a slow write can carry the turn past
@@ -637,7 +726,10 @@
         }).then(function (value) {
           if (value === NOT_RUN) return;
           out.reads.push({ name: c.name, args: c.args });
-          results[c.id] = { id: c.id, ok: true, value: sized(value) };
+          return pictured(value).then(function (p) {
+            results[c.id] = { id: c.id, ok: true, value: sized(p.value) };
+            if (p.pictures) results[c.id].pictures = p.pictures;
+          });
         }, function (e) { results[c.id] = { id: c.id, ok: false, error: refused(e) }; });
       })).then(function () {
         var stopped = null;
@@ -675,13 +767,18 @@
             var running;
             try { running = Promise.resolve(tool.run(c.args, call)); } catch (e) { running = Promise.reject(e); }
             return running.then(function (value) {
-              var kept = sized(value);
-              ran[key] = kept;
+              // The change is done: it's recorded now, whatever happens to its pictures after.
+              ran[key] = { note: 'Done.' };
               var entry = { name: c.name, args: c.args, result: value };
               if (value && typeof value === 'object' && value.partly) entry.partly = String(value.partly);
               out.done.push(entry);
-              results[c.id] = { id: c.id, ok: true, value: kept };
               tell({ kind: 'write', name: c.name, args: c.args, result: value });
+              return pictured(value).then(function (p) { return { p: p }; });
+            }).then(function (got) {
+              var kept = sized(got.p.value);
+              ran[key] = kept;
+              results[c.id] = { id: c.id, ok: true, value: kept };
+              if (got.p.pictures) results[c.id].pictures = got.p.pictures;
               return next(i + 1);
             }, function (e) { return fail(refused(e)); });
           }, function (e) { return fail(refused(e)); });
@@ -696,13 +793,15 @@
       if (left <= 500) return stop('time');
       // The step's own time: what's left of an app's turn, or the step's guard against a call that has stopped.
       var stepTime = Math.min(left, ACT_STEP_SECONDS * 1000);
-      var body = { request: String(request), about: o.about || '', now: present(), tools: declared, steps: steps,
+      var body = { request: String(request), about: o.about || '', now: present(), tools: declared, steps: sentSteps(),
                    history: (o.history || []).slice(-3), seconds: Math.floor(stepTime / 1000) };
       if (o.context !== undefined) body.context = jsonState(o.context);
       if (route) body.route = route;
       if (steps.length >= limits.steps - 1) body.final = true;
       if (o.model) body.model = o.model;
       post('/ai/act', body, stepTime, Date.now() + stepTime).then(function (answer) {
+        // What the step cost through a key, paid for whether or not its answer still runs.
+        if (typeof answer.cost === 'number' && answer.cost > 0) out.cost += answer.cost;
         // Cancelled, out of time or already over: an answer that comes late runs nothing.
         if (late()) return;
         route = answer.route || route;
@@ -728,6 +827,7 @@
         }) ? offered.map(function (c) { return c.trim(); }) : [];
         stop(null);
       }, function (e) {
+        if (e && typeof e.cost === 'number' && e.cost > 0) out.cost += e.cost;
         var message = refused(e);
         stop(/reach Lamp|Lamp isn't|wasn't put here|put it on this device/.test(message) ? 'unreachable' : 'error', message);
       });
@@ -928,8 +1028,12 @@
 
     /** options: data (text from outside the app, for the AI to work on, not obey), images (pictures to look at),
      *  system, json (true asks for JSON and gives back the parsed value: describe its exact shape in the prompt),
-     *  model (an id `LampAI.models()` lists, such as "engine:claude:opus" or "openrouter:sonnet"; a bare "opus" is
-     *  Opus through whatever answers by default). */
+     *  schema (a JSON Schema the answer must fit, which holds every AI to it, for large JSON: implies json; each object
+     *  lists every property in `required` and has `additionalProperties: false`), model (an id `LampAI.models()` lists,
+     *  such as "engine:claude:opus" or "openrouter:sonnet"; a bare "opus" is Opus through whatever answers by
+     *  default), signal (an AbortSignal: aborting it fails the ask at once with `error.stopped`), details (true
+     *  resolves { answer, seconds, cost }: `cost` in US dollars, 0 when the person's own Claude Code or Codex
+     *  answered). */
     ask: function (prompt, options) { return call(prompt, options); },
 
     /**
@@ -957,7 +1061,7 @@
      *       model: picker.value,                                           // optional: an id from LampAI.models()
      *       onStep: function (s) { … }                                     // { kind, name, args[, result] }
      *     });
-     *     turn.result.then(function (r) { r.reply; r.ask; r.choices; r.done; r.failed; r.skipped; r.stopped; });
+     *     turn.result.then(function (r) { r.reply; r.ask; r.choices; r.done; r.failed; r.skipped; r.stopped; r.seconds; r.cost; });
      *     turn.cancel();
      *
      * An argument's shape is { type: 'string' | 'integer' | 'number' | 'boolean' | 'date' | 'datetime' | 'list',
@@ -971,10 +1075,18 @@
      * `call.key` is the same for the same write in the same turn: make the ids of what a write creates from it, or
      * keep it on the change, so a write asked again (or a turn tried again with `turn: r.turn`) finds it instead of
      * adding a second. The same write in one turn while the page is open isn't run twice; nothing promises more.
-     * Throw `LampAI.refuse('sentence')` from `check` or `run` to say why something can't be done.
+     * A tool's result can show the AI what the app drew, in the middle of the turn: put `LampAI.picture(canvas)` in it
+     * (`{ rendered: [LampAI.picture(front), LampAI.picture(side)] }`), and the AI sees each picture in its place, so it
+     * can build, look and fix in one turn. The newest few are shown, smaller than drawn; older ones the AI has seen.
+     * Throw `LampAI.refuse('sentence')` from `check` or `run` to say why something can't be done. `seconds` is how long
+     * the turn took and `cost` what it cost in US dollars (0 when the person's own Claude Code or Codex answered).
      */
     act: function (request, options) { return act(request, options); },
 
+    /** A picture for the AI to look at, in what a tool of `act` returns: a <canvas>, an <img>, or a picture's address
+     *  (a data: URL or a file of the app's). */
+    picture: function (source) { return new ActPicture(source); },
+
     /**
      * What the person said, as text: `audio` is a WAV in base64 as `LampVoice.listen` gives it (16 kHz mono 16-bit,
      * at most 15 seconds), transcribed by Lamp on the Mac with the AI set up there (OpenRouter's Scribe), which

package/cli/openrouter.mjs

@@ -6,7 +6,18 @@
 const ENDPOINT = "https://openrouter.ai/api/v1/chat/completions";
 /** The model lamp answers with when none is named: the one Lamp's app offers first for apps' text. */
 export const DEFAULT_MODEL = "anthropic/claude-sonnet-5.5";
-const FILTERED = "The AI provider's content filter cut the reply off. Try rewording the request.";
+/** The image models an app may choose for its pictures, by OpenRouter's ids, first the one used when it chose none:
+ *  Lamp's app's list (`Icons.artModels`), kept the same by hand. */
+export const IMAGE_MODELS = [
+  "google/gemini-3.1-flash-image",
+  "google/gemini-nano-banana-2.1",
+  "google/gemini-3-pro-image",
+  "openai/gpt-image-2.5-sunburst",
+  "openai/gpt-image-2.5-flare",
+  "openai/gpt-image-2",
+  "google/gemini-3.1-flash-lite-image",
+];
+const FILTERED ="The AI provider's content filter cut the reply off. Try rewording the request.";
 
 /** Why OpenRouter gave no answer: `kind` "refused" (with its `status` and `reply`), "filtered", "timeout" or "stopped". */
 export class OpenRouterTrouble extends Error {

package/cli/page.js

@@ -25,16 +25,19 @@
   // (`/__lamp/ai`, with the person's own Claude Code or Codex), through the same shape of bridge lamp.js takes from
   // Lamp's Android app: postMessage of a request, an onmessage with its answer.
   if (!window.lampGatewayHost && !(window.webkit && window.webkit.messageHandlers && window.webkit.messageHandlers.lampGateway)) {
-    const gateway = { onmessage: null };
+    const gateway = { onmessage: null }, asking = new Map();
     gateway.postMessage = (text) => {
       let asked;
       try { asked = JSON.parse(text); } catch (e) { return; }
+      // The page stopped waiting (lamp.js's `cancel`): the request ends, which stops lamp's answer.
+      if (asked.cancel) { const going = asking.get(asked.id); if (going) { going.cancelled = true; going.abort(); } return; }
       const answer = (status, body) => { if (typeof gateway.onmessage === 'function') gateway.onmessage({ data: JSON.stringify({ id: asked.id, status, body }) }); };
       const stop = new AbortController();
+      asking.set(asked.id, stop);
       const timer = later(() => stop.abort(), Math.max(1000, Number(asked.timeout) || 240000));
       ask('/__lamp/ai', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Lamp-Page': '1' }, body: JSON.stringify({ path: asked.path, body: asked.body }), signal: stop.signal })
-        .then((r) => r.json()).then((r) => answer(r.status, r.body), () => answer(502, JSON.stringify({ error: 'lamp dev took too long to answer.' })))
-        .finally(() => unlater(timer));
+        .then((r) => r.json()).then((r) => answer(r.status, r.body), () => { if (!stop.cancelled) answer(502, JSON.stringify({ error: 'lamp dev took too long to answer.' })); })
+        .finally(() => { unlater(timer); asking.delete(asked.id); });
     };
     try { Object.defineProperty(window, 'lampGatewayHost', { value: gateway, configurable: true }); } catch (e) {}
   }

package/cli/skill/SKILL.md

@@ -36,7 +36,7 @@ looked earlier, `--reopen` puts it right.
 a page (saved data with `LampStore`, the app's files and more). Read Lamp's `web-app` skill before writing it, and
 the comments in `lamp.js` for how each block is used; never edit `lamp.js`. Its AI (`LampAI`: ask, act, decide) is
 answered by lamp itself on this Mac, with the person's own Claude Code or Codex (or OpenRouter, with their key in `OPENROUTER_API_KEY`), in every page lamp runs here (`dev`,
-`open`, `look`, `use`); pictures and hearing need Lamp's app. Shared on lamp.dev, it's answered on the computer the page is played on while Lamp is open there or `{{lamp}} ai` runs there in a terminal, once the person allows that page. Never put a key or a model provider's address in the page. A page that wants none of Lamp's needs no starting files: write its
+`open`, `look`, `use`). Pictures (`LampAI.makeImage` and its edits) are made there with the OpenRouter key, or with Codex; not with Claude Code alone. Hearing needs Lamp's app. Shared on lamp.dev, it's answered on the computer the page is played on while Lamp is open there or `{{lamp}} ai` runs there in a terminal, once the person allows that page. Never put a key or a model provider's address in the page. A page that wants none of Lamp's needs no starting files: write its
 `index.html` in an empty folder.
 
 ## A Mac app

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lampdotdev/lamp",
-  "version": "0.36.2-nightly.40",
+  "version": "0.36.2-nightly.43",
   "description": "Runs a web app or game from its folder and puts each saved change into the open page.",
   "type": "module",
   "bin": {