@lampdotdev/lamp 0.36.2-nightly.41 → 0.36.2-nightly.44

package/cli/lamps/web-starter/lamp.js

@@ -138,8 +138,20 @@
       delete asked[m.id];
       waiting({ status: m.status, body: m.body });
     };
-    bridge = { postMessage: function (m) {
-      return new Promise(function (resolve) { var id = ++askId; asked[id] = resolve; gate.postMessage(JSON.stringify({ id: id, path: m.path, body: m.body, timeout: m.timeout })); });
+    bridge = { postMessage: function (m, signal) {
+      return new Promise(function (resolve) {
+        var id = ++askId;
+        asked[id] = resolve;
+        gate.postMessage(JSON.stringify({ id: id, path: m.path, body: m.body, timeout: m.timeout }));
+        // A page that stops waiting tells the host, which can stop the answer (lamp dev does), and hears nothing after.
+        function stop() {
+          if (!asked[id]) return;
+          delete asked[id];
+          gate.postMessage(JSON.stringify({ id: id, cancel: true }));
+          resolve({ status: 499, body: '{}' });
+        }
+        if (signal) { if (signal.aborted) stop(); else signal.addEventListener('abort', stop); }
+      });
     } };
   }
   var preferred = null;
@@ -305,14 +317,14 @@
   // with a little more for the answer to come back.
   function AI_WAIT(seconds) { return (typeof seconds === 'number' && seconds > 0 ? Math.min(seconds, 600) : 600) * 1000 + 15000; }
 
-  function post(path, body, timeout, deadline) {
+  function post(path, body, timeout, deadline, signal) {
     timeout = timeout || 240000;
     var done;
     if (!bridge && shared()) {
       done = nearby(path, body, timeout, deadline);
     } else if (bridge) {
       // The TV preview on the Mac answers for Lamp itself.
-      done = bridge.postMessage({ path: path, body: JSON.stringify(body), timeout: timeout }).then(function (r) {
+      done = bridge.postMessage({ path: path, body: JSON.stringify(body), timeout: timeout }, signal).then(function (r) {
         var reply = {};
         try { reply = JSON.parse(r.body) || {}; } catch (e) {}
         return { status: r.status, reply: reply };
@@ -328,10 +340,15 @@
       });
     }
     return done.then(function (answer) {
+      // Stopped by the page: no failure of the AI's, so nothing is said about it.
+      if (signal && signal.aborted) throw stoppedFailure();
       if (answer.status === 200) return answer.reply;
       var message = answer.reply.error || 'The AI request failed.';
       if (window.console) console.error('LampAI ' + path + ' failed: ' + message);
-      throw new Failure(message, answer.reply.needs || null);
+      var failed = new Failure(message, answer.reply.needs || null);
+      // What was billed before it failed (a retry, a timeout), for a turn's `cost`.
+      if (typeof answer.reply.cost === 'number' && answer.reply.cost > 0) failed.cost = answer.reply.cost;
+      throw failed;
     });
   }
 
@@ -346,14 +363,14 @@
     });
   }
 
-  /** Downscales to at most 1568px on the long side (the most the model uses) and JPEG-encodes. Takes an <img>, a
-   *  <canvas>, or a picture's address (a data: URL or a file of the app's). */
-  function encode(images) {
+  /** Downscales to at most 1568px on the long side (the most the model uses), or `side`, and JPEG-encodes. Takes an
+   *  <img>, a <canvas>, or a picture's address (a data: URL or a file of the app's). */
+  function encode(images, side) {
     return Promise.all((images || []).map(function (image) {
       var ready = typeof image === 'string' ? loadImage(image) : Promise.resolve(image);
       return ready.then(function (source) {
         var w = source.naturalWidth || source.width, h = source.naturalHeight || source.height;
-        var scale = Math.min(1, 1568 / Math.max(w, h, 1));
+        var scale = Math.min(1, (side || 1568) / Math.max(w, h, 1));
         var canvas = document.createElement('canvas');
         canvas.width = Math.max(1, Math.round(w * scale));
         canvas.height = Math.max(1, Math.round(h * scale));
@@ -483,21 +500,44 @@
     score: function (question, levels) { return { type: 'score', instructions: question, criteria: levels }; }
   };
 
+  function stoppedFailure() { var f = new Failure('Stopped.'); f.stopped = true; return f; }
+
+  // A promise the page can stop waiting for with an AbortSignal: it fails at once with `stopped`, whatever comes after.
+  function stoppable(promise, signal) {
+    if (!signal) return promise;
+    return new Promise(function (resolve, reject) {
+      function stop() { reject(stoppedFailure()); }
+      // Settled either way, so a stopped ask's own failure is never left unhandled.
+      if (signal.aborted) { promise.then(null, function () {}); return stop(); }
+      signal.addEventListener('abort', stop);
+      promise.then(function (v) { signal.removeEventListener('abort', stop); if (!signal.aborted) resolve(v); },
+                   function (e) { signal.removeEventListener('abort', stop); if (!signal.aborted) reject(e); });
+    });
+  }
+
   function call(prompt, options) {
-    var o = options || {}, text = prompt, rules = [o.system || '', present()];
+    var o = options || {}, text = prompt, rules = [o.system || '', present()], started = Date.now(), cost = 0;
+    var schema = o.schema && typeof o.schema === 'object' ? o.schema : null, json = !!o.json || !!schema;
     if (o.data) { var f = fenced(prompt, o.data); text = f.prompt; rules.push(f.rules); }
-    return encode(o.images).then(function (images) {
-      var body = { prompt: text, system: rules.filter(function (r) { return r; }).join('\n\n'), json: !!o.json, images: images };
+    return stoppable(encode(o.images).then(function (images) {
+      // Stopped before or while its pictures were read: nothing is asked.
+      if (o.signal && o.signal.aborted) throw stoppedFailure();
+      var body = { prompt: text, system: rules.filter(function (r) { return r; }).join('\n\n'), json: json, images: images };
+      if (schema) body.schema = schema;
       if (o.model) body.model = o.model;
       // No limit of the page's own unless it asks for one: Lamp's guard ends only a call that stopped answering.
       if (typeof o.seconds === 'number' && o.seconds > 0) body.seconds = o.seconds;
-      return post('/ai', body, AI_WAIT(body.seconds));
+      return post('/ai', body, AI_WAIT(body.seconds), null, o.signal);
     }).then(function (reply) {
       if (typeof reply.text !== 'string') throw new Failure('The AI request failed.');
-      if (!o.json) return reply.text;
-      try { return JSON.parse(reply.text.replace(/^\s*```(?:json)?\s*|\s*```\s*$/g, '')); }
-      catch (e) { throw new Failure('The AI\'s answer wasn\'t in the expected format. Try again.'); }
-    });
+      cost = typeof reply.cost === 'number' && reply.cost > 0 ? reply.cost : 0;
+      var answer = reply.text;
+      if (json) {
+        try { answer = JSON.parse(reply.text.replace(/^\s*```(?:json)?\s*|\s*```\s*$/g, '')); }
+        catch (e) { throw new Failure('The AI\'s answer wasn\'t in the expected format. Try again.'); }
+      }
+      return o.details ? { answer: answer, seconds: Math.round((Date.now() - started) / 100) / 10, cost: cost } : answer;
+    }), o.signal);
   }
 
   function image(prompt, images, aspectRatio) {
@@ -538,6 +578,12 @@
   // What a turn's reads give back in all, which every later step carries again: kept under what a step may carry
   // (`MAX_BODY` in lamp's app-ai.mjs), with room for the request, the tools and the steps' own words.
   var ACT_TURN_RESULTS = 1200000;
+  // The pictures a step carries from its tools' results (`LampAI.picture`): the newest few, each at most this many
+  // pixels on its long side, together at most this many bytes, beside the text results under the step's limit
+  // (`MAX_PICTURES`, `MAX_BODY` in lamp's app-ai.mjs). Older ones the model has seen go as no longer shown.
+  var ACT_PICTURES = 6, ACT_PICTURE_SIDE = 1024, ACT_PICTURE_BYTES = 1600000;
+  /** A picture in a tool's result, for the AI to look at (`LampAI.picture`). */
+  function ActPicture(source) { this.source = source; }
   var ACT_STEP_SECONDS = 600;
   var actRan = {};      // turn id -> call key -> the result a write gave, while this copy of the page is open
   function actId() { return 't' + Date.now().toString(36) + Math.random().toString(36).slice(2, 8); }
@@ -596,8 +642,50 @@
     }
     var ran = actRan[id];
     var started = Date.now(), deadline = asked ? started + Math.max(1, asked) * 1000 : Infinity;
-    var out = { turn: id, reply: '', ask: false, choices: [], done: [], failed: [], skipped: [], reads: [], stopped: null, error: '' };
-    var steps = [], route = null, writes = 0, over = false, settle;
+    var out = { turn: id, reply: '', ask: false, choices: [], done: [], failed: [], skipped: [], reads: [], stopped: null, error: '', seconds: 0, cost: 0 };
+    var steps = [], route = null, writes = 0, over = false, settle, pictureCount = 0;
+    // A result with its pictures taken out: each one sent beside it, scaled down, and named in its place
+    // ("[picture 2]", numbered across the turn), so the model sees what the app drew in the middle of the turn.
+    function pictured(value) {
+      var found = [];
+      function walk(v, depth) {
+        if (v instanceof ActPicture) { found.push({ source: v.source, number: ++pictureCount }); return '[picture ' + pictureCount + ']'; }
+        if (!v || typeof v !== 'object' || depth > 8) return v;
+        if (isList(v)) return v.map(function (x) { return walk(x, depth + 1); });
+        var proto = Object.getPrototypeOf(v);
+        if (proto !== Object.prototype && proto !== null) return v;
+        var copy = {};
+        for (var k in v) if (Object.prototype.hasOwnProperty.call(v, k)) copy[k] = walk(v[k], depth + 1);
+        return copy;
+      }
+      var kept = walk(value, 0);
+      if (!found.length) return Promise.resolve({ value: value });
+      // A picture that never loads costs the turn nothing: after a few seconds the result goes without it.
+      var slow = new Promise(function (resolve, reject) { browserTimeout(function () { reject(new Error('slow')); }, 10000); });
+      return Promise.race([encode(found.map(function (f) { return f.source; }), ACT_PICTURE_SIDE), slow]).then(function (encoded) {
+        return { value: kept, pictures: encoded.map(function (e, i) { return { media_type: e.media_type, data: e.data, number: found[i].number }; }) };
+      }, function () {
+        return { value: { result: kept, note: 'Its pictures couldn\'t be read, so they aren\'t shown.' } };
+      });
+    }
+    // The steps as sent: only the newest pictures go with their data, within the step's room for them.
+    function sentSteps() {
+      // Chosen newest first by number across every result, whatever order the results are in.
+      var room = ACT_PICTURE_BYTES, chosen = {}, all = [];
+      steps.forEach(function (s) { s.results.forEach(function (r) { (r.pictures || []).forEach(function (p) { all.push(p); }); }); });
+      all.sort(function (a, b) { return b.number - a.number; }).forEach(function (p) {
+        if (Object.keys(chosen).length < ACT_PICTURES && p.data.length <= room) { chosen[p.number] = true; room -= p.data.length; }
+      });
+      return steps.map(function (s) {
+        return { calls: s.calls, results: s.results.map(function (r) {
+          if (!r.pictures) return r;
+          var copy = {};
+          for (var k in r) if (Object.prototype.hasOwnProperty.call(r, k)) copy[k] = r[k];
+          copy.pictures = r.pictures.map(function (p) { return { media_type: p.media_type, number: p.number, data: chosen[p.number] ? p.data : null }; });
+          return copy;
+        }) };
+      });
+    }
     var declared = Object.keys(tools).map(function (name) {
       var t = tools[name];
       return { name: name, about: t.about || '', reads: !!t.reads, args: t.args || {} };
@@ -608,6 +696,7 @@
       over = true;
       out.stopped = reason || null;
       if (error) out.error = error;
+      out.seconds = Math.round((Date.now() - started) / 100) / 10;
       settle(out);
     }
     // Over, or past the deadline by the clock even if its timer hasn't run yet (a slow write can carry the turn past
@@ -622,72 +711,92 @@
 
     // Reads run together; then the writes, one after another, each checked right before it runs. A write that
     // fails stops the writes after it in this step, which the model hears were skipped.
-    function runCalls(calls, step) {
-      var results = {}, reads = [], changes = [];
-      calls.forEach(function (c) { (tools[c.name] && tools[c.name].reads ? reads : changes).push(c); });
-      return Promise.all(reads.map(function (c) {
+    // A step's calls, run as they come (`stream`) and in the order the AI wrote them: reads side by side run together,
+    // a write waits for every call before it, and a read after a write sees it. A write that fails skips the writes
+    // after it in the step. A call Lamp held back as not fitting (`problem`) isn't run: the AI hears why, and a change of
+    // that kind fails in its place.
+    function stepRunner(step) {
+      var results = {}, chain = Promise.resolve(), reading = [], stopped = null;
+      function read(c) {
         if (late()) return null;
         var tool = tools[c.name], bad = actProblem(tool, c.name, c.args);
         if (bad) { results[c.id] = { id: c.id, ok: false, error: bad }; return null; }
         tell({ kind: 'read', name: c.name, args: c.args });
         var NOT_RUN = {};
         return Promise.resolve().then(function () {
-          // Reads start together, but each one only while there's time: an earlier one can be slow.
+          // Each read starts only while there's time: an earlier one can be slow.
           return late() ? NOT_RUN : tool.run(c.args, { turn: id, step: step, id: c.id });
         }).then(function (value) {
           if (value === NOT_RUN) return;
           out.reads.push({ name: c.name, args: c.args });
-          results[c.id] = { id: c.id, ok: true, value: sized(value) };
+          return pictured(value).then(function (p) {
+            results[c.id] = { id: c.id, ok: true, value: sized(p.value) };
+            if (p.pictures) results[c.id].pictures = p.pictures;
+          });
         }, function (e) { results[c.id] = { id: c.id, ok: false, error: refused(e) }; });
-      })).then(function () {
-        var stopped = null;
-        function next(i) {
-          if (i >= changes.length) return results;
-          var c = changes[i], tool = tools[c.name];
-          function skip(reason) {
-            out.skipped.push({ name: c.name, args: c.args, reason: reason });
-            results[c.id] = { id: c.id, ok: false, skipped: reason };
-            return next(i + 1);
-          }
-          function fail(reason) {
-            out.failed.push({ name: c.name, args: c.args, reason: reason });
-            results[c.id] = { id: c.id, ok: false, error: reason };
-            stopped = stopped || 'An earlier change in this step didn\'t go through.';
-            return next(i + 1);
-          }
-          if (late()) return results;
-          if (stopped) return skip(stopped);
-          var bad = actProblem(tool, c.name, c.args);
-          if (bad) return fail(bad);
-          var key = 'k' + hash(id + '|' + c.name + '|' + canonical(c.args));
-          // The same change asked again in this turn gets its first result, and doesn't run again.
-          // Its result is in an earlier step, which every request carries: pointed to, not sent again.
-          if (ran.hasOwnProperty(key)) { results[c.id] = { id: c.id, ok: true, repeat: true, value: { note: 'Already done in this request: its result is in an earlier step.' } }; return next(i + 1); }
-          if (writes >= limits.writes) return fail('Over this turn\'s limit of ' + limits.writes + ' changes.');
-          var call = { turn: id, key: key, step: step, id: c.id };
-          return Promise.resolve().then(function () { return tool.check ? tool.check(c.args, call) : null; }).then(function (problem) {
-            if (problem) return fail(String(problem));
-            // Cancel and the time are looked at again right before the change, never in the middle of it.
-            if (late()) return results;
-            tell({ kind: 'write', name: c.name, args: c.args });
-            if (late()) return results;
-            writes++;
-            var running;
-            try { running = Promise.resolve(tool.run(c.args, call)); } catch (e) { running = Promise.reject(e); }
-            return running.then(function (value) {
-              var kept = sized(value);
+      }
+      function write(c) {
+        var tool = tools[c.name];
+        function skip(reason) {
+          out.skipped.push({ name: c.name, args: c.args, reason: reason });
+          results[c.id] = { id: c.id, ok: false, skipped: reason };
+        }
+        function fail(reason) {
+          out.failed.push({ name: c.name, args: c.args, reason: reason });
+          results[c.id] = { id: c.id, ok: false, error: reason };
+          stopped = stopped || 'An earlier change in this step didn\'t go through.';
+        }
+        if (late()) return;
+        if (stopped) return skip(stopped);
+        // A change Lamp held back as not fitting failed, in its place.
+        if (typeof c.problem === 'string') return fail(c.problem);
+        var bad = actProblem(tool, c.name, c.args);
+        if (bad) return fail(bad);
+        var key = 'k' + hash(id + '|' + c.name + '|' + canonical(c.args));
+        // The same change asked again in this turn gets its first result, and doesn't run again.
+        // Its result is in an earlier step, which every request carries: pointed to, not sent again.
+        if (ran.hasOwnProperty(key)) { results[c.id] = { id: c.id, ok: true, repeat: true, value: { note: 'Already done in this request: its result is in an earlier step.' } }; return; }
+        if (writes >= limits.writes) return fail('Over this turn\'s limit of ' + limits.writes + ' changes.');
+        var call = { turn: id, key: key, step: step, id: c.id };
+        return Promise.resolve().then(function () { return tool.check ? tool.check(c.args, call) : null; }).then(function (problem) {
+          if (problem) return fail(String(problem));
+          // Cancel and the time are looked at again right before the change, never in the middle of it.
+          if (late()) return;
+          tell({ kind: 'write', name: c.name, args: c.args });
+          if (late()) return;
+          writes++;
+          var running;
+          try { running = Promise.resolve(tool.run(c.args, call)); } catch (e) { running = Promise.reject(e); }
+          return running.then(function (value) {
+            // The change is done: it's recorded now, whatever happens to its pictures after.
+            ran[key] = { note: 'Done.' };
+            var entry = { name: c.name, args: c.args, result: value };
+            if (value && typeof value === 'object' && value.partly) entry.partly = String(value.partly);
+            out.done.push(entry);
+            tell({ kind: 'write', name: c.name, args: c.args, result: value });
+            return pictured(value).then(function (p) {
+              var kept = sized(p.value);
               ran[key] = kept;
-              var entry = { name: c.name, args: c.args, result: value };
-              if (value && typeof value === 'object' && value.partly) entry.partly = String(value.partly);
-              out.done.push(entry);
               results[c.id] = { id: c.id, ok: true, value: kept };
-              tell({ kind: 'write', name: c.name, args: c.args, result: value });
-              return next(i + 1);
-            }, function (e) { return fail(refused(e)); });
-          }, function (e) { return fail(refused(e)); });
-        }
-        return next(0);
-      });
+              if (p.pictures) results[c.id].pictures = p.pictures;
+            });
+          }, function (e) { fail(refused(e)); });
+        }, function (e) { fail(refused(e)); });
+      }
+      return {
+        add: function (c) {
+          if (typeof c.problem === 'string' && tools[c.name] && tools[c.name].reads) { results[c.id] = { id: c.id, ok: false, error: c.problem }; return; }
+          if (tools[c.name] && tools[c.name].reads) {
+            var going = chain.then(function () { return read(c); });
+            reading.push(going);
+          } else {
+            var before = [chain].concat(reading);
+            reading = [];
+            chain = Promise.all(before).then(function () { return write(c); });
+          }
+        },
+        finish: function () { return Promise.all([chain].concat(reading)).then(function () { return results; }); }
+      };
     }
 
     function step() {
@@ -696,23 +805,35 @@
       if (left <= 500) return stop('time');
       // The step's own time: what's left of an app's turn, or the step's guard against a call that has stopped.
       var stepTime = Math.min(left, ACT_STEP_SECONDS * 1000);
-      var body = { request: String(request), about: o.about || '', now: present(), tools: declared, steps: steps,
+      var body = { request: String(request), about: o.about || '', now: present(), tools: declared, steps: sentSteps(),
                    history: (o.history || []).slice(-3), seconds: Math.floor(stepTime / 1000) };
       if (o.context !== undefined) body.context = jsonState(o.context);
       if (route) body.route = route;
       if (steps.length >= limits.steps - 1) body.final = true;
       if (o.model) body.model = o.model;
-      post('/ai/act', body, stepTime, Date.now() + stepTime).then(function (answer) {
+      // Streamed: each call comes as soon as the AI has written it, and runs then; the page follows the step for the
+      // rest. A Lamp from before streaming answers the whole step at once, which runs the same way.
+      body.stream = true;
+      var n = steps.length + 1, calls = [], ids = {}, runner = stepRunner(n), until = Date.now() + stepTime;
+      function heard(answer) {
+        // What the step cost through a key, paid for whether or not its answer still runs.
+        if (typeof answer.cost === 'number' && answer.cost > 0) out.cost += answer.cost;
         // Cancelled, out of time or already over: an answer that comes late runs nothing.
         if (late()) return;
         route = answer.route || route;
-        if (answer.calls && answer.calls.length) {
-          var calls = answer.calls, n = steps.length + 1, ids = {};
-          for (var i = 0; i < calls.length; i++) {
-            if (ids[calls[i].id]) return stop('error', 'The AI gave two calls the same id.');
-            ids[calls[i].id] = true;
-          }
-          return runCalls(calls, n).then(function (results) {
+        var given = answer.calls || [];
+        for (var i = 0; i < given.length; i++) {
+          if (ids[given[i].id]) return stop('error', 'The AI gave two calls the same id.');
+          ids[given[i].id] = true;
+          calls.push(given[i]);
+          runner.add(given[i]);
+        }
+        if (answer.more === true && typeof answer.stream === 'string') {
+          var have = calls.length;
+          return post('/ai/act', { follow: answer.stream, have: have }, Math.max(1000, until - Date.now()), until).then(heard, failed);
+        }
+        if (calls.length) {
+          return runner.finish().then(function (results) {
             if (late()) return;
             steps.push({ calls: calls, results: calls.map(function (c) { return results[c.id] || { id: c.id, ok: false, skipped: 'Not run.' }; }) });
             if (steps.length >= limits.steps) return stop('steps');
@@ -727,10 +848,13 @@
           return typeof c === 'string' && c.trim() && c.length <= 24 && !/[<>*_`#\[\]{}|\\\n\r\t]/.test(c);
         }) ? offered.map(function (c) { return c.trim(); }) : [];
         stop(null);
-      }, function (e) {
+      }
+      function failed(e) {
+        if (e && typeof e.cost === 'number' && e.cost > 0) out.cost += e.cost;
         var message = refused(e);
         stop(/reach Lamp|Lamp isn't|wasn't put here|put it on this device/.test(message) ? 'unreachable' : 'error', message);
-      });
+      }
+      post('/ai/act', body, stepTime, until).then(heard, failed);
     }
     step();
     return { id: id, result: result, cancel: function () { stop('cancelled'); } };
@@ -928,8 +1052,12 @@
 
     /** options: data (text from outside the app, for the AI to work on, not obey), images (pictures to look at),
      *  system, json (true asks for JSON and gives back the parsed value: describe its exact shape in the prompt),
-     *  model (an id `LampAI.models()` lists, such as "engine:claude:opus" or "openrouter:sonnet"; a bare "opus" is
-     *  Opus through whatever answers by default). */
+     *  schema (a JSON Schema the answer must fit, which holds every AI to it, for large JSON: implies json; each object
+     *  lists every property in `required` and has `additionalProperties: false`), model (an id `LampAI.models()` lists,
+     *  such as "engine:claude:opus" or "openrouter:sonnet"; a bare "opus" is Opus through whatever answers by
+     *  default), signal (an AbortSignal: aborting it fails the ask at once with `error.stopped`), details (true
+     *  resolves { answer, seconds, cost }: `cost` in US dollars, 0 when the person's own Claude Code or Codex
+     *  answered). */
     ask: function (prompt, options) { return call(prompt, options); },
 
     /**
@@ -957,24 +1085,34 @@
      *       model: picker.value,                                           // optional: an id from LampAI.models()
      *       onStep: function (s) { … }                                     // { kind, name, args[, result] }
      *     });
-     *     turn.result.then(function (r) { r.reply; r.ask; r.choices; r.done; r.failed; r.skipped; r.stopped; });
+     *     turn.result.then(function (r) { r.reply; r.ask; r.choices; r.done; r.failed; r.skipped; r.stopped; r.seconds; r.cost; });
      *     turn.cancel();
      *
      * An argument's shape is { type: 'string' | 'integer' | 'number' | 'boolean' | 'date' | 'datetime' | 'list',
-     * oneOf: [strings], min, max, optional, about, of (a list's item) }. A tool with `reads: true` changes nothing;
-     * reads in one step run together. Writes run one after another, each with its `check` right before it: a write
-     * that fails stops the writes after it in that step (`skipped`). Cancel and the time stop the turn between
-     * calls, never in the middle of a write; an answer that comes after runs nothing.
+     * oneOf: [strings], min, max, optional, about, of (a list's item) }. A tool with `reads: true` changes nothing.
+     * Each call runs as soon as the AI has written it, before the rest of its answer has come, so a build shows as
+     * it's made; calls run in the order the AI wrote them. Reads side by side run together; a write waits for the
+     * calls before it, with its `check` right before it, and a read after it sees it. A write that fails stops the
+     * writes after it in that step (`skipped`). Cancel and the time stop the turn between calls, never in the middle
+     * of a write; an answer that comes after runs nothing.
      *
      * Say what happened from `done`, `failed` and `skipped`, never from `reply`: they're what the page ran. A question
      * back (`ask`) may come with `choices`, two to four short answers to offer as buttons; with none, any answer goes.
      * `call.key` is the same for the same write in the same turn: make the ids of what a write creates from it, or
      * keep it on the change, so a write asked again (or a turn tried again with `turn: r.turn`) finds it instead of
      * adding a second. The same write in one turn while the page is open isn't run twice; nothing promises more.
-     * Throw `LampAI.refuse('sentence')` from `check` or `run` to say why something can't be done.
+     * A tool's result can show the AI what the app drew, in the middle of the turn: put `LampAI.picture(canvas)` in it
+     * (`{ rendered: [LampAI.picture(front), LampAI.picture(side)] }`), and the AI sees each picture in its place, so it
+     * can build, look and fix in one turn. The newest few are shown, smaller than drawn; older ones the AI has seen.
+     * Throw `LampAI.refuse('sentence')` from `check` or `run` to say why something can't be done. `seconds` is how long
+     * the turn took and `cost` what it cost in US dollars (0 when the person's own Claude Code or Codex answered).
      */
     act: function (request, options) { return act(request, options); },
 
+    /** A picture for the AI to look at, in what a tool of `act` returns: a <canvas>, an <img>, or a picture's address
+     *  (a data: URL or a file of the app's). */
+    picture: function (source) { return new ActPicture(source); },
+
     /**
      * What the person said, as text: `audio` is a WAV in base64 as `LampVoice.listen` gives it (16 kHz mono 16-bit,
      * at most 15 seconds), transcribed by Lamp on the Mac with the AI set up there (OpenRouter's Scribe), which

package/cli/openrouter.mjs

@@ -81,6 +81,84 @@ export async function chat({ key, body, seconds = 120, signal, fetcher = fetch }
   throw new OpenRouterTrouble("refused", "The AI didn't answer.", { cost });
 }
 
+/**
+ * One chat completion streamed (`stream` on `LampAI.act`): `onCall(name, argumentsText)` hears each tool call, in order,
+ * as soon as its arguments are whole (they read as a JSON object, which can't go on after its closing brace), or when the
+ * answer ends. Providers may send several calls' pieces interleaved, so a later call starting says nothing. Resolves as `chat` does,
+ * `{message, cost}`, the whole message. Asked once: a provider's filter cutting it off is `filtered`, since calls may
+ * already have gone to the page.
+ */
+export async function chatStream({ key, body, seconds = 120, signal, fetcher = fetch, onCall = () => {} }) {
+  const timeout = AbortSignal.timeout(Math.max(1000, Math.floor(seconds * 1000)));
+  let answer;
+  try {
+    answer = await fetcher(ENDPOINT, { method: "POST", body: JSON.stringify({ ...body, stream: true }),
+      headers: { Authorization: `Bearer ${key || ""}`, "Content-Type": "application/json", "X-Title": "Lamp" },
+      signal: signal ? AbortSignal.any([signal, timeout]) : timeout });
+  } catch (error) {
+    if (signal?.aborted) throw new OpenRouterTrouble("stopped", "Stopped.");
+    if (error?.name === "TimeoutError") throw new OpenRouterTrouble("timeout", "This ran out of time.");
+    throw new OpenRouterTrouble("refused", "Couldn't reach OpenRouter.");
+  }
+  if (answer.status !== 200 || !answer.body) {
+    let reply = {};
+    try { reply = await answer.json(); } catch {}
+    throw new OpenRouterTrouble("refused", "OpenRouter didn't answer.", { status: answer.status, reply });
+  }
+  const calls = [], told = new Set();
+  let content = "", cost = 0, finish = null, rest = "";
+  const whole = (call) => { try { const args = JSON.parse(call.function.arguments); return !!call.function.name && !!args && typeof args === "object" && !Array.isArray(args); } catch { return false; } };
+  // In order: a call waits for every one before it, so they run as they were written.
+  const tell = (all = false) => {
+    for (let i = told.size; i < calls.length; i++) {
+      if (!calls[i] || (!all && !whole(calls[i]))) return;
+      told.add(i);
+      onCall(calls[i].function.name, calls[i].function.arguments);
+    }
+  };
+  const take = (line) => {
+    if (!line.startsWith("data:")) return;
+    const data = line.slice(5).trim();
+    if (!data || data === "[DONE]") return;
+    let chunk;
+    try { chunk = JSON.parse(data); } catch { return; }
+    if (chunk.usage) cost += Number(chunk.usage.cost) || 0;
+    if (chunk.error) throw new OpenRouterTrouble("refused", "OpenRouter didn't answer.", { status: Number(chunk.error.code) || 0, reply: { error: chunk.error }, cost });
+    const choice = Array.isArray(chunk.choices) ? chunk.choices[0] : null;
+    if (!choice) return;
+    if (choice.finish_reason) finish = choice.finish_reason;
+    const delta = choice.delta ?? {};
+    if (typeof delta.content === "string") content += delta.content;
+    for (const part of Array.isArray(delta.tool_calls) ? delta.tool_calls : []) {
+      const i = Number.isInteger(part.index) ? part.index : calls.length;
+      if (!calls[i]) calls[i] = { id: part.id || `call_${i}`, type: "function", function: { name: "", arguments: "" } };
+      if (part.id) calls[i].id = part.id;
+      if (part.function?.name) calls[i].function.name += part.function.name;
+      if (part.function?.arguments) calls[i].function.arguments += part.function.arguments;
+    }
+    tell();
+  };
+  try {
+    const decoder = new TextDecoder();
+    for await (const bytes of answer.body) {
+      rest += decoder.decode(bytes, { stream: true });
+      const lines = rest.split("\n");
+      rest = lines.pop();
+      for (const line of lines) take(line.trim());
+    }
+    if (rest.trim()) take(rest.trim());
+  } catch (error) {
+    if (error instanceof OpenRouterTrouble) throw error;
+    if (signal?.aborted) throw new OpenRouterTrouble("stopped", "Stopped.", { cost });
+    if (error?.name === "TimeoutError" || timeout.aborted) throw new OpenRouterTrouble("timeout", "This ran out of time.", { cost });
+    throw new OpenRouterTrouble("refused", "OpenRouter stopped answering.", { cost });
+  }
+  if (finish === "content_filter") throw new OpenRouterTrouble("filtered", FILTERED, { cost });
+  tell(true);
+  const message = { content, ...(calls.length ? { tool_calls: calls.filter(Boolean) } : {}) };
+  return { message, cost };
+}
+
 /** A text answer: `{text, cost}`. */
 export async function text(given) {
   const { message, cost } = await chat({ ...given, body: textBody(given) });

package/cli/page.js

@@ -25,16 +25,19 @@
   // (`/__lamp/ai`, with the person's own Claude Code or Codex), through the same shape of bridge lamp.js takes from
   // Lamp's Android app: postMessage of a request, an onmessage with its answer.
   if (!window.lampGatewayHost && !(window.webkit && window.webkit.messageHandlers && window.webkit.messageHandlers.lampGateway)) {
-    const gateway = { onmessage: null };
+    const gateway = { onmessage: null }, asking = new Map();
     gateway.postMessage = (text) => {
       let asked;
       try { asked = JSON.parse(text); } catch (e) { return; }
+      // The page stopped waiting (lamp.js's `cancel`): the request ends, which stops lamp's answer.
+      if (asked.cancel) { const going = asking.get(asked.id); if (going) { going.cancelled = true; going.abort(); } return; }
       const answer = (status, body) => { if (typeof gateway.onmessage === 'function') gateway.onmessage({ data: JSON.stringify({ id: asked.id, status, body }) }); };
       const stop = new AbortController();
+      asking.set(asked.id, stop);
       const timer = later(() => stop.abort(), Math.max(1000, Number(asked.timeout) || 240000));
       ask('/__lamp/ai', { method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Lamp-Page': '1' }, body: JSON.stringify({ path: asked.path, body: asked.body }), signal: stop.signal })
-        .then((r) => r.json()).then((r) => answer(r.status, r.body), () => answer(502, JSON.stringify({ error: 'lamp dev took too long to answer.' })))
-        .finally(() => unlater(timer));
+        .then((r) => r.json()).then((r) => answer(r.status, r.body), () => { if (!stop.cancelled) answer(502, JSON.stringify({ error: 'lamp dev took too long to answer.' })); })
+        .finally(() => { unlater(timer); asking.delete(asked.id); });
     };
     try { Object.defineProperty(window, 'lampGatewayHost', { value: gateway, configurable: true }); } catch (e) {}
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lampdotdev/lamp",
-  "version": "0.36.2-nightly.41",
+  "version": "0.36.2-nightly.44",
   "description": "Runs a web app or game from its folder and puts each saved change into the open page.",
   "type": "module",
   "bin": {