@gui-chat-plugin/sequence 0.5.3 → 0.6.0

package/CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## 0.6.0
+
+- **An HTML or Markdown slide can hold a generated picture.** Give `imagePrompt` with `html` or
+  `markdown`, and mark where the picture goes with `{{image}}`: `<img src="{{image}}" class="…">` in
+  HTML, `![what it shows]({{image}})` in Markdown. The picture is drawn (one per slide) and put in
+  place when the slide is shown (`withSlideImage`, a `data:` URL, which the slide's policy allows);
+  the saved HTML keeps the mark, and the picture is saved apart (`imagePath`). A Markdown slide
+  with the picture on a line of its own is two columns, the text on the left and the picture on
+  the right, filling the slide's height. Without `{{image}}`, an `imagePrompt` sent with `html` or
+  `markdown` is still left out; `{{image}}` without `imagePrompt` is refused. A picture that can't
+  be drawn leaves the slide shown without it, and the model is told. A host that makes a movie of
+  a slideshow puts the saved picture in the same way (`withSlideImage` with the file as a `data:`
+  URL).
+
+## 0.5.4
+
+- **A story is told aloud, panel by panel.** Asked to tell a panel and call the next one in the same
+  reply, OpenAI's Realtime and Grok called it without a word: six panels went by in 25 seconds,
+  silent, while the pictures were drawn. The cast and each panel are now told in a reply of their
+  own, and the host's sequence keeper asks for the next panel when it ends, as slides have been since
+  0.5.3. Checked in MulmoChat (Storyteller role, mock images): every panel told aloud, 52–73 words
+  a panel on OpenAI's Realtime, 57–69 on Grok, 28–38 on Gemini Live. Interactive panels (with
+  choices) and the last panel are unchanged. A host needs `createSequenceKeeper` for a story to go
+  on by itself, as for a slideshow.
+
 ## 0.5.3
 
 - **A slide is explained in a reply of its own, and the host's sequence keeper asks for the next

package/README.md

@@ -47,9 +47,10 @@ asks the model to go on when it ends a reply mid-slideshow, and passes `keeper.u
 `context.userSpokeAt`. A slideshow needs it: a slide's instructions ask the model only to explain
 the slide, in a reply of its own, and the keeper asks for the next one when that reply ends (asked
 for both in one reply, voice models announced the next slide instead of explaining this one).
-Without a keeper the slideshow stops after each slide until the user says to go on. Stories and
-step-by-step guides go on without it: a panel's instructions ask for the next panel, and a guide
-waits for the user anyway.
+Stories work the same way: the cast and each panel are told in a reply of their own, and the keeper
+asks for the next panel. Without a keeper, a slideshow or a story stops after each step until the
+user says to go on. Step-by-step guides and interactive panels (with choices) wait for the user
+anyway.
 
 Step results also set `instructionsRequired`, so a turn-based host (text chat) takes a turn for the
 model to explain the step.
@@ -73,6 +74,21 @@ small tables; HTML for designed layouts, comparisons, timelines and diagrams. Pi
 words, numbers and math badly. A slide sent with more than one is one kind: `html` first, then
 `markdown`, then `chart`, then the picture.
 
+### A picture on an HTML or Markdown slide
+
+An HTML or Markdown slide can hold one generated picture: `imagePrompt` with `html` or `markdown`,
+and `{{image}}` where the picture goes (`<img src="{{image}}" class="…">`, `![what it shows]({{image}})`).
+`execute()` draws it with the host's `generateImage` and returns it as for a picture slide
+(`imageData`, `imagePath`); the slide's `html` keeps `{{image}}`, in the result and the record.
+`withSlideImage(html, imageData)` (core entry) puts the picture in its place, a `data:` URL, which
+the slide's policy allows; the View does this, and so must a host that shows or films the saved
+HTML itself (reading `imagePath` as a `data:` URL). A Markdown slide whose picture is on a line of
+its own is laid out in two columns: the text on the left, the picture on the right, filling the
+slide's height (`object-fit: cover`, so the picture is asked to keep its subject in the middle).
+Without `{{image}}`, an `imagePrompt` sent with `html` or `markdown` is left out; `{{image}}`
+without `imagePrompt` is refused. When the picture can't be drawn, the slide is shown without it
+and the model is told.
+
 ### Markdown slides
 
 `markdown` is GitHub-flavored Markdown with TeX math between `$…$` (inline) or `$$…$$` (a
@@ -134,7 +150,7 @@ that an HTML step is on the screen; a guide step asked for again is then shown a
 
 Saved through `files.artifacts`, with their pictures' paths:
 
-- `slideshows/<id>.json`: `{ id, title, mode, totalSlides, slides: { "1": { title, imagePrompt, imagePath } } }`; a slide that isn't a picture has an `imagePrompt` of `""` and no `imagePath`, and has `html` (an HTML slide), `html` and `markdown` (a Markdown slide) or `chart` (a chart slide)
+- `slideshows/<id>.json`: `{ id, title, mode, totalSlides, slides: { "1": { title, imagePrompt, imagePath } } }`; a slide that isn't a picture has `html` (an HTML slide), `html` and `markdown` (a Markdown slide) or `chart` (a chart slide), and an `imagePrompt` of `""` and no `imagePath` unless its `html` places a picture at `{{image}}`
 - `storyboards/<id>.json`: `{ id, title, style, totalPanels, interactive, characters: [{ name, description, imagePath }], panels: { "1": { caption, characters, imagePrompt, imagePath, choices } } }`
 
 IDs are 12 hex digits. The results name them, so a later tool call can refer to a slideshow or a

package/dist/core/definitions.d.ts

@@ -14,7 +14,8 @@ export interface SlideArgs {
     slide: number;
     totalSlides: number;
     title: string;
-    /** The picture's prompt; "" for a slide that isn't a picture. */
+    /** The picture's prompt: a picture slide's, or the picture an HTML or
+     *  Markdown slide places at {{image}}; "" for a slide without one. */
     imagePrompt: string;
     /** An HTML slide's body. */
     html?: string;
@@ -26,6 +27,7 @@ export interface SlideArgs {
 }
 export declare const SLIDE_ARGS_ERROR = "presentSlide needs slide and totalSlides (whole numbers, slide from 1 to totalSlides), and one of imagePrompt, html, markdown or chart";
 export declare const SLIDE_HTML_TOO_LONG = "presentSlide's slide is too long: keep its html, markdown or chart under 60000 characters";
+export declare const SLIDE_IMAGE_WITHOUT_PROMPT = "presentSlide's slide has {{image}} but no imagePrompt: give imagePrompt for the picture that goes there, or take {{image}} out";
 export declare const SLIDE_CHART_INVALID: string;
 /** The slide arguments, or what is wrong with them (said to the model). A
  *  slide is one kind: html first, then markdown, then chart, then a
@@ -42,11 +44,13 @@ export interface Slideshow {
     totalSlides: number;
     slides: Record<string, {
         title: string;
-        /** "" for a slide that isn't a picture. */
+        /** "" for a slide without a picture. */
         imagePrompt: string;
+        /** A picture slide's picture, or the one an HTML or Markdown slide
+         *  places at {{image}} (withSlideImage). */
         imagePath?: string;
         /** An HTML slide's body (slideHtmlDocument makes it a page); for a
-         *  Markdown slide, the HTML made of it. */
+         *  Markdown slide, the HTML made of it. Keeps {{image}}. */
         html?: string;
         /** A Markdown slide's text, as the model wrote it. */
         markdown?: string;
@@ -62,7 +66,7 @@ export interface SlideData {
     html?: string;
     markdown?: string;
     chart?: SlideChart;
-    /** The picture's prompt; "" for a slide that isn't a picture. */
+    /** The picture's prompt; "" for a slide without a picture. */
     prompt: string;
     slideshowId: string;
     slide: number;
@@ -110,7 +114,7 @@ export interface StoryboardArgs {
 }
 /** The arguments, or what is wrong with them (said to the model). */
 export declare function parseStoryboardArgs(args: Record<string, unknown>): StoryboardArgs | string;
-export declare const castShownInstructions: ({ title, id }: Storyboard) => string;
+export declare const castShownInstructions: ({ title }: Storyboard) => string;
 /** The cast's result data. */
 export interface CastData {
     storyboardId: string;

package/dist/core/index.d.ts

@@ -4,7 +4,7 @@ export declare const presentSlidePluginCore: SequencePluginCore;
 export declare const defineStoryboardPluginCore: SequencePluginCore;
 export declare const presentPanelPluginCore: SequencePluginCore;
 export * from "./definitions";
-export { MAX_SLIDE_HTML, SLIDE_ANIMATION_KINDS, SLIDE_AUTO_END_SECONDS, SLIDE_CSS_ANIMATIONS_FINISHED, SLIDE_FIT_SCRIPT, SLIDE_HEIGHT, SLIDE_WIDTH, slideHtmlDocument, } from "./slideHtml";
+export { MAX_SLIDE_HTML, SLIDE_ANIMATION_KINDS, SLIDE_AUTO_END_SECONDS, SLIDE_CSS_ANIMATIONS_FINISHED, SLIDE_FIT_SCRIPT, SLIDE_HEIGHT, SLIDE_WIDTH, slideHtmlDocument, SLIDE_IMAGE_MARK, hasSlideImageMark, withSlideImage, } from "./slideHtml";
 export { SLIDE_CHART_TYPES, chartSlideDocument, type SlideChart, } from "./chartSlide";
 export { markdownSlideHtml } from "./markdownSlide";
 export { SLIDESHOWS_DIR, STORYBOARDS_DIR, loadRecord } from "./records";

package/dist/core/slideHtml.d.ts

@@ -39,5 +39,15 @@ export declare const neutralizeSlideHtml: (html: string) => string;
  *  that frame, so a host that makes a movie of the slide adds these to the
  *  HTML it passes on (a zero-length animation is at its end even paused). */
 export declare const SLIDE_CSS_ANIMATIONS_FINISHED = "*, *::before, *::after { animation-play-state: paused !important; animation-duration: 0s !important; animation-delay: 0s !important; animation-iteration-count: 1 !important; animation-fill-mode: both !important; transition: none !important; }";
+export declare const SLIDE_IMAGE_MARK = "{{image}}";
+/** Whether a slide's HTML has a place for a picture: an src or a CSS url()
+ *  of {{image}} outside <pre> and <code>, where withSlideImage puts it. A
+ *  Markdown slide is asked about the HTML made of it, so the answer is
+ *  marked's reading of the Markdown. */
+export declare const hasSlideImageMark: (html: string) => boolean;
+/** The slide's HTML with its picture in place of {{image}}, outside <pre>
+ *  and <code>: imageData when it is a data: URL of an image, else a
+ *  transparent pixel. */
+export declare const withSlideImage: (html: string, imageData?: string) => string;
 /** The page an HTML slide is shown as. */
 export declare function slideHtmlDocument(html: string): string;