@datalayer/agent-runtimes 1.3.79 → 1.3.81

package/lib/chat/assistant/formats/acs.d.ts

@@ -1,4 +1,4 @@
-import { type AssistantCharacterData } from './types';
+import { type AssistantCharacterData, type AssistantMouthShape } from './types';
 export declare const ACS_SIGNATURE = 2882382787;
 export declare const ACF_SIGNATURE = 2882382788;
 export interface AcsLocator {
@@ -10,6 +10,26 @@ export interface AcsFrameImage {
     x: number;
     y: number;
 }
+/**
+ * A mouth overlay (ACSOVERLAYINFO), drawn only while the character speaks:
+ * its image over the frame's top image, or in its place.
+ */
+export interface AcsOverlay {
+    /** 0 closed, 1 to 4 wide open, 5 medium, 6 narrow (`MOUTH_SHAPES`). */
+    type: number;
+    /** Whether the overlay takes the place of the frame's top image. */
+    replaceTop: boolean;
+    image: number;
+    x: number;
+    y: number;
+    /**
+     * What the mouth is drawn over in place of the frame's images, when the
+     * file says (an `.aca` gives the frame without its top image).
+     */
+    base?: AcsFrameImage[];
+}
+/** The mouth shapes of the format, by overlay type. */
+export declare const MOUTH_SHAPES: readonly AssistantMouthShape[];
 export interface AcsFrame {
     /** Images, as stored: the first is drawn on top. */
     images: AcsFrameImage[];
@@ -23,6 +43,8 @@ export interface AcsFrame {
         frameIndex: number;
         probability: number;
     }[];
+    /** Its mouth overlays, by shape; none for most frames. */
+    overlays: AcsOverlay[];
 }
 export interface AcsAnimation {
     name: string;
@@ -57,6 +79,33 @@ export interface AcsImage {
     height: number;
     pixels: Uint8Array;
 }
+/**
+ * Little-endian reads that refuse, in a sentence naming the file kind, to
+ * run past the end. `terminated`: whether a STRING ends with a null
+ * character, as in an `.acs` (an `.acf`'s do not).
+ */
+export declare class AgentReader {
+    readonly bytes: Uint8Array;
+    readonly kind: string;
+    readonly terminated: boolean;
+    private readonly view;
+    pos: number;
+    constructor(bytes: Uint8Array, kind?: string, terminated?: boolean);
+    need(count: number, what: string): void;
+    seek(locator: AcsLocator, what: string): void;
+    u8(what: string): number;
+    u16(what: string): number;
+    i16(what: string): number;
+    u32(what: string): number;
+    skip(count: number, what: string): void;
+    take(count: number, what: string): Uint8Array;
+    locator(what: string): AcsLocator;
+    /**
+     * STRING: a ULONG count of UTF-16 characters, then, in an `.acs`, a null
+     * terminator.
+     */
+    string(what: string): string;
+}
 /**
  * Read the tables of an `.acs`: the character, its animations and frames,
  * where its images and sounds are. Images are decoded by `decodeAcsImage`.
@@ -64,6 +113,23 @@ export interface AcsImage {
 export declare function parseAcs(buffer: ArrayBuffer): AcsFile;
 /** Decode one image of an `.acs` into palette indices, top row first. */
 export declare function decodeAcsImage(file: AcsFile, index: number): AcsImage;
+/**
+ * What a Microsoft Agent character is drawn from, whichever file it came in:
+ * an `.acs` (`parseAcs`), or an `.acf` and its `.aca` files (`readAcfCharacter`).
+ */
+export interface AgentCharacterSource {
+    name: string;
+    width: number;
+    height: number;
+    transparentIndex: number;
+    palette: Uint32Array;
+    states: Record<string, string[]>;
+    animations: AcsAnimation[];
+    /** Image `index`, decoded. */
+    decode: (index: number) => AcsImage;
+    /** The bytes of sound `index` (RIFF WAVE). */
+    sound: (index: number) => Uint8Array;
+}
 /** One cell of the sprite sheet: the frames that look the same share it. */
 export interface AcsSpriteLayout {
     columns: number;
@@ -72,17 +138,29 @@ export interface AcsSpriteLayout {
     cells: AcsFrameImage[][];
     /** For each animation, for each frame, its cell, or -1 for an empty frame. */
     frameCells: number[][];
+    /**
+     * For each animation, for each frame, the cell of each mouth it has — the
+     * frame with that overlay drawn over its top image, or in its place.
+     */
+    mouthCells: Array<Array<Partial<Record<AssistantMouthShape, number>>>>;
 }
-/** Assign a cell to each distinct, non-empty frame. */
-export declare function layoutAcsSprite(file: AcsFile): AcsSpriteLayout;
+/** The images of a frame with a mouth: over the top image, or in its place. */
+export declare function mouthImages(frame: Pick<AcsFrame, 'images'>, overlay: AcsOverlay): AcsFrameImage[];
+/** Assign a cell to each distinct, non-empty frame, and to each mouth. */
+export declare function layoutAcsSprite(file: Pick<AgentCharacterSource, 'width' | 'height' | 'animations'>): AcsSpriteLayout;
 /**
  * Composite a cell as Agent draws a frame: its images from last to first,
  * each at its offset, the transparent colour left clear. RGBA, frame-sized.
  */
-export declare function composeAcsCell(file: AcsFile, images: AcsFrameImage[], decode?: (index: number) => AcsImage): Uint8ClampedArray;
+export declare function composeAcsCell(file: Pick<AgentCharacterSource, 'width' | 'height' | 'transparentIndex' | 'palette'>, images: AcsFrameImage[], decode?: (index: number) => AcsImage): Uint8ClampedArray;
+/**
+ * Draw a Microsoft Agent character in the page: its sprite sheet (an object
+ * URL), its animations with their mouths, its sounds as object URLs.
+ */
+export declare function drawAgentCharacter(source: AgentCharacterSource): Promise<AssistantCharacterData>;
 /**
- * Read a Microsoft Agent character (`.acs`) a person picked: its animations,
- * a sprite sheet drawn in the page (an object URL), its sounds as object
- * URLs. Nothing leaves the page.
+ * Read a Microsoft Agent character (`.acs`) a person picked: its animations
+ * and their mouths, a sprite sheet drawn in the page (an object URL), its
+ * sounds as object URLs. Nothing leaves the page.
  */
 export declare function readAcsCharacter(buffer: ArrayBuffer): Promise<AssistantCharacterData>;

package/lib/chat/assistant/formats/acs.js

@@ -27,22 +27,41 @@ const STYLE_BALLOON = 0x200;
 /** The widest a sprite sheet is laid out, and the largest it may be. */
 const SHEET_MAX_WIDTH = 4096;
 const SHEET_MAX_SIDE = 16384;
-class Reader {
+/** The mouth shapes of the format, by overlay type. */
+export const MOUTH_SHAPES = [
+    'closed',
+    'wide1',
+    'wide2',
+    'wide3',
+    'wide4',
+    'medium',
+    'narrow',
+];
+/**
+ * Little-endian reads that refuse, in a sentence naming the file kind, to
+ * run past the end. `terminated`: whether a STRING ends with a null
+ * character, as in an `.acs` (an `.acf`'s do not).
+ */
+export class AgentReader {
     bytes;
+    kind;
+    terminated;
     view;
     pos = 0;
-    constructor(bytes) {
+    constructor(bytes, kind = '.acs', terminated = true) {
         this.bytes = bytes;
+        this.kind = kind;
+        this.terminated = terminated;
         this.view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
     }
     need(count, what) {
         if (count < 0 || this.pos + count > this.bytes.length) {
-            throw new AssistantCharacterFormatError(`This .acs ends early: ${what} at byte ${this.pos} runs past the end of the file.`);
+            throw new AssistantCharacterFormatError(`This ${this.kind} ends early: ${what} at byte ${this.pos} runs past the end of the file.`);
         }
     }
     seek(locator, what) {
         if (locator.offset + locator.size > this.bytes.length) {
-            throw new AssistantCharacterFormatError(`This .acs is damaged: ${what} is said to be at byte ${locator.offset}, past the end of the file.`);
+            throw new AssistantCharacterFormatError(`This ${this.kind} is damaged: ${what} is said to be at byte ${locator.offset}, past the end of the file.`);
         }
         this.pos = locator.offset;
     }
@@ -81,17 +100,21 @@ class Reader {
     locator(what) {
         return { offset: this.u32(what), size: this.u32(what) };
     }
-    /** STRING: a ULONG count of UTF-16 characters, then a null terminator. */
+    /**
+     * STRING: a ULONG count of UTF-16 characters, then, in an `.acs`, a null
+     * terminator.
+     */
     string(what) {
         const count = this.u32(what);
         if (count === 0)
             return '';
-        this.need((count + 1) * 2, what);
+        const stored = this.terminated ? count + 1 : count;
+        this.need(stored * 2, what);
         let out = '';
         for (let k = 0; k < count; k++) {
             out += String.fromCharCode(this.view.getUint16(this.pos + k * 2, true));
         }
-        this.pos += (count + 1) * 2;
+        this.pos += stored * 2;
         return out;
     }
 }
@@ -190,17 +213,26 @@ function readFrames(r, animation) {
         for (let k = 0; k < branchCount; k++) {
             branches.push({ frameIndex: r.u16(where), probability: r.u16(where) });
         }
+        const overlays = [];
         const overlayCount = r.u8(where);
         for (let k = 0; k < overlayCount; k++) {
-            // Mouth overlays, drawn only while Agent speaks: read past, not kept.
-            // Type, replace flag, image index, an unknown byte; then the region flag.
-            r.skip(5, where);
+            // A mouth overlay, drawn only while the character speaks: its type,
+            // whether it replaces the top image, its image, an unknown byte, the
+            // region flag, its offset, its size (halved, and not needed: the
+            // image has its own), its region.
+            const type = r.u8(where);
+            const replaceTop = r.u8(where) !== 0;
+            const image = r.u16(where);
+            r.skip(1, where);
             const hasRegion = r.u8(where) !== 0;
-            r.skip(8, where);
+            const x = r.i16(where);
+            const y = r.i16(where);
+            r.skip(4, where);
             if (hasRegion)
                 r.skip(r.u32(where), where);
+            overlays.push({ type, replaceTop, image, x, y });
         }
-        frames.push({ images, sound, duration, exitBranch, branches });
+        frames.push({ images, sound, duration, exitBranch, branches, overlays });
     }
     return frames;
 }
@@ -213,10 +245,10 @@ export function parseAcs(buffer) {
     if (bytes.length < 36) {
         throw new AssistantCharacterFormatError(`This is not a Microsoft Agent character: the file is ${bytes.length} bytes, shorter than the header of an .acs.`);
     }
-    const r = new Reader(bytes);
+    const r = new AgentReader(bytes);
     const signature = r.u32('the signature');
     if (signature === ACF_SIGNATURE) {
-        throw new AssistantCharacterFormatError('This is a Microsoft Agent .acf, whose animations are in separate .aca files: .acf is not read yet, load the .acs of the character.');
+        throw new AssistantCharacterFormatError('This is a Microsoft Agent .acf, whose animations are in separate .aca files: pick the .acf together with its .aca files.');
     }
     if (signature !== ACS_SIGNATURE) {
         throw new AssistantCharacterFormatError(`This is not a Microsoft Agent character (.acs): it starts with 0x${signature.toString(16).padStart(8, '0')}, not 0xabcdabc3.`);
@@ -276,6 +308,11 @@ export function parseAcs(buffer) {
             if (frame.sound >= sounds.length) {
                 throw new AssistantCharacterFormatError(`This .acs is damaged: frame ${f} of ${name} plays sound ${frame.sound}, and there are ${sounds.length}.`);
             }
+            for (const overlay of frame.overlays) {
+                if (overlay.image >= images.length) {
+                    throw new AssistantCharacterFormatError(`This .acs is damaged: a mouth of frame ${f} of ${name} draws image ${overlay.image}, and there are ${images.length}.`);
+                }
+            }
             for (const branch of frame.branches) {
                 if (branch.frameIndex >= frames.length) {
                     throw new AssistantCharacterFormatError(`This .acs is damaged: frame ${f} of ${name} branches to frame ${branch.frameIndex}, and there are ${frames.length}.`);
@@ -304,7 +341,7 @@ export function decodeAcsImage(file, index) {
     if (!entry) {
         throw new AssistantCharacterFormatError(`This .acs has no image ${index}.`);
     }
-    const r = new Reader(file.bytes);
+    const r = new AgentReader(file.bytes);
     const what = `image ${index}`;
     r.seek(entry.locator, what);
     r.skip(1, what);
@@ -334,21 +371,39 @@ export function decodeAcsImage(file, index) {
     }
     return { width, height, pixels };
 }
-/** Assign a cell to each distinct, non-empty frame. */
+/** The images of a frame with a mouth: over the top image, or in its place. */
+export function mouthImages(frame, overlay) {
+    const mouth = { image: overlay.image, x: overlay.x, y: overlay.y };
+    const under = overlay.base ?? (overlay.replaceTop ? frame.images.slice(1) : frame.images);
+    return [mouth, ...under];
+}
+/** Assign a cell to each distinct, non-empty frame, and to each mouth. */
 export function layoutAcsSprite(file) {
     const cells = [];
     const byKey = new Map();
-    const frameCells = file.animations.map(animation => animation.frames.map(frame => {
-        if (frame.images.length === 0)
+    const cellOf = (images) => {
+        if (images.length === 0)
             return -1;
-        const key = frame.images.map(i => `${i.image},${i.x},${i.y}`).join(';');
+        const key = images.map(i => `${i.image},${i.x},${i.y}`).join(';');
         let cell = byKey.get(key);
         if (cell === undefined) {
             cell = cells.length;
-            cells.push(frame.images);
+            cells.push(images);
             byKey.set(key, cell);
         }
         return cell;
+    };
+    const frameCells = file.animations.map(animation => animation.frames.map(frame => cellOf(frame.images)));
+    const mouthCells = file.animations.map(animation => animation.frames.map((frame, f) => {
+        const mouths = {};
+        for (const overlay of frame.overlays) {
+            const shape = MOUTH_SHAPES[overlay.type];
+            if (!shape) {
+                throw new AssistantCharacterFormatError(`This character is damaged: a mouth of frame ${f} of ${animation.name} is of type ${overlay.type}, where 0 to 6 are expected.`);
+            }
+            mouths[shape] = cellOf(mouthImages(frame, overlay));
+        }
+        return mouths;
     }));
     const count = Math.max(1, cells.length);
     // Near square, and no wider than SHEET_MAX_WIDTH.
@@ -356,9 +411,9 @@ export function layoutAcsSprite(file) {
     const rows = Math.ceil(count / columns);
     if (columns * file.width > SHEET_MAX_SIDE ||
         rows * file.height > SHEET_MAX_SIDE) {
-        throw new AssistantCharacterFormatError(`This .acs has ${cells.length} distinct frames of ${file.width}×${file.height} pixels, more than one sprite sheet can hold.`);
+        throw new AssistantCharacterFormatError(`This character has ${cells.length} distinct frames of ${file.width}×${file.height} pixels, more than one sprite sheet can hold.`);
     }
-    return { columns, rows, cells, frameCells };
+    return { columns, rows, cells, frameCells, mouthCells };
 }
 /**
  * Composite a cell as Agent draws a frame: its images from last to first,
@@ -403,7 +458,7 @@ function makeSheet(width, height) {
         canvas = element;
         toBlob = () => new Promise((resolve, reject) => element.toBlob(blob => blob
             ? resolve(blob)
-            : reject(new AssistantCharacterFormatError('The sprite sheet of this .acs could not be drawn.')), 'image/png'));
+            : reject(new AssistantCharacterFormatError('The sprite sheet of this character could not be drawn.')), 'image/png'));
     }
     else {
         throw new AssistantCharacterFormatError('This page has no canvas to draw the character on.');
@@ -415,19 +470,17 @@ function makeSheet(width, height) {
     return { context, toBlob };
 }
 /**
- * Read a Microsoft Agent character (`.acs`) a person picked: its animations,
- * a sprite sheet drawn in the page (an object URL), its sounds as object
- * URLs. Nothing leaves the page.
+ * Draw a Microsoft Agent character in the page: its sprite sheet (an object
+ * URL), its animations with their mouths, its sounds as object URLs.
  */
-export async function readAcsCharacter(buffer) {
-    const file = parseAcs(buffer);
-    const layout = layoutAcsSprite(file);
-    const { width, height } = file;
+export async function drawAgentCharacter(source) {
+    const layout = layoutAcsSprite(source);
+    const { width, height } = source;
     const decoded = new Map();
     const decode = (index) => {
         let image = decoded.get(index);
         if (!image) {
-            image = decodeAcsImage(file, index);
+            image = source.decode(index);
             decoded.set(index, image);
         }
         return image;
@@ -439,17 +492,18 @@ export async function readAcsCharacter(buffer) {
     });
     layout.cells.forEach((images, cell) => {
         const data = sheet.context.createImageData(width, height);
-        data.data.set(composeAcsCell(file, images, decode));
+        data.data.set(composeAcsCell(source, images, decode));
         const { x, y } = cellAt(cell);
         sheet.context.putImageData(data, x, y);
     });
     const sprite = objectUrl(await sheet.toBlob());
     const used = new Set();
     const animations = {};
-    file.animations.forEach((animation, a) => {
+    source.animations.forEach((animation, a) => {
         animations[animation.name] = {
             frames: animation.frames.map((frame, f) => {
                 const cell = layout.frameCells[a][f];
+                const mouths = Object.entries(layout.mouthCells[a][f]);
                 if (frame.sound >= 0)
                     used.add(frame.sound);
                 return {
@@ -468,6 +522,11 @@ export async function readAcsCharacter(buffer) {
                         frame.exitBranch < animation.frames.length
                         ? { exitBranch: frame.exitBranch }
                         : {}),
+                    ...(mouths.length
+                        ? {
+                            mouths: Object.fromEntries(mouths.map(([shape, at]) => [shape, cellAt(at)])),
+                        }
+                        : {}),
                 };
             }),
         };
@@ -476,18 +535,31 @@ export async function readAcsCharacter(buffer) {
     if (used.size > 0) {
         sounds = {};
         for (const index of [...used].sort((x, y) => x - y)) {
-            const { offset, size } = file.sounds[index];
-            sounds[String(index)] = objectUrl(new Blob([file.bytes.slice(offset, offset + size)], {
-                type: 'audio/wav',
-            }));
+            sounds[String(index)] = objectUrl(new Blob([source.sound(index).slice()], { type: 'audio/wav' }));
         }
     }
     return {
-        name: file.name,
+        name: source.name,
         frameSize: { width, height },
         sprite,
         animations,
         ...(sounds ? { sounds } : {}),
-        authoredStates: file.states,
+        authoredStates: source.states,
     };
 }
+/**
+ * Read a Microsoft Agent character (`.acs`) a person picked: its animations
+ * and their mouths, a sprite sheet drawn in the page (an object URL), its
+ * sounds as object URLs. Nothing leaves the page.
+ */
+export async function readAcsCharacter(buffer) {
+    const file = parseAcs(buffer);
+    return drawAgentCharacter({
+        ...file,
+        decode: index => decodeAcsImage(file, index),
+        sound: index => {
+            const { offset, size } = file.sounds[index];
+            return file.bytes.subarray(offset, offset + size);
+        },
+    });
+}

package/lib/chat/assistant/formats/files.d.ts

@@ -1,8 +1,8 @@
 import { type AssistantCharacterData } from './types';
 /** What a file picker for a character accepts. */
-export declare const CHARACTER_FILES_ACCEPT = ".acs,.js,.png,.gif,.webp,.jpg,.jpeg";
+export declare const CHARACTER_FILES_ACCEPT = ".acs,.acf,.aca,.js,.png,.gif,.webp,.jpg,.jpeg";
 /** The sentence for a pick that is not a character. */
-export declare const CHARACTER_FILES_EXPECTED = "Pick one .acs file, or a clippy.js character: its agent.js and its map image (and a sounds file if you have one).";
+export declare const CHARACTER_FILES_EXPECTED = "Pick one .acs file; or an .acf file with its .aca files; or a clippy.js character: its agent.js and its map image (and a sounds file if you have one).";
 /** A picked file, as much of `File` as the reader needs. */
 export type PickedFile = Blob & {
     name: string;
@@ -11,6 +11,10 @@ export type PickedFile = Blob & {
 export declare function characterFilesOf(files: ReadonlyArray<PickedFile>): {
     kind: 'acs';
     acs: PickedFile;
+} | {
+    kind: 'acf';
+    acf: PickedFile;
+    acas: PickedFile[];
 } | {
     kind: 'clippy';
     agentJs: PickedFile;

package/lib/chat/assistant/formats/files.js

@@ -4,25 +4,32 @@
  */
 /**
  * The files a person picked, read as one character (LOOP T-26): one
- * Microsoft Agent `.acs`, or a clippy.js character's `agent.js` and map
- * image (and its sounds file, if there is one). Read in the page; nothing is
+ * Microsoft Agent `.acs`; a Microsoft Agent `.acf` with the `.aca` files it
+ * names; or a clippy.js character's `agent.js` and map image (and its
+ * sounds file, if there is one). Read in the page; nothing is
  * sent anywhere.
  *
  * @module chat/assistant/formats/files
  */
+import { readAcfCharacter } from './acf';
 import { readAcsCharacter } from './acs';
 import { readClippyCharacter } from './clippy';
 import { AssistantCharacterFormatError, } from './types';
 /** What a file picker for a character accepts. */
-export const CHARACTER_FILES_ACCEPT = '.acs,.js,.png,.gif,.webp,.jpg,.jpeg';
+export const CHARACTER_FILES_ACCEPT = '.acs,.acf,.aca,.js,.png,.gif,.webp,.jpg,.jpeg';
 /** The sentence for a pick that is not a character. */
-export const CHARACTER_FILES_EXPECTED = 'Pick one .acs file, or a clippy.js character: its agent.js and its map image (and a sounds file if you have one).';
+export const CHARACTER_FILES_EXPECTED = 'Pick one .acs file; or an .acf file with its .aca files; or a clippy.js character: its agent.js and its map image (and a sounds file if you have one).';
 /** Which of the picked files make the character, or why none do. */
 export function characterFilesOf(files) {
     const acs = files.find(file => file.name.toLowerCase().endsWith('.acs'));
     if (acs) {
         return { kind: 'acs', acs };
     }
+    const acf = files.find(file => file.name.toLowerCase().endsWith('.acf'));
+    if (acf) {
+        const acas = files.filter(file => file.name.toLowerCase().endsWith('.aca'));
+        return { kind: 'acf', acf, acas };
+    }
     const agentJs = files.find(file => file.name.toLowerCase() === 'agent.js');
     const map = files.find(file => /\.(png|gif|webp|jpe?g)$/i.test(file.name));
     const sounds = files.find(file => /^sounds-.*\.js$/i.test(file.name));
@@ -40,6 +47,13 @@ export async function readCharacterFiles(files) {
     if (picked.kind === 'acs') {
         return readAcsCharacter(await picked.acs.arrayBuffer());
     }
+    if (picked.kind === 'acf') {
+        const acas = {};
+        for (const aca of picked.acas) {
+            acas[aca.name] = await aca.arrayBuffer();
+        }
+        return readAcfCharacter(await picked.acf.arrayBuffer(), acas);
+    }
     return readClippyCharacter({
         agentJs: await picked.agentJs.text(),
         mapPng: picked.map,

package/lib/chat/assistant/formats/index.d.ts

@@ -1,14 +1,17 @@
 /**
  * Character files a person brings (LOOP T-26): clippy.js sprite maps and
- * Microsoft Agent `.acs` files, read in the page into one shape.
+ * Microsoft Agent `.acs` files, or `.acf` files with their `.aca` files,
+ * read in the page into one shape.
  *
  * @module chat/assistant/formats
  */
 export * from './types';
 export { readClippyCharacter, readClippyAgent, readClippySounds, } from './clippy';
 export type { ClippyCharacterFiles } from './clippy';
-export { readAcsCharacter, parseAcs, decodeAcsImage, layoutAcsSprite, composeAcsCell, } from './acs';
-export type { AcsFile, AcsAnimation, AcsFrame, AcsImage } from './acs';
+export { readAcsCharacter, parseAcs, decodeAcsImage, layoutAcsSprite, composeAcsCell, drawAgentCharacter, mouthImages, MOUTH_SHAPES, } from './acs';
+export type { AcsFile, AcsAnimation, AcsFrame, AcsImage, AcsOverlay, AgentCharacterSource, } from './acs';
+export { readAcfCharacter, parseAcf, parseAca } from './acf';
+export type { AcfFile, AcfAnimationEntry, AcaFile } from './acf';
 export { decompressAgentData } from './agentCompression';
 export { stateAnimations } from './stateAnimations';
 export { CHARACTER_FILES_ACCEPT, CHARACTER_FILES_EXPECTED, characterFilesOf, readCharacterFiles, } from './files';

package/lib/chat/assistant/formats/index.js

@@ -4,13 +4,15 @@
  */
 /**
  * Character files a person brings (LOOP T-26): clippy.js sprite maps and
- * Microsoft Agent `.acs` files, read in the page into one shape.
+ * Microsoft Agent `.acs` files, or `.acf` files with their `.aca` files,
+ * read in the page into one shape.
  *
  * @module chat/assistant/formats
  */
 export * from './types';
 export { readClippyCharacter, readClippyAgent, readClippySounds, } from './clippy';
-export { readAcsCharacter, parseAcs, decodeAcsImage, layoutAcsSprite, composeAcsCell, } from './acs';
+export { readAcsCharacter, parseAcs, decodeAcsImage, layoutAcsSprite, composeAcsCell, drawAgentCharacter, mouthImages, MOUTH_SHAPES, } from './acs';
+export { readAcfCharacter, parseAcf, parseAca } from './acf';
 export { decompressAgentData } from './agentCompression';
 export { stateAnimations } from './stateAnimations';
 export { CHARACTER_FILES_ACCEPT, CHARACTER_FILES_EXPECTED, characterFilesOf, readCharacterFiles, } from './files';

package/lib/chat/assistant/formats/types.d.ts

@@ -4,6 +4,11 @@
  *
  * @module chat/assistant/formats/types
  */
+/**
+ * The shapes of a mouth while the character speaks, as Microsoft Agent
+ * names its overlays: closed, wide open (four), medium, narrow.
+ */
+export type AssistantMouthShape = 'closed' | 'wide1' | 'wide2' | 'wide3' | 'wide4' | 'medium' | 'narrow';
 /** One frame of an animation. */
 export interface AssistantCharacterFrame {
     /** How long the frame shows, in milliseconds. */
@@ -26,6 +31,15 @@ export interface AssistantCharacterFrame {
     }[];
     /** The frame to go to when the animation is asked to end early. */
     exitBranch?: number;
+    /**
+     * The frame with each mouth the character has for it, drawn while it
+     * speaks in place of `images`: the sprite offset of each (Microsoft Agent's
+     * mouth overlays, composited into the sheet). Most frames have none.
+     */
+    mouths?: Partial<Record<AssistantMouthShape, {
+        x: number;
+        y: number;
+    }>>;
 }
 /** A named sequence of frames. */
 export interface AssistantCharacterAnimation {

package/lib/chat/assistant/state.d.ts

@@ -103,6 +103,8 @@ export declare function latestSaying(items: readonly unknown[]): AssistantSaying
  * popup's button calls it back — for the session, or for good; the header's
  * *Floating assistant* always calls it back.
  */
+/** The openings of the mouth, by level: shut, a little, half, wide. */
+export declare function mouthOpening(level: number): 0 | 1 | 2 | 3;
 export type AssistantAway = 'none' | 'page' | 'session' | 'always';
 /** Where the choice is kept: the session's storage, or for good. */
 export declare const ASSISTANT_AWAY_KEY = "datalayer-assistant-away";

package/lib/chat/assistant/state.js

@@ -125,6 +125,15 @@ export function latestSaying(items) {
     }
     return undefined;
 }
+/**
+ * How long the assistant is away, once sent (T-27): for the page — the
+ * popup's button calls it back — for the session, or for good; the header's
+ * *Floating assistant* always calls it back.
+ */
+/** The openings of the mouth, by level: shut, a little, half, wide. */
+export function mouthOpening(level) {
+    return level < 0.08 ? 0 : level < 0.3 ? 1 : level < 0.6 ? 2 : 3;
+}
 /** Where the choice is kept: the session's storage, or for good. */
 export const ASSISTANT_AWAY_KEY = 'datalayer-assistant-away';
 /** The away a page starts with, from what was kept. Storage may be refused. */

package/lib/chat/base/ChatBase.js

@@ -61,6 +61,7 @@ import { useNotebookTools } from '../../tools/adapters/agent-runtimes/notebookHo
 import { useAgentInspectorSink } from '../../components/inspector/agentInspector';
 import { chatInspectorRecorder, } from '../../components/inspector/chatInspect';
 import { loopMessageChange, speakerOf, withLoopMessage } from './loopMessage';
+import { loopStepOf, withLoopStep } from './loopStep';
 // Tracks pending prompts already auto-sent for a given conversation scope.
 // This prevents layout-driven unmount/remount cycles from re-sending prompts.
 const sentPendingPromptKeys = new Set();
@@ -2684,6 +2685,12 @@ showToolApprovalBanner = true, pendingApprovals: pendingApprovalsProp, onApprove
                     }
                     break;
                 case 'activity': {
+                    // A step of an application's code, drawn as it runs (LOOP P-16).
+                    const step = loopStepOf(event.activity);
+                    if (step) {
+                        setDisplayItems(prev => withLoopStep(prev, step));
+                        break;
+                    }
                     // An application's code changed a message it sent: its author
                     // said, or the message taken away (LOOP P-15).
                     const change = loopMessageChange(event.activity);