agentix-cli 0.94.1 → 0.95.0

package/apps/mac-voice/Sources/AgentXVoice/Character.swift

@@ -51,6 +51,10 @@ final class CharacterHost {
     private var rest = CGPoint.zero
     /// Its body on screen: where it can be taken hold of.
     private var body = NSRect.zero
+    /// Its colours, and the point under it in screen points: play mode
+    /// (#505) takes it from there.
+    var stops: [NSColor] { view.stops }
+    var foot: NSPoint { NSPoint(x: CGFloat(sim.x), y: rest.y) }
     /// Carried by the pointer: where the pointer took hold, and where it
     /// stood then, which is not where it rests once it has stepped aside.
     private var carried: (from: NSPoint, rest: CGPoint)?

package/apps/mac-voice/Sources/AgentXVoice/Config.swift

@@ -35,6 +35,13 @@ enum Config {
         set { UserDefaults.standard.set(newValue, forKey: "animatedOrb") }
     }
 
+    /// "Play mode" in the menu (#505): the character may play on a frozen
+    /// picture of the screen when asked. Off by default.
+    static var playMode: Bool {
+        get { UserDefaults.standard.bool(forKey: "playMode") }
+        set { UserDefaults.standard.set(newValue, forKey: "playMode") }
+    }
+
     /// Where the pill was last dragged to (its bottom-left corner, in
     /// screen points). Nil: the default corner.
     static var pillOrigin: CGPoint? {

package/apps/mac-voice/Sources/AgentXVoice/PlayMath.swift

@@ -0,0 +1,250 @@
+import Foundation
+
+/// Play mode (#505): the character leaves the bottom edge and plays on a
+/// frozen picture of the screen. It walks along a line of text as on a
+/// floor, jumps to another line, eats words and wipes a line. These are
+/// the numbers only: where the words are, what it does and when, where it
+/// is at a given moment and which letters are gone. Foundation only, so
+/// the tests can run it without a window.
+///
+/// Everything is in the picture's points, origin top-left, y downwards.
+enum PlayMath {
+    typealias M = CharacterMath
+
+    struct Point: Equatable { var x = 0.0, y = 0.0 }
+
+    struct Rect: Equatable {
+        var x = 0.0, y = 0.0, w = 0.0, h = 0.0
+        var maxX: Double { x + w }
+        var maxY: Double { y + h }
+        var midX: Double { x + w / 2 }
+        var midY: Double { y + h / 2 }
+    }
+
+    struct Word: Equatable {
+        var text: String
+        var rect: Rect
+    }
+
+    /// One line of text as it was read, left to right, and the colour of
+    /// the page around it (0xRRGGBB): what shows where a word is gone.
+    struct Line: Equatable {
+        var words: [Word]
+        var rect: Rect
+        var paper: UInt32 = 0xFFFFFF
+    }
+
+    /// One move of a script.
+    enum Step: Equatable {
+        /// Jump onto the start of a line.
+        case jump(line: Int)
+        /// Walk along the top of a line to its end.
+        case walk(line: Int)
+        /// Go through a line and eat these words, letter by letter.
+        case eat(line: Int, words: Range<Int>)
+        /// Wipe a whole line with a cloth.
+        case wipe(line: Int)
+        /// Stand still, facing you, for so many seconds.
+        case rest(Double)
+    }
+
+    /// Points per drawing unit on the page: a body 40 points across,
+    /// smaller than on the bottom edge so it fits between lines.
+    static let unit = 0.4
+    /// The middle of its body above its feet, and its top, in points.
+    static let middle = (M.Pose().lift + 50) * unit
+    static let top = (M.Pose().lift + 100) * unit + 8
+    /// Points a second, and seconds a letter.
+    static let walkSpeed = 220.0
+    static let wipeSpeed = 260.0
+    static let perLetter = 0.09
+    /// Its mouth, ahead of its middle.
+    static let mouth = 10.0
+
+    /// A word's letters: the word's rectangle cut into equal parts, one a
+    /// character. An estimate, close enough to take them one at a time.
+    static func letters(of word: Word) -> [Rect] {
+        let n = max(word.text.count, 1), w = word.rect.w / Double(n)
+        return (0..<n).map { Rect(x: word.rect.x + Double($0) * w, y: word.rect.y, w: w, h: word.rect.h) }
+    }
+
+    /// The colour most of `samples` share: the page around a line. Close
+    /// shades count as one, so the soft edge of a letter does not win.
+    static func common(_ samples: [UInt32]) -> UInt32? {
+        var count: [UInt32: Int] = [:]
+        for s in samples { count[s & 0xF8F8F8, default: 0] += 1 }
+        guard let best = count.max(by: { ($0.value, $0.key) < ($1.value, $1.key) })?.key else { return nil }
+        return samples.first { $0 & 0xF8F8F8 == best }
+    }
+
+    /// The built-in script: the line nearest the middle of the page and
+    /// the two under it in the same column. It walks the first, eats the
+    /// start of the second and wipes the third. Lines too close to the top
+    /// to stand on, too short or of one word are left alone.
+    static func demo(_ lines: [Line], width: Double, height: Double) -> [Step] {
+        let fit = lines.indices.filter { lines[$0].words.count >= 2 && lines[$0].rect.w >= 120 && lines[$0].rect.y >= top + 40 }
+        func far(_ i: Int) -> Double { hypot(lines[i].rect.midX - width / 2, lines[i].rect.midY - height / 2) }
+        guard let first = fit.min(by: { far($0) < far($1) }) else { return [] }
+        let a = lines[first].rect
+        let under = fit.filter { lines[$0].rect.y > a.y + a.h / 2 && lines[$0].rect.x < a.maxX && lines[$0].rect.maxX > a.x }
+            .sorted { lines[$0].rect.y < lines[$1].rect.y }
+        let picked = [first] + under.prefix(2)
+        var steps: [Step] = [.jump(line: first), .walk(line: first)]
+        if picked.count > 1 { steps += [.eat(line: picked[1], words: 0..<min(3, lines[picked[1]].words.count))] }
+        else { steps += [.eat(line: first, words: 0..<min(2, lines[first].words.count))] }
+        if picked.count > 2 { steps += [.wipe(line: picked[2])] }
+        return steps + [.rest(1.2)]
+    }
+}
+
+/// A script laid out in time: every move has its start and its end, so
+/// where the character is, and what is gone, depends on the time alone.
+/// The same script gives the same frames on every run.
+struct Play {
+    typealias P = PlayMath
+    typealias M = CharacterMath
+
+    enum Kind { case jump, walk, eat, wipe, rest }
+
+    struct Move {
+        var kind: Kind
+        var from, to: P.Point
+        var start, duration: Double
+        var line = 0
+        var words = 0..<0
+        var end: Double { start + duration }
+    }
+
+    struct Frame {
+        /// Its feet.
+        var at = P.Point()
+        /// -1 turned left, 0 facing you, 1 turned right.
+        var face = 0.0
+        var pose = M.Pose()
+        var t = 0.0
+        /// What is covered with the page's colour, and which colour.
+        var gone: [(rect: P.Rect, paper: UInt32)] = []
+        /// The cloth, while it wipes.
+        var cloth: P.Rect?
+        var done = false
+    }
+
+    let lines: [P.Line]
+    private(set) var moves: [Move] = []
+    var duration: Double { moves.last?.end ?? 0 }
+
+    /// `start` is where its feet are when play begins. Steps that name a
+    /// line or a word that is not there are skipped.
+    init(lines: [P.Line], steps: [P.Step], start: P.Point) {
+        self.lines = lines
+        var at = start, now = 0.0
+        func add(_ kind: Kind, to: P.Point, _ seconds: Double, line: Int = 0, words: Range<Int> = 0..<0) -> Move {
+            Move(kind: kind, from: at, to: to, start: now, duration: seconds, line: line, words: words)
+        }
+        /// Get to `to` first: a jump, unless it is already there.
+        func reach(_ to: P.Point, into list: inout [Move]) {
+            guard at != to else { return }
+            let m = add(.jump, to: to, 0.5 + hypot(to.x - at.x, to.y - at.y) / 900)
+            list.append(m); at = to; now = m.end
+        }
+        var list: [Move] = []
+        for step in steps {
+            switch step {
+            case .jump(let i):
+                guard lines.indices.contains(i) else { continue }
+                reach(P.Point(x: lines[i].rect.x, y: lines[i].rect.y), into: &list)
+            case .walk(let i):
+                guard lines.indices.contains(i) else { continue }
+                let r = lines[i].rect
+                if at.y != r.y || at.x < r.x || at.x > r.maxX { reach(P.Point(x: r.x, y: r.y), into: &list) }
+                let m = add(.walk, to: P.Point(x: r.maxX, y: r.y), (r.maxX - at.x) / P.walkSpeed, line: i)
+                list.append(m); at = m.to; now = m.end
+            case .eat(let i, let words):
+                guard lines.indices.contains(i), !words.isEmpty, words.upperBound <= lines[i].words.count else { continue }
+                // In the line, its middle at the middle of the letters.
+                let y = lines[i].rect.midY + P.middle
+                let first = lines[i].words[words.lowerBound].rect, last = lines[i].words[words.upperBound - 1].rect
+                reach(P.Point(x: first.x - P.mouth - 4, y: y), into: &list)
+                let count = lines[i].words[words].reduce(0) { $0 + max($1.text.count, 1) }
+                let m = add(.eat, to: P.Point(x: last.maxX, y: y), Double(count) * P.perLetter, line: i, words: words)
+                list.append(m); at = m.to; now = m.end
+            case .wipe(let i):
+                guard lines.indices.contains(i) else { continue }
+                let r = lines[i].rect
+                reach(P.Point(x: r.x, y: r.y), into: &list)
+                let m = add(.wipe, to: P.Point(x: r.maxX, y: r.y), r.w / P.wipeSpeed, line: i)
+                list.append(m); at = m.to; now = m.end
+            case .rest(let seconds):
+                let m = add(.rest, to: at, max(seconds, 0))
+                list.append(m); now = m.end
+            }
+        }
+        moves = list
+    }
+
+    func frame(at t: Double) -> Frame {
+        guard let i = moves.lastIndex(where: { $0.start <= t }) else { return Frame(done: true) }
+        let m = moves[i], p = M.span(t, m.start, m.end)
+        var f = Frame(t: t, done: t >= duration)
+
+        f.at = P.Point(x: m.from.x + (m.to.x - m.from.x) * p, y: m.from.y + (m.to.y - m.from.y) * p)
+        if m.kind == .jump {
+            // An arc over both ends, kept inside the picture.
+            let rise = 30 + hypot(m.to.x - m.from.x, m.to.y - m.from.y) * 0.12
+            f.at.y -= min(rise, max(min(m.from.y, m.to.y) - P.top, 0)) * 4 * p * (1 - p)
+        }
+
+        // The look of this move, reached from the look of the one before
+        // in a fifth of a second: the eyes and the body do not snap.
+        let into = M.ease(M.span(t, m.start, m.start + 0.2))
+        let before = i > 0 ? look(moves[i - 1]) : look(m)
+        let now = look(m)
+        f.face = before.face + (now.face - before.face) * into
+        f.pose = M.blend(before.pose, now.pose, eyes: into, body: into, marks: into)
+        switch m.kind {
+        case .walk: f.pose.lift += 3 * into * abs(sin(2 * .pi * t * 2.2))
+        case .jump:
+            let air = sin(.pi * p)
+            f.pose.sy *= 1 + 0.12 * air; f.pose.sx *= 1 - 0.08 * air
+        case .eat:
+            let bite = into * (0.5 + 0.5 * sin(2 * .pi * t / P.perLetter))
+            f.pose.sy *= 1 - 0.08 * bite; f.pose.sx *= 1 + 0.05 * bite
+        case .wipe: f.pose.tilt += 5 * into * sin(2 * .pi * t * 5)
+        case .rest: break
+        }
+
+        for done in moves[...i] {
+            let x = done.end <= t ? done.to.x : f.at.x
+            switch done.kind {
+            case .eat:
+                for word in lines[done.line].words[done.words] {
+                    let gone = P.letters(of: word).filter { $0.maxX <= x + P.mouth }
+                    guard let last = gone.last else { continue }
+                    f.gone.append((cover(P.Rect(x: word.rect.x, y: word.rect.y, w: last.maxX - word.rect.x, h: word.rect.h)),
+                                   lines[done.line].paper))
+                }
+            case .wipe:
+                let r = lines[done.line].rect
+                if x > r.x { f.gone.append((cover(P.Rect(x: r.x, y: r.y, w: min(x, r.maxX) - r.x, h: r.h)), lines[done.line].paper)) }
+                if done.end > t { f.cloth = P.Rect(x: x - 9, y: r.y - 2, w: 18, h: r.h + 4) }
+            default: break
+            }
+        }
+        return f
+    }
+
+    /// A little more than the letters, so nothing of them shows at the edges.
+    private func cover(_ r: P.Rect) -> P.Rect { P.Rect(x: r.x - 1, y: r.y - 2, w: r.w + 2, h: r.h + 4) }
+
+    /// Which way it faces during a move, and its pose without the motion.
+    private func look(_ m: Move) -> (face: Double, pose: M.Pose) {
+        let way: Double = m.to.x >= m.from.x ? 1 : -1
+        switch m.kind {
+        case .jump: return (way, M.Pose(tilt: 6 * way, gy: -2))
+        case .walk: return (way, M.Pose(tilt: 7 * way))
+        case .eat: return (way, M.Pose(size: 1.12))
+        case .wipe: return (way, M.Pose(tilt: 12 * way, gy: 4, lid: 0.3))
+        case .rest: return (0, M.Pose())
+        }
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/PlayRead.swift

@@ -0,0 +1,98 @@
+import AppKit
+import Vision
+
+/// Play mode's stage (#505): one picture of the screen and the lines of
+/// text on it. The picture is read here, on this Mac, by Apple's Vision
+/// framework; it is kept in memory only and nothing is uploaded.
+///
+/// Taking the picture needs the Screen Recording permission. Without it
+/// macOS gives a picture of the desktop alone rather than an error, so the
+/// permission is checked first.
+enum PlayRead {
+    static var allowed: Bool { CGPreflightScreenCaptureAccess() }
+
+    /// Shows the system's request, once; after that macOS only answers no.
+    static func ask() { CGRequestScreenCaptureAccess() }
+
+    /// What is on `screen` now.
+    static func picture(of screen: NSScreen) -> CGImage? {
+        // Screens are measured from the bottom of the first one; the
+        // picture is asked for from its top.
+        guard let first = NSScreen.screens.first else { return nil }
+        let f = screen.frame
+        let rect = CGRect(x: f.minX, y: first.frame.maxY - f.maxY, width: f.width, height: f.height)
+        return CGWindowListCreateImage(rect, .optionOnScreenOnly, kCGNullWindowID, [.bestResolution])
+    }
+
+    /// The lines of text in `image`, top to bottom, in points of a picture
+    /// `size` across with its origin top-left. Slow (a second or so for a
+    /// whole screen): call it off the main thread.
+    static func lines(in image: CGImage, size: CGSize) -> [PlayMath.Line] {
+        let request = VNRecognizeTextRequest()
+        request.recognitionLevel = .accurate
+        request.recognitionLanguages = ["fr-FR", "en-US"]
+        // Where the letters are matters here, not what they spell.
+        request.usesLanguageCorrection = false
+        guard (try? VNImageRequestHandler(cgImage: image, options: [:]).perform([request])) != nil,
+              let found = request.results else { return [] }
+        let pixels = Pixels(image, size: size)
+
+        var lines: [PlayMath.Line] = []
+        for observation in found {
+            guard let text = observation.topCandidates(1).first else { continue }
+            let s = text.string
+            var words: [PlayMath.Word] = []
+            var i = s.startIndex
+            while i < s.endIndex {
+                guard !s[i].isWhitespace else { i = s.index(after: i); continue }
+                let end = s[i...].firstIndex(where: \.isWhitespace) ?? s.endIndex
+                // Vision's boxes are 0…1 from the bottom-left of the image.
+                if let b = (try? text.boundingBox(for: i..<end))?.boundingBox {
+                    words.append(PlayMath.Word(text: String(s[i..<end]),
+                                               rect: PlayMath.Rect(x: b.minX * size.width, y: (1 - b.maxY) * size.height,
+                                                                   w: b.width * size.width, h: b.height * size.height)))
+                }
+                i = end
+            }
+            guard let first = words.first, let last = words.last else { continue }
+            let top = words.map(\.rect.y).min()!, bottom = words.map(\.rect.maxY).max()!
+            let rect = PlayMath.Rect(x: first.rect.x, y: top, w: last.rect.maxX - first.rect.x, h: bottom - top)
+            lines.append(PlayMath.Line(words: words, rect: rect, paper: pixels?.paper(around: rect) ?? 0xFFFFFF))
+        }
+        return lines.sorted { abs($0.rect.y - $1.rect.y) > 6 ? $0.rect.y < $1.rect.y : $0.rect.x < $1.rect.x }
+    }
+
+    /// The picture at one pixel a point, to read colours from.
+    private struct Pixels {
+        let data: [UInt8]
+        let width, height: Int
+
+        init?(_ image: CGImage, size: CGSize) {
+            let w = Int(size.width), h = Int(size.height)
+            guard w > 0, h > 0 else { return nil }
+            var bytes = [UInt8](repeating: 0, count: w * h * 4)
+            let drawn = bytes.withUnsafeMutableBytes { buffer -> Bool in
+                guard let ctx = CGContext(data: buffer.baseAddress, width: w, height: h, bitsPerComponent: 8,
+                                          bytesPerRow: w * 4, space: CGColorSpace(name: CGColorSpace.sRGB)!,
+                                          bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return false }
+                ctx.draw(image, in: CGRect(x: 0, y: 0, width: w, height: h))
+                return true
+            }
+            guard drawn else { return nil }
+            data = bytes; width = w; height = h
+        }
+
+        /// The page's colour around a line: what most points just above
+        /// and just under it share.
+        func paper(around r: PlayMath.Rect) -> UInt32? {
+            var samples: [UInt32] = []
+            for y in [Int(r.y) - 3, Int(r.maxY) + 3] where y >= 0 && y < height {
+                for x in stride(from: max(Int(r.x), 0), to: min(Int(r.maxX), width), by: 3) {
+                    let i = (y * width + x) * 4
+                    samples.append(UInt32(data[i]) << 16 | UInt32(data[i + 1]) << 8 | UInt32(data[i + 2]))
+                }
+            }
+            return PlayMath.common(samples)
+        }
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/PlayStage.swift

@@ -0,0 +1,164 @@
+import AppKit
+
+/// What play mode (#505) draws over the picture: the page's colour where
+/// words are gone, the cloth, and the character. The context is flipped,
+/// in the picture's points.
+enum PlayDraw {
+    static func draw(_ frame: Play.Frame, in ctx: CGContext, stops: [NSColor]) {
+        for gone in frame.gone {
+            let c = gone.paper
+            ctx.setFillColor(CGColor(srgbRed: CGFloat(c >> 16 & 0xFF) / 255, green: CGFloat(c >> 8 & 0xFF) / 255,
+                                     blue: CGFloat(c & 0xFF) / 255, alpha: 1))
+            ctx.fill(cg(gone.rect))
+        }
+        if let cloth = frame.cloth {
+            let path = CGPath(roundedRect: cg(cloth), cornerWidth: 4, cornerHeight: 4, transform: nil)
+            ctx.addPath(path)
+            ctx.setFillColor(stops[4].cgColor)
+            ctx.fillPath()
+            ctx.addPath(path)
+            ctx.setStrokeColor(stops[1].cgColor)
+            ctx.setLineWidth(1)
+            ctx.strokePath()
+        }
+        let shown = CharacterSim.Frame(pose: frame.pose, x: frame.at.x, face: frame.face, t: frame.t)
+        CharacterDraw.draw(shown, in: ctx, edge: CGPoint(x: frame.at.x, y: frame.at.y), origin: 0,
+                           unit: CGFloat(PlayMath.unit), stops: stops)
+    }
+
+    private static func cg(_ r: PlayMath.Rect) -> CGRect { CGRect(x: r.x, y: r.y, width: r.w, height: r.h) }
+}
+
+/// The stage: a frozen picture of the screen laid over everything, with
+/// the character playing on it. The real page is never touched: nothing
+/// is clicked, typed or changed, and what looks eaten or wiped is only
+/// painted over on the picture.
+///
+/// The page behind goes stale while the picture is up, so play ends on
+/// the first real input: any key, a click, a scroll, another app or Space
+/// coming forward. Unlike the character on its edge, this window does
+/// take keys and clicks, for that one purpose.
+@MainActor
+final class PlayHost {
+    private var window: NSPanel?
+    private var timer: Timer?
+    private var observers: [(NotificationCenter, NSObjectProtocol)] = []
+    private var script: Play?
+    private var began = 0.0
+    /// Raised by each `start`, so a text read that outlives its play can
+    /// tell that the picture now up is not the one it read.
+    private(set) var number = 0
+    /// Told each time a play is over.
+    var onEnd: (() -> Void)?
+
+    var running: Bool { window != nil }
+
+    /// Lay `picture` over `screen`, the character waiting at `foot` (the
+    /// picture's points) until `play(_:)` gives it a script: reading the
+    /// text takes a moment, and the page must not move meanwhile.
+    func start(picture: CGImage, on screen: NSScreen, stops: [NSColor], at foot: PlayMath.Point) {
+        guard !running else { return }
+        script = nil
+        number += 1
+        let panel = PlayWindow(contentRect: screen.frame, styleMask: [.borderless, .nonactivatingPanel],
+                               backing: .buffered, defer: false)
+        panel.level = .screenSaver
+        panel.collectionBehavior = [.fullScreenAuxiliary]
+        panel.hasShadow = false
+        panel.hidesOnDeactivate = false
+
+        let bounds = NSRect(origin: .zero, size: screen.frame.size)
+        let backdrop = NSView(frame: bounds)
+        backdrop.wantsLayer = true
+        backdrop.layer?.contents = picture
+        let view = PlayView(frame: bounds)
+        view.stops = stops
+        view.shown = Play.Frame(at: foot, pose: CharacterMath.pose(.working))
+        view.onInput = { [weak self] in self?.end("a key or a click") }
+        backdrop.addSubview(view)
+        panel.contentView = backdrop
+        panel.setFrame(screen.frame, display: true)
+        window = panel
+        panel.makeKeyAndOrderFront(nil)
+        panel.makeFirstResponder(view)
+
+        func watch(_ center: NotificationCenter, _ name: Notification.Name, _ object: Any?, _ why: String) {
+            observers.append((center, center.addObserver(forName: name, object: object, queue: .main) { [weak self] _ in
+                MainActor.assumeIsolated { self?.end(why) }
+            }))
+        }
+        watch(.default, NSWindow.didResignKeyNotification, panel, "another window came forward")
+        watch(NSWorkspace.shared.notificationCenter, NSWorkspace.activeSpaceDidChangeNotification, nil, "the Space changed")
+        watch(.default, NSApplication.didChangeScreenParametersNotification, nil, "the screens changed")
+
+        let shownAt = ProcessInfo.processInfo.systemUptime
+        let frames = Timer(timeInterval: 1.0 / 30, repeats: true) { [weak self, weak view] _ in
+            MainActor.assumeIsolated {
+                guard let self, let view else { return }
+                let now = ProcessInfo.processInfo.systemUptime
+                if let script = self.script {
+                    view.shown = script.frame(at: now - self.began)
+                } else {
+                    // Its thinking dots hop while it waits.
+                    view.shown.t = now - shownAt
+                }
+                view.needsDisplay = true
+                if view.shown.done { self.end("the script ended") }
+            }
+        }
+        RunLoop.main.add(frames, forMode: .common)
+        timer = frames
+    }
+
+    /// The script to play on the picture that is up. Nothing happens if
+    /// the play has ended meanwhile.
+    func play(_ script: Play) {
+        guard running, self.script == nil else { return }
+        self.script = script
+        began = ProcessInfo.processInfo.systemUptime
+        Log.info("play: started, \(script.lines.count) lines read, \(String(format: "%.1f", script.duration)) s")
+    }
+
+    /// Take the picture away and drop it. Safe to call when no play runs.
+    func end(_ why: String) {
+        guard let panel = window else { return }
+        window = nil
+        timer?.invalidate()
+        timer = nil
+        for (center, observer) in observers { center.removeObserver(observer) }
+        observers = []
+        panel.orderOut(nil)
+        panel.contentView?.layer?.contents = nil
+        panel.contentView = nil
+        Log.info("play: ended, \(why)")
+        onEnd?()
+    }
+}
+
+/// Borderless, and still able to take the keys that end the play.
+private final class PlayWindow: NSPanel {
+    override var canBecomeKey: Bool { true }
+}
+
+/// The drawing over the picture. Every key, click and scroll ends the play.
+private final class PlayView: NSView {
+    var shown = Play.Frame()
+    var stops = CharacterDraw.stops(tint: Brand.accent, colors: nil)
+    var onInput: (() -> Void)?
+
+    override var isFlipped: Bool { true }
+    override var acceptsFirstResponder: Bool { true }
+    override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
+
+    override func draw(_ dirtyRect: NSRect) {
+        guard let ctx = NSGraphicsContext.current?.cgContext else { return }
+        PlayDraw.draw(shown, in: ctx, stops: stops)
+    }
+
+    override func keyDown(with event: NSEvent) { onInput?() }
+    override func performKeyEquivalent(with event: NSEvent) -> Bool { onInput?(); return true }
+    override func mouseDown(with event: NSEvent) { onInput?() }
+    override func rightMouseDown(with event: NSEvent) { onInput?() }
+    override func otherMouseDown(with event: NSEvent) { onInput?() }
+    override func scrollWheel(with event: NSEvent) { onInput?() }
+}

package/apps/mac-voice/Sources/AgentXVoice/StatusMenu.swift

@@ -55,6 +55,9 @@ final class StatusMenu: NSObject, NSMenuDelegate {
     /// Bring a hidden character back, and let its bubble show again.
     var onShowCharacter: (() -> Void)?
     var onSettings: (() -> Void)?
+    /// Play mode (#505): whether a play can start now, and start one.
+    var playReady: () -> Bool = { false }
+    var onPlay: (() -> Void)?
 
     override init() {
         super.init()
@@ -248,6 +251,14 @@ final class StatusMenu: NSObject, NSMenuDelegate {
         let orb = action("Animated orb", #selector(toggleOrb), key: "")
         orb.state = Config.animatedOrb ? .on : .off
         menu.addItem(orb)
+        let mode = action("Play mode", #selector(togglePlayMode), key: "")
+        mode.state = Config.playMode ? .on : .off
+        menu.addItem(mode)
+        if Config.playMode {
+            let play = action("Play on this page", #selector(startPlay), key: "")
+            play.isEnabled = playReady()
+            menu.addItem(play)
+        }
         // The character hides with its bubble, and has no bubble at rest.
         switch characterVisible() {
         case true?: menu.addItem(action("Hide character", #selector(hidePill), key: ""))
@@ -313,6 +324,9 @@ final class StatusMenu: NSObject, NSMenuDelegate {
         onAnimatedOrbChanged?(Config.animatedOrb)
     }
 
+    @objc private func togglePlayMode() { Config.playMode.toggle() }
+    @objc private func startPlay() { onPlay?() }
+
     @objc private func hidePill() { onHidePill?() }
     @objc private func showCharacter() { onShowCharacter?() }
     @objc private func resetPosition() { onResetPosition?() }

package/apps/mac-voice/Sources/AgentXVoice/main.swift

@@ -20,6 +20,8 @@ final class App: NSObject, NSApplicationDelegate {
     private let statusMenu = StatusMenu()
     /// Stands in for the pill's orb when voice.look is "character".
     private let character = CharacterHost()
+    /// The character playing on a frozen picture of the screen (#505).
+    private let play = PlayHost()
     /// The agent whose by-name answer is on screen now, while no turn runs.
     private var asideSpeaker: String?
     private let recorder = Recorder()
@@ -216,6 +218,52 @@ final class App: NSObject, NSApplicationDelegate {
         Log.info("settings: talk \(saved.general.hotkeys.talk), speech to text \(saved.general.stt)")
     }
 
+    /// A play can start: the character shows and may move, and the
+    /// assistant has nothing to say or hear.
+    private var canPlay: Bool {
+        character.onScreen == true && Config.animatedOrb
+            && !NSWorkspace.shared.accessibilityDisplayShouldReduceMotion
+            && !busy && !recorder.isRecording && !play.running
+    }
+
+    /// "Play on this page" (#505): one picture of the character's screen, the
+    /// lines of text read on it, and the character plays the built-in
+    /// script on the picture. The page itself is never touched.
+    private func startPlay() {
+        let foot = character.foot
+        guard canPlay, let screen = NSScreen.screens.first(where: { $0.frame.contains(foot) }) ?? NSScreen.screens.first
+        else { return }
+        guard PlayRead.allowed else {
+            PlayRead.ask()
+            Log.warn("play: no Screen Recording permission; allow AgentX Voice in System Settings, then quit and reopen it")
+            panel.render(.error("Allow Screen Recording"))
+            resetSoon()
+            return
+        }
+        let stops = character.stops, size = screen.frame.size
+        let start = PlayMath.Point(x: foot.x - screen.frame.minX, y: screen.frame.maxY - foot.y)
+        // Off its edge, so it is not in the picture twice.
+        character.setShown(false)
+        Task { @MainActor in
+            try? await Task.sleep(nanoseconds: 150_000_000)
+            guard let picture = PlayRead.picture(of: screen), !busy, !recorder.isRecording else {
+                character.setShown(settings?.general.look == "character")
+                return
+            }
+            play.start(picture: picture, on: screen, stops: stops, at: start)
+            let number = play.number
+            let asked = ProcessInfo.processInfo.systemUptime
+            let lines = await Task.detached { PlayRead.lines(in: picture, size: size) }.value
+            Log.info("play: \(lines.count) lines read in \(Int((ProcessInfo.processInfo.systemUptime - asked) * 1000)) ms")
+            // Ended while its text was read, and maybe started again since:
+            // these lines belong to a picture that is gone.
+            guard play.running, play.number == number else { return }
+            let steps = PlayMath.demo(lines, width: size.width, height: size.height)
+            if steps.isEmpty { play.end("no line of text to play on") }
+            else { play.play(Play(lines: lines, steps: steps, start: start)) }
+        }
+    }
+
     /// Who the pill names and whose colour its orb wears: the agent a
     /// shortcut is asking, a by-name answer being spoken, else the agent
     /// our turn is asking, else the target.
@@ -337,6 +385,14 @@ final class App: NSObject, NSApplicationDelegate {
         }
         panel.onLook = { [weak self] state, tint, colors in
             self?.character.show(state.activity, tint: tint, colors: colors)
+            // The assistant has something to show: play gives way.
+            if state.activity != .idle { self?.play.end("the assistant is busy") }
+        }
+        statusMenu.playReady = { [weak self] in self?.canPlay ?? false }
+        statusMenu.onPlay = { [weak self] in self?.startPlay() }
+        play.onEnd = { [weak self] in
+            guard let self else { return }
+            self.character.setShown(self.settings?.general.look == "character")
         }
         panel.onShown = { [weak self] in self?.character.redraw() }
         panel.onDismiss = { [weak self] in self?.dismissPill() }
@@ -548,6 +604,7 @@ final class App: NSObject, NSApplicationDelegate {
         // A held key overrides any hands-free window that is open, so the
         // two ways of talking never fight over the microphone.
         stopPolling()
+        play.end("the talk key")
         guard !recorder.isRecording else { return }
         // A dismissed pill comes back with the talk key.
         summonPill()

package/apps/mac-voice/build.sh

@@ -18,7 +18,7 @@ rm -rf build && mkdir -p "$BIN" "$APP/Contents/Resources"
 swiftc -O \
   -o "$BIN/AgentXVoice" \
   Sources/AgentXVoice/*.swift \
-  -framework AppKit -framework AVFoundation -framework Carbon -framework CoreML -framework SwiftUI \
+  -framework AppKit -framework AVFoundation -framework Carbon -framework CoreML -framework SwiftUI -framework Vision \
   -target arm64-apple-macosx14.0
 
 # The on-device models from a terminal: fetch them, transcribe a file, see