agentix-cli 0.94.1 → 0.96.0

package/apps/mac-voice/Sources/AgentXVoice/Character.swift

@@ -51,6 +51,10 @@ final class CharacterHost {
     private var rest = CGPoint.zero
     /// Its body on screen: where it can be taken hold of.
     private var body = NSRect.zero
+    /// Its colours, and the point under it in screen points: play mode
+    /// (#505) takes it from there.
+    var stops: [NSColor] { view.stops }
+    var foot: NSPoint { NSPoint(x: CGFloat(sim.x), y: rest.y) }
     /// Carried by the pointer: where the pointer took hold, and where it
     /// stood then, which is not where it rests once it has stepped aside.
     private var carried: (from: NSPoint, rest: CGPoint)?

package/apps/mac-voice/Sources/AgentXVoice/Config.swift

@@ -35,18 +35,35 @@ enum Config {
         set { UserDefaults.standard.set(newValue, forKey: "animatedOrb") }
     }
 
+    /// "Play mode" in the menu (#505): the character may play on a frozen
+    /// picture of the screen when asked. Off by default.
+    static var playMode: Bool {
+        get { UserDefaults.standard.bool(forKey: "playMode") }
+        set { UserDefaults.standard.set(newValue, forKey: "playMode") }
+    }
+
     /// Where the pill was last dragged to (its bottom-left corner, in
     /// screen points). Nil: the default corner.
     static var pillOrigin: CGPoint? {
-        get {
-            guard let xy = UserDefaults.standard.array(forKey: "pillOrigin") as? [Double], xy.count == 2
-            else { return nil }
-            return CGPoint(x: xy[0], y: xy[1])
-        }
-        set {
-            if let p = newValue { UserDefaults.standard.set([Double(p.x), Double(p.y)], forKey: "pillOrigin") }
-            else { UserDefaults.standard.removeObject(forKey: "pillOrigin") }
-        }
+        get { point("pillOrigin") }
+        set { setPoint(newValue, "pillOrigin") }
+    }
+
+    /// Where the pill reduced to its orb was last dragged to. Nil: the
+    /// middle of the bottom edge. Kept apart from the pill's own place.
+    static var orbOrigin: CGPoint? {
+        get { point("orbOrigin") }
+        set { setPoint(newValue, "orbOrigin") }
+    }
+
+    private static func point(_ key: String) -> CGPoint? {
+        guard let xy = UserDefaults.standard.array(forKey: key) as? [Double], xy.count == 2 else { return nil }
+        return CGPoint(x: xy[0], y: xy[1])
+    }
+
+    private static func setPoint(_ p: CGPoint?, _ key: String) {
+        if let p { UserDefaults.standard.set([Double(p.x), Double(p.y)], forKey: key) }
+        else { UserDefaults.standard.removeObject(forKey: key) }
     }
 
     /// Where the character was last dragged to (the point under it, in

package/apps/mac-voice/Sources/AgentXVoice/Panel.swift

@@ -36,6 +36,12 @@ final class Panel: NSPanel {
     let separator = NSBox()
     /// Hidden by close, Esc or the menu until the next talk key.
     var dismissed = false
+    /// Reduced to the orb alone (#457): a small circle with no words and no
+    /// buttons, dragged and remembered apart from the pill. A click, a
+    /// call or an answer to read opens the full pill again.
+    private(set) var reduced = false
+    /// Told when the pill reduces or opens.
+    var onReduced: ((Bool) -> Void)?
     /// Above zero while the app moves or resizes the pill, so only a drag
     /// is remembered.
     var placing = 0
@@ -136,7 +142,10 @@ final class Panel: NSPanel {
             let now = NSEvent.mouseLocation
             let moved = hypot(now.x - start.x, now.y - start.y)
             let onRow = convertToScreen(row.convert(row.bounds, to: nil)).contains(start)
-            if moved < 4 && onRow { onClick?() }
+            // Reduced, a click opens the pill; it does not start talking.
+            if moved < 4 && onRow {
+                if reduced { MainActor.assumeIsolated { setReduced(false) } } else { onClick?() }
+            }
         }
         super.mouseUp(with: event)
     }
@@ -201,6 +210,14 @@ final class Panel: NSPanel {
             }
         }
 
+        /// A call must not be missed behind an orb: it opens the full pill.
+        var opensFullPill: Bool {
+            switch self {
+            case .ringing, .onCall: return true
+            default: return false
+            }
+        }
+
         /// Idle is an invitation and belongs in meta type; everything else
         /// is a running commentary and belongs in body type.
         var isMeta: Bool { if case .idle = self { return true }; return false }
@@ -312,6 +329,9 @@ final class Panel: NSPanel {
         let answerHeight = max(0, bounds.height - h)
         let above = growth.above
         row.frame = NSRect(x: 0, y: above ? 0 : answerHeight, width: bounds.width, height: h)
+        // Reduced, the orb is all there is: in the middle of its circle.
+        orb.frame.origin.x = reduced ? (bounds.width - Self.orbFrame) / 2 : 28 - Self.orbFrame / 2
+        if reduced { return }
         closeButton.frame.origin.x = bounds.width - 28
         // The mini orbs sit between the words and the close button, and
         // the words give up that room while they show.
@@ -345,17 +365,41 @@ final class Panel: NSPanel {
         }
         let screens = NSScreen.screens.map(\.visibleFrame)
         let fallback = (NSScreen.main ?? NSScreen.screens.first)?.visibleFrame ?? .zero
-        place(PillPlacement.clamp(saved: Config.pillOrigin, size: frame.size, screens: screens, fallback: fallback))
+        place(PillPlacement.clamp(saved: reduced ? Config.orbOrigin : Config.pillOrigin, size: frame.size,
+                                  screens: screens, fallback: fallback,
+                                  rest: reduced ? PillPlacement.orbOrigin : PillPlacement.defaultOrigin))
     }
 
-    /// Back to the bottom-right corner, forgetting where it was dragged.
+    /// Back to where it first sat (the pill: the bottom-right corner; the
+    /// orb: the middle of the bottom edge), forgetting where it was dragged.
     /// Not while it is the character's bubble: nothing would move, and
     /// the place kept for the orb look would be lost.
     @MainActor
     func resetPosition() {
         guard bubble == nil else { return }
-        Config.pillOrigin = nil
+        if reduced { Config.orbOrigin = nil } else { Config.pillOrigin = nil }
+        restorePosition()
+    }
+
+    /// Reduce the pill to its orb, or open it again. The two forms keep
+    /// their own places. Nothing to reduce to while the character stands
+    /// in for the orb.
+    @MainActor
+    func setReduced(_ on: Bool) {
+        guard reduced != on, !on || PillMenu.canReduce(showsOrb: showsOrb) else { return }
+        collapse(animated: false)
+        reduced = on
+        clip?.isHidden = on
+        closeButton.isHidden = true
+        miniOrbs.isHidden = on || busyCount == 0
+        (contentView as? Surface)?.shape(circle: on ? PillPlacement.orbSize.height : nil)
+        placing += 1
+        setContentSize(on ? PillPlacement.orbSize : Self.size)
+        placing -= 1
         restorePosition()
+        invalidateShadow()
+        render(current)
+        onReduced?(on)
     }
 
     /// The character's speech bubble (#491): the pill sits above `head`,
@@ -396,6 +440,7 @@ final class Panel: NSPanel {
     @objc private func moved() {
         MainActor.assumeIsolated {
             guard placing == 0 else { return }
+            if reduced { Config.orbOrigin = frame.origin; return }
             Config.pillOrigin = expanded ? collapsedFrame().origin : frame.origin
         }
     }
@@ -427,6 +472,7 @@ final class Panel: NSPanel {
     @MainActor
     func setShowsOrb(_ on: Bool) {
         guard showsOrb != on else { return }
+        if !on { setReduced(false) }
         showsOrb = on
         orb.isHidden = !on
         orb.setOnScreen(on && isVisible)
@@ -451,7 +497,7 @@ final class Panel: NSPanel {
     @objc private func closeClicked() { onDismiss?() }
 
     override func mouseEntered(with event: NSEvent) {
-        closeButton.isHidden = false
+        closeButton.isHidden = reduced
         hovering = true
         MainActor.assumeIsolated { armCollapse() }
     }
@@ -484,6 +530,7 @@ final class Panel: NSPanel {
     /// corruption that traps somewhere unrelated an hour later.
     @MainActor
     func render(_ state: State) {
+        if reduced && state.opensFullPill { setReduced(false) }
         current = state
         // The agent's colour, except where the state is the message: an
         // error, or notifications held.
@@ -514,7 +561,8 @@ final class Panel: NSPanel {
 
         // Grown into an answer, it stays until it collapses, idle or not;
         // so does a pill with agents still busy in its mini orbs.
-        if dismissed || (state.isMeta && !alwaysVisible && !expanded && busyCount == 0) { hide() } else { show() }
+        // Reduced, the orb is the assistant's place on screen: it stays, idle or not.
+        if dismissed || (state.isMeta && !alwaysVisible && !expanded && busyCount == 0 && !reduced) { hide() } else { show() }
         // After showing or hiding: a still character draws its bubble's
         // tail only while the bubble is on screen.
         onLook?(state, tint, colors)
@@ -526,6 +574,7 @@ final class Panel: NSPanel {
     @MainActor
     func showCall(_ mode: CallBar.Mode) {
         guard mode != callBar.mode else { return }
+        if mode != .hidden { setReduced(false) }
         callBar.show(mode)
         if let content = contentView { layoutContent(content.bounds) }
     }
@@ -536,7 +585,7 @@ final class Panel: NSPanel {
     func showBusy(_ orbs: [MiniOrbsModel.Orb], more: Int) {
         let count = orbs.isEmpty ? 0 : orbs.count + more
         miniOrbs.show(orbs, more: more)
-        miniOrbs.isHidden = count == 0
+        miniOrbs.isHidden = count == 0 || reduced
         if count != busyCount {
             busyCount = count
             if let content = contentView { layoutContent(content.bounds) }

package/apps/mac-voice/Sources/AgentXVoice/PanelAnswer.swift

@@ -23,6 +23,8 @@ extension Panel {
     @MainActor
     func grow(answerHeight: CGFloat) {
         guard !dismissed else { return }
+        // An answer to read, or one that asks: the full pill opens for it.
+        setReduced(false)
         let pill = collapsedFrame()
         guard let visible = visibleFrame(for: pill) else { return }
         let height = min(Self.size.height + min(answerHeight, cardMaxHeight), visible.height)

package/apps/mac-voice/Sources/AgentXVoice/PillMenu.swift

@@ -13,4 +13,13 @@ enum PillMenu {
     static func click(showPill: Bool, visible: Bool) -> Bool {
         !isChecked(showPill: showPill, visible: visible)
     }
+
+    /// The item that reduces the pill to its orb, or brings the pill back.
+    static func reduceTitle(reduced: Bool) -> String {
+        reduced ? "Show full pill" : "Reduce to orb"
+    }
+
+    /// Only a pill that has its orb can be reduced to it: while the
+    /// character stands in for the orb, there is nothing to reduce to.
+    static func canReduce(showsOrb: Bool) -> Bool { showsOrb }
 }

package/apps/mac-voice/Sources/AgentXVoice/PillPlacement.swift

@@ -13,13 +13,24 @@ enum PillPlacement {
         CGPoint(x: visible.maxX - size.width - inset, y: visible.minY + inset)
     }
 
+    /// The pill reduced to its orb: a circle as tall as the pill.
+    static let orbSize = CGSize(width: 54, height: 54)
+
+    /// Where the reduced orb first sits: the middle of the bottom edge of
+    /// `visible`, clear of the Dock.
+    static func orbOrigin(size: CGSize, visible: CGRect) -> CGPoint {
+        CGPoint(x: (visible.midX - size.width / 2).rounded(), y: visible.minY + inset)
+    }
+
     /// A saved position made safe: the pill lands fully on the screen it
     /// overlaps most. When it overlaps none (the monitor it was on has been
     /// unplugged, or the saved value is nonsense) it goes back to the
-    /// default corner of `fallback`. Nil `saved`: the default corner.
-    static func clamp(saved: CGPoint?, size: CGSize, screens: [CGRect], fallback: CGRect) -> CGPoint {
+    /// default place on `fallback`. Nil `saved`: the default place, which
+    /// is the bottom-right corner unless `rest` says otherwise.
+    static func clamp(saved: CGPoint?, size: CGSize, screens: [CGRect], fallback: CGRect,
+                      rest defaultOrigin: (CGSize, CGRect) -> CGPoint = PillPlacement.defaultOrigin) -> CGPoint {
         guard let saved, saved.x.isFinite, saved.y.isFinite else {
-            return defaultOrigin(size: size, visible: fallback)
+            return defaultOrigin(size, fallback)
         }
         let frame = CGRect(origin: saved, size: size)
         var best: CGRect?
@@ -31,7 +42,7 @@ enum PillPlacement {
             if area > bestArea { bestArea = area; best = screen }
         }
         guard let screen = best, bestArea > 0 else {
-            return defaultOrigin(size: size, visible: fallback)
+            return defaultOrigin(size, fallback)
         }
         return inside(frame, screen).origin
     }

package/apps/mac-voice/Sources/AgentXVoice/PlayMath.swift

@@ -0,0 +1,250 @@
+import Foundation
+
+/// Play mode (#505): the character leaves the bottom edge and plays on a
+/// frozen picture of the screen. It walks along a line of text as on a
+/// floor, jumps to another line, eats words and wipes a line. These are
+/// the numbers only: where the words are, what it does and when, where it
+/// is at a given moment and which letters are gone. Foundation only, so
+/// the tests can run it without a window.
+///
+/// Everything is in the picture's points, origin top-left, y downwards.
+enum PlayMath {
+    typealias M = CharacterMath
+
+    struct Point: Equatable { var x = 0.0, y = 0.0 }
+
+    struct Rect: Equatable {
+        var x = 0.0, y = 0.0, w = 0.0, h = 0.0
+        var maxX: Double { x + w }
+        var maxY: Double { y + h }
+        var midX: Double { x + w / 2 }
+        var midY: Double { y + h / 2 }
+    }
+
+    struct Word: Equatable {
+        var text: String
+        var rect: Rect
+    }
+
+    /// One line of text as it was read, left to right, and the colour of
+    /// the page around it (0xRRGGBB): what shows where a word is gone.
+    struct Line: Equatable {
+        var words: [Word]
+        var rect: Rect
+        var paper: UInt32 = 0xFFFFFF
+    }
+
+    /// One move of a script.
+    enum Step: Equatable {
+        /// Jump onto the start of a line.
+        case jump(line: Int)
+        /// Walk along the top of a line to its end.
+        case walk(line: Int)
+        /// Go through a line and eat these words, letter by letter.
+        case eat(line: Int, words: Range<Int>)
+        /// Wipe a whole line with a cloth.
+        case wipe(line: Int)
+        /// Stand still, facing you, for so many seconds.
+        case rest(Double)
+    }
+
+    /// Points per drawing unit on the page: a body 40 points across,
+    /// smaller than on the bottom edge so it fits between lines.
+    static let unit = 0.4
+    /// The middle of its body above its feet, and its top, in points.
+    static let middle = (M.Pose().lift + 50) * unit
+    static let top = (M.Pose().lift + 100) * unit + 8
+    /// Points a second, and seconds a letter.
+    static let walkSpeed = 220.0
+    static let wipeSpeed = 260.0
+    static let perLetter = 0.09
+    /// Its mouth, ahead of its middle.
+    static let mouth = 10.0
+
+    /// A word's letters: the word's rectangle cut into equal parts, one a
+    /// character. An estimate, close enough to take them one at a time.
+    static func letters(of word: Word) -> [Rect] {
+        let n = max(word.text.count, 1), w = word.rect.w / Double(n)
+        return (0..<n).map { Rect(x: word.rect.x + Double($0) * w, y: word.rect.y, w: w, h: word.rect.h) }
+    }
+
+    /// The colour most of `samples` share: the page around a line. Close
+    /// shades count as one, so the soft edge of a letter does not win.
+    static func common(_ samples: [UInt32]) -> UInt32? {
+        var count: [UInt32: Int] = [:]
+        for s in samples { count[s & 0xF8F8F8, default: 0] += 1 }
+        guard let best = count.max(by: { ($0.value, $0.key) < ($1.value, $1.key) })?.key else { return nil }
+        return samples.first { $0 & 0xF8F8F8 == best }
+    }
+
+    /// The built-in script: the line nearest the middle of the page and
+    /// the two under it in the same column. It walks the first, eats the
+    /// start of the second and wipes the third. Lines too close to the top
+    /// to stand on, too short or of one word are left alone.
+    static func demo(_ lines: [Line], width: Double, height: Double) -> [Step] {
+        let fit = lines.indices.filter { lines[$0].words.count >= 2 && lines[$0].rect.w >= 120 && lines[$0].rect.y >= top + 40 }
+        func far(_ i: Int) -> Double { hypot(lines[i].rect.midX - width / 2, lines[i].rect.midY - height / 2) }
+        guard let first = fit.min(by: { far($0) < far($1) }) else { return [] }
+        let a = lines[first].rect
+        let under = fit.filter { lines[$0].rect.y > a.y + a.h / 2 && lines[$0].rect.x < a.maxX && lines[$0].rect.maxX > a.x }
+            .sorted { lines[$0].rect.y < lines[$1].rect.y }
+        let picked = [first] + under.prefix(2)
+        var steps: [Step] = [.jump(line: first), .walk(line: first)]
+        if picked.count > 1 { steps += [.eat(line: picked[1], words: 0..<min(3, lines[picked[1]].words.count))] }
+        else { steps += [.eat(line: first, words: 0..<min(2, lines[first].words.count))] }
+        if picked.count > 2 { steps += [.wipe(line: picked[2])] }
+        return steps + [.rest(1.2)]
+    }
+}
+
+/// A script laid out in time: every move has its start and its end, so
+/// where the character is, and what is gone, depends on the time alone.
+/// The same script gives the same frames on every run.
+struct Play {
+    typealias P = PlayMath
+    typealias M = CharacterMath
+
+    enum Kind { case jump, walk, eat, wipe, rest }
+
+    struct Move {
+        var kind: Kind
+        var from, to: P.Point
+        var start, duration: Double
+        var line = 0
+        var words = 0..<0
+        var end: Double { start + duration }
+    }
+
+    struct Frame {
+        /// Its feet.
+        var at = P.Point()
+        /// -1 turned left, 0 facing you, 1 turned right.
+        var face = 0.0
+        var pose = M.Pose()
+        var t = 0.0
+        /// What is covered with the page's colour, and which colour.
+        var gone: [(rect: P.Rect, paper: UInt32)] = []
+        /// The cloth, while it wipes.
+        var cloth: P.Rect?
+        var done = false
+    }
+
+    let lines: [P.Line]
+    private(set) var moves: [Move] = []
+    var duration: Double { moves.last?.end ?? 0 }
+
+    /// `start` is where its feet are when play begins. Steps that name a
+    /// line or a word that is not there are skipped.
+    init(lines: [P.Line], steps: [P.Step], start: P.Point) {
+        self.lines = lines
+        var at = start, now = 0.0
+        func add(_ kind: Kind, to: P.Point, _ seconds: Double, line: Int = 0, words: Range<Int> = 0..<0) -> Move {
+            Move(kind: kind, from: at, to: to, start: now, duration: seconds, line: line, words: words)
+        }
+        /// Get to `to` first: a jump, unless it is already there.
+        func reach(_ to: P.Point, into list: inout [Move]) {
+            guard at != to else { return }
+            let m = add(.jump, to: to, 0.5 + hypot(to.x - at.x, to.y - at.y) / 900)
+            list.append(m); at = to; now = m.end
+        }
+        var list: [Move] = []
+        for step in steps {
+            switch step {
+            case .jump(let i):
+                guard lines.indices.contains(i) else { continue }
+                reach(P.Point(x: lines[i].rect.x, y: lines[i].rect.y), into: &list)
+            case .walk(let i):
+                guard lines.indices.contains(i) else { continue }
+                let r = lines[i].rect
+                if at.y != r.y || at.x < r.x || at.x > r.maxX { reach(P.Point(x: r.x, y: r.y), into: &list) }
+                let m = add(.walk, to: P.Point(x: r.maxX, y: r.y), (r.maxX - at.x) / P.walkSpeed, line: i)
+                list.append(m); at = m.to; now = m.end
+            case .eat(let i, let words):
+                guard lines.indices.contains(i), !words.isEmpty, words.upperBound <= lines[i].words.count else { continue }
+                // In the line, its middle at the middle of the letters.
+                let y = lines[i].rect.midY + P.middle
+                let first = lines[i].words[words.lowerBound].rect, last = lines[i].words[words.upperBound - 1].rect
+                reach(P.Point(x: first.x - P.mouth - 4, y: y), into: &list)
+                let count = lines[i].words[words].reduce(0) { $0 + max($1.text.count, 1) }
+                let m = add(.eat, to: P.Point(x: last.maxX, y: y), Double(count) * P.perLetter, line: i, words: words)
+                list.append(m); at = m.to; now = m.end
+            case .wipe(let i):
+                guard lines.indices.contains(i) else { continue }
+                let r = lines[i].rect
+                reach(P.Point(x: r.x, y: r.y), into: &list)
+                let m = add(.wipe, to: P.Point(x: r.maxX, y: r.y), r.w / P.wipeSpeed, line: i)
+                list.append(m); at = m.to; now = m.end
+            case .rest(let seconds):
+                let m = add(.rest, to: at, max(seconds, 0))
+                list.append(m); now = m.end
+            }
+        }
+        moves = list
+    }
+
+    func frame(at t: Double) -> Frame {
+        guard let i = moves.lastIndex(where: { $0.start <= t }) else { return Frame(done: true) }
+        let m = moves[i], p = M.span(t, m.start, m.end)
+        var f = Frame(t: t, done: t >= duration)
+
+        f.at = P.Point(x: m.from.x + (m.to.x - m.from.x) * p, y: m.from.y + (m.to.y - m.from.y) * p)
+        if m.kind == .jump {
+            // An arc over both ends, kept inside the picture.
+            let rise = 30 + hypot(m.to.x - m.from.x, m.to.y - m.from.y) * 0.12
+            f.at.y -= min(rise, max(min(m.from.y, m.to.y) - P.top, 0)) * 4 * p * (1 - p)
+        }
+
+        // The look of this move, reached from the look of the one before
+        // in a fifth of a second: the eyes and the body do not snap.
+        let into = M.ease(M.span(t, m.start, m.start + 0.2))
+        let before = i > 0 ? look(moves[i - 1]) : look(m)
+        let now = look(m)
+        f.face = before.face + (now.face - before.face) * into
+        f.pose = M.blend(before.pose, now.pose, eyes: into, body: into, marks: into)
+        switch m.kind {
+        case .walk: f.pose.lift += 3 * into * abs(sin(2 * .pi * t * 2.2))
+        case .jump:
+            let air = sin(.pi * p)
+            f.pose.sy *= 1 + 0.12 * air; f.pose.sx *= 1 - 0.08 * air
+        case .eat:
+            let bite = into * (0.5 + 0.5 * sin(2 * .pi * t / P.perLetter))
+            f.pose.sy *= 1 - 0.08 * bite; f.pose.sx *= 1 + 0.05 * bite
+        case .wipe: f.pose.tilt += 5 * into * sin(2 * .pi * t * 5)
+        case .rest: break
+        }
+
+        for done in moves[...i] {
+            let x = done.end <= t ? done.to.x : f.at.x
+            switch done.kind {
+            case .eat:
+                for word in lines[done.line].words[done.words] {
+                    let gone = P.letters(of: word).filter { $0.maxX <= x + P.mouth }
+                    guard let last = gone.last else { continue }
+                    f.gone.append((cover(P.Rect(x: word.rect.x, y: word.rect.y, w: last.maxX - word.rect.x, h: word.rect.h)),
+                                   lines[done.line].paper))
+                }
+            case .wipe:
+                let r = lines[done.line].rect
+                if x > r.x { f.gone.append((cover(P.Rect(x: r.x, y: r.y, w: min(x, r.maxX) - r.x, h: r.h)), lines[done.line].paper)) }
+                if done.end > t { f.cloth = P.Rect(x: x - 9, y: r.y - 2, w: 18, h: r.h + 4) }
+            default: break
+            }
+        }
+        return f
+    }
+
+    /// A little more than the letters, so nothing of them shows at the edges.
+    private func cover(_ r: P.Rect) -> P.Rect { P.Rect(x: r.x - 1, y: r.y - 2, w: r.w + 2, h: r.h + 4) }
+
+    /// Which way it faces during a move, and its pose without the motion.
+    private func look(_ m: Move) -> (face: Double, pose: M.Pose) {
+        let way: Double = m.to.x >= m.from.x ? 1 : -1
+        switch m.kind {
+        case .jump: return (way, M.Pose(tilt: 6 * way, gy: -2))
+        case .walk: return (way, M.Pose(tilt: 7 * way))
+        case .eat: return (way, M.Pose(size: 1.12))
+        case .wipe: return (way, M.Pose(tilt: 12 * way, gy: 4, lid: 0.3))
+        case .rest: return (0, M.Pose())
+        }
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/PlayRead.swift

@@ -0,0 +1,98 @@
+import AppKit
+import Vision
+
+/// Play mode's stage (#505): one picture of the screen and the lines of
+/// text on it. The picture is read here, on this Mac, by Apple's Vision
+/// framework; it is kept in memory only and nothing is uploaded.
+///
+/// Taking the picture needs the Screen Recording permission. Without it
+/// macOS gives a picture of the desktop alone rather than an error, so the
+/// permission is checked first.
+enum PlayRead {
+    static var allowed: Bool { CGPreflightScreenCaptureAccess() }
+
+    /// Shows the system's request, once; after that macOS only answers no.
+    static func ask() { CGRequestScreenCaptureAccess() }
+
+    /// What is on `screen` now.
+    static func picture(of screen: NSScreen) -> CGImage? {
+        // Screens are measured from the bottom of the first one; the
+        // picture is asked for from its top.
+        guard let first = NSScreen.screens.first else { return nil }
+        let f = screen.frame
+        let rect = CGRect(x: f.minX, y: first.frame.maxY - f.maxY, width: f.width, height: f.height)
+        return CGWindowListCreateImage(rect, .optionOnScreenOnly, kCGNullWindowID, [.bestResolution])
+    }
+
+    /// The lines of text in `image`, top to bottom, in points of a picture
+    /// `size` across with its origin top-left. Slow (a second or so for a
+    /// whole screen): call it off the main thread.
+    static func lines(in image: CGImage, size: CGSize) -> [PlayMath.Line] {
+        let request = VNRecognizeTextRequest()
+        request.recognitionLevel = .accurate
+        request.recognitionLanguages = ["fr-FR", "en-US"]
+        // Where the letters are matters here, not what they spell.
+        request.usesLanguageCorrection = false
+        guard (try? VNImageRequestHandler(cgImage: image, options: [:]).perform([request])) != nil,
+              let found = request.results else { return [] }
+        let pixels = Pixels(image, size: size)
+
+        var lines: [PlayMath.Line] = []
+        for observation in found {
+            guard let text = observation.topCandidates(1).first else { continue }
+            let s = text.string
+            var words: [PlayMath.Word] = []
+            var i = s.startIndex
+            while i < s.endIndex {
+                guard !s[i].isWhitespace else { i = s.index(after: i); continue }
+                let end = s[i...].firstIndex(where: \.isWhitespace) ?? s.endIndex
+                // Vision's boxes are 0…1 from the bottom-left of the image.
+                if let b = (try? text.boundingBox(for: i..<end))?.boundingBox {
+                    words.append(PlayMath.Word(text: String(s[i..<end]),
+                                               rect: PlayMath.Rect(x: b.minX * size.width, y: (1 - b.maxY) * size.height,
+                                                                   w: b.width * size.width, h: b.height * size.height)))
+                }
+                i = end
+            }
+            guard let first = words.first, let last = words.last else { continue }
+            let top = words.map(\.rect.y).min()!, bottom = words.map(\.rect.maxY).max()!
+            let rect = PlayMath.Rect(x: first.rect.x, y: top, w: last.rect.maxX - first.rect.x, h: bottom - top)
+            lines.append(PlayMath.Line(words: words, rect: rect, paper: pixels?.paper(around: rect) ?? 0xFFFFFF))
+        }
+        return lines.sorted { abs($0.rect.y - $1.rect.y) > 6 ? $0.rect.y < $1.rect.y : $0.rect.x < $1.rect.x }
+    }
+
+    /// The picture at one pixel a point, to read colours from.
+    private struct Pixels {
+        let data: [UInt8]
+        let width, height: Int
+
+        init?(_ image: CGImage, size: CGSize) {
+            let w = Int(size.width), h = Int(size.height)
+            guard w > 0, h > 0 else { return nil }
+            var bytes = [UInt8](repeating: 0, count: w * h * 4)
+            let drawn = bytes.withUnsafeMutableBytes { buffer -> Bool in
+                guard let ctx = CGContext(data: buffer.baseAddress, width: w, height: h, bitsPerComponent: 8,
+                                          bytesPerRow: w * 4, space: CGColorSpace(name: CGColorSpace.sRGB)!,
+                                          bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue) else { return false }
+                ctx.draw(image, in: CGRect(x: 0, y: 0, width: w, height: h))
+                return true
+            }
+            guard drawn else { return nil }
+            data = bytes; width = w; height = h
+        }
+
+        /// The page's colour around a line: what most points just above
+        /// and just under it share.
+        func paper(around r: PlayMath.Rect) -> UInt32? {
+            var samples: [UInt32] = []
+            for y in [Int(r.y) - 3, Int(r.maxY) + 3] where y >= 0 && y < height {
+                for x in stride(from: max(Int(r.x), 0), to: min(Int(r.maxX), width), by: 3) {
+                    let i = (y * width + x) * 4
+                    samples.append(UInt32(data[i]) << 16 | UInt32(data[i + 1]) << 8 | UInt32(data[i + 2]))
+                }
+            }
+            return PlayMath.common(samples)
+        }
+    }
+}