agentix-cli 0.98.0 → 0.100.0

package/apps/mac-voice/Sources/AgentXVoice/Guide.swift

@@ -0,0 +1,185 @@
+import AppKit
+
+/// Guiding (#482): the daemon's /voice/guide route, the wait on it, and
+/// the mark the character leaves on what it shows. Where it stands is in
+/// GuideMath.swift; Character.swift moves it there.
+enum GuideClient {
+    struct Command: Decodable {
+        struct Rect: Decodable { let x, y, width, height: Double }
+        let seq: Int
+        /// Nil: back to where it rests.
+        let rect: Rect?
+        let mark: String
+    }
+
+    /// GET /voice/guide: the command after `seq`, as soon as there is
+    /// one; the current one after 25 s; nil when the daemon cannot be
+    /// reached or is too old to know the route.
+    static func next(after seq: Int) async -> Command? {
+        guard let url = URL(string: "\(Config.daemonURL)/voice/guide?after=\(seq)") else { return nil }
+        var req = URLRequest(url: url)
+        req.timeoutInterval = 35
+        guard let (data, response) = try? await URLSession.shared.data(for: req),
+              (response as? HTTPURLResponse)?.statusCode == 200 else { return nil }
+        return try? JSONDecoder().decode(Command.self, from: data)
+    }
+}
+
+/// Waits on the daemon while the character is on screen. The wait is
+/// also how the daemon knows there is a character to send.
+@MainActor
+final class GuideWatcher {
+    private var seq = 0
+    private var away = false
+    private var task: Task<Void, Never>?
+    /// The character is on screen.
+    var wanted: (() -> Bool)?
+    /// Show this, in AppKit coordinates, or go home.
+    var onCommand: ((NSRect?, GuideMath.Mark) -> Void)?
+
+    /// (Re)start the wait. Called again when the character comes on screen
+    /// or leaves it, so a wait already open is ended: cancelling it closes
+    /// its socket, and the daemon stops counting a character as there.
+    func start() {
+        task?.cancel()
+        task = Task { [weak self] in
+            while !Task.isCancelled {
+                guard let self else { return }
+                await self.turn()
+            }
+        }
+    }
+
+    private func turn() async {
+        guard wanted?() == true, let command = await GuideClient.next(after: seq) else {
+            // Hidden, or the daemon is gone: nothing is left on screen.
+            if away { away = false; onCommand?(nil, .none) }
+            try? await Task.sleep(nanoseconds: 2_000_000_000)
+            return
+        }
+        guard command.seq != seq else { return }
+        seq = command.seq
+        let rect = command.rect.map {
+            GuideMath.toAppKit(CGRect(x: $0.x, y: $0.y, width: $0.width, height: $0.height),
+                               primaryHeight: NSScreen.screens.first?.frame.maxY ?? 0)
+        }
+        away = rect != nil
+        Log.info("guide: \(rect.map { "to \(Int($0.minX)),\(Int($0.minY)) \(Int($0.width))x\(Int($0.height)) \(command.mark)" } ?? "home")")
+        onCommand?(rect, GuideMath.Mark(rawValue: command.mark) ?? .box)
+    }
+}
+
+/// What the character was sent to show (#482): it stands beside it until
+/// it is sent home, taken hold of or hidden; its mark; its glide there and back.
+@MainActor
+final class CharacterGuide {
+    private(set) var showing: NSRect?
+    private let mark = GuideMark()
+    /// On its way up or down, to what it shows or back; and the last frame.
+    private var gliding = false
+    private var ticked = 0.0
+
+    func show(_ rect: NSRect, _ kind: GuideMath.Mark, color: NSColor, animated: Bool) {
+        showing = rect
+        gliding = true
+        mark.show(kind, around: rect, color: color, animated: animated)
+    }
+
+    /// Back to where it rests, gliding there unless it was taken hold of.
+    func end(glide: Bool = true) {
+        gliding = glide && showing != nil
+        showing = nil
+        mark.hide()
+    }
+
+    /// Where it stands beside what it shows, on that thing's screen.
+    func stand(screens: [NSScreen], body: CGFloat, tall: CGFloat) -> CGPoint? {
+        guard let showing, let first = screens.first else { return nil }
+        let middle = CGPoint(x: showing.midX, y: showing.midY)
+        return GuideMath.stand(beside: showing, in: (screens.first { $0.frame.contains(middle) } ?? first).visibleFrame,
+                               body: body, tall: tall, inset: PillPlacement.characterInset)
+    }
+
+    /// Its height this frame: sent somewhere or back, it glides up or down
+    /// to `y`; otherwise it is there at once.
+    func height(from: CGFloat, toward y: CGFloat, now: Double, still: Bool) -> CGFloat {
+        let h = gliding && !still ? GuideMath.glide(from, toward: y, dt: now - ticked) : y
+        gliding = h != y
+        ticked = now
+        return h
+    }
+}
+
+/// The mark on what the character shows: drawn on its own click-through
+/// window over the page, which is never touched.
+@MainActor
+final class GuideMark {
+    /// Room around the outline for the stroke.
+    private static let margin: CGFloat = 6
+    private let window: NSPanel
+    private let shape = CAShapeLayer()
+
+    init() {
+        window = NSPanel(contentRect: .zero, styleMask: [.borderless, .nonactivatingPanel], backing: .buffered, defer: false)
+        window.level = .floating
+        window.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .stationary]
+        window.isOpaque = false
+        window.backgroundColor = .clear
+        window.hasShadow = false
+        window.hidesOnDeactivate = false
+        window.ignoresMouseEvents = true
+        let view = NSView()
+        view.wantsLayer = true
+        view.layer?.addSublayer(shape)
+        window.contentView = view
+        shape.lineWidth = 3
+        shape.lineCap = .round
+        shape.lineJoin = .round
+    }
+
+    /// Draw `mark` around `rect`. Animated, it is drawn on once the
+    /// character has had time to get there.
+    func show(_ mark: GuideMath.Mark, around rect: NSRect, color: NSColor, animated: Bool) {
+        // Kept to the screen it is on, whatever size it was sent.
+        let screen = NSScreen.screens.first { $0.frame.intersects(rect) }?.frame ?? .null
+        let outline = GuideMath.outline(mark, around: rect).intersection(screen)
+        guard !outline.isNull, !outline.isEmpty else { return hide() }
+        let frame = outline.insetBy(dx: -Self.margin, dy: -Self.margin)
+        window.setFrame(frame, display: false)
+        let inner = CGRect(x: Self.margin, y: Self.margin, width: outline.width, height: outline.height)
+        let path = CGMutablePath()
+        switch mark {
+        case .circle: path.addEllipse(in: inner)
+        case .underline: path.move(to: inner.origin); path.addLine(to: CGPoint(x: inner.maxX, y: inner.minY))
+        default: path.addRoundedRect(in: inner, cornerWidth: min(6, inner.width / 2), cornerHeight: min(6, inner.height / 2))
+        }
+        shape.frame = CGRect(origin: .zero, size: frame.size)
+        shape.path = path
+        shape.strokeColor = color.cgColor
+        shape.fillColor = mark == .box ? color.withAlphaComponent(0.12).cgColor : nil
+        shape.removeAllAnimations()
+        if animated {
+            let draw = CABasicAnimation(keyPath: "strokeEnd")
+            draw.fromValue = 0
+            draw.toValue = 1
+            draw.beginTime = CACurrentMediaTime() + 0.35
+            draw.duration = 0.45
+            draw.fillMode = .backwards
+            draw.timingFunction = CAMediaTimingFunction(name: .easeInEaseOut)
+            shape.add(draw, forKey: "draw")
+            let appear = CABasicAnimation(keyPath: "opacity")
+            appear.fromValue = 0
+            appear.toValue = 1
+            appear.beginTime = draw.beginTime
+            appear.duration = 0.15
+            appear.fillMode = .backwards
+            shape.add(appear, forKey: "appear")
+        }
+        window.orderFrontRegardless()
+    }
+
+    func hide() {
+        shape.removeAllAnimations()
+        window.orderOut(nil)
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/GuideMath.swift

@@ -0,0 +1,49 @@
+import CoreGraphics
+import Foundation
+
+/// Guiding (#482): the answering agent sends the character to something
+/// on screen. Where it stands to show it, and the mark it leaves there.
+/// Kept free of AppKit so the tests can check it without a window.
+/// Rectangles are in screen coordinates, origin bottom-left, as AppKit
+/// uses them, once toAppKit has turned them.
+enum GuideMath {
+    enum Mark: String {
+        case box, circle, underline, none
+    }
+
+    /// Accessibility → AppKit: the daemon's rectangles are measured from
+    /// the top of the primary display, whichever screen they are on.
+    static func toAppKit(_ r: CGRect, primaryHeight: CGFloat) -> CGRect {
+        CGRect(x: r.minX, y: primaryHeight - r.maxY, width: r.width, height: r.height)
+    }
+
+    /// The point under the character while it shows `rect`: beside it,
+    /// its body level with the middle. On the left, away from which its
+    /// bubble reaches, so the bubble does not cover what it shows; on the
+    /// right when the left has no room in `visible`.
+    static func stand(beside rect: CGRect, in visible: CGRect, body: CGFloat, tall: CGFloat,
+                      inset: CGFloat, gap: CGFloat = 14) -> CGPoint {
+        let left = rect.minX - gap - body / 2
+        let x = left >= visible.minX + inset ? left : rect.maxX + gap + body / 2
+        return CGPoint(x: x, y: rect.midY - tall / 2)
+    }
+
+    /// What the mark is drawn along, around `rect`: a box a little wider
+    /// than it, an oval that clears its corners, or a line under it.
+    static func outline(_ mark: Mark, around rect: CGRect) -> CGRect {
+        switch mark {
+        case .box: return rect.insetBy(dx: -5, dy: -5)
+        // An oval through a rectangle's corners is √2 times its size.
+        case .circle: return rect.insetBy(dx: -(rect.width * 0.21 + 6), dy: -(rect.height * 0.21 + 6))
+        case .underline: return CGRect(x: rect.minX - 2, y: rect.minY - 5, width: rect.width + 4, height: 0)
+        case .none: return .null
+        }
+    }
+
+    /// One frame of its way up or down to where it is sent: most of the
+    /// way in a third of a second, and there once it is within a point.
+    static func glide(_ y: CGFloat, toward goal: CGFloat, dt: Double) -> CGFloat {
+        let next = y + (goal - y) * CGFloat(1 - exp(-7 * min(max(dt, 0), 0.1)))
+        return abs(goal - next) < 1 ? goal : next
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/Meets.swift

@@ -0,0 +1,39 @@
+import CoreGraphics
+import Foundation
+
+/// What the character meets on the live screen (#539): the sides of the
+/// windows that stand on its line. Read from the window list, which gives
+/// each window's rectangle and no picture, title or text, so it needs no
+/// permission.
+enum Meets {
+    /// A window smaller than this is not something to stop at.
+    static let least = CGSize(width: 80, height: 40)
+
+    /// Where the sides of `windows`, front to back, cross `band`: each
+    /// side only where no window in front covers it. One set of
+    /// coordinates for both.
+    static func edges(of windows: [CGRect], band: ClosedRange<CGFloat>) -> [Double] {
+        var found: [Double] = []
+        for (i, window) in windows.enumerated() {
+            let low = max(window.minY, band.lowerBound), high = min(window.maxY, band.upperBound)
+            guard window.width >= least.width, window.height >= least.height, low < high else { continue }
+            let y = (low + high) / 2
+            for x in [window.minX, window.maxX] {
+                let covered = windows[..<i].contains { $0.minX < x && x < $0.maxX && $0.minY <= y && y < $0.maxY }
+                if !covered { found.append(Double(x)) }
+            }
+        }
+        return found
+    }
+
+    /// The ordinary windows on screen, front to back, measured from the
+    /// top left of the main screen.
+    static func windows() -> [CGRect] {
+        let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? []
+        return list.compactMap { window in
+            guard window[kCGWindowLayer as String] as? Int == 0, window[kCGWindowAlpha as String] as? Double != 0,
+                  let bounds = window[kCGWindowBounds as String] else { return nil }
+            return CGRect(dictionaryRepresentation: bounds as! CFDictionary)
+        }
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/Panel.swift

@@ -570,6 +570,15 @@ final class Panel: NSPanel {
         onRender?(state)
     }
 
+    /// On screen with something to use: an answer, an error or the call
+    /// buttons. The character does not play with the pointer then (#537).
+    @MainActor
+    var holdsSomething: Bool {
+        guard isVisible else { return false }
+        if case .error = current { return true }
+        return expanded || callBar.mode != .hidden
+    }
+
     /// The call buttons for this moment of a call; `.hidden` outside one.
     @MainActor
     func showCall(_ mode: CallBar.Mode) {

package/apps/mac-voice/Sources/AgentXVoice/PointerPlay.swift

@@ -0,0 +1,137 @@
+import Foundation
+
+/// Play with the pointer (#505): with "Play mode" ticked, an idle
+/// character does more than step aside. When the pointer moves within
+/// its sight it starts a game: it follows it along the edge, crouches
+/// and jumps at it like a cat, or runs away from it. Between games it
+/// rests, and the usual rules hold. A click in its sight makes it jump.
+///
+/// It only decides where the character wants to be and how it crouches
+/// and hops; `CharacterSim` moves it. Foundation only, and the same
+/// games in the same order on every run, so the tests can count on them.
+struct PointerPlay {
+    typealias M = CharacterMath
+
+    enum Game: Equatable { case follow, pounce, flee }
+
+    struct Out: Equatable {
+        /// Where it wants to be along the edge. Nil: no game, the usual rules.
+        var target: Double?
+        /// 0…1 crouched, and the height of a hop in drawing units.
+        var crouch = 0.0
+        var hop = 0.0
+        /// It came down on the pointer just now.
+        var caught = false
+    }
+
+    /// How far along the edge and above it the pointer is seen, in points.
+    static let sight = (x: 520.0, y: 360.0)
+    /// A pointer slower than this, in points a second, starts no game.
+    static let stir = 80.0
+    /// Following, it stops this short of the pointer.
+    static let gap = 70.0
+    /// It jumps at a pointer no further than this, and no higher.
+    static let leap = (x: 220.0, y: 150.0)
+    /// It has caught a pointer this close to where it comes down.
+    static let grip = 45.0
+    /// Running away, it puts this much between the pointer and itself.
+    static let far = 320.0
+    /// Seconds: the crouch before a jump, the jump, running away, and
+    /// the jump a click gives it.
+    static let wind = 0.5, air = 0.6, flight = 2.5, start = 0.35
+    /// Drawing units: how high it jumps at the pointer, and at a click.
+    static let high = 70.0, jolt = 34.0
+
+    private(set) var game: Game?
+    private(set) var games = 0
+    private var began = 0.0, until = 0.0, next = 0.0
+    private var jumpTo: Double?
+    private var seen: (x: Double, y: Double)?
+    private var down = false
+    private var jolted: Double?
+
+    /// One frame. `x` is where the character is; `pointer` along the edge
+    /// and above it, nil on another screen; `down` the mouse button.
+    mutating func step(_ now: Double, _ dt: Double, x: Double, pointer: (x: Double, y: Double)?,
+                       down: Bool, range: ClosedRange<Double>) -> Out {
+        let clicked = down && !self.down
+        self.down = down
+        let before = seen
+        seen = pointer
+        guard let p = pointer, abs(p.x - x) < Self.sight.x, abs(p.y) < Self.sight.y else {
+            if game != nil { end(now) }
+            return Out(hop: hop(now))
+        }
+        func inside(_ v: Double) -> Double { min(max(v, range.lowerBound), range.upperBound) }
+
+        // A click startles it: a small jump, and the game is over.
+        if clicked && jumpTo == nil {
+            jolted = now
+            if game != nil { end(now) }
+        }
+
+        if game == nil, now >= next, jolted == nil, let was = before, dt > 0,
+           hypot(p.x - was.x, p.y - was.y) / dt > Self.stir {
+            games += 1
+            let pick = M.rnd(games, 7)
+            let reach = abs(p.x - x) < Self.leap.x && abs(p.y) < Self.leap.y
+            game = pick < 0.5 ? .follow : pick < 0.8 ? (reach ? .pounce : .follow) : .flee
+            began = now
+            jumpTo = nil
+            until = now + (game == .follow ? 4 + 3 * M.rnd(games, 2) : game == .flee ? Self.flight : Self.wind + Self.air)
+        }
+
+        var out = Out(hop: hop(now))
+        switch game {
+        case nil:
+            break
+        case .follow:
+            // On the side it is on, a little short of the pointer.
+            out.target = inside(p.x + (x >= p.x ? Self.gap : -Self.gap))
+        case .pounce:
+            let t = now - began
+            if t < Self.wind {
+                out.target = x
+                out.crouch = M.ease(t / (Self.wind * 0.8))
+            } else {
+                // It jumps at where the pointer was when it let go.
+                if jumpTo == nil { jumpTo = inside(p.x) }
+                let a = M.span(t, Self.wind, Self.wind + Self.air)
+                out.target = jumpTo
+                out.crouch = 1 - M.ease(a / 0.25)
+                out.hop = Self.high * sin(.pi * a)
+            }
+        case .flee:
+            out.target = abs(p.x - x) < Self.far ? M.aside(x: x, pointer: p.x, clear: Self.far, range: range) : x
+        }
+        if game != nil && now >= until {
+            out.caught = game == .pounce && abs(p.x - x) < Self.grip && abs(p.y) < Self.leap.y
+            end(now)
+        }
+        return out
+    }
+
+    /// The assistant has work, or play is off: no game, and none at once
+    /// after. The button is still followed, so one held through it is no click.
+    mutating func stop(_ now: Double, down: Bool) {
+        if game != nil { end(now) }
+        self.down = down
+        jolted = nil
+        seen = nil
+    }
+
+    /// A game is over: a rest of five to thirteen seconds before the next.
+    private mutating func end(_ now: Double) {
+        game = nil
+        jumpTo = nil
+        next = now + 5 + 8 * M.rnd(games, 3)
+    }
+
+    /// The jump a click gives it.
+    private mutating func hop(_ now: Double) -> Double {
+        guard let at = jolted else { return 0 }
+        let a = (now - at) / Self.start
+        if a >= 1 { jolted = nil; return 0 }
+        return Self.jolt * sin(.pi * a)
+    }
+}

package/apps/mac-voice/Sources/AgentXVoice/main.swift

@@ -85,6 +85,7 @@ final class App: NSObject, NSApplicationDelegate {
 
     // An agent ringing the owner (#321).
     private let callWatcher = CallWatcher()
+    private let guideWatcher = GuideWatcher()
     private let ringer = Ringer()
     /// The call ringing on the pill now.
     private var ringingCall: IncomingCall?
@@ -392,6 +393,7 @@ final class App: NSObject, NSApplicationDelegate {
             panel.attach(head: head, visible: visible)
             return panel.isVisible ? panel.frame : nil
         }
+        character.bubbleShows = { [weak self] in self?.panel.holdsSomething ?? false }
         panel.onLook = { [weak self] state, tint, colors in
             self?.character.show(state.activity, tint: tint, colors: colors)
             // The assistant has something to show: play gives way.
@@ -507,6 +509,11 @@ final class App: NSObject, NSApplicationDelegate {
             return CallModel.pollQuery(ringing: self.ringingCall?.id ?? self.answeringCall, canRing: self.canRing)
         }
         callWatcher.start()
+        // The answering agent sends the character to something (#482).
+        guideWatcher.wanted = { [weak self] in self?.character.onScreen == true }
+        guideWatcher.onCommand = { [weak self] rect, mark in self?.character.guide(to: rect, mark: mark) }
+        guideWatcher.start()
+        character.onScreenChanged = { [weak self] in self?.guideWatcher.start() }
 
         // The target does not wait for the microphone: the menu shows it
         // either way.

package/apps/mac-voice/build.sh

@@ -32,10 +32,20 @@ swiftc -O \
 
 cp Resources/Info.plist "$APP/Contents/Info.plist"
 
-# Ad-hoc signature. Unsigned bundles get a fresh TCC identity on every
-# rebuild, so macOS re-asks for the microphone every single launch.
-codesign --force --sign - --identifier tn.acme.agentx.voice "$APP" 2>/dev/null \
-  || echo "warning: codesign failed; expect repeated microphone prompts"
+# Signing. Unsigned bundles get a fresh TCC identity on every launch. An
+# ad-hoc signature (the default) holds only for this one build: macOS keys
+# a permission to the build's hash, so every new build is asked again for
+# the microphone and Screen Recording. AGENTX_SIGN_IDENTITY names a
+# code-signing certificate in the keychain (a self-signed one is enough)
+# whose permissions carry over to the next build.
+IDENTITY="${AGENTX_SIGN_IDENTITY:--}"
+if [ "$IDENTITY" = "-" ]; then
+  codesign --force --sign - --identifier tn.acme.agentx.voice "$APP" 2>/dev/null \
+    || echo "warning: codesign failed; expect repeated microphone prompts"
+  echo "note: ad-hoc signature; macOS asks again for the microphone and Screen Recording after this install (set AGENTX_SIGN_IDENTITY to keep them)"
+else
+  codesign --force --sign "$IDENTITY" --identifier tn.acme.agentx.voice "$APP"
+fi
 
 echo "built: $APP"
 echo "run:   open \"$APP\"     (or ./build/AgentX\\ Voice.app/Contents/MacOS/AgentXVoice for logs)"