@lattices/cli 0.4.10 → 0.4.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +13 -13
  3. package/{app → apps/mac}/Lattices.app/Contents/Info.plist +10 -2
  4. package/{app → apps/mac}/Lattices.app/Contents/MacOS/Lattices +0 -0
  5. package/{app → apps/mac}/Package.swift +2 -1
  6. package/apps/mac/Resources/Pets/assistant-spark/pet.json +62 -0
  7. package/apps/mac/Resources/Pets/assistant-spark/spritesheet.webp +0 -0
  8. package/apps/mac/Resources/Pets/scout-ranger/pet.json +6 -0
  9. package/apps/mac/Resources/Pets/scout-ranger/spritesheet.webp +0 -0
  10. package/apps/mac/Sources/AppShell/AppActivationCoordinator.swift +27 -0
  11. package/apps/mac/Sources/AppShell/AppDelegate.swift +189 -0
  12. package/apps/mac/Sources/AppShell/AppServicesBootstrap.swift +25 -0
  13. package/{app → apps/mac}/Sources/AppShell/AppShellView.swift +18 -3
  14. package/{app → apps/mac}/Sources/AppShell/AppUpdater.swift +4 -3
  15. package/apps/mac/Sources/AppShell/HotkeyBootstrap.swift +87 -0
  16. package/{app → apps/mac}/Sources/AppShell/LatticesRuntime.swift +43 -0
  17. package/{app → apps/mac}/Sources/AppShell/MainView.swift +116 -63
  18. package/apps/mac/Sources/AppShell/MenuBarController.swift +177 -0
  19. package/{app → apps/mac}/Sources/AppShell/OnboardingView.swift +72 -60
  20. package/apps/mac/Sources/AppShell/PermissionsAssistantView.swift +366 -0
  21. package/apps/mac/Sources/AppShell/PermissionsAssistantWindow.swift +70 -0
  22. package/{app → apps/mac}/Sources/AppShell/Preferences.swift +37 -2
  23. package/{app → apps/mac}/Sources/AppShell/SettingsView.swift +815 -156
  24. package/{app → apps/mac}/Sources/AppShell/SettingsWindow.swift +10 -0
  25. package/apps/mac/Sources/AppShell/WorkspaceInspectorPresenter.swift +13 -0
  26. package/{app → apps/mac}/Sources/Core/Actions/HotkeyStore.swift +6 -1
  27. package/{app → apps/mac}/Sources/Core/Actions/IntentEngine.swift +2 -0
  28. package/{app → apps/mac}/Sources/Core/Daemon/DaemonServer.swift +5 -0
  29. package/{app → apps/mac}/Sources/Core/Daemon/LatticesApi.swift +365 -0
  30. package/{app → apps/mac}/Sources/Core/Desktop/OcrModel.swift +17 -13
  31. package/apps/mac/Sources/Core/Desktop/WindowCapture.swift +33 -0
  32. package/{app → apps/mac}/Sources/Core/Desktop/WindowDragSnapController.swift +18 -217
  33. package/{app → apps/mac}/Sources/Core/Desktop/WindowPreviewStore.swift +4 -5
  34. package/{app → apps/mac}/Sources/Core/Desktop/WindowTiler.swift +19 -13
  35. package/apps/mac/Sources/Core/Input/EventTapBreaker.swift +124 -0
  36. package/apps/mac/Sources/Core/Input/EventTapThread.swift +54 -0
  37. package/apps/mac/Sources/Core/Input/InputCaptureResetCenter.swift +20 -0
  38. package/apps/mac/Sources/Core/Input/KeyboardRemapController.swift +335 -0
  39. package/apps/mac/Sources/Core/Input/KeyboardRemapStore.swift +141 -0
  40. package/{app → apps/mac}/Sources/Core/Input/MouseGestureConfig.swift +155 -20
  41. package/apps/mac/Sources/Core/Input/MouseGestureController.swift +2271 -0
  42. package/apps/mac/Sources/Core/Input/MouseShortcutStore.swift +170 -0
  43. package/apps/mac/Sources/Core/Input/SecureEventInputMonitor.swift +39 -0
  44. package/apps/mac/Sources/Core/Input/ShapeRecognizer.swift +624 -0
  45. package/apps/mac/Sources/Core/Input/TapBudgetMeter.swift +56 -0
  46. package/{app → apps/mac}/Sources/Core/Overlays/ScreenMap/ScreenMapState.swift +8 -8
  47. package/apps/mac/Sources/Core/Overlays/ScreenOverlayCanvasController.swift +1264 -0
  48. package/{app → apps/mac}/Sources/Core/Overlays/Voice/VoiceCommandWindow.swift +11 -23
  49. package/{app → apps/mac}/Sources/Core/Pi/PiChatDock.swift +90 -43
  50. package/{app → apps/mac}/Sources/Core/Pi/PiChatSession.swift +676 -43
  51. package/{app → apps/mac}/Sources/Core/Pi/PiProviderSetupCallout.swift +5 -5
  52. package/{app → apps/mac}/Sources/Core/Pi/PiWorkspaceView.swift +93 -44
  53. package/apps/mac/Sources/Core/System/Capability.swift +79 -0
  54. package/{app → apps/mac}/Sources/Core/System/PermissionChecker.swift +43 -8
  55. package/{app → apps/mac}/Sources/Core/Voice/AudioProvider.swift +225 -56
  56. package/bin/handsoff-infer.ts +14 -5
  57. package/bin/handsoff-worker.ts +11 -7
  58. package/bin/infer.ts +406 -0
  59. package/bin/lattices-app.ts +57 -7
  60. package/bin/lattices-dev +40 -1
  61. package/bin/lattices.ts +1 -1
  62. package/docs/agent-execution-plan.md +9 -9
  63. package/docs/api.md +119 -0
  64. package/docs/app.md +1 -0
  65. package/docs/companion-deck.md +1 -1
  66. package/docs/gesture-customization-proposal.md +520 -0
  67. package/docs/mouse-gestures.md +79 -0
  68. package/docs/overview.md +2 -2
  69. package/docs/presentation-execution-review.md +9 -9
  70. package/docs/proposals/LAT-001-gesture-visual-customization.md +522 -0
  71. package/docs/proposals/LAT-002-shared-overlay-canvas.md +353 -0
  72. package/docs/proposals/LAT-003-menu-bar-controller-architecture.md +291 -0
  73. package/docs/proposals/LAT-004-interactive-overlay-actors.md +534 -0
  74. package/docs/reference/dewey.config.ts +74 -0
  75. package/docs/reference/install-agent.md +79 -0
  76. package/docs/repo-structure.md +100 -0
  77. package/docs/voice-error-model.md +7 -7
  78. package/docs/voice.md +18 -0
  79. package/package.json +23 -13
  80. package/swift/Package.swift +20 -0
  81. package/swift/Sources/DeckKit/DeckAction.swift +51 -0
  82. package/swift/Sources/DeckKit/DeckBridgeSecurity.swift +152 -0
  83. package/swift/Sources/DeckKit/DeckCockpit.swift +82 -0
  84. package/swift/Sources/DeckKit/DeckHost.swift +7 -0
  85. package/swift/Sources/DeckKit/DeckManifest.swift +145 -0
  86. package/swift/Sources/DeckKit/DeckRuntimeSnapshot.swift +533 -0
  87. package/swift/Sources/DeckKit/DeckTrackpad.swift +63 -0
  88. package/swift/Sources/DeckKit/DeckValue.swift +93 -0
  89. package/swift/Sources/DeckKit/DeckVoiceError.swift +88 -0
  90. package/swift/Tests/DeckKitTests/DeckKitTests.swift +286 -0
  91. package/app/Sources/AppShell/AppDelegate.swift +0 -408
  92. package/app/Sources/Core/Input/KeyboardRemapController.swift +0 -184
  93. package/app/Sources/Core/Input/KeyboardRemapStore.swift +0 -84
  94. package/app/Sources/Core/Input/MouseGestureController.swift +0 -1203
  95. package/app/Sources/Core/Input/MouseShortcutStore.swift +0 -107
  96. /package/{app → apps/mac}/Info.plist +0 -0
  97. /package/{app → apps/mac}/Lattices.app/Contents/Resources/AppIcon.icns +0 -0
  98. /package/{app → apps/mac}/Lattices.app/Contents/Resources/tap.wav +0 -0
  99. /package/{app → apps/mac}/Lattices.app/Contents/_CodeSignature/CodeResources +0 -0
  100. /package/{app → apps/mac}/Lattices.entitlements +0 -0
  101. /package/{app → apps/mac}/Resources/tap.wav +0 -0
  102. /package/{app → apps/mac}/Sources/AppShell/App.swift +0 -0
  103. /package/{app → apps/mac}/Sources/AppShell/CliActionLauncher.swift +0 -0
  104. /package/{app → apps/mac}/Sources/AppShell/HomeDashboardView.swift +0 -0
  105. /package/{app → apps/mac}/Sources/AppShell/KeyRecorderView.swift +0 -0
  106. /package/{app → apps/mac}/Sources/AppShell/MainWindow.swift +0 -0
  107. /package/{app → apps/mac}/Sources/Core/Actions/HotkeyManager.swift +0 -0
  108. /package/{app → apps/mac}/Sources/Core/Actions/IntentSchema.swift +0 -0
  109. /package/{app → apps/mac}/Sources/Core/Actions/Intents/CreateLayerIntent.swift +0 -0
  110. /package/{app → apps/mac}/Sources/Core/Actions/Intents/DistributeIntent.swift +0 -0
  111. /package/{app → apps/mac}/Sources/Core/Actions/Intents/FocusIntent.swift +0 -0
  112. /package/{app → apps/mac}/Sources/Core/Actions/Intents/HelpIntent.swift +0 -0
  113. /package/{app → apps/mac}/Sources/Core/Actions/Intents/KillIntent.swift +0 -0
  114. /package/{app → apps/mac}/Sources/Core/Actions/Intents/LatticeIntent.swift +0 -0
  115. /package/{app → apps/mac}/Sources/Core/Actions/Intents/LaunchIntent.swift +0 -0
  116. /package/{app → apps/mac}/Sources/Core/Actions/Intents/ListSessionsIntent.swift +0 -0
  117. /package/{app → apps/mac}/Sources/Core/Actions/Intents/ListWindowsIntent.swift +0 -0
  118. /package/{app → apps/mac}/Sources/Core/Actions/Intents/ScanIntent.swift +0 -0
  119. /package/{app → apps/mac}/Sources/Core/Actions/Intents/SearchIntent.swift +0 -0
  120. /package/{app → apps/mac}/Sources/Core/Actions/Intents/SwitchLayerIntent.swift +0 -0
  121. /package/{app → apps/mac}/Sources/Core/Actions/Intents/TileIntent.swift +0 -0
  122. /package/{app → apps/mac}/Sources/Core/Actions/PaletteCommand.swift +0 -0
  123. /package/{app → apps/mac}/Sources/Core/Actions/VoiceIntentResolver.swift +0 -0
  124. /package/{app → apps/mac}/Sources/Core/Companion/CompanionActivityLog.swift +0 -0
  125. /package/{app → apps/mac}/Sources/Core/Companion/CompanionKeyboardController.swift +0 -0
  126. /package/{app → apps/mac}/Sources/Core/Companion/LatticesCompanionBridgeServer.swift +0 -0
  127. /package/{app → apps/mac}/Sources/Core/Companion/LatticesCompanionCockpit.swift +0 -0
  128. /package/{app → apps/mac}/Sources/Core/Companion/LatticesCompanionSecurityCoordinator.swift +0 -0
  129. /package/{app → apps/mac}/Sources/Core/Companion/LatticesCompanionTrackpadController.swift +0 -0
  130. /package/{app → apps/mac}/Sources/Core/Companion/LatticesDeckHost.swift +0 -0
  131. /package/{app → apps/mac}/Sources/Core/Daemon/DaemonProtocol.swift +0 -0
  132. /package/{app → apps/mac}/Sources/Core/Desktop/AccessibilityTextExtractor.swift +0 -0
  133. /package/{app → apps/mac}/Sources/Core/Desktop/AppTypeClassifier.swift +0 -0
  134. /package/{app → apps/mac}/Sources/Core/Desktop/DesktopModel.swift +0 -0
  135. /package/{app → apps/mac}/Sources/Core/Desktop/DesktopModelTypes.swift +0 -0
  136. /package/{app → apps/mac}/Sources/Core/Desktop/InventoryManager.swift +0 -0
  137. /package/{app → apps/mac}/Sources/Core/Desktop/InventoryPath.swift +0 -0
  138. /package/{app → apps/mac}/Sources/Core/Desktop/MouseFinder.swift +0 -0
  139. /package/{app → apps/mac}/Sources/Core/Desktop/OcrStore.swift +0 -0
  140. /package/{app → apps/mac}/Sources/Core/Desktop/PlacementSpec.swift +0 -0
  141. /package/{app → apps/mac}/Sources/Core/Desktop/SessionWindowLocator.swift +0 -0
  142. /package/{app → apps/mac}/Sources/Core/Desktop/TilePickerView.swift +0 -0
  143. /package/{app → apps/mac}/Sources/Core/Desktop/WindowPreviewCard.swift +0 -0
  144. /package/{app → apps/mac}/Sources/Core/Desktop/WindowSelectionStore.swift +0 -0
  145. /package/{app → apps/mac}/Sources/Core/Input/KeyboardRemapConfig.swift +0 -0
  146. /package/{app → apps/mac}/Sources/Core/Input/MouseInputDeviceStore.swift +0 -0
  147. /package/{app → apps/mac}/Sources/Core/Input/MouseInputEventViewer.swift +0 -0
  148. /package/{app → apps/mac}/Sources/Core/Overlays/AppWindowShell.swift +0 -0
  149. /package/{app → apps/mac}/Sources/Core/Overlays/CommandMode/CommandModeState.swift +0 -0
  150. /package/{app → apps/mac}/Sources/Core/Overlays/CommandMode/CommandModeView.swift +0 -0
  151. /package/{app → apps/mac}/Sources/Core/Overlays/CommandMode/CommandModeWindow.swift +0 -0
  152. /package/{app → apps/mac}/Sources/Core/Overlays/CommandPalette/CommandPaletteView.swift +0 -0
  153. /package/{app → apps/mac}/Sources/Core/Overlays/CommandPalette/CommandPaletteWindow.swift +0 -0
  154. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/CheatSheetHUD.swift +0 -0
  155. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/HUDBottomBar.swift +0 -0
  156. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/HUDController.swift +0 -0
  157. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/HUDLeftBar.swift +0 -0
  158. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/HUDMinimap.swift +0 -0
  159. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/HUDRightBar.swift +0 -0
  160. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/HUDState.swift +0 -0
  161. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/HUDTopBar.swift +0 -0
  162. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/LauncherHUD.swift +0 -0
  163. /package/{app → apps/mac}/Sources/Core/Overlays/HUD/LayerBezel.swift +0 -0
  164. /package/{app → apps/mac}/Sources/Core/Overlays/OmniSearch/OmniSearchState.swift +0 -0
  165. /package/{app → apps/mac}/Sources/Core/Overlays/OmniSearch/OmniSearchView.swift +0 -0
  166. /package/{app → apps/mac}/Sources/Core/Overlays/OmniSearch/OmniSearchWindow.swift +0 -0
  167. /package/{app → apps/mac}/Sources/Core/Overlays/OverlayPanelShell.swift +0 -0
  168. /package/{app → apps/mac}/Sources/Core/Overlays/ScreenMap/ScreenMapView.swift +0 -0
  169. /package/{app → apps/mac}/Sources/Core/Overlays/ScreenMap/ScreenMapWindowController.swift +0 -0
  170. /package/{app → apps/mac}/Sources/Core/Pi/PiAuthNextStepCard.swift +0 -0
  171. /package/{app → apps/mac}/Sources/Core/Pi/PiAuthPromptCard.swift +0 -0
  172. /package/{app → apps/mac}/Sources/Core/Pi/PiInstallCallout.swift +0 -0
  173. /package/{app → apps/mac}/Sources/Core/System/DiagnosticLog.swift +0 -0
  174. /package/{app → apps/mac}/Sources/Core/System/EventBus.swift +0 -0
  175. /package/{app → apps/mac}/Sources/Core/System/ProcessModel.swift +0 -0
  176. /package/{app → apps/mac}/Sources/Core/System/ProcessQuery.swift +0 -0
  177. /package/{app → apps/mac}/Sources/Core/System/SystemTelemetryMonitor.swift +0 -0
  178. /package/{app → apps/mac}/Sources/Core/Voice/AdvisorLearningStore.swift +0 -0
  179. /package/{app → apps/mac}/Sources/Core/Voice/AgentSession.swift +0 -0
  180. /package/{app → apps/mac}/Sources/Core/Voice/HandsOffSession.swift +0 -0
  181. /package/{app → apps/mac}/Sources/Core/Voice/VoiceChatView.swift +0 -0
  182. /package/{app → apps/mac}/Sources/Core/Voice/VoxClient.swift +0 -0
  183. /package/{app → apps/mac}/Sources/Core/Workspace/Project.swift +0 -0
  184. /package/{app → apps/mac}/Sources/Core/Workspace/ProjectScanner.swift +0 -0
  185. /package/{app → apps/mac}/Sources/Core/Workspace/SessionLayerStore.swift +0 -0
  186. /package/{app → apps/mac}/Sources/Core/Workspace/SessionManager.swift +0 -0
  187. /package/{app → apps/mac}/Sources/Core/Workspace/Terminal/Terminal.swift +0 -0
  188. /package/{app → apps/mac}/Sources/Core/Workspace/Terminal/TerminalQuery.swift +0 -0
  189. /package/{app → apps/mac}/Sources/Core/Workspace/Terminal/TerminalSynthesizer.swift +0 -0
  190. /package/{app → apps/mac}/Sources/Core/Workspace/Tmux/TmuxModel.swift +0 -0
  191. /package/{app → apps/mac}/Sources/Core/Workspace/Tmux/TmuxQuery.swift +0 -0
  192. /package/{app → apps/mac}/Sources/Core/Workspace/WorkspaceManager.swift +0 -0
  193. /package/{app → apps/mac}/Sources/UI/ActionRow.swift +0 -0
  194. /package/{app → apps/mac}/Sources/UI/OrphanRow.swift +0 -0
  195. /package/{app → apps/mac}/Sources/UI/ProjectRow.swift +0 -0
  196. /package/{app → apps/mac}/Sources/UI/TabGroupRow.swift +0 -0
  197. /package/{app → apps/mac}/Sources/UI/Theme.swift +0 -0
  198. /package/{app → apps/mac}/Tests/StageDragTests.swift +0 -0
  199. /package/{app → apps/mac}/Tests/StageJoinTests.swift +0 -0
  200. /package/{app → apps/mac}/Tests/StageManagerTests.swift +0 -0
  201. /package/{app → apps/mac}/Tests/StageTileTests.swift +0 -0
@@ -12,12 +12,22 @@ final class SettingsWindowController {
12
12
 
13
13
  func show() {
14
14
  ScreenMapWindowController.shared.showPage(.settings)
15
+ DispatchQueue.main.async {
16
+ NotificationCenter.default.post(name: .latticesShowGeneralSettings, object: nil)
17
+ }
15
18
  }
16
19
 
17
20
  func showCompanion() {
18
21
  ScreenMapWindowController.shared.showPage(.companionSettings)
19
22
  }
20
23
 
24
+ func showAssistant() {
25
+ ScreenMapWindowController.shared.showPage(.settings)
26
+ DispatchQueue.main.async {
27
+ NotificationCenter.default.post(name: .latticesShowAssistantSettings, object: nil)
28
+ }
29
+ }
30
+
21
31
  func close() {
22
32
  ScreenMapWindowController.shared.close()
23
33
  }
@@ -0,0 +1,13 @@
1
+ import AppKit
2
+
3
+ enum WorkspaceInspectorPresenter {
4
+ static func show() {
5
+ guard let entry = DesktopModel.shared.frontmostWindow(),
6
+ entry.app != "Lattices" else {
7
+ ScreenMapWindowController.shared.showPage(.screenMap)
8
+ return
9
+ }
10
+
11
+ ScreenMapWindowController.shared.showWindow(wid: entry.wid)
12
+ }
13
+ }
@@ -25,6 +25,7 @@ enum HotkeyAction: String, CaseIterable, Codable {
25
25
  case unifiedWindow
26
26
  case hud
27
27
  case mouseFinder
28
+ case overlayActors
28
29
  // Layers
29
30
  case layer1, layer2, layer3, layer4, layer5, layer6, layer7, layer8, layer9
30
31
  case layerNext, layerPrev, layerTag
@@ -47,6 +48,7 @@ enum HotkeyAction: String, CaseIterable, Codable {
47
48
  case .unifiedWindow: return "Workspace Home"
48
49
  case .hud: return "HUD"
49
50
  case .mouseFinder: return "Find Mouse"
51
+ case .overlayActors: return "Toggle Overlay Actors"
50
52
  case .layer1: return "Layer 1"
51
53
  case .layer2: return "Layer 2"
52
54
  case .layer3: return "Layer 3"
@@ -80,7 +82,7 @@ enum HotkeyAction: String, CaseIterable, Codable {
80
82
 
81
83
  var group: HotkeyGroup {
82
84
  switch self {
83
- case .palette, .screenMap, .bezel, .cheatSheet, .desktopInventory, .omniSearch, .voiceCommand, .handsOff, .unifiedWindow, .hud, .mouseFinder: return .app
85
+ case .palette, .screenMap, .bezel, .cheatSheet, .desktopInventory, .omniSearch, .voiceCommand, .handsOff, .unifiedWindow, .hud, .mouseFinder, .overlayActors: return .app
84
86
  case .layer1, .layer2, .layer3, .layer4, .layer5,
85
87
  .layer6, .layer7, .layer8, .layer9,
86
88
  .layerNext, .layerPrev, .layerTag: return .layers
@@ -101,6 +103,7 @@ enum HotkeyAction: String, CaseIterable, Codable {
101
103
  case .unifiedWindow: return 207
102
104
  case .hud: return 208
103
105
  case .mouseFinder: return 209
106
+ case .overlayActors: return 210
104
107
  case .layer1: return 101
105
108
  case .layer2: return 102
106
109
  case .layer3: return 103
@@ -240,6 +243,7 @@ class HotkeyStore: ObservableObject {
240
243
  // App
241
244
  bind(.palette, 46, cmdShift) // Cmd+Shift+M
242
245
  bind(.unifiedWindow, 18, hyper) // Hyper+1 (Workspace Home)
246
+ bind(.screenMap, 37, hyper) // Hyper+L (Layout)
243
247
  bind(.bezel, 19, hyper) // Hyper+2
244
248
  bind(.hud, 20, hyper) // Hyper+3 (HUD overlay)
245
249
  bind(.desktopInventory, 5, hyper) // Hyper+G
@@ -249,6 +253,7 @@ class HotkeyStore: ObservableObject {
249
253
  bind(.omniSearch, 23, hyper) // Hyper+5
250
254
  bind(.cheatSheet, 22, hyper) // Hyper+6
251
255
  bind(.mouseFinder, 26, hyper) // Hyper+7
256
+ bind(.overlayActors, 28, hyper) // Hyper+8
252
257
 
253
258
  // Layers: Cmd+Option+1-9
254
259
  let layerKeyCodes: [UInt32] = [18, 19, 20, 21, 23, 22, 26, 28, 25]
@@ -845,6 +845,8 @@ struct ClaudeResolvedIntent {
845
845
  let slots: [String: JSON]
846
846
  }
847
847
 
848
+ typealias ResolvedIntent = ClaudeResolvedIntent
849
+
848
850
  struct ClaudeAgentPlan {
849
851
  let steps: [ClaudeResolvedIntent]
850
852
  let reasoning: String
@@ -121,6 +121,11 @@ final class DaemonServer: ObservableObject {
121
121
  }
122
122
  guard clientFd >= 0 else { return }
123
123
 
124
+ // A client can disconnect immediately after a large response. On Darwin,
125
+ // writing to that socket can otherwise raise SIGPIPE and terminate the app.
126
+ var noSigPipe: Int32 = 1
127
+ setsockopt(clientFd, SOL_SOCKET, SO_NOSIGPIPE, &noSigPipe, socklen_t(MemoryLayout<Int32>.size))
128
+
124
129
  let id = UUID()
125
130
  let client = WebSocketClient(id: id, fd: clientFd)
126
131
 
@@ -287,6 +287,13 @@ final class LatticesApi {
287
287
  Field(name: "tmuxSessionCount", type: "int", required: true, description: "Active tmux session count"),
288
288
  ]))
289
289
 
290
+ api.model(ApiModel(name: "OverlayLayer", fields: [
291
+ Field(name: "id", type: "string", required: true, description: "Overlay layer identifier"),
292
+ Field(name: "kind", type: "string", required: true, description: "Overlay kind: toast, label, highlight, pet"),
293
+ Field(name: "owner", type: "string", required: true, description: "Layer owner namespace"),
294
+ Field(name: "expiresAt", type: "double", required: false, description: "Expiration timestamp (Unix seconds)"),
295
+ ]))
296
+
290
297
  // ── Endpoints: Read ─────────────────────────────────────
291
298
 
292
299
  api.register(Endpoint(
@@ -815,6 +822,96 @@ final class LatticesApi {
815
822
  }
816
823
  ))
817
824
 
825
+ api.register(Endpoint(
826
+ method: "overlay.publish",
827
+ description: "Publish a transient visual layer on the invisible screen overlay canvas",
828
+ access: .mutate,
829
+ params: [
830
+ Param(name: "kind", type: "string", required: true, description: "toast, label, highlight, or pet"),
831
+ Param(name: "id", type: "string", required: false, description: "Stable layer id; generated if omitted"),
832
+ Param(name: "text", type: "string", required: false, description: "Toast/label text"),
833
+ Param(name: "detail", type: "string", required: false, description: "Secondary toast/label text"),
834
+ Param(name: "message", type: "string", required: false, description: "Pet speech/message"),
835
+ Param(name: "glyph", type: "string", required: false, description: "Pet glyph, emoji, or short symbol"),
836
+ Param(name: "petId", type: "string", required: false, description: "Bundled pet id from Resources/Pets"),
837
+ Param(name: "state", type: "string", required: false, description: "Pet animation state"),
838
+ Param(name: "name", type: "string", required: false, description: "Pet name"),
839
+ Param(name: "x", type: "double", required: false, description: "Screen-local x coordinate"),
840
+ Param(name: "y", type: "double", required: false, description: "Screen-local y coordinate"),
841
+ Param(name: "w", type: "double", required: false, description: "Highlight width"),
842
+ Param(name: "h", type: "double", required: false, description: "Highlight height"),
843
+ Param(name: "placement", type: "string", required: false, description: "top, bottom, center, cursor, or point"),
844
+ Param(name: "style", type: "string", required: false, description: "info, success, warning, danger, or playful"),
845
+ Param(name: "display", type: "int", required: false, description: "Display index; omit for all displays"),
846
+ Param(name: "ttlMs", type: "int", required: false, description: "Time to live in milliseconds"),
847
+ Param(name: "opacity", type: "double", required: false, description: "Opacity 0-1"),
848
+ Param(name: "zIndex", type: "int", required: false, description: "Layer ordering"),
849
+ Param(name: "dismissible", type: "bool", required: false, description: "Whether click-away dismissal removes the layer"),
850
+ ],
851
+ returns: .object(model: "OverlayLayer"),
852
+ handler: { params in
853
+ try Self.publishOverlay(params)
854
+ }
855
+ ))
856
+
857
+ api.register(Endpoint(
858
+ method: "overlay.clear",
859
+ description: "Clear overlay layers published through the daemon API",
860
+ access: .mutate,
861
+ params: [
862
+ Param(name: "id", type: "string", required: false, description: "Specific layer id to clear"),
863
+ Param(name: "owner", type: "string", required: false, description: "Owner namespace to clear; defaults to agentApi"),
864
+ ],
865
+ returns: .ok,
866
+ handler: { params in
867
+ try Self.clearOverlay(params)
868
+ }
869
+ ))
870
+
871
+ api.register(Endpoint(
872
+ method: "overlay.actor.publish",
873
+ description: "Create or update a small generative overlay actor",
874
+ access: .mutate,
875
+ params: [
876
+ Param(name: "id", type: "string", required: false, description: "Stable actor id; generated if omitted"),
877
+ Param(name: "renderer", type: "string", required: false, description: "Renderer type; sprite is currently supported"),
878
+ Param(name: "asset", type: "string", required: false, description: "Bundled sprite asset id"),
879
+ Param(name: "state", type: "string", required: false, description: "Actor state or animation name"),
880
+ Param(name: "name", type: "string", required: false, description: "Actor display name"),
881
+ Param(name: "message", type: "string", required: false, description: "Attached message text"),
882
+ Param(name: "x", type: "double", required: false, description: "Screen-local x coordinate"),
883
+ Param(name: "y", type: "double", required: false, description: "Screen-local y coordinate"),
884
+ Param(name: "placement", type: "string", required: false, description: "top, bottom, center, cursor, or point"),
885
+ Param(name: "style", type: "string", required: false, description: "info, success, warning, danger, or playful"),
886
+ Param(name: "display", type: "int", required: false, description: "Display index; omit for all displays"),
887
+ Param(name: "ttlMs", type: "int", required: false, description: "Time to live in milliseconds; omit or pass 0 for persistent"),
888
+ Param(name: "opacity", type: "double", required: false, description: "Opacity 0-1"),
889
+ Param(name: "zIndex", type: "int", required: false, description: "Layer ordering"),
890
+ Param(name: "dismissible", type: "bool", required: false, description: "Whether click-away dismissal removes the actor; defaults false"),
891
+ ],
892
+ returns: .object(model: "OverlayLayer"),
893
+ handler: { params in
894
+ try Self.publishOverlayActor(params)
895
+ }
896
+ ))
897
+
898
+ api.register(Endpoint(
899
+ method: "overlay.actor.moveTo",
900
+ description: "Move an overlay actor with app-owned easing",
901
+ access: .mutate,
902
+ params: [
903
+ Param(name: "id", type: "string", required: true, description: "Actor id"),
904
+ Param(name: "x", type: "double", required: true, description: "Target screen-local x coordinate"),
905
+ Param(name: "y", type: "double", required: true, description: "Target screen-local y coordinate"),
906
+ Param(name: "durationMs", type: "int", required: false, description: "Animation duration in milliseconds"),
907
+ Param(name: "easing", type: "string", required: false, description: "linear, easeInOut, or spring"),
908
+ ],
909
+ returns: .ok,
910
+ handler: { params in
911
+ try Self.moveOverlayActor(params)
912
+ }
913
+ ))
914
+
818
915
  api.register(Endpoint(
819
916
  method: "daemon.status",
820
917
  description: "Get daemon status including uptime and counts",
@@ -1918,6 +2015,274 @@ final class LatticesApi {
1918
2015
  }
1919
2016
 
1920
2017
  private extension LatticesApi {
2018
+ static func publishOverlay(_ params: JSON?) throws -> JSON {
2019
+ guard let params else {
2020
+ throw RouterError.missingParam("kind")
2021
+ }
2022
+ guard let kind = params["kind"]?.stringValue?.lowercased(), !kind.isEmpty else {
2023
+ throw RouterError.missingParam("kind")
2024
+ }
2025
+
2026
+ let id = params["id"]?.stringValue ?? "agent-\(UUID().uuidString)"
2027
+ let style = try parseOverlayStyle(params["style"]?.stringValue)
2028
+ let placement = try parseOverlayPlacement(params["placement"]?.stringValue)
2029
+ let screen = try parseOverlayScreen(params["display"]?.intValue)
2030
+ let point = parseOverlayPoint(params)
2031
+ let opacity = CGFloat(max(0.05, min(params["opacity"]?.numericDouble ?? 1.0, 1.0)))
2032
+ let zIndex = params["zIndex"]?.intValue ?? 500
2033
+ let ttlMs = params["ttlMs"]?.intValue ?? defaultOverlayTTL(for: kind)
2034
+ let expiresAt = ttlMs > 0 ? Date().addingTimeInterval(Double(ttlMs) / 1000.0) : nil
2035
+ let payload: ScreenOverlayPayload
2036
+
2037
+ switch kind {
2038
+ case "toast":
2039
+ let text = try requiredString(params, "text")
2040
+ payload = .toast(ScreenOverlayTextPayload(
2041
+ text: text,
2042
+ detail: params["detail"]?.stringValue,
2043
+ point: point,
2044
+ placement: placement,
2045
+ style: style
2046
+ ))
2047
+ case "label":
2048
+ let text = try requiredString(params, "text")
2049
+ payload = .label(ScreenOverlayTextPayload(
2050
+ text: text,
2051
+ detail: params["detail"]?.stringValue,
2052
+ point: point,
2053
+ placement: placement == .top ? .point : placement,
2054
+ style: style
2055
+ ))
2056
+ case "highlight":
2057
+ guard let rect = parseOverlayRect(params) else {
2058
+ throw RouterError.custom("highlight requires x, y, w, and h")
2059
+ }
2060
+ payload = .highlight(ScreenOverlayHighlightPayload(
2061
+ rect: rect,
2062
+ label: params["text"]?.stringValue ?? params["label"]?.stringValue,
2063
+ style: style,
2064
+ cornerRadius: CGFloat(params["cornerRadius"]?.numericDouble ?? 10)
2065
+ ))
2066
+ case "pet":
2067
+ let glyph = params["glyph"]?.stringValue ?? "✦"
2068
+ payload = .pet(ScreenOverlayPetPayload(
2069
+ glyph: String(glyph.prefix(4)),
2070
+ petID: params["petId"]?.stringValue,
2071
+ state: params["state"]?.stringValue,
2072
+ name: params["name"]?.stringValue,
2073
+ message: params["message"]?.stringValue ?? params["text"]?.stringValue,
2074
+ point: point,
2075
+ placement: placement,
2076
+ style: style,
2077
+ isDragging: false,
2078
+ dismissible: params["dismissible"]?.boolValue ?? true
2079
+ ))
2080
+ default:
2081
+ throw RouterError.custom("Unsupported overlay kind: \(kind)")
2082
+ }
2083
+
2084
+ let layer = ScreenOverlayLayerSnapshot(
2085
+ id: ScreenOverlayLayerID(id),
2086
+ owner: .agentApi,
2087
+ screen: screen,
2088
+ zIndex: zIndex,
2089
+ opacity: opacity,
2090
+ payload: payload,
2091
+ expiresAt: expiresAt
2092
+ )
2093
+
2094
+ runOnMain {
2095
+ ScreenOverlayCanvasController.shared.publishLayer(layer)
2096
+ }
2097
+
2098
+ var result: [String: JSON] = [
2099
+ "id": .string(id),
2100
+ "kind": .string(kind),
2101
+ "owner": .string(ScreenOverlayOwner.agentApi.rawValue),
2102
+ ]
2103
+ if let expiresAt {
2104
+ result["expiresAt"] = .double(expiresAt.timeIntervalSince1970)
2105
+ }
2106
+ return .object(result)
2107
+ }
2108
+
2109
+ static func clearOverlay(_ params: JSON?) throws -> JSON {
2110
+ let owner = try parseOverlayOwner(params?["owner"]?.stringValue)
2111
+ if let id = params?["id"]?.stringValue, !id.isEmpty {
2112
+ runOnMain {
2113
+ ScreenOverlayCanvasController.shared.removeLayer(id: ScreenOverlayLayerID(id))
2114
+ }
2115
+ } else {
2116
+ runOnMain {
2117
+ ScreenOverlayCanvasController.shared.removeLayers(owner: owner)
2118
+ }
2119
+ }
2120
+ return .object(["ok": .bool(true)])
2121
+ }
2122
+
2123
+ static func publishOverlayActor(_ params: JSON?) throws -> JSON {
2124
+ guard let params else {
2125
+ throw RouterError.missingParam("id")
2126
+ }
2127
+ let id = params["id"]?.stringValue ?? "actor-\(UUID().uuidString)"
2128
+ let renderer = params["renderer"]?.stringValue?.lowercased() ?? "sprite"
2129
+ guard renderer == "sprite" || renderer == "pet" else {
2130
+ throw RouterError.custom("Unsupported overlay actor renderer: \(renderer)")
2131
+ }
2132
+
2133
+ let style = try parseOverlayStyle(params["style"]?.stringValue)
2134
+ let point = parseOverlayPoint(params)
2135
+ let placement = try parseOverlayPlacement(params["placement"]?.stringValue ?? (point == nil ? "bottom" : "point"))
2136
+ let screen = try parseOverlayScreen(params["display"]?.intValue)
2137
+ let opacity = CGFloat(max(0.05, min(params["opacity"]?.numericDouble ?? 1.0, 1.0)))
2138
+ let zIndex = params["zIndex"]?.intValue ?? 520
2139
+ let ttlMs = params["ttlMs"]?.intValue ?? 0
2140
+ let expiresAt = ttlMs > 0 ? Date().addingTimeInterval(Double(ttlMs) / 1000.0) : nil
2141
+ let asset = params["asset"]?.stringValue ?? params["petId"]?.stringValue
2142
+ let message = params["message"]?.stringValue ?? params["text"]?.stringValue
2143
+
2144
+ let layer = ScreenOverlayLayerSnapshot(
2145
+ id: ScreenOverlayLayerID(id),
2146
+ owner: .agentApi,
2147
+ screen: screen,
2148
+ zIndex: zIndex,
2149
+ opacity: opacity,
2150
+ payload: .pet(ScreenOverlayPetPayload(
2151
+ glyph: String((params["glyph"]?.stringValue ?? "✦").prefix(4)),
2152
+ petID: asset,
2153
+ state: params["state"]?.stringValue ?? "idle",
2154
+ name: params["name"]?.stringValue,
2155
+ message: message,
2156
+ point: point,
2157
+ placement: placement,
2158
+ style: style,
2159
+ isDragging: false,
2160
+ dismissible: params["dismissible"]?.boolValue ?? false
2161
+ )),
2162
+ expiresAt: expiresAt
2163
+ )
2164
+
2165
+ runOnMain {
2166
+ ScreenOverlayCanvasController.shared.publishLayer(layer)
2167
+ }
2168
+
2169
+ var result: [String: JSON] = [
2170
+ "id": .string(id),
2171
+ "kind": .string("actor"),
2172
+ "owner": .string(ScreenOverlayOwner.agentApi.rawValue),
2173
+ "renderer": .string(renderer),
2174
+ ]
2175
+ if let expiresAt {
2176
+ result["expiresAt"] = .double(expiresAt.timeIntervalSince1970)
2177
+ }
2178
+ return .object(result)
2179
+ }
2180
+
2181
+ static func moveOverlayActor(_ params: JSON?) throws -> JSON {
2182
+ guard let params else {
2183
+ throw RouterError.missingParam("id")
2184
+ }
2185
+ let id = try requiredString(params, "id")
2186
+ guard let x = params["x"]?.numericDouble else {
2187
+ throw RouterError.missingParam("x")
2188
+ }
2189
+ guard let y = params["y"]?.numericDouble else {
2190
+ throw RouterError.missingParam("y")
2191
+ }
2192
+ let durationMs = params["durationMs"]?.intValue ?? 700
2193
+ let easing = params["easing"]?.stringValue
2194
+ var moved = false
2195
+ runOnMain {
2196
+ moved = ScreenOverlayCanvasController.shared.moveLayer(
2197
+ id: ScreenOverlayLayerID(id),
2198
+ to: CGPoint(x: x, y: y),
2199
+ durationMs: durationMs,
2200
+ easing: easing
2201
+ )
2202
+ }
2203
+ guard moved else {
2204
+ throw RouterError.custom("Overlay actor not found or not movable: \(id)")
2205
+ }
2206
+ return .object([
2207
+ "ok": .bool(true),
2208
+ "id": .string(id),
2209
+ "x": .double(x),
2210
+ "y": .double(y),
2211
+ ])
2212
+ }
2213
+
2214
+ static func runOnMain(_ work: @escaping () -> Void) {
2215
+ if Thread.isMainThread {
2216
+ work()
2217
+ } else {
2218
+ DispatchQueue.main.sync(execute: work)
2219
+ }
2220
+ }
2221
+
2222
+ static func parseOverlayScreen(_ displayIndex: Int?) throws -> ScreenOverlayScreenTarget {
2223
+ guard let displayIndex else { return .all }
2224
+ guard displayIndex >= 0, displayIndex < NSScreen.screens.count else {
2225
+ throw RouterError.custom("Invalid display index: \(displayIndex)")
2226
+ }
2227
+ return .screen(id: ScreenOverlayCanvasController.screenID(for: NSScreen.screens[displayIndex]))
2228
+ }
2229
+
2230
+ static func parseOverlayPoint(_ params: JSON) -> CGPoint? {
2231
+ guard let x = params["x"]?.numericDouble,
2232
+ let y = params["y"]?.numericDouble else { return nil }
2233
+ return CGPoint(x: x, y: y)
2234
+ }
2235
+
2236
+ static func parseOverlayRect(_ params: JSON) -> CGRect? {
2237
+ guard let x = params["x"]?.numericDouble,
2238
+ let y = params["y"]?.numericDouble,
2239
+ let w = params["w"]?.numericDouble,
2240
+ let h = params["h"]?.numericDouble else { return nil }
2241
+ return CGRect(x: x, y: y, width: w, height: h)
2242
+ }
2243
+
2244
+ static func parseOverlayPlacement(_ value: String?) throws -> ScreenOverlayPlacement {
2245
+ let raw = value?.lowercased() ?? ScreenOverlayPlacement.top.rawValue
2246
+ guard let placement = ScreenOverlayPlacement(rawValue: raw) else {
2247
+ throw RouterError.custom("Unsupported overlay placement: \(raw)")
2248
+ }
2249
+ return placement
2250
+ }
2251
+
2252
+ static func parseOverlayStyle(_ value: String?) throws -> ScreenOverlayStyle {
2253
+ let raw = value?.lowercased() ?? ScreenOverlayStyle.info.rawValue
2254
+ guard let style = ScreenOverlayStyle(rawValue: raw) else {
2255
+ throw RouterError.custom("Unsupported overlay style: \(raw)")
2256
+ }
2257
+ return style
2258
+ }
2259
+
2260
+ static func parseOverlayOwner(_ value: String?) throws -> ScreenOverlayOwner {
2261
+ let raw = value ?? ScreenOverlayOwner.agentApi.rawValue
2262
+ guard let owner = ScreenOverlayOwner(rawValue: raw) else {
2263
+ throw RouterError.custom("Unsupported overlay owner: \(raw)")
2264
+ }
2265
+ return owner
2266
+ }
2267
+
2268
+ static func defaultOverlayTTL(for kind: String) -> Int {
2269
+ switch kind {
2270
+ case "highlight":
2271
+ return 2500
2272
+ case "pet":
2273
+ return 4200
2274
+ default:
2275
+ return 2800
2276
+ }
2277
+ }
2278
+
2279
+ static func requiredString(_ params: JSON, _ key: String) throws -> String {
2280
+ guard let value = params[key]?.stringValue, !value.isEmpty else {
2281
+ throw RouterError.missingParam(key)
2282
+ }
2283
+ return value
2284
+ }
2285
+
1921
2286
  static func decodeDeckActionRequest(from json: JSON?) throws -> DeckActionRequest {
1922
2287
  guard let json else {
1923
2288
  throw RouterError.missingParam("actionID")
@@ -86,9 +86,15 @@ final class OcrModel: ObservableObject {
86
86
  func setEnabled(_ on: Bool) {
87
87
  enabled = on
88
88
  prefs.ocrEnabled = on
89
- if on && timer == nil {
90
- start()
91
- } else if !on {
89
+ if on {
90
+ // User intentionally turning on OCR clears any prior snooze and is
91
+ // the moment we ask macOS for Screen Recording.
92
+ prefs.clearDismissal(Capability.screenSearch.rawValue)
93
+ if timer == nil {
94
+ PermissionChecker.shared.requestScreenRecording()
95
+ start()
96
+ }
97
+ } else {
92
98
  stop()
93
99
  }
94
100
  }
@@ -207,11 +213,10 @@ final class OcrModel: ObservableObject {
207
213
 
208
214
  // Phase 1: capture + hash all windows (cheap)
209
215
  for win in windows {
210
- if let cgImage = CGWindowListCreateImage(
211
- .null,
212
- .optionIncludingWindow,
213
- CGWindowID(win.wid),
214
- [.boundsIgnoreFraming, .bestResolution]
216
+ if let cgImage = WindowCapture.image(
217
+ listOption: .optionIncludingWindow,
218
+ windowID: CGWindowID(win.wid),
219
+ imageOption: [.boundsIgnoreFraming, .bestResolution]
215
220
  ) {
216
221
  let hash = self.imageHash(cgImage)
217
222
  newHashes[win.wid] = hash
@@ -314,11 +319,10 @@ final class OcrModel: ObservableObject {
314
319
 
315
320
  let win = windows[index]
316
321
 
317
- if let cgImage = CGWindowListCreateImage(
318
- .null,
319
- .optionIncludingWindow,
320
- CGWindowID(win.wid),
321
- [.boundsIgnoreFraming, .bestResolution]
322
+ if let cgImage = WindowCapture.image(
323
+ listOption: .optionIncludingWindow,
324
+ windowID: CGWindowID(win.wid),
325
+ imageOption: [.boundsIgnoreFraming, .bestResolution]
322
326
  ) {
323
327
  let hash = imageHash(cgImage)
324
328
  newHashes[win.wid] = hash
@@ -0,0 +1,33 @@
1
+ import CoreGraphics
2
+ import Darwin
3
+
4
+ enum WindowCapture {
5
+ // Transitional wrapper for the old CoreGraphics window snapshot API.
6
+ // macOS 26 rejects direct references because ScreenCaptureKit is the supported path;
7
+ // this can return nil if Apple removes the symbol, so preview/OCR callers must degrade.
8
+ private typealias CGWindowListCreateImageFn = @convention(c) (
9
+ CGRect,
10
+ CGWindowListOption,
11
+ CGWindowID,
12
+ CGWindowImageOption
13
+ ) -> Unmanaged<CGImage>?
14
+
15
+ private static let createImage: CGWindowListCreateImageFn? = {
16
+ guard let handle = dlopen("/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics", RTLD_LAZY) else {
17
+ return nil
18
+ }
19
+ guard let symbol = dlsym(handle, "CGWindowListCreateImage") else {
20
+ return nil
21
+ }
22
+ return unsafeBitCast(symbol, to: CGWindowListCreateImageFn.self)
23
+ }()
24
+
25
+ static func image(
26
+ bounds: CGRect = .null,
27
+ listOption: CGWindowListOption,
28
+ windowID: CGWindowID,
29
+ imageOption: CGWindowImageOption
30
+ ) -> CGImage? {
31
+ createImage?(bounds, listOption, windowID, imageOption)?.takeRetainedValue()
32
+ }
33
+ }