@arach/lattices 0.2.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +172 -86
  3. package/apps/mac/Info.plist +43 -0
  4. package/apps/mac/Lattices.app/Contents/Info.plist +43 -0
  5. package/apps/mac/Lattices.app/Contents/MacOS/Lattices +0 -0
  6. package/apps/mac/Lattices.app/Contents/Resources/AppIcon.icns +0 -0
  7. package/apps/mac/Lattices.app/Contents/Resources/docs/assistant-knowledge.md +130 -0
  8. package/apps/mac/Lattices.app/Contents/Resources/tap.wav +0 -0
  9. package/apps/mac/Lattices.app/Contents/_CodeSignature/CodeResources +150 -0
  10. package/apps/mac/Lattices.entitlements +21 -0
  11. package/apps/mac/Resources/Pets/assistant-spark/pet.json +62 -0
  12. package/apps/mac/Resources/Pets/assistant-spark/spritesheet.webp +0 -0
  13. package/apps/mac/Resources/Pets/scout-ranger/pet.json +6 -0
  14. package/apps/mac/Resources/Pets/scout-ranger/spritesheet.webp +0 -0
  15. package/apps/mac/Resources/tap.wav +0 -0
  16. package/assets/AppIcon.icns +0 -0
  17. package/bin/assistant-intelligence.ts +912 -0
  18. package/bin/cli/capture.ts +252 -0
  19. package/bin/cli/daemon.ts +22 -0
  20. package/bin/cli/helpers.ts +105 -0
  21. package/bin/cli/layer.ts +178 -0
  22. package/bin/cli/runs.ts +43 -0
  23. package/bin/cli/search.ts +141 -0
  24. package/bin/cli/session.ts +32 -0
  25. package/bin/client.ts +17 -0
  26. package/bin/cua.ts +26 -0
  27. package/bin/{daemon-client.js → daemon-client.ts} +49 -30
  28. package/bin/handsoff-infer.ts +96 -0
  29. package/bin/handsoff-worker.ts +531 -0
  30. package/bin/infer.ts +424 -0
  31. package/bin/keychain.ts +75 -0
  32. package/bin/lattices-app.ts +655 -0
  33. package/bin/lattices-build +125 -0
  34. package/bin/lattices-build-env.ts +77 -0
  35. package/bin/lattices-dev +362 -0
  36. package/bin/lattices.ts +3260 -0
  37. package/bin/project-twin.ts +645 -0
  38. package/docs/agent-execution-plan.md +562 -0
  39. package/docs/agent-layer-guide.md +207 -0
  40. package/docs/agents.md +233 -0
  41. package/docs/ai-chat-ux-review.md +416 -0
  42. package/docs/api.md +1041 -47
  43. package/docs/app.md +96 -13
  44. package/docs/assistant-knowledge.md +130 -0
  45. package/docs/companion-deck.md +209 -0
  46. package/docs/component-extraction-roadmap.md +392 -0
  47. package/docs/concepts.md +13 -12
  48. package/docs/config.md +83 -10
  49. package/docs/gesture-customization-proposal.md +520 -0
  50. package/docs/handsoff-test-scenarios.md +84 -0
  51. package/docs/hyperspace-grid-snappiness.md +210 -0
  52. package/docs/layers.md +176 -28
  53. package/docs/mouse-gestures.md +244 -0
  54. package/docs/ocr.md +21 -9
  55. package/docs/overview.md +42 -23
  56. package/docs/presentation-execution-review.md +491 -0
  57. package/docs/prompts/hands-off-system.md +382 -0
  58. package/docs/prompts/hands-off-turn.md +30 -0
  59. package/docs/prompts/voice-advisor.md +31 -0
  60. package/docs/prompts/voice-fallback.md +23 -0
  61. package/docs/proposals/LAT-001-gesture-visual-customization.md +522 -0
  62. package/docs/proposals/LAT-002-shared-overlay-canvas.md +353 -0
  63. package/docs/proposals/LAT-003-menu-bar-controller-architecture.md +291 -0
  64. package/docs/proposals/LAT-004-interactive-overlay-actors.md +534 -0
  65. package/docs/proposals/LAT-005-action-runtime-product-spine.md +914 -0
  66. package/docs/proposals/LAT-006-followup-gaps.md +103 -0
  67. package/docs/proposals/LAT-006-runs-and-capture-in-lattices.md +566 -0
  68. package/docs/proposals/LAT-007-unified-app-shell.md +128 -0
  69. package/docs/quickstart.md +8 -12
  70. package/docs/reference/dewey.config.ts +74 -0
  71. package/docs/reference/install-agent.md +79 -0
  72. package/docs/release.md +172 -0
  73. package/docs/repo-structure.md +100 -0
  74. package/docs/terminal-kit.md +87 -0
  75. package/docs/tiling-reference.md +224 -0
  76. package/docs/twins.md +138 -0
  77. package/docs/voice-command-protocol.md +278 -0
  78. package/docs/voice-error-model.md +73 -0
  79. package/docs/voice.md +221 -0
  80. package/package.json +69 -16
  81. package/packages/npm/sdk/cua.d.mts +1 -0
  82. package/packages/npm/sdk/cua.d.ts +188 -0
  83. package/packages/npm/sdk/cua.mjs +376 -0
  84. package/app/Lattices.app/Contents/Info.plist +0 -24
  85. package/app/Package.swift +0 -13
  86. package/app/Sources/ActionRow.swift +0 -61
  87. package/app/Sources/App.swift +0 -10
  88. package/app/Sources/AppDelegate.swift +0 -234
  89. package/app/Sources/AppShellView.swift +0 -62
  90. package/app/Sources/AppTypeClassifier.swift +0 -70
  91. package/app/Sources/AppWindowShell.swift +0 -63
  92. package/app/Sources/CheatSheetHUD.swift +0 -332
  93. package/app/Sources/CommandModeState.swift +0 -1362
  94. package/app/Sources/CommandModeView.swift +0 -1405
  95. package/app/Sources/CommandModeWindow.swift +0 -192
  96. package/app/Sources/CommandPaletteView.swift +0 -307
  97. package/app/Sources/CommandPaletteWindow.swift +0 -134
  98. package/app/Sources/DaemonProtocol.swift +0 -101
  99. package/app/Sources/DaemonServer.swift +0 -414
  100. package/app/Sources/DesktopModel.swift +0 -121
  101. package/app/Sources/DesktopModelTypes.swift +0 -71
  102. package/app/Sources/DiagnosticLog.swift +0 -271
  103. package/app/Sources/EventBus.swift +0 -30
  104. package/app/Sources/HotkeyManager.swift +0 -250
  105. package/app/Sources/HotkeyStore.swift +0 -338
  106. package/app/Sources/InventoryManager.swift +0 -35
  107. package/app/Sources/InventoryPath.swift +0 -43
  108. package/app/Sources/KeyRecorderView.swift +0 -210
  109. package/app/Sources/LatticesApi.swift +0 -1125
  110. package/app/Sources/MainView.swift +0 -467
  111. package/app/Sources/MainWindow.swift +0 -83
  112. package/app/Sources/OcrModel.swift +0 -309
  113. package/app/Sources/OcrStore.swift +0 -295
  114. package/app/Sources/OmniSearchState.swift +0 -283
  115. package/app/Sources/OmniSearchView.swift +0 -288
  116. package/app/Sources/OmniSearchWindow.swift +0 -105
  117. package/app/Sources/OrphanRow.swift +0 -129
  118. package/app/Sources/PaletteCommand.swift +0 -419
  119. package/app/Sources/PermissionChecker.swift +0 -125
  120. package/app/Sources/Preferences.swift +0 -92
  121. package/app/Sources/ProcessModel.swift +0 -199
  122. package/app/Sources/ProcessQuery.swift +0 -151
  123. package/app/Sources/Project.swift +0 -28
  124. package/app/Sources/ProjectRow.swift +0 -368
  125. package/app/Sources/ProjectScanner.swift +0 -121
  126. package/app/Sources/ScreenMapState.swift +0 -2387
  127. package/app/Sources/ScreenMapView.swift +0 -2820
  128. package/app/Sources/ScreenMapWindowController.swift +0 -89
  129. package/app/Sources/SessionManager.swift +0 -72
  130. package/app/Sources/SettingsView.swift +0 -1053
  131. package/app/Sources/SettingsWindow.swift +0 -20
  132. package/app/Sources/TabGroupRow.swift +0 -178
  133. package/app/Sources/Terminal.swift +0 -259
  134. package/app/Sources/TerminalQuery.swift +0 -156
  135. package/app/Sources/TerminalSynthesizer.swift +0 -200
  136. package/app/Sources/Theme.swift +0 -163
  137. package/app/Sources/TilePickerView.swift +0 -209
  138. package/app/Sources/TmuxModel.swift +0 -53
  139. package/app/Sources/TmuxQuery.swift +0 -81
  140. package/app/Sources/WindowTiler.swift +0 -1755
  141. package/app/Sources/WorkspaceManager.swift +0 -434
  142. package/bin/lattices-app.js +0 -221
  143. package/bin/lattices.js +0 -1418
@@ -0,0 +1,73 @@
1
+ # Voice Error Model
2
+
3
+ ## Goal + anchors
4
+
5
+ Use one error vocabulary for Mac voice capture/execution and iPad relay/status. The canonical protocol says Lattices borrows Vox for capture and never owns the mic directly (`docs/voice-command-protocol.md:5-7`), but the shared runtime already has a cross-platform `DeckVoiceState` slot (`swift/Sources/DeckKit/DeckRuntimeSnapshot.swift:49-68`). Current Mac code exposes local strings (`VoxError`, `executionResult`) instead of structured errors (`apps/mac/Sources/VoxClient.swift:43-59`, `apps/mac/Sources/AudioProvider.swift:343-349`); iPad has only a generic `errorMessage` (`apps/ios/Sources/DeckStore.swift:18`). Normalize at DeckKit, then let each surface render the same object.
6
+
7
+ ## Error structure
8
+
9
+ Prefer `DeckVoiceError` now; if later reused for trackpad/deck actions, lift the same shape to `LatsError`.
10
+
11
+ ```swift
12
+ public struct DeckVoiceError: Codable, Equatable, Identifiable, Sendable {
13
+ public var id: String // uuid or request id
14
+ public var code: DeckVoiceErrorCode
15
+ public var severity: DeckErrorSeverity
16
+ public var recoverable: Bool
17
+ public var retry: DeckRetryHint?
18
+ public var source: DeckErrorSource // mac, ipad, vox, daemon, intent, bridge
19
+ public var owner: String? // e.g. "Vox", "Lattices", "Bridge"
20
+ public var message: String // cockpit copy, already human-readable
21
+ public var remediation: DeckRemediationAction?
22
+ public var occurredAt: Date
23
+ public var detail: String? // diagnostic-only
24
+ }
25
+
26
+ public enum DeckVoiceErrorCode: String, Codable, Sendable { case vox_unreachable, daemon_unreachable, network, connection_lost, mic_denied, accessibility_denied, mic_busy, no_active_target, vox_not_running, vox_loading, intent_unresolved, action_failed, transcription_failed, empty_transcript, language_unsupported }
27
+ public enum DeckErrorSeverity: String, Codable, Sendable { case info, warning, error, blocked }
28
+ public enum DeckRetryHint: String, Codable, Sendable { case silent, immediate, afterLaunch, userAction }
29
+ public enum DeckErrorSource: String, Codable, Sendable { case mac, ipad, vox, daemon, intent, bridge }
30
+ public enum DeckRemediationAction: Codable, Equatable, Sendable {
31
+ case openVox, openSystemSettings(kind: String), retryVoice, openDiagnostics, chooseTarget
32
+ }
33
+ ```
34
+
35
+ Add `var error: DeckVoiceError?` and optionally `var lastError: DeckVoiceError?` to `DeckVoiceState`, preserving existing `phase`, transcript, and provider fields (`swift/Sources/DeckKit/DeckRuntimeSnapshot.swift:49-68`). Codes should stay stable string raw values for JSON logs and iPad bridge snapshots.
36
+
37
+ ## Categories
38
+
39
+ | Category | Codes | Recovery rule |
40
+ |---|---|---|
41
+ | Connection | `vox_unreachable`, `daemon_unreachable`, `network`, `connection_lost` | Recoverable unless active capture was lost. Silent reconnect when idle; visible banner during `listening`/`transcribing`. |
42
+ | Permission | `mic_denied`, `accessibility_denied` | Needs user action. `mic_denied` is Mac/Vox-owned; iPad only relays it. `accessibility_denied` blocks execution/navigation. |
43
+ | State | `mic_busy { owner }`, `no_active_target`, `vox_not_running`, `vox_loading` | Usually recoverable. `mic_busy` waits for owner; `vox_not_running` supports launch-on-demand. |
44
+ | Execution | `intent_unresolved`, `action_failed`, `transcription_failed` | Recoverable by retry or edited command; escalate to log if repeated. |
45
+ | Validation | `empty_transcript`, `language_unsupported` | Recoverable; no scary chrome. Treat as a missed command, not a crash. |
46
+
47
+ Copy examples: `Mic in use by Vox — finish memo first`, `No target window`, `Vox offline — starting`, `Connection lost — press again`, `Intent not found`.
48
+
49
+ ## Presentation patterns
50
+
51
+ **Mac VoiceCommandWindow.** Keep the three-column cockpit. The top mic bar already owns live state (`connecting...`, `processing...`; `apps/mac/Sources/VoiceCommandWindow.swift:692-719`); render the active error as a compact red/amber status chip there. The center column uses `commandSection` cards (`apps/mac/Sources/VoiceCommandWindow.swift:1287-1304`): show a single `blocked`/`needs action` card only when the user can do something. The footer already has key chips (`apps/mac/Sources/VoiceCommandWindow.swift:1308-1348`); replace the generic command list with contextual remediation: `⌥ Retry`, `Return Open Vox`, `⌘, Permissions`. Logs stay in the right rail, using existing level colors (`apps/mac/Sources/VoiceCommandWindow.swift:1112-1150`).
52
+
53
+ **Mac HUD.** `HUDTopBar.voiceStatus` already has dot, label, transcript, response (`apps/mac/Sources/HUDTopBar.swift:134-198`). Add severity tint: green idle/listening, amber connecting/recoverable, red blocked. For active voice errors, HUD shows a one-line banner in the top bar; no sheet.
54
+
55
+ **iPad Home.** Add `HomeVoiceOverlay` as the full voice modal for active relay: title row `VOICE`, phase, transcript, Mac owner, and one remediation button. The bottom bar already has dense status slots and `hold·space` (`apps/ios/Sources/Home/HomeBottomBar.swift:58-68`, `apps/ios/Sources/Home/HomeBottomBar.swift:129-148`); render idle/recoverable errors inline there (`voice · reconnecting`, `voice · Vox offline`). Use a deck overlay banner only when an issued iPad action failed. Use sheets only for permissions/pairing because they need human action. This follows the chrome rule: do not remove noisy UI; replace it with state that answers “what am I controlling, who is listening, what failed?” (`/Users/arach/.claude/projects/-Users-arach-dev-lattices/memory/feedback_chrome_design.md:11-13`).
56
+
57
+ ## Unhappy-path prescriptions
58
+
59
+ **Launch Vox on demand.** Spec flow is detect installed/not running, open Vox, show `Starting Vox...`, wait up to 10s, retry `startDictation`, then fail with manual-open copy (`docs/voice-command-protocol.md:73-89`). Current Mac waits 2s after `connect()` (`apps/mac/Sources/VoiceCommandWindow.swift:290-313`); design target is `vox_not_running` → `vox_loading` → retry → either clear error or `vox_unreachable` with `openVox`.
60
+
61
+ **Mic busy.** Preserve owner attribution from protocol (`docs/voice-command-protocol.md:127-135`). `mic_busy(owner: "Vox")` is warning, recoverable, retry hint `userAction`; message: `Mic in use by Vox — finish memo first`. If owner is unknown: `Mic busy — wait for current recording`.
62
+
63
+ **Connection recovery.** If idle, reconnect silently and write log only. If active, show red `Connection lost`; do not auto-retry captured audio because Vox cancels dropped sockets (`docs/voice-command-protocol.md:174-188`). iPad shows `Mac voice link lost` if bridge lost, not `network` unless the iPad transport failed.
64
+
65
+ **JSONL.** Add `~/.lattices/voice.jsonl` beside `lattices.log` (current log path is `~/.lattices/lattices.log`; `apps/mac/Sources/DiagnosticLog.swift:40-59`). Shape:
66
+
67
+ ```json
68
+ {"ts":"2026-04-27T14:03:11.120Z","platform":"mac","sessionId":"...","phase":"listening","event":"error","error":{"code":"mic_busy","severity":"warning","recoverable":true,"source":"vox","owner":"Vox","message":"Mic in use by Vox — finish memo first"},"transcript":null,"intent":null,"durationMs":820}
69
+ ```
70
+
71
+ ## Cross-platform conventions
72
+
73
+ Tone: terse cockpit, no apologies. Prefer noun-state-action: `Vox offline — starting`, `No target — pick window`, `Access denied — enable Accessibility`. Tint maps to existing palettes: Mac `Palette.detach` amber and `Palette.kill` red (`apps/mac/Sources/Theme.swift:19-23`); iPad `LatsPalette.amber/red` (`apps/ios/Sources/LatsDeckScreen.swift:19-25`). Icons: `mic.fill` live, `mic.slash` denied, `waveform.badge.exclamationmark` transcription, `bolt.trianglebadge.exclamationmark` execution, `wifi.exclamationmark` connection, `scope` target. Ownership: Mac owns Vox, mic, Accessibility, intent execution, and JSONL. iPad owns relay/bridge/network presentation and never claims direct mic capture.
package/docs/voice.md ADDED
@@ -0,0 +1,221 @@
1
+ ---
2
+ title: Voice Commands
3
+ description: Natural language voice control for window management
4
+ order: 7
5
+ ---
6
+
7
+ Voice commands let you control Lattices by speaking. Press **Hyper+D**
8
+ to open the voice command window, hold **Option** to speak, release to
9
+ stop. Lattices transcribes your speech via Vox,
10
+ matches it to an intent, and executes it.
11
+
12
+ ## Quick start
13
+
14
+ 1. Install Vox (provides mic + transcription)
15
+ 2. Connect an Assistant provider in **Settings > AI**
16
+ 3. Press **Hyper+D** to open the voice command window
17
+ 4. Hold **Option** and speak a command
18
+ 5. Release **Option** — Lattices transcribes and executes
19
+
20
+ ## Keyboard shortcuts
21
+
22
+ | Key | Action |
23
+ |-----|--------|
24
+ | **Hyper+D** | Open/close voice command window |
25
+ | **⌥ (hold)** | Push-to-talk — hold to record, release to stop |
26
+ | **Tab** | Arm/disarm the mic |
27
+ | **Escape** | Cancel recording or dismiss window |
28
+
29
+ ## Built-in commands
30
+
31
+ ### Search
32
+
33
+ Find windows by app name, title, content, or category.
34
+
35
+ ```
36
+ "Find all vox windows"
37
+ "Find terminals" → expands to iTerm, Terminal, Warp, etc.
38
+ "Show me all browsers" → expands to Safari, Chrome, Firefox, Arc, etc.
39
+ "Where is my editor?" → expands to VS Code, Cursor, Xcode, etc.
40
+ ```
41
+
42
+ Category synonyms are built in — saying "terminals", "browsers", "editors",
43
+ "chat", "music", "mail", or "notes" automatically expands to search for
44
+ the actual app names.
45
+
46
+ ### Tile
47
+
48
+ Move windows to screen positions.
49
+
50
+ ```
51
+ "Tile this left"
52
+ "Snap to the right half"
53
+ "Maximize the window"
54
+ "Put this in the top right corner"
55
+ ```
56
+
57
+ Voice tiling should resolve into the same canonical daemon mutation used
58
+ by other agent surfaces: `window.place`.
59
+
60
+ ### Focus
61
+
62
+ Bring a window or app to the front.
63
+
64
+ ```
65
+ "Focus Safari"
66
+ "Switch to Slack"
67
+ "Go to the lattices window"
68
+ ```
69
+
70
+ ### Open / Launch
71
+
72
+ Open applications or project workspaces.
73
+
74
+ ```
75
+ "Open Spotify"
76
+ "Launch the vox project"
77
+ ```
78
+
79
+ ### Kill
80
+
81
+ Close windows or quit applications.
82
+
83
+ ```
84
+ "Kill this window"
85
+ "Close Safari"
86
+ "Quit Spotify"
87
+ ```
88
+
89
+ ### Scan
90
+
91
+ Trigger an OCR scan of visible windows.
92
+
93
+ ```
94
+ "Scan the screen"
95
+ "Read what's on screen"
96
+ ```
97
+
98
+ ### Other
99
+
100
+ ```
101
+ "List all windows"
102
+ "Show my sessions"
103
+ "Switch to layer 2"
104
+ "Help"
105
+ ```
106
+
107
+ ## AI advisor
108
+
109
+ Every voice command can ask the selected Assistant provider for
110
+ commentary and follow-up suggestions in the **AI corner** (bottom-right
111
+ of the voice command window).
112
+
113
+ When local matching handles the command well, the AI corner shows
114
+ "no AI needed" with an optional "ask AI" button. When the advisor
115
+ has something useful, it shows a one-line comment and an actionable
116
+ suggestion button.
117
+
118
+ ### How it works
119
+
120
+ 1. You speak a command
121
+ 2. Local intent matching runs immediately (fast, free)
122
+ 3. The selected Assistant provider runs in parallel when connected
123
+ 4. If the advisor suggests something, a button appears in the AI corner
124
+ 5. Click the suggestion to execute it
125
+ 6. If you engage with a suggestion that the local matcher missed,
126
+ it's recorded in `~/.lattices/advisor-learning.jsonl` for future
127
+ improvement
128
+
129
+ ## Hands-off inference
130
+
131
+ Hands-off voice uses the shared inference wrapper in `bin/infer.ts`.
132
+ By default it chooses the lowest-latency configured provider and, when
133
+ Groq credentials are present, uses `groq/llama-3.1-8b-instant`.
134
+
135
+ Credentials are read from process env, `.env.local`, `.env`,
136
+ `~/.lattices/inference.json`, then `~/.config/speakeasy/settings.json`.
137
+ For Groq, either `GROQ_API_KEY` or the common typo `GROK_API_KEY` works
138
+ when the key has Groq's `gsk_` prefix.
139
+
140
+ Override the voice engine if needed:
141
+
142
+ ```bash
143
+ LATTICES_VOICE_PROVIDER=groq
144
+ LATTICES_VOICE_MODEL=llama-3.1-8b-instant
145
+ ```
146
+
147
+ ## Configuration
148
+
149
+ Open **Settings > AI** to configure:
150
+
151
+ | Setting | Default | Description |
152
+ |---------|---------|-------------|
153
+ | Assistant provider | OpenAI Codex | Provider used by the in-app chat and provider-backed voice advisor. Supports OAuth providers such as GitHub Copilot and OpenAI Codex plus API-key providers. |
154
+ | Pi runtime | Auto-detected | Installs or refreshes the provider runtime used by the in-app assistant. |
155
+ | Provider credentials | Not authenticated | Sign in with OAuth or save an API key locally for the selected provider. |
156
+
157
+ ## Layout
158
+
159
+ The voice command window has four sections:
160
+
161
+ | Section | Position | Content |
162
+ |---------|----------|---------|
163
+ | **History** | Left column | Past commands with expandable details |
164
+ | **Voice Command** | Center column | Current transcript, matched intent, results |
165
+ | **Log** | Top-right | Rolling diagnostic log (last 12 entries) |
166
+ | **AI Corner** | Bottom-right | Advisor commentary, suggestions, provider readiness |
167
+
168
+ ## Search architecture
169
+
170
+ Voice search uses the same backend as `lattices search`:
171
+
172
+ 1. **Quick search** — window titles, app names, session tags (instant)
173
+ 2. **Complete search** — adds terminal cwd/processes + OCR content
174
+ 3. **Synonym expansion** — category terms like "terminals" expand to
175
+ actual app names before searching
176
+ 4. **Query cleanup** — strips natural language qualifiers ("and sort by...",
177
+ "please", "for me") before searching
178
+
179
+ ## Processing resilience
180
+
181
+ - **15-second timeout** — if processing doesn't complete, returns to idle
182
+ - **Cancellation on dismiss** — closing the window cancels in-flight work
183
+ - **Double-execution prevention** — streaming and stop callbacks can't
184
+ both fire the intent
185
+
186
+ ## Advisor learning
187
+
188
+ When the local matcher fails but the AI advisor suggests something that
189
+ you engage with, the interaction is recorded:
190
+
191
+ ```
192
+ ~/.lattices/advisor-learning.jsonl
193
+ ```
194
+
195
+ Each line is a JSON object:
196
+
197
+ ```json
198
+ {
199
+ "timestamp": "2026-03-15T18:30:00.000Z",
200
+ "transcript": "find all terminals",
201
+ "localIntent": "search",
202
+ "localSlots": {"query": "terminals"},
203
+ "localResultCount": 0,
204
+ "advisorIntent": "search",
205
+ "advisorSlots": {"query": "iterm"},
206
+ "advisorLabel": "Search iTerm"
207
+ }
208
+ ```
209
+
210
+ This dataset captures where the local system falls short and what the
211
+ right answer was. Future work can mine it for automatic synonym
212
+ mappings and phrase pattern improvements.
213
+
214
+ ## Requirements
215
+
216
+ - **Vox** — provides microphone access and
217
+ speech-to-text transcription
218
+ - **Assistant provider** — enables provider-backed AI suggestions from
219
+ **Settings > AI** (optional, voice commands still work without it)
220
+ - **Accessibility** permission — for window tiling and focus
221
+ - **Screen Recording** permission — for window discovery
package/package.json CHANGED
@@ -1,19 +1,37 @@
1
1
  {
2
2
  "name": "@arach/lattices",
3
- "version": "0.2.0",
4
- "description": "macOS workspace manager with window tiling, OCR, and a 30-method daemon API for AI agents",
3
+ "version": "0.6.1",
4
+ "description": "macOS workspace manager menu bar app, CLI, and WebSocket daemon API for tiling, search, layers, and agent control",
5
+ "homepage": "https://lattices.dev",
6
+ "bugs": {
7
+ "url": "https://github.com/arach/lattices/issues"
8
+ },
9
+ "publishConfig": {
10
+ "access": "public",
11
+ "provenance": true
12
+ },
13
+ "packageManager": "bun@1.3.11",
14
+ "engines": {
15
+ "node": ">=18"
16
+ },
5
17
  "bin": {
6
- "lattices": "./bin/lattices.js",
7
- "lattices-app": "./bin/lattices-app.js"
18
+ "lattices": "./bin/lattices.ts",
19
+ "lattices-app": "./bin/lattices-app.ts"
8
20
  },
9
21
  "keywords": [
22
+ "lattices",
23
+ "macos",
24
+ "workspace",
25
+ "window-manager",
10
26
  "tmux",
27
+ "daemon",
28
+ "ai-agents",
11
29
  "claude",
12
- "dev-server",
13
30
  "developer-tools",
14
31
  "cli",
15
32
  "terminal",
16
- "multiplexer"
33
+ "ocr",
34
+ "tiling"
17
35
  ],
18
36
  "repository": {
19
37
  "type": "git",
@@ -21,22 +39,57 @@
21
39
  },
22
40
  "license": "MIT",
23
41
  "exports": {
24
- ".": "./bin/lattices.js",
25
- "./daemon-client": "./bin/daemon-client.js"
42
+ ".": "./bin/client.ts",
43
+ "./assistant-intelligence": "./bin/assistant-intelligence.ts",
44
+ "./cua": "./bin/cua.ts",
45
+ "./daemon-client": "./bin/daemon-client.ts",
46
+ "./project-twin": "./bin/project-twin.ts"
26
47
  },
27
48
  "scripts": {
28
- "dev": "bun --cwd docs-site dev"
49
+ "dev": "bun --cwd apps/site dev",
50
+ "site:build": "bun --cwd apps/site build",
51
+ "studio:dev": "bun --cwd apps/studio dev",
52
+ "studio:build": "bun --cwd apps/studio build",
53
+ "docs:agent": "bun --cwd apps/site agent-docs",
54
+ "check": "bun run check:types && bun run check:app",
55
+ "check:types": "tsc --noEmit",
56
+ "check:app": "env CLANG_MODULE_CACHE_PATH=/tmp/lattices-clang-cache SWIFTPM_TESTS_MODULECACHE=/tmp/lattices-swiftpm-cache swift build --package-path apps/mac",
57
+ "test": "node --experimental-strip-types --test tests/cli.test.mjs",
58
+ "test:cli": "node --experimental-strip-types --test tests/cli.test.mjs",
59
+ "test:e2e": "node --experimental-strip-types --test tests/e2e-daemon.test.mjs",
60
+ "test:e2e:voice": "node --experimental-strip-types tests/eval-voice.js",
61
+ "typecheck": "bun run check:types",
62
+ "build:app-bundle": "bash ./bin/lattices-build package",
63
+ "build:dev-app": "bash ./bin/lattices-build dev",
64
+ "build:dist": "bash ./bin/lattices-build dist",
65
+ "build:dist:local": "bash ./bin/lattices-build dist:local",
66
+ "prepack": "bash ./bin/lattices-build package"
29
67
  },
30
68
  "type": "module",
31
- "engines": {
32
- "node": ">=18"
33
- },
34
69
  "os": ["darwin"],
35
70
  "files": [
36
71
  "bin",
37
- "app/Package.swift",
38
- "app/Sources",
39
- "app/Lattices.app/Contents/Info.plist",
72
+ "packages/npm/sdk/cua.mjs",
73
+ "packages/npm/sdk/cua.d.ts",
74
+ "packages/npm/sdk/cua.d.mts",
75
+ "apps/mac/Info.plist",
76
+ "apps/mac/Lattices.app",
77
+ "apps/mac/Lattices.entitlements",
78
+ "apps/mac/Resources",
79
+ "assets/AppIcon.icns",
40
80
  "docs"
41
- ]
81
+ ],
82
+ "devDependencies": {
83
+ "bun-types": "^1.3.10",
84
+ "typescript": "^5.9.3"
85
+ },
86
+ "dependencies": {
87
+ "@ai-sdk/anthropic": "^3.0.58",
88
+ "@ai-sdk/google": "^3.0.43",
89
+ "@ai-sdk/openai": "^3.0.41",
90
+ "@ai-sdk/xai": "^3.0.67",
91
+ "@arach/speakeasy": "^0.2.8",
92
+ "ai": "^6.0.116",
93
+ "zod": "^3.25.76"
94
+ }
42
95
  }
@@ -0,0 +1 @@
1
+ export * from "./cua.d.ts";
@@ -0,0 +1,188 @@
1
+ import type { z } from "zod";
2
+
3
+ export type ComputerTreatment = "observe" | "stage" | "present" | "execute";
4
+ export type ComputerClickTransport =
5
+ | "auto"
6
+ | "ax"
7
+ | "accessibility"
8
+ | "pointer"
9
+ | "mouse"
10
+ | "hardware";
11
+ export type CursorStyle = "spotlight" | "pulse" | "marker";
12
+ export type CursorShape =
13
+ | "arrow"
14
+ | "chevron"
15
+ | "facet"
16
+ | "shard"
17
+ | "wedge"
18
+ | "prism"
19
+ | "notch"
20
+ | "needle"
21
+ | "petal"
22
+ | "kite";
23
+ export type CursorSize = "tiny" | "small" | "regular" | "large";
24
+ export type CursorTrail = "none" | "thread" | "ribbon" | "spark" | "comet" | "route";
25
+ export type CursorMotion =
26
+ | "glide"
27
+ | "snap"
28
+ | "float"
29
+ | "rush"
30
+ | "crawl"
31
+ | "accelerate"
32
+ | "teleport"
33
+ | "spring"
34
+ | "magnet"
35
+ | "slingshot";
36
+ export type CursorTrajectory = "straight" | "soft" | "arc" | "swoop" | "overshoot";
37
+ export type CursorGlow = "none" | "soft" | "halo" | "comet";
38
+ export type CursorIdle =
39
+ | "still"
40
+ | "breathe"
41
+ | "wiggle"
42
+ | "orbit"
43
+ | "hover"
44
+ | "nod"
45
+ | "drift"
46
+ | "shimmer"
47
+ | "blink"
48
+ | "tremble";
49
+ export type CursorEdge =
50
+ | "none"
51
+ | "pulse"
52
+ | "ripple"
53
+ | "tick"
54
+ | "reticle"
55
+ | "blink"
56
+ | "spark"
57
+ | "underline"
58
+ | "echo"
59
+ | "scan"
60
+ | "pin";
61
+ export type CursorSound = "none" | "tick" | "click" | "engage" | "chime";
62
+ export type CaptionPlacement =
63
+ | "top-left"
64
+ | "top-right"
65
+ | "bottom-left"
66
+ | "bottom-right"
67
+ | "top-center"
68
+ | "top"
69
+ | "center"
70
+ | "middle"
71
+ | "near-cursor"
72
+ | "cursor";
73
+
74
+ export interface WindowTarget {
75
+ wid?: number;
76
+ app?: string;
77
+ title?: string;
78
+ }
79
+
80
+ export interface ActionBase {
81
+ treatment?: ComputerTreatment;
82
+ dryRun?: boolean;
83
+ capture?: boolean;
84
+ source?: string;
85
+ }
86
+
87
+ export interface PointTarget {
88
+ x?: number;
89
+ y?: number;
90
+ xRatio?: number;
91
+ yRatio?: number;
92
+ }
93
+
94
+ export interface ComputerClickParams extends WindowTarget, PointTarget, ActionBase {
95
+ button?: "left" | "right" | "secondary" | "context";
96
+ transport?: ComputerClickTransport;
97
+ axLabel?: string;
98
+ targetText?: string;
99
+ noFocus?: boolean;
100
+ label?: string;
101
+ }
102
+
103
+ export interface ComputerMagicCursorParams extends WindowTarget, PointTarget, ActionBase {
104
+ style?: CursorStyle;
105
+ appearance?: CursorStyle;
106
+ shape?: CursorShape;
107
+ angleDeg?: number;
108
+ size?: CursorSize;
109
+ color?: string;
110
+ durationMs?: number;
111
+ label?: string;
112
+ caption?: string;
113
+ captionTitle?: string;
114
+ captionBody?: string;
115
+ captionDetail?: string;
116
+ captionTags?: string;
117
+ captionMode?: "auto" | "selection";
118
+ captionEyebrow?: string;
119
+ captionLeadMs?: number;
120
+ captionSound?: CursorSound;
121
+ captionPlacement?: CaptionPlacement;
122
+ captionMargin?: number;
123
+ captionX?: number;
124
+ captionY?: number;
125
+ captionXRatio?: number;
126
+ captionYRatio?: number;
127
+ captionLeftRatio?: number;
128
+ captionTopRatio?: number;
129
+ sound?: CursorSound;
130
+ sfx?: CursorSound;
131
+ showCaption?: boolean;
132
+ captionSelections?: boolean;
133
+ treatmentLabel?: string;
134
+ variant?: string;
135
+ trail?: CursorTrail;
136
+ pathStyle?: CursorTrail;
137
+ motion?: CursorMotion;
138
+ trajectory?: CursorTrajectory;
139
+ glow?: CursorGlow;
140
+ bloom?: CursorGlow;
141
+ idle?: CursorIdle;
142
+ settle?: CursorIdle;
143
+ presence?: CursorIdle;
144
+ edge?: CursorEdge;
145
+ edgeEffect?: CursorEdge;
146
+ arrival?: CursorEdge;
147
+ typewriter?: boolean;
148
+ typing?: boolean;
149
+ typeIntervalMs?: number;
150
+ typingIntervalMs?: number;
151
+ text?: string;
152
+ append?: boolean;
153
+ fromX?: number;
154
+ fromY?: number;
155
+ fromXRatio?: number;
156
+ fromYRatio?: number;
157
+ }
158
+
159
+ export interface CuaClientOptions {
160
+ defaultTimeoutMs?: number;
161
+ }
162
+
163
+ export interface CuaClient {
164
+ click(params: ComputerClickParams): Promise<unknown>;
165
+ magicCursor(params: ComputerMagicCursorParams): Promise<unknown>;
166
+ }
167
+
168
+ export declare const computerTreatmentSchema: z.ZodType<ComputerTreatment>;
169
+ export declare const computerClickTransportSchema: z.ZodType<ComputerClickTransport>;
170
+ export declare const cursorStyleSchema: z.ZodType<CursorStyle>;
171
+ export declare const cursorShapeSchema: z.ZodType<CursorShape>;
172
+ export declare const cursorSizeSchema: z.ZodType<CursorSize>;
173
+ export declare const cursorTrailSchema: z.ZodType<CursorTrail>;
174
+ export declare const cursorMotionSchema: z.ZodType<CursorMotion>;
175
+ export declare const cursorTrajectorySchema: z.ZodType<CursorTrajectory>;
176
+ export declare const cursorGlowSchema: z.ZodType<CursorGlow>;
177
+ export declare const cursorIdleSchema: z.ZodType<CursorIdle>;
178
+ export declare const cursorEdgeSchema: z.ZodType<CursorEdge>;
179
+ export declare const cursorSoundSchema: z.ZodType<CursorSound>;
180
+ export declare const captionPlacementSchema: z.ZodType<CaptionPlacement>;
181
+ export declare const computerClickParamsSchema: z.ZodType<ComputerClickParams>;
182
+ export declare const computerMagicCursorParamsSchema: z.ZodType<ComputerMagicCursorParams>;
183
+
184
+ export declare function createCuaClient(options?: CuaClientOptions): CuaClient;
185
+ export declare function click(params: ComputerClickParams): Promise<unknown>;
186
+ export declare function magicCursor(params: ComputerMagicCursorParams): Promise<unknown>;
187
+
188
+ export declare const cua: CuaClient;