@arach/lattices 0.2.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +172 -86
- package/apps/mac/Info.plist +43 -0
- package/apps/mac/Lattices.app/Contents/Info.plist +43 -0
- package/apps/mac/Lattices.app/Contents/MacOS/Lattices +0 -0
- package/apps/mac/Lattices.app/Contents/Resources/AppIcon.icns +0 -0
- package/apps/mac/Lattices.app/Contents/Resources/docs/assistant-knowledge.md +130 -0
- package/apps/mac/Lattices.app/Contents/Resources/tap.wav +0 -0
- package/apps/mac/Lattices.app/Contents/_CodeSignature/CodeResources +150 -0
- package/apps/mac/Lattices.entitlements +21 -0
- package/apps/mac/Resources/Pets/assistant-spark/pet.json +62 -0
- package/apps/mac/Resources/Pets/assistant-spark/spritesheet.webp +0 -0
- package/apps/mac/Resources/Pets/scout-ranger/pet.json +6 -0
- package/apps/mac/Resources/Pets/scout-ranger/spritesheet.webp +0 -0
- package/apps/mac/Resources/tap.wav +0 -0
- package/assets/AppIcon.icns +0 -0
- package/bin/assistant-intelligence.ts +912 -0
- package/bin/cli/capture.ts +252 -0
- package/bin/cli/daemon.ts +22 -0
- package/bin/cli/helpers.ts +105 -0
- package/bin/cli/layer.ts +178 -0
- package/bin/cli/runs.ts +43 -0
- package/bin/cli/search.ts +141 -0
- package/bin/cli/session.ts +32 -0
- package/bin/client.ts +17 -0
- package/bin/cua.ts +26 -0
- package/bin/{daemon-client.js → daemon-client.ts} +49 -30
- package/bin/handsoff-infer.ts +96 -0
- package/bin/handsoff-worker.ts +531 -0
- package/bin/infer.ts +424 -0
- package/bin/keychain.ts +75 -0
- package/bin/lattices-app.ts +655 -0
- package/bin/lattices-build +125 -0
- package/bin/lattices-build-env.ts +77 -0
- package/bin/lattices-dev +362 -0
- package/bin/lattices.ts +3260 -0
- package/bin/project-twin.ts +645 -0
- package/docs/agent-execution-plan.md +562 -0
- package/docs/agent-layer-guide.md +207 -0
- package/docs/agents.md +233 -0
- package/docs/ai-chat-ux-review.md +416 -0
- package/docs/api.md +1041 -47
- package/docs/app.md +96 -13
- package/docs/assistant-knowledge.md +130 -0
- package/docs/companion-deck.md +209 -0
- package/docs/component-extraction-roadmap.md +392 -0
- package/docs/concepts.md +13 -12
- package/docs/config.md +83 -10
- package/docs/gesture-customization-proposal.md +520 -0
- package/docs/handsoff-test-scenarios.md +84 -0
- package/docs/hyperspace-grid-snappiness.md +210 -0
- package/docs/layers.md +176 -28
- package/docs/mouse-gestures.md +244 -0
- package/docs/ocr.md +21 -9
- package/docs/overview.md +42 -23
- package/docs/presentation-execution-review.md +491 -0
- package/docs/prompts/hands-off-system.md +382 -0
- package/docs/prompts/hands-off-turn.md +30 -0
- package/docs/prompts/voice-advisor.md +31 -0
- package/docs/prompts/voice-fallback.md +23 -0
- package/docs/proposals/LAT-001-gesture-visual-customization.md +522 -0
- package/docs/proposals/LAT-002-shared-overlay-canvas.md +353 -0
- package/docs/proposals/LAT-003-menu-bar-controller-architecture.md +291 -0
- package/docs/proposals/LAT-004-interactive-overlay-actors.md +534 -0
- package/docs/proposals/LAT-005-action-runtime-product-spine.md +914 -0
- package/docs/proposals/LAT-006-followup-gaps.md +103 -0
- package/docs/proposals/LAT-006-runs-and-capture-in-lattices.md +566 -0
- package/docs/proposals/LAT-007-unified-app-shell.md +128 -0
- package/docs/quickstart.md +8 -12
- package/docs/reference/dewey.config.ts +74 -0
- package/docs/reference/install-agent.md +79 -0
- package/docs/release.md +172 -0
- package/docs/repo-structure.md +100 -0
- package/docs/terminal-kit.md +87 -0
- package/docs/tiling-reference.md +224 -0
- package/docs/twins.md +138 -0
- package/docs/voice-command-protocol.md +278 -0
- package/docs/voice-error-model.md +73 -0
- package/docs/voice.md +221 -0
- package/package.json +69 -16
- package/packages/npm/sdk/cua.d.mts +1 -0
- package/packages/npm/sdk/cua.d.ts +188 -0
- package/packages/npm/sdk/cua.mjs +376 -0
- package/app/Lattices.app/Contents/Info.plist +0 -24
- package/app/Package.swift +0 -13
- package/app/Sources/ActionRow.swift +0 -61
- package/app/Sources/App.swift +0 -10
- package/app/Sources/AppDelegate.swift +0 -234
- package/app/Sources/AppShellView.swift +0 -62
- package/app/Sources/AppTypeClassifier.swift +0 -70
- package/app/Sources/AppWindowShell.swift +0 -63
- package/app/Sources/CheatSheetHUD.swift +0 -332
- package/app/Sources/CommandModeState.swift +0 -1362
- package/app/Sources/CommandModeView.swift +0 -1405
- package/app/Sources/CommandModeWindow.swift +0 -192
- package/app/Sources/CommandPaletteView.swift +0 -307
- package/app/Sources/CommandPaletteWindow.swift +0 -134
- package/app/Sources/DaemonProtocol.swift +0 -101
- package/app/Sources/DaemonServer.swift +0 -414
- package/app/Sources/DesktopModel.swift +0 -121
- package/app/Sources/DesktopModelTypes.swift +0 -71
- package/app/Sources/DiagnosticLog.swift +0 -271
- package/app/Sources/EventBus.swift +0 -30
- package/app/Sources/HotkeyManager.swift +0 -250
- package/app/Sources/HotkeyStore.swift +0 -338
- package/app/Sources/InventoryManager.swift +0 -35
- package/app/Sources/InventoryPath.swift +0 -43
- package/app/Sources/KeyRecorderView.swift +0 -210
- package/app/Sources/LatticesApi.swift +0 -1125
- package/app/Sources/MainView.swift +0 -467
- package/app/Sources/MainWindow.swift +0 -83
- package/app/Sources/OcrModel.swift +0 -309
- package/app/Sources/OcrStore.swift +0 -295
- package/app/Sources/OmniSearchState.swift +0 -283
- package/app/Sources/OmniSearchView.swift +0 -288
- package/app/Sources/OmniSearchWindow.swift +0 -105
- package/app/Sources/OrphanRow.swift +0 -129
- package/app/Sources/PaletteCommand.swift +0 -419
- package/app/Sources/PermissionChecker.swift +0 -125
- package/app/Sources/Preferences.swift +0 -92
- package/app/Sources/ProcessModel.swift +0 -199
- package/app/Sources/ProcessQuery.swift +0 -151
- package/app/Sources/Project.swift +0 -28
- package/app/Sources/ProjectRow.swift +0 -368
- package/app/Sources/ProjectScanner.swift +0 -121
- package/app/Sources/ScreenMapState.swift +0 -2387
- package/app/Sources/ScreenMapView.swift +0 -2820
- package/app/Sources/ScreenMapWindowController.swift +0 -89
- package/app/Sources/SessionManager.swift +0 -72
- package/app/Sources/SettingsView.swift +0 -1053
- package/app/Sources/SettingsWindow.swift +0 -20
- package/app/Sources/TabGroupRow.swift +0 -178
- package/app/Sources/Terminal.swift +0 -259
- package/app/Sources/TerminalQuery.swift +0 -156
- package/app/Sources/TerminalSynthesizer.swift +0 -200
- package/app/Sources/Theme.swift +0 -163
- package/app/Sources/TilePickerView.swift +0 -209
- package/app/Sources/TmuxModel.swift +0 -53
- package/app/Sources/TmuxQuery.swift +0 -81
- package/app/Sources/WindowTiler.swift +0 -1755
- package/app/Sources/WorkspaceManager.swift +0 -434
- package/bin/lattices-app.js +0 -221
- package/bin/lattices.js +0 -1418
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# Voice Error Model
|
|
2
|
+
|
|
3
|
+
## Goal + anchors
|
|
4
|
+
|
|
5
|
+
Use one error vocabulary for Mac voice capture/execution and iPad relay/status. The canonical protocol says Lattices borrows Vox for capture and never owns the mic directly (`docs/voice-command-protocol.md:5-7`), but the shared runtime already has a cross-platform `DeckVoiceState` slot (`swift/Sources/DeckKit/DeckRuntimeSnapshot.swift:49-68`). Current Mac code exposes local strings (`VoxError`, `executionResult`) instead of structured errors (`apps/mac/Sources/VoxClient.swift:43-59`, `apps/mac/Sources/AudioProvider.swift:343-349`); iPad has only a generic `errorMessage` (`apps/ios/Sources/DeckStore.swift:18`). Normalize at DeckKit, then let each surface render the same object.
|
|
6
|
+
|
|
7
|
+
## Error structure
|
|
8
|
+
|
|
9
|
+
Prefer `DeckVoiceError` now; if later reused for trackpad/deck actions, lift the same shape to `LatsError`.
|
|
10
|
+
|
|
11
|
+
```swift
|
|
12
|
+
public struct DeckVoiceError: Codable, Equatable, Identifiable, Sendable {
|
|
13
|
+
public var id: String // uuid or request id
|
|
14
|
+
public var code: DeckVoiceErrorCode
|
|
15
|
+
public var severity: DeckErrorSeverity
|
|
16
|
+
public var recoverable: Bool
|
|
17
|
+
public var retry: DeckRetryHint?
|
|
18
|
+
public var source: DeckErrorSource // mac, ipad, vox, daemon, intent, bridge
|
|
19
|
+
public var owner: String? // e.g. "Vox", "Lattices", "Bridge"
|
|
20
|
+
public var message: String // cockpit copy, already human-readable
|
|
21
|
+
public var remediation: DeckRemediationAction?
|
|
22
|
+
public var occurredAt: Date
|
|
23
|
+
public var detail: String? // diagnostic-only
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
public enum DeckVoiceErrorCode: String, Codable, Sendable { case vox_unreachable, daemon_unreachable, network, connection_lost, mic_denied, accessibility_denied, mic_busy, no_active_target, vox_not_running, vox_loading, intent_unresolved, action_failed, transcription_failed, empty_transcript, language_unsupported }
|
|
27
|
+
public enum DeckErrorSeverity: String, Codable, Sendable { case info, warning, error, blocked }
|
|
28
|
+
public enum DeckRetryHint: String, Codable, Sendable { case silent, immediate, afterLaunch, userAction }
|
|
29
|
+
public enum DeckErrorSource: String, Codable, Sendable { case mac, ipad, vox, daemon, intent, bridge }
|
|
30
|
+
public enum DeckRemediationAction: Codable, Equatable, Sendable {
|
|
31
|
+
case openVox, openSystemSettings(kind: String), retryVoice, openDiagnostics, chooseTarget
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Add `var error: DeckVoiceError?` and optionally `var lastError: DeckVoiceError?` to `DeckVoiceState`, preserving existing `phase`, transcript, and provider fields (`swift/Sources/DeckKit/DeckRuntimeSnapshot.swift:49-68`). Codes should stay stable string raw values for JSON logs and iPad bridge snapshots.
|
|
36
|
+
|
|
37
|
+
## Categories
|
|
38
|
+
|
|
39
|
+
| Category | Codes | Recovery rule |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| Connection | `vox_unreachable`, `daemon_unreachable`, `network`, `connection_lost` | Recoverable unless active capture was lost. Silent reconnect when idle; visible banner during `listening`/`transcribing`. |
|
|
42
|
+
| Permission | `mic_denied`, `accessibility_denied` | Needs user action. `mic_denied` is Mac/Vox-owned; iPad only relays it. `accessibility_denied` blocks execution/navigation. |
|
|
43
|
+
| State | `mic_busy { owner }`, `no_active_target`, `vox_not_running`, `vox_loading` | Usually recoverable. `mic_busy` waits for owner; `vox_not_running` supports launch-on-demand. |
|
|
44
|
+
| Execution | `intent_unresolved`, `action_failed`, `transcription_failed` | Recoverable by retry or edited command; escalate to log if repeated. |
|
|
45
|
+
| Validation | `empty_transcript`, `language_unsupported` | Recoverable; no scary chrome. Treat as a missed command, not a crash. |
|
|
46
|
+
|
|
47
|
+
Copy examples: `Mic in use by Vox — finish memo first`, `No target window`, `Vox offline — starting`, `Connection lost — press again`, `Intent not found`.
|
|
48
|
+
|
|
49
|
+
## Presentation patterns
|
|
50
|
+
|
|
51
|
+
**Mac VoiceCommandWindow.** Keep the three-column cockpit. The top mic bar already owns live state (`connecting...`, `processing...`; `apps/mac/Sources/VoiceCommandWindow.swift:692-719`); render the active error as a compact red/amber status chip there. The center column uses `commandSection` cards (`apps/mac/Sources/VoiceCommandWindow.swift:1287-1304`): show a single `blocked`/`needs action` card only when the user can do something. The footer already has key chips (`apps/mac/Sources/VoiceCommandWindow.swift:1308-1348`); replace the generic command list with contextual remediation: `⌥ Retry`, `Return Open Vox`, `⌘, Permissions`. Logs stay in the right rail, using existing level colors (`apps/mac/Sources/VoiceCommandWindow.swift:1112-1150`).
|
|
52
|
+
|
|
53
|
+
**Mac HUD.** `HUDTopBar.voiceStatus` already has dot, label, transcript, response (`apps/mac/Sources/HUDTopBar.swift:134-198`). Add severity tint: green idle/listening, amber connecting/recoverable, red blocked. For active voice errors, HUD shows a one-line banner in the top bar; no sheet.
|
|
54
|
+
|
|
55
|
+
**iPad Home.** Add `HomeVoiceOverlay` as the full voice modal for active relay: title row `VOICE`, phase, transcript, Mac owner, and one remediation button. The bottom bar already has dense status slots and `hold·space` (`apps/ios/Sources/Home/HomeBottomBar.swift:58-68`, `apps/ios/Sources/Home/HomeBottomBar.swift:129-148`); render idle/recoverable errors inline there (`voice · reconnecting`, `voice · Vox offline`). Use a deck overlay banner only when an issued iPad action failed. Use sheets only for permissions/pairing because they need human action. This follows the chrome rule: do not remove noisy UI; replace it with state that answers “what am I controlling, who is listening, what failed?” (`/Users/arach/.claude/projects/-Users-arach-dev-lattices/memory/feedback_chrome_design.md:11-13`).
|
|
56
|
+
|
|
57
|
+
## Unhappy-path prescriptions
|
|
58
|
+
|
|
59
|
+
**Launch Vox on demand.** Spec flow is detect installed/not running, open Vox, show `Starting Vox...`, wait up to 10s, retry `startDictation`, then fail with manual-open copy (`docs/voice-command-protocol.md:73-89`). Current Mac waits 2s after `connect()` (`apps/mac/Sources/VoiceCommandWindow.swift:290-313`); design target is `vox_not_running` → `vox_loading` → retry → either clear error or `vox_unreachable` with `openVox`.
|
|
60
|
+
|
|
61
|
+
**Mic busy.** Preserve owner attribution from protocol (`docs/voice-command-protocol.md:127-135`). `mic_busy(owner: "Vox")` is warning, recoverable, retry hint `userAction`; message: `Mic in use by Vox — finish memo first`. If owner is unknown: `Mic busy — wait for current recording`.
|
|
62
|
+
|
|
63
|
+
**Connection recovery.** If idle, reconnect silently and write log only. If active, show red `Connection lost`; do not auto-retry captured audio because Vox cancels dropped sockets (`docs/voice-command-protocol.md:174-188`). iPad shows `Mac voice link lost` if bridge lost, not `network` unless the iPad transport failed.
|
|
64
|
+
|
|
65
|
+
**JSONL.** Add `~/.lattices/voice.jsonl` beside `lattices.log` (current log path is `~/.lattices/lattices.log`; `apps/mac/Sources/DiagnosticLog.swift:40-59`). Shape:
|
|
66
|
+
|
|
67
|
+
```json
|
|
68
|
+
{"ts":"2026-04-27T14:03:11.120Z","platform":"mac","sessionId":"...","phase":"listening","event":"error","error":{"code":"mic_busy","severity":"warning","recoverable":true,"source":"vox","owner":"Vox","message":"Mic in use by Vox — finish memo first"},"transcript":null,"intent":null,"durationMs":820}
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Cross-platform conventions
|
|
72
|
+
|
|
73
|
+
Tone: terse cockpit, no apologies. Prefer noun-state-action: `Vox offline — starting`, `No target — pick window`, `Access denied — enable Accessibility`. Tint maps to existing palettes: Mac `Palette.detach` amber and `Palette.kill` red (`apps/mac/Sources/Theme.swift:19-23`); iPad `LatsPalette.amber/red` (`apps/ios/Sources/LatsDeckScreen.swift:19-25`). Icons: `mic.fill` live, `mic.slash` denied, `waveform.badge.exclamationmark` transcription, `bolt.trianglebadge.exclamationmark` execution, `wifi.exclamationmark` connection, `scope` target. Ownership: Mac owns Vox, mic, Accessibility, intent execution, and JSONL. iPad owns relay/bridge/network presentation and never claims direct mic capture.
|
package/docs/voice.md
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: Voice Commands
|
|
3
|
+
description: Natural language voice control for window management
|
|
4
|
+
order: 7
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
Voice commands let you control Lattices by speaking. Press **Hyper+D**
|
|
8
|
+
to open the voice command window, hold **Option** to speak, release to
|
|
9
|
+
stop. Lattices transcribes your speech via Vox,
|
|
10
|
+
matches it to an intent, and executes it.
|
|
11
|
+
|
|
12
|
+
## Quick start
|
|
13
|
+
|
|
14
|
+
1. Install Vox (provides mic + transcription)
|
|
15
|
+
2. Connect an Assistant provider in **Settings > AI**
|
|
16
|
+
3. Press **Hyper+D** to open the voice command window
|
|
17
|
+
4. Hold **Option** and speak a command
|
|
18
|
+
5. Release **Option** — Lattices transcribes and executes
|
|
19
|
+
|
|
20
|
+
## Keyboard shortcuts
|
|
21
|
+
|
|
22
|
+
| Key | Action |
|
|
23
|
+
|-----|--------|
|
|
24
|
+
| **Hyper+D** | Open/close voice command window |
|
|
25
|
+
| **⌥ (hold)** | Push-to-talk — hold to record, release to stop |
|
|
26
|
+
| **Tab** | Arm/disarm the mic |
|
|
27
|
+
| **Escape** | Cancel recording or dismiss window |
|
|
28
|
+
|
|
29
|
+
## Built-in commands
|
|
30
|
+
|
|
31
|
+
### Search
|
|
32
|
+
|
|
33
|
+
Find windows by app name, title, content, or category.
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
"Find all vox windows"
|
|
37
|
+
"Find terminals" → expands to iTerm, Terminal, Warp, etc.
|
|
38
|
+
"Show me all browsers" → expands to Safari, Chrome, Firefox, Arc, etc.
|
|
39
|
+
"Where is my editor?" → expands to VS Code, Cursor, Xcode, etc.
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Category synonyms are built in — saying "terminals", "browsers", "editors",
|
|
43
|
+
"chat", "music", "mail", or "notes" automatically expands to search for
|
|
44
|
+
the actual app names.
|
|
45
|
+
|
|
46
|
+
### Tile
|
|
47
|
+
|
|
48
|
+
Move windows to screen positions.
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
"Tile this left"
|
|
52
|
+
"Snap to the right half"
|
|
53
|
+
"Maximize the window"
|
|
54
|
+
"Put this in the top right corner"
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Voice tiling should resolve into the same canonical daemon mutation used
|
|
58
|
+
by other agent surfaces: `window.place`.
|
|
59
|
+
|
|
60
|
+
### Focus
|
|
61
|
+
|
|
62
|
+
Bring a window or app to the front.
|
|
63
|
+
|
|
64
|
+
```
|
|
65
|
+
"Focus Safari"
|
|
66
|
+
"Switch to Slack"
|
|
67
|
+
"Go to the lattices window"
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Open / Launch
|
|
71
|
+
|
|
72
|
+
Open applications or project workspaces.
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
"Open Spotify"
|
|
76
|
+
"Launch the vox project"
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Kill
|
|
80
|
+
|
|
81
|
+
Close windows or quit applications.
|
|
82
|
+
|
|
83
|
+
```
|
|
84
|
+
"Kill this window"
|
|
85
|
+
"Close Safari"
|
|
86
|
+
"Quit Spotify"
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### Scan
|
|
90
|
+
|
|
91
|
+
Trigger an OCR scan of visible windows.
|
|
92
|
+
|
|
93
|
+
```
|
|
94
|
+
"Scan the screen"
|
|
95
|
+
"Read what's on screen"
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
### Other
|
|
99
|
+
|
|
100
|
+
```
|
|
101
|
+
"List all windows"
|
|
102
|
+
"Show my sessions"
|
|
103
|
+
"Switch to layer 2"
|
|
104
|
+
"Help"
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## AI advisor
|
|
108
|
+
|
|
109
|
+
Every voice command can ask the selected Assistant provider for
|
|
110
|
+
commentary and follow-up suggestions in the **AI corner** (bottom-right
|
|
111
|
+
of the voice command window).
|
|
112
|
+
|
|
113
|
+
When local matching handles the command well, the AI corner shows
|
|
114
|
+
"no AI needed" with an optional "ask AI" button. When the advisor
|
|
115
|
+
has something useful, it shows a one-line comment and an actionable
|
|
116
|
+
suggestion button.
|
|
117
|
+
|
|
118
|
+
### How it works
|
|
119
|
+
|
|
120
|
+
1. You speak a command
|
|
121
|
+
2. Local intent matching runs immediately (fast, free)
|
|
122
|
+
3. The selected Assistant provider runs in parallel when connected
|
|
123
|
+
4. If the advisor suggests something, a button appears in the AI corner
|
|
124
|
+
5. Click the suggestion to execute it
|
|
125
|
+
6. If you engage with a suggestion that the local matcher missed,
|
|
126
|
+
it's recorded in `~/.lattices/advisor-learning.jsonl` for future
|
|
127
|
+
improvement
|
|
128
|
+
|
|
129
|
+
## Hands-off inference
|
|
130
|
+
|
|
131
|
+
Hands-off voice uses the shared inference wrapper in `bin/infer.ts`.
|
|
132
|
+
By default it chooses the lowest-latency configured provider and, when
|
|
133
|
+
Groq credentials are present, uses `groq/llama-3.1-8b-instant`.
|
|
134
|
+
|
|
135
|
+
Credentials are read from process env, `.env.local`, `.env`,
|
|
136
|
+
`~/.lattices/inference.json`, then `~/.config/speakeasy/settings.json`.
|
|
137
|
+
For Groq, either `GROQ_API_KEY` or the common typo `GROK_API_KEY` works
|
|
138
|
+
when the key has Groq's `gsk_` prefix.
|
|
139
|
+
|
|
140
|
+
Override the voice engine if needed:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
LATTICES_VOICE_PROVIDER=groq
|
|
144
|
+
LATTICES_VOICE_MODEL=llama-3.1-8b-instant
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
## Configuration
|
|
148
|
+
|
|
149
|
+
Open **Settings > AI** to configure:
|
|
150
|
+
|
|
151
|
+
| Setting | Default | Description |
|
|
152
|
+
|---------|---------|-------------|
|
|
153
|
+
| Assistant provider | OpenAI Codex | Provider used by the in-app chat and provider-backed voice advisor. Supports OAuth providers such as GitHub Copilot and OpenAI Codex plus API-key providers. |
|
|
154
|
+
| Pi runtime | Auto-detected | Installs or refreshes the provider runtime used by the in-app assistant. |
|
|
155
|
+
| Provider credentials | Not authenticated | Sign in with OAuth or save an API key locally for the selected provider. |
|
|
156
|
+
|
|
157
|
+
## Layout
|
|
158
|
+
|
|
159
|
+
The voice command window has four sections:
|
|
160
|
+
|
|
161
|
+
| Section | Position | Content |
|
|
162
|
+
|---------|----------|---------|
|
|
163
|
+
| **History** | Left column | Past commands with expandable details |
|
|
164
|
+
| **Voice Command** | Center column | Current transcript, matched intent, results |
|
|
165
|
+
| **Log** | Top-right | Rolling diagnostic log (last 12 entries) |
|
|
166
|
+
| **AI Corner** | Bottom-right | Advisor commentary, suggestions, provider readiness |
|
|
167
|
+
|
|
168
|
+
## Search architecture
|
|
169
|
+
|
|
170
|
+
Voice search uses the same backend as `lattices search`:
|
|
171
|
+
|
|
172
|
+
1. **Quick search** — window titles, app names, session tags (instant)
|
|
173
|
+
2. **Complete search** — adds terminal cwd/processes + OCR content
|
|
174
|
+
3. **Synonym expansion** — category terms like "terminals" expand to
|
|
175
|
+
actual app names before searching
|
|
176
|
+
4. **Query cleanup** — strips natural language qualifiers ("and sort by...",
|
|
177
|
+
"please", "for me") before searching
|
|
178
|
+
|
|
179
|
+
## Processing resilience
|
|
180
|
+
|
|
181
|
+
- **15-second timeout** — if processing doesn't complete, returns to idle
|
|
182
|
+
- **Cancellation on dismiss** — closing the window cancels in-flight work
|
|
183
|
+
- **Double-execution prevention** — streaming and stop callbacks can't
|
|
184
|
+
both fire the intent
|
|
185
|
+
|
|
186
|
+
## Advisor learning
|
|
187
|
+
|
|
188
|
+
When the local matcher fails but the AI advisor suggests something that
|
|
189
|
+
you engage with, the interaction is recorded:
|
|
190
|
+
|
|
191
|
+
```
|
|
192
|
+
~/.lattices/advisor-learning.jsonl
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Each line is a JSON object:
|
|
196
|
+
|
|
197
|
+
```json
|
|
198
|
+
{
|
|
199
|
+
"timestamp": "2026-03-15T18:30:00.000Z",
|
|
200
|
+
"transcript": "find all terminals",
|
|
201
|
+
"localIntent": "search",
|
|
202
|
+
"localSlots": {"query": "terminals"},
|
|
203
|
+
"localResultCount": 0,
|
|
204
|
+
"advisorIntent": "search",
|
|
205
|
+
"advisorSlots": {"query": "iterm"},
|
|
206
|
+
"advisorLabel": "Search iTerm"
|
|
207
|
+
}
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
This dataset captures where the local system falls short and what the
|
|
211
|
+
right answer was. Future work can mine it for automatic synonym
|
|
212
|
+
mappings and phrase pattern improvements.
|
|
213
|
+
|
|
214
|
+
## Requirements
|
|
215
|
+
|
|
216
|
+
- **Vox** — provides microphone access and
|
|
217
|
+
speech-to-text transcription
|
|
218
|
+
- **Assistant provider** — enables provider-backed AI suggestions from
|
|
219
|
+
**Settings > AI** (optional, voice commands still work without it)
|
|
220
|
+
- **Accessibility** permission — for window tiling and focus
|
|
221
|
+
- **Screen Recording** permission — for window discovery
|
package/package.json
CHANGED
|
@@ -1,19 +1,37 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@arach/lattices",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "macOS workspace manager
|
|
3
|
+
"version": "0.6.1",
|
|
4
|
+
"description": "macOS workspace manager — menu bar app, CLI, and WebSocket daemon API for tiling, search, layers, and agent control",
|
|
5
|
+
"homepage": "https://lattices.dev",
|
|
6
|
+
"bugs": {
|
|
7
|
+
"url": "https://github.com/arach/lattices/issues"
|
|
8
|
+
},
|
|
9
|
+
"publishConfig": {
|
|
10
|
+
"access": "public",
|
|
11
|
+
"provenance": true
|
|
12
|
+
},
|
|
13
|
+
"packageManager": "bun@1.3.11",
|
|
14
|
+
"engines": {
|
|
15
|
+
"node": ">=18"
|
|
16
|
+
},
|
|
5
17
|
"bin": {
|
|
6
|
-
"lattices": "./bin/lattices.
|
|
7
|
-
"lattices-app": "./bin/lattices-app.
|
|
18
|
+
"lattices": "./bin/lattices.ts",
|
|
19
|
+
"lattices-app": "./bin/lattices-app.ts"
|
|
8
20
|
},
|
|
9
21
|
"keywords": [
|
|
22
|
+
"lattices",
|
|
23
|
+
"macos",
|
|
24
|
+
"workspace",
|
|
25
|
+
"window-manager",
|
|
10
26
|
"tmux",
|
|
27
|
+
"daemon",
|
|
28
|
+
"ai-agents",
|
|
11
29
|
"claude",
|
|
12
|
-
"dev-server",
|
|
13
30
|
"developer-tools",
|
|
14
31
|
"cli",
|
|
15
32
|
"terminal",
|
|
16
|
-
"
|
|
33
|
+
"ocr",
|
|
34
|
+
"tiling"
|
|
17
35
|
],
|
|
18
36
|
"repository": {
|
|
19
37
|
"type": "git",
|
|
@@ -21,22 +39,57 @@
|
|
|
21
39
|
},
|
|
22
40
|
"license": "MIT",
|
|
23
41
|
"exports": {
|
|
24
|
-
".": "./bin/
|
|
25
|
-
"./
|
|
42
|
+
".": "./bin/client.ts",
|
|
43
|
+
"./assistant-intelligence": "./bin/assistant-intelligence.ts",
|
|
44
|
+
"./cua": "./bin/cua.ts",
|
|
45
|
+
"./daemon-client": "./bin/daemon-client.ts",
|
|
46
|
+
"./project-twin": "./bin/project-twin.ts"
|
|
26
47
|
},
|
|
27
48
|
"scripts": {
|
|
28
|
-
"dev": "bun --cwd
|
|
49
|
+
"dev": "bun --cwd apps/site dev",
|
|
50
|
+
"site:build": "bun --cwd apps/site build",
|
|
51
|
+
"studio:dev": "bun --cwd apps/studio dev",
|
|
52
|
+
"studio:build": "bun --cwd apps/studio build",
|
|
53
|
+
"docs:agent": "bun --cwd apps/site agent-docs",
|
|
54
|
+
"check": "bun run check:types && bun run check:app",
|
|
55
|
+
"check:types": "tsc --noEmit",
|
|
56
|
+
"check:app": "env CLANG_MODULE_CACHE_PATH=/tmp/lattices-clang-cache SWIFTPM_TESTS_MODULECACHE=/tmp/lattices-swiftpm-cache swift build --package-path apps/mac",
|
|
57
|
+
"test": "node --experimental-strip-types --test tests/cli.test.mjs",
|
|
58
|
+
"test:cli": "node --experimental-strip-types --test tests/cli.test.mjs",
|
|
59
|
+
"test:e2e": "node --experimental-strip-types --test tests/e2e-daemon.test.mjs",
|
|
60
|
+
"test:e2e:voice": "node --experimental-strip-types tests/eval-voice.js",
|
|
61
|
+
"typecheck": "bun run check:types",
|
|
62
|
+
"build:app-bundle": "bash ./bin/lattices-build package",
|
|
63
|
+
"build:dev-app": "bash ./bin/lattices-build dev",
|
|
64
|
+
"build:dist": "bash ./bin/lattices-build dist",
|
|
65
|
+
"build:dist:local": "bash ./bin/lattices-build dist:local",
|
|
66
|
+
"prepack": "bash ./bin/lattices-build package"
|
|
29
67
|
},
|
|
30
68
|
"type": "module",
|
|
31
|
-
"engines": {
|
|
32
|
-
"node": ">=18"
|
|
33
|
-
},
|
|
34
69
|
"os": ["darwin"],
|
|
35
70
|
"files": [
|
|
36
71
|
"bin",
|
|
37
|
-
"
|
|
38
|
-
"
|
|
39
|
-
"
|
|
72
|
+
"packages/npm/sdk/cua.mjs",
|
|
73
|
+
"packages/npm/sdk/cua.d.ts",
|
|
74
|
+
"packages/npm/sdk/cua.d.mts",
|
|
75
|
+
"apps/mac/Info.plist",
|
|
76
|
+
"apps/mac/Lattices.app",
|
|
77
|
+
"apps/mac/Lattices.entitlements",
|
|
78
|
+
"apps/mac/Resources",
|
|
79
|
+
"assets/AppIcon.icns",
|
|
40
80
|
"docs"
|
|
41
|
-
]
|
|
81
|
+
],
|
|
82
|
+
"devDependencies": {
|
|
83
|
+
"bun-types": "^1.3.10",
|
|
84
|
+
"typescript": "^5.9.3"
|
|
85
|
+
},
|
|
86
|
+
"dependencies": {
|
|
87
|
+
"@ai-sdk/anthropic": "^3.0.58",
|
|
88
|
+
"@ai-sdk/google": "^3.0.43",
|
|
89
|
+
"@ai-sdk/openai": "^3.0.41",
|
|
90
|
+
"@ai-sdk/xai": "^3.0.67",
|
|
91
|
+
"@arach/speakeasy": "^0.2.8",
|
|
92
|
+
"ai": "^6.0.116",
|
|
93
|
+
"zod": "^3.25.76"
|
|
94
|
+
}
|
|
42
95
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./cua.d.ts";
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import type { z } from "zod";
|
|
2
|
+
|
|
3
|
+
export type ComputerTreatment = "observe" | "stage" | "present" | "execute";
|
|
4
|
+
export type ComputerClickTransport =
|
|
5
|
+
| "auto"
|
|
6
|
+
| "ax"
|
|
7
|
+
| "accessibility"
|
|
8
|
+
| "pointer"
|
|
9
|
+
| "mouse"
|
|
10
|
+
| "hardware";
|
|
11
|
+
export type CursorStyle = "spotlight" | "pulse" | "marker";
|
|
12
|
+
export type CursorShape =
|
|
13
|
+
| "arrow"
|
|
14
|
+
| "chevron"
|
|
15
|
+
| "facet"
|
|
16
|
+
| "shard"
|
|
17
|
+
| "wedge"
|
|
18
|
+
| "prism"
|
|
19
|
+
| "notch"
|
|
20
|
+
| "needle"
|
|
21
|
+
| "petal"
|
|
22
|
+
| "kite";
|
|
23
|
+
export type CursorSize = "tiny" | "small" | "regular" | "large";
|
|
24
|
+
export type CursorTrail = "none" | "thread" | "ribbon" | "spark" | "comet" | "route";
|
|
25
|
+
export type CursorMotion =
|
|
26
|
+
| "glide"
|
|
27
|
+
| "snap"
|
|
28
|
+
| "float"
|
|
29
|
+
| "rush"
|
|
30
|
+
| "crawl"
|
|
31
|
+
| "accelerate"
|
|
32
|
+
| "teleport"
|
|
33
|
+
| "spring"
|
|
34
|
+
| "magnet"
|
|
35
|
+
| "slingshot";
|
|
36
|
+
export type CursorTrajectory = "straight" | "soft" | "arc" | "swoop" | "overshoot";
|
|
37
|
+
export type CursorGlow = "none" | "soft" | "halo" | "comet";
|
|
38
|
+
export type CursorIdle =
|
|
39
|
+
| "still"
|
|
40
|
+
| "breathe"
|
|
41
|
+
| "wiggle"
|
|
42
|
+
| "orbit"
|
|
43
|
+
| "hover"
|
|
44
|
+
| "nod"
|
|
45
|
+
| "drift"
|
|
46
|
+
| "shimmer"
|
|
47
|
+
| "blink"
|
|
48
|
+
| "tremble";
|
|
49
|
+
export type CursorEdge =
|
|
50
|
+
| "none"
|
|
51
|
+
| "pulse"
|
|
52
|
+
| "ripple"
|
|
53
|
+
| "tick"
|
|
54
|
+
| "reticle"
|
|
55
|
+
| "blink"
|
|
56
|
+
| "spark"
|
|
57
|
+
| "underline"
|
|
58
|
+
| "echo"
|
|
59
|
+
| "scan"
|
|
60
|
+
| "pin";
|
|
61
|
+
export type CursorSound = "none" | "tick" | "click" | "engage" | "chime";
|
|
62
|
+
export type CaptionPlacement =
|
|
63
|
+
| "top-left"
|
|
64
|
+
| "top-right"
|
|
65
|
+
| "bottom-left"
|
|
66
|
+
| "bottom-right"
|
|
67
|
+
| "top-center"
|
|
68
|
+
| "top"
|
|
69
|
+
| "center"
|
|
70
|
+
| "middle"
|
|
71
|
+
| "near-cursor"
|
|
72
|
+
| "cursor";
|
|
73
|
+
|
|
74
|
+
export interface WindowTarget {
|
|
75
|
+
wid?: number;
|
|
76
|
+
app?: string;
|
|
77
|
+
title?: string;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
export interface ActionBase {
|
|
81
|
+
treatment?: ComputerTreatment;
|
|
82
|
+
dryRun?: boolean;
|
|
83
|
+
capture?: boolean;
|
|
84
|
+
source?: string;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export interface PointTarget {
|
|
88
|
+
x?: number;
|
|
89
|
+
y?: number;
|
|
90
|
+
xRatio?: number;
|
|
91
|
+
yRatio?: number;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export interface ComputerClickParams extends WindowTarget, PointTarget, ActionBase {
|
|
95
|
+
button?: "left" | "right" | "secondary" | "context";
|
|
96
|
+
transport?: ComputerClickTransport;
|
|
97
|
+
axLabel?: string;
|
|
98
|
+
targetText?: string;
|
|
99
|
+
noFocus?: boolean;
|
|
100
|
+
label?: string;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface ComputerMagicCursorParams extends WindowTarget, PointTarget, ActionBase {
|
|
104
|
+
style?: CursorStyle;
|
|
105
|
+
appearance?: CursorStyle;
|
|
106
|
+
shape?: CursorShape;
|
|
107
|
+
angleDeg?: number;
|
|
108
|
+
size?: CursorSize;
|
|
109
|
+
color?: string;
|
|
110
|
+
durationMs?: number;
|
|
111
|
+
label?: string;
|
|
112
|
+
caption?: string;
|
|
113
|
+
captionTitle?: string;
|
|
114
|
+
captionBody?: string;
|
|
115
|
+
captionDetail?: string;
|
|
116
|
+
captionTags?: string;
|
|
117
|
+
captionMode?: "auto" | "selection";
|
|
118
|
+
captionEyebrow?: string;
|
|
119
|
+
captionLeadMs?: number;
|
|
120
|
+
captionSound?: CursorSound;
|
|
121
|
+
captionPlacement?: CaptionPlacement;
|
|
122
|
+
captionMargin?: number;
|
|
123
|
+
captionX?: number;
|
|
124
|
+
captionY?: number;
|
|
125
|
+
captionXRatio?: number;
|
|
126
|
+
captionYRatio?: number;
|
|
127
|
+
captionLeftRatio?: number;
|
|
128
|
+
captionTopRatio?: number;
|
|
129
|
+
sound?: CursorSound;
|
|
130
|
+
sfx?: CursorSound;
|
|
131
|
+
showCaption?: boolean;
|
|
132
|
+
captionSelections?: boolean;
|
|
133
|
+
treatmentLabel?: string;
|
|
134
|
+
variant?: string;
|
|
135
|
+
trail?: CursorTrail;
|
|
136
|
+
pathStyle?: CursorTrail;
|
|
137
|
+
motion?: CursorMotion;
|
|
138
|
+
trajectory?: CursorTrajectory;
|
|
139
|
+
glow?: CursorGlow;
|
|
140
|
+
bloom?: CursorGlow;
|
|
141
|
+
idle?: CursorIdle;
|
|
142
|
+
settle?: CursorIdle;
|
|
143
|
+
presence?: CursorIdle;
|
|
144
|
+
edge?: CursorEdge;
|
|
145
|
+
edgeEffect?: CursorEdge;
|
|
146
|
+
arrival?: CursorEdge;
|
|
147
|
+
typewriter?: boolean;
|
|
148
|
+
typing?: boolean;
|
|
149
|
+
typeIntervalMs?: number;
|
|
150
|
+
typingIntervalMs?: number;
|
|
151
|
+
text?: string;
|
|
152
|
+
append?: boolean;
|
|
153
|
+
fromX?: number;
|
|
154
|
+
fromY?: number;
|
|
155
|
+
fromXRatio?: number;
|
|
156
|
+
fromYRatio?: number;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export interface CuaClientOptions {
|
|
160
|
+
defaultTimeoutMs?: number;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export interface CuaClient {
|
|
164
|
+
click(params: ComputerClickParams): Promise<unknown>;
|
|
165
|
+
magicCursor(params: ComputerMagicCursorParams): Promise<unknown>;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export declare const computerTreatmentSchema: z.ZodType<ComputerTreatment>;
|
|
169
|
+
export declare const computerClickTransportSchema: z.ZodType<ComputerClickTransport>;
|
|
170
|
+
export declare const cursorStyleSchema: z.ZodType<CursorStyle>;
|
|
171
|
+
export declare const cursorShapeSchema: z.ZodType<CursorShape>;
|
|
172
|
+
export declare const cursorSizeSchema: z.ZodType<CursorSize>;
|
|
173
|
+
export declare const cursorTrailSchema: z.ZodType<CursorTrail>;
|
|
174
|
+
export declare const cursorMotionSchema: z.ZodType<CursorMotion>;
|
|
175
|
+
export declare const cursorTrajectorySchema: z.ZodType<CursorTrajectory>;
|
|
176
|
+
export declare const cursorGlowSchema: z.ZodType<CursorGlow>;
|
|
177
|
+
export declare const cursorIdleSchema: z.ZodType<CursorIdle>;
|
|
178
|
+
export declare const cursorEdgeSchema: z.ZodType<CursorEdge>;
|
|
179
|
+
export declare const cursorSoundSchema: z.ZodType<CursorSound>;
|
|
180
|
+
export declare const captionPlacementSchema: z.ZodType<CaptionPlacement>;
|
|
181
|
+
export declare const computerClickParamsSchema: z.ZodType<ComputerClickParams>;
|
|
182
|
+
export declare const computerMagicCursorParamsSchema: z.ZodType<ComputerMagicCursorParams>;
|
|
183
|
+
|
|
184
|
+
export declare function createCuaClient(options?: CuaClientOptions): CuaClient;
|
|
185
|
+
export declare function click(params: ComputerClickParams): Promise<unknown>;
|
|
186
|
+
export declare function magicCursor(params: ComputerMagicCursorParams): Promise<unknown>;
|
|
187
|
+
|
|
188
|
+
export declare const cua: CuaClient;
|