@lattices/cli 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/README.md +85 -9
  2. package/app/Package.swift +8 -1
  3. package/app/Sources/AdvisorLearningStore.swift +90 -0
  4. package/app/Sources/AgentSession.swift +377 -0
  5. package/app/Sources/AppDelegate.swift +44 -12
  6. package/app/Sources/AppShellView.swift +81 -8
  7. package/app/Sources/AudioProvider.swift +386 -0
  8. package/app/Sources/CheatSheetHUD.swift +261 -19
  9. package/app/Sources/DaemonProtocol.swift +13 -0
  10. package/app/Sources/DaemonServer.swift +8 -0
  11. package/app/Sources/DesktopModel.swift +164 -5
  12. package/app/Sources/DesktopModelTypes.swift +2 -0
  13. package/app/Sources/DiagnosticLog.swift +104 -2
  14. package/app/Sources/EventBus.swift +1 -0
  15. package/app/Sources/HUDBottomBar.swift +279 -0
  16. package/app/Sources/HUDController.swift +1158 -0
  17. package/app/Sources/HUDLeftBar.swift +849 -0
  18. package/app/Sources/HUDMinimap.swift +179 -0
  19. package/app/Sources/HUDRightBar.swift +774 -0
  20. package/app/Sources/HUDState.swift +367 -0
  21. package/app/Sources/HUDTopBar.swift +243 -0
  22. package/app/Sources/HandsOffSession.swift +733 -0
  23. package/app/Sources/HomeDashboardView.swift +125 -0
  24. package/app/Sources/HotkeyManager.swift +2 -0
  25. package/app/Sources/HotkeyStore.swift +45 -9
  26. package/app/Sources/IntentEngine.swift +925 -0
  27. package/app/Sources/Intents/CreateLayerIntent.swift +54 -0
  28. package/app/Sources/Intents/DistributeIntent.swift +56 -0
  29. package/app/Sources/Intents/FocusIntent.swift +69 -0
  30. package/app/Sources/Intents/HelpIntent.swift +41 -0
  31. package/app/Sources/Intents/KillIntent.swift +47 -0
  32. package/app/Sources/Intents/LatticeIntent.swift +78 -0
  33. package/app/Sources/Intents/LaunchIntent.swift +67 -0
  34. package/app/Sources/Intents/ListSessionsIntent.swift +32 -0
  35. package/app/Sources/Intents/ListWindowsIntent.swift +30 -0
  36. package/app/Sources/Intents/ScanIntent.swift +52 -0
  37. package/app/Sources/Intents/SearchIntent.swift +190 -0
  38. package/app/Sources/Intents/SwitchLayerIntent.swift +50 -0
  39. package/app/Sources/Intents/TileIntent.swift +61 -0
  40. package/app/Sources/LatticesApi.swift +1235 -30
  41. package/app/Sources/LauncherHUD.swift +348 -0
  42. package/app/Sources/MainView.swift +147 -44
  43. package/app/Sources/OcrModel.swift +34 -1
  44. package/app/Sources/OmniSearchState.swift +99 -102
  45. package/app/Sources/OnboardingView.swift +457 -0
  46. package/app/Sources/PermissionChecker.swift +2 -12
  47. package/app/Sources/PiChatDock.swift +454 -0
  48. package/app/Sources/PiChatSession.swift +815 -0
  49. package/app/Sources/PiWorkspaceView.swift +364 -0
  50. package/app/Sources/PlacementSpec.swift +195 -0
  51. package/app/Sources/Preferences.swift +59 -0
  52. package/app/Sources/ProjectScanner.swift +1 -1
  53. package/app/Sources/ScreenMapState.swift +701 -55
  54. package/app/Sources/ScreenMapView.swift +843 -103
  55. package/app/Sources/ScreenMapWindowController.swift +22 -0
  56. package/app/Sources/SessionLayerStore.swift +285 -0
  57. package/app/Sources/SessionManager.swift +4 -1
  58. package/app/Sources/SettingsView.swift +186 -3
  59. package/app/Sources/Theme.swift +9 -8
  60. package/app/Sources/TmuxModel.swift +7 -0
  61. package/app/Sources/TmuxQuery.swift +27 -3
  62. package/app/Sources/VoiceChatView.swift +192 -0
  63. package/app/Sources/VoiceCommandWindow.swift +1594 -0
  64. package/app/Sources/VoiceIntentResolver.swift +671 -0
  65. package/app/Sources/VoxClient.swift +454 -0
  66. package/app/Sources/WindowTiler.swift +348 -87
  67. package/app/Sources/WorkspaceManager.swift +127 -18
  68. package/bin/client.ts +16 -0
  69. package/bin/{daemon-client.js → daemon-client.ts} +49 -30
  70. package/bin/handsoff-infer.ts +280 -0
  71. package/bin/handsoff-worker.ts +731 -0
  72. package/bin/{lattices-app.js → lattices-app.ts} +67 -32
  73. package/bin/lattices-dev +160 -0
  74. package/bin/{lattices.js → lattices.ts} +600 -137
  75. package/bin/project-twin.ts +645 -0
  76. package/docs/agent-execution-plan.md +562 -0
  77. package/docs/agents.md +142 -0
  78. package/docs/api.md +153 -34
  79. package/docs/app.md +29 -1
  80. package/docs/config.md +5 -1
  81. package/docs/handsoff-test-scenarios.md +84 -0
  82. package/docs/layers.md +20 -20
  83. package/docs/ocr.md +14 -5
  84. package/docs/overview.md +5 -1
  85. package/docs/presentation-execution-review.md +491 -0
  86. package/docs/prompts/hands-off-system.md +374 -0
  87. package/docs/prompts/hands-off-turn.md +30 -0
  88. package/docs/prompts/voice-advisor.md +31 -0
  89. package/docs/prompts/voice-fallback.md +23 -0
  90. package/docs/tiling-reference.md +167 -0
  91. package/docs/twins.md +138 -0
  92. package/docs/voice-command-protocol.md +278 -0
  93. package/docs/voice.md +219 -0
  94. package/package.json +21 -10
  95. package/bin/client.js +0 -4
@@ -0,0 +1,54 @@
1
+ import Foundation
2
+
3
+ struct CreateLayerIntent: LatticeIntent {
4
+ static let name = "create_layer"
5
+ static let title = "Save the current layout as a named layer"
6
+
7
+ static let phrases = [
8
+ // create
9
+ "create a layer called {name}",
10
+ "create layer called {name}",
11
+ "create a layer {name}",
12
+ "create layer {name}",
13
+ "create new layer {name}",
14
+ // make
15
+ "make a layer called {name}",
16
+ "make layer called {name}",
17
+ "make a layer {name}",
18
+ "make layer {name}",
19
+ "make a new layer {name}",
20
+ // save
21
+ "save this layout as {name}",
22
+ "save layout as {name}",
23
+ "save as layer {name}",
24
+ "save as {name}",
25
+ // name / snapshot
26
+ "name this layer {name}",
27
+ "new layer called {name}",
28
+ "new layer {name}",
29
+ // No-arg variants
30
+ "create a layer",
31
+ "create layer",
32
+ "save this layout",
33
+ "save layout",
34
+ "snapshot",
35
+ "snapshot this",
36
+ "remember this layout",
37
+ "save this workspace",
38
+ ]
39
+
40
+ static let slots = [
41
+ SlotDef(name: "name", type: .string, required: false),
42
+ ]
43
+
44
+ func perform(slots: [String: JSON]) throws -> JSON {
45
+ var params: [String: JSON] = [:]
46
+ if let name = slots["name"]?.stringValue {
47
+ params["name"] = .string(name)
48
+ }
49
+ return try LatticesApi.shared.dispatch(
50
+ method: "layer.create",
51
+ params: .object(params)
52
+ )
53
+ }
54
+ }
@@ -0,0 +1,56 @@
1
+ import Foundation
2
+
3
+ struct DistributeIntent: LatticeIntent {
4
+ static let name = "distribute"
5
+ static let title = "Distribute windows evenly across the screen"
6
+
7
+ static let phrases = [
8
+ // Primary: spread / distribute
9
+ "distribute",
10
+ "distribute windows",
11
+ "distribute everything",
12
+ "spread",
13
+ "spread out",
14
+ "spread out the windows",
15
+ "spread them out",
16
+ // organize / arrange
17
+ "organize",
18
+ "organize windows",
19
+ "organize my windows",
20
+ "organize everything",
21
+ "arrange",
22
+ "arrange windows",
23
+ "arrange my windows",
24
+ "arrange everything",
25
+ "arrange them evenly",
26
+ // tidy / clean
27
+ "tidy",
28
+ "tidy up",
29
+ "tidy up the desktop",
30
+ "tidy up my windows",
31
+ "clean up",
32
+ "clean up the layout",
33
+ "clean up my desktop",
34
+ // other
35
+ "even out",
36
+ "even out the windows",
37
+ "fix the layout",
38
+ "reset the layout",
39
+ "make a grid",
40
+ "grid layout",
41
+ "line them up",
42
+ "line everything up",
43
+ "get everything organized",
44
+ "clean up the windows",
45
+ "clean up windows",
46
+ ]
47
+
48
+ static let slots: [SlotDef] = []
49
+
50
+ func perform(slots: [String: JSON]) throws -> JSON {
51
+ DispatchQueue.main.async {
52
+ WindowTiler.distributeVisible()
53
+ }
54
+ return .object(["ok": .bool(true)])
55
+ }
56
+ }
@@ -0,0 +1,69 @@
1
+ import Foundation
2
+
3
+ struct FocusIntent: LatticeIntent {
4
+ static let name = "focus"
5
+ static let title = "Focus and raise a window"
6
+
7
+ static let phrases = [
8
+ // Primary operator: show
9
+ "show {app}",
10
+ "show me {app}",
11
+ "show me the {app}",
12
+ // focus
13
+ "focus {app}",
14
+ "focus on {app}",
15
+ "focus the {app}",
16
+ // switch / go
17
+ "switch to {app}",
18
+ "switch over to {app}",
19
+ "go to {app}",
20
+ "go back to {app}",
21
+ // bring / raise / pull
22
+ "bring up {app}",
23
+ "bring up the {app}",
24
+ "bring forward {app}",
25
+ "raise {app}",
26
+ "raise the {app}",
27
+ "pull up {app}",
28
+ "pull up the {app}",
29
+ // natural
30
+ "i want to see {app}",
31
+ "let me see {app}",
32
+ "take me to {app}",
33
+ "give me {app}",
34
+ "give me the {app}",
35
+ // activate / jump
36
+ "activate {app}",
37
+ "activate the {app}",
38
+ "jump to {app}",
39
+ ]
40
+
41
+ static let slots = [
42
+ SlotDef(name: "app", type: .app, required: true),
43
+ ]
44
+
45
+ func perform(slots: [String: JSON]) throws -> JSON {
46
+ guard let app = slots["app"]?.stringValue else {
47
+ throw IntentError.missingSlot("app")
48
+ }
49
+
50
+ // Use unified search — single source of truth
51
+ let result = try LatticesApi.shared.dispatch(
52
+ method: "lattices.search",
53
+ params: .object([
54
+ "query": .string(app),
55
+ "sources": .array([.string("titles"), .string("apps"), .string("sessions"), .string("cwd"), .string("tmux")]),
56
+ ])
57
+ )
58
+ if case .array(let items) = result, let first = items.first,
59
+ let wid = first["wid"]?.uint32Value {
60
+ let pid = first["pid"]?.intValue ?? Int(DesktopModel.shared.windows[wid]?.pid ?? 0)
61
+ DispatchQueue.main.async {
62
+ WindowTiler.focusWindow(wid: wid, pid: Int32(pid))
63
+ }
64
+ return .object(["ok": .bool(true), "focused": .string(app), "wid": .int(Int(wid))])
65
+ }
66
+
67
+ return .object(["ok": .bool(false), "reason": .string("No window found for '\(app)'")])
68
+ }
69
+ }
@@ -0,0 +1,41 @@
1
+ import Foundation
2
+
3
+ struct HelpIntent: LatticeIntent {
4
+ static let name = "help"
5
+ static let title = "Show available commands and usage"
6
+
7
+ static let phrases = [
8
+ "help",
9
+ "help me",
10
+ "what can i do",
11
+ "what can you do",
12
+ "what commands are there",
13
+ "what are the commands",
14
+ "how does this work",
15
+ "how do i use this",
16
+ "show me the commands",
17
+ "what can i say",
18
+ "what are my options",
19
+ ]
20
+
21
+ static let slots: [SlotDef] = []
22
+
23
+ func perform(slots: [String: JSON]) throws -> JSON {
24
+ let commands = [
25
+ "find <query> — search windows by name, title, or content",
26
+ "show <app> — focus an app window",
27
+ "open <project> — launch a project or app",
28
+ "tile <position> — tile the current window (left, right, top-left, etc.)",
29
+ "maximize — make the current window full screen",
30
+ "distribute — arrange all visible windows in a grid",
31
+ "scan — OCR all visible windows",
32
+ "kill <session> — stop a tmux session",
33
+ "list windows — show all open windows",
34
+ "list sessions — show running tmux sessions",
35
+ ]
36
+ return .object([
37
+ "commands": .array(commands.map { .string($0) }),
38
+ "hint": .string("You can also say things naturally — 'where's my Slack?' or 'tidy up the windows'"),
39
+ ])
40
+ }
41
+ }
@@ -0,0 +1,47 @@
1
+ import Foundation
2
+
3
+ struct KillIntent: LatticeIntent {
4
+ static let name = "kill"
5
+ static let title = "Stop a running session"
6
+
7
+ static let phrases = [
8
+ // Primary operator: kill
9
+ "kill {session}",
10
+ "kill the {session}",
11
+ "kill the {session} session",
12
+ "kill that",
13
+ // stop
14
+ "stop {session}",
15
+ "stop the {session}",
16
+ "stop the {session} session",
17
+ // shut down
18
+ "shut down {session}",
19
+ "shut down the {session}",
20
+ "shut it down",
21
+ // close
22
+ "close {session}",
23
+ "close the {session}",
24
+ "close the {session} session",
25
+ // terminate
26
+ "terminate {session}",
27
+ "terminate the {session}",
28
+ // end
29
+ "end {session}",
30
+ "end the {session}",
31
+ "end the {session} session",
32
+ ]
33
+
34
+ static let slots = [
35
+ SlotDef(name: "session", type: .session, required: true),
36
+ ]
37
+
38
+ func perform(slots: [String: JSON]) throws -> JSON {
39
+ guard let session = slots["session"]?.stringValue else {
40
+ throw IntentError.missingSlot("session")
41
+ }
42
+ return try LatticesApi.shared.dispatch(
43
+ method: "session.kill",
44
+ params: .object(["name": .string(session)])
45
+ )
46
+ }
47
+ }
@@ -0,0 +1,78 @@
1
+ import Foundation
2
+
3
+ // MARK: - Core Protocol
4
+
5
+ /// A self-contained voice intent. Each intent declares its phrases, slots, and execution.
6
+ protocol LatticeIntent {
7
+ static var name: String { get }
8
+ static var title: String { get }
9
+ static var phrases: [String] { get } // e.g. "find {query}", "search for {query}"
10
+ static var slots: [SlotDef] { get }
11
+
12
+ func perform(slots: [String: JSON]) throws -> JSON
13
+ }
14
+
15
+ // MARK: - Slot Definition
16
+
17
+ enum SlotType {
18
+ case string // Free-form text
19
+ case position // Tile position (left, right, maximize, etc.)
20
+ case app // Running app name
21
+ case session // Active tmux session
22
+ case layer // Layer name
23
+ case enumerated([String]) // Fixed set of values
24
+ }
25
+
26
+ struct SlotDef {
27
+ let name: String
28
+ let type: SlotType
29
+ let required: Bool
30
+ let defaultValue: JSON?
31
+
32
+ init(name: String, type: SlotType, required: Bool = true, defaultValue: JSON? = nil) {
33
+ self.name = name
34
+ self.type = type
35
+ self.required = required
36
+ self.defaultValue = defaultValue
37
+ }
38
+ }
39
+
40
+ // MARK: - Compiled Phrase Template
41
+
42
+ struct CompiledPhrase {
43
+ let original: String // "find {query}"
44
+ let regex: NSRegularExpression // ^find (.+)$
45
+ let slotNames: [String] // ["query"] in capture-group order
46
+ let intentName: String
47
+ }
48
+
49
+ // MARK: - Match Result
50
+
51
+ struct IntentMatch {
52
+ let intentName: String
53
+ let slots: [String: JSON]
54
+ let confidence: Double
55
+ let matchedPhrase: String
56
+ }
57
+
58
+ // MARK: - Phrase Matcher
59
+
60
+ final class PhraseMatcher {
61
+ static let shared = PhraseMatcher()
62
+
63
+ private init() {
64
+ DiagnosticLog.shared.info("PhraseMatcher: semantic resolver active (\(IntentEngine.shared.definitions().count) intents)")
65
+ }
66
+
67
+ func match(text: String) -> IntentMatch? {
68
+ VoiceIntentResolver.shared.match(text: text)
69
+ }
70
+
71
+ func execute(_ match: IntentMatch) throws -> JSON {
72
+ try VoiceIntentResolver.shared.execute(match)
73
+ }
74
+
75
+ func catalog() -> JSON {
76
+ VoiceIntentResolver.shared.catalog()
77
+ }
78
+ }
@@ -0,0 +1,67 @@
1
+ import AppKit
2
+ import Foundation
3
+
4
+ struct LaunchIntent: LatticeIntent {
5
+ static let name = "launch"
6
+ static let title = "Launch a project or app"
7
+
8
+ static let phrases = [
9
+ // Primary operator: open
10
+ "open {project}",
11
+ "open up {project}",
12
+ "open my {project}",
13
+ "open the {project}",
14
+ // launch
15
+ "launch {project}",
16
+ "launch the {project}",
17
+ "launch my {project}",
18
+ // start
19
+ "start {project}",
20
+ "start up {project}",
21
+ "start the {project}",
22
+ "start my {project}",
23
+ "start working on {project}",
24
+ // work on
25
+ "work on {project}",
26
+ "work on the {project}",
27
+ "begin {project}",
28
+ "begin working on {project}",
29
+ // fire / spin / boot
30
+ "fire up {project}",
31
+ "spin up {project}",
32
+ "boot up {project}",
33
+ // load / run
34
+ "load {project}",
35
+ "load up {project}",
36
+ "run {project}",
37
+ "run the {project}",
38
+ ]
39
+
40
+ static let slots = [
41
+ SlotDef(name: "project", type: .string, required: true),
42
+ ]
43
+
44
+ func perform(slots: [String: JSON]) throws -> JSON {
45
+ guard let project = slots["project"]?.stringValue else {
46
+ throw IntentError.missingSlot("project")
47
+ }
48
+
49
+ // Try to find the project by scanning known project paths
50
+ let scanner = ProjectScanner.shared
51
+ if let found = scanner.projects.first(where: {
52
+ $0.name.lowercased().contains(project.lowercased())
53
+ }) {
54
+ // Launch via session manager
55
+ let result = try LatticesApi.shared.dispatch(
56
+ method: "session.launch",
57
+ params: .object(["path": .string(found.path)])
58
+ )
59
+ return result
60
+ }
61
+
62
+ // Fallback: try as an app name
63
+ let app = project.prefix(1).uppercased() + project.dropFirst()
64
+ NSWorkspace.shared.launchApplication(app)
65
+ return .object(["ok": .bool(true), "launched": .string(app)])
66
+ }
67
+ }
@@ -0,0 +1,32 @@
1
+ import Foundation
2
+
3
+ struct ListSessionsIntent: LatticeIntent {
4
+ static let name = "list_sessions"
5
+ static let title = "List running tmux sessions"
6
+
7
+ static let phrases = [
8
+ "list sessions",
9
+ "list all sessions",
10
+ "list my sessions",
11
+ "what sessions",
12
+ "what sessions are running",
13
+ "what's running",
14
+ "whats running",
15
+ "what is running",
16
+ "which projects",
17
+ "which projects are active",
18
+ "show sessions",
19
+ "show my sessions",
20
+ "show my projects",
21
+ "show me what's running",
22
+ "show me whats running",
23
+ "how many sessions",
24
+ "any sessions running",
25
+ ]
26
+
27
+ static let slots: [SlotDef] = []
28
+
29
+ func perform(slots: [String: JSON]) throws -> JSON {
30
+ try LatticesApi.shared.dispatch(method: "sessions.list", params: nil)
31
+ }
32
+ }
@@ -0,0 +1,30 @@
1
+ import Foundation
2
+
3
+ struct ListWindowsIntent: LatticeIntent {
4
+ static let name = "list_windows"
5
+ static let title = "List all open windows"
6
+
7
+ static let phrases = [
8
+ "list windows",
9
+ "list all windows",
10
+ "list my windows",
11
+ "what windows",
12
+ "what windows are open",
13
+ "what's open",
14
+ "which windows",
15
+ "which windows are visible",
16
+ "show all windows",
17
+ "show me all windows",
18
+ "show me all the windows",
19
+ "how many windows",
20
+ "count windows",
21
+ "what do i have open",
22
+ "what's visible",
23
+ ]
24
+
25
+ static let slots: [SlotDef] = []
26
+
27
+ func perform(slots: [String: JSON]) throws -> JSON {
28
+ try LatticesApi.shared.dispatch(method: "windows.list", params: nil)
29
+ }
30
+ }
@@ -0,0 +1,52 @@
1
+ import Foundation
2
+
3
+ struct ScanIntent: LatticeIntent {
4
+ static let name = "scan"
5
+ static let title = "Rescan screen text (OCR)"
6
+
7
+ static let phrases = [
8
+ // Primary operator: scan
9
+ "scan",
10
+ "scan the screen",
11
+ "scan everything",
12
+ "scan all",
13
+ "scan my screen",
14
+ "scan my windows",
15
+ "scan all windows",
16
+ // rescan
17
+ "rescan",
18
+ "rescan the screen",
19
+ "rescan everything",
20
+ // read
21
+ "read the screen",
22
+ "read my screen",
23
+ "read all windows",
24
+ // ocr
25
+ "ocr",
26
+ "ocr scan",
27
+ "run ocr",
28
+ // update / refresh / capture
29
+ "update screen text",
30
+ "refresh screen text",
31
+ "capture screen text",
32
+ "capture text",
33
+ // natural
34
+ "what's on my screen",
35
+ "what's on screen",
36
+ "what is on my screen",
37
+ "show me what's on the screen",
38
+ "show me what is on the screen",
39
+ "index the screen",
40
+ "do a scan",
41
+ "give me a scan",
42
+ "give me a fresh scan",
43
+ "quick scan",
44
+ "a fresh scan",
45
+ ]
46
+
47
+ static let slots: [SlotDef] = []
48
+
49
+ func perform(slots: [String: JSON]) throws -> JSON {
50
+ try LatticesApi.shared.dispatch(method: "ocr.scan", params: nil)
51
+ }
52
+ }