ft-scout 9.0.5 → 9.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/.firebase/hosting.d2Vi.cache +9 -8
  2. package/AUTONOMOUS_FEATURES.md +192 -0
  3. package/README.md +82 -40
  4. package/bin/src/commands/agent.d.ts.map +1 -1
  5. package/bin/src/commands/agent.js +43 -2
  6. package/bin/src/commands/agent.js.map +1 -1
  7. package/bin/src/commands/login.d.ts.map +1 -1
  8. package/bin/src/commands/login.js +3 -1
  9. package/bin/src/commands/login.js.map +1 -1
  10. package/bin/src/engine/actionVerifier.d.ts.map +1 -1
  11. package/bin/src/engine/actionVerifier.js +9 -1
  12. package/bin/src/engine/actionVerifier.js.map +1 -1
  13. package/bin/src/engine/agentEngine.d.ts.map +1 -1
  14. package/bin/src/engine/agentEngine.js +230 -85
  15. package/bin/src/engine/agentEngine.js.map +1 -1
  16. package/bin/src/engine/browser/browserController.d.ts.map +1 -1
  17. package/bin/src/engine/browser/browserController.js +53 -13
  18. package/bin/src/engine/browser/browserController.js.map +1 -1
  19. package/bin/src/engine/computer/computerController.d.ts.map +1 -1
  20. package/bin/src/engine/computer/computerController.js +43 -11
  21. package/bin/src/engine/computer/computerController.js.map +1 -1
  22. package/bin/src/engine/computer/scoutDriver.cs +325 -37
  23. package/bin/src/engine/computer/scout_driver.exe +0 -0
  24. package/bin/src/engine/planner.d.ts.map +1 -1
  25. package/bin/src/engine/planner.js +81 -2
  26. package/bin/src/engine/planner.js.map +1 -1
  27. package/bin/src/engine/runtime/agentRuntime.d.ts.map +1 -1
  28. package/bin/src/engine/runtime/agentRuntime.js +66 -5
  29. package/bin/src/engine/runtime/agentRuntime.js.map +1 -1
  30. package/bin/src/engine/safety.d.ts.map +1 -1
  31. package/bin/src/engine/safety.js +4 -3
  32. package/bin/src/engine/safety.js.map +1 -1
  33. package/bin/src/engine/scoutAudioRecorder.cs +215 -0
  34. package/bin/src/engine/scout_audio_recorder.exe +0 -0
  35. package/bin/src/engine/toolRegistry.d.ts.map +1 -1
  36. package/bin/src/engine/toolRegistry.js +65 -0
  37. package/bin/src/engine/toolRegistry.js.map +1 -1
  38. package/bin/src/engine/voiceEngine.d.ts.map +1 -1
  39. package/bin/src/engine/voiceEngine.js +333 -13
  40. package/bin/src/engine/voiceEngine.js.map +1 -1
  41. package/bin/src/index.js +5 -4
  42. package/bin/src/index.js.map +1 -1
  43. package/bin/src/utils/branding.js +1 -1
  44. package/bin/src/utils/holoseneAuth.d.ts.map +1 -1
  45. package/bin/src/utils/holoseneAuth.js +169 -41
  46. package/bin/src/utils/holoseneAuth.js.map +1 -1
  47. package/firebase.json +8 -0
  48. package/package.json +1 -1
  49. package/web/app.js +3 -3
  50. package/web/styles.css +2 -2
@@ -1,8 +1,9 @@
1
- web_cli.html,1789281384143,d251153dac472e4e4ea21d00f881b575e81a63822da13006e720c376cfbe24af
2
- styles.css,1789281384145,f1c7c32071cbb851d4f0a3023375765c3a5630106878c5fcad8456adb98e138a
3
- ScoutFavicon.png,1786186569805,41a9ca9abf7acd2976e78c72a72285b49cc0e64b2b5eeb2c2b88c69628fc618a
4
- authentication.html,1789281384176,678dbfb5438ff13b0a6f8d759c1af77d5aac926ff44ea7815f181e80674d183a
5
- app.js,1789281384176,f281ff2c345623a652af75a4db565bb35377bfd452a5e5af940c362c65d2f52f
6
- admin.html,1789281384176,72bb3b4ab7dbbe6ff32bfb5a51e8c79ef0bece4e2f1fc91218296df17e9b34f4
7
- paid_plans.html,1789283167535,0708329edc4938577250c92fb1a294d09ea38778b8edec990b0b80adc839e49f
8
- index.html,1789282851111,f7812b28faad4cc0ebf3062272dfadc297ab257d3d73ff254b69780e5401934a
1
+ web_cli.html,1789281384143,d251153dac472e4e4ea21d00f881b575e81a63822da13006e720c376cfbe24af
2
+ styles.css,1789281384145,f1c7c32071cbb851d4f0a3023375765c3a5630106878c5fcad8456adb98e138a
3
+ ScoutFavicon.png,1786186569805,41a9ca9abf7acd2976e78c72a72285b49cc0e64b2b5eeb2c2b88c69628fc618a
4
+ paid_plans.html,1789283167535,0708329edc4938577250c92fb1a294d09ea38778b8edec990b0b80adc839e49f
5
+ index.html,1789282851111,f7812b28faad4cc0ebf3062272dfadc297ab257d3d73ff254b69780e5401934a
6
+ authentication.html,1789281384176,678dbfb5438ff13b0a6f8d759c1af77d5aac926ff44ea7815f181e80674d183a
7
+ app.js,1789284464838,bfddc8f457e7a68a6ff158425cd5b23132e8e15700c7ecd22850fbe0455ff7c0
8
+ admin.html,1789281384176,72bb3b4ab7dbbe6ff32bfb5a51e8c79ef0bece4e2f1fc91218296df17e9b34f4
9
+ oauth.html,1789285036283,f53de5de706d1ce49d2af18e07e1fce39f01fe6a5c65ce525ba678809a1c5bfb
@@ -0,0 +1,192 @@
1
+ # 🚀 Scout Autonomous Agent — Features & Architecture Guide
2
+
3
+ Scout is an advanced, multi-modal autonomous agent platform built directly into FT-CLI. It bridges traditional developer terminal workflows with modern web automation and full operating-system desktop control (GUI).
4
+
5
+ ---
6
+
7
+ ## 📑 Table of Contents
8
+
9
+ 1. [Architectural Overview](#architectural-overview)
10
+ 2. [Desktop & Computer-Use (GUI Control)](#1-desktop--computer-use-gui-control)
11
+ 3. [Browser Automation & Web Scraping](#2-browser-automation--web-scraping)
12
+ 4. [Multi-Step Planning & Cognitive Scratchpad](#3-multi-step-planning--cognitive-scratchpad)
13
+ 5. [Action Verification & Self-Healing Engine](#4-action-verification--self-healing-engine)
14
+ 6. [Safety & Human-in-the-Loop Risk Classifier](#5-safety--human-in-the-loop-risk-classifier)
15
+ 7. [Developer & Workspace Baseline Operations](#6-developer--workspace-baseline-operations)
16
+ 8. [Subscription Tier Capabilities (Free vs Pro / Enterprise)](#subscription-tier-capabilities)
17
+
18
+ ---
19
+
20
+ ## Architectural Overview
21
+
22
+ Scout executes a continuous **Perceive ➔ Plan ➔ Act ➔ Verify ➔ Recover** operational loop:
23
+
24
+ ```mermaid
25
+ flowchart TD
26
+ Goal[User High-Level Objective] --> Planner[TaskPlanner & DAG Decomposition]
27
+ Planner --> Scratchpad[Cognitive Scratchpad & State Checkpoint]
28
+ Scratchpad --> SelectTool[Tool Selection Hierarchy]
29
+ SelectTool --> RiskCheck{RiskClassifier & Safety Gate}
30
+ RiskCheck -- High/Critical Risk --> Approval[ApprovalManager: Human Confirmation]
31
+ Approval -- Approved --> Dispatcher[Agent Runtime Dispatcher]
32
+ RiskCheck -- Low/Medium Risk --> Dispatcher
33
+ Dispatcher --> Execute[Execution: Native Computer / Browser / CLI Tools]
34
+ Execute --> Verify[ActionVerifier: Post-Execution Verification]
35
+ Verify -- Success --> Checkpoint[Save Task State & Next Step]
36
+ Verify -- Failure --> Recovery[Error Recovery Strategy: Retry / Alternative]
37
+ Recovery --> Dispatcher
38
+ Checkpoint --> Done{All Steps Complete?}
39
+ Done -- No --> Scratchpad
40
+ Done -- Yes --> Complete[Task Complete Notification]
41
+ ```
42
+
43
+ ---
44
+
45
+ ## 1. 🖥️ Desktop & Computer-Use (GUI Control)
46
+
47
+ Scout provides deep operating system control powered by the high-performance native `scout_driver.exe` engine (Win32 API interop) and Node.js controller integration (`SystemComputerController`).
48
+
49
+ ### Core Features
50
+
51
+ - **Multi-Monitor Screen Perception:**
52
+ - `computer_screenshot` / `screenshot`: Captures high-resolution screen frames to perceive visible windows, buttons, and visual state.
53
+ - **Precision Mouse Automation:**
54
+ - `computer_click`: Sends left, right, or middle mouse clicks to exact screen coordinates `(x, y)`.
55
+ - `computer_double_click`: Opens files, selects words, or triggers desktop shortcuts via double-click.
56
+ - `computer_mouse_move`: Translates the cursor smoothly or instantly across display bounds.
57
+ - `computer_mouse_down` & `computer_mouse_up`: Granular mouse button holds and releases for complex interactions.
58
+ - `computer_drag`: Drags items from `(fromX, fromY)` to destination coordinates `(toX, toY)`.
59
+ - `computer_scroll`: Dispatches vertical and horizontal mouse-wheel scroll deltas.
60
+ - **Keyboard & Shortcut Dispatch:**
61
+ - `computer_type`: Emits text strings directly into focused input controls or editor windows.
62
+ - `computer_key_press`: Presses single functional keys (Enter, Escape, Tab, Backspace, Arrow keys, etc.).
63
+ - `computer_hotkey`: Emits multi-key chords (e.g. `["Control", "s"]` to save, `["Alt", "F4"]` to close, `["Alt", "Tab"]` to switch tasks).
64
+ - **Application & Window Orchestration:**
65
+ - `computer_open_app`: Launches desktop applications (VS Code, Notepad, File Explorer, Terminal, etc.).
66
+ - `computer_close_app`: Gracefully terminates or closes targeted desktop processes and windows.
67
+ - `computer_switch_window`: Brings specific applications into foreground focus by matching title or process.
68
+ - `computer_inspect_ui`: Enumerates active open desktop windows and highlights the current foreground window.
69
+ - `computer_wait`: Pauses execution (100ms - 30,000ms) to allow GUI windows, animations, and transitions to settle.
70
+ - `computer_diagnostic`: Collects display resolution, DPI scaling metrics, and driver connection status.
71
+
72
+ ---
73
+
74
+ ## 2. 🌐 Browser Automation & Web Scraping
75
+
76
+ Implemented in [`browserController.ts`](file:///c:/Users/rishi/FT-CLI/bin/src/engine/browser/browserController.ts), Scout includes an intelligent browser automation controller capable of navigating websites, analyzing semantic DOM trees, and extracting data.
77
+
78
+ ### Core Features
79
+
80
+ - **Page Navigation & State:**
81
+ - `browser_navigate`: Loads HTTP/HTTPS URLs, handles navigation history, and constructs interactive DOM trees.
82
+ - `browser_inspect`: Discovers interactive elements (links, buttons, form fields) and assigns lightweight reference IDs (e.g., `[ref=1]`).
83
+ - **Web Form Interaction:**
84
+ - `browser_click`: Clicks web elements identified by CSS selector, visible label text, or ref ID.
85
+ - `browser_type`: Enters query strings or credentials into active form fields and textareas.
86
+ - `browser_select`: Selects options from dropdown menus (`<select>`) by visible text or value.
87
+ - `browser_scroll`: Scrolls viewport up or down to trigger infinite scroll or lazy-loaded assets.
88
+ - **Data Scraping & File Downloads:**
89
+ - `browser_content`: Strips out HTML markup and extracts clean structured text, tables, and article content.
90
+ - `browser_screenshot`: Captures a visual snapshot of the current web page viewport.
91
+ - `browser_download`: Automatically downloads invoices, reports, PDFs, or files from web links and saves them directly to the workspace `downloads/` directory.
92
+
93
+ #### Example Implementation (`browserController.ts`):
94
+ ```typescript
95
+ // Content Extraction & Document Download
96
+ public async extract_content(selector?: string): Promise<BrowserActionResult> {
97
+ const page = this.mockPages.get(this.currentUrl);
98
+ const text = page
99
+ ? page.html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim()
100
+ : `Content of ${this.currentUrl}`;
101
+ return { success: true, output: text, url: this.currentUrl, title: this.currentTitle };
102
+ }
103
+
104
+ public async download_file(urlOrSelector: string, destPath?: string): Promise<BrowserActionResult> {
105
+ // Discovers links, downloads files, and writes to workspace disk
106
+ const fileName = path.basename(targetUrl).includes('.') ? path.basename(targetUrl) : 'downloaded_invoice.pdf';
107
+ const finalPath = destPath ? path.resolve(this.workspaceDir, destPath) : path.resolve(this.workspaceDir, 'downloads', fileName);
108
+ fs.mkdirSync(path.dirname(finalPath), { recursive: true });
109
+ // Stream data and persist to disk...
110
+ }
111
+ ```
112
+
113
+ ---
114
+
115
+ ## 3. 🧠 Multi-Step Planning & Cognitive Scratchpad
116
+
117
+ Complex tasks require structured reasoning, plan tracking, and persistent state across agent steps.
118
+
119
+ - **Dynamic Task Planning (`TaskPlanner`):**
120
+ - Parses open-ended goals into structured steps with explicit dependencies.
121
+ - Dynamically adapts when intermediate steps succeed, fail, or reveal new context.
122
+ - **Cognitive Scratchpad Memory (`scratchpad`):**
123
+ - Exclusive to **Pro & Enterprise** tiers.
124
+ - Provides working memory via `write`, `read`, `append`, and `clear` operations.
125
+ - Persisted locally to `.ft/scratchpad.md` to prevent LLM context overflow during long executions.
126
+ - **Task State Checkpoints (`TaskStateManager`):**
127
+ - Saves full task execution histories and artifact logs to `.ft/tasks/<task_id>.json`.
128
+ - Enables clean task resumption, history reviews, and rollback capabilities.
129
+
130
+ ---
131
+
132
+ ## 4. 🔍 Action Verification & Self-Healing Engine
133
+
134
+ Scout doesn't simply fire tools and assume success. Every action is verified through [`actionVerifier.ts`](file:///c:/Users/rishi/FT-CLI/bin/src/engine/actionVerifier.ts).
135
+
136
+ ### Verification Types
137
+ 1. `file_exists`: Verifies target file is present on the filesystem and contains non-empty contents.
138
+ 2. `app_active`: Verifies that an opened application is running and holds foreground focus.
139
+ 3. `url_matches`: Verifies the browser navigated to the expected target domain/path.
140
+ 4. `element_exists`: Confirms targeted DOM elements rendered on the page.
141
+
142
+ ### Autonomous Recovery Strategy
143
+ When verification fails:
144
+ - **`RETRY`**: Re-attempts the action if caused by transient loading latency.
145
+ - **`ALTERNATIVE`**: Selects a secondary tool or selector if the primary approach is blocked.
146
+ - **`ABORT`**: Gracefully yields to the user if irreversible roadblocks occur.
147
+
148
+ ### Intent Preservation
149
+ If a user requests GUI action (e.g. *"Open Notepad and type my notes"*), Scout strictly honors the intent and will **not** cheat by performing silent background file writes unless explicitly permitted.
150
+
151
+ ---
152
+
153
+ ## 5. 🛡️ Safety & Human-in-the-Loop Risk Classifier
154
+
155
+ Implemented in [`safety.ts`](file:///c:/Users/rishi/FT-CLI/bin/src/engine/safety.ts), Scout protects user systems from accidental disruption or destructive operations.
156
+
157
+ | Risk Level | Triggering Tools / Patterns | Behavior |
158
+ | :--- | :--- | :--- |
159
+ | **LOW** | `read_file`, `browser_inspect`, `computer_screenshot`, `file_info` | Auto-executed silently. |
160
+ | **MEDIUM** | `write_file`, `edit_file`, `browser_type`, `browser_click` | Executed with standard logging. |
161
+ | **HIGH** | `computer_close_app`, destructive hotkeys (`Alt+F4`), process termination | **Execution pauses.** Modal/CLI prompt requires user approval. |
162
+ | **CRITICAL** | System-wide deletion (`rmdir /s /q`, formatting), registry edits | **Blocked / Strict explicit consent required.** |
163
+
164
+ ---
165
+
166
+ ## 6. 💼 Developer & Workspace Baseline Operations
167
+
168
+ In addition to autonomous GUI and web tools, Scout provides a full suite of developer tools:
169
+
170
+ - **File Operations:** `read_file`, `write_file`, `edit_file`, `multi_edit_file`, `file_info`.
171
+ - **Search & Inspection:** `grep_search`, `glob_search`, `tree_view`, `list_dir`.
172
+ - **Command & Version Control:** `run_command`, `git_diff`.
173
+ - **Interaction:** `ask_user`, `task_completed`.
174
+
175
+ ---
176
+
177
+ ## Subscription Tier Capabilities
178
+
179
+ | Feature Capability | Free Tier | Pro Plan | Enterprise Plan |
180
+ | :--- | :---: | :---: | :---: |
181
+ | **Code Reading & File Editing** | ✅ Full | ✅ Full | ✅ Full |
182
+ | **File / Directory Creation** | ✅ Full | ✅ Full | ✅ Full |
183
+ | **Terminal & Build Commands** | ✅ Full | ✅ Full | ✅ Full |
184
+ | **AI Cognitive Scratchpad** | ❌ Blocked | ✅ Full | ✅ Full |
185
+ | **Autonomous Browser Automation** | ❌ Blocked | ✅ Full | ✅ Full |
186
+ | **Autonomous Computer & GUI Control** | ❌ Blocked | ✅ Full | ✅ Full |
187
+ | **Multi-App System Automation** | ❌ Blocked | ✅ Full | ✅ Full |
188
+ | **Dedicated Concurrency & Support** | Community | Priority | 24/7 Dedicated |
189
+
190
+ ---
191
+
192
+ *Authored by Front Terrain Engineering. For upgrades or enterprise deployment, visit [https://scout.frontterrain.com](https://scout.frontterrain.com).*
package/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # scout (FrontTerrain CLI v9.0.5)
1
+ # scout (FrontTerrain CLI v9.0.7)
2
2
 
3
3
  **FrontTerrain (`scout`)** — The Autonomous AI Principal Software Engineer & Codebase Co-Pilot in your terminal.
4
4
 
@@ -228,37 +228,67 @@ Scout is built as a high-agency autonomous operator capable of operating externa
228
228
 
229
229
  ## Scout Autonomous Tool Registry
230
230
 
231
- ### Computer & Application Control Tools
232
- | Tool | Description |
233
- |---|---|
234
- | `computer_screenshot` | Capture display screen image and inspect active foreground window |
235
- | `computer_click` | Move cursor and click mouse at coordinate (left, right, middle) |
236
- | `computer_double_click` | Double-click mouse at specified screen coordinate |
237
- | `computer_type` | Ensure target window focus and type text keystrokes via OS keyboard |
238
- | `computer_key_press` | Press individual keyboard key (`enter`, `tab`, `escape`, `backspace`) |
239
- | `computer_hotkey` | Trigger key combination (`["ctrl", "s"]`, `["alt", "tab"]`) |
240
- | `computer_open_app` | Launch desktop application (Notepad, Excel, Chrome, Terminal) and focus window |
241
- | `computer_switch_window` | Switch foreground active window to target application |
242
- | `computer_inspect_ui` | Inspect active window coordinates, title, and UI state |
243
-
244
- ### Workspace & Engineering Tools
245
- | Tool | Description |
246
- |---|---|
247
- | `read_file` | Read content & line ranges of workspace files |
248
- | `write_file` | Create new files or overwrite existing source code |
249
- | `edit_file` | Search & replace exact target code snippets |
250
- | `multi_edit_file` | Apply batch snippet replacements in a single atomic tool call |
251
- | `run_command` | Execute live terminal commands (`npm test`, `tsc`, `pytest`) |
252
- | `list_dir` | List contents of workspace directories |
253
- | `tree_view` | Generate visual ASCII directory tree structures for workspace paths |
254
- | `file_info` | Get file metadata, exact line counts, size, & modification timestamps |
255
- | `grep_search` | String/regex pattern search across entire codebase |
256
- | `glob_search` | Filter workspace files by file glob patterns |
257
- | `fetch_url` | Fetch web API documentation, specs, or external web URLs |
258
- | `git_diff` | Query uncommitted workspace git diff directly inside execution loop |
259
- | `ask_user` | Request missing secret credentials or API keys |
260
- | `speak_text` | Read text phrases out loud using Text-to-Speech (TTS) voice synthesis |
261
- | `task_completed` | Conclude autonomous goal execution with verification and report |
231
+ > 📘 **Full Architecture & Specifications**: For complete deep-dive documentation, flowcharts, and schemas, see [AUTONOMOUS_FEATURES.md](./AUTONOMOUS_FEATURES.md).
232
+
233
+ ### Desktop & Computer-Use Control Tools
234
+ | Tool | Description | Tier Availability |
235
+ |---|---|:---:|
236
+ | `computer` | Unified multi-action desktop dispatcher (click, drag, type, hotkey, open, etc.) | Pro / Enterprise |
237
+ | `computer_screenshot` | Capture display screen image and inspect active foreground window | Pro / Enterprise |
238
+ | `computer_click` | Move cursor and click mouse at coordinate (left, right, middle) | Pro / Enterprise |
239
+ | `computer_double_click` | Double-click mouse at specified screen coordinate | Pro / Enterprise |
240
+ | `computer_right_click` | Right-click mouse at screen coordinate to open context menus | Pro / Enterprise |
241
+ | `computer_mouse_move` | Move mouse cursor to specific display (x, y) coordinates | Pro / Enterprise |
242
+ | `computer_mouse_down` | Hold down mouse button for drag operations | Pro / Enterprise |
243
+ | `computer_mouse_up` | Release mouse button | Pro / Enterprise |
244
+ | `computer_drag` | Drag mouse cursor from `(fromX, fromY)` to `(toX, toY)` | Pro / Enterprise |
245
+ | `computer_scroll` | Scroll vertical or horizontal mouse wheel deltas | Pro / Enterprise |
246
+ | `computer_type` | Ensure target window focus and type text keystrokes via OS keyboard | Pro / Enterprise |
247
+ | `computer_key_press` | Press individual keyboard key (`enter`, `tab`, `escape`, `backspace`) | Pro / Enterprise |
248
+ | `computer_hotkey` | Trigger key combination (`["ctrl", "s"]`, `["alt", "f4"]`, `["alt", "tab"]`) | Pro / Enterprise |
249
+ | `computer_open_app` | Launch desktop application (Notepad, Excel, Chrome, Terminal) and focus window | Pro / Enterprise |
250
+ | `computer_close_app` | Gracefully close application or window by title/executable | Pro / Enterprise |
251
+ | `computer_switch_window` | Switch foreground active window to target application | Pro / Enterprise |
252
+ | `computer_inspect_ui` | Inspect active window coordinates, title, and UI state | Pro / Enterprise |
253
+ | `computer_wait` | Pause execution to allow desktop applications or animations to settle | Pro / Enterprise |
254
+ | `computer_diagnostic` | Retrieve system display resolution, DPI scaling, and Scout driver health | Pro / Enterprise |
255
+
256
+ ### Browser Automation & Web Scraping Tools
257
+ | Tool | Description | Tier Availability |
258
+ |---|---|:---:|
259
+ | `browser_navigate` | Open URL, handle navigation history, and construct semantic DOM tree | Pro / Enterprise |
260
+ | `browser_inspect` | Inspect clickable buttons, links, inputs, and assign element ref IDs (`[ref=1]`) | Pro / Enterprise |
261
+ | `browser_click` | Click element by CSS selector, visible label text, or ref ID | Pro / Enterprise |
262
+ | `browser_type` | Type text strings into active form fields and search inputs | Pro / Enterprise |
263
+ | `browser_select` | Choose option from dropdown `<select>` menus by value or visible text | Pro / Enterprise |
264
+ | `browser_scroll` | Scroll webpage up or down to load lazy-loaded elements | Pro / Enterprise |
265
+ | `browser_content` | Extract clean, structured visible text and tabular data from page | Pro / Enterprise |
266
+ | `browser_screenshot` | Capture visual viewport image of active browser page | Pro / Enterprise |
267
+ | `browser_download` | Automatically download web files, invoices, PDFs to workspace `downloads/` | Pro / Enterprise |
268
+
269
+ ### Cognitive Scratchpad & Reasoning Tools
270
+ | Tool | Description | Tier Availability |
271
+ |---|---|:---:|
272
+ | `scratchpad` | Private working memory for multi-step plans, hypotheses, and state (`write`, `read`, `append`, `clear`) | Pro / Enterprise |
273
+
274
+ ### Workspace & Engineering Tools (Developer Baseline)
275
+ | Tool | Description | Tier Availability |
276
+ |---|---|:---:|
277
+ | `read_file` | Read content & line ranges of workspace files | All Tiers (Free, Pro, Enterprise) |
278
+ | `write_file` | Create new files or overwrite existing source code | All Tiers (Free, Pro, Enterprise) |
279
+ | `edit_file` | Search & replace exact target code snippets | All Tiers (Free, Pro, Enterprise) |
280
+ | `multi_edit_file` | Apply batch snippet replacements in a single atomic tool call | All Tiers (Free, Pro, Enterprise) |
281
+ | `run_command` | Execute live terminal commands (`npm test`, `tsc`, `pytest`) | All Tiers (Free, Pro, Enterprise) |
282
+ | `list_dir` | List contents of workspace directories | All Tiers (Free, Pro, Enterprise) |
283
+ | `tree_view` | Generate visual ASCII directory tree structures for workspace paths | All Tiers (Free, Pro, Enterprise) |
284
+ | `file_info` | Get file metadata, exact line counts, size, & modification timestamps | All Tiers (Free, Pro, Enterprise) |
285
+ | `grep_search` | String/regex pattern search across entire codebase | All Tiers (Free, Pro, Enterprise) |
286
+ | `glob_search` | Filter workspace files by file glob patterns | All Tiers (Free, Pro, Enterprise) |
287
+ | `fetch_url` | Fetch web API documentation, specs, or external web URLs | All Tiers (Free, Pro, Enterprise) |
288
+ | `git_diff` | Query uncommitted workspace git diff directly inside execution loop | All Tiers (Free, Pro, Enterprise) |
289
+ | `ask_user` | Request missing secret credentials or API keys | All Tiers (Free, Pro, Enterprise) |
290
+ | `speak_text` | Read text phrases out loud via ElevenLabs AI Human Voice (Pro/Enterprise) or Windows SAPI5 (Free) | All Tiers (Free, Pro, Enterprise) |
291
+ | `task_completed` | Conclude autonomous goal execution with verification and report | All Tiers (Free, Pro, Enterprise) |
262
292
 
263
293
  ---
264
294
 
@@ -273,11 +303,17 @@ Scout is built for uninterrupted autonomy across multiple LLM providers:
273
303
 
274
304
  ---
275
305
 
276
- ## Dual Interaction Modes (Text & Voice)
306
+ ## Conversational Voice Co-Pilot & Audio Engine
277
307
 
278
- When starting interactive sessions in `scout agent`, `scout init` (ask mode), or `scout search`, Scout presents interactive options:
279
- 1. **Text Interaction**: Standard command/prompt input -> Text response in terminal.
280
- 2. **Audio Interaction**: Voice command input via microphone Speech-to-Text (STT) -> Text response + Audio response spoken out loud via Text-to-Speech (TTS).
308
+ When starting interactive sessions in `scout agent` (`-V` or `--voice`), `scout init` (ask mode), or `scout search`, Scout presents full two-way audio dialogue:
309
+
310
+ 1. **Native C# High-Performance Audio Capture**: Built-in standalone Win32 `waveIn` audio recording engine (`scout_audio_recorder.exe`) compiled directly via native Windows toolchain without requiring Python or external heavy runtime dependencies.
311
+ 2. **Real-Time Voice Activity Detection (VAD)**: Intelligent RMS audio power monitoring with a 2.5-second silence debounce cutoff, allowing developers to speak natural multi-sentence thoughts without mid-sentence interruptions.
312
+ 3. **Groq Whisper STT with Developer Vocabulary Hints**: Lightning-fast speech-to-text powered by `whisper-large-v3-turbo` with technical prompt biasing (e.g., *Scout, Chrome, Notepad, VS Code, TypeScript, Python*) to prevent mishearing developer commands.
313
+ 4. **Ultra-Realistic ElevenLabs Human Voice TTS**:
314
+ - **Pro & Enterprise Tiers**: Full access to **ElevenLabs AI Voice Synthesis** (`eleven_turbo_v2_5`, PCM 24kHz / WAV container) delivering human-quality, natural speech dialogue.
315
+ - **Developer Free Tier**: Fully supported with local Windows SAPI5 speech synthesis (zero API billing cost).
316
+ 5. **Conversational Co-Pilot Persona**: Instant spoken acknowledgment (e.g., *"On it! Opening Chrome for you."*) and warm conversational completion summaries rather than dry robotic execution logs.
281
317
 
282
318
  ---
283
319
 
@@ -287,7 +323,7 @@ Inside an active `scout agent` interactive session, type any slash command to tr
287
323
 
288
324
  - `/help` — Display comprehensive guide to all 15 tools and interactive slash commands.
289
325
  - `/voice` — Activate microphone Speech-to-Text (STT) listener to speak your prompt into CLI.
290
- - `/speak <text>` — Read entered text out loud via Text-to-Speech (TTS) voice synthesis.
326
+ - `/speak <text>` — Read entered text out loud via ElevenLabs AI (Pro/Enterprise) or SAPI5 (Free).
291
327
  - `/tree [dir]` — Render an interactive visual directory tree of any workspace folder.
292
328
  - `/status` or `/stats` — View real-time session telemetry (steps executed, tool call count, modified files, elapsed time, branch info).
293
329
  - `/test [cmd]` — Auto-detect and run project verification test suite directly inside session context.
@@ -301,12 +337,18 @@ Inside an active `scout agent` interactive session, type any slash command to tr
301
337
 
302
338
  ---
303
339
 
304
- ## What's New in v9.0.5
305
- - **Autonomous Computer-Use & Desktop App Operator**: Native desktop application control (Notepad, Excel, VS Code, Browser) with full mouse, keyboard, hotkey, and window focus management across Windows, macOS, and Linux.
340
+ ## What's New in v9.0.7
341
+ - **Ultra-Realistic ElevenLabs AI Voice Engine**: Integrated human-grade voice synthesis via ElevenLabs for Pro and Enterprise subscribers, bringing lifelike conversational audio feedback directly to the CLI co-pilot.
342
+ - **Native C# Audio Recorder with Real-Time VAD**: Standalone Win32 `waveIn` audio recording binary (`scout_audio_recorder.exe`) with RMS energy detection and 2.5s silence debounce cutoff.
343
+ - **Zero-Latency Groq Whisper STT**: Integrated Groq `whisper-large-v3-turbo` with specialized developer vocabulary hints to eliminate speech recognition errors.
344
+ - **Intelligent Single-Step App Intent Engine (`checkSimpleAppAction`)**: Natural language comprehension for desktop app commands across English and Hinglish (e.g., *"open chrome"*, *"chrome open kar do"*, *"launch vscode"*), terminating immediately on Step 1 without unnecessary repository file exploration.
345
+ - **Autonomous Computer-Use & Desktop App Operator**: Native desktop application control (Notepad, Excel, VS Code, Browser) with full mouse, keyboard, hotkey, and window focus management across Windows, macOS, and Linux powered by the native `scout_driver.exe` engine.
346
+ - **Autonomous Browser Automation & Document Download Engine**: Fully automated web browsing (`browser_navigate`, `browser_inspect`, `browser_click`, `browser_type`, `browser_select`, `browser_scroll`, `browser_content`, `browser_screenshot`, `browser_download`) with DOM extraction and automatic file downloads to `downloads/`.
347
+ - **Cognitive Scratchpad & Working Memory (`scratchpad`)**: Long-horizon multi-step reasoning memory (`write`, `read`, `append`, `clear`) saved to `.ft/scratchpad.md`, keeping model context concise and focused.
306
348
  - **Strict Execution Intent Alignment (`USER INTENT > TOOL CONVENIENCE`)**: Differentiates between outcome and execution requirements; strictly blocks silent filesystem shortcuts when application interaction is requested.
307
349
  - **Bounded Computer Recovery Loop**: Structured action feedback (`{ success, action, error, recoverable }`) with automated re-observation, window refocusing, and retries.
308
350
  - **Action Verifier & Anti-Substitute Path Rejection**: Verifies true destination (resolving Windows OneDrive/Desktop and `~/Desktop`) and rejects substitute files created in the workspace root.
309
- - **Interactive Web Dashboard & Sync Server (`scout dashboard`)**: Launch a local web GUI to visually inspect repository maps, audit logs, plan details, and live agent controls.
351
+ - **Interactive Web Dashboard & Sync Server (`scout dashboard`)**: Launch a local web GUI to visually inspect repository maps, audit logs, plan details, live agent controls, and subscription tiers.
310
352
  - **Developer Authentication Guard (`scout login` / `signup` / `auth`)**: Unified account management with Firebase and Holosene token authentication, securing command execution and session persistence.
311
353
  - **Quota & Subscription Management (`scout quota` / `plan`)**: Easily inspect real-time usage quotas, track billing renewal cycles, switch between tiers, or configure BYOK (Bring-Your-Own-Key) access.
312
354
  - **Automated Model Benchmark Suite (`scout benchmark`)**: Run live end-to-end latency and throughput benchmarks (tokens/sec) across all configured LLM providers (Nemotron, Qwen, Gemini, GPT).
@@ -1 +1 @@
1
- {"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["agent.ts"],"names":[],"mappings":"AA6BA,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACpC,WAAW,EAAE,MAAM,CAAC;CACvB;AAKD,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,QAKpD;AAED,wBAAgB,cAAc,IAAI,YAAY,EAAE,CAO/C;AAED,MAAM,WAAW,YAAY;IACzB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IACpC,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC;IACpC,IAAI,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7B,GAAG,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/B;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAsDlE;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAG3E;AAED,MAAM,WAAW,gBAAgB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,UAAU,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACvE,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CACtC;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAoF5F;AAED,wBAAgB,yBAAyB,CACrC,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,MAAM,EAAE,GACpB,MAAM,EAAE,CAgGV;AAED,wBAAsB,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAMxF;AAED,wBAAsB,WAAW,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,iBA4nC3E"}
1
+ {"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["agent.ts"],"names":[],"mappings":"AA6BA,MAAM,WAAW,YAAY;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IACpC,WAAW,EAAE,MAAM,CAAC;CACvB;AA6CD,wBAAgB,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,QAKpD;AAED,wBAAgB,cAAc,IAAI,YAAY,EAAE,CAO/C;AAED,MAAM,WAAW,YAAY;IACzB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IACpC,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,SAAS,CAAC;IACpC,IAAI,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC3B,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,MAAM,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7B,GAAG,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC1B,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IAC5B,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC/B;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAsDlE;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAG3E;AAED,MAAM,WAAW,gBAAgB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,UAAU,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;IACvE,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAChC,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;CACtC;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,gBAAgB,CAoF5F;AAED,wBAAgB,yBAAyB,CACrC,UAAU,EAAE,MAAM,EAClB,aAAa,EAAE,MAAM,GAAG,SAAS,EACjC,SAAS,EAAE,MAAM,EAAE,GACpB,MAAM,EAAE,CAgGV;AAED,wBAAsB,iBAAiB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAMxF;AAED,wBAAsB,WAAW,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,iBA+nC3E"}
@@ -18,6 +18,44 @@ import { promisify } from 'util';
18
18
  const execAsync = promisify(exec);
19
19
  const FT_DIR = path.join(process.cwd(), '.ft', 'scout');
20
20
  const SCAN_CACHE = path.join(FT_DIR, 'scan-report.json');
21
+ function getConversationalAcknowledgment(goal) {
22
+ const lower = goal.toLowerCase().trim();
23
+ if (/^(hi|hello|hey|sup|namaste)/i.test(lower)) {
24
+ return "Hey there! How can I help you today?";
25
+ }
26
+ if (lower.startsWith('open ') || lower.startsWith('launch ')) {
27
+ const app = goal.slice(5).trim();
28
+ return `On it! Opening ${app} for you.`;
29
+ }
30
+ if (lower.startsWith('close ') || lower.startsWith('kill ')) {
31
+ const app = goal.slice(6).trim();
32
+ return `Sure, closing ${app}.`;
33
+ }
34
+ if (lower.startsWith('create ') || lower.startsWith('make ')) {
35
+ return "Understood! Getting started on creating that.";
36
+ }
37
+ if (lower.startsWith('fix ') || lower.startsWith('debug ')) {
38
+ return "Let's track that down. Inspecting the issue now.";
39
+ }
40
+ if (lower.startsWith('run ') || lower.startsWith('test ')) {
41
+ return "Running that right away.";
42
+ }
43
+ const cleanGoal = goal.replace(/["'`]/g, '');
44
+ const shortGoal = cleanGoal.length > 30 ? cleanGoal.slice(0, 30) + '...' : cleanGoal;
45
+ return `Got it! Working on ${shortGoal}.`;
46
+ }
47
+ function formatConversationalClosing(summary) {
48
+ let text = summary.trim();
49
+ text = text.replace(/^(execution|task|action)\s*(completed|finished|summary)?\s*:\s*/i, '');
50
+ text = text.replace(/^mock execution completed successfully for goal:\s*"[^"]*"\.?\s*/i, '');
51
+ text = text.replace(/`[^`]+`/g, (m) => m.replace(/`/g, ''));
52
+ text = text.replace(/\[[^\]]+\]/g, '');
53
+ text = text.replace(/\s+/g, ' ').trim();
54
+ if (!text.endsWith('.') && !text.endsWith('!') && !text.endsWith('?')) {
55
+ text += '.';
56
+ }
57
+ return text;
58
+ }
21
59
  export function saveScanReport(issues) {
22
60
  if (!fs.existsSync(FT_DIR)) {
23
61
  fs.mkdirSync(FT_DIR, { recursive: true });
@@ -342,6 +380,8 @@ ${chalk.cyan('/exit')} - Exit session
342
380
  if (transcript && transcript.trim()) {
343
381
  rawGoal = transcript.trim();
344
382
  safeNote(` Voice Command Received: "${chalk.bold.green(rawGoal)}"`, ' Scout STT Input');
383
+ const ack = getConversationalAcknowledgment(rawGoal);
384
+ speakText(ack, { async: true });
345
385
  }
346
386
  else {
347
387
  safeNote(chalk.yellow('No speech transcript detected or listener timed out.'), ' Scout Audio Interaction');
@@ -1087,8 +1127,9 @@ ${execResult.summary}
1087
1127
  `.trim();
1088
1128
  safeNote(renderMarkdown(teardownMd), ' Scout Execution Teardown Report');
1089
1129
  if (isAudioMode && execResult?.summary) {
1090
- safeNote(` Speaking summary response out loud...`, ' Scout Audio Output');
1091
- await speakText(execResult.summary);
1130
+ safeNote(` Speaking conversational response out loud...`, ' Scout Audio Output');
1131
+ const spokenClosing = formatConversationalClosing(execResult.summary);
1132
+ await speakText(spokenClosing);
1092
1133
  }
1093
1134
  if (modifiedFiles.length === 0)
1094
1135
  continue;