@equinor/fusion-framework-cli-plugin-copilot 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/CHANGELOG.md +24 -0
  2. package/LICENSE +21 -0
  3. package/README.md +228 -0
  4. package/dist/esm/commands/app/command.js +176 -0
  5. package/dist/esm/commands/app/command.js.map +1 -0
  6. package/dist/esm/commands/app/eval.js +92 -0
  7. package/dist/esm/commands/app/eval.js.map +1 -0
  8. package/dist/esm/commands/app/format.js +93 -0
  9. package/dist/esm/commands/app/format.js.map +1 -0
  10. package/dist/esm/commands/app/login.js +71 -0
  11. package/dist/esm/commands/app/login.js.map +1 -0
  12. package/dist/esm/commands/app/prompts/execute-step.prompt.js +44 -0
  13. package/dist/esm/commands/app/prompts/execute-step.prompt.js.map +1 -0
  14. package/dist/esm/commands/app/prompts/judge.prompt.js +116 -0
  15. package/dist/esm/commands/app/prompts/judge.prompt.js.map +1 -0
  16. package/dist/esm/commands/app/prompts/plan.prompt.js +47 -0
  17. package/dist/esm/commands/app/prompts/plan.prompt.js.map +1 -0
  18. package/dist/esm/commands/app/prompts/step.prompt.js +42 -0
  19. package/dist/esm/commands/app/prompts/step.prompt.js.map +1 -0
  20. package/dist/esm/commands/app/prompts/system.dev-server.prompt.js +21 -0
  21. package/dist/esm/commands/app/prompts/system.dev-server.prompt.js.map +1 -0
  22. package/dist/esm/commands/app/prompts/system.prompt.js +77 -0
  23. package/dist/esm/commands/app/prompts/system.prompt.js.map +1 -0
  24. package/dist/esm/commands/app/server.js +83 -0
  25. package/dist/esm/commands/app/server.js.map +1 -0
  26. package/dist/esm/commands/app/session-logger.js +171 -0
  27. package/dist/esm/commands/app/session-logger.js.map +1 -0
  28. package/dist/esm/commands/app/tools/click.js +27 -0
  29. package/dist/esm/commands/app/tools/click.js.map +1 -0
  30. package/dist/esm/commands/app/tools/errors.js +24 -0
  31. package/dist/esm/commands/app/tools/errors.js.map +1 -0
  32. package/dist/esm/commands/app/tools/eval-js.js +36 -0
  33. package/dist/esm/commands/app/tools/eval-js.js.map +1 -0
  34. package/dist/esm/commands/app/tools/fill.js +28 -0
  35. package/dist/esm/commands/app/tools/fill.js.map +1 -0
  36. package/dist/esm/commands/app/tools/find.js +44 -0
  37. package/dist/esm/commands/app/tools/find.js.map +1 -0
  38. package/dist/esm/commands/app/tools/get-styles.js +36 -0
  39. package/dist/esm/commands/app/tools/get-styles.js.map +1 -0
  40. package/dist/esm/commands/app/tools/get-url.js +24 -0
  41. package/dist/esm/commands/app/tools/get-url.js.map +1 -0
  42. package/dist/esm/commands/app/tools/go-back.js +15 -0
  43. package/dist/esm/commands/app/tools/go-back.js.map +1 -0
  44. package/dist/esm/commands/app/tools/hover.js +24 -0
  45. package/dist/esm/commands/app/tools/hover.js.map +1 -0
  46. package/dist/esm/commands/app/tools/index.js +32 -0
  47. package/dist/esm/commands/app/tools/index.js.map +1 -0
  48. package/dist/esm/commands/app/tools/navigate.js +31 -0
  49. package/dist/esm/commands/app/tools/navigate.js.map +1 -0
  50. package/dist/esm/commands/app/tools/press-key.js +24 -0
  51. package/dist/esm/commands/app/tools/press-key.js.map +1 -0
  52. package/dist/esm/commands/app/tools/read-file.js +98 -0
  53. package/dist/esm/commands/app/tools/read-file.js.map +1 -0
  54. package/dist/esm/commands/app/tools/registry.js +34 -0
  55. package/dist/esm/commands/app/tools/registry.js.map +1 -0
  56. package/dist/esm/commands/app/tools/reload.js +15 -0
  57. package/dist/esm/commands/app/tools/reload.js.map +1 -0
  58. package/dist/esm/commands/app/tools/screenshot.js +133 -0
  59. package/dist/esm/commands/app/tools/screenshot.js.map +1 -0
  60. package/dist/esm/commands/app/tools/scroll.js +41 -0
  61. package/dist/esm/commands/app/tools/scroll.js.map +1 -0
  62. package/dist/esm/commands/app/tools/select.js +25 -0
  63. package/dist/esm/commands/app/tools/select.js.map +1 -0
  64. package/dist/esm/commands/app/tools/snapshot.js +24 -0
  65. package/dist/esm/commands/app/tools/snapshot.js.map +1 -0
  66. package/dist/esm/commands/app/tools/type-text.js +29 -0
  67. package/dist/esm/commands/app/tools/type-text.js.map +1 -0
  68. package/dist/esm/commands/app/tools/types.js +2 -0
  69. package/dist/esm/commands/app/tools/types.js.map +1 -0
  70. package/dist/esm/commands/app/tools/wait.js +39 -0
  71. package/dist/esm/commands/app/tools/wait.js.map +1 -0
  72. package/dist/esm/commands/app/tools/write-file.js +98 -0
  73. package/dist/esm/commands/app/tools/write-file.js.map +1 -0
  74. package/dist/esm/commands/app/types.js +2 -0
  75. package/dist/esm/commands/app/types.js.map +1 -0
  76. package/dist/esm/eval-resolve.js +83 -0
  77. package/dist/esm/eval-resolve.js.map +1 -0
  78. package/dist/esm/index.js +18 -0
  79. package/dist/esm/index.js.map +1 -0
  80. package/dist/esm/utils/agent-browser.js +67 -0
  81. package/dist/esm/utils/agent-browser.js.map +1 -0
  82. package/dist/esm/utils/browser-poll.js +26 -0
  83. package/dist/esm/utils/browser-poll.js.map +1 -0
  84. package/dist/esm/utils/daemon.js +83 -0
  85. package/dist/esm/utils/daemon.js.map +1 -0
  86. package/dist/esm/utils/index.js +6 -0
  87. package/dist/esm/utils/index.js.map +1 -0
  88. package/dist/esm/utils/process.js +30 -0
  89. package/dist/esm/utils/process.js.map +1 -0
  90. package/dist/esm/utils/server.js +65 -0
  91. package/dist/esm/utils/server.js.map +1 -0
  92. package/dist/esm/version.js +3 -0
  93. package/dist/esm/version.js.map +1 -0
  94. package/dist/tsconfig.tsbuildinfo +1 -0
  95. package/dist/types/commands/app/command.d.ts +7 -0
  96. package/dist/types/commands/app/eval.d.ts +9 -0
  97. package/dist/types/commands/app/format.d.ts +14 -0
  98. package/dist/types/commands/app/login.d.ts +14 -0
  99. package/dist/types/commands/app/prompts/execute-step.prompt.d.ts +13 -0
  100. package/dist/types/commands/app/prompts/judge.prompt.d.ts +15 -0
  101. package/dist/types/commands/app/prompts/plan.prompt.d.ts +13 -0
  102. package/dist/types/commands/app/prompts/step.prompt.d.ts +12 -0
  103. package/dist/types/commands/app/prompts/system.dev-server.prompt.d.ts +12 -0
  104. package/dist/types/commands/app/prompts/system.prompt.d.ts +12 -0
  105. package/dist/types/commands/app/server.d.ts +26 -0
  106. package/dist/types/commands/app/session-logger.d.ts +14 -0
  107. package/dist/types/commands/app/tools/click.d.ts +9 -0
  108. package/dist/types/commands/app/tools/errors.d.ts +9 -0
  109. package/dist/types/commands/app/tools/eval-js.d.ts +13 -0
  110. package/dist/types/commands/app/tools/fill.d.ts +9 -0
  111. package/dist/types/commands/app/tools/find.d.ts +9 -0
  112. package/dist/types/commands/app/tools/get-styles.d.ts +13 -0
  113. package/dist/types/commands/app/tools/get-url.d.ts +9 -0
  114. package/dist/types/commands/app/tools/go-back.d.ts +9 -0
  115. package/dist/types/commands/app/tools/hover.d.ts +9 -0
  116. package/dist/types/commands/app/tools/index.d.ts +9 -0
  117. package/dist/types/commands/app/tools/navigate.d.ts +9 -0
  118. package/dist/types/commands/app/tools/press-key.d.ts +9 -0
  119. package/dist/types/commands/app/tools/read-file.d.ts +13 -0
  120. package/dist/types/commands/app/tools/registry.d.ts +28 -0
  121. package/dist/types/commands/app/tools/reload.d.ts +9 -0
  122. package/dist/types/commands/app/tools/screenshot.d.ts +9 -0
  123. package/dist/types/commands/app/tools/scroll.d.ts +9 -0
  124. package/dist/types/commands/app/tools/select.d.ts +9 -0
  125. package/dist/types/commands/app/tools/snapshot.d.ts +9 -0
  126. package/dist/types/commands/app/tools/type-text.d.ts +9 -0
  127. package/dist/types/commands/app/tools/types.d.ts +12 -0
  128. package/dist/types/commands/app/tools/wait.d.ts +9 -0
  129. package/dist/types/commands/app/tools/write-file.d.ts +19 -0
  130. package/dist/types/commands/app/types.d.ts +68 -0
  131. package/dist/types/eval-resolve.d.ts +31 -0
  132. package/dist/types/index.d.ts +15 -0
  133. package/dist/types/utils/agent-browser.d.ts +26 -0
  134. package/dist/types/utils/browser-poll.d.ts +9 -0
  135. package/dist/types/utils/daemon.d.ts +16 -0
  136. package/dist/types/utils/index.d.ts +5 -0
  137. package/dist/types/utils/process.d.ts +21 -0
  138. package/dist/types/utils/server.d.ts +26 -0
  139. package/dist/types/version.d.ts +1 -0
  140. package/package.json +57 -0
  141. package/src/commands/app/command.ts +204 -0
  142. package/src/commands/app/eval.ts +114 -0
  143. package/src/commands/app/format.ts +110 -0
  144. package/src/commands/app/login.ts +92 -0
  145. package/src/commands/app/prompts/execute-step.prompt.ts +48 -0
  146. package/src/commands/app/prompts/judge.prompt.ts +121 -0
  147. package/src/commands/app/prompts/plan.prompt.ts +48 -0
  148. package/src/commands/app/prompts/step.prompt.ts +45 -0
  149. package/src/commands/app/prompts/system.dev-server.prompt.ts +22 -0
  150. package/src/commands/app/prompts/system.prompt.ts +78 -0
  151. package/src/commands/app/server.ts +105 -0
  152. package/src/commands/app/session-logger.ts +187 -0
  153. package/src/commands/app/tools/click.ts +28 -0
  154. package/src/commands/app/tools/errors.ts +26 -0
  155. package/src/commands/app/tools/eval-js.ts +40 -0
  156. package/src/commands/app/tools/fill.ts +29 -0
  157. package/src/commands/app/tools/find.ts +49 -0
  158. package/src/commands/app/tools/get-styles.ts +40 -0
  159. package/src/commands/app/tools/get-url.ts +26 -0
  160. package/src/commands/app/tools/go-back.ts +16 -0
  161. package/src/commands/app/tools/hover.ts +25 -0
  162. package/src/commands/app/tools/index.ts +57 -0
  163. package/src/commands/app/tools/navigate.ts +32 -0
  164. package/src/commands/app/tools/press-key.ts +25 -0
  165. package/src/commands/app/tools/read-file.ts +113 -0
  166. package/src/commands/app/tools/registry.ts +38 -0
  167. package/src/commands/app/tools/reload.ts +16 -0
  168. package/src/commands/app/tools/screenshot.ts +168 -0
  169. package/src/commands/app/tools/scroll.ts +45 -0
  170. package/src/commands/app/tools/select.ts +26 -0
  171. package/src/commands/app/tools/snapshot.ts +27 -0
  172. package/src/commands/app/tools/type-text.ts +30 -0
  173. package/src/commands/app/tools/types.ts +15 -0
  174. package/src/commands/app/tools/wait.ts +43 -0
  175. package/src/commands/app/tools/write-file.ts +104 -0
  176. package/src/commands/app/types.ts +76 -0
  177. package/src/eval-resolve.ts +113 -0
  178. package/src/index.ts +20 -0
  179. package/src/utils/agent-browser.ts +66 -0
  180. package/src/utils/browser-poll.ts +27 -0
  181. package/src/utils/daemon.ts +84 -0
  182. package/src/utils/index.ts +5 -0
  183. package/src/utils/process.ts +33 -0
  184. package/src/utils/server.ts +66 -0
  185. package/src/version.ts +2 -0
  186. package/tsconfig.json +13 -0
package/CHANGELOG.md ADDED
@@ -0,0 +1,24 @@
1
+ # @equinor/fusion-framework-cli-plugin-copilot
2
+
3
+ ## 1.0.0
4
+
5
+ ### Minor Changes
6
+
7
+ - abffa53: Add the `copilot app eval` CLI plugin for agentic Fusion application evaluation with the GitHub Copilot SDK and `agent-browser`.
8
+
9
+ The new plugin can read eval markdown files, run a Fusion app, collect browser evidence, and return structured pass or fail verdicts with screenshots, snapshots, and console error checks. The login flow is available through both `--login` and `--logon`.
10
+
11
+ Refs https://github.com/equinor/fusion-core-tasks/issues/724
12
+
13
+ ### Patch Changes
14
+
15
+ - abffa53: Improve the cookbook eval agent system prompt so it reasons over user stories or provided context, plans browser steps, executes them agentically, and produces more explicit criterion-by-criterion verdicts.
16
+ - Updated dependencies [abffa53]
17
+ - Updated dependencies [abffa53]
18
+ - Updated dependencies [abffa53]
19
+ - Updated dependencies [ae92f13]
20
+ - Updated dependencies [abffa53]
21
+ - Updated dependencies [c123c39]
22
+ - Updated dependencies [3de232c]
23
+ - Updated dependencies [32bcf83]
24
+ - @equinor/fusion-framework-cli@14.0.0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2022 Equinor
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,228 @@
1
+ # @equinor/fusion-framework-cli-plugin-copilot
2
+
3
+ Agentic evaluation plugin for the Fusion Framework CLI. It uses the [GitHub Copilot SDK](https://www.npmjs.com/package/@github/copilot-sdk) to create a conversational session in which an LLM autonomously drives [`agent-browser`](https://www.npmjs.com/package/agent-browser) to navigate, interact with, and judge a running Fusion application against eval criteria written in plain markdown.
4
+
5
+ ## When to use this
6
+
7
+ - You have a Fusion application (cookbook or production app) and want to verify it works end-to-end.
8
+ - You want an LLM to autonomously explore and validate acceptance criteria without writing Playwright scripts.
9
+ - You want structured, evidence-backed pass/fail verdicts with screenshots, snapshots, and computed style checks.
10
+
11
+ ## How it works
12
+
13
+ The plugin follows a single-session agentic workflow: the Copilot agent reads your eval markdown, decides what to check, drives the browser through tool calls, collects evidence, and produces a structured JSON verdict in one session.
14
+
15
+ ```mermaid
16
+ flowchart LR
17
+ A[CLI] -->|starts| B[Dev Server]
18
+ A -->|reads| C[eval/*.md]
19
+ A -->|creates| D[Copilot Session]
20
+ D -->|system prompt +\neval markdown| E[LLM]
21
+ E <-->|tool calls| F[agent-browser]
22
+ F <-->|headless Chrome| B
23
+ E -->|JSON verdict| A
24
+ ```
25
+
26
+ ### Eval lifecycle
27
+
28
+ ```mermaid
29
+ sequenceDiagram
30
+ participant CLI as ffc copilot app eval
31
+ participant Server as Dev Server
32
+ participant SDK as Copilot SDK
33
+ participant LLM as LLM Agent
34
+ participant AB as agent-browser
35
+ participant Browser as Headless Chrome
36
+
37
+ CLI->>CLI: Resolve eval files from app/eval/*.md
38
+ CLI->>Server: Start dev server (ffc app serve)
39
+ CLI->>CLI: Wait for server ready
40
+ CLI->>AB: Reset daemon (kill stale sessions)
41
+
42
+ loop For each eval file
43
+ CLI->>CLI: Create timestamped run directory
44
+ CLI->>SDK: Create session with browser tools
45
+ SDK->>LLM: System prompt + eval markdown
46
+
47
+ loop Agent autonomously evaluates
48
+ LLM->>AB: browser_navigate(url)
49
+ AB->>Browser: Open page
50
+ LLM->>AB: browser_wait(networkidle)
51
+ LLM->>AB: browser_snapshot()
52
+ AB-->>LLM: Accessibility tree
53
+ LLM->>AB: browser_get_styles(selector)
54
+ AB-->>LLM: Computed CSS values
55
+ LLM->>AB: browser_screenshot()
56
+ AB-->>LLM: Image data
57
+ LLM->>AB: browser_errors()
58
+ AB-->>LLM: Console errors
59
+ end
60
+
61
+ LLM-->>CLI: JSON verdict {pass, reasoning, steps}
62
+ CLI->>CLI: Save verdict + evidence artifacts
63
+ end
64
+
65
+ CLI->>AB: Reset daemon
66
+ CLI->>Server: Stop dev server
67
+ CLI->>CLI: Exit 0 (all pass) or 1 (any fail)
68
+ ```
69
+
70
+ ## Quick start
71
+
72
+ ```bash
73
+ # Evaluate all eval files in a cookbook
74
+ ffc copilot app eval ./cookbooks/app-react
75
+
76
+ # Or use the workspace-root shortcut
77
+ pnpm eval:app ./cookbooks/app-react
78
+
79
+ # Run a specific eval
80
+ ffc copilot app eval ./cookbooks/app-react --eval smoke
81
+
82
+ # Use a specific LLM model
83
+ ffc copilot app eval ./cookbooks/app-react --model claude-sonnet-4
84
+
85
+ # Point at an already-running server
86
+ ffc copilot app eval . --url http://localhost:3000/apps/my-app
87
+ ```
88
+
89
+ ### MSAL login (first-time setup)
90
+
91
+ If the application requires authentication, run the login flow once to persist MSAL tokens in the Chrome profile:
92
+
93
+ ```bash
94
+ ffc copilot app eval ./cookbooks/app-react --login
95
+ # Alias
96
+ ffc copilot app eval ./cookbooks/app-react --logon
97
+ ```
98
+
99
+ This opens a **headed** browser for interactive login. After you authenticate, press Ctrl+C. Subsequent eval runs reuse the saved session automatically.
100
+
101
+ ## Writing eval files
102
+
103
+ Eval files are plain markdown files placed in the app's `eval/` directory. The plugin passes the raw markdown directly to the LLM as the user prompt. No special parsing is required.
104
+
105
+ ```
106
+ my-app/
107
+ ├── eval/
108
+ │ ├── smoke.md
109
+ │ └── accessibility.md
110
+ ├── src/
111
+ └── package.json
112
+ ```
113
+
114
+ ### Eval file format
115
+
116
+ Write eval files as you would a user story or test plan. The LLM reads the markdown and decides how to verify each criterion.
117
+
118
+ ```markdown
119
+ ---
120
+ name: smoke-test
121
+ ---
122
+
123
+ ## User Story
124
+
125
+ As a user, I need to see the application load successfully so I know
126
+ the deployment is healthy.
127
+
128
+ ## Acceptance Criteria
129
+
130
+ - must see a heading with the application name
131
+ - must not have any JavaScript errors in the console
132
+ - should load within 5 seconds
133
+ - should display navigation elements
134
+ ```
135
+
136
+ Both story-driven formats (user story + acceptance criteria) and instruction-driven formats (numbered steps + assertions) work. The LLM adapts to the structure you provide.
137
+
138
+ ## CLI reference
139
+
140
+ ```
141
+ ffc copilot app eval <path> [options]
142
+ ```
143
+
144
+ | Option | Description | Default |
145
+ |--------|-------------|---------|
146
+ | `<path>` | Path to the Fusion application directory | required |
147
+ | `--eval <name-or-path>` | Run a specific eval by name or file path | All files in `eval/` |
148
+ | `--port <port>` | Port for the app dev server | `3000` |
149
+ | `--host <host>` | Host for the app dev server | `0.0.0.0` |
150
+ | `--url <url>` | Skip server start and use an already-running URL | none |
151
+ | `--verbose` | Show `agent-browser` commands and app server output | `false` |
152
+ | `--login`, `--logon` | Open a headed browser for interactive MSAL login | `false` |
153
+ | `-m, --model <model>` | LLM model to use, for example `claude-sonnet-4` | SDK default |
154
+ | `-o, --output <dir>` | Output directory for run artifacts | `.tmp/copilot/` |
155
+
156
+ ## Run artifacts
157
+
158
+ Each eval run creates a timestamped directory containing evidence and metadata:
159
+
160
+ ```
161
+ .tmp/copilot/smoke-test_143052/
162
+ ├── copilot-log.jsonl # Tool call log (timestamp, tool name, arguments)
163
+ ├── copilot-response.md # Full LLM response text
164
+ ├── verdict.json # Structured pass/fail verdict
165
+ └── evidence/
166
+ ├── snapshot.txt # Accessibility tree snapshot
167
+ ├── errors.txt # JavaScript console errors
168
+ ├── url.txt # Final page URL
169
+ ├── screenshot-*.jpg # Visual evidence screenshots
170
+ ├── styles-*.txt # Computed CSS style captures
171
+ └── eval-*.json # JavaScript evaluation results
172
+ ```
173
+
174
+ ## Available browser tools
175
+
176
+ The LLM can use 18 browser tools during an eval session:
177
+
178
+ | Tool | Description |
179
+ |------|-------------|
180
+ | `browser_navigate` | Open a URL in the browser |
181
+ | `browser_snapshot` | Capture an accessibility tree with element refs (`@e1`, `@e2`) |
182
+ | `browser_screenshot` | Take a screenshot and return the image to the model |
183
+ | `browser_get_styles` | Get computed CSS styles of an element |
184
+ | `browser_eval` | Evaluate a JavaScript expression in the page context |
185
+ | `browser_click` | Click an element by ref, CSS selector, or text |
186
+ | `browser_fill` | Fill a form field (clears existing content first) |
187
+ | `browser_type` | Type text character by character |
188
+ | `browser_press_key` | Press a keyboard key (Enter, Tab, Escape, etc.) |
189
+ | `browser_hover` | Hover over an element |
190
+ | `browser_select` | Select an option from a dropdown |
191
+ | `browser_scroll` | Scroll the page or scroll an element into view |
192
+ | `browser_wait` | Wait for load state, text, element, or timeout |
193
+ | `browser_find` | Semantic element lookup by role, text, or label |
194
+ | `browser_errors` | Get JavaScript console errors |
195
+ | `browser_get_url` | Get the current page URL |
196
+ | `browser_go_back` | Navigate back in browser history |
197
+ | `browser_reload` | Reload the current page |
198
+
199
+ ## Package structure
200
+
201
+ ```mermaid
202
+ graph TD
203
+ subgraph "@equinor/fusion-framework-cli-plugin-copilot"
204
+ IDX[index.ts — plugin entry point]
205
+ CMD[commands/app/command.ts — CLI command tree]
206
+ EVAL[commands/app/eval.ts — Copilot session runner]
207
+ LOGIN[commands/app/login.ts — MSAL login flow]
208
+ RESOLVE[eval-resolve.ts — eval file discovery]
209
+ TOOLS[commands/app/tools/ — 18 browser tool wrappers]
210
+ UTILS[utils/ — agent-browser, daemon, server helpers]
211
+ end
212
+
213
+ IDX -->|registers| CMD
214
+ CMD -->|--login| LOGIN
215
+ CMD -->|resolves| RESOLVE
216
+ CMD -->|runs| EVAL
217
+ EVAL -->|creates| TOOLS
218
+ EVAL -->|uses| UTILS
219
+ LOGIN -->|uses| UTILS
220
+ TOOLS -->|calls| UTILS
221
+ ```
222
+
223
+ ## Prerequisites
224
+
225
+ - **Node.js** >= 20
226
+ - **pnpm** as the workspace package manager
227
+ - **agent-browser** installed globally or available on `PATH`
228
+ - **GitHub Copilot** access in VS Code, because the SDK authenticates through the GitHub Copilot extension
@@ -0,0 +1,176 @@
1
+ import { createCommand } from 'commander';
2
+ import { basename } from 'node:path';
3
+ import { resetDaemon, cleanup, createRunDir } from '../../utils/index.js';
4
+ import { runLogin } from './login.js';
5
+ import { resolveEvalCommandInput, resolveEvalFiles } from '../../eval-resolve.js';
6
+ import { createSession } from './eval.js';
7
+ import { startAppServer } from './server.js';
8
+ import { attachSessionLogger } from './session-logger.js';
9
+ import { formatVerdict } from './format.js';
10
+ import { existsSync, readFileSync } from 'node:fs';
11
+ import { createPlanPrompt } from './prompts/plan.prompt.js';
12
+ import { createStepPrompt } from './prompts/step.prompt.js';
13
+ import { createJudgePrompt } from './prompts/judge.prompt.js';
14
+ import chalk from 'chalk';
15
+ /**
16
+ * CLI command: `copilot app eval <path-or-eval>`
17
+ *
18
+ * Evaluates a Fusion application using the GitHub Copilot SDK with
19
+ * `agent-browser` exposed as tools. The Copilot agent autonomously
20
+ * navigates, interacts, and judges the application against eval criteria.
21
+ */
22
+ const _appEvalCommand = createCommand('eval')
23
+ .description('Evaluate a Fusion app using Copilot SDK with agent-browser tools')
24
+ .argument('<path-or-eval>', 'Path to the Fusion application directory or an eval markdown file')
25
+ .option('--eval <name-or-path>', 'Specific eval to run (name or file path)')
26
+ .option('--port <port>', 'Port for the app server', '3333')
27
+ .option('--host <host>', 'Host for the app server', '0.0.0.0')
28
+ .option('--url <url>', 'Skip server start, use an already-running URL')
29
+ .option('--verbose', 'Show detailed output', false)
30
+ .option('--logon, --login', 'Open a headed browser for interactive MSAL login', false)
31
+ .option('-m, --model <model>', 'LLM model to use (e.g. claude-sonnet-4)')
32
+ .option('-r, --reasoning <effort>', 'Reasoning effort: low, medium, high, xhigh')
33
+ .option('-o, --output <dir>', 'Output directory for run artifacts')
34
+ .addHelpText('after', [
35
+ '',
36
+ 'Uses the GitHub Copilot SDK to create an agentic session where the LLM',
37
+ 'autonomously drives agent-browser to verify eval criteria.',
38
+ '',
39
+ 'Examples:',
40
+ ' $ ffc copilot app eval ./cookbooks/app-react',
41
+ ' $ ffc copilot app eval . --eval smoke',
42
+ ' $ ffc copilot app eval . --eval 06-people-api-page.md',
43
+ ' $ ffc copilot app eval ./cookbooks/app-react-router/eval/06-people-api-page.md',
44
+ ' $ ffc copilot app eval . --model claude-sonnet-4',
45
+ ' $ ffc copilot app eval . --login',
46
+ ' $ ffc copilot app eval . --logon',
47
+ ' $ ffc copilot app eval . --url http://localhost:3333/apps/my-app',
48
+ ].join('\n'))
49
+ .action(async (appOrEvalPath, options) => {
50
+ const resolvedInput = resolveEvalCommandInput(appOrEvalPath);
51
+ const absAppPath = resolvedInput.appPath;
52
+ const port = parseInt(options.port, 10);
53
+ const verbose = options.verbose;
54
+ if (resolvedInput.evalFile && options.eval) {
55
+ console.error('❌ Do not combine a positional eval markdown file with --eval');
56
+ process.exit(1);
57
+ }
58
+ // ── Login mode: open headed browser for interactive MSAL login ──
59
+ if (options.login) {
60
+ await runLogin(absAppPath, port, options.host);
61
+ return;
62
+ }
63
+ // Resolve eval files
64
+ const evalFiles = resolvedInput.evalFile
65
+ ? [resolvedInput.evalFile]
66
+ : resolveEvalFiles(absAppPath, options.eval);
67
+ console.log(`📋 Found ${evalFiles.length} eval(s): ${evalFiles.map((f) => basename(f)).join(', ')}`);
68
+ // Start the app dev server (unless --url provides an already-running URL)
69
+ const { serverProcess, appUrl } = await startAppServer(absAppPath, {
70
+ port,
71
+ host: options.host,
72
+ url: options.url,
73
+ verbose,
74
+ });
75
+ // Kill any stale daemon and clean up profile locks before the run
76
+ resetDaemon();
77
+ // Track active session logger + session for SIGINT cleanup
78
+ let logger = null;
79
+ let activeSession = null;
80
+ process.on('SIGINT', () => {
81
+ logger?.stop();
82
+ console.log('\n\n👋 Shutting down...');
83
+ activeSession?.disconnect();
84
+ resetDaemon();
85
+ cleanup(serverProcess);
86
+ process.exit(130);
87
+ });
88
+ let failures = 0;
89
+ try {
90
+ for (const evalFilePath of evalFiles) {
91
+ const evalName = basename(evalFilePath, '.md');
92
+ const query = readFileSync(evalFilePath, 'utf-8').trim();
93
+ console.log(`\n${'═'.repeat(60)}`);
94
+ console.log(`📄 Running eval: ${evalName}`);
95
+ console.log(`${'═'.repeat(60)}`);
96
+ const runDir = createRunDir(evalName, options.output);
97
+ const ctx = { url: appUrl, outDir: runDir };
98
+ try {
99
+ const sessionConfig = {
100
+ model: options.model,
101
+ ...(options.reasoning ? { reasoningEffort: options.reasoning } : {}),
102
+ };
103
+ // Single session with all tools (browser + file)
104
+ const session = await createSession({ ctx, config: sessionConfig });
105
+ activeSession = session;
106
+ logger = attachSessionLogger(session, { requestedModel: options.model });
107
+ // ── Phase 1: Plan ──────────────────────────────────────────
108
+ await session.sendAndWait({ prompt: createPlanPrompt(query, ctx) }, 300_000);
109
+ const planPath = `${runDir}/plan.json`;
110
+ if (!existsSync(planPath)) {
111
+ console.error(chalk.red(`❌ Agent did not produce ${planPath}. The planning phase may have failed silently.`));
112
+ session.disconnect();
113
+ logger.stop();
114
+ failures++;
115
+ continue;
116
+ }
117
+ const plan = JSON.parse(readFileSync(planPath, 'utf-8'));
118
+ console.log(plan.summary);
119
+ // ── Phase 2: Execute + Evaluate each step ──────────────────
120
+ for (const step of plan.steps) {
121
+ console.log(chalk.cyan(`\n── Scenario: ${step.scenario}`));
122
+ for (const c of step.criteria) {
123
+ console.log(chalk.dim(` • ${c}`));
124
+ }
125
+ await session.sendAndWait({ prompt: createStepPrompt(step, ctx) }, 300_000);
126
+ }
127
+ // ── Phase 3: Judge ─────────────────────────────────────────
128
+ console.log(chalk.cyan('\n── Judging results...'));
129
+ await session.sendAndWait({ prompt: createJudgePrompt(ctx) }, 300_000);
130
+ session.disconnect();
131
+ logger.stop();
132
+ // Print the verdict
133
+ const verdictPath = `${runDir}/verdict.json`;
134
+ if (existsSync(verdictPath)) {
135
+ const verdict = JSON.parse(readFileSync(verdictPath, 'utf-8'));
136
+ console.log(formatVerdict(verdict));
137
+ if (!verdict.pass)
138
+ failures++;
139
+ }
140
+ else {
141
+ console.error(chalk.red(`❌ Judge did not produce ${verdictPath}.`));
142
+ failures++;
143
+ }
144
+ }
145
+ catch (err) {
146
+ logger?.stop();
147
+ const msg = err instanceof Error ? err.message : String(err);
148
+ console.error(chalk.red(`\n❌ Eval "${evalName}" failed: ${msg}`));
149
+ if (err instanceof Error && err.stack) {
150
+ console.error(chalk.dim(err.stack));
151
+ }
152
+ failures++;
153
+ }
154
+ }
155
+ }
156
+ finally {
157
+ logger?.stop();
158
+ resetDaemon();
159
+ cleanup(serverProcess);
160
+ process.exit(failures > 0 ? 1 : 0);
161
+ }
162
+ });
163
+ /** Intermediate `app` command group: `copilot app` */
164
+ const _appCommand = createCommand('app')
165
+ .description('Copilot-powered application commands')
166
+ .addCommand(_appEvalCommand);
167
+ /**
168
+ * Top-level `copilot` command group.
169
+ *
170
+ * Subcommands:
171
+ * app eval — Evaluate a Fusion app with Copilot SDK + agent-browser tools
172
+ */
173
+ export const copilotCommand = createCommand('copilot')
174
+ .description('GitHub Copilot SDK powered commands')
175
+ .addCommand(_appCommand);
176
+ //# sourceMappingURL=command.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"command.js","sourceRoot":"","sources":["../../../../src/commands/app/command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B;;;;;;GAMG;AACH,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,CAAC;KAC1C,WAAW,CAAC,kEAAkE,CAAC;KAC/E,QAAQ,CAAC,gBAAgB,EAAE,mEAAmE,CAAC;KAC/F,MAAM,CAAC,uBAAuB,EAAE,0CAA0C,CAAC;KAC3E,MAAM,CAAC,eAAe,EAAE,yBAAyB,EAAE,MAAM,CAAC;KAC1D,MAAM,CAAC,eAAe,EAAE,yBAAyB,EAAE,SAAS,CAAC;KAC7D,MAAM,CAAC,aAAa,EAAE,+CAA+C,CAAC;KACtE,MAAM,CAAC,WAAW,EAAE,sBAAsB,EAAE,KAAK,CAAC;KAClD,MAAM,CAAC,kBAAkB,EAAE,kDAAkD,EAAE,KAAK,CAAC;KACrF,MAAM,CAAC,qBAAqB,EAAE,yCAAyC,CAAC;KACxE,MAAM,CAAC,0BAA0B,EAAE,4CAA4C,CAAC;KAChF,MAAM,CAAC,oBAAoB,EAAE,oCAAoC,CAAC;KAClE,WAAW,CACV,OAAO,EACP;IACE,EAAE;IACF,wEAAwE;IACxE,4DAA4D;IAC5D,EAAE;IACF,WAAW;IACX,gDAAgD;IAChD,yCAAyC;IACzC,yDAAyD;IACzD,kFAAkF;IAClF,oDAAoD;IACpD,oCAAoC;IACpC,oCAAoC;IACpC,oEAAoE;CACrE,CAAC,IAAI,CAAC,IAAI,CAAC,CACb;KACA,MAAM,CAAC,KAAK,EAAE,aAAqB,EAAE,OAA2B,EAAE,EAAE;IACnE,MAAM,aAAa,GAAG,uBAAuB,CAAC,aAAa,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC;IACzC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAEhC,IAAI,aAAa,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3C,OAAO,CAAC,KAAK,CAAC,8DAA8D,CAAC,CAAC;QAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,mEAAmE;IACnE,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,MAAM,QAAQ,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/C,OAAO;IACT,CAAC;IAED,qBAAqB;IACrB,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ;QACtC,CAAC,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC;QAC1B,CAAC,CAAC,gBAAgB,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CACT,YAAY,SAAS,CAAC,MAAM,aAAa,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACxF,CAAC;IAEF,0EAA0E;IAC1E,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE;QACjE,IAAI;QACJ,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,OAAO;KACR,CAAC,CAAC;IAEH,kEAAkE;IAClE,WAAW,EAAE,CAAC;IAEd,2DAA2D;IAC3D,IAAI,MAAM,GAAkD,IAAI,CAAC;IACjE,IAAI,aAAa,GAAqD,IAAI,CAAC;IAE3E,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,MAAM,EAAE,IAAI,EAAE,CAAC;QACf,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACvC,aAAa,EAAE,UAAU,EAAE,CAAC;QAC5B,WAAW,EAAE,CAAC;QACd,OAAO,CAAC,aAAa,CAAC,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,CAAC;QACH,KAAK,MAAM,YAAY,IAAI,SAAS,EAAE,CAAC;YACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;YAC/C,MAAM,KAAK,GAAG,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;YACzD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAEjC,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAEtD,MAAM,GAAG,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;YAE5C,IAAI,CAAC;gBACH,MAAM,aAAa,GAAG;oBACpB,KAAK,EAAE,OAAO,CAAC,KAAK;oBACpB,GAAG,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACrE,CAAC;gBAEF,iDAAiD;gBACjD,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,CAAC,CAAC;gBACpE,aAAa,GAAG,OAAO,CAAC;gBACxB,MAAM,GAAG,mBAAmB,CAAC,OAAO,EAAE,EAAE,cAAc,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;gBAEzE,8DAA8D;gBAC9D,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBAE7E,MAAM,QAAQ,GAAG,GAAG,MAAM,YAAY,CAAC;gBACvC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC1B,OAAO,CAAC,KAAK,CACX,KAAK,CAAC,GAAG,CACP,2BAA2B,QAAQ,gDAAgD,CACpF,CACF,CAAC;oBACF,OAAO,CAAC,UAAU,EAAE,CAAC;oBACrB,MAAM,CAAC,IAAI,EAAE,CAAC;oBACd,QAAQ,EAAE,CAAC;oBACX,SAAS;gBACX,CAAC;gBAED,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAS,CAAC;gBACjE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAE1B,8DAA8D;gBAC9D,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;oBAC3D,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;wBAC9B,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;oBACtC,CAAC;oBACD,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,gBAAgB,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBAC9E,CAAC;gBAED,8DAA8D;gBAC9D,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC,CAAC;gBACnD,MAAM,OAAO,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;gBAEvE,OAAO,CAAC,UAAU,EAAE,CAAC;gBACrB,MAAM,CAAC,IAAI,EAAE,CAAC;gBAEd,oBAAoB;gBACpB,MAAM,WAAW,GAAG,GAAG,MAAM,eAAe,CAAC;gBAC7C,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;oBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAY,CAAC;oBAC1E,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;oBACpC,IAAI,CAAC,OAAO,CAAC,IAAI;wBAAE,QAAQ,EAAE,CAAC;gBAChC,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,2BAA2B,WAAW,GAAG,CAAC,CAAC,CAAC;oBACpE,QAAQ,EAAE,CAAC;gBACb,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,EAAE,IAAI,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,QAAQ,aAAa,GAAG,EAAE,CAAC,CAAC,CAAC;gBAClE,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;oBACtC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;gBACtC,CAAC;gBACD,QAAQ,EAAE,CAAC;YACb,CAAC;QACH,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,EAAE,IAAI,EAAE,CAAC;QACf,WAAW,EAAE,CAAC;QACd,OAAO,CAAC,aAAa,CAAC,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,sDAAsD;AACtD,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC;KACrC,WAAW,CAAC,sCAAsC,CAAC;KACnD,UAAU,CAAC,eAAe,CAAC,CAAC;AAE/B;;;;;GAKG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,aAAa,CAAC,SAAS,CAAC;KACnD,WAAW,CAAC,qCAAqC,CAAC;KAClD,UAAU,CAAC,WAAW,CAAC,CAAC"}
@@ -0,0 +1,92 @@
1
+ import { mkdirSync } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { ab, abErrorMessage } from '../../utils/index.js';
4
+ import { createAgentBrowserTools } from './tools/index.js';
5
+ import { createWriteFileTool, createAppendFileTool } from './tools/write-file.js';
6
+ import { createReadFileTool } from './tools/read-file.js';
7
+ import { createSystemPrompt } from './prompts/system.prompt.js';
8
+ import { createDevServerContext } from './prompts/system.dev-server.prompt.js';
9
+ /**
10
+ * Lazily imports the Copilot SDK to avoid module resolution errors at CLI startup.
11
+ *
12
+ * The SDK depends on `vscode-jsonrpc/node` which may fail under certain Node versions
13
+ * when loaded eagerly via static imports.
14
+ *
15
+ * @returns The Copilot SDK module namespace
16
+ */
17
+ async function loadCopilotSdk() {
18
+ return await import('@github/copilot-sdk');
19
+ }
20
+ export const createSession = async (options) => {
21
+ const { ctx, config, defaultToolTimeoutMs } = options;
22
+ const { CopilotClient, defineTool, approveAll } = await loadCopilotSdk();
23
+ const client = new CopilotClient({ autoStart: true });
24
+ await client.start();
25
+ const fileTools = [
26
+ createReadFileTool(ctx.outDir, defineTool),
27
+ createWriteFileTool(ctx.outDir, defineTool),
28
+ createAppendFileTool(ctx.outDir, defineTool),
29
+ ];
30
+ const tools = [
31
+ ...createBrowserTools(defineTool, {
32
+ cwd: ctx.outDir,
33
+ timeoutMs: defaultToolTimeoutMs,
34
+ }),
35
+ ...fileTools,
36
+ ...(config?.tools ?? []),
37
+ ];
38
+ // Exclude dangerous built-in tools; names vary across model providers
39
+ // so a blocklist is more robust than an allowlist.
40
+ const excludedTools = [
41
+ 'bash',
42
+ 'list_bash',
43
+ 'read_bash',
44
+ 'write_bash',
45
+ 'stop_bash',
46
+ 'sql',
47
+ 'web_fetch',
48
+ 'skill',
49
+ 'list_agents',
50
+ 'read_agent',
51
+ 'glob',
52
+ 'rg',
53
+ 'view',
54
+ 'task',
55
+ ];
56
+ const session = await client.createSession({
57
+ ...config,
58
+ tools,
59
+ excludedTools,
60
+ systemMessage: {
61
+ mode: 'replace',
62
+ content: `${createSystemPrompt(ctx)}\n\n## Dev Server${createDevServerContext(ctx)}`,
63
+ },
64
+ streaming: true,
65
+ onPermissionRequest: approveAll,
66
+ });
67
+ return session;
68
+ };
69
+ /**
70
+ * Creates the `agent-browser` tool definitions for the Copilot SDK session.
71
+ *
72
+ * Each tool wraps an `agent-browser` CLI subcommand so the Copilot agent can
73
+ * autonomously navigate and inspect a running Fusion application.
74
+ *
75
+ * @param defineTool - The `defineTool` helper from the Copilot SDK
76
+ * @returns Array of Copilot SDK tool definitions for browser interaction
77
+ */
78
+ function createBrowserTools(defineTool, options) {
79
+ const outDir = join(options?.cwd ?? process.cwd(), 'evidence');
80
+ mkdirSync(outDir, { recursive: true });
81
+ const invoke = (args, timeoutMs = options?.timeoutMs) => {
82
+ try {
83
+ return ab(args, timeoutMs);
84
+ }
85
+ catch (err) {
86
+ const msg = abErrorMessage(err);
87
+ return `Error: ${msg}`;
88
+ }
89
+ };
90
+ return createAgentBrowserTools({ outDir, invoke }, defineTool);
91
+ }
92
+ //# sourceMappingURL=eval.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"eval.js","sourceRoot":"","sources":["../../../../src/commands/app/eval.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,EAAE,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC1D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAClF,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAG1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uCAAuC,CAAC;AAE/E;;;;;;;GAOG;AACH,KAAK,UAAU,cAAc;IAC3B,OAAO,MAAM,MAAM,CAAC,qBAAqB,CAAC,CAAC;AAC7C,CAAC;AAWD,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuB,EAA2B,EAAE;IACtF,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,oBAAoB,EAAE,GAAG,OAAO,CAAC;IAEtD,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,MAAM,cAAc,EAAE,CAAC;IACzE,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IAErB,MAAM,SAAS,GAAG;QAChB,kBAAkB,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;QAC1C,mBAAmB,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;QAC3C,oBAAoB,CAAC,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC;KAC7C,CAAC;IAEF,MAAM,KAAK,GAAG;QACZ,GAAG,kBAAkB,CAAC,UAAU,EAAE;YAChC,GAAG,EAAE,GAAG,CAAC,MAAM;YACf,SAAS,EAAE,oBAAoB;SAChC,CAAC;QACF,GAAG,SAAS;QACZ,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC;KACzB,CAAC;IAEF,sEAAsE;IACtE,mDAAmD;IACnD,MAAM,aAAa,GAAG;QACpB,MAAM;QACN,WAAW;QACX,WAAW;QACX,YAAY;QACZ,WAAW;QACX,KAAK;QACL,WAAW;QACX,OAAO;QACP,aAAa;QACb,YAAY;QACZ,MAAM;QACN,IAAI;QACJ,MAAM;QACN,MAAM;KACP,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;QACzC,GAAG,MAAM;QACT,KAAK;QACL,aAAa;QACb,aAAa,EAAE;YACb,IAAI,EAAE,SAAS;YACf,OAAO,EAAE,GAAG,kBAAkB,CAAC,GAAG,CAAC,oBAAoB,sBAAsB,CAAC,GAAG,CAAC,EAAE;SACrF;QACD,SAAS,EAAE,IAAI;QACf,mBAAmB,EAAE,UAAU;KAChC,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAEF;;;;;;;;GAQG;AACH,SAAS,kBAAkB,CACzB,UAAoE,EACpE,OAA8C;IAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,EAAE,UAAU,CAAC,CAAC;IAC/D,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,CAAC,IAAc,EAAE,SAAS,GAAG,OAAO,EAAE,SAAS,EAAU,EAAE;QACxE,IAAI,CAAC;YACH,OAAO,EAAE,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;YAChC,OAAO,UAAU,GAAG,EAAE,CAAC;QACzB,CAAC;IACH,CAAC,CAAC;IACF,OAAO,uBAAuB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,UAAU,CAAC,CAAC;AACjE,CAAC"}
@@ -0,0 +1,93 @@
1
+ import chalk from 'chalk';
2
+ // ── Plan ────────────────────────────────────────────────────────────────────
3
+ /** Renders a structured plan as a human-readable summary. */
4
+ export function formatPlan(plan) {
5
+ const lines = [
6
+ '',
7
+ chalk.bold(`📋 ${plan.summary}`),
8
+ '',
9
+ ...plan.steps.flatMap((s, si) => [
10
+ ` ${chalk.dim(`${si + 1}.`)} ${s.scenario}`,
11
+ ...s.criteria.map((c) => chalk.dim(` • ${truncate(c, 70)}`)),
12
+ ]),
13
+ '',
14
+ ];
15
+ return lines.join('\n');
16
+ }
17
+ // ── Step result (execution line) ────────────────────────────────────────────
18
+ /** Renders a single step execution result. */
19
+ export function formatStepResult(result) {
20
+ const icon = statusIcon(result.ok);
21
+ const files = result.evidence.length ? chalk.dim(` [${result.evidence.join(', ')}]`) : '';
22
+ return ` ${icon} ${result.criterion}\n${chalk.dim(` ${result.note}`)}${files}`;
23
+ }
24
+ // ── Verdict ─────────────────────────────────────────────────────────────────
25
+ /** Renders the final verdict as a full-width coloured report. */
26
+ export function formatVerdict(verdict) {
27
+ const bar = '═'.repeat(60);
28
+ const banner = verdict.pass
29
+ ? chalk.bgGreen.black.bold(' ✅ PASS ')
30
+ : chalk.bgRed.white.bold(' ❌ FAIL ');
31
+ const lines = [
32
+ '',
33
+ bar,
34
+ ` ${banner}`,
35
+ bar,
36
+ '',
37
+ ...verdict.steps.map((s) => ` ${statusIcon(s.ok)} ${s.criterion}\n${chalk.dim(` ${s.note}`)}`),
38
+ '',
39
+ chalk.dim(` ${verdict.reasoning}`),
40
+ '',
41
+ ];
42
+ if (verdict.ux && verdict.ux.length > 0) {
43
+ lines.push(chalk.bold(' 💡 UX Feedback'), '');
44
+ for (const hint of verdict.ux) {
45
+ lines.push(chalk.yellow(` • ${hint}`));
46
+ }
47
+ lines.push('');
48
+ }
49
+ return lines.join('\n');
50
+ }
51
+ // ── Dispatcher ──────────────────────────────────────────────────────────────
52
+ /**
53
+ * Attempts to detect the JSON shape of an assistant message and format it.
54
+ *
55
+ * Returns `null` when the content is not a recognised structure, so the
56
+ * caller can fall back to plain-text rendering.
57
+ */
58
+ export function tryFormatMessage(content) {
59
+ let parsed;
60
+ try {
61
+ parsed = JSON.parse(content.trim());
62
+ }
63
+ catch {
64
+ return null;
65
+ }
66
+ if (!parsed || typeof parsed !== 'object')
67
+ return null;
68
+ // Verdict: { pass, reasoning, steps[] }
69
+ if ('pass' in parsed && 'steps' in parsed && Array.isArray(parsed.steps)) {
70
+ return formatVerdict(parsed);
71
+ }
72
+ // Plan: { summary, steps[] }
73
+ if ('summary' in parsed && 'steps' in parsed && Array.isArray(parsed.steps)) {
74
+ return formatPlan(parsed);
75
+ }
76
+ // Step result: { criterion, ok, note }
77
+ if ('criterion' in parsed && 'ok' in parsed && 'note' in parsed) {
78
+ return formatStepResult(parsed);
79
+ }
80
+ return null;
81
+ }
82
+ // ── Helpers ─────────────────────────────────────────────────────────────────
83
+ function statusIcon(ok) {
84
+ if (ok === true)
85
+ return chalk.green('✔');
86
+ if (ok === false)
87
+ return chalk.red('✖');
88
+ return chalk.yellow('⚠'); // "blocked", "flaky", etc.
89
+ }
90
+ function truncate(str, max) {
91
+ return str.length > max ? `${str.slice(0, max - 1)}…` : str;
92
+ }
93
+ //# sourceMappingURL=format.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"format.js","sourceRoot":"","sources":["../../../../src/commands/app/format.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAI1B,+EAA+E;AAE/E,6DAA6D;AAC7D,MAAM,UAAU,UAAU,CAAC,IAAU;IACnC,MAAM,KAAK,GAAG;QACZ,EAAE;QACF,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;QAChC,EAAE;QACF,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC;YAC/B,KAAK,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;YAC5C,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;SACjE,CAAC;QACF,EAAE;KACH,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,+EAA+E;AAE/E,8CAA8C;AAC9C,MAAM,UAAU,gBAAgB,CAAC,MAAkB;IACjD,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1F,OAAO,KAAK,IAAI,KAAK,MAAM,CAAC,SAAS,KAAK,KAAK,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC;AACvF,CAAC;AAED,+EAA+E;AAE/E,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,OAAgB;IAC5C,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI;QACzB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;QACtC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAEvC,MAAM,KAAK,GAAG;QACZ,EAAE;QACF,GAAG;QACH,KAAK,MAAM,EAAE;QACb,GAAG;QACH,EAAE;QACF,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAClB,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAC/E;QACD,EAAE;QACF,KAAK,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,SAAS,EAAE,CAAC;QACnC,EAAE;KACH,CAAC;IAEF,IAAI,OAAO,CAAC,EAAE,IAAI,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,+EAA+E;AAE/E;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAEvD,wCAAwC;IACxC,IAAI,MAAM,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,IAAI,KAAK,CAAC,OAAO,CAAE,MAAkB,CAAC,KAAK,CAAC,EAAE,CAAC;QACtF,OAAO,aAAa,CAAC,MAAiB,CAAC,CAAC;IAC1C,CAAC;IAED,6BAA6B;IAC7B,IAAI,SAAS,IAAI,MAAM,IAAI,OAAO,IAAI,MAAM,IAAI,KAAK,CAAC,OAAO,CAAE,MAAe,CAAC,KAAK,CAAC,EAAE,CAAC;QACtF,OAAO,UAAU,CAAC,MAAc,CAAC,CAAC;IACpC,CAAC;IAED,uCAAuC;IACvC,IAAI,WAAW,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC;QAChE,OAAO,gBAAgB,CAAC,MAAoB,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,+EAA+E;AAE/E,SAAS,UAAU,CAAC,EAAoB;IACtC,IAAI,EAAE,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,2BAA2B;AACvD,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,GAAW;IACxC,OAAO,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9D,CAAC"}