ace-test-runner-e2e 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.ace-defaults/e2e-runner/config.yml +70 -0
- data/.ace-defaults/nav/protocols/guide-sources/ace-test-runner-e2e.yml +11 -0
- data/.ace-defaults/nav/protocols/skill-sources/ace-test-runner-e2e.yml +19 -0
- data/.ace-defaults/nav/protocols/tmpl-sources/ace-test-runner-e2e.yml +12 -0
- data/.ace-defaults/nav/protocols/wfi-sources/ace-test-runner-e2e.yml +11 -0
- data/CHANGELOG.md +1166 -0
- data/LICENSE +21 -0
- data/README.md +42 -0
- data/Rakefile +15 -0
- data/exe/ace-test-e2e +15 -0
- data/exe/ace-test-e2e-sh +67 -0
- data/exe/ace-test-e2e-suite +13 -0
- data/handbook/guides/e2e-testing.g.md +124 -0
- data/handbook/guides/scenario-yml-reference.g.md +182 -0
- data/handbook/guides/tc-authoring.g.md +131 -0
- data/handbook/skills/as-e2e-create/SKILL.md +30 -0
- data/handbook/skills/as-e2e-fix/SKILL.md +35 -0
- data/handbook/skills/as-e2e-manage/SKILL.md +31 -0
- data/handbook/skills/as-e2e-plan-changes/SKILL.md +30 -0
- data/handbook/skills/as-e2e-review/SKILL.md +35 -0
- data/handbook/skills/as-e2e-rewrite/SKILL.md +31 -0
- data/handbook/skills/as-e2e-run/SKILL.md +48 -0
- data/handbook/skills/as-e2e-setup-sandbox/SKILL.md +34 -0
- data/handbook/templates/ace-taskflow-fixture.template.md +322 -0
- data/handbook/templates/agent-experience-report.template.md +89 -0
- data/handbook/templates/metadata.template.yml +49 -0
- data/handbook/templates/scenario.yml.template.yml +60 -0
- data/handbook/templates/tc-file.template.md +45 -0
- data/handbook/templates/test-report.template.md +94 -0
- data/handbook/workflow-instructions/e2e/analyze-failures.wf.md +126 -0
- data/handbook/workflow-instructions/e2e/create.wf.md +395 -0
- data/handbook/workflow-instructions/e2e/execute.wf.md +253 -0
- data/handbook/workflow-instructions/e2e/fix.wf.md +166 -0
- data/handbook/workflow-instructions/e2e/manage.wf.md +179 -0
- data/handbook/workflow-instructions/e2e/plan-changes.wf.md +255 -0
- data/handbook/workflow-instructions/e2e/review.wf.md +286 -0
- data/handbook/workflow-instructions/e2e/rewrite.wf.md +281 -0
- data/handbook/workflow-instructions/e2e/run.wf.md +355 -0
- data/handbook/workflow-instructions/e2e/setup-sandbox.wf.md +461 -0
- data/lib/ace/test/end_to_end_runner/atoms/display_helpers.rb +234 -0
- data/lib/ace/test/end_to_end_runner/atoms/prompt_builder.rb +199 -0
- data/lib/ace/test/end_to_end_runner/atoms/result_parser.rb +166 -0
- data/lib/ace/test/end_to_end_runner/atoms/skill_prompt_builder.rb +166 -0
- data/lib/ace/test/end_to_end_runner/atoms/skill_result_parser.rb +244 -0
- data/lib/ace/test/end_to_end_runner/atoms/suite_report_prompt_builder.rb +103 -0
- data/lib/ace/test/end_to_end_runner/atoms/tc_fidelity_validator.rb +39 -0
- data/lib/ace/test/end_to_end_runner/atoms/test_case_parser.rb +108 -0
- data/lib/ace/test/end_to_end_runner/cli/commands/run_suite.rb +130 -0
- data/lib/ace/test/end_to_end_runner/cli/commands/run_test.rb +156 -0
- data/lib/ace/test/end_to_end_runner/models/test_case.rb +47 -0
- data/lib/ace/test/end_to_end_runner/models/test_result.rb +115 -0
- data/lib/ace/test/end_to_end_runner/models/test_scenario.rb +90 -0
- data/lib/ace/test/end_to_end_runner/molecules/affected_detector.rb +92 -0
- data/lib/ace/test/end_to_end_runner/molecules/config_loader.rb +75 -0
- data/lib/ace/test/end_to_end_runner/molecules/failure_finder.rb +203 -0
- data/lib/ace/test/end_to_end_runner/molecules/fixture_copier.rb +35 -0
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb +121 -0
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb +182 -0
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb +321 -0
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb +131 -0
- data/lib/ace/test/end_to_end_runner/molecules/progress_display_manager.rb +172 -0
- data/lib/ace/test/end_to_end_runner/molecules/report_writer.rb +259 -0
- data/lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb +254 -0
- data/lib/ace/test/end_to_end_runner/molecules/setup_executor.rb +181 -0
- data/lib/ace/test/end_to_end_runner/molecules/simple_display_manager.rb +72 -0
- data/lib/ace/test/end_to_end_runner/molecules/suite_progress_display_manager.rb +223 -0
- data/lib/ace/test/end_to_end_runner/molecules/suite_report_writer.rb +277 -0
- data/lib/ace/test/end_to_end_runner/molecules/suite_simple_display_manager.rb +116 -0
- data/lib/ace/test/end_to_end_runner/molecules/test_discoverer.rb +136 -0
- data/lib/ace/test/end_to_end_runner/molecules/test_executor.rb +332 -0
- data/lib/ace/test/end_to_end_runner/organisms/suite_orchestrator.rb +830 -0
- data/lib/ace/test/end_to_end_runner/organisms/test_orchestrator.rb +442 -0
- data/lib/ace/test/end_to_end_runner/version.rb +9 -0
- data/lib/ace/test/end_to_end_runner.rb +71 -0
- metadata +220 -0
|
@@ -0,0 +1,461 @@
|
|
|
1
|
+
---
|
|
2
|
+
doc-type: workflow
|
|
3
|
+
title: E2E Sandbox Setup Workflow
|
|
4
|
+
purpose: E2E test environment setup workflow
|
|
5
|
+
ace-docs:
|
|
6
|
+
last-updated: 2026-03-12
|
|
7
|
+
last-checked: 2026-03-21
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# E2E Sandbox Setup Workflow
|
|
11
|
+
|
|
12
|
+
## Purpose
|
|
13
|
+
|
|
14
|
+
Set up a safe, isolated environment for E2E tests that:
|
|
15
|
+
- Isolates from main project (no pollution)
|
|
16
|
+
- Handles external APIs safely (test tokens, limited scope)
|
|
17
|
+
- Ensures cleanup on success AND failure
|
|
18
|
+
- Captures outputs for debugging
|
|
19
|
+
|
|
20
|
+
Setup is owned by scenario configuration and fixtures. Runner TC files must not re-implement environment bootstrapping.
|
|
21
|
+
|
|
22
|
+
## When to Use
|
|
23
|
+
|
|
24
|
+
- Before running E2E tests that touch filesystem
|
|
25
|
+
- When E2E test needs git repository
|
|
26
|
+
- When E2E test calls external APIs
|
|
27
|
+
- When test creates resources that need cleanup
|
|
28
|
+
|
|
29
|
+
**Note:** For CLI pipeline runs (`ace-test-e2e`), `SetupExecutor` handles sandbox creation automatically using the 6-phase deterministic pipeline. This workflow is the reference for what `SetupExecutor` implements and for manual runs guided by `ace-bundle wfi://e2e/run`.
|
|
30
|
+
|
|
31
|
+
## Prerequisites
|
|
32
|
+
|
|
33
|
+
- `ace-b36ts` available for unique IDs
|
|
34
|
+
- Required tools installed for the specific test
|
|
35
|
+
- Test tokens available (if external APIs needed)
|
|
36
|
+
|
|
37
|
+
## Workflow Steps
|
|
38
|
+
|
|
39
|
+
### Step 1: Create Isolated Directory
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
# Generate unique timestamp ID
|
|
43
|
+
TIMESTAMP_ID="$(ace-b36ts encode)"
|
|
44
|
+
|
|
45
|
+
# Extract short names from test metadata
|
|
46
|
+
# Note: params are lowercase (package, test_id) - use them directly
|
|
47
|
+
SHORT_PKG="${package#ace-}" # Remove ace- prefix
|
|
48
|
+
SHORT_ID="$(echo "$test_id" | tr '[:upper:]' '[:lower:]' | tr '-' '')" # ts001
|
|
49
|
+
|
|
50
|
+
# Create sandbox and reports directories
|
|
51
|
+
TEST_DIR=".ace-local/test-e2e/${TIMESTAMP_ID}-${SHORT_PKG}-${SHORT_ID}"
|
|
52
|
+
REPORTS_DIR="${TEST_DIR}-reports"
|
|
53
|
+
|
|
54
|
+
mkdir -p "$TEST_DIR"
|
|
55
|
+
mkdir -p "$REPORTS_DIR"
|
|
56
|
+
|
|
57
|
+
# Record for cleanup
|
|
58
|
+
echo "$TEST_DIR" > "$REPORTS_DIR/sandbox-path.txt"
|
|
59
|
+
echo "$(date -Iseconds)" > "$REPORTS_DIR/start-time.txt"
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
### Step 2: Isolate Environment
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
cd "$TEST_DIR"
|
|
66
|
+
|
|
67
|
+
# Critical: Isolate from main project
|
|
68
|
+
export PROJECT_ROOT_PATH="$TEST_DIR"
|
|
69
|
+
export ACE_TEST_MODE=true
|
|
70
|
+
export ACE_CONFIG_PATH="$TEST_DIR/.ace"
|
|
71
|
+
|
|
72
|
+
# Record environment for debugging
|
|
73
|
+
env | grep -E "^(PROJECT|ACE|PATH)" > "$REPORTS_DIR/environment.txt"
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
### Step 3: Initialize Git (if needed)
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
if [ "$NEEDS_GIT" = "true" ]; then
|
|
80
|
+
git init --quiet .
|
|
81
|
+
git config user.email "test@example.com"
|
|
82
|
+
git config user.name "E2E Test"
|
|
83
|
+
|
|
84
|
+
# Verify isolation
|
|
85
|
+
if ! PROJECT_ROOT=$(git rev-parse --show-toplevel 2>/dev/null); then
|
|
86
|
+
echo "ERROR: Failed to get git root (git not initialized properly)"
|
|
87
|
+
exit 1
|
|
88
|
+
fi
|
|
89
|
+
if [ "$PROJECT_ROOT" != "$TEST_DIR" ]; then
|
|
90
|
+
echo "ERROR: Git not isolated, found: $PROJECT_ROOT"
|
|
91
|
+
exit 1
|
|
92
|
+
fi
|
|
93
|
+
fi
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Step 3.1: Sandbox Isolation Checkpoint (MANDATORY)
|
|
97
|
+
|
|
98
|
+
> **STOP - Verify Before Continuing**
|
|
99
|
+
>
|
|
100
|
+
> Before proceeding to test data creation or test execution, you MUST verify sandbox isolation.
|
|
101
|
+
> Failure to verify will result in polluting the main repository with test artifacts.
|
|
102
|
+
|
|
103
|
+
**Run these verification commands:**
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
echo "=== SANDBOX ISOLATION CHECK ==="
|
|
107
|
+
|
|
108
|
+
# Check 1: Current directory must be under .ace-local/test-e2e/
|
|
109
|
+
CURRENT_DIR="$(pwd)"
|
|
110
|
+
if [[ "$CURRENT_DIR" == *".ace-local/test-e2e/"* ]]; then
|
|
111
|
+
echo "PASS: Working directory is inside sandbox"
|
|
112
|
+
echo " Location: $CURRENT_DIR"
|
|
113
|
+
else
|
|
114
|
+
echo "FAIL: NOT in sandbox!"
|
|
115
|
+
echo " Current: $CURRENT_DIR"
|
|
116
|
+
echo " Expected: Should contain '.ace-local/test-e2e/'"
|
|
117
|
+
echo " ACTION: STOP - Do not proceed. Re-run Environment Setup."
|
|
118
|
+
fi
|
|
119
|
+
|
|
120
|
+
# Check 2: Git remote must be empty (fresh isolated repo)
|
|
121
|
+
if git rev-parse --git-dir >/dev/null 2>&1; then
|
|
122
|
+
REMOTES=$(git remote -v 2>/dev/null)
|
|
123
|
+
if [ -z "$REMOTES" ]; then
|
|
124
|
+
echo "PASS: No git remotes (isolated repo)"
|
|
125
|
+
else
|
|
126
|
+
echo "FAIL: Git remotes found - NOT an isolated repo!"
|
|
127
|
+
echo " Remotes: $REMOTES"
|
|
128
|
+
echo " ACTION: STOP - You are in the main repository."
|
|
129
|
+
fi
|
|
130
|
+
else
|
|
131
|
+
echo "PASS: No git repo in sandbox (tools use PROJECT_ROOT_PATH)"
|
|
132
|
+
fi
|
|
133
|
+
|
|
134
|
+
# Check 3: Project root markers should NOT exist
|
|
135
|
+
if [ -f "CLAUDE.md" ] || [ -f "Gemfile" ] || [ -d ".ace-taskflow" ]; then
|
|
136
|
+
echo "FAIL: Main project markers found - NOT an isolated repo!"
|
|
137
|
+
echo " ACTION: STOP - You are in the main repository."
|
|
138
|
+
else
|
|
139
|
+
echo "PASS: No main project markers (expected for sandbox)"
|
|
140
|
+
fi
|
|
141
|
+
|
|
142
|
+
echo "=== END ISOLATION CHECK ==="
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
**Interpretation:**
|
|
146
|
+
- **All checks PASS**: Continue to Step 4 (Handle External APIs) or Step 5 (Create Test Data)
|
|
147
|
+
- **Any check FAILS**:
|
|
148
|
+
1. STOP immediately - do NOT execute any test commands
|
|
149
|
+
2. Return to project root: `cd "$PROJECT_ROOT"`
|
|
150
|
+
3. Re-read and re-execute Environment Setup
|
|
151
|
+
4. Re-run this checkpoint until all checks pass
|
|
152
|
+
|
|
153
|
+
### Step 4: Handle External APIs Safely
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
# Check for test tokens (never use production)
|
|
157
|
+
if [ -n "$EXTERNAL_API_REQUIRED" ]; then
|
|
158
|
+
# GitHub example
|
|
159
|
+
if [ -z "$TEST_GITHUB_TOKEN" ]; then
|
|
160
|
+
echo "SKIP: No test GitHub token available"
|
|
161
|
+
echo "Set TEST_GITHUB_TOKEN for full E2E coverage"
|
|
162
|
+
exit 0
|
|
163
|
+
fi
|
|
164
|
+
|
|
165
|
+
# Use test token, not production
|
|
166
|
+
export GITHUB_TOKEN="$TEST_GITHUB_TOKEN"
|
|
167
|
+
|
|
168
|
+
# Verify token scope is limited
|
|
169
|
+
SCOPES=$(gh auth status 2>&1 | grep "Token scopes" || echo "unknown")
|
|
170
|
+
echo "Token scopes: $SCOPES" >> "$REPORTS_DIR/api-config.txt"
|
|
171
|
+
fi
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
### Step 5: Create Test Data
|
|
175
|
+
|
|
176
|
+
```bash
|
|
177
|
+
# Create test files from heredocs (reproducible)
|
|
178
|
+
cat > "$TEST_DIR/test-file.rb" << 'EOF'
|
|
179
|
+
# Test file content
|
|
180
|
+
class Example
|
|
181
|
+
def hello
|
|
182
|
+
"Hello, World!"
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
EOF
|
|
186
|
+
|
|
187
|
+
# Create test configuration
|
|
188
|
+
mkdir -p "$TEST_DIR/.ace/tool"
|
|
189
|
+
cat > "$TEST_DIR/.ace/tool/config.yml" << 'EOF'
|
|
190
|
+
setting: test-value
|
|
191
|
+
EOF
|
|
192
|
+
|
|
193
|
+
# Record test data for debugging
|
|
194
|
+
find "$TEST_DIR" -type f > "$REPORTS_DIR/test-files.txt"
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
### Step 6: Execute with Safety
|
|
198
|
+
|
|
199
|
+
```bash
|
|
200
|
+
# Timeout protection
|
|
201
|
+
TIMEOUT_SECONDS="${TIMEOUT:-300}"
|
|
202
|
+
|
|
203
|
+
# Capture all output
|
|
204
|
+
timeout "$TIMEOUT_SECONDS" $COMMAND \
|
|
205
|
+
> "$REPORTS_DIR/stdout.txt" \
|
|
206
|
+
2> "$REPORTS_DIR/stderr.txt"
|
|
207
|
+
EXIT_CODE=$?
|
|
208
|
+
|
|
209
|
+
# Record exit code
|
|
210
|
+
echo "$EXIT_CODE" > "$REPORTS_DIR/exit-code.txt"
|
|
211
|
+
|
|
212
|
+
# Check for timeout
|
|
213
|
+
if [ "$EXIT_CODE" -eq 124 ]; then
|
|
214
|
+
echo "FAIL: Command timed out after ${TIMEOUT_SECONDS}s"
|
|
215
|
+
fi
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
### Step 7: Verify Results
|
|
219
|
+
|
|
220
|
+
```bash
|
|
221
|
+
# Explicit PASS/FAIL assertions
|
|
222
|
+
if [ "$EXIT_CODE" -eq 0 ]; then
|
|
223
|
+
echo "PASS: Exit code 0" >> "$REPORTS_DIR/results.txt"
|
|
224
|
+
else
|
|
225
|
+
echo "FAIL: Exit code $EXIT_CODE (expected 0)" >> "$REPORTS_DIR/results.txt"
|
|
226
|
+
fi
|
|
227
|
+
|
|
228
|
+
# Check expected output
|
|
229
|
+
if grep -q "expected pattern" "$REPORTS_DIR/stdout.txt"; then
|
|
230
|
+
echo "PASS: Output contains expected pattern" >> "$REPORTS_DIR/results.txt"
|
|
231
|
+
else
|
|
232
|
+
echo "FAIL: Output missing expected pattern" >> "$REPORTS_DIR/results.txt"
|
|
233
|
+
fi
|
|
234
|
+
|
|
235
|
+
# Check files created
|
|
236
|
+
if [ -f "$TEST_DIR/expected-output.txt" ]; then
|
|
237
|
+
echo "PASS: Output file created" >> "$REPORTS_DIR/results.txt"
|
|
238
|
+
else
|
|
239
|
+
echo "FAIL: Output file not created" >> "$REPORTS_DIR/results.txt"
|
|
240
|
+
fi
|
|
241
|
+
```
|
|
242
|
+
|
|
243
|
+
### Step 8: Cleanup
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
# Record end time
|
|
247
|
+
echo "$(date -Iseconds)" > "$REPORTS_DIR/end-time.txt"
|
|
248
|
+
|
|
249
|
+
# Restore environment
|
|
250
|
+
unset PROJECT_ROOT_PATH
|
|
251
|
+
unset ACE_TEST_MODE
|
|
252
|
+
unset ACE_CONFIG_PATH
|
|
253
|
+
unset GITHUB_TOKEN
|
|
254
|
+
|
|
255
|
+
# Cleanup API resources (if created)
|
|
256
|
+
if [ -f "$REPORTS_DIR/resources-created.txt" ]; then
|
|
257
|
+
while read -r resource; do
|
|
258
|
+
# Delete resource (implementation depends on API)
|
|
259
|
+
echo "Cleaning up: $resource"
|
|
260
|
+
done < "$REPORTS_DIR/resources-created.txt"
|
|
261
|
+
fi
|
|
262
|
+
|
|
263
|
+
# Determine sandbox disposition
|
|
264
|
+
if grep -q "FAIL" "$REPORTS_DIR/results.txt"; then
|
|
265
|
+
echo "Sandbox preserved for debugging: $TEST_DIR"
|
|
266
|
+
echo "Reports: $REPORTS_DIR"
|
|
267
|
+
else
|
|
268
|
+
# Optional: Clean up on success
|
|
269
|
+
if [ "$CLEANUP_ON_SUCCESS" = "true" ]; then
|
|
270
|
+
rm -rf "$TEST_DIR"
|
|
271
|
+
echo "Sandbox cleaned up"
|
|
272
|
+
else
|
|
273
|
+
echo "Sandbox preserved: $TEST_DIR"
|
|
274
|
+
fi
|
|
275
|
+
fi
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
## Output Files
|
|
279
|
+
|
|
280
|
+
After running, reports directory contains:
|
|
281
|
+
|
|
282
|
+
```
|
|
283
|
+
{timestamp}-{pkg}-{id}-reports/
|
|
284
|
+
├── sandbox-path.txt # Path to test directory
|
|
285
|
+
├── start-time.txt # When test started
|
|
286
|
+
├── end-time.txt # When test ended
|
|
287
|
+
├── environment.txt # Environment variables
|
|
288
|
+
├── test-files.txt # Files in sandbox
|
|
289
|
+
├── api-config.txt # API configuration (if used)
|
|
290
|
+
├── stdout.txt # Command stdout
|
|
291
|
+
├── stderr.txt # Command stderr
|
|
292
|
+
├── exit-code.txt # Command exit code
|
|
293
|
+
├── results.txt # PASS/FAIL assertions
|
|
294
|
+
└── resources-created.txt # API resources to clean up
|
|
295
|
+
```
|
|
296
|
+
|
|
297
|
+
## Safety Checklist
|
|
298
|
+
|
|
299
|
+
Before running:
|
|
300
|
+
|
|
301
|
+
- [ ] Using test tokens, not production
|
|
302
|
+
- [ ] API scopes are minimal
|
|
303
|
+
- [ ] Timeout is set
|
|
304
|
+
- [ ] PROJECT_ROOT_PATH isolates sandbox
|
|
305
|
+
- [ ] Cleanup is defined for failure case
|
|
306
|
+
|
|
307
|
+
After running:
|
|
308
|
+
|
|
309
|
+
- [ ] Results recorded
|
|
310
|
+
- [ ] Resources cleaned up
|
|
311
|
+
- [ ] Environment restored
|
|
312
|
+
- [ ] Sandbox preserved (failure) or cleaned (success)
|
|
313
|
+
|
|
314
|
+
## Integration
|
|
315
|
+
|
|
316
|
+
### With E2E Test Files
|
|
317
|
+
|
|
318
|
+
Add setup directives to `scenario.yml`:
|
|
319
|
+
|
|
320
|
+
```yaml
|
|
321
|
+
# scenario.yml
|
|
322
|
+
setup:
|
|
323
|
+
- git-init
|
|
324
|
+
- run: "cp $PROJECT_ROOT_PATH/mise.toml mise.toml && mise trust mise.toml"
|
|
325
|
+
- copy-fixtures
|
|
326
|
+
- agent-env:
|
|
327
|
+
PROJECT_ROOT_PATH: "."
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
### With `ace-bundle wfi://e2e/run`
|
|
331
|
+
|
|
332
|
+
This workflow is called automatically when running E2E tests.
|
|
333
|
+
|
|
334
|
+
## Standard Setup Script
|
|
335
|
+
|
|
336
|
+
This is the **authoritative copy-executable script** for sandbox setup. Other workflows (like `run-e2e-test.wf.md`) should reference this section rather than duplicating the logic.
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
#!/bin/bash
|
|
340
|
+
# E2E Sandbox Setup - Standard Script
|
|
341
|
+
# Usage: Source or copy-execute in E2E test workflows
|
|
342
|
+
|
|
343
|
+
# Capture project root before any cd operations
|
|
344
|
+
PROJECT_ROOT="$(pwd)"
|
|
345
|
+
|
|
346
|
+
# Generate unique timestamp ID
|
|
347
|
+
TIMESTAMP_ID="$(ace-b36ts encode)"
|
|
348
|
+
|
|
349
|
+
# Derive short names (adjust PACKAGE and TEST_ID as needed)
|
|
350
|
+
SHORT_PKG="${PACKAGE#ace-}"
|
|
351
|
+
SHORT_ID=$(echo "$TEST_ID" | sed 's/TS-[A-Z]*-/ts/' | tr '[:upper:]' '[:lower:]')
|
|
352
|
+
|
|
353
|
+
# Create sandbox and reports directories
|
|
354
|
+
TEST_DIR="$PROJECT_ROOT/.ace-local/test-e2e/${TIMESTAMP_ID}-${SHORT_PKG}-${SHORT_ID}"
|
|
355
|
+
REPORTS_DIR="${TEST_DIR}-reports"
|
|
356
|
+
|
|
357
|
+
mkdir -p "$TEST_DIR"
|
|
358
|
+
mkdir -p "$REPORTS_DIR"
|
|
359
|
+
|
|
360
|
+
# Record for cleanup
|
|
361
|
+
echo "$TEST_DIR" > "$REPORTS_DIR/sandbox-path.txt"
|
|
362
|
+
echo "$(date -Iseconds)" > "$REPORTS_DIR/start-time.txt"
|
|
363
|
+
|
|
364
|
+
# Navigate to sandbox
|
|
365
|
+
cd "$TEST_DIR"
|
|
366
|
+
|
|
367
|
+
# Set isolation environment
|
|
368
|
+
export PROJECT_ROOT_PATH="$TEST_DIR"
|
|
369
|
+
export ACE_TEST_MODE=true
|
|
370
|
+
export ACE_CONFIG_PATH="$TEST_DIR/.ace"
|
|
371
|
+
|
|
372
|
+
# Record environment for debugging
|
|
373
|
+
env | grep -E "^(PROJECT|ACE|PATH)" > "$REPORTS_DIR/environment.txt"
|
|
374
|
+
|
|
375
|
+
# Initialize git if needed
|
|
376
|
+
if [ "$NEEDS_GIT" = "true" ]; then
|
|
377
|
+
git init --quiet .
|
|
378
|
+
git config user.email "test@example.com"
|
|
379
|
+
git config user.name "E2E Test"
|
|
380
|
+
fi
|
|
381
|
+
|
|
382
|
+
# === SANDBOX ISOLATION CHECKPOINT ===
|
|
383
|
+
echo "=== SANDBOX ISOLATION CHECK ==="
|
|
384
|
+
|
|
385
|
+
# Check 1: Path
|
|
386
|
+
CURRENT_DIR="$(pwd)"
|
|
387
|
+
if [[ "$CURRENT_DIR" == *".ace-local/test-e2e/"* ]]; then
|
|
388
|
+
echo "PASS: Working directory is inside sandbox"
|
|
389
|
+
else
|
|
390
|
+
echo "FAIL: NOT in sandbox! Current: $CURRENT_DIR"
|
|
391
|
+
exit 1
|
|
392
|
+
fi
|
|
393
|
+
|
|
394
|
+
# Check 2: Git remotes
|
|
395
|
+
if git rev-parse --git-dir >/dev/null 2>&1; then
|
|
396
|
+
REMOTES=$(git remote -v 2>/dev/null)
|
|
397
|
+
if [ -z "$REMOTES" ]; then
|
|
398
|
+
echo "PASS: No git remotes (isolated repo)"
|
|
399
|
+
else
|
|
400
|
+
echo "FAIL: Git remotes found - NOT isolated!"
|
|
401
|
+
exit 1
|
|
402
|
+
fi
|
|
403
|
+
else
|
|
404
|
+
echo "PASS: No git repo in sandbox (tools use PROJECT_ROOT_PATH)"
|
|
405
|
+
fi
|
|
406
|
+
|
|
407
|
+
# Check 3: Project markers
|
|
408
|
+
if [ -f "CLAUDE.md" ] || [ -f "Gemfile" ] || [ -d ".ace-taskflow" ]; then
|
|
409
|
+
echo "FAIL: Main project markers found!"
|
|
410
|
+
exit 1
|
|
411
|
+
else
|
|
412
|
+
echo "PASS: No main project markers"
|
|
413
|
+
fi
|
|
414
|
+
|
|
415
|
+
echo "=== ISOLATION VERIFIED - SAFE TO PROCEED ==="
|
|
416
|
+
|
|
417
|
+
# Export variables for use by calling script
|
|
418
|
+
export PROJECT_ROOT TEST_DIR REPORTS_DIR TIMESTAMP_ID
|
|
419
|
+
```
|
|
420
|
+
|
|
421
|
+
**Expected Variables After Setup:**
|
|
422
|
+
- `PROJECT_ROOT` - Original project directory (for accessing binaries like `$PROJECT_ROOT/bin/ace-lint`)
|
|
423
|
+
- `TEST_DIR` - Sandbox directory (current working directory after setup)
|
|
424
|
+
- `REPORTS_DIR` - Reports directory for test outputs
|
|
425
|
+
- `TIMESTAMP_ID` - Unique identifier for this test run
|
|
426
|
+
|
|
427
|
+
## Using `ace-test-e2e-sh` After Setup
|
|
428
|
+
|
|
429
|
+
After Environment Setup completes, all subsequent bash blocks (Test Data, Test Cases) MUST use the `ace-test-e2e-sh` wrapper to ensure sandbox isolation persists across separate shell invocations.
|
|
430
|
+
|
|
431
|
+
**Why:** Each `bash` block in a test scenario runs in a fresh shell. The `cd "$TEST_DIR"` from Environment Setup does not carry over. The wrapper enforces the correct working directory and `PROJECT_ROOT_PATH` for every command.
|
|
432
|
+
|
|
433
|
+
**Single command:**
|
|
434
|
+
```bash
|
|
435
|
+
ace-test-e2e-sh "$TEST_DIR" git add README.md
|
|
436
|
+
```
|
|
437
|
+
|
|
438
|
+
**Multi-command block (heredoc):**
|
|
439
|
+
```bash
|
|
440
|
+
ace-test-e2e-sh "$TEST_DIR" bash << 'SANDBOX'
|
|
441
|
+
cat > README.md << 'EOF'
|
|
442
|
+
# Test Repository
|
|
443
|
+
EOF
|
|
444
|
+
git add README.md
|
|
445
|
+
git commit -m "Initial commit"
|
|
446
|
+
SANDBOX
|
|
447
|
+
```
|
|
448
|
+
|
|
449
|
+
**Worktree tests:** Use `$REPO_DIR` (a subdirectory of `$TEST_DIR`) instead:
|
|
450
|
+
```bash
|
|
451
|
+
ace-test-e2e-sh "$REPO_DIR" git status
|
|
452
|
+
```
|
|
453
|
+
|
|
454
|
+
**Skip wrapping for:**
|
|
455
|
+
- The Environment Setup block itself (it creates and enters the sandbox)
|
|
456
|
+
- Report-writing blocks (Section 7) that use absolute `$REPORT_DIR` paths
|
|
457
|
+
|
|
458
|
+
## See Also
|
|
459
|
+
|
|
460
|
+
- [E2E Testing Guide](guide://e2e-testing)
|
|
461
|
+
- [Test Suite Health](guide://test-suite-health)
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ace
|
|
4
|
+
module Test
|
|
5
|
+
module EndToEndRunner
|
|
6
|
+
module Atoms
|
|
7
|
+
# Pure formatting functions for E2E test display output.
|
|
8
|
+
# No I/O, no side effects — all methods return strings.
|
|
9
|
+
module DisplayHelpers
|
|
10
|
+
ANSI_COLORS = {
|
|
11
|
+
green: "\033[32m",
|
|
12
|
+
red: "\033[31m",
|
|
13
|
+
yellow: "\033[33m",
|
|
14
|
+
cyan: "\033[36m",
|
|
15
|
+
gray: "\033[90m",
|
|
16
|
+
reset: "\033[0m"
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
19
|
+
SEPARATOR = "=" * 65
|
|
20
|
+
DOUBLE_SEPARATOR = "\u2550" * 65
|
|
21
|
+
|
|
22
|
+
# Unicode and ASCII fallback icons for progress display
|
|
23
|
+
UNICODE_ICONS = {
|
|
24
|
+
waiting: "\u00b7", # · (middle dot)
|
|
25
|
+
running: "\u22ef", # ⋯ (midline horizontal ellipsis)
|
|
26
|
+
check: "\u2713", # ✓ (check mark)
|
|
27
|
+
cross: "\u2717" # ✗ (ballot x)
|
|
28
|
+
}.freeze
|
|
29
|
+
|
|
30
|
+
ASCII_ICONS = {
|
|
31
|
+
waiting: ".",
|
|
32
|
+
running: "...",
|
|
33
|
+
check: "OK",
|
|
34
|
+
cross: "XX"
|
|
35
|
+
}.freeze
|
|
36
|
+
|
|
37
|
+
module_function
|
|
38
|
+
|
|
39
|
+
# Detect if the current terminal supports Unicode
|
|
40
|
+
# Checks LANG/LC_ALL environment variables for UTF-8 encoding
|
|
41
|
+
# @return [Boolean] true if terminal likely supports Unicode
|
|
42
|
+
def unicode_terminal?
|
|
43
|
+
lang = ENV["LANG"] || ENV["LC_ALL"] || ""
|
|
44
|
+
lang.include?("UTF-8") || lang.include?("utf-8")
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# @return [String] 65-char double-line separator (═ or =)
|
|
48
|
+
def double_separator
|
|
49
|
+
unicode_terminal? ? DOUBLE_SEPARATOR : SEPARATOR
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @param success [Boolean]
|
|
53
|
+
# @return [String] "✓" or "✗" (or ASCII fallback)
|
|
54
|
+
def status_icon(success)
|
|
55
|
+
icons = unicode_terminal? ? UNICODE_ICONS : ASCII_ICONS
|
|
56
|
+
success ? icons[:check] : icons[:cross]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Get the waiting icon for progress display
|
|
60
|
+
# @return [String] "·" or "."
|
|
61
|
+
def waiting_icon
|
|
62
|
+
unicode_terminal? ? UNICODE_ICONS[:waiting] : ASCII_ICONS[:waiting]
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Get the running icon for progress display
|
|
66
|
+
# @return [String] "⋯" or "..."
|
|
67
|
+
def running_icon
|
|
68
|
+
unicode_terminal? ? UNICODE_ICONS[:running] : ASCII_ICONS[:running]
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Right-aligned elapsed seconds for columnar output
|
|
72
|
+
# @param seconds [Numeric]
|
|
73
|
+
# @return [String] e.g. " 10.7s"
|
|
74
|
+
def format_elapsed(seconds)
|
|
75
|
+
sprintf("%5.1fs", seconds)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Human-readable duration for summary lines
|
|
79
|
+
# @param seconds [Numeric]
|
|
80
|
+
# @return [String] e.g. "1m 50s" or "10.70s"
|
|
81
|
+
def format_duration(seconds)
|
|
82
|
+
if seconds >= 60
|
|
83
|
+
"#{(seconds / 60).floor}m #{(seconds % 60).round(0)}s"
|
|
84
|
+
else
|
|
85
|
+
sprintf("%.2fs", seconds)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Test case count display suffix
|
|
90
|
+
# @param result [Models::TestResult]
|
|
91
|
+
# @return [String] e.g. " 0/8 cases" or ""
|
|
92
|
+
def tc_count_display(result)
|
|
93
|
+
return "" if result.total_count == 0
|
|
94
|
+
|
|
95
|
+
" #{result.passed_count}/#{result.total_count} cases"
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# @return [String] 65-char separator line
|
|
99
|
+
def separator
|
|
100
|
+
SEPARATOR
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Wrap text in ANSI color codes (or return plain text)
|
|
104
|
+
# @param text [String]
|
|
105
|
+
# @param color_name [Symbol] one of :green, :red, :yellow, :cyan, :gray
|
|
106
|
+
# @param use_color [Boolean]
|
|
107
|
+
# @return [String]
|
|
108
|
+
def color(text, color_name, use_color: true)
|
|
109
|
+
return text unless use_color
|
|
110
|
+
|
|
111
|
+
"#{ANSI_COLORS[color_name]}#{text}#{ANSI_COLORS[:reset]}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Duration formatted for suite-level display (minute-range values)
|
|
115
|
+
# @param seconds [Numeric]
|
|
116
|
+
# @return [String] e.g. "4m 25s" or "45.3s"
|
|
117
|
+
def format_suite_duration(seconds)
|
|
118
|
+
if seconds >= 60
|
|
119
|
+
"#{(seconds / 60).floor}m %02ds" % (seconds % 60).round(0)
|
|
120
|
+
else
|
|
121
|
+
sprintf("%.1fs", seconds)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Right-aligned elapsed for suite columnar output (wider field)
|
|
126
|
+
# @param seconds [Numeric]
|
|
127
|
+
# @return [String] e.g. " 4m 25s" (7 chars wide)
|
|
128
|
+
def format_suite_elapsed(seconds)
|
|
129
|
+
sprintf("%7s", format_suite_duration(seconds))
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Build a columnar test result line for suite display
|
|
133
|
+
# @param icon [String] status icon (may include ANSI)
|
|
134
|
+
# @param elapsed [Numeric] seconds
|
|
135
|
+
# @param package [String] package name
|
|
136
|
+
# @param test_name [String] test name
|
|
137
|
+
# @param cases_str [String] e.g. "5/5 cases" or ""
|
|
138
|
+
# @param pkg_width [Integer] column width for package
|
|
139
|
+
# @param name_width [Integer] column width for test name
|
|
140
|
+
# @return [String]
|
|
141
|
+
def format_suite_test_line(icon, elapsed, package, test_name, cases_str, pkg_width:, name_width:)
|
|
142
|
+
time_col = format_suite_elapsed(elapsed)
|
|
143
|
+
pkg_col = package.ljust(pkg_width)
|
|
144
|
+
name_col = test_name.ljust(name_width)
|
|
145
|
+
"#{icon} #{time_col} #{pkg_col} #{name_col} #{cases_str}"
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# Build the full suite summary block
|
|
149
|
+
# @param results_data [Hash] with keys :total, :passed, :failed, :errors, :packages, :duration, :failed_details
|
|
150
|
+
# @param use_color [Boolean]
|
|
151
|
+
# @return [Array<String>] lines to print
|
|
152
|
+
def format_suite_summary(results_data, use_color: false)
|
|
153
|
+
lines = ["", double_separator]
|
|
154
|
+
|
|
155
|
+
failed_details = results_data[:failed_details] || []
|
|
156
|
+
if failed_details.any?
|
|
157
|
+
lines << "Failed tests:"
|
|
158
|
+
failed_details.each do |detail|
|
|
159
|
+
lines << " - #{detail[:package]}/#{detail[:test_name]}: #{detail[:cases]}"
|
|
160
|
+
end
|
|
161
|
+
lines << ""
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
lines << "Duration: #{format_suite_duration(results_data[:duration])}"
|
|
165
|
+
lines << "Tests: #{results_data[:passed]} passed, #{results_data[:failed] + results_data[:errors]} failed"
|
|
166
|
+
|
|
167
|
+
total_tc = results_data[:total_cases] || 0
|
|
168
|
+
passed_tc = results_data[:passed_cases] || 0
|
|
169
|
+
if total_tc > 0
|
|
170
|
+
failed_tc = total_tc - passed_tc
|
|
171
|
+
pct = (passed_tc * 100.0 / total_tc).round(0)
|
|
172
|
+
lines << "Test cases: #{passed_tc} passed, #{failed_tc} failed (#{pct}%)"
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
lines << ""
|
|
176
|
+
lines << if results_data[:failed] + results_data[:errors] == 0
|
|
177
|
+
color("\u2713 ALL TESTS PASSED", :green, use_color: use_color)
|
|
178
|
+
else
|
|
179
|
+
color("\u2717 SOME TESTS FAILED", :red, use_color: use_color)
|
|
180
|
+
end
|
|
181
|
+
lines << double_separator
|
|
182
|
+
lines
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Format summary lines for display after a test run
|
|
186
|
+
# @param results [Array<Models::TestResult>]
|
|
187
|
+
# @param duration [Numeric] total duration in seconds
|
|
188
|
+
# @param report_path [String]
|
|
189
|
+
# @param use_color [Boolean]
|
|
190
|
+
# @return [Array<String>] lines to print
|
|
191
|
+
def format_summary_lines(results, duration, report_path, use_color: false)
|
|
192
|
+
passed = results.count(&:success?)
|
|
193
|
+
failed = results.size - passed
|
|
194
|
+
total_tc = results.sum(&:total_count)
|
|
195
|
+
total_passed_tc = results.sum(&:passed_count)
|
|
196
|
+
total_failed_tc = total_tc - total_passed_tc
|
|
197
|
+
|
|
198
|
+
lines = [separator]
|
|
199
|
+
lines << "Duration: #{format_duration(duration)}"
|
|
200
|
+
lines << "Tests: #{passed} passed, #{failed} failed"
|
|
201
|
+
|
|
202
|
+
if total_tc > 0
|
|
203
|
+
pct = (total_passed_tc * 100.0 / total_tc).round(0)
|
|
204
|
+
lines << "Test cases: #{total_passed_tc} passed, #{total_failed_tc} failed (#{pct}%)"
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
lines << "Report: #{report_path}"
|
|
208
|
+
lines << ""
|
|
209
|
+
|
|
210
|
+
lines << if failed == 0
|
|
211
|
+
color("\u2713 ALL TESTS PASSED", :green, use_color: use_color)
|
|
212
|
+
else
|
|
213
|
+
color("\u2717 SOME TESTS FAILED", :red, use_color: use_color)
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
lines << separator
|
|
217
|
+
lines
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# Format a single-test result line
|
|
221
|
+
# @param result [Models::TestResult]
|
|
222
|
+
# @param use_color [Boolean]
|
|
223
|
+
# @return [String]
|
|
224
|
+
def format_single_result(result, use_color: false)
|
|
225
|
+
icon = color(status_icon(result.success?), result.success? ? :green : :red, use_color: use_color)
|
|
226
|
+
tc = tc_count_display(result)
|
|
227
|
+
|
|
228
|
+
"Result: #{icon} #{result.status.upcase}#{tc}"
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|