tryouts 3.3.2 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +15 -4
- data/lib/tryouts/cli/formatters/agent.rb +450 -0
- data/lib/tryouts/cli/formatters/factory.rb +5 -0
- data/lib/tryouts/cli/formatters/token_budget.rb +157 -0
- data/lib/tryouts/cli/formatters.rb +2 -0
- data/lib/tryouts/cli/opts.rb +74 -7
- data/lib/tryouts/test_runner.rb +8 -2
- data/lib/tryouts/version.rb +1 -1
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6b0f0a0ca7de83a58c4e822d127a963bb1fb08523d9146b24c8b33f8a2c12ff2
|
4
|
+
data.tar.gz: 2f4e61611e23176da42cc9bbf82a92a2cc1e9d6049943fd4d4827dcd2b34c84f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1505d26c47b4fb3fb75675425e6aaac3e1ba81b6f39ac8ae47622ae43acd0c19a13a16e82486663de0201f21bf7b997281744fc6d69cae6116e67065803b4a39
|
7
|
+
data.tar.gz: ca57f3c38348023c57f5e382fe8b45e29877393fae155a718c921550b4a2200b0fe37c1c31be4f41dcbc934527d8c6b53e74fbe88fcc574c53cbc3d765bb3534
|
data/README.md
CHANGED
@@ -1,17 +1,21 @@
|
|
1
|
-
# Tryouts v3
|
1
|
+
# Tryouts v3
|
2
2
|
|
3
3
|
**Ruby tests that read like documentation.**
|
4
4
|
|
5
5
|
A modern test framework for Ruby that uses comments to define expectations. Tryouts are meant to double as documentation, so the Ruby code should be plain and reminiscent of real code.
|
6
6
|
|
7
|
+
> [!NOTE]
|
8
|
+
> **Agent-Optimized Output**: Tryouts includes specialized output modes for LLM consumption with `--agent` flag, providing structured, token-efficient test results that are 60-80% smaller than traditional output while preserving debugging context.
|
9
|
+
|
7
10
|
> [!WARNING]
|
8
|
-
> Version 3.0+ uses Ruby's Prism parser and pattern matching, requiring Ruby 3.
|
11
|
+
> Version 3.0+ uses Ruby's Prism parser and pattern matching, requiring Ruby 3.2+
|
9
12
|
|
10
13
|
## Key Features
|
11
14
|
|
12
15
|
- **Documentation-style tests** using comment-based expectations (`#=>`)
|
13
16
|
- **Great expectation syntax** for more expressive assertions (`#==>` for true, `#=/=>` for false, `#=:>` for class/module)
|
14
17
|
- **Framework integration** write with tryouts syntax, run with RSpec or Minitest
|
18
|
+
- **Agent-optimized output** structured, token-efficient output for LLM consumption
|
15
19
|
- **Enhanced error reporting** with line numbers and context
|
16
20
|
|
17
21
|
## Installation
|
@@ -117,6 +121,13 @@ try -v # verbose (includes source code and return values)
|
|
117
121
|
try -q # quiet mode
|
118
122
|
try -f # show failures only
|
119
123
|
try -D # debug mode
|
124
|
+
|
125
|
+
# Agent-optimized output for LLMs
|
126
|
+
try --agent # structured, token-efficient output
|
127
|
+
try --agent --agent-focus summary # show only counts and problem files
|
128
|
+
try --agent --agent-focus first-failure # show first failure per file
|
129
|
+
try --agent --agent-focus critical # show only errors/exceptions
|
130
|
+
try --agent --agent-limit 1000 # limit output to 1000 tokens
|
120
131
|
```
|
121
132
|
|
122
133
|
### Exit Codes
|
@@ -127,14 +138,14 @@ try -D # debug mode
|
|
127
138
|
|
128
139
|
## Requirements
|
129
140
|
|
130
|
-
- **Ruby >= 3.2
|
141
|
+
- **Ruby >= 3.2** (for Prism parser and pattern matching)
|
131
142
|
- **RSpec** or **Minitest** (optional, for framework integration)
|
132
143
|
|
133
144
|
## Modern Architecture (v3+)
|
134
145
|
|
135
146
|
### Core Components
|
136
147
|
|
137
|
-
- **Prism Parser**:
|
148
|
+
- **Prism Parser**: Native Ruby parsing with pattern matching for line classification
|
138
149
|
- **Data Structures**: Immutable `Data.define` classes for test representation
|
139
150
|
- **Framework Translators**: Convert tryouts to RSpec/Minitest format
|
140
151
|
- **CLI**: Modern command-line interface with framework selection
|
@@ -0,0 +1,450 @@
|
|
1
|
+
# lib/tryouts/cli/formatters/agent.rb
|
2
|
+
|
3
|
+
require_relative 'token_budget'
|
4
|
+
|
5
|
+
class Tryouts
|
6
|
+
class CLI
|
7
|
+
# Agent-optimized formatter designed for LLM context management
|
8
|
+
# Features:
|
9
|
+
# - Token budget awareness
|
10
|
+
# - Structured YAML-like output
|
11
|
+
# - No redundant file paths
|
12
|
+
# - Smart truncation
|
13
|
+
# - Hierarchical organization
|
14
|
+
class AgentFormatter
|
15
|
+
include FormatterInterface
|
16
|
+
|
17
|
+
def initialize(options = {})
|
18
|
+
super
|
19
|
+
@budget = TokenBudget.new(options[:agent_limit] || TokenBudget::DEFAULT_LIMIT)
|
20
|
+
@focus_mode = options[:agent_focus] || :failures
|
21
|
+
@collected_files = []
|
22
|
+
@current_file_data = nil
|
23
|
+
@total_stats = { files: 0, tests: 0, failures: 0, errors: 0, elapsed: 0 }
|
24
|
+
@output_rendered = false
|
25
|
+
|
26
|
+
# No colors in agent mode for cleaner parsing
|
27
|
+
@use_colors = false
|
28
|
+
end
|
29
|
+
|
30
|
+
# Phase-level output - collect data, don't output immediately
|
31
|
+
def phase_header(message, file_count: nil)
|
32
|
+
# Store file count for later use, but only store actual file count
|
33
|
+
if file_count && message.include?("FILES")
|
34
|
+
@total_stats[:files] = file_count
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
# File-level operations - start collecting file data
|
39
|
+
def file_start(file_path, context_info: {})
|
40
|
+
@current_file_data = {
|
41
|
+
path: relative_path(file_path),
|
42
|
+
tests: 0,
|
43
|
+
failures: [],
|
44
|
+
errors: [],
|
45
|
+
passed: 0
|
46
|
+
}
|
47
|
+
end
|
48
|
+
|
49
|
+
def file_end(file_path, context_info: {})
|
50
|
+
# Finalize current file data
|
51
|
+
if @current_file_data
|
52
|
+
@collected_files << @current_file_data
|
53
|
+
@current_file_data = nil
|
54
|
+
end
|
55
|
+
# REMOVED: No longer attempts to render here to avoid premature output
|
56
|
+
end
|
57
|
+
|
58
|
+
def file_parsed(_file_path, test_count:, setup_present: false, teardown_present: false)
|
59
|
+
@current_file_data[:tests] = test_count if @current_file_data
|
60
|
+
@total_stats[:tests] += test_count
|
61
|
+
end
|
62
|
+
|
63
|
+
def file_result(_file_path, total_tests:, failed_count:, error_count:, elapsed_time: nil)
|
64
|
+
# Always update global totals
|
65
|
+
@total_stats[:failures] += failed_count
|
66
|
+
@total_stats[:errors] += error_count
|
67
|
+
@total_stats[:elapsed] += elapsed_time if elapsed_time
|
68
|
+
|
69
|
+
# Update per-file data when available
|
70
|
+
if @current_file_data
|
71
|
+
@current_file_data[:passed] = total_tests - failed_count - error_count
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
|
76
|
+
# Test-level operations - collect failure data
|
77
|
+
def test_result(result_packet)
|
78
|
+
return unless @current_file_data
|
79
|
+
|
80
|
+
# For summary mode, we still need to collect failures for counting, just don't build detailed data
|
81
|
+
if result_packet.failed? || result_packet.error?
|
82
|
+
if @focus_mode == :summary
|
83
|
+
# Just track counts for summary
|
84
|
+
if result_packet.error?
|
85
|
+
@current_file_data[:errors] << { basic: true }
|
86
|
+
else
|
87
|
+
@current_file_data[:failures] << { basic: true }
|
88
|
+
end
|
89
|
+
else
|
90
|
+
# Build detailed failure data for other modes
|
91
|
+
failure_data = build_failure_data(result_packet)
|
92
|
+
|
93
|
+
if result_packet.error?
|
94
|
+
@current_file_data[:errors] << failure_data
|
95
|
+
else
|
96
|
+
@current_file_data[:failures] << failure_data
|
97
|
+
end
|
98
|
+
|
99
|
+
# Mark truncation for first-failure mode (handle limiting in render phase)
|
100
|
+
if (@focus_mode == :first_failure || @focus_mode == :'first-failure') &&
|
101
|
+
(@current_file_data[:failures].size + @current_file_data[:errors].size) > 1
|
102
|
+
@current_file_data[:truncated] = true
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
# Summary operations - reliable trigger for rendering
|
109
|
+
def batch_summary(failure_collector)
|
110
|
+
# This becomes the single, reliable trigger for rendering
|
111
|
+
grand_total(
|
112
|
+
total_tests: @total_stats[:tests],
|
113
|
+
failed_count: @collected_files.sum { |f| f[:failures].size },
|
114
|
+
error_count: @collected_files.sum { |f| f[:errors].size },
|
115
|
+
successful_files: @collected_files.size - @collected_files.count { |f| f[:failures].any? || f[:errors].any? },
|
116
|
+
total_files: @collected_files.size,
|
117
|
+
elapsed_time: @total_stats[:elapsed]
|
118
|
+
) unless @output_rendered
|
119
|
+
end
|
120
|
+
|
121
|
+
def grand_total(total_tests:, failed_count:, error_count:, successful_files:, total_files:, elapsed_time:)
|
122
|
+
return if @output_rendered # Prevent double rendering
|
123
|
+
|
124
|
+
@total_stats.merge!(
|
125
|
+
tests: total_tests,
|
126
|
+
failures: failed_count,
|
127
|
+
errors: error_count,
|
128
|
+
successful_files: successful_files,
|
129
|
+
total_files: total_files,
|
130
|
+
elapsed: elapsed_time
|
131
|
+
)
|
132
|
+
|
133
|
+
# Now render all collected data
|
134
|
+
render_agent_output
|
135
|
+
@output_rendered = true
|
136
|
+
end
|
137
|
+
|
138
|
+
# Override live status - not needed for agent mode
|
139
|
+
def live_status_capabilities
|
140
|
+
{
|
141
|
+
supports_coordination: false,
|
142
|
+
output_frequency: :none,
|
143
|
+
requires_tty: false
|
144
|
+
}
|
145
|
+
end
|
146
|
+
|
147
|
+
private
|
148
|
+
|
149
|
+
def build_failure_data(result_packet)
|
150
|
+
test_case = result_packet.test_case
|
151
|
+
|
152
|
+
failure_data = {
|
153
|
+
line: (test_case.first_expectation_line || test_case.line_range&.first || 0) + 1,
|
154
|
+
test: test_case.description.to_s.empty? ? 'unnamed test' : test_case.description.to_s
|
155
|
+
}
|
156
|
+
|
157
|
+
case result_packet.status
|
158
|
+
when :error
|
159
|
+
error = result_packet.error
|
160
|
+
failure_data[:error] = error ? "#{error.class.name}: #{error.message}" : 'unknown error'
|
161
|
+
when :failed
|
162
|
+
if result_packet.expected_results.any? && result_packet.actual_results.any?
|
163
|
+
expected = @budget.smart_truncate(result_packet.first_expected, max_tokens: 25)
|
164
|
+
actual = @budget.smart_truncate(result_packet.first_actual, max_tokens: 25)
|
165
|
+
failure_data[:expected] = expected
|
166
|
+
failure_data[:got] = actual
|
167
|
+
|
168
|
+
# Add diff for strings if budget allows
|
169
|
+
if result_packet.first_expected.is_a?(String) &&
|
170
|
+
result_packet.first_actual.is_a?(String) &&
|
171
|
+
@budget.has_budget?
|
172
|
+
failure_data[:diff] = generate_simple_diff(result_packet.first_expected, result_packet.first_actual)
|
173
|
+
end
|
174
|
+
else
|
175
|
+
failure_data[:reason] = 'test failed'
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
failure_data
|
180
|
+
end
|
181
|
+
|
182
|
+
def generate_simple_diff(expected, actual)
|
183
|
+
return nil unless @budget.remaining > 100 # Only if we have decent budget left
|
184
|
+
|
185
|
+
# Simple line-by-line diff
|
186
|
+
exp_lines = expected.split("\n")
|
187
|
+
act_lines = actual.split("\n")
|
188
|
+
|
189
|
+
diff_lines = []
|
190
|
+
diff_lines << "- #{act_lines.first}" if act_lines.any?
|
191
|
+
diff_lines << "+ #{exp_lines.first}" if exp_lines.any?
|
192
|
+
|
193
|
+
diff_result = diff_lines.join("\n")
|
194
|
+
return @budget.fit_text(diff_result) if @budget.would_exceed?(diff_result)
|
195
|
+
diff_result
|
196
|
+
end
|
197
|
+
|
198
|
+
def render_agent_output
|
199
|
+
case @focus_mode
|
200
|
+
when :summary
|
201
|
+
render_summary_only
|
202
|
+
when :critical
|
203
|
+
render_critical_only
|
204
|
+
else
|
205
|
+
render_full_structured
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def render_summary_only
|
210
|
+
output = []
|
211
|
+
|
212
|
+
# Count failures manually from collected file data (same as other render methods)
|
213
|
+
failed_count = @collected_files.sum { |f| f[:failures].size }
|
214
|
+
error_count = @collected_files.sum { |f| f[:errors].size }
|
215
|
+
issues_count = failed_count + error_count
|
216
|
+
passed_count = [@total_stats[:tests] - issues_count, 0].max
|
217
|
+
|
218
|
+
if issues_count > 0
|
219
|
+
status = "FAIL: #{issues_count}/#{@total_stats[:tests]} tests"
|
220
|
+
details = []
|
221
|
+
details << "#{failed_count} failed" if failed_count > 0
|
222
|
+
details << "#{error_count} errors" if error_count > 0
|
223
|
+
status += " (#{details.join(', ')}, #{passed_count} passed)"
|
224
|
+
else
|
225
|
+
status = "PASS: #{@total_stats[:tests]} tests passed"
|
226
|
+
end
|
227
|
+
|
228
|
+
status += " (#{format_time(@total_stats[:elapsed])})" if @total_stats[:elapsed]
|
229
|
+
|
230
|
+
output << status
|
231
|
+
|
232
|
+
# Show which files had failures
|
233
|
+
files_with_issues = @collected_files.select { |f| f[:failures].any? || f[:errors].any? }
|
234
|
+
if files_with_issues.any?
|
235
|
+
output << ""
|
236
|
+
output << "Files with issues:"
|
237
|
+
files_with_issues.each do |file_data|
|
238
|
+
issue_count = file_data[:failures].size + file_data[:errors].size
|
239
|
+
output << " #{file_data[:path]}: #{issue_count} issue#{'s' if issue_count != 1}"
|
240
|
+
end
|
241
|
+
end
|
242
|
+
|
243
|
+
puts output.join("\n")
|
244
|
+
end
|
245
|
+
|
246
|
+
def render_critical_only
|
247
|
+
# Only show errors (exceptions), skip assertion failures
|
248
|
+
critical_files = @collected_files.select { |f| f[:errors].any? }
|
249
|
+
|
250
|
+
if critical_files.empty?
|
251
|
+
puts "No critical errors found"
|
252
|
+
return
|
253
|
+
end
|
254
|
+
|
255
|
+
output = []
|
256
|
+
output << "CRITICAL: #{critical_files.size} file#{'s' if critical_files.size != 1} with errors"
|
257
|
+
output << ""
|
258
|
+
|
259
|
+
critical_files.each do |file_data|
|
260
|
+
unless @budget.has_budget?
|
261
|
+
output << "... (truncated due to token limit)"
|
262
|
+
break
|
263
|
+
end
|
264
|
+
|
265
|
+
output << "#{file_data[:path]}:"
|
266
|
+
|
267
|
+
file_data[:errors].each do |error|
|
268
|
+
error_line = " L#{error[:line]}: #{error[:error]}"
|
269
|
+
if @budget.would_exceed?(error_line)
|
270
|
+
output << @budget.fit_text(error_line)
|
271
|
+
else
|
272
|
+
output << error_line
|
273
|
+
@budget.consume(error_line)
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
output << ""
|
278
|
+
end
|
279
|
+
|
280
|
+
puts output.join("\n")
|
281
|
+
end
|
282
|
+
|
283
|
+
def render_full_structured
|
284
|
+
output = []
|
285
|
+
|
286
|
+
# Header with overall stats
|
287
|
+
issues_count = @total_stats[:failures] + @total_stats[:errors]
|
288
|
+
passed_count = [@total_stats[:tests] - issues_count, 0].max
|
289
|
+
|
290
|
+
files_count = if @total_stats[:files].to_i > 0
|
291
|
+
@total_stats[:files]
|
292
|
+
else
|
293
|
+
@total_stats[:total_files] || @collected_files.size
|
294
|
+
end
|
295
|
+
|
296
|
+
if issues_count > 0
|
297
|
+
status_line = "FAIL: #{issues_count}/#{@total_stats[:tests]} tests (#{files_count} files, #{format_time(@total_stats[:elapsed])})"
|
298
|
+
else
|
299
|
+
status_line = "PASS: #{@total_stats[:tests]} tests (#{files_count} files, #{format_time(@total_stats[:elapsed])})"
|
300
|
+
end
|
301
|
+
|
302
|
+
# Always include status line
|
303
|
+
output << status_line
|
304
|
+
@budget.force_consume(status_line)
|
305
|
+
|
306
|
+
# Only show files with issues (unless focus is different)
|
307
|
+
files_to_show = case @focus_mode
|
308
|
+
when :failures, :first_failure
|
309
|
+
@collected_files.select { |f| f[:failures].any? || f[:errors].any? }
|
310
|
+
else
|
311
|
+
@collected_files.select { |f| f[:failures].any? || f[:errors].any? }
|
312
|
+
end
|
313
|
+
|
314
|
+
if files_to_show.any?
|
315
|
+
output << ""
|
316
|
+
@budget.consume("\n")
|
317
|
+
|
318
|
+
files_to_show.each do |file_data|
|
319
|
+
break unless @budget.has_budget?
|
320
|
+
|
321
|
+
file_section = render_file_section(file_data)
|
322
|
+
if @budget.would_exceed?(file_section)
|
323
|
+
# Try to fit what we can
|
324
|
+
truncated = @budget.fit_text(file_section, preserve_suffix: "\n ... (truncated)")
|
325
|
+
output << truncated if truncated.length > 20 # Only if meaningful content remains
|
326
|
+
break
|
327
|
+
else
|
328
|
+
output << file_section
|
329
|
+
@budget.consume(file_section)
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
# Final summary line
|
335
|
+
summary = "Summary: #{passed_count} passed, #{@total_stats[:failures]} failed"
|
336
|
+
summary += ", #{@total_stats[:errors]} errors" if @total_stats[:errors] > 0
|
337
|
+
summary += " in #{@total_stats[:files]} files"
|
338
|
+
|
339
|
+
output << ""
|
340
|
+
output << summary
|
341
|
+
|
342
|
+
puts output.join("\n")
|
343
|
+
end
|
344
|
+
|
345
|
+
def render_file_section(file_data)
|
346
|
+
lines = []
|
347
|
+
|
348
|
+
# File header
|
349
|
+
lines << "#{file_data[:path]}:"
|
350
|
+
|
351
|
+
# For first-failure mode, only show first error or failure
|
352
|
+
if @focus_mode == :first_failure || @focus_mode == :'first-failure'
|
353
|
+
shown_count = 0
|
354
|
+
|
355
|
+
# Show first error
|
356
|
+
if file_data[:errors].any? && shown_count == 0
|
357
|
+
error = file_data[:errors].first
|
358
|
+
lines << " L#{error[:line]}: #{error[:error]}"
|
359
|
+
lines << " Test: #{error[:test]}" if error[:test] != 'unnamed test'
|
360
|
+
shown_count += 1
|
361
|
+
end
|
362
|
+
|
363
|
+
# Show first failure if no error was shown
|
364
|
+
if file_data[:failures].any? && shown_count == 0
|
365
|
+
failure = file_data[:failures].first
|
366
|
+
line_parts = [" L#{failure[:line]}:"]
|
367
|
+
|
368
|
+
if failure[:expected] && failure[:got]
|
369
|
+
line_parts << "expected #{failure[:expected]}, got #{failure[:got]}"
|
370
|
+
elsif failure[:reason]
|
371
|
+
line_parts << failure[:reason]
|
372
|
+
end
|
373
|
+
|
374
|
+
lines << line_parts.join(' ')
|
375
|
+
lines << " Test: #{failure[:test]}" if failure[:test] != 'unnamed test'
|
376
|
+
|
377
|
+
# Add diff if available and budget allows
|
378
|
+
if failure[:diff] && @budget.remaining > 50
|
379
|
+
lines << " Diff:"
|
380
|
+
failure[:diff].split("\n").each { |diff_line| lines << " #{diff_line}" }
|
381
|
+
end
|
382
|
+
end
|
383
|
+
|
384
|
+
# Show truncation notice
|
385
|
+
total_issues = file_data[:errors].size + file_data[:failures].size
|
386
|
+
if total_issues > 1
|
387
|
+
lines << " ... (#{total_issues - 1} more failures not shown)"
|
388
|
+
end
|
389
|
+
else
|
390
|
+
# Normal mode - show all errors and failures
|
391
|
+
# Errors first (more critical)
|
392
|
+
file_data[:errors].each do |error|
|
393
|
+
next if error[:basic] # Skip basic entries from summary mode
|
394
|
+
lines << " L#{error[:line]}: #{error[:error]}"
|
395
|
+
lines << " Test: #{error[:test]}" if error[:test] != 'unnamed test'
|
396
|
+
end
|
397
|
+
|
398
|
+
# Then failures
|
399
|
+
file_data[:failures].each do |failure|
|
400
|
+
next if failure[:basic] # Skip basic entries from summary mode
|
401
|
+
line_parts = [" L#{failure[:line]}:"]
|
402
|
+
|
403
|
+
if failure[:expected] && failure[:got]
|
404
|
+
line_parts << "expected #{failure[:expected]}, got #{failure[:got]}"
|
405
|
+
elsif failure[:reason]
|
406
|
+
line_parts << failure[:reason]
|
407
|
+
end
|
408
|
+
|
409
|
+
lines << line_parts.join(' ')
|
410
|
+
lines << " Test: #{failure[:test]}" if failure[:test] != 'unnamed test'
|
411
|
+
|
412
|
+
# Add diff if available and budget allows
|
413
|
+
if failure[:diff] && @budget.remaining > 50
|
414
|
+
lines << " Diff:"
|
415
|
+
failure[:diff].split("\n").each { |diff_line| lines << " #{diff_line}" }
|
416
|
+
end
|
417
|
+
end
|
418
|
+
|
419
|
+
# Show truncation notice if applicable
|
420
|
+
if file_data[:truncated]
|
421
|
+
lines << " ... (more failures not shown)"
|
422
|
+
end
|
423
|
+
end
|
424
|
+
|
425
|
+
lines.join("\n")
|
426
|
+
end
|
427
|
+
|
428
|
+
def relative_path(file_path)
|
429
|
+
# Remove leading path components to save tokens
|
430
|
+
path = Pathname.new(file_path).relative_path_from(Pathname.pwd).to_s
|
431
|
+
# If relative path is longer, use just filename
|
432
|
+
path.include?('../') ? File.basename(file_path) : path
|
433
|
+
rescue
|
434
|
+
File.basename(file_path)
|
435
|
+
end
|
436
|
+
|
437
|
+
def format_time(seconds)
|
438
|
+
return '0ms' unless seconds
|
439
|
+
|
440
|
+
if seconds < 0.001
|
441
|
+
"#{(seconds * 1_000_000).round}μs"
|
442
|
+
elsif seconds < 1
|
443
|
+
"#{(seconds * 1000).round}ms"
|
444
|
+
else
|
445
|
+
"#{seconds.round(2)}s"
|
446
|
+
end
|
447
|
+
end
|
448
|
+
end
|
449
|
+
end
|
450
|
+
end
|
@@ -12,6 +12,11 @@ class Tryouts
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def self.create_formatter(options = {})
|
15
|
+
# Check for agent mode first (takes precedence)
|
16
|
+
if options[:agent]
|
17
|
+
return AgentFormatter.new(options)
|
18
|
+
end
|
19
|
+
|
15
20
|
# Map boolean flags to format symbols if format not explicitly set
|
16
21
|
format = options[:format]&.to_sym || determine_format_from_flags(options)
|
17
22
|
|
@@ -0,0 +1,157 @@
|
|
1
|
+
# lib/tryouts/cli/formatters/token_budget.rb
|
2
|
+
|
3
|
+
class Tryouts
|
4
|
+
class CLI
|
5
|
+
# Token budget tracking for agent-optimized output
|
6
|
+
class TokenBudget
|
7
|
+
DEFAULT_LIMIT = 5000
|
8
|
+
BUFFER_PERCENT = 0.05 # 5% buffer to avoid going over
|
9
|
+
|
10
|
+
attr_reader :limit, :used, :remaining
|
11
|
+
|
12
|
+
def initialize(limit = DEFAULT_LIMIT)
|
13
|
+
@limit = limit
|
14
|
+
@used = 0
|
15
|
+
@buffer_size = (@limit * BUFFER_PERCENT).to_i
|
16
|
+
end
|
17
|
+
|
18
|
+
# Estimate tokens in text (rough approximation: 1 token ≈ 4 characters)
|
19
|
+
def estimate_tokens(text)
|
20
|
+
return 0 if text.nil? || text.empty?
|
21
|
+
|
22
|
+
(text.length / 4.0).ceil
|
23
|
+
end
|
24
|
+
|
25
|
+
# Check if text would exceed budget
|
26
|
+
def would_exceed?(text)
|
27
|
+
token_count = estimate_tokens(text)
|
28
|
+
(@used + token_count) > (@limit - @buffer_size)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Add text to budget if within limits
|
32
|
+
def consume(text)
|
33
|
+
return false if would_exceed?(text)
|
34
|
+
|
35
|
+
@used += estimate_tokens(text)
|
36
|
+
true
|
37
|
+
end
|
38
|
+
|
39
|
+
# Force consume (for critical information that must be included)
|
40
|
+
def force_consume(text)
|
41
|
+
@used += estimate_tokens(text)
|
42
|
+
true
|
43
|
+
end
|
44
|
+
|
45
|
+
# Get remaining budget
|
46
|
+
def remaining
|
47
|
+
[@limit - @used - @buffer_size, 0].max
|
48
|
+
end
|
49
|
+
|
50
|
+
# Check if we have budget remaining
|
51
|
+
def has_budget?
|
52
|
+
remaining > 0
|
53
|
+
end
|
54
|
+
|
55
|
+
# Get utilization percentage
|
56
|
+
def utilization
|
57
|
+
(@used.to_f / @limit * 100).round(1)
|
58
|
+
end
|
59
|
+
|
60
|
+
# Try to fit text within remaining budget by truncating
|
61
|
+
def fit_text(text, preserve_suffix: nil)
|
62
|
+
token_count = estimate_tokens(text)
|
63
|
+
|
64
|
+
return text if token_count <= remaining
|
65
|
+
return '' unless has_budget?
|
66
|
+
|
67
|
+
# Calculate how many characters we can fit
|
68
|
+
max_chars = remaining * 4
|
69
|
+
|
70
|
+
if preserve_suffix
|
71
|
+
suffix_chars = preserve_suffix.length
|
72
|
+
return preserve_suffix if max_chars <= suffix_chars
|
73
|
+
|
74
|
+
available_chars = max_chars - suffix_chars - 3 # 3 for "..."
|
75
|
+
return "#{text[0, available_chars]}...#{preserve_suffix}"
|
76
|
+
else
|
77
|
+
return text[0, max_chars - 3] + '...' if max_chars > 3
|
78
|
+
return ''
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
# Smart truncate for different data types
|
83
|
+
def smart_truncate(value, max_tokens: nil)
|
84
|
+
max_tokens ||= [remaining / 2, 50].min # Use half remaining or 50, whichever is smaller
|
85
|
+
max_chars = [max_tokens.to_i * 4, 0].max
|
86
|
+
|
87
|
+
case value
|
88
|
+
when String
|
89
|
+
return value if value.length <= max_chars
|
90
|
+
return '...' if max_chars <= 3
|
91
|
+
"#{value[0, max_chars - 3]}..."
|
92
|
+
when Array
|
93
|
+
if estimate_tokens(value.inspect) <= max_tokens
|
94
|
+
value.inspect
|
95
|
+
else
|
96
|
+
# Show first few elements
|
97
|
+
truncated = []
|
98
|
+
char_count = 2 # for "[]"
|
99
|
+
|
100
|
+
value.each do |item|
|
101
|
+
item_str = item.inspect
|
102
|
+
if char_count + item_str.length + 2 <= max_chars - 10 # 10 chars for ", ..."
|
103
|
+
truncated << item
|
104
|
+
char_count += item_str.length + 2 # +2 for ", "
|
105
|
+
else
|
106
|
+
break
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
"[#{truncated.map(&:inspect).join(', ')}, ...#{value.size - truncated.size} more]"
|
111
|
+
end
|
112
|
+
when Hash
|
113
|
+
if estimate_tokens(value.inspect) <= max_tokens
|
114
|
+
value.inspect
|
115
|
+
else
|
116
|
+
# Show first few key-value pairs
|
117
|
+
truncated = {}
|
118
|
+
char_count = 2 # for "{}"
|
119
|
+
|
120
|
+
value.each do |key, val|
|
121
|
+
pair_str = "#{key.inspect}=>#{val.inspect}"
|
122
|
+
if char_count + pair_str.length + 2 <= max_chars - 10
|
123
|
+
truncated[key] = val
|
124
|
+
char_count += pair_str.length + 2
|
125
|
+
else
|
126
|
+
break
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
"{#{truncated.map { |k, v| "#{k.inspect}=>#{v.inspect}" }.join(', ')}, ...#{value.size - truncated.size} more}"
|
131
|
+
end
|
132
|
+
else
|
133
|
+
smart_truncate(value.to_s, max_tokens: max_tokens)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
# Distribution strategy for budget allocation
|
138
|
+
def allocate_budget
|
139
|
+
{
|
140
|
+
summary: (@limit * 0.20).to_i, # 20% for file summaries
|
141
|
+
failures: (@limit * 0.60).to_i, # 60% for failure details
|
142
|
+
context: (@limit * 0.15).to_i, # 15% for additional context
|
143
|
+
buffer: (@limit * 0.05).to_i # 5% buffer
|
144
|
+
}
|
145
|
+
end
|
146
|
+
|
147
|
+
# Reset budget
|
148
|
+
def reset
|
149
|
+
@used = 0
|
150
|
+
end
|
151
|
+
|
152
|
+
def to_s
|
153
|
+
"TokenBudget[#{@used}/#{@limit} tokens (#{utilization}%)]"
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
@@ -6,4 +6,6 @@ require_relative 'formatters/quiet'
|
|
6
6
|
require_relative 'formatters/verbose'
|
7
7
|
require_relative 'formatters/test_run_state'
|
8
8
|
require_relative 'formatters/tty_status_display'
|
9
|
+
require_relative 'formatters/token_budget'
|
10
|
+
require_relative 'formatters/agent'
|
9
11
|
require_relative 'formatters/factory'
|
data/lib/tryouts/cli/opts.rb
CHANGED
@@ -15,19 +15,72 @@ class Tryouts
|
|
15
15
|
try --direct --shared-context test_try.rb # Explicit shared context
|
16
16
|
try --generate-rspec test_try.rb # Output RSpec code only
|
17
17
|
try --inspect test_try.rb # Inspect file structure and validation
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
18
|
+
try --agent test_try.rb # Agent-optimized structured output
|
19
|
+
try --agent --agent-limit 10000 tests/ # Agent mode with 10K token limit
|
20
|
+
|
21
|
+
Agent Output Modes:
|
22
|
+
--agent # Structured, token-efficient output
|
23
|
+
--agent-focus summary # Show counts and problem files only
|
24
|
+
--agent-focus first-failure # Show first failure per file
|
25
|
+
--agent-focus critical # Show errors/exceptions only
|
26
|
+
--agent-limit 1000 # Limit output to 1000 tokens
|
27
|
+
|
28
|
+
File Naming & Organization:
|
29
|
+
Files must end with '_try.rb' or '.try.rb' (e.g., auth_service_try.rb, user_model.try.rb)
|
30
|
+
Auto-discovery searches: ./try/, ./tryouts/, ./*_try.rb, ./*.try.rb patterns
|
31
|
+
Organize by feature/module: try/models/, try/services/, try/api/
|
32
|
+
|
33
|
+
Testcase Structure (3 required parts)
|
34
|
+
## This is the description
|
35
|
+
echo 'This is ruby code under test'
|
36
|
+
true
|
37
|
+
#=> true # this is the expected result
|
38
|
+
|
39
|
+
File Structure (3 sections):
|
40
|
+
# Setup section (optional) - runs once before all tests
|
41
|
+
@shared_var = "available to all test cases"
|
42
|
+
|
43
|
+
## TEST: Feature description
|
44
|
+
# Test case body with plain Ruby code
|
45
|
+
result = some_operation()
|
46
|
+
#=> expected_value
|
47
|
+
|
48
|
+
# Teardown section (optional) - runs once after all tests
|
49
|
+
|
50
|
+
Context Guidelines:
|
51
|
+
Shared Context (default): Instance variables persist across test cases
|
52
|
+
- Use for: Integration testing, stateful scenarios, realistic workflows
|
53
|
+
- Caution: Test order matters, state accumulates
|
54
|
+
|
55
|
+
Fresh Context (--rspec/--minitest): Each test gets isolated environment
|
56
|
+
- Use for: Unit testing, independent test cases
|
57
|
+
- Setup variables copied to each test, but changes don't persist
|
58
|
+
|
59
|
+
Writing Quality Tryouts:
|
60
|
+
- Use realistic, plain Ruby code (avoid mocks, test harnesses)
|
61
|
+
- Test descriptions start with ##, be specific about what's being tested
|
62
|
+
- One result per test case (last expression is the result)
|
63
|
+
- Use appropriate expectation types for clarity (#==> for boolean, #=:> for types)
|
64
|
+
- Keep tests focused and readable - they serve as documentation
|
23
65
|
|
24
66
|
Great Expectations System:
|
25
|
-
Multiple expectation types are supported for different testing needs.
|
26
|
-
|
27
67
|
#=> Value equality #==> Must be true #=/=> Must be false
|
28
68
|
#=|> True OR false #=!> Must raise error #=:> Type matching
|
29
69
|
#=~> Regex matching #=%> Time constraints #=*> Non-nil result
|
30
70
|
#=1> STDOUT content #=2> STDERR content #=<> Intentional failure
|
71
|
+
|
72
|
+
Exception Testing:
|
73
|
+
# Method 1: Rescue and test exception
|
74
|
+
begin
|
75
|
+
risky_operation
|
76
|
+
rescue StandardError => e
|
77
|
+
e.class
|
78
|
+
end
|
79
|
+
#=> StandardError
|
80
|
+
|
81
|
+
# Method 2: Let it raise and test with #=!>
|
82
|
+
risky_operation
|
83
|
+
#=!> error.is_a?(StandardError)
|
31
84
|
HELP
|
32
85
|
|
33
86
|
class << self
|
@@ -70,6 +123,20 @@ class Tryouts
|
|
70
123
|
options[:parallel_threads] = threads.to_i if threads && threads.to_i > 0
|
71
124
|
end
|
72
125
|
|
126
|
+
opts.separator "\nAgent-Optimized Output:"
|
127
|
+
opts.on('-a', '--agent', 'Agent-optimized structured output for LLM context management') do
|
128
|
+
options[:agent] = true
|
129
|
+
end
|
130
|
+
opts.on('--agent-limit TOKENS', Integer, 'Limit total output to token budget (default: 5000)') do |limit|
|
131
|
+
options[:agent] = true
|
132
|
+
options[:agent_limit] = limit
|
133
|
+
end
|
134
|
+
opts.on('--agent-focus TYPE', %w[failures first-failure summary critical],
|
135
|
+
'Focus mode: failures, first-failure, summary, critical (default: failures)') do |focus|
|
136
|
+
options[:agent] = true
|
137
|
+
options[:agent_focus] = focus.to_sym
|
138
|
+
end
|
139
|
+
|
73
140
|
opts.separator "\nParser Options:"
|
74
141
|
opts.on('--enhanced-parser', 'Use enhanced parser with inhouse comment extraction (default)') { options[:parser] = :enhanced }
|
75
142
|
opts.on('--legacy-parser', 'Use legacy prism parser') { options[:parser] = :prism }
|
data/lib/tryouts/test_runner.rb
CHANGED
@@ -38,7 +38,13 @@ class Tryouts
|
|
38
38
|
result = process_files
|
39
39
|
show_failure_summary
|
40
40
|
show_grand_total if @global_tally[:aggregator].get_file_counts[:total] > 1
|
41
|
-
|
41
|
+
|
42
|
+
# For agent critical mode, only count errors as failures
|
43
|
+
if @options[:agent] && (@options[:agent_focus] == :critical || @options[:agent_focus] == 'critical')
|
44
|
+
@global_tally[:aggregator].get_display_counts[:errors]
|
45
|
+
else
|
46
|
+
result
|
47
|
+
end
|
42
48
|
end
|
43
49
|
|
44
50
|
private
|
@@ -107,7 +113,7 @@ class Tryouts
|
|
107
113
|
executor = Concurrent::ThreadPoolExecutor.new(
|
108
114
|
min_threads: 1,
|
109
115
|
max_threads: pool_size,
|
110
|
-
max_queue:
|
116
|
+
max_queue: @files.length, # Queue size must accommodate all files
|
111
117
|
fallback_policy: :abort # Raise exception if pool and queue are exhausted
|
112
118
|
)
|
113
119
|
|
data/lib/tryouts/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tryouts
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Delano Mandelbaum
|
@@ -137,6 +137,7 @@ files:
|
|
137
137
|
- lib/tryouts.rb
|
138
138
|
- lib/tryouts/cli.rb
|
139
139
|
- lib/tryouts/cli/formatters.rb
|
140
|
+
- lib/tryouts/cli/formatters/agent.rb
|
140
141
|
- lib/tryouts/cli/formatters/base.rb
|
141
142
|
- lib/tryouts/cli/formatters/compact.rb
|
142
143
|
- lib/tryouts/cli/formatters/factory.rb
|
@@ -144,6 +145,7 @@ files:
|
|
144
145
|
- lib/tryouts/cli/formatters/output_manager.rb
|
145
146
|
- lib/tryouts/cli/formatters/quiet.rb
|
146
147
|
- lib/tryouts/cli/formatters/test_run_state.rb
|
148
|
+
- lib/tryouts/cli/formatters/token_budget.rb
|
147
149
|
- lib/tryouts/cli/formatters/tty_status_display.rb
|
148
150
|
- lib/tryouts/cli/formatters/verbose.rb
|
149
151
|
- lib/tryouts/cli/modes/generate.rb
|