tryouts 3.5.0 → 3.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b692cc3a63ac86e060b52e77c246305be0f209dbf06b31b1b08deefbc06434a
-  data.tar.gz: 156e86aa4bde377aa2158f3a059983a6fe5f58796f71545c9895a9b57668b126
+  metadata.gz: 958254585920c850e17a18e13699040fad97ec8610f3c2141b32899016dc4194
+  data.tar.gz: 3666350beb13d17cf24bf184350fd88746cebbf7b7efc109c3e5af7f8d5c6c3b
 SHA512:
-  metadata.gz: d289a9b5ccd6694bd63fe4ca2a0c8aaf610d702e74380bf4975ae78ffaf377d0f82a9ec481dbc51105de9f3b07681c32eaa8a1c3566a1eebd0a861796eaff1e3
-  data.tar.gz: a0db3ebf76b5a291e6ea9d81f1f6e268a95320d28040a10f2225171b221f04bb2ed525bae3f79785d50f4269eb7c9be1dec242012bbca0a6d4588ac9e7fdd362
+  metadata.gz: 1a1091a28256b97c210f8a0a970db3a78d715e1b195e95a5a844e8f32b7d65f7e6d5cb49209baaf2c061b02500ad6bf2d71968cbcb76c16206108eabfa8039f3
+  data.tar.gz: 9613b25213146c5da6dbd5153fa101e8139e205f06fb5bcddb8fa172a82fc76dffff56f79d69609c4764623ac9bf0682a0ba8ef4e76fd49517920a9b3e1141af

data/README.md

@@ -132,18 +132,15 @@ try --agent --agent-limit 1000          # limit output to 1000 tokens
 
 #### Why Not Pipe Test Output Directly to AI?
 
-Raw test output creates several problems when working with AI assistants:
+I mean, you could. If that already works well, you could probably still benefit from an agent that is able to focus on the critical information for the task. And the extra context window space.
 
-- **Token bloat**: Verbose formatting wastes 60-80% of your context window on styling
-- **Signal vs noise**: Important failures get buried in passing test details and framework boilerplate
-- **Inconsistent parsing**: AI struggles with varying output formats across different test runs
-- **Context overflow**: Large test suites exceed AI token limits, truncating critical information
+Raw test output creates problems when working with AI assistants: high token usage with inconsistent parsing across different runs, where the same logical failure might be interpreted differently, making it difficult to reliably produce and analyze results consistently.
 
-#### TOPA: A Better Approach
+#### TOPAZ: A Better Approach
 
-Tryouts' `--agent` mode inspired the development of **TOPA (Test Output Protocol for AI)** - a standardized format optimized for AI analysis. The [tpane](https://github.com/delano/tpane) tool implements this protocol, transforming any test framework's output into structured, token-efficient formats.
+Tryouts' `--agent` mode inspired the development of **TOPAZ (Test Output Protocol for AI Zealots)** - a standardized format optimized for AI analysis. The [tpane](https://github.com/delano/tpane) tool implements this protocol, transforming any test framework's output into structured, token-efficient formats.
 
-Instead of overwhelming AI with raw output, TOPA provides clean semantic data focusing on what actually needs attention - failures, errors, and actionable context.
+Instead of overwhelming AI with raw output, TOPAZ provides clean semantic data focusing on what actually needs attention - failures, errors, and actionable context.
 
 ### Exit Codes
 

data/exe/try

@@ -38,10 +38,16 @@ require_relative '../lib/tryouts'
 lib_glob = File.join(Dir.pwd, '{lib,../lib,.}')
 Tryouts.update_load_path(lib_glob) if Tryouts.respond_to?(:update_load_path)
 
+# Capture original command for agent mode before ARGV gets modified
+original_command = [$0] + ARGV
+
 # Parse args and run CLI
 begin
   files, options = Tryouts::CLI.parse_args(ARGV)
 
+  # Add original command to options for agent formatter
+  options[:original_command] = original_command
+
   # Expand files if directories are given, preserving line specs
   expanded_files = []
   files.each do |file_or_dir|

data/lib/tryouts/cli/formatters/agent.rb

@@ -4,13 +4,42 @@ require_relative 'token_budget'
 
 class Tryouts
   class CLI
-    # Agent-optimized formatter designed for LLM context management
-    # Features:
-    # - Token budget awareness
-    # - Structured YAML-like output
-    # - No redundant file paths
-    # - Smart truncation
-    # - Hierarchical organization
+    # TOPAZ (Test Output Protocol for AI Zealots) Formatter
+    #
+    # Language-agnostic test output format designed for LLM context management.
+    # This formatter implements the TOPAZ v1.0 specification for structured,
+    # token-efficient test result communication.
+    #
+    # TOPAZ Features:
+    # - Language-agnostic field naming (snake_case, hierarchical)
+    # - Standardized execution context (runtime, environment, VCS)
+    # - Token budget awareness with smart truncation
+    # - Cross-platform compatibility (CI/CD, package managers)
+    # - Structured failure reporting with diffs
+    # - Protocol versioning for forward compatibility
+    #
+    # Field Specifications:
+    # - command: Exact command executed
+    # - process_id: System process identifier
+    # - runtime: Language, version, platform info
+    # - package_manager: Dependency management system
+    # - version_control: VCS branch/commit info
+    # - environment: Normalized env vars (ci_system, app_env, etc.)
+    # - test_framework: Framework name, isolation mode, parser
+    # - execution_flags: Runtime flags in normalized form
+    # - protocol: TOPAZ version and configuration
+    # - project: Auto-detected project type
+    # - test_discovery: File pattern matching rules
+    #
+    # Compatible with: Ruby/RSpec/Minitest, Python/pytest/unittest,
+    # JavaScript/Jest/Mocha, Java/JUnit, Go, C#/NUnit, and more.
+    #
+    # Language Adaptation Examples:
+    # Python: runtime.language=python, package_manager.name=pip/poetry/conda
+    # Node.js: runtime.language=javascript, package_manager.name=npm/yarn/pnpm
+    # Java: runtime.language=java, package_manager.name=maven/gradle
+    # Go: runtime.language=go, package_manager.name=go_modules
+    # C#: runtime.language=csharp, package_manager.name=nuget/dotnet
     class AgentFormatter
       include FormatterInterface
 
@@ -20,7 +49,7 @@ class Tryouts
         @focus_mode = options[:agent_focus] || :failures
         @collected_files = []
         @current_file_data = nil
-        @total_stats = { files: 0, tests: 0, failures: 0, errors: 0, elapsed: 0 }
+        @total_stats = { files: 0, tests: 0, failures: 0, errors: 0, elapsed_time: 0 }
         @output_rendered = false
         @options = options  # Store all options for execution context display
         @all_warnings = []  # Store warnings globally for execution details
@@ -91,7 +120,7 @@ class Tryouts
         # Always update global totals
         @total_stats[:failures] += failed_count
         @total_stats[:errors] += error_count
-        @total_stats[:elapsed] += elapsed_time if elapsed_time
+        @total_stats[:elapsed_time] += elapsed_time if elapsed_time
 
         # Update per-file data - file_result is called AFTER file_end, so data is in @collected_files
         relative_file_path = relative_path(file_path)
@@ -146,7 +175,7 @@ class Tryouts
           error_count: @collected_files.sum { |f| f[:errors].size },
           successful_files: @collected_files.size - @collected_files.count { |f| f[:failures].any? || f[:errors].any? },
           total_files: @collected_files.size,
-          elapsed_time: @total_stats[:elapsed]
+          elapsed_time: @total_stats[:elapsed_time]
         ) unless @output_rendered
       end
 
@@ -159,7 +188,7 @@ class Tryouts
           errors: error_count,
           successful_files: successful_files,
           total_files: total_files,
-          elapsed: elapsed_time
+          elapsed_time: elapsed_time,
         )
 
         # Now render all collected data
@@ -249,6 +278,12 @@ class Tryouts
       def render_summary_only
         output = []
 
+        time_str = if @total_stats[:elapsed_time] < 2.0
+          " (#{(@total_stats[:elapsed_time] * 1000).round}ms)"
+        else
+          " (#{@total_stats[:elapsed_time].round(2)}s)"
+        end
+
         # Add execution context header for agent clarity
         output << render_execution_context
         output << ""
@@ -264,13 +299,13 @@ class Tryouts
           details = []
           details << "#{failed_count} failed" if failed_count > 0
           details << "#{error_count} errors" if error_count > 0
-          status_parts << "FAIL: #{issues_count}/#{@total_stats[:tests]} tests (#{details.join(', ')}, #{passed_count} passed)"
+          status_parts << "FAIL: #{issues_count}/#{@total_stats[:tests]} tests (#{details.join(', ')}, #{passed_count} passed#{time_str})"
         else
           # Agent doesn't need output in the positive case (i.e. for passing
           # tests). It just fills out the context window.
         end
 
-        status_parts << "(#{format_time(@total_stats[:elapsed])})" if @total_stats[:elapsed]
+        status_parts << "(#{format_time(@total_stats[:elapsed_time])})" if @total_stats[:elapsed_time]
 
         output << status_parts.join(" ")
 
@@ -302,6 +337,12 @@ class Tryouts
         # Only show errors (exceptions), skip assertion failures
         critical_files = @collected_files.select { |f| f[:errors].any? }
 
+        time_str = if @total_stats[:elapsed_time] < 2.0
+          " (#{(@total_stats[:elapsed_time] * 1000).round}ms)"
+        else
+          " (#{@total_stats[:elapsed_time].round(2)}s)"
+        end
+
         output = []
 
         # Add execution context header for agent clarity
@@ -314,7 +355,7 @@ class Tryouts
           return
         end
 
-        output << "CRITICAL: #{critical_files.size} file#{'s' if critical_files.size != 1} with errors"
+        output << "CRITICAL: #{critical_files.size} file#{'s' if critical_files.size != 1} with errors#{time_str}"
         output << ""
 
         critical_files.each do |file_data|
@@ -344,6 +385,12 @@ class Tryouts
       def render_full_structured
         output = []
 
+        time_str = if @total_stats[:elapsed_time] < 2.0
+          " (#{(@total_stats[:elapsed_time] * 1000).round}ms)"
+        else
+          " (#{@total_stats[:elapsed_time].round(2)}s)"
+        end
+
         # Add execution context header for agent clarity
         output << render_execution_context
         output << ""
@@ -379,7 +426,7 @@ class Tryouts
         summary = "Summary: \n"
         summary += "#{passed_count} testcases passed, #{failed_count} failed"
         summary += ", #{error_count} errors" if error_count > 0
-        summary += " in #{@total_stats[:files]} files"
+        summary += " in #{@total_stats[:files]} files#{time_str}"
 
         output << summary
 
@@ -506,9 +553,41 @@ class Tryouts
 
       def render_execution_context
         context_lines = []
-        context_lines << "EXECUTION DETAILS:"
+        context_lines << "EXECUTION_CONTEXT:"
+
+        # Command that was executed
+        if @options[:original_command]
+          command_str = @options[:original_command].join(' ')
+          context_lines << "  command: #{command_str}"
+        end
 
-        # Framework and context mode
+        # Compact system info on one line when possible
+        context_lines << "  pid: #{Process.pid} | pwd: #{Dir.pwd}"
+
+        # Runtime - compact format
+        platform = RUBY_PLATFORM.gsub(/darwin\d+/, 'darwin')  # Simplify darwin25 -> darwin
+        context_lines << "  runtime: ruby #{RUBY_VERSION} (#{platform}); tryouts #{Tryouts::VERSION}"
+
+        # Package manager - only if present, compact format
+        if defined?(Bundler)
+          context_lines << "  package_manager: bundler #{Bundler::VERSION}"
+        end
+
+        # Version control - compact single line with timeout protection
+        git_info = safe_git_info
+        if git_info[:branch] && git_info[:commit] && !git_info[:branch].empty? && !git_info[:commit].empty?
+          context_lines << "  vcs: git #{git_info[:branch]}@#{git_info[:commit]}"
+        end
+
+        # Environment - only non-defaults
+        env_vars = build_environment_context
+        if env_vars.any?
+          # Compact key=value format
+          env_str = env_vars.map { |k, v| "#{k}=#{v}" }.join(', ')
+          context_lines << "  environment: #{env_str}"
+        end
+
+        # Test framework - compact critical info only
         framework = @options[:framework] || :direct
         shared_context = if @options.key?(:shared_context)
           @options[:shared_context]
@@ -522,26 +601,24 @@ class Tryouts
           end
         end
 
-        context_lines << "  Framework: #{framework}"
-        context_lines << "  Context mode: #{shared_context ? 'shared (variables persist across test cases)' : 'fresh (each test case isolated)'}"
-
-        # Parser type
-        parser = @options[:parser] || :enhanced
-        context_lines << "  Parser: #{parser}"
+        isolation = shared_context ? 'shared' : 'isolated'
+        context_lines << "  test_framework: #{framework} (#{isolation})"
 
-        # Other relevant flags
-        flags = []
-        flags << "verbose" if @options[:verbose]
-        flags << "fails-only" if @options[:fails_only]
-        flags << "debug" if @options[:debug]
-        flags << "stack-traces" if @options[:stack_traces]
-        flags << "parallel(#{@options[:parallel_threads] || 'auto'})" if @options[:parallel]
-        flags << "line-spec" if @options[:line_spec]
+        # Execution flags - only if non-standard
+        flags = build_execution_flags
+        if flags.any?
+          context_lines << "  flags: #{flags.join(', ')}"
+        end
 
-        context_lines << "  Flags: #{flags.any? ? flags.join(', ') : 'none'}" if flags.any?
+        # TOPAZ protocol - compact
+        context_lines << "  protocol: TOPAZ v0.3 | focus: #{@focus_mode} | limit: #{@budget.limit}"
 
-        # Agent-specific settings
-        context_lines << "  Agent mode: focus=#{@focus_mode}, limit=#{@budget.limit} tokens"
+        # File count being tested
+        if @collected_files && @collected_files.any?
+          context_lines << "  files_under_test: #{@collected_files.size}"
+        elsif @total_stats[:files] && @total_stats[:files] > 0
+          context_lines << "  files_under_test: #{@total_stats[:files]}"
+        end
 
         # Add syntax errors if any (these prevent test execution)
         if @syntax_errors.any?
@@ -571,6 +648,101 @@ class Tryouts
 
         context_lines.join("\n")
       end
+
+      # Build environment context with language-agnostic keys
+      def build_environment_context
+        env_vars = {}
+
+        # CI/CD detection - prioritize most specific
+        if ENV['GITHUB_ACTIONS']
+          env_vars['CI'] = 'github'
+        elsif ENV['GITLAB_CI']
+          env_vars['CI'] = 'gitlab'
+        elsif ENV['JENKINS_URL']
+          env_vars['CI'] = 'jenkins'
+        elsif ENV['CI']
+          env_vars['CI'] = 'true'
+        end
+
+        # Runtime environment - only if not default
+        if ENV['RAILS_ENV'] && ENV['RAILS_ENV'] != 'development'
+          env_vars['ENV'] = ENV['RAILS_ENV']
+        elsif ENV['RACK_ENV'] && ENV['RACK_ENV'] != 'development'
+          env_vars['ENV'] = ENV['RACK_ENV']
+        elsif ENV['NODE_ENV'] && ENV['NODE_ENV'] != 'development'
+          env_vars['ENV'] = ENV['NODE_ENV']
+        end
+
+        # Coverage - simplified
+        env_vars['COV'] = '1' if ENV['COVERAGE'] || ENV['SIMPLECOV']
+
+        # Test seed for reproducibility
+        env_vars['SEED'] = ENV['SEED'] if ENV['SEED']
+
+        env_vars
+      end
+
+      # Build execution flags in language-agnostic format
+      def build_execution_flags
+        flags = []
+        flags << "verbose" if @options[:verbose]
+        flags << "fails-only" if @options[:fails_only]
+        flags << "debug" if @options[:debug]
+        flags << "traces" if @options[:stack_traces] && !@options[:debug]  # debug implies traces
+        flags << "parallel" if @options[:parallel]
+        flags << "line-spec" if @options[:line_spec]
+        flags << "strict" if @options[:strict]
+        flags << "quiet" if @options[:quiet]
+        flags
+      end
+
+      # Get test discovery patterns in language-agnostic format
+      def get_test_discovery_patterns
+        patterns = []
+
+        # Ruby/Tryouts patterns
+        patterns.concat([
+          "**/*_try.rb",
+          "**/*.try.rb",
+          "try/**/*.rb",
+          "tryouts/**/*.rb"
+        ])
+
+        # TOPA-compatible patterns for other languages:
+        # Python: ["**/*_test.py", "**/test_*.py", "tests/**/*.py"]
+        # JavaScript: ["**/*.test.js", "**/*.spec.js", "__tests__/**/*.js"]
+        # Java: ["**/*Test.java", "**/Test*.java", "src/test/**/*.java"]
+        # Go: ["**/*_test.go"]
+        # C#: ["**/*Test.cs", "**/*Tests.cs"]
+        # PHP: ["**/*Test.php", "tests/**/*.php"]
+        # Rust: ["**/*_test.rs", "tests/**/*.rs"]
+
+        patterns
+      end
+
+      private
+
+      # Safely get git information with timeout protection
+      def safe_git_info
+        # Check if we're in a git repository
+        return {} unless File.directory?('.git') || system('git rev-parse --git-dir >/dev/null 2>&1')
+
+        require 'timeout'
+
+        Timeout.timeout(2) do
+          branch = `git rev-parse --abbrev-ref HEAD 2>/dev/null`.strip
+          commit = `git rev-parse --short HEAD 2>/dev/null`.strip
+
+          # Validate output to prevent injection
+          branch = nil unless branch =~ /\A[\w\-\/\.]+\z/
+          commit = nil unless commit =~ /\A[a-f0-9]+\z/i
+
+          { branch: branch, commit: commit }
+        end
+      rescue Timeout::Error, StandardError
+        # Return empty hash on any error (timeout, permission, etc.)
+        {}
+      end
     end
   end
 end

data/lib/tryouts/cli/opts.rb

@@ -10,20 +10,20 @@ class Tryouts
         Minitest:   Fresh context (each test isolated)
 
       Examples:
-        try test_try.rb                          # Tryouts test runner with shared context
-        try --rspec test_try.rb                  # RSpec with fresh context
-        try --direct --shared-context test_try.rb # Explicit shared context
-        try --generate-rspec test_try.rb         # Output RSpec code only
-        try --inspect test_try.rb                # Inspect file structure and validation
-        try --agent test_try.rb                  # Agent-optimized structured output
-        try --agent --agent-limit 10000 tests/  # Agent mode with 10K token limit
+        try test_try.rb                             # Tryouts test runner with shared context
+        try --rspec test_try.rb                     # RSpec with fresh context
+        try --direct --shared-context test_try.rb   # Explicit shared context
+        try --generate-rspec test_try.rb            # Output RSpec code only
+        try --inspect test_try.rb                   # Inspect file structure and validation
+        try --agent test_try.rb                     # Agent-optimized structured output
+        try --agent --agent-limit 10000 tests/      # Agent mode with 10K token limit
 
       Agent Output Modes:
-        --agent                                  # Structured, token-efficient output
-        --agent-focus summary                    # Show counts and problem files only
-        --agent-focus first-failure              # Show first failure per file
-        --agent-focus critical                   # Show errors/exceptions only
-        --agent-limit 1000                      # Limit output to 1000 tokens
+        --agent                                     # Structured, token-efficient output
+        --agent-focus summary                       # Show counts and problem files only
+        --agent-focus first-failure                 # Show first failure per file
+        --agent-focus critical                      # Show errors/exceptions only
+        --agent-limit 1000                          # Limit output to 1000 tokens
 
       File Naming & Organization:
         Files must end with '_try.rb' or '.try.rb' (e.g., auth_service_try.rb, user_model.try.rb)
@@ -37,7 +37,7 @@ class Tryouts
         #=> true  # this is the expected result
 
       File Structure (3 sections):
-        # Setup section (optional) - runs once before all tests
+        # Setup section (optional) - code before first testcase runs once before all tests
         @shared_var = "available to all test cases"
 
         ## TEST: Feature description
@@ -45,9 +45,9 @@ class Tryouts
         result = some_operation()
         #=> expected_value
 
-        # Teardown section (optional) - runs once after all tests
+        # Teardown section (optional) - code after last testcase runs once after all tests
 
-      Context Guidelines:
+      Execution Context:
         Shared Context (default): Instance variables persist across test cases
           - Use for: Integration testing, stateful scenarios, realistic workflows
           - Caution: Test order matters, state accumulates

data/lib/tryouts/expectation_evaluators/result_type.rb

@@ -40,6 +40,21 @@ class Tryouts
         # Try to evaluate in test context first, then fallback to global context for constants
         begin
           expected_class = eval_expectation_content(@expectation.content, expectation_result)
+
+          # If the evaluated result is not a class/module (e.g., shadowed constant),
+          # fall back to global context resolution
+          unless expected_class.is_a?(Class) || expected_class.is_a?(Module)
+            # Attempt to get the constant from global context
+            # This handles cases where a local variable/method shadows a class constant
+            constant_name = @expectation.content.strip
+            # Ensure the constant name is valid before attempting const_get
+            if constant_name =~ /\A[A-Z][\w:]*\z/
+              expected_class = Object.const_get(constant_name)
+            else
+              # If it's not a valid constant name, the evaluation failed
+              raise TypeError, "Expected a Class or Module, got #{expected_class.class}"
+            end
+          end
         rescue NameError => e
           # If we can't find the constant in test context, try global context
           # This is common for exception classes like ArgumentError, StandardError, etc.

data/lib/tryouts/file_processor.rb

@@ -1,6 +1,6 @@
 # lib/tryouts/file_processor.rb
 
-require_relative 'parsers/prism_parser'
+require_relative 'parsers/legacy_parser'
 require_relative 'parsers/enhanced_parser'
 require_relative 'test_executor'
 require_relative 'cli/modes/inspect'
@@ -84,7 +84,7 @@ class Tryouts
       when :enhanced
         EnhancedParser.new(file, options)
       when :prism
-        PrismParser.new(file, options)
+        LegacyParser.new(file, options)
       end
     end
 

data/lib/tryouts/parser_warning.rb

@@ -22,5 +22,15 @@ class Tryouts
         suggestion: "Use explicit '## Description' to clarify test structure"
       )
     end
+
+    def self.malformed_expectation(line_number:, syntax:, context:)
+      new(
+        type: :malformed_expectation,
+        message: "Malformed expectation syntax '#=#{syntax}>' at line #{line_number}",
+        line_number: line_number,
+        context: context,
+        suggestion: "Use valid expectation syntax like #=>, #==>, #=:>, #=!>, etc."
+      )
+    end
   end
 end