ask-rails-harness 0.1.1 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fbfa4dd4bde386eb1600fd036a12cad723ad335365b46cfadd97e040707674e9
-  data.tar.gz: a2af592dae8fec3fe8a6f4fefae6e252456b5dbbe510787fc86e7bc9066f60b2
+  metadata.gz: 6b7c1f9342c45ef3d4472772717fcc6ff302b35999e74806bd8964510f729f2f
+  data.tar.gz: 017c0f1b89629f435bde77c637c96e866caffb3be86f27d99adcf6431cc79cb3
 SHA512:
-  metadata.gz: 3dfc9d467a70c966cb3f4400605c7625bb1792d4f5da096aaba508dae32cf09429e4da281b91b8022d390f5dfa54cd9c5dc495dd550e0aab0a3c3b5daa3ff32f
-  data.tar.gz: 40589044b3cb0e691e08b036fdedc19ed8f3727f2e57af909a820e6ee2c442f2bde7c94313a8a18bd526a5b8ba9653e4acb8248342f81eed18477ba99a9c6624
+  metadata.gz: 4a3c0d99ae4b7f7725365827e4707fc56ca680bcee0fe7d0617c03bc7aa4e8c14039ee858016ad867648e4c5dec545acaec1b1b7934f6b3ed20d72d6a9d122db
+  data.tar.gz: 0b9c8b0b65b3baab555bde4c7e938c447b6b0527ea073a0abc3f9179d3ce8836e4f49221b601b20fd95446d33b40571f49f61bc25380baeba2cfe162a087b7bc

data/CHANGELOG.md

@@ -1,3 +1,51 @@
+## [0.2.1] — 2026-08-10
+
+### Fixed
+
+- **`run_tests` strips `RAILS_MAX_THREADS` for spawned test runs** — harness
+  servers may run with a deliberately small pool (e.g. `RAILS_MAX_THREADS=1`
+  in their MCP config). Without this, the cap leaked into `bin/rails test`
+  children, where it would serialize parallel tests on a single connection.
+  Test runs now always get the app's normal pool sizes.
+
+## [0.2.0] — 2026-08-10
+
+### Added
+
+- **`run_tests` tool** — runs the app's test suite and returns structured
+  results (summary counts plus per-test file/line/message for failures)
+  instead of raw terminal output. Detects minitest vs rspec (`Gemfile.lock` +
+  `spec/`), supports `file:`/`name:` filters, `failed_only:` reruns persisted
+  from the previous run (`tmp/test/.ask/last-failures.json`), and a `timeout:`
+  that kills the run and reports `timed_out` with the artifact path. Minitest
+  gets machine-readable results via a bundled reporter/plugin; rspec uses its
+  built-in JSON formatter.
+- **`MinitestJsonReporter` + minitest plugin** — `lib/minitest/ask_rails_harness_plugin.rb`
+  is auto-discovered by minitest and registers the JSON reporter only when
+  `ASK_TEST_JSON_PATH` is set (i.e. when the harness invoked the run), so
+  ordinary `bin/rails test` runs are untouched. Supports both the minitest 5
+  (`ask_rails_harness_plugin_init`) and minitest 6 (`plugin_ask_rails_harness_init`)
+  plugin init conventions. Full human output always lands at
+  `tmp/test/.ask/last-test.log`.
+
+### Fixed
+
+- **`ReadLog` crashed with `NoMethodError: undefined method 'env' for module Ask::Rails`** —
+  `read_log.rb` used bare `Rails.env`, which resolves lexically to the
+  `Ask::Rails` module inside the gem's own namespace. Now `::Rails.env`
+  (matching `query_database.rb`).
+- **Audit-log failure warnings were silently swallowed** — `audit_log.rb` used
+  the same bare-`Rails` lookup, so `defined?(Rails.logger)` was always false.
+  Now `::Rails.logger&.warn` with a nil-safe guard.
+
+## [0.1.1] — 2026-07-31
+
+### Changed
+
+- Use `Ask::Agent::Policies::Permissions` for environment hooks (requires
+  ask-agent >= 0.28.0).
+- Require `time` for time-stdlib methods (`iso8601`, `parse`).
+
 ## [0.1.0] — 2026-07-25
 
 ### Changed

data/lib/ask/rails/harness/audit_log.rb

@@ -166,8 +166,10 @@ module Ask
                VALUES (#{values.join(', ')})"
             )
           rescue StandardError => e
-            # Silently fail — audit log should never crash the caller
-            Rails.logger.warn("[ask-rails-harness] Audit log write failed: #{e.message}") if defined?(Rails.logger)
+            # Silently fail — audit log should never crash the caller.
+            # ::Rails avoids the bare-`Rails` constant resolving to Ask::Rails;
+            # `&.` guards against Rails.logger returning nil (no app booted).
+            ::Rails.logger&.warn("[ask-rails-harness] Audit log write failed: #{e.message}") if defined?(::Rails.logger)
           end
 
           # Reset cached table check (useful in tests)

data/lib/ask/rails/harness/minitest_json_reporter.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "minitest"
+require "json"
+
+module Ask
+  module Rails
+    module Harness
+      # Writes a machine-readable JSON summary of a minitest run.
+      #
+      # Registered by the ask_rails_harness minitest plugin when the
+      # ASK_TEST_JSON_PATH env var is set (see the run_tests tool). Produces
+      # stable structured results — test name, klass, source file/line, status,
+      # and a message head — so the harness never has to parse terminal output.
+      #
+      # The reporter is inert unless ASK_TEST_JSON_PATH is set, so ordinary
+      # `bin/rails test` runs in apps that ship the harness are unaffected.
+      class MinitestJsonReporter < Minitest::AbstractReporter
+        def initialize(path = ENV["ASK_TEST_JSON_PATH"])
+          super()
+          @path = path
+          @results = []
+        end
+
+        def record(result)
+          @results << result
+        end
+
+        def report
+          return if @path.nil? || @path.to_s.empty?
+          File.write(@path, JSON.pretty_generate(build_report))
+        end
+
+        # Never influences the process exit status — pass/fail is decided by
+        # minitest's own summary reporter.
+        def passed?
+          true
+        end
+
+        private
+
+        def build_report
+          tests = @results.map { |result| test_entry(result) }
+          {
+            "framework" => "minitest",
+            "run" => tests.size,
+            "failures" => tests.count { |t| t["status"] == "failed" },
+            "errors" => tests.count { |t| t["status"] == "error" },
+            "skips" => tests.count { |t| t["status"] == "skipped" },
+            "tests" => tests
+          }
+        end
+
+        def test_entry(result)
+          file, line = result.source_location
+          failure = result.failure
+          {
+            "name" => result.name,
+            "klass" => result.klass,
+            "file" => file,
+            "line" => line,
+            "time" => result.time,
+            "status" => status_of(result),
+            "message" => failure ? message_of(failure) : nil
+          }
+        end
+
+        def status_of(result)
+          return "skipped" if result.skipped?
+          return "error" if result.error?
+          return "passed" if result.passed?
+          "failed"
+        end
+
+        # First lines of the failure message — enough for an agent to act on
+        # without dumping full backtraces into the report.
+        def message_of(failure)
+          failure.message.to_s.lines.first(3).map(&:strip).reject(&:empty?).join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/ask/rails/harness/tools/read_log.rb

@@ -50,7 +50,9 @@ module Ask
 
           def resolve_log_path(custom_path)
             return rails_root.join(custom_path) if custom_path
-            rails_root.join("log", "#{Rails.env}.log")
+            # Use ::Rails (not Rails) — bare `Rails` inside the Ask::Rails::*
+            # namespace resolves to the Ask::Rails module itself.
+            rails_root.join("log", "#{::Rails.env}.log")
           end
 
           # Read from rotated archives too: log/production.log, .1, .2.gz, etc.

data/lib/ask/rails/harness/tools/run_tests.rb

@@ -0,0 +1,249 @@
+# frozen_string_literal: true
+
+require "json"
+require "timeout"
+
+module Ask
+  module Rails
+    module Harness
+      module Tools
+        class RunTests < Ask::Rails::Harness::Tool
+          description "Run the app's test suite and return structured results — summary counts " \
+                      "plus per-test file/line/message for failures, never raw terminal output. " \
+                      "Minitest gets a JSON reporter via the bundled minitest plugin; rspec uses " \
+                      "its built-in JSON formatter. Rerun only the previous run's failures with " \
+                      "failed_only."
+
+          param :file, type: :string, desc: "Test file path(s) relative to the app root (comma-separated for multiple)", required: false
+          param :name, type: :string, desc: "Test name pattern: minitest --name (string or /regex/), rspec -e", required: false
+          param :failed_only, type: :boolean, desc: "Rerun only the tests that failed in the previous run", required: false
+          param :timeout, type: :integer, desc: "Max seconds to wait before killing the run (default 300)", required: false
+
+          DEFAULT_TIMEOUT = 300
+          ARTIFACT_DIR = %w[tmp test .ask].freeze
+
+          def execute(file: nil, name: nil, failed_only: false, timeout: DEFAULT_TIMEOUT)
+            files = split_files(file)
+            framework = detect_framework
+
+            failed_tests = failed_only ? load_failed_tests : nil
+            if failed_only && failed_tests.empty?
+              return Ask::Result.failure("No failed tests from the previous run to rerun.")
+            end
+
+            artifact_dir = rails_root.join(*ARTIFACT_DIR).tap(&:mkpath)
+            log_path = artifact_dir.join("last-test.log")
+            json_path = artifact_dir.join("last-test.json")
+            status_path = artifact_dir.join("last-failures.json")
+
+            command, env = build_command(framework, files, name, failed_tests, json_path)
+            outcome = run(command, env, log_path, timeout)
+
+            results = parse_results(framework, json_path)
+            unless results
+              # A killed run can't produce results — report it structurally
+              # instead of failing, so the agent still gets the artifact path.
+              return Ask::Result.ok(data: timed_out_report(framework, command, env, outcome, status_path, log_path)) if outcome[:timed_out]
+
+              return Ask::Result.failure(
+                "Test run finished without machine-readable results (#{framework}); " \
+                "full output at #{rel(log_path)}"
+              )
+            end
+
+            report = build_report(framework, command, env, outcome, results, status_path, log_path)
+            Ask::Result.ok(data: report)
+          end
+
+          private
+
+          def split_files(file)
+            return [] if file.nil? || file.to_s.strip.empty?
+            file.split(",").map(&:strip).reject(&:empty?)
+          end
+
+          # Prefer rspec when it's in the bundle AND a spec/ dir exists;
+          # otherwise assume minitest (the Rails default).
+          def detect_framework
+            lockfile = rails_root.join("Gemfile.lock")
+            rspec = lockfile.exist? && lockfile.read.include?("rspec-rails")
+            rspec && rails_root.join("spec").directory? ? :rspec : :minitest
+          end
+
+          def build_command(framework, files, name, failed_tests, json_path)
+            if framework == :rspec
+              args = ["bundle", "exec", "rspec"]
+              args.concat(files)
+              args.concat(failed_tests.map { |t| "#{t[:file]}:#{t[:line]}" }) if failed_tests
+              args.concat(["-e", name]) if name
+              args.concat(["--format", "json", "--out", json_path.to_s])
+              [args, {}]
+            else
+              # Rails' `rails test` passes CLI args straight to minitest (files
+              # are required by Rails' runner, options parsed by minitest). The
+              # JSON reporter is injected via RUBYOPT: minitest 6 dropped both
+              # plugin auto-discovery and the -r option, so the plugin file is
+              # required by absolute path at process start (it pushes its
+              # extension; init_plugins registers the reporter later).
+              args = ["bin/rails", "test"]
+              args.concat(files)
+              args.concat(["-n", name]) if name
+              args.concat(["-n", name_pattern(failed_tests)]) if failed_tests
+              # minitest 6 dropped both plugin auto-discovery and the -r
+              # option. Activate the app's bundle first, then require the
+              # plugin by absolute path — it pushes its extension, and
+              # init_plugins registers the JSON reporter later.
+              env = {
+                "ASK_TEST_JSON_PATH" => json_path.to_s,
+                "RUBYOPT" => "-rbundler/setup -r#{minitest_plugin_path}"
+              }
+              [args, env]
+            end
+          end
+
+          def minitest_plugin_path
+            spec = Gem.loaded_specs["ask-rails-harness"]
+            spec ||= Gem::Specification.find_by_name("ask-rails-harness")
+            File.join(spec.full_gem_path, "lib", "minitest", "ask_rails_harness_plugin.rb")
+          end
+
+          # Minitest --name accepts a regexp; alternation runs exactly the
+          # failed tests.
+          def name_pattern(failed_tests)
+            escaped = failed_tests.map { |t| Regexp.escape(t[:test_name]) }
+            "/#{escaped.join('|')}/"
+          end
+
+          def run(command, env, log_path, timeout)
+            # The harness server may run with a deliberately small pool
+            # (e.g. RAILS_MAX_THREADS=1 in its MCP config). Test runs are a
+            # separate concern — let them use the app's normal pool sizes.
+            child_env = env.merge("RAILS_MAX_THREADS" => nil)
+            pid = Process.spawn(child_env, *command, chdir: rails_root.to_s,
+                                out: [log_path.to_s, "w"], err: [:child, :out])
+            status = nil
+            timed_out = false
+            begin
+              Timeout.timeout(timeout) { status = Process.wait2(pid).last }
+            rescue Timeout::Error
+              timed_out = true
+              begin
+                Process.kill("TERM", pid)
+                sleep 0.2
+                Process.kill("KILL", pid)
+              rescue Errno::ESRCH, Errno::EPERM
+                # Process already gone — nothing to kill.
+              end
+              status = Process.wait2(pid).last rescue nil
+            end
+            { exit_status: status&.exitstatus, timed_out: timed_out }
+          end
+
+          def build_report(framework, command, env, outcome, results, status_path, log_path)
+            summary = results[:summary]
+            failed_tests = results[:failed_tests]
+            persist_failed_tests(framework, failed_tests, status_path)
+
+            {
+              framework: framework.to_s,
+              command: full_command(command, env),
+              exit_status: outcome[:exit_status],
+              timed_out: outcome[:timed_out],
+              summary: summary,
+              failed_tests: failed_tests,
+              artifact: rel(log_path),
+              next: summary[:failures] + summary[:errors] > 0 ? "run_tests(failed_only: true)" : nil
+            }
+          end
+
+          def timed_out_report(framework, command, env, outcome, status_path, log_path)
+            persist_failed_tests(framework, [], status_path)
+            {
+              framework: framework.to_s,
+              command: full_command(command, env),
+              exit_status: outcome[:exit_status],
+              timed_out: true,
+              summary: nil,
+              failed_tests: nil,
+              artifact: rel(log_path),
+              next: nil
+            }
+          end
+
+          # The minitest options are part of the command itself (Rails passes
+          # CLI args through to minitest), so the reported command is the full
+          # invocation.
+          def full_command(command, env)
+            command.join(" ")
+          end
+
+          def parse_results(framework, json_path)
+            return nil unless json_path.exist?
+            payload = JSON.parse(json_path.read)
+            framework == :rspec ? parse_rspec(payload) : parse_minitest(payload)
+          rescue JSON::ParserError
+            nil
+          end
+
+          def parse_minitest(payload)
+            tests = payload["tests"] || []
+            summary = {
+              run: payload.fetch("run", tests.size),
+              failures: payload.fetch("failures", 0),
+              errors: payload.fetch("errors", 0),
+              skips: payload.fetch("skips", 0)
+            }
+            failed_tests = tests.filter_map do |t|
+              next unless %w[failed error].include?(t["status"])
+              { file: t["file"], test_name: t["name"], line: t["line"], message: t["message"] }
+            end
+            { summary: summary, failed_tests: failed_tests }
+          end
+
+          def parse_rspec(payload)
+            examples = payload["examples"] || []
+            summary_payload = payload["summary"] || {}
+            failed_examples = examples.select { |e| e["status"] == "failed" }
+            pending = examples.count { |e| e["status"] == "pending" }
+            summary = {
+              run: summary_payload.fetch("example_count", examples.size),
+              failures: summary_payload.fetch("failure_count", failed_examples.size),
+              errors: 0,
+              skips: summary_payload.fetch("pending_count", pending)
+            }
+            failed_tests = failed_examples.map do |e|
+              exception = e["exception"] || {}
+              {
+                file: e["file_path"],
+                test_name: e["full_description"],
+                line: e["line_number"],
+                message: exception["message"]
+              }
+            end
+            { summary: summary, failed_tests: failed_tests }
+          end
+
+          def persist_failed_tests(framework, failed_tests, status_path)
+            status_path.write(JSON.pretty_generate(framework: framework.to_s, failed_tests: failed_tests))
+          end
+
+          def load_failed_tests
+            path = rails_root.join(*ARTIFACT_DIR, "last-failures.json")
+            return [] unless path.exist?
+            JSON.parse(path.read).fetch("failed_tests", []).map { |t| symbolize_keys(t) }
+          rescue JSON::ParserError
+            []
+          end
+
+          def symbolize_keys(hash)
+            hash.each_with_object({}) { |(k, v), h| h[k.to_sym] = v }
+          end
+
+          def rel(path)
+            path.relative_path_from(rails_root).to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ask/rails/harness/version.rb

@@ -3,7 +3,7 @@
 module Ask
   module Rails
     module Harness
-      VERSION = "0.1.1"
+      VERSION = "0.2.1"
     end
   end
 end

data/lib/ask/rails/harness.rb

@@ -174,6 +174,8 @@ require_relative "harness/tools/read_model"
 require_relative "harness/tools/read_log"
 require_relative "harness/tools/schema_graph"
 require_relative "harness/tools/route_inspector"
+require_relative "harness/tools/run_tests"
+require_relative "harness/minitest_json_reporter"
 
 # Railtie is loaded only when Rails is fully available
 if defined?(::Rails::Railtie)
@@ -186,5 +188,5 @@ Ask::Rails::Harness::CORE_RAILS_TOOLS = [
   Ask::Rails::Harness::Tools::SearchCodebase, Ask::Rails::Harness::Tools::ReadRoutes,
   Ask::Rails::Harness::Tools::QueryDatabase, Ask::Rails::Harness::Tools::ReadModel,
   Ask::Rails::Harness::Tools::ReadLog, Ask::Rails::Harness::Tools::SchemaGraph,
-  Ask::Rails::Harness::Tools::RouteInspector
+  Ask::Rails::Harness::Tools::RouteInspector, Ask::Rails::Harness::Tools::RunTests
 ].freeze

data/lib/minitest/ask_rails_harness_plugin.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# Minitest plugin for ask-rails-harness.
+#
+# Registers the JSON reporter when a run was started by the harness's
+# run_tests tool (ASK_TEST_JSON_PATH set); ordinary test runs are untouched.
+#
+# Loading differs by minitest version:
+#   - minitest 5 auto-discovers `minitest/*_plugin.rb` files and calls
+#     init_plugins with `#{name}_plugin_init`.
+#   - minitest 6 dropped auto-discovery (and the -r option); the run_tests
+#     tool injects this file via RUBYOPT="-r <absolute path>" and
+#     init_plugins dispatches `plugin_#{name}_init` instead.
+# Either way, pushing the extension below (idempotent) makes init_plugins
+# dispatch to the matching init method after the composite reporter exists —
+# the only reliable point to append a reporter.
+# __dir__-based requires keep the file loadable before Bundler.setup.
+require File.expand_path("../ask/rails/harness/minitest_json_reporter", __dir__)
+
+module Minitest
+  # Minitest 6 convention: init_plugins calls plugin_#{name}_init.
+  def self.plugin_ask_rails_harness_init(options)
+    register_ask_rails_harness_json_reporter(options)
+  end
+
+  # Minitest 5 convention: init_plugins calls #{name}_plugin_init.
+  def self.ask_rails_harness_plugin_init(options)
+    register_ask_rails_harness_json_reporter(options)
+  end
+
+  def self.register_ask_rails_harness_json_reporter(_options)
+    path = ENV["ASK_TEST_JSON_PATH"]
+    return if path.nil? || path.to_s.empty?
+
+    Minitest.reporter << Ask::Rails::Harness::MinitestJsonReporter.new(path)
+  end
+end
+
+Minitest.extensions << "ask_rails_harness" unless Minitest.extensions.include?("ask_rails_harness")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-rails-harness
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
 platform: ruby
 authors:
 - Kaka Ruto
@@ -160,6 +160,7 @@ files:
 - lib/ask/rails/harness/configuration.rb
 - lib/ask/rails/harness/engine.rb
 - lib/ask/rails/harness/environment_permissions.rb
+- lib/ask/rails/harness/minitest_json_reporter.rb
 - lib/ask/rails/harness/persistence.rb
 - lib/ask/rails/harness/railtie.rb
 - lib/ask/rails/harness/service_discovery.rb
@@ -171,6 +172,7 @@ files:
 - lib/ask/rails/harness/tools/read_routes.rb
 - lib/ask/rails/harness/tools/route_inspector.rb
 - lib/ask/rails/harness/tools/run_command.rb
+- lib/ask/rails/harness/tools/run_tests.rb
 - lib/ask/rails/harness/tools/schema_graph.rb
 - lib/ask/rails/harness/tools/search_codebase.rb
 - lib/ask/rails/harness/version.rb
@@ -181,6 +183,7 @@ files:
 - lib/generators/ask/rails/harness/install/templates/audit_log_migration.rb
 - lib/generators/ask/rails/harness/install/templates/initializer.rb
 - lib/generators/ask/rails/harness/install/templates/migration.rb
+- lib/minitest/ask_rails_harness_plugin.rb
 homepage: https://github.com/ask-rb/ask-rails-harness
 licenses:
 - MIT