testament-probe-ruby 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 337fe8994c7ab64268d451d7b9f91e671251036eade4ed01905f42a10dd366f8
+  data.tar.gz: 5faf2fb1f40a1df3cbed28907b6871940ec05d498a284d955fa1b06fc2f78c18
+SHA512:
+  metadata.gz: 55f1e5ff0529789880754d15acd90be7ee2a58d4104983800d5c592175f74de4a97597e3e49f1009dcc0c6f2114ada4ca4c1687ff00e1730ab841cdacdd6763c
+  data.tar.gz: 4c6d77433a6e09c18b446672af24937812a71c7a49c0cdd31213d2a89b544c5ca9634b576806d74027a0107db3328dc5b6b68d705442d35fc8b0102d2b3d05d9

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Yudai Takada
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# testament-probe-ruby
+
+Small Ruby-side companion for collecting per-test line coverage and assertion
+trace evidence consumed by testament.
+
+## Usage
+
+Add the probe to the test process before tests run:
+
+```ruby
+require "testament/probe"
+```
+
+By default the probe writes `.testament/per-test-coverage.json` and
+`.testament/trace.json`. Set `TESTAMENT_PROBE_OUTPUT` or
+`TESTAMENT_TRACE_OUTPUT` to write another path.
+
+The trace output records executed project lines, lines executed while assertion
+methods are active, and recent executed lines observed when assertions begin.
+`TESTAMENT_PROJECT_ROOT` controls the project-root filter, and
+`TESTAMENT_TRACE_WINDOW` controls how many recent lines are attributed to an
+assertion.
+
+Line/call tracing adds noticeable overhead to the test run. Set
+`TESTAMENT_TRACE=0` to collect per-test line coverage without trace evidence.
+
+The probe installs hooks for RSpec and Minitest when those constants are loaded.
+Load the probe after the test framework is required.

data/lib/testament/probe/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Testament
+  module Probe
+    VERSION = "0.1.0"
+  end
+end

data/lib/testament/probe.rb

@@ -0,0 +1,248 @@
+require "coverage"
+require "fileutils"
+require "json"
+require_relative "probe/version"
+
+module Testament
+  module Probe
+    DEFAULT_OUTPUT = ".testament/per-test-coverage.json"
+    DEFAULT_TRACE_OUTPUT = ".testament/trace.json"
+    DEFAULT_TRACE_WINDOW = 200
+    ASSERTION_METHODS = %i[
+      expect should should_not to not_to to_not
+      assert assert_equal assert_same assert_nil assert_not_nil assert_empty
+      assert_includes assert_match assert_operator assert_predicate
+      assert_instance_of assert_kind_of assert_raises assert_throws
+      assert_output assert_silent
+      refute refute_equal refute_same refute_nil refute_empty
+      refute_includes refute_match refute_operator refute_predicate
+      refute_instance_of refute_kind_of
+      must_be must_equal must_include must_match must_raise must_be_nil
+      wont_be wont_equal wont_include wont_match wont_be_nil
+    ].freeze
+
+    class << self
+      def install!(
+        output: ENV.fetch("TESTAMENT_PROBE_OUTPUT", DEFAULT_OUTPUT),
+        trace_output: ENV.fetch("TESTAMENT_TRACE_OUTPUT", DEFAULT_TRACE_OUTPUT)
+      )
+        @output = output
+        @trace_output = trace_output
+        @root = File.expand_path(ENV.fetch("TESTAMENT_PROJECT_ROOT", Dir.pwd))
+        @probe_file = File.expand_path(__FILE__)
+        @trace_window = ENV.fetch("TESTAMENT_TRACE_WINDOW", DEFAULT_TRACE_WINDOW).to_i
+        @trace_enabled = ENV.fetch("TESTAMENT_TRACE", "1") != "0"
+        start_coverage
+        capture_coverage
+        install_rspec if defined?(RSpec)
+        install_minitest if defined?(Minitest::Test)
+        at_exit { write! }
+      end
+
+      def record(case_id)
+        start_coverage
+        start_trace
+        previous_case = @current_case
+        previous_recent_lines = @recent_lines
+        previous_assertion_depth = @assertion_depth
+        @recent_lines = []
+        @assertion_depth = 0
+        @current_case = case_id
+        trace_cases[case_id]
+        yield
+      ensure
+        merge_case(case_id, capture_coverage) if case_id
+        @recent_lines = previous_recent_lines || []
+        @assertion_depth = previous_assertion_depth || 0
+        @current_case = previous_case
+      end
+
+      def write!
+        output = @output || DEFAULT_OUTPUT
+        FileUtils.mkdir_p(File.dirname(output))
+        File.write(output, JSON.pretty_generate("cases" => cases))
+
+        trace_output = @trace_output || DEFAULT_TRACE_OUTPUT
+        FileUtils.mkdir_p(File.dirname(trace_output))
+        File.write(trace_output, JSON.pretty_generate("cases" => trace_cases))
+      end
+
+      private
+
+      def cases
+        @cases ||= {}
+      end
+
+      def trace_cases
+        @trace_cases ||= Hash.new do |cases, case_id|
+          cases[case_id] = { "executed" => {}, "checked" => {} }
+        end
+      end
+
+      def start_coverage
+        return false if coverage_running?
+
+        Coverage.start(lines: true)
+        true
+      end
+
+      def start_trace
+        return unless @trace_enabled
+
+        @tracepoint ||= TracePoint.new(:line, :call, :c_call, :return, :c_return) do |event|
+          case event.event
+          when :line
+            record_trace_line(event.path, event.lineno)
+          when :call, :c_call
+            enter_assertion if assertion_method?(event.method_id, event.defined_class)
+          when :return, :c_return
+            leave_assertion if assertion_method?(event.method_id, event.defined_class)
+          end
+        end
+        @tracepoint.enable unless @tracepoint.enabled?
+      end
+
+      def coverage_running?
+        Coverage.respond_to?(:running?) && Coverage.running?
+      end
+
+      def capture_coverage
+        Coverage.result(stop: false, clear: true)
+      rescue ArgumentError
+        Coverage.result
+      end
+
+      def merge_case(case_id, coverage)
+        normalized = normalize_coverage(coverage)
+        return if normalized.empty?
+
+        existing = cases.fetch(case_id, {})
+        normalized.each do |path, lines|
+          merged = (existing.fetch(path, []) + lines).uniq.sort
+          existing[path] = merged
+        end
+        cases[case_id] = existing
+      end
+
+      def record_trace_line(path, line)
+        return unless @current_case
+
+        path = normalize_trace_path(path)
+        return unless path
+
+        trace = trace_cases[@current_case]
+        append_trace_line(trace["executed"], path, line)
+        append_trace_line(trace["checked"], path, line) if assertion_active?
+        @recent_lines ||= []
+        @recent_lines << [path, line]
+        @recent_lines.shift while @recent_lines.length > trace_window
+      end
+
+      def enter_assertion
+        mark_recent_lines_checked
+        @assertion_depth = assertion_depth + 1
+      end
+
+      def leave_assertion
+        @assertion_depth = [assertion_depth - 1, 0].max
+      end
+
+      def assertion_active?
+        assertion_depth.positive?
+      end
+
+      def assertion_depth
+        @assertion_depth ||= 0
+      end
+
+      def mark_recent_lines_checked
+        return unless @current_case
+
+        checked = trace_cases[@current_case]["checked"]
+        @recent_lines.each do |path, line|
+          append_trace_line(checked, path, line)
+        end
+      end
+
+      def append_trace_line(files, path, line)
+        lines = files.fetch(path, [])
+        lines << line
+        files[path] = lines.uniq.sort
+      end
+
+      def normalize_trace_path(path)
+        return if path.nil? || path.empty?
+        return if path.start_with?("<")
+
+        expanded = File.expand_path(path)
+        root = @root || Dir.pwd
+        return if expanded == @probe_file
+        return unless expanded == root || expanded.start_with?("#{root}/")
+
+        expanded.delete_prefix("#{root}/")
+      end
+
+      def assertion_method?(method_id, defined_class)
+        return false unless ASSERTION_METHODS.include?(method_id)
+
+        owner = defined_class.to_s
+        owner.include?("RSpec") || owner.include?("Minitest") || owner.include?("Test::Unit")
+      end
+
+      def trace_window
+        [@trace_window || DEFAULT_TRACE_WINDOW, 1].max
+      end
+
+      def normalize_coverage(coverage)
+        coverage.each_with_object({}) do |(path, value), result|
+          lines = line_hits(value)
+          covered = lines.each_with_index.filter_map do |hits, index|
+            index + 1 if hits && hits.positive?
+          end
+          result[path] = covered unless covered.empty?
+        end
+      end
+
+      def line_hits(value)
+        if value.is_a?(Hash)
+          unless value.key?(:lines) || value.key?("lines")
+            warn_missing_line_coverage
+            return []
+          end
+          value.fetch(:lines, value.fetch("lines", []))
+        else
+          value || []
+        end
+      end
+
+      def warn_missing_line_coverage
+        return if @warned_missing_line_coverage
+
+        warn "testament probe: Coverage is active without lines mode; per-test coverage will be empty"
+        @warned_missing_line_coverage = true
+      end
+
+      def install_rspec
+        RSpec.configure do |config|
+          config.around(:each) do |example|
+            Testament::Probe.record(example.full_description) { example.run }
+          end
+        end
+      end
+
+      def install_minitest
+        return if Minitest::Test < MinitestIntegration
+
+        Minitest::Test.prepend(MinitestIntegration)
+      end
+    end
+
+    module MinitestIntegration
+      def run
+        Testament::Probe.record("#{self.class}##{name}") { super }
+      end
+    end
+  end
+end
+
+Testament::Probe.install!

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: testament-probe-ruby
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yudai Takada
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: Collects per-test line coverage and assertion trace evidence for the
+  testament test-quality analyzer.
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.md
+files:
+- LICENSE.txt
+- README.md
+- lib/testament/probe.rb
+- lib/testament/probe/version.rb
+homepage: https://github.com/ydah/testament
+licenses:
+- MIT
+metadata:
+  source_code_uri: https://github.com/ydah/testament
+  homepage_uri: https://github.com/ydah/testament
+  bug_tracker_uri: https://github.com/ydah/testament/issues
+  allowed_push_host: https://rubygems.org
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: Per-test Ruby coverage and trace probe for testament
+test_files: []