selective-ruby-core 0.2.9 → 0.2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5d925ba84e14e58006c41c23c87f716b46f84ccd37df32b158b1340e3ad7309
-  data.tar.gz: 43c0d8bd524fc6b59bb829c211099dd56cd6d19540ddc682abad1a40fd76298b
+  metadata.gz: 9f6dbb5b565e9daa33dd9734e130efafcbb431372ee74eafc3b1e61356b94f2a
+  data.tar.gz: 9a4252a640d2a396d6d8b6cde0817566c9940ff569bceeb049260a062f6b9e73
 SHA512:
-  metadata.gz: 36103afe3d448cb4fb17d9bfdaebc93f3ac64a57e89b793f24df570ec107e4d99daa54e9f97935e9583c06b52ff07699b1391404d253f148f2d188c3f8c9e651
-  data.tar.gz: 005ee1f9f38d12e87a609465b77933fb4ad3ac7a77c1aa28a7f191a3257c07df38597342f756d8318b577b5df1eef7ba1e75cbcb8b90df032b1c133b1c2c95bc
+  metadata.gz: 5db64c1685e6c52cf3bca4d08c27b5cc3461f2162b8e9d587fd87872c7f73012d96a14c2260d7b2e6650661e6a07163a0ed379aa6019459beb91591e4ea79dcb
+  data.tar.gz: 4f0755272db64264a15727ce35a1d8a896953a0f192617d9b9eff0afbe46b3d8f7f1211e11de45cc4b9987e91d6b5092a6636024e6fefe1a240e585e63da2984

data/Rakefile

@@ -5,6 +5,26 @@ require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
+# Builds the optional test map tracer into lib/ for development and CI.
+# Installed gems build it through `spec.extensions` instead.
+desc "Compile the native test map tracer"
+task :compile do
+  require "fileutils"
+  require "rbconfig"
+  ext_dir = File.expand_path("ext/selective_tracer", __dir__)
+  build_dir = File.expand_path("tmp/selective_tracer", __dir__)
+  FileUtils.mkdir_p(build_dir)
+  Dir.chdir(build_dir) do
+    # extconf must not inherit this process's bundle; it only needs mkmf.
+    Bundler.with_unbundled_env do
+      sh RbConfig.ruby, File.join(ext_dir, "extconf.rb")
+      sh "make"
+    end
+    built = Dir["selective_tracer.{#{RbConfig::CONFIG["DLEXT"]},bundle,so}"].first
+    FileUtils.cp(built, File.expand_path("lib", __dir__)) if built
+  end
+end
+
 require "standard/rake"
 
 task default: %i[spec standard]

data/ext/selective_tracer/extconf.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# The tracer is optional. A customer without a C toolchain (or on an
+# unsupported Ruby) must still be able to install selective-ruby-core and run
+# their suite exactly as before, so any failed check writes a no-op Makefile
+# instead of failing `gem install`. That includes compiling the tracer itself
+# here first: an error in `make` would fail the install, so a Ruby whose
+# headers the source doesn't build against must be caught before it. At
+# runtime the missing library simply means "this runner can't record a test
+# map".
+
+require "mkmf"
+
+def write_noop_makefile(reason)
+  warn "selective_tracer: skipping native build (#{reason}); test map recording will be unavailable"
+  File.write("Makefile", <<~MAKE)
+    all:
+    \t@true
+    install:
+    \t@true
+    clean:
+    \t@true
+  MAKE
+end
+
+begin
+  if RUBY_ENGINE != "ruby"
+    write_noop_makefile("#{RUBY_ENGINE} is not supported")
+  elsif !have_header("ruby/debug.h") || !have_func("rb_profile_frames", "ruby/debug.h")
+    write_noop_makefile("required VM hook APIs are missing")
+  else
+    $CFLAGS << " -O2 -std=c99 -Wall -Wno-unused-parameter" # standard:disable Style/GlobalVars -- mkmf's interface
+    if try_compile(File.read(File.expand_path("selective_tracer.c", __dir__)))
+      create_makefile("selective_tracer")
+    else
+      write_noop_makefile("the tracer does not compile on Ruby #{RUBY_VERSION}; see mkmf.log")
+    end
+  end
+rescue => e
+  write_noop_makefile(e.message)
+end

data/ext/selective_tracer/selective_tracer.c

@@ -0,0 +1,336 @@
+// selective_tracer — records which source files a single test executes.
+//
+// A test map is built from one question asked of every test: "which files
+// did you run?". This extension answers it with two raw VM event hooks that
+// never call back into Ruby while the test runs:
+//
+//   * RUBY_EVENT_LINE | CALL | B_CALL, registered globally so work done on
+//     *any* thread is
+//     attributed to the running test. Capybara drives a Puma server thread;
+//     a hook scoped to the test's own thread (what rotoscope does) would map
+//     every system test to nothing and let selection skip them wrongly.
+//     Selective runs one test at a time per process, so "everything that ran
+//     between start and stop" is the right attribution. Stray background
+//     threads can only add files, which makes selection more conservative,
+//     never less. CALL and B_CALL are not redundant with LINE: an endless
+//     method (`def total = price * qty`) and a one-line block executed from
+//     another file emit no line event of their own, so a line-only hook never
+//     ties a test to the file they live in.
+//
+//   * RUBY_INTERNAL_EVENT_NEWOBJ (optional), which records the class of every
+//     plain object allocated. A class whose file only ran at boot — an
+//     ActiveRecord model with no methods of its own — executes no lines
+//     during a test, so line events alone would never tie the test to it.
+//     The classes are resolved to source files in Ruby after the test stops.
+//
+// Why not Ruby's Coverage module: it is process-global, so it cannot run next
+// to SimpleCov, and while it is enabled Bootsnap cannot use its compile cache.
+// Why not TracePoint from Ruby: every event re-enters the interpreter. The
+// design follows what Datadog learned building datadog-ci-rb (line hook with
+// pointer caching, allocation tracking for code-less classes), written for
+// Selective's needs: several project roots, several ignored prefixes, and no
+// Ruby API use inside the allocation hook.
+
+#include <ruby.h>
+#include <ruby/debug.h>
+#include <ruby/st.h>
+
+#include <stdbool.h>
+#include <string.h>
+
+#define PATH_CACHE_SIZE 1024   // power of two
+#define KLASS_CACHE_SIZE 4096  // power of two
+#define MAX_PREFIXES 32
+#define EXEC_EVENTS (RUBY_EVENT_LINE | RUBY_EVENT_CALL | RUBY_EVENT_B_CALL)
+
+struct prefix_list {
+  int count;
+  char *values[MAX_PREFIXES];
+  long lengths[MAX_PREFIXES];
+};
+
+struct tracer {
+  // { path(String) => true } for the test being recorded.
+  VALUE files;
+
+  // Absolute paths are kept only under one of `roots` and outside every
+  // `ignored` prefix (bundled gems, Selective's own gems, tmp dirs).
+  struct prefix_list roots;
+  struct prefix_list ignored;
+
+  // Consecutive line events almost always come from the same file, and the
+  // path pointer of an iseq is stable, so a pointer comparison settles most
+  // events. The strings are marked so a freed path's address can't be reused
+  // for a different file and wrongly treated as "already seen".
+  VALUE last_path;
+  VALUE path_cache[PATH_CACHE_SIZE];
+
+  bool allocations;
+  bool running;
+
+  // Classes allocated during the test. Keys are class VALUEs, pinned by mark.
+  st_table *klasses;
+  VALUE last_klass;
+  VALUE klass_cache[KLASS_CACHE_SIZE];
+};
+
+static int mark_klass_i(st_data_t key, st_data_t _value, st_data_t _arg) {
+  rb_gc_mark((VALUE)key);
+  return ST_CONTINUE;
+}
+
+static void tracer_mark(void *ptr) {
+  struct tracer *t = ptr;
+  rb_gc_mark(t->files);
+  rb_gc_mark(t->last_path);
+  for (int i = 0; i < PATH_CACHE_SIZE; i++) rb_gc_mark(t->path_cache[i]);
+  rb_gc_mark(t->last_klass);
+  for (int i = 0; i < KLASS_CACHE_SIZE; i++) rb_gc_mark(t->klass_cache[i]);
+  if (t->klasses) st_foreach(t->klasses, mark_klass_i, 0);
+}
+
+static void free_prefixes(struct prefix_list *list) {
+  for (int i = 0; i < list->count; i++) xfree(list->values[i]);
+  list->count = 0;
+}
+
+static void tracer_free(void *ptr) {
+  struct tracer *t = ptr;
+  // No hook cleanup needed: the VM marks every registered hook's data, so a
+  // tracer that is still running is reachable and can't be freed.
+  free_prefixes(&t->roots);
+  free_prefixes(&t->ignored);
+  if (t->klasses) st_free_table(t->klasses);
+  xfree(t);
+}
+
+static size_t tracer_memsize(const void *ptr) {
+  return sizeof(struct tracer);
+}
+
+static const rb_data_type_t tracer_type = {
+    .wrap_struct_name = "selective_tracer",
+    .function = {.dmark = tracer_mark, .dfree = tracer_free, .dsize = tracer_memsize},
+    .flags = RUBY_TYPED_FREE_IMMEDIATELY};
+
+static void reset_caches(struct tracer *t) {
+  t->last_path = Qnil;
+  for (int i = 0; i < PATH_CACHE_SIZE; i++) t->path_cache[i] = Qnil;
+  t->last_klass = Qnil;
+  for (int i = 0; i < KLASS_CACHE_SIZE; i++) t->klass_cache[i] = Qnil;
+}
+
+static VALUE tracer_alloc(VALUE klass) {
+  struct tracer *t;
+  VALUE self = TypedData_Make_Struct(klass, struct tracer, &tracer_type, t);
+  t->files = Qnil;
+  t->roots.count = 0;
+  t->ignored.count = 0;
+  t->allocations = false;
+  t->running = false;
+  t->klasses = NULL;
+  reset_caches(t);
+  t->files = rb_hash_new();
+  t->klasses = st_init_numtable();
+  return self;
+}
+
+static struct tracer *get_tracer(VALUE self) {
+  struct tracer *t;
+  TypedData_Get_Struct(self, struct tracer, &tracer_type, t);
+  return t;
+}
+
+static bool has_dir_prefix(const char *path, long path_len, const char *prefix, long prefix_len) {
+  if (prefix_len > path_len || memcmp(prefix, path, prefix_len) != 0) return false;
+  // "/app" must not match "/application"; a prefix given with a trailing
+  // slash already carries the boundary.
+  return prefix_len == path_len || prefix[prefix_len - 1] == '/' || path[prefix_len] == '/';
+}
+
+static bool path_included(struct tracer *t, const char *path, long len) {
+  if (len == 0) return false;
+
+  if (path[0] != '/') {
+    // Relative paths ("./spec/x_spec.rb" from `load`) are resolved and
+    // re-checked in Ruby. Pseudo-paths like "(eval at ...)" and
+    // "<internal:kernel>" are never project files.
+    return path[0] != '(' && path[0] != '<';
+  }
+
+  bool in_root = false;
+  for (int i = 0; i < t->roots.count; i++) {
+    if (has_dir_prefix(path, len, t->roots.values[i], t->roots.lengths[i])) {
+      in_root = true;
+      break;
+    }
+  }
+  if (!in_root) return false;
+
+  for (int i = 0; i < t->ignored.count; i++) {
+    if (has_dir_prefix(path, len, t->ignored.values[i], t->ignored.lengths[i])) return false;
+  }
+  return true;
+}
+
+static inline size_t cache_slot(uintptr_t ptr, size_t size) {
+  return ((ptr >> 4) ^ (ptr >> 12)) & (size - 1);
+}
+
+static void on_exec(rb_event_flag_t event, VALUE data, VALUE self, ID id, VALUE klass) {
+  struct tracer *t = RTYPEDDATA_DATA(data);
+
+  const char *current = rb_sourcefile();
+  if (current == NULL) return;
+
+  if (t->last_path != Qnil && RSTRING_PTR(t->last_path) == current) return;
+
+  size_t slot = cache_slot((uintptr_t)current, PATH_CACHE_SIZE);
+  VALUE cached = t->path_cache[slot];
+  if (cached != Qnil && RSTRING_PTR(cached) == current) {
+    t->last_path = cached;
+    return;
+  }
+
+  // First time this test sees this file: fetch the frame's path String so
+  // the pointer we cache is owned by a marked object.
+  VALUE frame;
+  if (rb_profile_frames(0, 1, &frame, NULL) != 1) return;
+
+  VALUE path = rb_profile_frame_path(frame);
+  if (NIL_P(path) || !RB_TYPE_P(path, T_STRING)) return;
+
+  t->last_path = path;
+  t->path_cache[slot] = path;
+
+  if (path_included(t, RSTRING_PTR(path), RSTRING_LEN(path))) {
+    rb_hash_aset(t->files, path, Qtrue);
+  }
+}
+
+// Runs inside object allocation: no Ruby method calls and no Ruby object
+// allocation are allowed here. rb_obj_class, rb_mod_name and st_* do neither.
+static void on_newobj(VALUE data, rb_trace_arg_t *targ) {
+  VALUE obj = rb_tracearg_object(targ);
+  enum ruby_value_type type = rb_type(obj);
+  if (type != RUBY_T_OBJECT && type != RUBY_T_STRUCT) return;
+
+  VALUE klass = rb_obj_class(obj);
+  if (klass == 0 || NIL_P(klass)) return;
+
+  struct tracer *t = RTYPEDDATA_DATA(data);
+  if (t->last_klass == klass) return;
+
+  size_t slot = cache_slot((uintptr_t)klass, KLASS_CACHE_SIZE);
+  if (t->klass_cache[slot] == klass) {
+    t->last_klass = klass;
+    return;
+  }
+
+  t->last_klass = klass;
+  t->klass_cache[slot] = klass;
+
+  if (st_is_member(t->klasses, (st_data_t)klass)) return;
+  if (NIL_P(rb_mod_name(klass))) return;  // anonymous class: no file to resolve
+
+  st_insert(t->klasses, (st_data_t)klass, 1);
+}
+
+static void load_prefixes(struct prefix_list *list, VALUE ary, const char *name) {
+  if (NIL_P(ary)) return;
+  Check_Type(ary, T_ARRAY);
+  long n = RARRAY_LEN(ary);
+  if (n > MAX_PREFIXES) rb_raise(rb_eArgError, "at most %d %s are supported", MAX_PREFIXES, name);
+
+  for (long i = 0; i < n; i++) {
+    VALUE s = rb_ary_entry(ary, i);
+    Check_Type(s, T_STRING);
+    long len = RSTRING_LEN(s);
+    if (len == 0) continue;
+    char *copy = ALLOC_N(char, len + 1);
+    memcpy(copy, RSTRING_PTR(s), len);
+    copy[len] = '\0';
+    list->values[list->count] = copy;
+    list->lengths[list->count] = len;
+    list->count++;
+  }
+}
+
+// NativeTracer.new(roots: [...], ignored: [...], allocations: true/false)
+static VALUE tracer_initialize(int argc, VALUE *argv, VALUE self) {
+  VALUE opts;
+  rb_scan_args(argc, argv, "1", &opts);
+  Check_Type(opts, T_HASH);
+
+  struct tracer *t = get_tracer(self);
+  free_prefixes(&t->roots);
+  free_prefixes(&t->ignored);
+
+  load_prefixes(&t->roots, rb_hash_aref(opts, ID2SYM(rb_intern("roots"))), "roots");
+  load_prefixes(&t->ignored, rb_hash_aref(opts, ID2SYM(rb_intern("ignored"))), "ignored prefixes");
+  if (t->roots.count == 0) rb_raise(rb_eArgError, "roots must contain at least one path");
+
+  t->allocations = RTEST(rb_hash_aref(opts, ID2SYM(rb_intern("allocations"))));
+  return self;
+}
+
+static VALUE tracer_start(VALUE self) {
+  struct tracer *t = get_tracer(self);
+  if (t->running) rb_raise(rb_eRuntimeError, "tracer is already running");
+
+  rb_hash_clear(t->files);
+  st_clear(t->klasses);
+  reset_caches(t);
+
+  rb_add_event_hook2((rb_event_hook_func_t)on_exec, EXEC_EVENTS, self, RUBY_EVENT_HOOK_FLAG_SAFE);
+  if (t->allocations) {
+    rb_add_event_hook2((rb_event_hook_func_t)on_newobj, RUBY_INTERNAL_EVENT_NEWOBJ, self,
+                       RUBY_EVENT_HOOK_FLAG_SAFE | RUBY_EVENT_HOOK_FLAG_RAW_ARG);
+  }
+  t->running = true;
+  return self;
+}
+
+static int collect_klass_i(st_data_t key, st_data_t _value, st_data_t ary) {
+  rb_ary_push((VALUE)ary, (VALUE)key);
+  return ST_CONTINUE;
+}
+
+// Returns [files, classes]: the Hash of executed paths and the Array of named
+// classes allocated during the test. Both are fresh objects owned by Ruby.
+static VALUE tracer_stop(VALUE self) {
+  struct tracer *t = get_tracer(self);
+  if (!t->running) rb_raise(rb_eRuntimeError, "tracer is not running");
+
+  rb_remove_event_hook_with_data((rb_event_hook_func_t)on_exec, self);
+  if (t->allocations) rb_remove_event_hook_with_data((rb_event_hook_func_t)on_newobj, self);
+  t->running = false;
+
+  VALUE files = t->files;
+  t->files = rb_hash_new();
+
+  VALUE klasses = rb_ary_new_capa((long)t->klasses->num_entries);
+  st_foreach(t->klasses, collect_klass_i, (st_data_t)klasses);
+  st_clear(t->klasses);
+  reset_caches(t);
+
+  return rb_assoc_new(files, klasses);
+}
+
+static VALUE tracer_running_p(VALUE self) {
+  return get_tracer(self)->running ? Qtrue : Qfalse;
+}
+
+void Init_selective_tracer(void) {
+  VALUE mSelective = rb_define_module("Selective");
+  VALUE mRuby = rb_define_module_under(mSelective, "Ruby");
+  VALUE mCore = rb_define_module_under(mRuby, "Core");
+  VALUE mTestMap = rb_define_module_under(mCore, "TestMap");
+  VALUE cTracer = rb_define_class_under(mTestMap, "NativeTracer", rb_cObject);
+
+  rb_define_alloc_func(cTracer, tracer_alloc);
+  rb_define_method(cTracer, "initialize", tracer_initialize, -1);
+  rb_define_method(cTracer, "start", tracer_start, 0);
+  rb_define_method(cTracer, "stop", tracer_stop, 0);
+  rb_define_method(cTracer, "running?", tracer_running_p, 0);
+}

data/lib/selective/ruby/core/controller.rb

@@ -24,10 +24,9 @@ module Selective
 
         def initialize(runner_class, runner_args, debug: false, log: false)
           @debug = debug
+          @log = log
           @runner = runner_class.new(runner_args, method(:test_case_callback))
           @retries = 0
-          @runner_id = safe_filename(get_runner_id)
-          @logger = init_logger(log)
         end
 
         def start(reconnect: false)
@@ -67,7 +66,19 @@ module Selective
 
         private
 
-        attr_reader :runner, :pipe, :transport_pid, :retries, :logger, :runner_id, :diff
+        attr_reader :runner, :pipe, :transport_pid, :retries, :diff
+
+        # Resolved on first use rather than in the constructor: `selective
+        # exec` (the plain framework run behind the manifest dry-run) never
+        # connects, so it must not require the CI environment that naming a
+        # runner validates.
+        def runner_id
+          @runner_id ||= safe_filename(get_runner_id)
+        end
+
+        def logger
+          @logger ||= init_logger(@log)
+        end
 
         def get_runner_id
           runner_id = build_env["runner_id"]
@@ -90,7 +101,7 @@ module Selective
             message = pipe.read
             response = JSON.parse(message, symbolize_names: true)
 
-            @logger.info("Received Command: #{response}")
+            logger.info("Received Command: #{response}")
             break if handle_command(response) == :break
           end
         end
@@ -140,6 +151,13 @@ module Selective
               "framework" => runner.framework,
               "framework_version" => runner.framework_version,
               "framework_wrapper_version" => runner.wrapper_version,
+              "runtime_version" => RUBY_VERSION,
+              # "record", "select" or "0": whether this runner can record a
+              # test map, only run selected subsets, or neither.
+              "test_map" => TestMap.capability,
+              # Forces this run to record (e.g. a nightly full run), however
+              # fresh the current map is.
+              "test_map_record" => ENV["SELECTIVE_TEST_MAP_RECORD"].to_s
             }.merge(metadata: metadata.to_json)
           end
         end
@@ -271,8 +289,11 @@ module Selective
 
         def handle_test_manifest(_data)
           self.class.restore_reporting!
-          @logger.info("Sending Response: test_manifest")
+          logger.info("Sending Response: test_manifest")
           data = {test_cases: runner.manifest["examples"] || runner.manifest["test_cases"]}
+          if TestMap.tree_requested? && (tree = test_map_tree)
+            data[:test_map_tree] = tree
+          end
           num_commits = build_env["num_commits"] || 1000
           if (diff = get_diff(num_commits))
             data[:modified_test_files] = modified_test_files(diff)
@@ -281,6 +302,22 @@ module Selective
           write({type: "test_manifest", data: data})
         end
 
+        # Sent by servers that support test maps, before any tests are
+        # assigned. `tree: true` asks for a file-hash snapshot with the
+        # manifest; `record: true` turns on tracing for every test this runner
+        # executes.
+        def handle_configure_test_map(data)
+          recorder = TestMap.configure(
+            record: data[:record] == true,
+            tree: data[:tree] == true,
+            allocations: data[:allocations] != false
+          )
+          logger.info("Test map configured: record=#{!recorder.nil?} tree=#{TestMap.tree_requested?}")
+          if data[:record] == true && recorder.nil?
+            print_warning("Selective asked this runner to record a test map, but the native tracer isn't available. Tests will run normally.")
+          end
+        end
+
         def handle_run_test_cases(data)
           runner.run_test_cases(data[:test_case_ids])
         end
@@ -307,6 +344,8 @@ module Selective
             runner.finish unless exit_status.is_a?(Integer)
           end
 
+          upload_test_map
+
           kill_transport
           pipe.delete_pipes
           exit(exit_status || runner.exit_status)
@@ -315,12 +354,40 @@ module Selective
           :break
         end
 
+        def test_map_tree
+          TestMap::Tree.snapshot
+        rescue => e
+          logger.warn("Could not snapshot the repository for test selection: #{e.message}")
+          nil
+        end
+
+        def upload_test_map
+          recorder = TestMap.recorder
+          return if recorder.nil? || recorder.empty?
+
+          uploader = TestMap::Uploader.new(
+            host: build_env["host"],
+            api_key: build_env["api_key"],
+            run_id: build_env["run_id"],
+            run_attempt: build_env["run_attempt"],
+            runner_id: runner_id,
+            logger: logger
+          )
+          if uploader.upload(recorder.to_h)
+            logger.info("Uploaded test map fragment: #{recorder.test_count} tests")
+          else
+            print_warning("Selective could not upload this runner's test map (#{uploader.last_error&.message}). Its tests will run on the next build rather than be skipped.")
+          end
+        rescue => e
+          logger.warn("Test map upload failed: #{e.class}: #{e.message}")
+        end
+
         def correlated_files(diff, num_commits)
           Selective::Ruby::Core::FileCorrelator.new(diff, num_commits, build_env["target_branch"]).correlate
         end
 
         def test_case_callback(test_case)
-          @logger.info("Sending Response: test_case_result: #{test_case[:id]}")
+          logger.info("Sending Response: test_case_result: #{test_case[:id]}")
           write({type: "test_case_result", data: test_case})
         end
 

data/lib/selective/ruby/core/test_map/recorder.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Selective
+  module Ruby
+    module Core
+      module TestMap
+        # Accumulates "test id => files" for every test this runner executes and
+        # serialises it as a fragment. Paths are interned into a table so a
+        # fragment carries each path once, not once per test that touched it.
+        class Recorder
+          FORMAT_VERSION = 1
+
+          attr_reader :tracer
+
+          def initialize(tracer:)
+            @tracer = tracer
+            @paths = []
+            @path_index = {}
+            @tests = {}
+          end
+
+          # Retries and reruns can run a test twice in one process. Keep the
+          # union: a dependency seen on any attempt is a real dependency.
+          def record(test_id)
+            result, files = tracer.trace { yield }
+            indexes = (@tests[test_id] ||= Set.new)
+            files.each { |path| indexes << intern(path) }
+            result
+          end
+
+          def test_count
+            @tests.size
+          end
+
+          def empty?
+            @tests.empty?
+          end
+
+          def to_h
+            {
+              version: FORMAT_VERSION,
+              files: @paths,
+              tests: @tests.map { |id, indexes| {id: id, deps: indexes.to_a.sort} }
+            }
+          end
+
+          private
+
+          def intern(path)
+            @path_index[path] ||= begin
+              @paths << path
+              @paths.size - 1
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/selective/ruby/core/test_map/tracer.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Selective
+  module Ruby
+    module Core
+      module TestMap
+        # Ruby face of the native tracer: decides which paths count as the
+        # project's own code, and turns what the tracer saw into sorted,
+        # repo-relative paths that line up with `git ls-tree`.
+        class Tracer
+          attr_reader :repo_root
+
+          def initialize(allocations: true, repo_root: Tree.repo_root, pwd: Dir.pwd)
+            @repo_root = repo_root || File.realpath(pwd)
+            @roots = unique_dirs([@repo_root, pwd])
+            # Only directories *inside* a root need ignoring (everything else is
+            # outside the roots already). An ancestor would ignore the whole
+            # project, e.g. GEM_HOME=/work with the repo at /work/app.
+            @ignored = unique_dirs(ignored_dirs).select do |dir|
+              @roots.any? { |root| dir.start_with?("#{root}/") }
+            end
+            @native = NativeTracer.new(roots: @roots, ignored: @ignored, allocations: allocations)
+            @relative_paths = {}
+            @class_files = {}.compare_by_identity
+          end
+
+          # Runs the block with tracing on and returns [result, files], where
+          # files is a Set of repo-relative paths.
+          def trace
+            @native.start
+            begin
+              result = yield
+            ensure
+              files, klasses = @native.stop
+            end
+            [result, collect(files, klasses)]
+          end
+
+          private
+
+          def collect(files, klasses)
+            paths = Set.new
+            files.each_key do |path|
+              rel = relative(path)
+              paths << rel if rel
+            end
+            klasses.each do |klass|
+              files_for_class(klass).each { |rel| paths << rel }
+            end
+            paths
+          end
+
+          # A class's own file, plus those of its project-defined ancestors: a
+          # test that builds a `User` depends on `ApplicationRecord` too.
+          def files_for_class(klass)
+            @class_files[klass] ||= klass.ancestors.filter_map do |mod|
+              name = mod.name
+              next if name.nil?
+
+              location = Object.const_source_location(name)&.first
+              location && relative(location)
+            rescue NameError, ArgumentError, TypeError
+              nil
+            end.uniq.freeze
+          end
+
+          def relative(path)
+            return @relative_paths[path] if @relative_paths.key?(path)
+
+            @relative_paths[path] = compute_relative(path)
+          end
+
+          def compute_relative(path)
+            abs = File.expand_path(path)
+            return nil if ignored?(abs)
+
+            rel = strip_root(abs)
+            if rel.nil? && File.exist?(abs)
+              real = File.realpath(abs)
+              rel = strip_root(real) unless ignored?(real)
+            end
+            rel
+          end
+
+          def strip_root(abs)
+            prefix = "#{@repo_root}/"
+            abs.start_with?(prefix) ? abs.delete_prefix(prefix) : nil
+          end
+
+          def ignored?(abs)
+            @ignored.any? { |dir| abs == dir || abs.start_with?("#{dir}/") }
+          end
+
+          # Code the project runs but doesn't own: installed gems (including a
+          # vendored bundle inside the repo) and Selective's own gems, which
+          # live inside the repo when developed through path/symlinks.
+          def ignored_dirs
+            dirs = [Gem.dir, *Gem.path]
+            dirs << Bundler.bundle_path.to_s if defined?(Bundler) && Bundler.respond_to?(:bundle_path)
+            Gem.loaded_specs.each do |name, spec|
+              dirs << spec.full_gem_path if name.start_with?("selective-ruby-")
+            end
+            extra = ENV["SELECTIVE_TEST_MAP_IGNORE"].to_s.split(",").map(&:strip).reject(&:empty?)
+            dirs.concat(extra.map { |d| File.expand_path(d) })
+          end
+
+          def unique_dirs(dirs)
+            dirs.compact.flat_map do |dir|
+              expanded = File.expand_path(dir.to_s)
+              real = File.exist?(expanded) ? File.realpath(expanded) : expanded
+              [expanded, real]
+            end.map { |d| d.chomp("/") }.reject(&:empty?).uniq
+          end
+        end
+      end
+    end
+  end
+end

data/lib/selective/ruby/core/test_map/tree.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Selective
+  module Ruby
+    module Core
+      module TestMap
+        # A content-hash snapshot of the repository: every file's git blob id.
+        #
+        # This is what makes selection independent of diffs. The server
+        # compares these hashes with the ones stored alongside the map, so there
+        # is no base commit to pick, no merge-base to compute and no fetch depth
+        # to get right. It works in a depth-1 clone, and for uncommitted edits
+        # (a developer or an agent running locally) it hashes what is actually
+        # on disk rather than what HEAD says.
+        module Tree
+          module_function
+
+          def repo_root
+            out, status = Open3.capture2e("git", "rev-parse", "--show-toplevel")
+            status.success? ? out.strip : nil
+          rescue SystemCallError
+            nil
+          end
+
+          # => {sha:, prefix:, files: {path => blob}, dirty: [paths]} or nil
+          # when the working directory isn't a git checkout.
+          def snapshot(pwd: Dir.pwd)
+            root = repo_root
+            return nil if root.nil?
+
+            files = committed_files(root)
+            return nil if files.nil?
+
+            dirty = apply_working_tree_changes(root, files)
+            {
+              sha: head_sha(root),
+              prefix: prefix_for(root, pwd),
+              files: files,
+              dirty: dirty
+            }
+          end
+
+          # Where the test command runs, relative to the repository root, e.g.
+          # "services/api/" in a monorepo. The server uses it to line manifest
+          # paths (relative to the command) up with tree paths (relative to the
+          # root).
+          def prefix_for(root, pwd)
+            real = File.realpath(pwd)
+            return "" if real == root
+
+            real.start_with?("#{root}/") ? "#{real.delete_prefix("#{root}/")}/" : ""
+          end
+
+          def head_sha(root)
+            git(root, "rev-parse", "HEAD")&.strip
+          end
+
+          def committed_files(root)
+            out = git(root, "ls-tree", "-r", "-z", "--full-tree", "HEAD")
+            return nil if out.nil?
+
+            out.split("\0").each_with_object({}) do |entry, files|
+              meta, path = entry.split("\t", 2)
+              next if path.nil?
+
+              _mode, _type, blob = meta.split(" ")
+              files[path] = blob
+            end
+          end
+
+          # Folds uncommitted edits to tracked files into the snapshot. Returns
+          # the dirty paths so the server can report that the run used a
+          # working tree.
+          #
+          # Untracked files are deliberately left out. They add nothing — a new
+          # test file is "new" to the map through the manifest, and a new app
+          # file can't be a recorded dependency — and they churn: runner logs,
+          # tmp/, coverage output. Hashing them made a local log file look like
+          # a changed file no test uses, which runs the whole suite.
+          def apply_working_tree_changes(root, files)
+            out = git(root, "status", "--porcelain=v1", "-z", "--untracked-files=no", "--no-renames")
+            return [] if out.nil? || out.empty?
+
+            changed = []
+            deleted = []
+            out.split("\0").each do |entry|
+              next if entry.length < 4
+
+              status = entry[0, 2]
+              path = entry[3..]
+              if status.include?("D") && !File.exist?(File.join(root, path))
+                files.delete(path)
+                deleted << path
+              else
+                changed << path
+              end
+            end
+
+            hash_objects(root, changed).each { |path, blob| files[path] = blob }
+            changed + deleted
+          end
+
+          def hash_objects(root, paths)
+            existing = paths.select { |p| File.file?(File.join(root, p)) }
+            return {} if existing.empty?
+
+            out, status = Open3.capture2("git", "-C", root, "hash-object", "--stdin-paths", stdin_data: existing.join("\n") + "\n")
+            return {} unless status.success?
+
+            existing.zip(out.split("\n")).to_h
+          end
+
+          def git(root, *args)
+            out, status = Open3.capture2("git", "-C", root, *args)
+            status.success? ? out : nil
+          rescue SystemCallError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/selective/ruby/core/test_map/uploader.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "open3"
+require "rbconfig"
+require "tempfile"
+require "uri"
+require "zlib"
+
+module Selective
+  module Ruby
+    module Core
+      module TestMap
+        # Sends this runner's recorded fragment to the server over HTTPS.
+        #
+        # A map is far larger than anything else a runner sends (tests × files),
+        # so it doesn't go over the WebSocket. Upload failure never fails the
+        # build: the worst outcome is a map missing this runner's tests, and a
+        # test missing from the map is always run.
+        #
+        # The POST runs in a child Ruby process. The upload happens inside the
+        # test process, where suites commonly load WebMock or VCR, and those
+        # intercept every Net::HTTP request, this one included. A child started
+        # without Bundler's environment loads neither.
+        class Uploader
+          PATH = "/test_maps/fragments"
+          ATTEMPTS = 3
+
+          def initialize(host:, api_key:, run_id:, run_attempt:, runner_id:, logger: nil)
+            @uri = build_uri(host, run_id: run_id, run_attempt: run_attempt, runner_id: runner_id)
+            @api_key = api_key
+            @logger = logger
+          end
+
+          # Returns true on success.
+          def upload(fragment)
+            body = Zlib.gzip(JSON.generate(fragment))
+            attempt = 0
+            begin
+              attempt += 1
+              true if post(body)
+            rescue => e
+              @logger&.warn("Test map upload attempt #{attempt} failed: #{e.message}")
+              if attempt < ATTEMPTS
+                sleep(attempt)
+                retry
+              end
+              @last_error = e
+              false
+            end
+          end
+
+          attr_reader :last_error, :uri
+
+          private
+
+          # The request, as a standalone script for the child process. The API key
+          # arrives in the environment, never on the command line.
+          CHILD_SCRIPT = <<~RUBY
+            require "net/http"
+            require "uri"
+            uri = URI(ARGV.fetch(0))
+            http = Net::HTTP.new(uri.host, uri.port)
+            http.use_ssl = uri.scheme == "https"
+            http.open_timeout = 10
+            http.read_timeout = 120
+            request = Net::HTTP::Post.new(uri.request_uri)
+            request["authorization"] = ENV.fetch("SELECTIVE_TEST_MAP_UPLOAD_KEY")
+            request["content-type"] = "application/gzip"
+            request.body = File.binread(ARGV.fetch(1))
+            response = http.request(request)
+            puts response.code
+            warn response.body.to_s[0, 200] unless response.is_a?(Net::HTTPSuccess)
+            exit(response.is_a?(Net::HTTPSuccess) ? 0 : 1)
+          RUBY
+
+          # Keeps Bundler (and whatever it would require) out of the child.
+          CLEAN_ENV = %w[RUBYOPT RUBYLIB BUNDLE_GEMFILE BUNDLE_BIN_PATH BUNDLER_SETUP BUNDLER_VERSION GEM_HOME
+            GEM_PATH].to_h do |k|
+            [k, nil]
+          end
+
+          # Raises unless the server accepted the upload.
+          def post(body)
+            Tempfile.create(["selective-test-map", ".json.gz"], binmode: true) do |file|
+              file.write(body)
+              file.flush
+
+              env = CLEAN_ENV.merge("SELECTIVE_TEST_MAP_UPLOAD_KEY" => @api_key.to_s)
+              out, err, status = Open3.capture3(env, RbConfig.ruby, "-e", CHILD_SCRIPT, @uri.to_s, file.path)
+              raise "HTTP #{out.strip}: #{err.strip}" unless status.success?
+
+              true
+            end
+          end
+
+          # The runner is configured with the WebSocket host (ws:// or wss://);
+          # the upload goes to the same server over HTTP(S).
+          def build_uri(host, **params)
+            base = host.sub(%r{\Aws(s?)://}) { "http#{::Regexp.last_match(1)}://" }.chomp("/")
+            URI("#{base}#{PATH}?#{URI.encode_www_form(params)}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/selective/ruby/core/test_map.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Selective
+  module Ruby
+    module Core
+      # Test maps let Selective skip tests a change can't affect.
+      #
+      # A *recording* run traces every test with the native tracer and uploads
+      # "test id => repo files it executed" when the runner finishes. A later
+      # run sends a snapshot of the repository's file hashes with its manifest,
+      # and the server skips a test only when every file it depends on is
+      # byte-for-byte what it was when the map was recorded.
+      #
+      # Everything here is inert unless the server asks for it
+      # (`configure_test_map`), and recording additionally needs the optional
+      # native extension. Without either, the suite runs exactly as before.
+      module TestMap
+        class << self
+          # True when the native tracer compiled and loads on this Ruby.
+          def available?
+            return @available if defined?(@available)
+
+            @available = if disabled?
+              false
+            else
+              begin
+                require "selective_tracer"
+                true
+              rescue LoadError
+                false
+              end
+            end
+          end
+
+          # SELECTIVE_TEST_MAP_DISABLE=1 is the escape hatch: the runner then
+          # never records and never takes part in selection.
+          def disabled?
+            ENV["SELECTIVE_TEST_MAP_DISABLE"].to_s.match?(/\A(1|true|yes)\z/i)
+          end
+
+          # What this runner can do, as advertised to the server. Selection
+          # needs only a git snapshot, so a runner without the native tracer
+          # (unsupported Ruby, failed compile) still runs selected subsets
+          # from maps recorded elsewhere; it just can't record one.
+          def capability
+            return "0" if disabled?
+
+            available? ? "record" : "select"
+          end
+
+          # Recording is one-way for the life of the process: a reconnect can
+          # make the server re-send its configuration, and dropping a recorder
+          # would throw away every test traced so far.
+          def configure(record: false, tree: false, allocations: true)
+            @tree_requested = tree
+            if record && available?
+              @recorder ||= Recorder.new(tracer: Tracer.new(allocations: allocations))
+            end
+            @recorder
+          end
+
+          def recording?
+            !@recorder.nil?
+          end
+
+          def tree_requested?
+            @tree_requested == true
+          end
+
+          # Wraps one test. Adapters call this around the framework's own
+          # "run this test" so setup, the test body and teardown are all traced.
+          def around(test_id)
+            recorder = @recorder
+            return yield if recorder.nil?
+
+            recorder.record(test_id) { yield }
+          end
+
+          attr_reader :recorder
+
+          def reset!
+            @recorder = nil
+            @tree_requested = false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/selective/ruby/core/version.rb

@@ -3,7 +3,7 @@
 module Selective
   module Ruby
     module Core
-      VERSION = "0.2.9"
+      VERSION = "0.2.10"
     end
   end
 end

data/selective-ruby-core.gemspec

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative "lib/selective/ruby/core/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "selective-ruby-core"
+  spec.version = Selective::Ruby::Core::VERSION
+  spec.authors = ["Benjamin Wood", "Nate Vick"]
+  spec.email = ["ben@hint.io", "nate@hint.io"]
+  spec.license = "MIT"
+
+  # Currently supported release platforms are:
+  # Linux: x86_64-linux, aarch64-linux
+  # MacOS: x86_64-darwin, arm64-darwin
+  if (platform = ENV["RELEASE_PLATFORM"])
+    spec.platform = platform
+  end
+
+  spec.summary = "Selective Ruby Client Core"
+  spec.description = "Selective is an intelligent test runner for your current CI provider. Get real-time test results, intelligent ordering based on code changes, shorter run times, automatic flake detection, the ability to re-enqueue failed tests, and more."
+  spec.homepage = "https://www.selective.ci"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "http://github.com/selectiveci/selective-ruby-core"
+  spec.metadata["changelog_uri"] = "https://github.com/selectiveci/selective-ruby-core/blob/main/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (File.expand_path(f) == __FILE__) || f.start_with?(*%w[bin/ test/ spec/ features/ log/ .git .circleci appveyor .rspec .ruby-version .standard.yml README.md CHANGELOG.md CODE_OF_CONDUCT.md Gemfile])
+    end
+  end
+
+  # Add the transport binary
+  unless ENV["RELEASE_PLATFORM"].nil?
+    spec.files << "lib/bin/transport"
+  end
+
+  # The optional test map tracer. extconf.rb writes a no-op Makefile when it
+  # can't build (no compiler, unsupported Ruby), so installation never fails
+  # because of it; the runner then reports it can't record.
+  spec.extensions = ["ext/selective_tracer/extconf.rb"]
+
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  spec.add_dependency("logger", "~> 1.6")
+  spec.add_dependency("zeitwerk", "~> 2.6.12")
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: selective-ruby-core
 version: !ruby/object:Gem::Version
-  version: 0.2.9
+  version: 0.2.10
 platform: ruby
 authors:
 - Benjamin Wood
 - Nate Vick
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2026-05-03 00:00:00.000000000 Z
+date: 2026-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logger
@@ -48,12 +48,15 @@ email:
 - nate@hint.io
 executables:
 - selective
-extensions: []
+extensions:
+- ext/selective_tracer/extconf.rb
 extra_rdoc_files: []
 files:
 - LICENSE
 - Rakefile
 - exe/selective
+- ext/selective_tracer/extconf.rb
+- ext/selective_tracer/selective_tracer.c
 - lib/bin/build_env.sh
 - lib/bin/file_correlation_collector.sh
 - lib/selective-ruby-core.rb
@@ -61,7 +64,13 @@ files:
 - lib/selective/ruby/core/file_correlator.rb
 - lib/selective/ruby/core/helper.rb
 - lib/selective/ruby/core/named_pipe.rb
+- lib/selective/ruby/core/test_map.rb
+- lib/selective/ruby/core/test_map/recorder.rb
+- lib/selective/ruby/core/test_map/tracer.rb
+- lib/selective/ruby/core/test_map/tree.rb
+- lib/selective/ruby/core/test_map/uploader.rb
 - lib/selective/ruby/core/version.rb
+- selective-ruby-core.gemspec
 homepage: https://www.selective.ci
 licenses:
 - MIT
@@ -69,7 +78,7 @@ metadata:
   homepage_uri: https://www.selective.ci
   source_code_uri: http://github.com/selectiveci/selective-ruby-core
   changelog_uri: https://github.com/selectiveci/selective-ruby-core/blob/main/CHANGELOG.md
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -85,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.5.22
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Selective Ruby Client Core
 test_files: []