polyrun 1.0.0

data/lib/polyrun/cli/prepare_command.rb

@@ -0,0 +1,71 @@
+require "json"
+require "open3"
+require "optparse"
+
+require_relative "prepare_recipe"
+
+module Polyrun
+  class CLI
+    module PrepareCommand
+      include PrepareRecipe
+
+      private
+
+      def cmd_prepare(argv, config_path)
+        dry = false
+        OptionParser.new do |opts|
+          opts.on("--dry-run", "Print steps only") { dry = true }
+        end.parse!(argv)
+
+        cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
+        prep = cfg.prepare
+        recipe = prep["recipe"] || prep[:recipe] || "default"
+        prep_env = (prep["env"] || prep[:env] || {}).transform_keys(&:to_s).transform_values(&:to_s)
+        child_env = prep_env.empty? ? nil : ENV.to_h.merge(prep_env)
+        manifest = prepare_build_manifest(recipe, dry, prep_env)
+
+        exit_code = prepare_dispatch_recipe(manifest, prep, recipe, dry, child_env)
+        return exit_code unless exit_code.nil?
+
+        prepare_write_artifact_manifest!(manifest, recipe, dry)
+      end
+
+      def prepare_write_artifact_manifest!(manifest, recipe, dry)
+        entries = (manifest["artifacts"] || []).map do |p|
+          {"path" => p, "kind" => (File.directory?(p) ? "directory" : "file")}
+        end
+        manifest["artifact_manifest_path"] = Polyrun::Prepare::Artifacts.write!(root: Dir.pwd, recipe: recipe, entries: entries, dry_run: dry)
+        Polyrun::Log.puts JSON.generate(manifest)
+        0
+      end
+
+      def prepare_build_manifest(recipe, dry, prep_env)
+        manifest = {
+          "recipe" => recipe,
+          "dry_run" => dry,
+          "artifacts" => [],
+          "executed" => !dry
+        }
+        manifest["env"] = prep_env unless prep_env.empty?
+        manifest
+      end
+
+      def prepare_dispatch_recipe(manifest, prep, recipe, dry, child_env)
+        case recipe
+        when "default", nil, ""
+          prepare_recipe_default(manifest, recipe)
+          nil
+        when "assets"
+          m, err = prepare_recipe_assets(manifest, prep, dry, child_env)
+          err
+        when "shell"
+          m, err = prepare_recipe_shell(manifest, prep, dry, child_env)
+          err
+        else
+          Polyrun::Log.warn "unknown prepare recipe: #{recipe}"
+          1
+        end
+      end
+    end
+  end
+end

data/lib/polyrun/cli/prepare_recipe.rb

@@ -0,0 +1,77 @@
+require "open3"
+
+module Polyrun
+  class CLI
+    module PrepareRecipe
+      private
+
+      def prepare_recipe_default(manifest, recipe)
+        Polyrun::Log.warn "polyrun prepare: default recipe (no side effects)" if @verbose
+        [manifest, nil]
+      end
+
+      def prepare_recipe_assets(manifest, prep, dry, child_env)
+        rails_root = File.expand_path(prep["rails_root"] || prep[:rails_root] || ".", Dir.pwd)
+        manifest["rails_root"] = rails_root
+        custom = prep["command"] || prep[:command]
+        if dry
+          manifest["actions"] = [
+            custom ? custom.to_s.strip : "bin/rails assets:precompile"
+          ]
+          manifest["executed"] = false
+          return [manifest, nil]
+        end
+        if custom && !custom.to_s.strip.empty?
+          _out, err, st = Open3.capture3(*([child_env].compact + ["sh", "-c", custom.to_s]), chdir: rails_root)
+          prepare_log_stderr(err)
+          unless st.success?
+            Polyrun::Log.warn "polyrun prepare: assets custom command failed (exit #{st.exitstatus})"
+            return [manifest, 1]
+          end
+        else
+          Polyrun::Prepare::Assets.precompile!(rails_root: rails_root, silent: !@verbose)
+        end
+        manifest["artifacts"] = [File.join(rails_root, "public", "assets").to_s]
+        [manifest, nil]
+      end
+
+      def prepare_shell_command_lines(prep)
+        command = prep["command"] || prep[:command]
+        commands = prep["commands"] || prep[:commands]
+        lines = []
+        lines.concat(Array(commands).map { |c| c.to_s.strip }.reject(&:empty?)) if commands
+        lines << command.to_s.strip if command && !command.to_s.strip.empty?
+        lines
+      end
+
+      def prepare_recipe_shell(manifest, prep, dry, child_env)
+        rails_root = File.expand_path(prep["rails_root"] || prep[:rails_root] || ".", Dir.pwd)
+        manifest["rails_root"] = rails_root
+        lines = prepare_shell_command_lines(prep)
+
+        if lines.empty?
+          Polyrun::Log.warn "polyrun prepare: shell recipe requires prepare.command and/or prepare.commands"
+          return [manifest, 1]
+        end
+        manifest["actions"] = lines
+        if dry
+          manifest["executed"] = false
+          return [manifest, nil]
+        end
+        lines.each_with_index do |line, i|
+          _out, err, st = Open3.capture3(*([child_env].compact + ["sh", "-c", line]), chdir: rails_root)
+          prepare_log_stderr(err)
+          unless st.success?
+            Polyrun::Log.warn "polyrun prepare: shell step #{i + 1} failed (exit #{st.exitstatus})"
+            return [manifest, 1]
+          end
+        end
+        [manifest, nil]
+      end
+
+      def prepare_log_stderr(err)
+        Polyrun::Log.warn err unless err.to_s.empty?
+      end
+    end
+  end
+end

data/lib/polyrun/cli/queue_command.rb

@@ -0,0 +1,101 @@
+require "json"
+require "optparse"
+
+module Polyrun
+  class CLI
+    module QueueCommand
+      private
+
+      # File-backed queue (spec_queue.md): init → claim batches → ack (ledger append-only).
+      def cmd_queue(argv)
+        dir = ".polyrun-queue"
+        paths_file = nil
+        timing_path = nil
+        worker = ENV["USER"] || "worker"
+        batch = 5
+        lease_id = nil
+
+        sub = argv.shift
+        Polyrun::Debug.log("queue: subcommand=#{sub.inspect}")
+        case sub
+        when "init"
+          queue_cmd_init(argv, dir, paths_file, timing_path)
+        when "claim"
+          queue_cmd_claim(argv, dir, worker, batch)
+        when "ack"
+          queue_cmd_ack(argv, dir, lease_id, worker)
+        when "status"
+          queue_cmd_status(argv, dir)
+        else
+          Polyrun::Log.warn "usage: polyrun queue {init|claim|ack|status} [options]"
+          2
+        end
+      end
+
+      def queue_cmd_init(argv, dir, paths_file, timing_path)
+        OptionParser.new do |opts|
+          opts.banner = "usage: polyrun queue init --paths-file P [--timing PATH] [--dir DIR]"
+          opts.on("--dir PATH") { |v| dir = v }
+          opts.on("--paths-file PATH") { |v| paths_file = v }
+          opts.on("--timing PATH") { |v| timing_path = v }
+        end.parse!(argv)
+        unless paths_file
+          Polyrun::Log.warn "queue init: need --paths-file"
+          return 2
+        end
+        items = Polyrun::Partition::Paths.read_lines(paths_file)
+        costs = timing_path ? Polyrun::Partition::Plan.load_timing_costs(File.expand_path(timing_path, Dir.pwd)) : nil
+        ordered = queue_init_ordered_items(items, costs)
+        Polyrun::Queue::FileStore.new(dir).init!(ordered)
+        Polyrun::Log.puts JSON.generate({"dir" => File.expand_path(dir), "count" => ordered.size})
+        0
+      end
+
+      def queue_init_ordered_items(items, costs)
+        if costs && !costs.empty?
+          dw = costs.values.sum / costs.size.to_f
+          items.sort_by { |p| [-queue_weight_for(p, costs, dw), p] }
+        else
+          items.sort
+        end
+      end
+
+      def queue_cmd_claim(argv, dir, worker, batch)
+        OptionParser.new do |opts|
+          opts.banner = "usage: polyrun queue claim [--dir DIR] [--worker ID] [--batch N]"
+          opts.on("--dir PATH") { |v| dir = v }
+          opts.on("--worker ID") { |v| worker = v }
+          opts.on("--batch N", Integer) { |v| batch = v }
+        end.parse!(argv)
+        r = Polyrun::Queue::FileStore.new(dir).claim!(worker_id: worker, batch_size: batch)
+        Polyrun::Log.puts JSON.generate(r)
+        0
+      end
+
+      def queue_cmd_ack(argv, dir, lease_id, worker)
+        OptionParser.new do |opts|
+          opts.banner = "usage: polyrun queue ack --lease ID [--dir DIR] [--worker ID]"
+          opts.on("--dir PATH") { |v| dir = v }
+          opts.on("--lease ID") { |v| lease_id = v }
+          opts.on("--worker ID") { |v| worker = v }
+        end.parse!(argv)
+        unless lease_id
+          Polyrun::Log.warn "queue ack: need --lease"
+          return 2
+        end
+        Polyrun::Queue::FileStore.new(dir).ack!(lease_id: lease_id, worker_id: worker)
+        Polyrun::Log.puts "ok"
+        0
+      end
+
+      def queue_cmd_status(argv, dir)
+        OptionParser.new do |opts|
+          opts.on("--dir PATH") { |v| dir = v }
+        end.parse!(argv)
+        s = Polyrun::Queue::FileStore.new(dir).status
+        Polyrun::Log.puts JSON.generate(s)
+        0
+      end
+    end
+  end
+end

data/lib/polyrun/cli/quick_command.rb

@@ -0,0 +1,13 @@
+module Polyrun
+  class CLI
+    module QuickCommand
+      private
+
+      def cmd_quick(argv)
+        require_relative "../quick/runner"
+        paths = argv.dup
+        Polyrun::Quick::Runner.run(paths: paths.empty? ? nil : paths, verbose: @verbose)
+      end
+    end
+  end
+end

data/lib/polyrun/cli/report_commands.rb

@@ -0,0 +1,94 @@
+require "json"
+require "optparse"
+
+module Polyrun
+  class CLI
+    module ReportCommands
+      private
+
+      def cmd_report_junit(argv)
+        inputs, output = report_junit_parse_inputs(argv)
+        inputs.uniq!
+        if inputs.empty?
+          Polyrun::Log.warn "report-junit: need -i FILE (existing path after glob expansion)"
+          return 2
+        end
+
+        inputs = inputs.map { |p| File.expand_path(p) }
+        return 2 unless report_junit_inputs_exist?(inputs)
+
+        out = report_junit_resolved_output(inputs, output)
+        path =
+          if inputs.size == 1
+            Polyrun::Reporting::Junit.write_from_json_file(inputs.first, output_path: out)
+          else
+            Polyrun::Reporting::Junit.merge_rspec_json_files(inputs, output_path: out)
+          end
+        Polyrun::Log.puts path
+        0
+      end
+
+      def report_junit_parse_inputs(argv)
+        inputs = []
+        output = nil
+        OptionParser.new do |opts|
+          opts.banner = "usage: polyrun report-junit -i FILE [-i FILE]... [-o PATH]"
+          opts.on("-i", "--input PATH", "RSpec JSON (repeatable; globs ok; multiple files merge examples)") do |v|
+            expand_merge_input_pattern(v).each { |x| inputs << x }
+          end
+          opts.on("-o", "--output PATH", "Default: <dir of first input>/junit.xml") { |v| output = v }
+        end.parse!(argv)
+        if inputs.empty? && argv.first
+          expand_merge_input_pattern(argv.first).each { |x| inputs << x }
+        end
+        [inputs, output]
+      end
+
+      def report_junit_inputs_exist?(inputs)
+        inputs.each do |p|
+          unless File.file?(p)
+            Polyrun::Log.warn "report-junit: not a file: #{p}"
+            return false
+          end
+        end
+        true
+      end
+
+      def report_junit_resolved_output(inputs, output)
+        if output
+          File.expand_path(output)
+        else
+          File.join(File.dirname(inputs.first), "junit.xml")
+        end
+      end
+
+      def cmd_report_timing(argv)
+        input = nil
+        out_file = nil
+        top = 30
+        OptionParser.new do |opts|
+          opts.banner = "usage: polyrun report-timing -i FILE [-o PATH] [--top N]"
+          opts.on("-i", "--input PATH", "Merged polyrun_timing.json (path => seconds)") { |v| input = v }
+          opts.on("-o", "--output PATH", "Write summary to file instead of stdout") { |v| out_file = v }
+          opts.on("--top N", Integer) { |v| top = v }
+        end.parse!(argv)
+        input ||= argv.first
+
+        unless input && File.file?(input)
+          Polyrun::Log.warn "report-timing: need -i FILE"
+          return 2
+        end
+
+        merged = JSON.parse(File.read(File.expand_path(input)))
+        text = Polyrun::Timing::Summary.format_slow_files(merged, top: top)
+        if out_file
+          File.write(File.expand_path(out_file), text)
+          Polyrun::Log.puts File.expand_path(out_file)
+        else
+          Polyrun::Log.print text
+        end
+        0
+      end
+    end
+  end
+end

data/lib/polyrun/cli/run_shards_command.rb

@@ -0,0 +1,88 @@
+require "optparse"
+require "rbconfig"
+
+require_relative "start_bootstrap"
+require_relative "run_shards_run"
+
+module Polyrun
+  class CLI
+    module RunShardsCommand
+      include StartBootstrap
+      include RunShardsRun
+
+      private
+
+      # Default and upper bound for parallel OS processes (POLYRUN_WORKERS / --workers).
+      DEFAULT_PARALLEL_WORKERS = 5
+      MAX_PARALLEL_WORKERS = 10
+
+      # Spawns N OS processes (not Ruby threads) with POLYRUN_SHARD_INDEX / POLYRUN_SHARD_TOTAL so
+      # {Coverage::Collector} writes coverage/polyrun-fragment-<shard>.json. Merge with merge-coverage.
+      def cmd_run_shards(argv, config_path)
+        run_shards_run!(argv, config_path)
+      end
+
+      # Same as run-shards with --merge-coverage; if you omit --, runs `bundle exec rspec`.
+      def cmd_parallel_rspec(argv, config_path)
+        sep = argv.index("--")
+        combined =
+          if sep
+            head = argv[0...sep]
+            tail = argv[sep..]
+            head + ["--merge-coverage"] + tail
+          else
+            argv + ["--merge-coverage", "--", "bundle", "exec", "rspec"]
+          end
+        Polyrun::Debug.log_kv(parallel_rspec: "combined argv", argv: combined)
+        cmd_run_shards(combined, config_path)
+      end
+
+      # Convenience alias: optional legacy script/build_spec_paths.rb (if present and partition.paths_build unset), then parallel-rspec.
+      def cmd_start(argv, config_path)
+        cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
+        code = start_bootstrap!(cfg, argv, config_path)
+        return code if code != 0
+
+        unless skip_build_spec_paths?
+          unless partition_paths_build?(cfg.partition)
+            build_script = File.expand_path("script/build_spec_paths.rb", Dir.pwd)
+            if File.file?(build_script)
+              ok = system({"RUBYOPT" => nil}, RbConfig.ruby, build_script)
+              return 1 unless ok
+            end
+          end
+        end
+        cmd_parallel_rspec(argv, config_path)
+      end
+
+      def partition_paths_build?(partition)
+        pb = partition["paths_build"] || partition[:paths_build]
+        pb.is_a?(Hash) && !pb.empty?
+      end
+
+      def skip_build_spec_paths?
+        v = ENV["POLYRUN_SKIP_BUILD_SPEC_PATHS"].to_s.downcase
+        %w[1 true yes].include?(v)
+      end
+
+      # ENV for a worker process: POLYRUN_SHARD_* plus per-shard database URLs from polyrun.yml or DATABASE_URL.
+      def shard_child_env(cfg:, workers:, shard:)
+        child_env = ENV.to_h.merge(
+          Polyrun::Database::Shard.env_map(shard_index: shard, shard_total: workers)
+        )
+        dh = cfg.databases
+        if dh.is_a?(Hash) && !dh.empty?
+          child_env.merge!(Polyrun::Database::UrlBuilder.env_exports_for_databases(dh, shard_index: shard))
+        elsif workers > 1 && (u = ENV["DATABASE_URL"]) && !u.to_s.strip.empty?
+          child_env["DATABASE_URL"] = Polyrun::Database::Shard.database_url_with_shard(u, shard)
+        end
+        child_env
+      end
+
+      def cmd_build_paths(config_path)
+        cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
+        Polyrun::Partition::PathsBuild.apply!(partition: cfg.partition, cwd: Dir.pwd)
+      end
+    end
+  end
+end

data/lib/polyrun/cli/run_shards_plan_boot_phases.rb

@@ -0,0 +1,91 @@
+require "shellwords"
+
+module Polyrun
+  class CLI
+    # Boot argv, then phase A (options + validate) and B (items + plan) for run-shards.
+    module RunShardsPlanBootPhases
+      private
+
+      # @return [:fail, Integer] | [:ok, Hash, Array<String>]
+      def run_shards_plan_phase_a(head, cmd, pc)
+        o = run_shards_plan_options(head, pc)
+        code = Polyrun::Partition::PathsBuild.apply!(partition: pc, cwd: Dir.pwd)
+        return [:fail, code] if code != 0
+
+        o[:timing_path] = run_shards_default_timing_path(pc, o[:timing_path], o[:strategy])
+        err = run_shards_validate_workers!(o)
+        return [:fail, err] if err
+
+        err = run_shards_validate_cmd(cmd)
+        return [:fail, err] if err
+
+        cmd = Shellwords.split(cmd.first) if cmd.size == 1 && cmd.first.include?(" ")
+
+        [:ok, o, cmd]
+      end
+
+      def run_shards_plan_phase_b(o, cmd, cfg, pc, run_t0, config_path)
+        items, paths_source, err = run_shards_resolve_items(o[:paths_file])
+        return [err, nil] if err
+
+        costs, strategy, err = run_shards_resolve_costs(o[:timing_path], o[:strategy])
+        return [err, nil] if err
+
+        run_shards_plan_ready_log(o, strategy, cmd, paths_source, items.size)
+
+        constraints = load_partition_constraints(pc, o[:constraints_path])
+        plan = run_shards_make_plan(items, o[:workers], strategy, o[:seed], costs, constraints)
+
+        run_shards_debug_shard_sizes(plan, o[:workers])
+        Polyrun::Log.warn "polyrun run-shards: #{items.size} paths → #{o[:workers]} workers (#{strategy})" if @verbose
+
+        parallel = o[:workers] > 1
+        run_shards_warn_parallel_banner(items.size, o[:workers], strategy) if parallel
+
+        [nil, run_shards_plan_context_hash(o, cmd, cfg, plan, run_t0, parallel, config_path)]
+      end
+
+      def run_shards_plan_boot(argv, config_path)
+        run_t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        sep = argv.index("--")
+        unless sep
+          Polyrun::Log.warn "polyrun run-shards: need -- before the command (e.g. run-shards --workers 5 -- bundle exec rspec)"
+          return [2, nil]
+        end
+
+        head = argv[0...sep]
+        cmd = argv[(sep + 1)..].map(&:to_s)
+        cfg = Polyrun::Config.load(path: config_path || ENV["POLYRUN_CONFIG"])
+        [run_t0, head, cmd, cfg, cfg.partition]
+      end
+
+      def run_shards_plan_ready_log(o, strategy, cmd, paths_source, item_count)
+        Polyrun::Debug.log_kv(
+          run_shards: "ready to partition",
+          workers: o[:workers],
+          strategy: strategy,
+          merge_coverage: o[:merge_coverage],
+          command: cmd,
+          timing_path: o[:timing_path],
+          paths_source: paths_source,
+          item_count: item_count
+        )
+      end
+
+      def run_shards_plan_context_hash(o, cmd, cfg, plan, run_t0, parallel, config_path)
+        {
+          workers: o[:workers],
+          cmd: cmd,
+          cfg: cfg,
+          plan: plan,
+          run_t0: run_t0,
+          parallel: parallel,
+          merge_coverage: o[:merge_coverage],
+          merge_output: o[:merge_output],
+          merge_format: o[:merge_format],
+          config_path: config_path
+        }
+      end
+    end
+  end
+end

data/lib/polyrun/cli/run_shards_plan_options.rb

@@ -0,0 +1,45 @@
+require "optparse"
+
+module Polyrun
+  class CLI
+    module RunShardsPlanOptions
+      private
+
+      def run_shards_plan_options(head, pc)
+        st = run_shards_plan_options_state(pc)
+        run_shards_plan_options_parse!(head, st)
+        st[:paths_file] ||= pc["paths_file"] || pc[:paths_file]
+        st
+      end
+
+      def run_shards_plan_options_state(pc)
+        {
+          workers: env_int("POLYRUN_WORKERS", RunShardsCommand::DEFAULT_PARALLEL_WORKERS),
+          paths_file: nil,
+          strategy: (pc["strategy"] || pc[:strategy] || "round_robin").to_s,
+          seed: pc["seed"] || pc[:seed],
+          timing_path: nil,
+          constraints_path: nil,
+          merge_coverage: false,
+          merge_output: nil,
+          merge_format: nil
+        }
+      end
+
+      def run_shards_plan_options_parse!(head, st)
+        OptionParser.new do |opts|
+          opts.banner = "usage: polyrun run-shards [--workers N] [--strategy NAME] [--paths-file P] [--timing P] [--constraints P] [--seed S] [--merge-coverage] [--merge-output P] [--merge-format LIST] [--] <command> [args...]"
+          opts.on("--workers N", Integer) { |v| st[:workers] = v }
+          opts.on("--strategy NAME", String) { |v| st[:strategy] = v }
+          opts.on("--seed VAL") { |v| st[:seed] = v }
+          opts.on("--paths-file PATH", String) { |v| st[:paths_file] = v }
+          opts.on("--constraints PATH", String) { |v| st[:constraints_path] = v }
+          opts.on("--timing PATH", "merged polyrun_timing.json; implies cost_binpack unless hrw/cost") { |v| st[:timing_path] = v }
+          opts.on("--merge-coverage", "After success, merge coverage/polyrun-fragment-*.json (Polyrun coverage must be enabled)") { st[:merge_coverage] = true }
+          opts.on("--merge-output PATH", String) { |v| st[:merge_output] = v }
+          opts.on("--merge-format LIST", String) { |v| st[:merge_format] = v }
+        end.parse!(head)
+      end
+    end
+  end
+end