canopus 0.4.0 → 0.5.0

data/lib/canopus/task/runner.rb

@@ -0,0 +1,314 @@
+# frozen_string_literal: true
+
+module Canopus
+  module Task
+    class Runner
+      Output = Data.define(:id, :label, :terminal, :presentation)
+      MAX_OUTPUTS = 32
+      MAX_CLOSERS = 32
+      STOP_GRACE_SECONDS = 0.25
+
+      attr_reader :active_index
+
+      def initialize(scrollback:, queue_limit_bytes:, terminal_factory: nil, report: nil)
+        unless scrollback.is_a?(Integer) && scrollback.between?(0, 1_000_000)
+          raise ArgumentError, "invalid task scrollback limit"
+        end
+        unless queue_limit_bytes.is_a?(Integer) && queue_limit_bytes.between?(65_536, 268_435_456)
+          raise ArgumentError, "invalid task queue limit"
+        end
+        @scrollback, @queue_limit_bytes = scrollback, queue_limit_bytes
+        @terminal_factory = terminal_factory || ->(**options) { Tarazed::PTY.new(**options) }
+        @report = report || ->(_error) {}
+        @entries, @completed = [], {}
+        @close_threads = {}.compare_by_identity
+        @closed_outputs = {}.compare_by_identity
+        @eof_outputs = {}.compare_by_identity
+        @blocked_outputs = {}.compare_by_identity
+        @active_index = @sequence = @poll_index = 0
+      end
+
+      def entries = @entries.dup.freeze
+      def active = @entries[@active_index]
+      def terminal = active&.terminal
+
+      def run(task)
+        raise Error, "task runner is closed" if @closed
+        validate_task!(task)
+        reap_closers
+        previous = @entries.find { |entry| entry.label == task.fetch("label") }
+        evicted = previous || eviction_candidate
+        ensure_closer_capacity! if evicted && !@closed_outputs.key?(evicted)
+        dimensions = @dimensions || [100, 12]
+        terminal = @terminal_factory.call(command: task.fetch("command"), cwd: task.fetch("cwd"),
+          env: task.fetch("env", {}),
+          columns: dimensions.first, rows: dimensions.last, scrollback: @scrollback,
+          queue_limit_bytes: @queue_limit_bytes)
+        @sequence += 1
+        entry = Output.new(@sequence, task.fetch("label"), terminal, task.fetch("presentation")).freeze
+        if evicted
+          index = @entries.index { |current| current.equal?(evicted) }
+          close_later(evicted)
+          @completed.delete(evicted.id)
+          @eof_outputs.delete(evicted)
+          @blocked_outputs.delete(evicted)
+          @entries[index] = entry
+          @closed_outputs.delete(evicted) unless @close_threads.key?(evicted)
+          @active_index = index
+        else
+          @entries << entry
+          @active_index = @entries.length - 1
+        end
+        entry
+      rescue StandardError
+        terminal&.close unless @entries.any? { |entry| entry.terminal.equal?(terminal) }
+        raise
+      end
+
+      def activate(index)
+        unless index.is_a?(Integer) && index.between?(0, @entries.length - 1)
+          raise IndexError, "task output tab outside panel"
+        end
+        @active_index = index
+        active
+      end
+
+      def remove(index = @active_index)
+        entry = @entries[index]
+        return unless entry
+
+        close_later(entry)
+        @entries.delete_at(index)
+        @completed.delete(entry.id)
+        @eof_outputs.delete(entry)
+        @blocked_outputs.delete(entry)
+        @closed_outputs.delete(entry) unless @close_threads.key?(entry)
+        @active_index = [index, @entries.length - 1].min.clamp(0, @entries.length)
+        entry
+      end
+
+      def stop(entry = active)
+        return false unless entry && @entries.any? { |current| current.equal?(entry) }
+        return false unless running?(entry)
+
+        close_later(entry, interrupt: true)
+        true
+      end
+
+      def running?(entry = active)
+        return false unless entry && !@closed_outputs.key?(entry)
+        terminal = entry.terminal
+        !terminal.respond_to?(:alive?) || terminal.alive?
+      rescue IOError, SystemCallError
+        false
+      end
+
+      def completed
+        @entries.filter_map do |entry|
+          next if @completed[entry.id] || running?(entry) || !@eof_outputs.key?(entry)
+          @completed[entry.id] = true
+          entry
+        end
+      end
+
+      def drain(max_bytes:, max_seconds: 0.004)
+        raise ArgumentError, "task drain limit must be positive" unless max_bytes.is_a?(Integer) && max_bytes.positive?
+        unless max_seconds.is_a?(Numeric) && max_seconds.finite? && max_seconds >= 0
+          raise ArgumentError, "task drain budget must be nonnegative and finite"
+        end
+        reap_closers
+        candidates = @entries.reject { |entry| @eof_outputs.key?(entry) }
+          .select { |entry| !@closed_outputs.key?(entry) || @blocked_outputs.key?(entry) }
+        return false if candidates.empty?
+
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + max_seconds
+        remaining, changed = max_bytes, false
+        order = candidates.rotate(@poll_index % candidates.length)
+        @poll_index += 1
+        order.each do |entry|
+          break if remaining <= 0 || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+          if @blocked_outputs.key?(entry)
+            ready = !block_given? || yield(entry, "".b, remaining_time(deadline)) != false
+            if ready
+              @blocked_outputs.delete(entry)
+              if @closed_outputs.key?(entry) && !@close_threads.key?(entry)
+                @eof_outputs[entry] = true
+                next
+              end
+            else
+              next
+            end
+          end
+          next if @closed_outputs.key?(entry)
+          next if remaining_time(deadline).zero?
+          current = entry.terminal
+          method = current.method(:read)
+          keywords = method.parameters.any? { |kind, _| [:key, :keyreq, :keyrest].include?(kind) }
+          data = if keywords
+            current.read(max_bytes: remaining,
+              max_seconds: [deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max)
+          else
+            current.read
+          end
+          ready = !block_given? || yield(entry, data, remaining_time(deadline)) != false
+          @blocked_outputs[entry] = true unless ready
+          @eof_outputs[entry] = true if data.nil? && ready
+          changed ||= !!(data && !data.empty?)
+          remaining -= data.bytesize if data
+        end
+        changed
+      end
+
+      def pending?
+        reap_closers
+        @entries.any? do |entry|
+          next running?(entry) if @eof_outputs.key?(entry)
+          if @closed_outputs.key?(entry)
+            next @blocked_outputs.key?(entry) || @close_threads.key?(entry)
+          end
+          @blocked_outputs.key?(entry) || !running?(entry) ||
+            entry.terminal.respond_to?(:pending?) && entry.terminal.pending?
+        end
+      end
+
+      def resize(columns, rows)
+        dimensions = [columns.to_i, rows.to_i]
+        return if dimensions.any? { |value| value <= 0 }
+        @dimensions = dimensions
+        @entries.each do |entry|
+          next if @closed_outputs.key?(entry) || (@sizes ||= {}.compare_by_identity)[entry.terminal] == dimensions
+          entry.terminal.resize(columns: dimensions.first, rows: dimensions.last)
+          @sizes[entry.terminal] = dimensions
+        end
+      end
+
+      def close
+        return if @closed
+        @closed = true
+        begin
+          @close_threads.each_value(&:join)
+          reap_closers
+          @entries.each do |entry|
+            next if @closed_outputs.key?(entry)
+            begin
+              close_later(entry)
+            rescue StandardError => error
+              record_close_error(error)
+              close_now(entry)
+            end
+          end
+        ensure
+          @close_threads.each_value do |thread|
+            thread.join
+          rescue StandardError => error
+            record_close_error(error)
+          end
+          @entries.each { |entry| close_now(entry) unless @closed_outputs.key?(entry) }
+          @entries.clear
+          @close_threads.clear
+          @closed_outputs.clear
+          @completed.clear
+          @eof_outputs.clear
+          @blocked_outputs.clear
+          @sizes&.clear
+        end
+        raise @close_error if @close_error
+        nil
+      end
+
+      private
+
+      def remaining_time(deadline)
+        [deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max
+      end
+
+      def validate_task!(task)
+        environment = task.is_a?(Hash) ? task.fetch("env", {}) : nil
+        valid = task.is_a?(Hash) && task["label"].is_a?(String) && !task["label"].empty? &&
+          task["command"].is_a?(Array) && !task["command"].empty? && task["command"].all? { |part| part.is_a?(String) } &&
+          task["cwd"].is_a?(String) && task["presentation"].is_a?(Hash) && valid_environment?(environment)
+        raise ArgumentError, "resolved task required" unless valid
+      end
+
+      def valid_environment?(environment)
+        return false unless environment.is_a?(Hash) && environment.length <= 4_096
+
+        bytes = 0
+        environment.all? do |key, value|
+          next false unless key.is_a?(String) && key.ascii_only? && key.match?(/\A[A-Za-z_][A-Za-z0-9_]*\z/)
+          next false unless value.nil? || value.is_a?(String) && value.valid_encoding? && !value.include?("\0")
+
+          bytes += key.bytesize + value.to_s.bytesize
+          bytes <= 1_048_576
+        end
+      end
+
+      def eviction_candidate
+        return if @entries.length < MAX_OUTPUTS
+        entry = @entries.find { |candidate| @eof_outputs.key?(candidate) && !running?(candidate) }
+        raise Error, "at most #{MAX_OUTPUTS} task outputs may run at once" unless entry
+        entry
+      end
+
+      def ensure_closer_capacity!
+        reap_closers
+        raise Error, "too many task processes are still closing" if @close_threads.length >= MAX_CLOSERS
+      end
+
+      def close_later(entry, interrupt: false)
+        return @close_threads[entry] if @closed_outputs.key?(entry)
+        ensure_closer_capacity!
+        thread = Thread.new do
+          Thread.current.report_on_exception = false
+          terminal = entry.terminal
+          begin
+            if interrupt && (!terminal.respond_to?(:alive?) || terminal.alive?)
+              terminal.signal("INT") if terminal.respond_to?(:signal)
+              deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + STOP_GRACE_SECONDS
+              sleep(0.01) while terminal.respond_to?(:alive?) && terminal.alive? &&
+                Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+            end
+          ensure
+            terminal.close if terminal.respond_to?(:close)
+          end
+        rescue StandardError => error
+          record_close_error(error)
+        end
+        @close_threads[entry] = thread
+        @closed_outputs[entry] = true
+        thread
+      end
+
+      def close_now(entry)
+        entry.terminal.close if entry.terminal.respond_to?(:close)
+      rescue StandardError => error
+        record_close_error(error)
+      ensure
+        @closed_outputs[entry] = true
+      end
+
+      def record_close_error(error)
+        @close_error ||= error
+        @report.call(error)
+      rescue StandardError => report_error
+        @close_error ||= report_error
+      end
+
+      def reap_closers
+        @close_threads.delete_if do |entry, thread|
+          next false unless thread.join(0)
+          @sizes&.delete(entry.terminal)
+          retained = @entries.any? { |current| current.equal?(entry) }
+          if retained
+            @eof_outputs[entry] = true unless @blocked_outputs.key?(entry)
+          else
+            @closed_outputs.delete(entry)
+            @eof_outputs.delete(entry)
+            @blocked_outputs.delete(entry)
+          end
+          true
+        end
+      end
+    end
+  end
+end

data/lib/canopus/test_runner/bounded_ignore.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module Canopus
+  module TestRunner
+    # Loads ignore files only for directories the bounded walker reaches.
+    class BoundedIgnore
+      MAX_FILES = 1_000
+      MAX_FILE_BYTES = 256 * 1_024
+      MAX_TOTAL_BYTES = 1 << 20
+      MAX_LINES = 10_000
+      MAX_LINE_BYTES = 4_096
+      MAX_GIT_POINTER_BYTES = 4_096
+
+      def initialize(root, delegate: nil, cancelled: -> { false })
+        @root = root
+        @delegate = delegate
+        @cancelled = cancelled
+        @matcher = Thuban::IgnoreMatcher.new.add("/.canopus/trash/\n")
+        @loaded = {}
+        @files = @bytes = @lines = 0
+      end
+
+      def ignored?(path, directory: false)
+        path = TestRunner.normalize_path(path)
+        return true unless path
+        return true if @cancelled.call
+        return safe_delegate(path, directory) if @delegate
+
+        bases_for(path).each do |base|
+          break if @cancelled.call
+          load_base(base)
+        end
+        return true if @cancelled.call
+
+        @matcher.ignored?(path, directory: directory)
+      rescue ArgumentError, EncodingError
+        true
+      end
+
+      private
+
+      def safe_delegate(path, directory)
+        @delegate.ignored?(path, directory: directory) || @matcher.ignored?(path, directory: directory)
+      rescue ArgumentError, EncodingError
+        true
+      end
+
+      def bases_for(path)
+        pieces = path.split("/")[0...-1]
+        ["", *pieces.each_index.map { |index| pieces[0..index].join("/") }]
+      end
+
+      def load_base(base)
+        return if @loaded[base]
+        @loaded[base] = true
+        load_repository_exclude if base.empty?
+        %w[.gitignore .ignore].each do |name|
+          break if @cancelled.call
+          load_file(File.join(@root, *base.split("/"), name), base, within: @root)
+        end
+      end
+
+      def load_repository_exclude
+        common = repository_common_directory
+        load_file(File.join(common, "info", "exclude"), "", within: common) if common
+      end
+
+      def repository_common_directory
+        git = File.join(@root, ".git")
+        entry = File.lstat(git)
+        return if entry.symlink?
+        git_directory = if entry.directory?
+          File.realpath(git)
+        elsif entry.file? && (pointer = control_file(git, within: @root))&.start_with?("gitdir: ")
+          real_directory(File.expand_path(pointer.delete_prefix("gitdir: "), @root))
+        end
+        return unless git_directory
+
+        common_file = File.join(git_directory, "commondir")
+        common = control_file(common_file, within: git_directory)
+        common ? real_directory(File.expand_path(common, git_directory)) : git_directory
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP, Errno::ENOTDIR
+        nil
+      end
+
+      def real_directory(path)
+        entry = File.lstat(path)
+        return if entry.symlink? || !entry.directory?
+
+        File.realpath(path)
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP, Errno::ENOTDIR
+        nil
+      end
+
+      def control_file(path, within:)
+        entry = File.lstat(path)
+        return if entry.symlink? || !entry.file? || entry.size > MAX_GIT_POINTER_BYTES
+        absolute = File.realpath(path)
+        return unless inside?(absolute, within)
+
+        source = File.open(absolute, "rb") { |file| file.read(MAX_GIT_POINTER_BYTES + 1) }
+        return if source.bytesize > MAX_GIT_POINTER_BYTES || source.include?("\0")
+        source = source.force_encoding(Encoding::UTF_8)
+        source.strip if source.valid_encoding?
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP, Errno::ENOTDIR, EncodingError
+        nil
+      end
+
+      def load_file(path, base, within:)
+        return if @files >= MAX_FILES || @bytes >= MAX_TOTAL_BYTES || @lines >= MAX_LINES
+        entry = File.lstat(path)
+        return if entry.symlink? || !entry.file? || entry.size > MAX_FILE_BYTES
+        absolute = File.realpath(path)
+        return unless inside?(absolute, within)
+
+        available = [MAX_FILE_BYTES, MAX_TOTAL_BYTES - @bytes].min
+        source = File.open(absolute, "rb") { |file| file.read(available + 1) }
+        return if source.bytesize > available || source.include?("\0")
+        source = source.force_encoding(Encoding::UTF_8)
+        return unless source.valid_encoding?
+
+        kept = +""
+        source.each_line do |line|
+          break if @cancelled.call || @lines >= MAX_LINES
+          next if line.bytesize > MAX_LINE_BYTES
+          kept << line
+          @lines += 1
+        end
+        @matcher = @matcher.add(kept, base: base) unless kept.empty? || @cancelled.call
+        @files += 1
+        @bytes += source.bytesize
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP, Errno::ENOTDIR, EncodingError
+        nil
+      end
+
+      def inside?(path, directory)
+        directory = File.realpath(directory)
+        path == directory || path.start_with?(directory + File::SEPARATOR)
+      rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP, Errno::ENOTDIR
+        false
+      end
+    end
+  end
+end

data/lib/canopus/test_runner/discovery.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Canopus
+  module TestRunner
+    class Discovery
+      MAX_FILES = 50_000
+      MAX_TEST_FILES = 2_000
+      MAX_FILE_BYTES = 1 << 20
+      MAX_TESTS = 10_000
+      MAX_DEPTH = 64
+      MAX_PATH_BYTES = TestRunner::MAX_PATH_BYTES
+      MAX_ADAPTERS = 16
+      MAX_NAME_BYTES = 4_096
+      MAX_GROUPS = 32
+
+      attr_reader :root
+
+      def initialize(root:, ignore: nil, adapters: [Minitest.new, RSpec.new])
+        @root = File.realpath(root)
+        raise ArgumentError, "test root must be a directory" unless File.directory?(@root)
+        raise ArgumentError, "ignore must respond to ignored?" if ignore && !ignore.respond_to?(:ignored?)
+        unless adapters.is_a?(Array) && !adapters.empty? && adapters.length <= MAX_ADAPTERS &&
+            adapters.all? { |adapter| adapter.respond_to?(:candidate?) && adapter.respond_to?(:discover) }
+          raise ArgumentError, "test adapters must support candidate? and discover"
+        end
+
+        @ignore, @adapters = ignore, adapters.dup.freeze
+      end
+
+      def discover(cancelled: -> { false })
+        raise ArgumentError, "cancelled must respond to call" unless cancelled.respond_to?(:call)
+
+        require "prism"
+        tests, files, candidates = [], 0, 0
+        ignore = BoundedIgnore.new(root, delegate: @ignore, cancelled: cancelled)
+        walker = SafeWalker.new(root, ignore: ignore, hidden: true,
+          follow_symlinks: false, max_depth: MAX_DEPTH, cancelled: cancelled)
+        walker.each do |relative|
+          break if cancelled.call || (files += 1) > MAX_FILES || tests.length >= MAX_TESTS
+          relative = TestRunner.normalize_path(relative)
+          next unless relative
+
+          adapters = @adapters.select { |adapter| adapter.candidate?(relative) }
+          next if adapters.empty?
+          break if (candidates += 1) > MAX_TEST_FILES
+
+          source = test_source(relative)
+          next unless source
+          parsed = Prism.parse(source)
+          next unless parsed.success?
+
+          adapters.each do |adapter|
+            remaining = MAX_TESTS - tests.length
+            break if remaining <= 0 || cancelled.call
+            discovered = adapter.discover(parsed.value, path: relative, limit: remaining, cancelled: cancelled)
+            next unless discovered.is_a?(Array)
+            discovered.first(remaining).each { |test| tests << test if valid_test?(test, relative) }
+          end
+        rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP, Errno::ENOTDIR, EncodingError, ArgumentError
+          next
+        end
+        tests.sort_by! { |test| [test.path, test.line, test.column, test.framework.to_s, test.name] }
+        tests.freeze
+      end
+
+      private
+
+      def valid_test?(test, relative)
+        test.is_a?(Test) && test.path == relative && [:minitest, :rspec].include?(test.framework) &&
+          valid_name?(test.name) && test.groups.is_a?(Array) && test.groups.length <= MAX_GROUPS &&
+          test.groups.all? { |group| valid_name?(group) } &&
+          test.line.is_a?(Integer) && test.line.positive? &&
+          [test.column, test.offset].all? { |value| value.is_a?(Integer) && value >= 0 } &&
+          valid_name?(test.selector)
+      end
+
+      def valid_name?(value)
+        value.is_a?(String) && value.valid_encoding? && !value.include?("\0") && value.bytesize <= MAX_NAME_BYTES
+      end
+
+      def test_source(relative)
+        candidate = File.join(root, relative)
+        entry = File.lstat(candidate)
+        return if entry.symlink? || !entry.file? || entry.size > MAX_FILE_BYTES
+
+        absolute = File.realpath(candidate)
+        prefix = root.end_with?(File::SEPARATOR) ? root : root + File::SEPARATOR
+        return unless absolute.start_with?(prefix)
+
+        source = File.open(absolute, "rb") { |file| file.read(MAX_FILE_BYTES + 1) }
+        return if source.bytesize > MAX_FILE_BYTES || source.include?("\0")
+        source = source.delete_prefix("\xEF\xBB\xBF".b).force_encoding(Encoding::UTF_8)
+        source if source.valid_encoding?
+      end
+    end
+  end
+end