ruby_raider 4.0.1 → 4.1.0

data/lib/utilities/flows/runner.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require 'English'
+require 'securerandom'
+require 'time'
+require_relative 'project'
+require_relative 'run_log'
+require_relative 'notifier'
+
+module RubyRaider
+  module Flows
+    # `raider flow run`: a thin wrapper around `bundle exec rspec flows/<name>_flow.rb`. The flow's own
+    # flows/support/run.rb records the run; this sets its env, retries the whole flow and maps the
+    # recorded status to the exit code.
+    # :reek:TooManyInstanceVariables { max_instance_variables: 6 } -- the project, injected collaborators and the last run line
+    class Runner
+      EXIT_CODES = { 'completed' => 0, 'failed' => 1, 'setup_error' => 2, 'busy' => 3 }.freeze
+      INPUT = /\A[a-z][a-z0-9_]{0,39}\z/
+      INPUT_RULE = 'use name=value, the name 1 to 40 of a-z, 0-9 and _ starting with a letter'
+      TRIGGERS = %w[manual schedule desktop].freeze
+      RETRY_WAIT = 30
+
+      class << self
+        # True once INT or TERM stopped the last spawned run: no retry after that
+        attr_accessor :stopped
+      end
+
+      # Runs argv with no shell, under the project's own bundle rather than the one raider runs in.
+      # The flow stays in the terminal's process group (it can read the terminal, and a closed terminal
+      # ends it). Its flows/support/run.rb turns the first INT into an Interrupt that ends the step and
+      # still records the run, and ignores the rest. nil when it could not start: no log line, so exit 2.
+      def self.spawn(env, argv, chdir)
+        self.stopped = false
+        pid = nil
+        forwarded = false
+        forward = lambda do
+          next if pid.nil? || forwarded || Gem.win_platform? # the Windows console already sent Ctrl-C to the flow
+
+          forwarded = true
+          interrupt(pid)
+        end
+        # Before spawning, so a Ctrl-C in between is not lost
+        previous = %w[INT TERM].to_h do |signal|
+          [signal, trap(signal) do
+            self.stopped = true
+            forward.call
+          end]
+        end
+        run = lambda do
+          pid = Process.spawn(env, *argv, chdir: chdir)
+          forward.call if stopped
+          Process.wait(pid)
+          $CHILD_STATUS.exitstatus
+        end
+        defined?(Bundler) ? Bundler.with_unbundled_env(&run) : run.call
+      rescue SystemCallError => e
+        warn("Could not start #{argv.first}: #{e.message}")
+        nil
+      ensure
+        previous&.each { |signal, handler| trap(signal, handler || 'DEFAULT') }
+      end
+
+      # INT, sent once: from a terminal the flow already has its own, which it ignores as a repeat
+      def self.interrupt(pid)
+        Process.kill('INT', pid)
+      rescue SystemCallError
+        nil
+      end
+
+      # notifier: nil makes a Notifier only when a run with notify: true needs one
+      def initialize(project, out: $stdout, spawner: Runner.method(:spawn), sleeper: ->(seconds) { sleep(seconds) }, notifier: nil)
+        @project = project
+        @out = out
+        @spawner = spawner
+        @sleeper = sleeper
+        @notifier = notifier
+        @last_line = nil
+      end
+
+      # Returns the exit code: 0 completed, 1 a step failed, 2 setup failed, 3 already running.
+      # notify: true sends one desktop notification when the run ends in neither completed nor busy.
+      def run(name, headed: false, retries: 0, inputs: [], trigger: 'manual', notify: false)
+        @project.flow_file(name)
+        raise Error, 'Flows are not set up in this project: run raider flow init' unless @project.set_up?
+
+        options = { headed: headed, inputs: input_env(inputs), trigger: checked_trigger(trigger) }
+        attempts = checked_retries(retries) + 1
+        status = attempt_runs(name, attempts, options)
+        # Not after a stop: whoever pressed Ctrl-C already knows
+        notify_failure(name, status, options[:trigger]) if notify && !Runner.stopped && !%w[completed busy].include?(status)
+        EXIT_CODES.fetch(status, 2)
+      end
+
+      private
+
+      def attempt_runs(name, attempts, options)
+        (1..attempts).each do |attempt|
+          status = attempt_run(name, attempt, options)
+          return stopped(name, status) if Runner.stopped
+          return status if %w[completed busy].include?(status) || attempt == attempts
+
+          @out.puts("Retrying in #{RETRY_WAIT} s (attempt #{attempt + 1} of #{attempts})")
+          @sleeper.call(RETRY_WAIT)
+        end
+        'setup_error'
+      end
+
+      # The notifier never raises, so a notification can't change the exit code
+      def notify_failure(name, status, trigger)
+        message = status == 'failed' ? "Failed at #{failed_step(@last_line&.fetch('failed_step', nil))}" : "Could not start: see #{start_output(name, trigger)}"
+        (@notifier ||= Notifier.new).notify("Ruby Raider flow #{name}", message)
+      end
+
+      # Only a scheduled run outside Windows has a scheduler.log: Windows tasks keep none
+      def start_output(name, trigger)
+        trigger == 'schedule' && !Gem.win_platform? ? ".raider/flows/#{name}/scheduler.log" : 'the run output'
+      end
+
+      def stopped(name, status)
+        @out.puts("Flow #{name} was stopped: no retry")
+        status
+      end
+
+      def attempt_run(name, attempt, options)
+        run_id = SecureRandom.hex(8)
+        # The literal name-built path ([a-z0-9_] only), not the realpath flow_file checked
+        argv = ['bundle', 'exec', 'rspec', "flows/#{name}_flow.rb", '--tag', 'flow']
+        code = @spawner.call(run_env(name, attempt, run_id, options), argv, @project.root)
+        line = RunLog.new(@project, name).find(run_id)
+        @last_line = line
+        report(name, line, code)
+        line ? line['status'].to_s : 'setup_error'
+      end
+
+      # Every RAIDER_FLOW_* and RAIDER_INPUT_* is set (or unset) here, so a value left in the shell never leaks into a run
+      def run_env(name, attempt, run_id, options)
+        headed = options[:headed]
+        stale_inputs = ENV.keys.grep(/\ARAIDER_INPUT_/).to_h { |key| [key, nil] }
+        stale_inputs.merge(
+          'BUNDLE_GEMFILE' => @project.path('Gemfile'),
+          'RAIDER_FLOW_NAME' => name,
+          'RAIDER_FLOW_ATTEMPT' => attempt.to_s,
+          'RAIDER_FLOW_TRIGGER' => options[:trigger],
+          'RAIDER_FLOW_RUN_ID' => run_id,
+          'RAIDER_FLOW_HEADED' => headed ? '1' : nil,
+          'HEADLESS' => headed ? nil : 'true',
+          'RETRY_COUNT' => '0'
+        ).merge(options[:inputs])
+      end
+
+      # name=value pairs; values reach the flow only through RAIDER_INPUT_<NAME>, never argv
+      def input_env(inputs)
+        Array(inputs).to_h do |pair|
+          key, value = pair.to_s.split('=', 2)
+          raise Error, "Invalid input #{key.to_s.inspect}: #{INPUT_RULE}" unless value && INPUT.match?(key.to_s)
+          raise Error, "Invalid input #{key.inspect}: the value contains a NUL byte" if value.include?("\0")
+
+          ["RAIDER_INPUT_#{key.to_s.upcase}", value]
+        end
+      end
+
+      def checked_trigger(trigger)
+        return trigger if TRIGGERS.include?(trigger)
+
+        raise Error, "Invalid trigger #{trigger.inspect}: use #{TRIGGERS.join(', ')}"
+      end
+
+      def checked_retries(retries)
+        return retries if retries.is_a?(Integer) && retries.between?(0, 3)
+
+        raise Error, "Invalid retries #{retries.inspect}: use 0 to 3"
+      end
+
+      def report(name, line, code)
+        return @out.puts("Flow #{name} did not record a run (exit #{code || 'none'}): see the output above") unless line
+
+        @out.puts("Flow #{name} #{Flows.printable(outcome(line))}")
+        results = line['results_dir']
+        @out.puts("Results: #{Flows.printable(results)}") if results.is_a?(String) && File.directory?(@project.path(results))
+      end
+
+      def outcome(line)
+        case line['status']
+        when 'completed' then "completed in #{duration(line)} s"
+        when 'failed' then "failed at #{failed_step(line['failed_step'])}"
+        when 'busy' then 'is already running'
+        else 'could not start: see the output above'
+        end
+      end
+
+      def failed_step(step)
+        step = {} unless step.is_a?(Hash)
+        number = step['number']
+        where = number ? "step #{number}" : "line #{step['line'] || '?'}"
+        title = step['title']
+        title ? "#{where} (#{title})" : where
+      end
+
+      def duration(line)
+        (Time.iso8601(line['finished_at'].to_s) - Time.iso8601(line['started_at'].to_s)).round(1)
+      rescue ArgumentError
+        '?'
+      end
+    end
+  end
+end

data/lib/utilities/flows/schedule.rb

@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'rbconfig'
+require 'time'
+require_relative 'project'
+require_relative 'runner'
+
+module RubyRaider
+  module Flows
+    # One scheduled flow: the rule (every/at/on), how it runs (retries, inputs) and what the OS entry
+    # runs (absolute ruby and bundle, GEM_HOME/GEM_PATH captured when it was scheduled). Immutable.
+    # :reek:TooManyInstanceVariables { max_instance_variables: 13 } -- the fields of a schedules.yml entry
+    # :reek:TooManyConstants { max_constants: 9 } -- the rule vocabulary, the OS entry parts and the yml keys
+    class Schedule
+      EVERY = %w[hour day weekday week].freeze
+      DAYS = %w[mon tue wed thu fri sat sun].freeze
+      DAYS_RULE = 'use mon, tue, wed, thu, fri, sat or sun'
+      CLOCK = /\A([01]\d|2[0-3]):([0-5]\d)\z/
+      MINUTE = /\A(?:[01]\d|2[0-3])?:([0-5]\d)\z/
+      GEM_ENV = %w[GEM_HOME GEM_PATH].freeze
+      SYSTEM_PATH = %w[/usr/bin /bin].freeze
+      OS_ID_PREFIX = 'dev.rubyraider.flow'
+      KEYS = %w[name every at on retries inputs notify os_id ruby bundle gem_env created_at].freeze
+
+      attr_reader :name, :every, :at, :on, :retries, :inputs, :notify, :os_id, :ruby, :bundle, :gem_env, :project_root, :created_at
+
+      def self.build(project:, name:, every:, at: nil, on: nil, retries: 0, inputs: [], notify: false, now: Time.now)
+        root = project.root
+        new(name: name, every: every, at: at, on: on, retries: retries, inputs: inputs, notify: notify, os_id: os_id_for(root, name),
+            ruby: RbConfig.ruby, bundle: bundle_path, gem_env: captured_gem_env,
+            project_root: root, created_at: now.getutc.iso8601)
+      end
+
+      # A schedules.yml entry; raises Error when it is not one this class wrote
+      def self.from_h(hash, project_root)
+        raise Error, 'A schedule entry must be a mapping' unless hash.is_a?(Hash)
+
+        fields = KEYS.to_h { |key| [key.to_sym, hash[key]] }
+        schedule = new(**fields, project_root: project_root)
+        # Strict: an entry copied from another project, or a moved project, is not this project's entry
+        raise Error, "Invalid os_id #{schedule.os_id.inspect} for #{schedule.name} in #{project_root}" unless schedule.os_id == os_id_for(project_root, schedule.name)
+
+        schedule
+      end
+
+      def self.os_id_for(project_root, name) = "#{OS_ID_PREFIX}.#{Digest::SHA256.hexdigest(project_root)[0, 10]}.#{name}"
+
+      def self.bundle_path
+        Gem.bin_path('bundler', 'bundle')
+      rescue Gem::GemNotFoundException, Gem::MissingSpecError => e
+        raise Error, "Bundler not found for this Ruby (#{e.message}): gem install bundler, then try again"
+      end
+
+      # The GEM_HOME and GEM_PATH raider started with (before bundle exec changed them), only when set
+      def self.captured_gem_env
+        env = defined?(Bundler) ? Bundler.original_env : ENV
+        GEM_ENV.each_with_object({}) do |key, captured|
+          value = env[key]
+          captured[key] = value if value.is_a?(String) && !value.empty?
+        end
+      end
+
+      def initialize(name:, every:, at:, on:, retries:, inputs:, os_id:, ruby:, bundle:, gem_env:, project_root:, created_at:, notify: false)
+        @name = checked_name(name)
+        @every = checked_every(every)
+        @at = checked_at(at)
+        @on = checked_on(on)
+        @retries = checked_retries(retries)
+        @inputs = checked_inputs(inputs)
+        @notify = checked_notify(notify)
+        @os_id = checked_os_id(os_id)
+        @ruby = checked_path('ruby', ruby)
+        @bundle = checked_path('bundle', bundle)
+        @gem_env = checked_gem_env(gem_env)
+        @project_root = checked_path('project_root', project_root)
+        @created_at = checked_text('created_at', created_at)
+        freeze
+      end
+
+      def hour = at[0, 2].to_i
+
+      def minute = at[3, 2].to_i
+
+      def days
+        case every
+        when 'weekday' then DAYS.first(5)
+        when 'week' then [on.to_s]
+        else DAYS
+        end
+      end
+
+      def argv
+        command = [ruby, bundle, 'exec', 'raider', 'flow', 'run', name, '--project', project_root, '--trigger', 'schedule']
+        command += ['--retries', retries.to_s] if retries.positive?
+        command << '--notify' if notify
+        command += ['--input', *inputs] unless inputs.empty?
+        command
+      end
+
+      def env
+        path = [File.dirname(ruby), File.dirname(bundle), *SYSTEM_PATH].uniq.join(File::PATH_SEPARATOR)
+        { 'PATH' => path, 'LANG' => 'en_US.UTF-8' }.merge(gem_env)
+      end
+
+      def working_directory = project_root
+
+      def log_path = File.join(project_root, '.raider', 'flows', name, 'scheduler.log')
+
+      def rule_text
+        case every
+        when 'hour' then "every hour at #{at[2, 3]}"
+        when 'week' then "every week on #{on} at #{at}"
+        else "every #{every} at #{at}"
+        end
+      end
+
+      def to_h
+        { 'name' => name, 'every' => every, 'at' => at, 'on' => on, 'retries' => retries, 'inputs' => inputs.dup, 'notify' => notify,
+          'os_id' => os_id, 'ruby' => ruby, 'bundle' => bundle, 'gem_env' => gem_env.dup, 'created_at' => created_at }
+      end
+
+      private
+
+      def checked_name(name)
+        return name.dup.freeze if name.is_a?(String) && Flows.valid_name?(name)
+
+        raise Error, "Invalid flow name #{name.inspect}: use 1 to 60 of a-z, 0-9 and _ (not env)"
+      end
+
+      def checked_every(every)
+        return every.dup.freeze if EVERY.include?(every)
+
+        raise Error, "Invalid every #{every.inspect}: use hour, day, weekday or week"
+      end
+
+      # HH:MM, or for every hour :MM or HH:MM (the hour ignored, :00 when left out), kept as HH:MM
+      def checked_at(at)
+        return clock(at) unless every == 'hour'
+        return '00:00' if at.nil?
+
+        match = MINUTE.match(at.to_s)
+        raise Error, "Invalid at #{at.inspect}: use :MM or HH:MM for every hour" unless match
+
+        "00:#{match[1]}"
+      end
+
+      def clock(at)
+        raise Error, "Every #{every} needs at: use HH:MM (24h)" if at.nil?
+        return at.dup.freeze if at.is_a?(String) && CLOCK.match?(at)
+
+        raise Error, "Invalid at #{at.inspect}: use HH:MM (24h)"
+      end
+
+      def checked_on(on)
+        unless every == 'week'
+          raise Error, 'on is only for every week' unless on.nil?
+
+          return nil
+        end
+        raise Error, "Every week needs on: #{DAYS_RULE}" if on.nil?
+        return on.dup.freeze if DAYS.include?(on)
+
+        raise Error, "Invalid on #{on.inspect}: #{DAYS_RULE}"
+      end
+
+      def checked_retries(retries)
+        return retries if retries.is_a?(Integer) && retries.between?(0, 3)
+
+        raise Error, "Invalid retries #{retries.inspect}: use 0 to 3"
+      end
+
+      # name=value pairs as `raider flow run --input` takes them; a control character could break an OS entry
+      def checked_inputs(inputs)
+        raise Error, 'Invalid inputs: use a list of name=value' unless inputs.is_a?(Array)
+
+        inputs.map do |pair|
+          raise Error, "Invalid input #{pair.inspect}: use name=value in valid UTF-8" unless pair.is_a?(String) && pair.valid_encoding?
+
+          key, value = pair.split('=', 2)
+          raise Error, "Invalid input #{key.to_s.inspect}: #{Runner::INPUT_RULE}" unless value && Runner::INPUT.match?(key.to_s)
+          raise Error, "Invalid input #{key.inspect}: the value contains a control character" if value.match?(/[[:cntrl:]]/)
+
+          pair.dup.freeze
+        end.freeze
+      end
+
+      # An entry written before notify existed has no key: off
+      def checked_notify(notify)
+        return notify == true if [true, false, nil].include?(notify)
+
+        raise Error, "Invalid notify #{notify.inspect}: use true or false"
+      end
+
+      def checked_os_id(os_id)
+        pattern = /\A#{Regexp.escape(OS_ID_PREFIX)}\.[0-9a-f]{10}\.#{Regexp.escape(name)}\z/
+        return os_id.dup.freeze if os_id.is_a?(String) && pattern.match?(os_id)
+
+        raise Error, "Invalid os_id #{os_id.inspect} for #{name}"
+      end
+
+      def checked_path(label, path)
+        return checked_text(label, path) if path.is_a?(String) && File.absolute_path?(path)
+
+        raise Error, "Invalid #{label} #{path.inspect}: use an absolute path"
+      end
+
+      def checked_gem_env(gem_env)
+        raise Error, 'Invalid gem_env: use a mapping of GEM_HOME and GEM_PATH' unless gem_env.is_a?(Hash) && (gem_env.keys - GEM_ENV).empty?
+
+        gem_env.to_h { |key, value| [key.dup.freeze, checked_text(key, value)] }.freeze
+      end
+
+      def checked_text(label, text)
+        return text.dup.freeze if text.is_a?(String) && text.valid_encoding? && !text.empty? && !text.match?(/[[:cntrl:]]/)
+
+        raise Error, "Invalid #{label} #{text.inspect}"
+      end
+    end
+  end
+end

data/lib/utilities/flows/schedule_store.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'securerandom'
+require 'yaml'
+require_relative 'schedule'
+
+module RubyRaider
+  module Flows
+    # .raider/flows/schedules.yml: the project's scheduled flows, sorted by name. The OS entries are made
+    # from it; it is written whole through a temp file and a rename, so a reader never sees half a file.
+    # :reek:MissingSafeMethod { exclude: [ writable! ] } -- a check that raises, with no safe twin to offer
+    class ScheduleStore
+      DIR = '.raider/flows'
+      FILE = 'schedules.yml'
+      NOFOLLOW = defined?(File::NOFOLLOW) ? File::NOFOLLOW : 0
+      NONBLOCK = defined?(File::NONBLOCK) ? File::NONBLOCK : 0
+
+      # What delete returns for an entry it removed but could not read: enough to remove its OS entry
+      class Removed
+        attr_reader :name, :os_id
+
+        def initialize(name, os_id)
+          @name = name
+          @os_id = os_id
+          freeze
+        end
+      end
+
+      def initialize(project)
+        @project = project
+        @file = project.path("#{DIR}/#{FILE}")
+      end
+
+      # Every valid entry, sorted by name; a broken entry or file is skipped
+      def all
+        entries = read_entries
+        return [] unless entries
+
+        entries.filter_map { |entry| from_entry(entry) }.sort_by(&:name)
+      end
+
+      def find(name) = all.find { |schedule| schedule.name == name }
+
+      # Adds the schedule, or replaces the one with its name
+      def save(schedule)
+        write(writable_entries.reject { |entry| named?(entry, schedule.name) } + [schedule.to_h])
+        schedule
+      end
+
+      # Raises the Error save and delete would, before anything else is changed
+      def writable!
+        writable_entries
+        nil
+      end
+
+      # Removes every entry with the name, readable or not. Returns the removed schedule, a Removed
+      # (name and os_id) when the entry could not be read, or nil when there was none.
+      def delete(name)
+        entries = writable_entries
+        named, kept = entries.partition { |entry| named?(entry, name) }
+        return nil if named.empty?
+
+        write(kept)
+        named.filter_map { |entry| from_entry(entry) }.first || Removed.new(name, Schedule.os_id_for(@project.root, name))
+      end
+
+      private
+
+      def from_entry(entry)
+        Schedule.from_h(entry, @project.root)
+      rescue Error
+        nil
+      end
+
+      def named?(entry, name) = entry.is_a?(Hash) && entry['name'] == name
+
+      # The raw entries, nil when the file is not a schedules file or can't be read (missing reads as empty)
+      def read_entries
+        text = read_text
+        return [] if text.nil?
+        return nil unless text
+
+        data = YAML.safe_load(text)
+        return [] if data.nil?
+
+        entries = data.is_a?(Hash) ? data['schedules'] : nil
+        entries.is_a?(Array) ? entries : nil
+      rescue Psych::Exception
+        nil
+      end
+
+      # The entries to rewrite, keeping ones this version can't read; refuses to overwrite a broken file
+      def writable_entries
+        read_entries || raise(Error, "#{DIR}/#{FILE} is not a schedules file: fix or remove it, then try again")
+      end
+
+      # Read through one handle that is not a link; nil when there is no file, false when it is a link,
+      # not a regular file or can't be read (so a write never replaces what it could not see)
+      def read_text
+        File.open(@file, File::RDONLY | NONBLOCK | NOFOLLOW) do |file|
+          file.stat.file? ? file.read.to_s.force_encoding(Encoding::UTF_8) : false
+        end
+      rescue Errno::ENOENT
+        nil
+      rescue SystemCallError
+        false
+      end
+
+      def write(entries)
+        dir = private_dir
+        sorted = entries.sort_by { |entry| entry.is_a?(Hash) ? entry['name'].to_s : '' }
+        temp = File.join(dir, ".#{FILE}.#{SecureRandom.hex(6)}.tmp")
+        File.open(temp, File::WRONLY | File::CREAT | File::EXCL | NOFOLLOW, 0o600) do |file|
+          file.write(YAML.dump({ 'schedules' => sorted }))
+          file.flush
+          file.fsync
+        end
+        File.rename(temp, @file)
+      ensure
+        FileUtils.rm_f(temp) if temp && File.exist?(temp)
+      end
+
+      # A 0700 folder inside the project that is not a link, as flows/support/run.rb makes it
+      def private_dir
+        dir = @project.path(DIR)
+        FileUtils.mkdir_p(dir)
+        raise Error, "#{DIR} must be a folder, not a link" unless File.lstat(dir).directory?
+
+        File.chmod(0o700, dir)
+        dir
+      end
+    end
+  end
+end