hecks 3.5.0 → 3.7.0

data/lib/hecks/tools/ci_gates.rb

@@ -23,11 +23,14 @@ module Hecks
       # @return [Integer] 0, or 1 when `--check` finds a region out of date
       # @raise [SystemExit] when a workflow has no marked region for a gate
       def main(argv, root: Tools::ROOT)
-        stale = projection(root).reject { |path, text| File.read(path) == text }.keys
+        files = projection(root)
+        stale = files.reject { |path, text| File.read(path) == text }.keys
         return report(stale, root) if argv.include?("--check")
 
-        stale.each { |path| File.write(path, projection(root).fetch(path)) }
-        stale.each { |path| puts "wrote #{path.delete_prefix("#{root}/")}" }
+        stale.each do |path|
+          File.write(path, files.fetch(path))
+          puts "wrote #{path.delete_prefix("#{root}/")}"
+        end
         puts "ci_gates: #{gates.size} gates, every region current" if stale.empty?
         0
       end
@@ -50,8 +53,8 @@ module Hecks
       # @raise [SystemExit] when a row's `mode` or `push` is not one the action understands
       def gates
         Hecks::Vocabulary.rows("CiGate").each do |gate|
-          abort "ci_gates: #{gate['name']} has mode #{gate['mode'].inspect}" unless MODES.include?(gate["mode"])
-          abort "ci_gates: #{gate['name']} has push #{gate['push'].inspect}" unless PUSHES.include?(gate["push"])
+          abort "ci_gates: #{gate["name"]} has mode #{gate["mode"].inspect}" unless MODES.include?(gate["mode"])
+          abort "ci_gates: #{gate["name"]} has push #{gate["push"].inspect}" unless PUSHES.include?(gate["push"])
         end
       end
 
@@ -64,7 +67,7 @@ module Hecks
           return 0
         end
 
-        warn "ci_gates: out of date: #{stale.map { |path| path.delete_prefix("#{root}/") }.join(', ')} " \
+        warn "ci_gates: out of date: #{stale.map { |path| path.delete_prefix("#{root}/") }.join(", ")} " \
              "(run hecks project_ci_gates)"
         1
       end
@@ -85,26 +88,37 @@ module Hecks
       # @return [String] the marked region: the detector job that answers `touched`
       def job(gate)
         name = gate.fetch("name")
-        [
-          "  # BEGIN GENERATED ci_gate #{name} (CiGate vocabulary; hecks project_ci_gates). Do not hand-edit.",
-          "  #{name}:",
-          "    runs-on: ubuntu-latest",
-          "    timeout-minutes: 10",
-          *job_condition(gate),
-          "    outputs:",
-          "      touched: ${{ steps.diff.outputs.touched }}",
-          "    steps:",
-          "      - uses: actions/checkout@v4",
-          "        with:",
-          "          # Full history: the diff needs both endpoints present as real objects.",
-          "          fetch-depth: 0",
-          "      - uses: ./.github/actions/setup-ruby",
-          "      - uses: ./.github/actions/hecks-environment",
-          "      - id: diff",
-          "        name: #{quoted(gate.fetch('label'))}",
-          "        run: bundle exec exe/hecks regeneration_run.decide_ci_gate gate=#{name} --wait",
-          "  # END GENERATED ci_gate #{name}"
-        ].join("\n")
+        [*job_header(gate, name), *JOB_BODY, *job_footer(gate, name)].join("\n")
+      end
+
+      # The job's static middle: its outputs and the steps up to the one that decides.
+      JOB_BODY = [
+        "    outputs:",
+        "      touched: ${{ steps.diff.outputs.touched }}",
+        "    steps:",
+        "      - uses: actions/checkout@v4",
+        "        with:",
+        "          # Full history: the diff needs both endpoints present as real objects.",
+        "          fetch-depth: 0",
+        "      - uses: ./.github/actions/setup-ruby",
+        "      - uses: ./.github/actions/hecks-environment",
+        "      - id: diff"
+      ].freeze
+
+      # @return [Array<String>] the marker, the job's name and runner, and its condition
+      def job_header(gate, name)
+        ["  # BEGIN GENERATED ci_gate #{name} (CiGate vocabulary; hecks project_ci_gates). Do not hand-edit.",
+         "  #{name}:",
+         "    runs-on: ubuntu-latest",
+         "    timeout-minutes: 10",
+         *job_condition(gate)]
+      end
+
+      # @return [Array<String>] the deciding step and the closing marker
+      def job_footer(gate, name)
+        ["        name: #{quoted(gate.fetch("label"))}",
+         "        run: bundle exec exe/hecks regeneration_run.decide_ci_gate gate=#{name} --wait",
+         "  # END GENERATED ci_gate #{name}"]
       end
 
       # @param gate [Hash{String => String}] a `CiGate` row

data/lib/hecks/tools/comment_style/baseline.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../../tools"
+
+module Hecks
+  module Tools
+    module CommentStyle
+      # The checked-in list of comment blocks already over `MAX_BLOCK` lines.
+      #
+      # Keyed by first-line text, not line number, so edits above a block don't break its key.
+      module Baseline
+        FILE = ".standardize_comments_baseline.json"
+
+        # The checkout this file lives in: the baseline's home when no `root:` is given.
+        ROOT = Tools::ROOT
+
+        PATH = File.join(ROOT, FILE)
+
+        module_function
+
+        # The baseline's location inside a checkout.
+        #
+        # @param root [String] the checkout's root directory
+        # @return [String] the baseline file's path
+        def path_in(root) = File.join(root, FILE)
+
+        # Reads the baseline from disk.
+        def load(path = PATH)
+          File.exist?(path) ? JSON.parse(File.read(path, encoding: Encoding::UTF_8)) : {}
+        end
+
+        # Writes a baseline in a stable order, so a rewrite shows only real changes in a diff.
+        def dump(blocks, path = PATH)
+          sorted = blocks.sort.to_h { |file, keys| [file, keys.sort.to_h] }
+          File.write(path, "#{JSON.pretty_generate(sorted)}\n")
+        end
+      end
+    end
+  end
+end

data/lib/hecks/tools/comment_style/caps_words.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require_relative "../../tools"
+
+module Hecks
+  module Tools
+    module CommentStyle
+      # Finds the all-caps words in a comment and decides how each should be written instead.
+      module CapsWords
+        private
+
+        # @return [Array<Array>] each shouted word as `[offset, word, replacement]`, in text order
+        def caps_words(comment)
+          masked = mask(comment)
+          matches = []
+          masked.scan(CAPS_WORD) { matches << Regexp.last_match }
+          sql = sql_keywords(comment, matches.map { |match| match[0] })
+          found = matches.filter_map { |match| shouted_word(comment, masked, match, sql) }
+          (drop_stray_articles(found) + possessives(masked)).sort_by(&:first)
+        end
+
+        def shouted_word(comment, masked, match, sql)
+          word = match[0]
+          return if word != "A" && (keep_capitals?(word) || sql.include?(word) || unknown?(word))
+
+          [match.begin(0), word, recase(word, sentence_start?(comment, masked, match.begin(0)))]
+        end
+
+        # `Finding`'s: the capital follows a code span, so `CAPS_WORD` never sees it.
+        def possessives(masked)
+          found = []
+          masked.scan(/(?<=#{FILLER})'S\b/o) { found << [Regexp.last_match.begin(0) + 1, "S", "s"] }
+          found
+        end
+
+        # An "A" is only emphasis when a word beside it is being fixed too.
+        def drop_stray_articles(found)
+          found.each_with_index.reject do |(offset, word, _), index|
+            word == "A" && !beside_fixed_word?(found, index, offset)
+          end.map(&:first)
+        end
+
+        def beside_fixed_word?(found, index, offset)
+          before = index.positive? ? found[index - 1] : nil
+          after = found[index + 1]
+          follows_fixed?(before, offset) || precedes_fixed?(after, offset)
+        end
+
+        def follows_fixed?(before, offset)
+          before && before[0] + before[1].length + 1 == offset && before[1] != "A"
+        end
+
+        def precedes_fixed?(after, offset)
+          after && after[0] == offset + 2 && after[1] != "A"
+        end
+
+        def recase(word, sentence_start)
+          bare, suffix = word.split("'", 2)
+          proper = proper_name(bare)
+          return [proper, suffix&.downcase].compact.join("'") if proper
+
+          sentence_start ? word.capitalize : word.downcase
+        end
+
+        # A shouted product or class name (`POSTGRESERA`) gets its real spelling
+        # back. An English word that is also a class name (`VALUE`) stays prose.
+        def proper_name(bare)
+          PROPER_NOUNS[bare] || (CommentStyle.class_names[bare] unless Dictionary.english?(bare))
+        end
+
+        # Neither English nor a known name, so lowercasing it would be a guess.
+        def unknown?(word)
+          bare = word.split("'", 2).first
+          !Dictionary.english?(word) && !proper_name(bare)
+        end
+
+        def keep_capitals?(word)
+          bare = word.sub(/'[A-Z]{1,2}\z/, "")
+          return false if PROPER_NOUNS.key?(bare)
+
+          ACRONYMS.include?(bare) || @constants.include?(bare) || bare.match?(/\A[IVX]+\z/)
+        end
+
+        def sentence_start?(comment, masked, offset)
+          before = masked[0, offset].sub(/\A#+\s*/, "")
+          return before.match?(/[.!?]["')\]]*\s+\z/) if before.match?(/[[:alnum:]#{FILLER}]/o)
+          return true if before.match?(/\A\s*(?:[-*•]|─+|\d+[.)])\s+\z/)
+          return false if masked[0, offset].match?(/\A#\s{3,}/)
+
+          after_sentence_end?(comment)
+        end
+
+        # Whether the line above, in the same paragraph, ends a sentence (or there is none).
+        def after_sentence_end?(comment)
+          previous = comment.full_line ? @by_line[comment.line - 1] : nil
+          return true unless previous&.full_line
+
+          body = previous.text.sub(/\A#+\s*/, "").strip
+          body.empty? || body.match?(/[.!?]["')\]]*\z/)
+        end
+      end
+    end
+  end
+end

data/lib/hecks/tools/comment_style/cli.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+require_relative "../../tools"
+require_relative "run"
+
+module Hecks
+  module Tools
+    module CommentStyle
+      # The command line of the Ruby comment linter, which `CommentStyle` extends itself with.
+      module Cli
+        # Builds the CLI's option parser, filling `options` as flags are seen.
+        def option_parser(options)
+          OptionParser.new do |opts|
+            opts.banner = "Usage: hecks check_comments [--report|--check|--fix] [--only a,b] PATH..."
+            mode_options(opts, options)
+            filter_options(opts, options)
+          end
+        end
+
+        # Prints `--report`, `--check`, or `--json` output for a finished run.
+        def render(run, options)
+          found = run.violations
+          print_run(run, found, options)
+          options[:mode] == :check && found.any? ? 1 : 0
+        end
+
+        # The CLI entry point: parses `argv` and runs the requested mode.
+        #
+        # Paths after a `--` are never read as flags, and a path that does not exist is refused.
+        def main(argv, root: Baseline::ROOT)
+          options, paths = parse_arguments(argv)
+          baseline_path = Baseline.path_in(root)
+          run = Run.new(paths, only: options[:only], baseline: Baseline.load(baseline_path))
+          case options[:mode]
+          when :fix then puts("rewrote #{run.fix!} files") || 0
+          when :write_baseline then write_baseline(run, baseline_path)
+          when :code_unchanged then report_code_changes(changed_since(run, options[:ref]))
+          else render(run, options)
+          end
+        end
+
+        # Rewrites the baseline file from the blocks over `MAX_BLOCK` lines that `run` finds.
+        def write_baseline(run, path = Baseline::PATH)
+          held = run.long_block_baseline
+          Baseline.dump(held, path)
+          puts "recorded #{held.values.sum(&:size)} blocks in #{held.size} files at #{path}"
+          0
+        end
+
+        # The files `run` finds changed since `ref`, refusing a ref that names no commit.
+        def changed_since(run, ref)
+          run.code_changed_since(ref)
+        rescue ArgumentError => e
+          abort e.message
+        end
+
+        # Prints `--code-unchanged`'s verdict.
+        def report_code_changes(changed)
+          changed.each { |path| puts "#{path}: code changed, not just comments" }
+          puts "code unchanged" if changed.empty?
+          changed.empty? ? 0 : 1
+        end
+
+        private
+
+        def print_run(run, found, options)
+          if options[:json]
+            puts JSON.pretty_generate(found.map(&:to_h))
+          elsif options[:mode] == :check
+            found.each { |v| puts "#{v.path}:#{v.line}: [#{v.category}] #{v.message}" }
+          else
+            puts run.report(top: options[:top])
+          end
+        end
+
+        def mode_options(opts, options)
+          opts.on("--report", "summary tables (default)") { options[:mode] = :report }
+          opts.on("--check", "list every violation, exit 1 if any") { options[:mode] = :check }
+          opts.on("--fix", "rewrite fixable categories: #{FIXABLE.join(", ")}") { options[:mode] = :fix }
+          opts.on("--write-baseline", "record every block over #{MAX_BLOCK} lines as tolerated") do
+            options[:mode] = :write_baseline
+          end
+        end
+
+        def filter_options(opts, options)
+          opts.on("--only LIST", Array, "limit to: #{CATEGORIES.keys.join(", ")}") { |list| options[:only] = list }
+          opts.on("--code-unchanged REF", "exit 1 if any file's code (not comments) differs from REF") do |ref|
+            options[:mode] = :code_unchanged
+            options[:ref] = ref
+          end
+          opts.on("--json", "machine-readable output") { options[:json] = true }
+          opts.on("--top N", Integer, "rows in the worst-files table") { |n| options[:top] = n }
+        end
+
+        # @return [Array(Hash, Array<String>)] the options and the paths, after refusing an unknown
+        #   category, a missing path, or no path at all
+        def parse_arguments(argv)
+          options = { mode: :report, top: 20 }
+          parser = option_parser(options)
+          paths = parser.parse(argv)
+          refuse_unknown_categories(options)
+          abort parser.help if paths.empty?
+          refuse_missing_paths(paths)
+          [options, paths]
+        end
+
+        def refuse_unknown_categories(options)
+          unknown = Array(options[:only]) - CATEGORIES.keys
+          abort "unknown categories: #{unknown.join(", ")}" unless unknown.empty?
+        end
+
+        def refuse_missing_paths(paths)
+          missing = paths.reject { |path| File.exist?(path) }
+          abort "no such path: #{missing.join(", ")}" unless missing.empty?
+        end
+      end
+    end
+  end
+end

data/lib/hecks/tools/comment_style/dictionary.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative "../../tools"
+
+module Hecks
+  module Tools
+    module CommentStyle
+      # Decides whether a capitalised token is an English word being shouted;
+      # anything else keeps its capitals and is reported for a person to look at.
+      module Dictionary
+        WORDS = "/usr/share/dict/words"
+
+        # Technical vocabulary the system word list lacks.
+        EXTRA = %w[
+          adversarial allowlist anymore app apps arg args arity async backend backtrace baseline
+          boolean bootable bootstrapping bluebook byte callee canonicalized changelog checklist
+          checkpoint codebase codemod config database dataset dedupe deduped enqueue evaluator
+          filename fixpoint formatting frontend glob grep handover hardcoded hecksagon heredoc held
+          hoc hostname idempotency idempotent inline keyword landmine lifecycle lookup mapping
+          memoised memoized memoizes metadata metaprogrammed monkeypatch monorepo multi mutex
+          namespace noop numeric op ops parameterized params pathname payload plugin postcondition
+          pre reachability reentrant refactor regex regexes rekey rekeyed repo restartable retrofit
+          reusable roadmap rollup runtime shortcut standalone stringified struct subprocess
+          subprocesses subroutine substring superuser symlink sync timestamp tmp tmpl touchpoint
+          trickiest triggerable tuple undecoded unsanitized untargeted upsert username webhook
+          widget workaround worklist worklists
+        ].freeze
+
+        SUFFIXES = [
+          ["", ""], ["s", ""], ["es", ""], ["ed", ""], ["d", ""], ["ing", ""], ["ing", "e"], ["ies", "y"],
+          ["ied", "y"], ["ly", ""], ["er", ""], ["est", ""], ["ier", "y"], ["ised", "ized"], ["ise", "ize"],
+          ["isation", "ization"], ["our", "or"], ["lling", "ling"], ["lled", "led"]
+        ].freeze
+
+        module_function
+
+        # Loads and memoizes the system word list, once per process.
+        def words
+          return @words if defined?(@words)
+
+          @words = File.exist?(WORDS) ? Set.new(File.foreach(WORDS, chomp: true).map(&:downcase)).merge(EXTRA) : nil
+        end
+
+        # Checks a capitalized token against the word list, stripped and stemmed.
+        # Always true when this machine has no word list to consult.
+        def english?(word)
+          return true unless words
+
+          bare = word.downcase.delete_suffix("n't").sub(/'.*\z/, "")
+          SUFFIXES.any? do |suffix, replacement|
+            next false unless bare.end_with?(suffix)
+
+            stem = bare.delete_suffix(suffix) + replacement
+            words.include?(stem) || words.include?(stem.sub(/(.)\1\z/, '\1'))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/tools/comment_style/fixing.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require_relative "../../tools"
+
+module Hecks
+  module Tools
+    module CommentStyle
+      # The rewrites for the fixable categories: all-caps words, long bold headings, long lines.
+      module Fixing
+        private
+
+        def fix_line(line, number, wanted)
+          comment = @by_line[number]
+          return line if comment.nil? || comment.text.match?(DIRECTIVE)
+
+          text = fixed_text(comment, wanted)
+          code = line.byteslice(0, comment.col)
+          return wrap(code, text) if wanted.include?("long_line") && comment.full_line
+
+          "#{code}#{text}#{"\n" if line.end_with?("\n")}"
+        end
+
+        def fixed_text(comment, wanted)
+          text = wanted.include?("all_caps") ? fix_caps(comment) : comment.text
+          wanted.include?("long_bold") ? unbold(text) : text
+        end
+
+        def unbold(text)
+          heading = text[BOLD_HEADING, 2]
+          return text unless heading && heading.split.size > MAX_HEADING_WORDS
+
+          text.sub(BOLD_HEADING) { "#{Regexp.last_match(1)}#{heading}" }
+        end
+
+        def fix_caps(comment)
+          text = comment.text.dup
+          found = caps_words(comment)
+          found.reverse_each { |offset, word, replacement| text[offset, word.length] = replacement }
+          first, last = heading_run(comment, found)
+          return text unless first
+
+          text.insert(last, "**").insert(first, "**")
+        end
+
+        # An all-caps run that opens a paragraph and stops at a dash or a full stop
+        # is that paragraph's heading, so it becomes bold rather than plain text.
+        def heading_run(comment, found)
+          masked = mask(comment)
+          return unless found.size >= 2 && paragraph_start?(comment, masked, found.first[0])
+
+          bounds = run_bounds(found, masked)
+          bounds if bounds && !documents_method?(comment)
+        end
+
+        # @return [Array(Integer, Integer), nil] where the first run of shouted words starts and
+        #   ends, when it reads as a heading
+        def run_bounds(found, masked)
+          run = first_run(found, masked)
+          finish = run_end(run, masked)
+          return unless run.size >= 2 && masked[finish..].match?(HEADING_END)
+          return if masked[run.first[0]...finish].split.size > MAX_HEADING_WORDS
+
+          [run.first[0], finish]
+        end
+
+        # The leading shouted words with only a gap between them.
+        def first_run(found, masked)
+          found.slice_when { |(a, word, _), (b, _, _)| !masked[(a + word.length)...b].match?(GAP) }.first
+        end
+
+        # Where a run of shouted words ends, taking in a trailing possessive `'s`.
+        def run_end(run, masked)
+          finish = run.last[0] + run.last[1].length
+          masked[finish, 2] == "'s" ? finish + 2 : finish
+        end
+
+        # The first line above a `def` has to be a sentence about the method, so a
+        # heading there stays plain text.
+        def documents_method?(comment)
+          line = comment.line
+          line += 1 while @by_line[line + 1]&.full_line
+          @lines[line].to_s.match?(/\A\s*(?:\w+\s+)?def\s/)
+        end
+
+        def paragraph_start?(comment, masked, offset)
+          return false if masked[0, offset].sub(/\A#+\s*/, "").match?(/[[:alnum:]#{FILLER}]/o)
+
+          above = comment.full_line ? @by_line[comment.line - 1] : nil
+          !above&.full_line || above.text.match?(/\A#+\s*\z/)
+        end
+
+        # Splits one over-long prose comment line in two. Lines with no safe
+        # break point (a long URL, an unbroken code span) are left for a person.
+        def wrap(indent, text)
+          whole = "#{indent}#{text}\n"
+          return whole if whole.chomp.length <= MAX_LINE
+
+          lead = wrap_lead(text)
+          cut = wrap_cut(indent, text, lead)
+          return whole unless cut
+
+          "#{indent}#{text[0, cut].rstrip}\n#{wrap(indent, "#{lead}#{text[cut..].strip}")}"
+        end
+
+        # An `@tag` line continues, indented, under its tag.
+        def wrap_lead(text)
+          lead = text[/\A#+\s*/]
+          text.match?(/\A#+\s*@\w/) ? "#{lead}  " : lead
+        end
+
+        # @return [Integer, nil] the offset to break at, nil when there is no safe one
+        def wrap_cut(indent, text, lead)
+          cut = text[0, MAX_LINE - indent.length + 1].rindex(" ")
+          cut unless cut.nil? || cut <= lead.length || text[0, cut].count("`").odd?
+        end
+      end
+    end
+  end
+end

data/lib/hecks/tools/comment_style/long_blocks.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "../../tools"
+
+module Hecks
+  module Tools
+    module CommentStyle
+      # The `long_block` category: runs of full-line comments longer than `MAX_BLOCK`, held
+      # against the baseline.
+      module LongBlocks
+        # Finds every run of consecutive full-line comments longer than `MAX_BLOCK`.
+        #
+        # @return [Array<Block>]
+        def long_blocks
+          return [] if generated?
+
+          seen = Hash.new(0)
+          long_runs = comment_runs.select { |run| run.size > MAX_BLOCK }
+          long_runs.map { |run| Block.new(run.first.line, run.size, block_key(run, seen)) }
+        end
+
+        private
+
+        def comment_runs
+          @comments.select(&:full_line).slice_when { |above, below| below.line != above.line + 1 }
+        end
+
+        # The text of a block's first line, without its `#`, is its key. A second block opened by
+        # the same text in this file gets a `#2` suffix, and so on.
+        def block_key(run, seen)
+          text = run.first.text.sub(/\A#+\s*/, "").strip[0, MAX_ANCHOR]
+          text = "(empty)" if text.empty?
+          seen[text] += 1
+          seen[text] == 1 ? text : "#{text} ##{seen[text]}"
+        end
+
+        # A block fails when it has no baseline entry, or has grown past the length recorded there.
+        def long_block_violations(baseline)
+          held = baseline.fetch(path.delete_prefix("./"), {})
+          long_blocks.filter_map do |block|
+            allowed = held[block.key]
+            next if allowed && block.lines <= allowed
+
+            detail = allowed ? "baselined at #{allowed}" : "over #{MAX_BLOCK}"
+            Violation.new(path, block.line, "long_block", "#{block.lines} lines, #{detail}: #{block.key}")
+          end
+        end
+      end
+    end
+  end
+end