lintus 0.1.0

data/lib/lintus/file_finder.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Lintus
+  # Resolves which files a run should look at, always as paths relative to the
+  # config root with forward slashes, in a stable order.
+  class FileFinder
+    attr_reader :root, :git
+
+    def initialize(root, git: Git.new(root))
+      @root = root
+      @git = git
+    end
+
+    # Every file in the tree: git's view when inside a repository, else a walk.
+    def all = sources(files_under(root))
+
+    def changed_since(ref)
+      require_repository!
+      sources(git.changed_files_since(ref))
+    end
+
+    def staged
+      require_repository!
+      sources(git.staged_files, existing_only: false) { |path| git.staged_content(path) }
+    end
+
+    # Paths given on the command line, resolved against the current directory.
+    def explicit(paths, from: Dir.pwd)
+      sources(paths.flat_map { |path| files_under(File.expand_path(path, from)) })
+    end
+
+    private
+
+    def require_repository!
+      raise Error, "#{root} is not a git repository: --diff and --staged need git" unless git.repository?
+    end
+
+    # Repository-relative paths of the files at or beneath an absolute path.
+    def files_under(absolute)
+      return [relativize(absolute)] unless File.directory?(absolute)
+
+      relative = relativize(absolute)
+      return git.files(relative) if git.repository?
+
+      Dir.glob("**/*", File::FNM_DOTMATCH, base: absolute)
+         .reject { |path| path == ".git" || path.start_with?(".git/") }
+         .map { |path| relative == "." ? path : File.join(relative, path) }
+    end
+
+    def sources(paths, existing_only: true, &reader)
+      reader ||= ->(path) { File.binread(File.join(root, path)) }
+
+      paths.sort.uniq.filter_map do |path|
+        next if existing_only && !File.file?(File.join(root, path))
+
+        SourceFile.new(path) { reader.call(path) }
+      end
+    end
+
+    def relativize(absolute)
+      relative = Pathname.new(absolute).relative_path_from(Pathname.new(root)).to_s
+      raise Error, "#{absolute} is outside the config root #{root}" if relative.start_with?("..")
+
+      relative
+    end
+  end
+end

data/lib/lintus/formatter/github.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Lintus
+  class Formatter
+    # GitHub Actions workflow commands: each offense becomes an annotation on
+    # its file in the pull request, then a plain summary for the job log.
+    class Github < Formatter
+      private
+
+      def offense_line(offense)
+        command(offense.severity, offense.message, file: offense.path, title: "lintus: #{offense.rule.id}")
+      end
+
+      def skipped_line(skipped)
+        command("notice", "Skipped: #{skipped.reason}", file: skipped.path, title: "lintus")
+      end
+
+      def failure_line(failure)
+        command("error", failure.error.message, file: failure.path, title: "lintus: request failed")
+      end
+
+      def command(level, message, **properties)
+        props = properties.map { |key, value| "#{key}=#{escape_property(value)}" }.join(",")
+        "::#{level} #{props}::#{escape_data(message)}"
+      end
+
+      def escape_data(value) = value.to_s.gsub("%", "%25").gsub("\r", "%0D").gsub("\n", "%0A")
+
+      def escape_property(value) = escape_data(value).gsub(":", "%3A").gsub(",", "%2C")
+    end
+  end
+end

data/lib/lintus/formatter/json.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Lintus
+  class Formatter
+    # Machine-readable output for other tools to consume.
+    class Json < Formatter
+      def render(report)
+        io.puts ::JSON.pretty_generate(
+          summary: {
+            files_inspected: report.checked.size,
+            offenses: report.offenses.size,
+            errors: report.errors.size,
+            warnings: report.warnings.size,
+            skipped: report.skipped.size,
+            failures: report.failures.size
+          },
+          offenses: report.sorted_offenses.map(&:to_h),
+          skipped: report.skipped.map { |skipped| { path: skipped.path, reason: skipped.reason } },
+          failures: report.failures.map { |failure| { path: failure.path, error: failure.error.message } }
+        )
+      end
+    end
+  end
+end

data/lib/lintus/formatter/text.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Lintus
+  class Formatter
+    # Human-readable output, one line per offense.
+    class Text < Formatter
+      private
+
+      def offense_line(offense)
+        "#{offense.path}: [#{offense.rule.id}] #{offense.message} (#{offense.severity}, noul #{offense.noul.round(2)})"
+      end
+
+      def skipped_line(skipped) = "#{skipped.path}: skipped, #{skipped.reason}"
+
+      def failure_line(failure) = "#{failure.path}: failed, #{failure.error.message}"
+
+      def before_summary(report)
+        io.puts if report.offenses.any? || report.skipped.any? || report.failures.any?
+      end
+    end
+  end
+end

data/lib/lintus/formatter.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Lintus
+  # Renders a Report to an IO. Line-oriented subclasses override the three
+  # `*_line` hooks; anything else overrides `render`.
+  class Formatter
+    NAMES = %w[text github json].freeze
+
+    def self.for(name)
+      raise Error, "unknown format #{name.inspect} (expected one of #{NAMES.join(", ")})" unless NAMES.include?(name.to_s)
+
+      const_get(name.to_s.capitalize)
+    end
+
+    attr_reader :io
+
+    def initialize(io = $stdout)
+      @io = io
+    end
+
+    def render(report)
+      report.sorted_offenses.each { |offense| io.puts offense_line(offense) }
+      report.skipped.each { |skipped| io.puts skipped_line(skipped) }
+      report.failures.each { |failure| io.puts failure_line(failure) }
+      before_summary(report)
+      io.puts summary(report)
+    end
+
+    private
+
+    def offense_line(offense) = raise(NotImplementedError)
+    def skipped_line(skipped) = raise(NotImplementedError)
+    def failure_line(failure) = raise(NotImplementedError)
+
+    def before_summary(report); end
+
+    def summary(report)
+      parts = ["#{pluralize(report.checked.size, "file")} inspected",
+               "#{pluralize(report.offenses.size, "offense")} detected"]
+      parts << "#{pluralize(report.skipped.size, "file")} skipped" if report.skipped.any?
+      parts << "#{pluralize(report.failures.size, "file")} failed" if report.failures.any?
+      parts.join(", ")
+    end
+
+    def pluralize(count, noun) = "#{count} #{noun}#{"s" unless count == 1}"
+  end
+end

data/lib/lintus/git.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Lintus
+  # A thin wrapper around the git commands file discovery needs.
+  class Git
+    class CommandError < Error; end
+
+    CHANGE_FILTER = "--diff-filter=ACMR"
+
+    attr_reader :root
+
+    def initialize(root)
+      @root = root
+    end
+
+    def repository? = run("rev-parse", "--is-inside-work-tree", allow_failure: true) == "true"
+
+    # Tracked and untracked files, honouring .gitignore, optionally under the given directories.
+    def files(*pathspecs) = lines("ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", *pathspecs)
+
+    def staged_files = lines("diff", "-z", "--cached", "--name-only", CHANGE_FILTER)
+
+    # Files that differ between the working tree and the merge base of `ref` and HEAD,
+    # plus untracked files. With `ref` at HEAD this is "what I have not committed yet".
+    def changed_files_since(ref)
+      base = run("merge-base", ref, "HEAD", allow_failure: true) || ref
+      lines("diff", "-z", "--name-only", CHANGE_FILTER, base) + lines("ls-files", "-z", "--others", "--exclude-standard")
+    end
+
+    def staged_content(path) = run("show", ":#{path}", chomp: false)
+
+    private
+
+    def lines(*)
+      run(*, chomp: false).split("\0").reject(&:empty?)
+    end
+
+    def run(*args, allow_failure: false, chomp: true)
+      stdout, stderr, status = Open3.capture3("git", *args, chdir: root)
+
+      unless status.success?
+        return nil if allow_failure
+
+        raise CommandError, "git #{args.join(" ")} failed: #{stderr.strip}"
+      end
+
+      chomp ? stdout.chomp : stdout
+    end
+  end
+end

data/lib/lintus/glob.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Lintus
+  # Matches repository-relative paths against the globs used in the config file.
+  #
+  # Patterns follow Ruby's File.fnmatch with pathname semantics ("**/" spans
+  # directories, "{a,b}" alternation is allowed), with two conveniences:
+  # a bare directory ("vendor" or "vendor/") matches everything beneath it,
+  # and a trailing "**" ("vendor/**") does the same.
+  module Glob
+    FLAGS = File::FNM_PATHNAME | File::FNM_EXTGLOB | File::FNM_DOTMATCH
+
+    # Every pattern is expanded once, however many paths it is matched against.
+    EXPANSIONS = Hash.new { |cache, pattern| cache[pattern] = expand(pattern) }
+
+    module_function
+
+    def match?(pattern, path)
+      EXPANSIONS[pattern].any? { |expanded| File.fnmatch?(expanded, path, FLAGS) }
+    end
+
+    def match_any?(patterns, path)
+      patterns.any? { |pattern| match?(pattern, path) }
+    end
+
+    def expand(pattern)
+      pattern = pattern.to_s.delete_prefix("./").delete_suffix("/")
+
+      if pattern.end_with?("**")
+        [pattern, "#{pattern}/*"]
+      elsif pattern.match?(/[*?\[{]/)
+        [pattern]
+      else
+        [pattern, "#{pattern}/**/*"]
+      end
+    end
+  end
+end

data/lib/lintus/offense.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Lintus
+  # A rule the model flagged on a file, with the probability it gave.
+  Offense = Struct.new(:path, :rule, :noul, keyword_init: true) do
+    def severity = rule.severity
+    def error? = rule.error?
+    def message = rule.description
+
+    def to_h
+      { path: path, rule: rule.id, severity: severity, message: message, noul: noul.round(3) }
+    end
+  end
+end

data/lib/lintus/report.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Lintus
+  # Everything a run produced: what was checked, what was flagged, what was skipped, what broke.
+  class Report
+    Skipped = Struct.new(:path, :reason, keyword_init: true)
+    Failure = Struct.new(:path, :error, keyword_init: true)
+
+    attr_reader :checked, :offenses, :skipped, :failures
+
+    def initialize
+      @checked = []
+      @offenses = []
+      @skipped = []
+      @failures = []
+      @mutex = Mutex.new
+    end
+
+    def record_checked(path, offenses)
+      synchronize do
+        checked << path
+        self.offenses.concat(offenses)
+      end
+    end
+
+    def record_skipped(path, reason)
+      synchronize { skipped << Skipped.new(path: path, reason: reason) }
+    end
+
+    def record_failure(path, error)
+      synchronize { failures << Failure.new(path: path, error: error) }
+    end
+
+    def errors = offenses.select(&:error?)
+    def warnings = offenses.reject(&:error?)
+
+    def sorted_offenses = offenses.sort_by { |offense| [offense.path, offense.rule.id] }
+
+    # 2 when a file could not be checked, 1 when offenses reach `fail_on`, else 0.
+    def exit_status(fail_on: "error")
+      return 2 if failures.any?
+
+      failing = case fail_on.to_s
+                when "warning" then offenses
+                when "never" then []
+                else errors
+                end
+      failing.any? ? 1 : 0
+    end
+
+    private
+
+    def synchronize(&) = @mutex.synchronize(&)
+  end
+end

data/lib/lintus/rule.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Lintus
+  # One entry of the `rules:` map in the config file.
+  #
+  # A rule is a noul question. It is asked against every file whose path
+  # matches its globs, and the file is an offense when the answer equals
+  # `offense_when` (true by default, so questions are phrased to describe the
+  # offense: "Does this file call sleep?").
+  class Rule
+    KEYS = %w[question description criteria threshold paths exclude severity offense_when].freeze
+    SEVERITIES = %w[error warning].freeze
+    ID_FORMAT = /\A[a-z][a-z0-9_]*\z/
+
+    attr_reader :id, :description, :question, :criteria, :threshold, :paths, :exclude, :severity, :offense_when
+
+    def initialize(id, attrs, default_paths: [], default_exclude: [])
+      @id = validate_id(id)
+      attrs = normalize(attrs)
+
+      @question = fetch_string(attrs, "question")
+      @description = attrs.fetch("description", @question).to_s
+      @criteria = build_criteria(attrs["criteria"])
+      @threshold = build_threshold(attrs["threshold"])
+      @paths = Schema.string_list(attrs.fetch("paths", default_paths))
+      @exclude = default_exclude + Schema.string_list(attrs["exclude"])
+      @severity = build_severity(attrs.fetch("severity", "error"))
+      @offense_when = build_offense_when(attrs.fetch("offense_when", true))
+    end
+
+    def applies_to?(path)
+      return false if Glob.match_any?(exclude, path)
+
+      paths.empty? || Glob.match_any?(paths, path)
+    end
+
+    def add_to(query)
+      options = { criteria: criteria, threshold: threshold }.compact
+      query.ask(id, question, **options)
+    end
+
+    def offense?(answer) = answer.result == offense_when
+
+    def error? = severity == "error"
+
+    private
+
+    def validate_id(id)
+      id = id.to_s
+      unless id.match?(ID_FORMAT)
+        raise ConfigError,
+              "invalid rule id #{id.inspect}: use snake_case (letters, digits, underscores)"
+      end
+
+      id
+    end
+
+    def normalize(attrs)
+      raise ConfigError, "rule #{id}: expected a map of attributes, got #{attrs.inspect}" unless attrs.is_a?(Hash)
+
+      attrs = attrs.transform_keys(&:to_s)
+      Schema.reject_unknown_keys!(attrs, KEYS, context: "rule #{id}")
+      attrs
+    end
+
+    def fetch_string(attrs, key)
+      value = attrs[key]
+      raise ConfigError, "rule #{id}: `#{key}` is required" if value.nil? || value.to_s.strip.empty?
+
+      value.to_s.strip
+    end
+
+    def build_criteria(criteria)
+      return nil if criteria.nil?
+      raise ConfigError, "rule #{id}: `criteria` must be a map with `true` and `false` keys" unless criteria.is_a?(Hash)
+
+      criteria.to_h { |key, value| [key.to_s, value.to_s] }
+    end
+
+    def build_threshold(threshold)
+      return nil if threshold.nil?
+
+      valid = threshold.is_a?(Numeric) && threshold.between?(0, 1)
+      raise ConfigError, "rule #{id}: `threshold` must be a number between 0 and 1" unless valid
+
+      threshold.to_f
+    end
+
+    def build_severity(severity)
+      severity = severity.to_s
+      unless SEVERITIES.include?(severity)
+        raise ConfigError,
+              "rule #{id}: `severity` must be one of #{SEVERITIES.join(", ")}"
+      end
+
+      severity
+    end
+
+    def build_offense_when(value)
+      raise ConfigError, "rule #{id}: `offense_when` must be true or false" unless [true, false].include?(value)
+
+      value
+    end
+  end
+end

data/lib/lintus/runner.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Lintus
+  # Sends each file to Jev once, with every rule that applies to it as a
+  # separate noul question, and turns the answers into offenses.
+  class Runner
+    RETRYABLE = [Jev::RateLimitError, Jev::OverloadedError].freeze
+
+    attr_reader :config, :jobs, :retries
+
+    def initialize(config, jobs: 4, retries: 3, sleeper: Kernel.method(:sleep))
+      @config = config
+      @jobs = [jobs.to_i, 1].max
+      @retries = retries
+      @sleeper = sleeper
+    end
+
+    # Pairs every file with the rules that apply to it, dropping files no rule covers.
+    def plan(files)
+      files.filter_map do |file|
+        rules = config.rules_for(file.path)
+        [file, rules] unless rules.empty?
+      end
+    end
+
+    # Checks the [file, rules] pairs from #plan concurrently.
+    def run(tasks)
+      report = Report.new
+      queue = Queue.new
+      tasks.each { |task| queue << task }
+      queue.close
+
+      workers = Array.new([jobs, tasks.size].min) do
+        Thread.new do
+          while (task = queue.pop)
+            check(*task, report)
+          end
+        end
+      end
+      workers.each(&:join)
+
+      report
+    end
+
+    def check(file, rules, report)
+      content = file.content
+      if (reason = skip_reason(content))
+        report.record_skipped(file.path, reason)
+        return
+      end
+
+      response = with_retries { perform(file, rules, content) }
+      report.record_checked(file.path, offenses_for(file, rules, response.answers))
+    rescue Jev::Error, Error => e
+      report.record_failure(file.path, e)
+    end
+
+    private
+
+    def skip_reason(content)
+      return "binary file" if content.include?("\0") || !content.valid_encoding?
+      if content.bytesize > config.max_file_size
+        return "larger than max_file_size (#{content.bytesize} > #{config.max_file_size} bytes)"
+      end
+
+      nil
+    end
+
+    def perform(file, rules, content)
+      Jev::Query.new("File: #{file.path}\n\n#{content}").perform do |query|
+        rules.each { |rule| rule.add_to(query) }
+      end
+    end
+
+    def offenses_for(file, rules, answers)
+      rules.filter_map do |rule|
+        answer = answers[rule.id]
+        Offense.new(path: file.path, rule: rule, noul: answer.noul) if rule.offense?(answer)
+      end
+    end
+
+    def with_retries
+      attempt = 0
+      begin
+        yield
+      rescue *RETRYABLE
+        raise if attempt >= retries
+
+        attempt += 1
+        @sleeper.call(backoff_for(attempt))
+        retry
+      end
+    end
+
+    def backoff_for(attempt) = (2**attempt) * (0.5 + (rand / 2))
+  end
+end

data/lib/lintus/schema.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Lintus
+  # Small checks shared by everything that reads the config file.
+  module Schema
+    module_function
+
+    def reject_unknown_keys!(hash, allowed, context:)
+      unknown = hash.keys - allowed
+      return if unknown.empty?
+
+      raise ConfigError, "#{context}: unknown key(s) #{unknown.join(", ")} (expected #{allowed.join(", ")})"
+    end
+
+    def string_list(value) = Array(value).map(&:to_s)
+  end
+end

data/lib/lintus/source_file.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Lintus
+  # A file to lint: a repository-relative path and a reader for its content.
+  # The reader is what lets `--staged` lint the index rather than the working tree.
+  class SourceFile
+    attr_reader :path
+
+    def initialize(path, &reader)
+      @path = path
+      @reader = reader
+    end
+
+    # Reads the content afresh on every call so a run holds only the files in flight.
+    def content
+      (+@reader.call).force_encoding(Encoding::UTF_8)
+    rescue SystemCallError, Git::CommandError => e
+      raise ReadError, "could not read #{path}: #{e.message}"
+    end
+  end
+end

data/lib/lintus/template.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module Lintus
+  # The starter config written by `lintus init`.
+  module Template
+    CONFIG = <<~YAML
+      # Lintus configuration. Every rule is a question asked of the Jev model
+      # about each matching file. Answer "true" means the file is an offense.
+      #
+      # Top-level `paths` and `exclude` apply to every rule; a rule can narrow
+      # them with its own `paths` and `exclude`. Globs are relative to this file.
+
+      paths:
+        - "**/*.rb"
+
+      exclude:
+        - "vendor/**"
+        - "node_modules/**"
+        - "tmp/**"
+
+      # Files larger than this (in bytes) are skipped rather than sent to the model.
+      max_file_size: 100000
+
+      rules:
+        no_debugging_leftovers:
+          description: Debugging statements must not be committed.
+          question: Does this file contain leftover debugging code, such as binding.irb, debugger, byebug, or a puts/p/pp used for debugging?
+          criteria:
+            "true": A debugger breakpoint or a throwaway print statement is present.
+            "false": Any output is deliberate program behaviour, or there is none.
+          severity: error
+
+        no_hardcoded_secrets:
+          description: Secrets must come from the environment or a credentials store, never source code.
+          question: Does this file contain a hardcoded secret, such as an API key, password, or private token?
+          criteria:
+            "true": A literal credential value is written in the source.
+            "false": Credentials are read from configuration, or there are none.
+          threshold: 0.7
+          severity: error
+
+        methods_are_documented:
+          description: Public classes should carry a short comment explaining their purpose.
+          question: Does every class or module defined in this file have a comment describing its responsibility?
+          # This rule is phrased positively, so the offense is a "false" answer.
+          offense_when: false
+          severity: warning
+    YAML
+  end
+end

data/lib/lintus/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Lintus
+  VERSION = "0.1.0"
+end

data/lib/lintus.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+require "jev"
+require "yaml"
+require "pathname"
+require_relative "lintus/version"
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect("cli" => "CLI")
+loader.setup
+
+# Lintus turns a YAML file of plain-language rules into a linter: every rule is
+# asked to the Jev model as a noul question against each file it applies to, and
+# an offense is reported wherever the answer says so.
+module Lintus
+  class Error < StandardError; end
+  class ConfigError < Error; end
+  class ReadError < Error; end
+
+  CONFIG_FILENAMES = %w[.lintus.yml lintus.yml .lintus.yaml lintus.yaml].freeze
+end